1 //
   2 // Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // X86 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // Previously set EBX, ESI, and EDI as save-on-entry for java code
  64 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  65 // Now that allocator is better, turn on ESI and EDI as SOE registers.
  66 
  67 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
  68 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
  69 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
  70 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
  71 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code
  72 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());
  73 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
  74 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());
  75 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());
  76 
  77 // Special Registers
  78 reg_def EFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());
  79 
  80 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the
  81 // allocator, and only shows up in the encodings.
  82 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  83 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  84 // Ok so here's the trick FPR1 is really st(0) except in the midst
  85 // of emission of assembly for a machnode. During the emission the fpu stack
  86 // is pushed making FPR1 == st(1) temporarily. However at any safepoint
  87 // the stack will not have this element so FPR1 == st(0) from the
  88 // oopMap viewpoint. This same weirdness with numbering causes
  89 // instruction encoding to have to play games with the register
  90 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation
  91 // where it does flt->flt moves to see an example
  92 //
  93 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());
  94 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());
  95 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());
  96 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());
  97 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());
  98 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());
  99 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());
 100 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());
 101 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());
 102 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());
 103 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());
 104 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());
 105 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());
 106 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());
 107 
 108 // XMM registers.  128-bit registers or 4 words each, labeled a-d.
 109 // Word a in each register holds a Float, words ab hold a Double.
 110 // We currently do not use the SIMD capabilities, so registers cd
 111 // are unused at the moment.
 112 reg_def XMM0a( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
 113 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next());
 114 reg_def XMM1a( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
 115 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next());
 116 reg_def XMM2a( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
 117 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next());
 118 reg_def XMM3a( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
 119 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next());
 120 reg_def XMM4a( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
 121 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next());
 122 reg_def XMM5a( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
 123 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next());
 124 reg_def XMM6a( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
 125 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next());
 126 reg_def XMM7a( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
 127 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next());
 128 
 129 // Specify priority of register selection within phases of register
 130 // allocation.  Highest priority is first.  A useful heuristic is to
 131 // give registers a low priority when they are required by machine
 132 // instructions, like EAX and EDX.  Registers which are used as
 133 // pairs must fall on an even boundary (witness the FPR#L's in this list).
 134 // For the Intel integer registers, the equivalent Long pairs are
 135 // EDX:EAX, EBX:ECX, and EDI:EBP.
 136 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,
 137                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,
 138                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,
 139                     FPR6L, FPR6H, FPR7L, FPR7H );
 140 
 141 alloc_class chunk1( XMM0a, XMM0b,
 142                     XMM1a, XMM1b,
 143                     XMM2a, XMM2b,
 144                     XMM3a, XMM3b,
 145                     XMM4a, XMM4b,
 146                     XMM5a, XMM5b,
 147                     XMM6a, XMM6b,
 148                     XMM7a, XMM7b, EFLAGS);
 149 
 150 
 151 //----------Architecture Description Register Classes--------------------------
 152 // Several register classes are automatically defined based upon information in
 153 // this architecture description.
 154 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 155 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 156 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 157 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 158 //
 159 // Class for all registers
 160 reg_class any_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);
 161 // Class for general registers
 162 reg_class e_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX);
 163 // Class for general registers which may be used for implicit null checks on win95
 164 // Also safe for use by tailjump. We don't want to allocate in rbp,
 165 reg_class e_reg_no_rbp(EAX, EDX, EDI, ESI, ECX, EBX);
 166 // Class of "X" registers
 167 reg_class x_reg(EBX, ECX, EDX, EAX);
 168 // Class of registers that can appear in an address with no offset.
 169 // EBP and ESP require an extra instruction byte for zero offset.
 170 // Used in fast-unlock
 171 reg_class p_reg(EDX, EDI, ESI, EBX);
 172 // Class for general registers not including ECX
 173 reg_class ncx_reg(EAX, EDX, EBP, EDI, ESI, EBX);
 174 // Class for general registers not including EAX
 175 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);
 176 // Class for general registers not including EAX or EBX.
 177 reg_class nabx_reg(EDX, EDI, ESI, ECX, EBP);
 178 // Class of EAX (for multiply and divide operations)
 179 reg_class eax_reg(EAX);
 180 // Class of EBX (for atomic add)
 181 reg_class ebx_reg(EBX);
 182 // Class of ECX (for shift and JCXZ operations and cmpLTMask)
 183 reg_class ecx_reg(ECX);
 184 // Class of EDX (for multiply and divide operations)
 185 reg_class edx_reg(EDX);
 186 // Class of EDI (for synchronization)
 187 reg_class edi_reg(EDI);
 188 // Class of ESI (for synchronization)
 189 reg_class esi_reg(ESI);
 190 // Singleton class for interpreter's stack pointer
 191 reg_class ebp_reg(EBP);
 192 // Singleton class for stack pointer
 193 reg_class sp_reg(ESP);
 194 // Singleton class for instruction pointer
 195 // reg_class ip_reg(EIP);
 196 // Singleton class for condition codes
 197 reg_class int_flags(EFLAGS);
 198 // Class of integer register pairs
 199 reg_class long_reg( EAX,EDX, ECX,EBX, EBP,EDI );
 200 // Class of integer register pairs that aligns with calling convention
 201 reg_class eadx_reg( EAX,EDX );
 202 reg_class ebcx_reg( ECX,EBX );
 203 // Not AX or DX, used in divides
 204 reg_class nadx_reg( EBX,ECX,ESI,EDI,EBP );
 205 
 206 // Floating point registers.  Notice FPR0 is not a choice.
 207 // FPR0 is not ever allocated; we use clever encodings to fake
 208 // a 2-address instructions out of Intels FP stack.
 209 reg_class flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );
 210 
 211 // make a register class for SSE registers
 212 reg_class xmm_reg(XMM0a, XMM1a, XMM2a, XMM3a, XMM4a, XMM5a, XMM6a, XMM7a);
 213 
 214 // make a double register class for SSE2 registers
 215 reg_class xdb_reg(XMM0a,XMM0b, XMM1a,XMM1b, XMM2a,XMM2b, XMM3a,XMM3b,
 216                   XMM4a,XMM4b, XMM5a,XMM5b, XMM6a,XMM6b, XMM7a,XMM7b );
 217 
 218 reg_class dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,
 219                    FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,
 220                    FPR7L,FPR7H );
 221 
 222 reg_class flt_reg0( FPR1L );
 223 reg_class dbl_reg0( FPR1L,FPR1H );
 224 reg_class dbl_reg1( FPR2L,FPR2H );
 225 reg_class dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,
 226                        FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );
 227 
 228 // XMM6 and XMM7 could be used as temporary registers for long, float and
 229 // double values for SSE2.
 230 reg_class xdb_reg6( XMM6a,XMM6b );
 231 reg_class xdb_reg7( XMM7a,XMM7b );
 232 %}
 233 
 234 
 235 //----------SOURCE BLOCK-------------------------------------------------------
 236 // This is a block of C++ code which provides values, functions, and
 237 // definitions necessary in the rest of the architecture description
 238 source_hpp %{
 239 // Must be visible to the DFA in dfa_x86_32.cpp
 240 extern bool is_operand_hi32_zero(Node* n);
 241 %}
 242 
 243 source %{
 244 #define   RELOC_IMM32    Assembler::imm_operand
 245 #define   RELOC_DISP32   Assembler::disp32_operand
 246 
 247 #define __ _masm.
 248 
 249 // How to find the high register of a Long pair, given the low register
 250 #define   HIGH_FROM_LOW(x) ((x)+2)
 251 
 252 // These masks are used to provide 128-bit aligned bitmasks to the XMM
 253 // instructions, to allow sign-masking or sign-bit flipping.  They allow
 254 // fast versions of NegF/NegD and AbsF/AbsD.
 255 
 256 // Note: 'double' and 'long long' have 32-bits alignment on x86.
 257 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
 258   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
 259   // of 128-bits operands for SSE instructions.
 260   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));
 261   // Store the value to a 128-bits operand.
 262   operand[0] = lo;
 263   operand[1] = hi;
 264   return operand;
 265 }
 266 
 267 // Buffer for 128-bits masks used by SSE instructions.
 268 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
 269 
 270 // Static initialization during VM startup.
 271 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
 272 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
 273 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
 274 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
 275 
 276 // Offset hacking within calls.
 277 static int pre_call_FPU_size() {
 278   if (Compile::current()->in_24_bit_fp_mode())
 279     return 6; // fldcw
 280   return 0;
 281 }
 282 
 283 static int preserve_SP_size() {
 284   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
 285 }
 286 
 287 // !!!!! Special hack to get all type of calls to specify the byte offset
 288 //       from the start of the call to the point where the return address
 289 //       will point.
 290 int MachCallStaticJavaNode::ret_addr_offset() {
 291   int offset = 5 + pre_call_FPU_size();  // 5 bytes from start of call to where return address points
 292   if (_method_handle_invoke)
 293     offset += preserve_SP_size();
 294   return offset;
 295 }
 296 
 297 int MachCallDynamicJavaNode::ret_addr_offset() {
 298   return 10 + pre_call_FPU_size();  // 10 bytes from start of call to where return address points
 299 }
 300 
 301 static int sizeof_FFree_Float_Stack_All = -1;
 302 
 303 int MachCallRuntimeNode::ret_addr_offset() {
 304   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");
 305   return sizeof_FFree_Float_Stack_All + 5 + pre_call_FPU_size();
 306 }
 307 
 308 // Indicate if the safepoint node needs the polling page as an input.
 309 // Since x86 does have absolute addressing, it doesn't.
 310 bool SafePointNode::needs_polling_address_input() {
 311   return false;
 312 }
 313 
 314 //
 315 // Compute padding required for nodes which need alignment
 316 //
 317 
 318 // The address of the call instruction needs to be 4-byte aligned to
 319 // ensure that it does not span a cache line so that it can be patched.
 320 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {
 321   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 322   current_offset += 1;      // skip call opcode byte
 323   return round_to(current_offset, alignment_required()) - current_offset;
 324 }
 325 
 326 // The address of the call instruction needs to be 4-byte aligned to
 327 // ensure that it does not span a cache line so that it can be patched.
 328 int CallStaticJavaHandleNode::compute_padding(int current_offset) const {
 329   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 330   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 331   current_offset += 1;      // skip call opcode byte
 332   return round_to(current_offset, alignment_required()) - current_offset;
 333 }
 334 
 335 // The address of the call instruction needs to be 4-byte aligned to
 336 // ensure that it does not span a cache line so that it can be patched.
 337 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {
 338   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 339   current_offset += 5;      // skip MOV instruction
 340   current_offset += 1;      // skip call opcode byte
 341   return round_to(current_offset, alignment_required()) - current_offset;
 342 }
 343 
 344 #ifndef PRODUCT
 345 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream* st ) const {
 346   st->print("INT3");
 347 }
 348 #endif
 349 
 350 // EMIT_RM()
 351 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 352   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);
 353   cbuf.insts()->emit_int8(c);
 354 }
 355 
 356 // EMIT_CC()
 357 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 358   unsigned char c = (unsigned char)( f1 | f2 );
 359   cbuf.insts()->emit_int8(c);
 360 }
 361 
 362 // EMIT_OPCODE()
 363 void emit_opcode(CodeBuffer &cbuf, int code) {
 364   cbuf.insts()->emit_int8((unsigned char) code);
 365 }
 366 
 367 // EMIT_OPCODE() w/ relocation information
 368 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {
 369   cbuf.relocate(cbuf.insts_mark() + offset, reloc);
 370   emit_opcode(cbuf, code);
 371 }
 372 
 373 // EMIT_D8()
 374 void emit_d8(CodeBuffer &cbuf, int d8) {
 375   cbuf.insts()->emit_int8((unsigned char) d8);
 376 }
 377 
 378 // EMIT_D16()
 379 void emit_d16(CodeBuffer &cbuf, int d16) {
 380   cbuf.insts()->emit_int16(d16);
 381 }
 382 
 383 // EMIT_D32()
 384 void emit_d32(CodeBuffer &cbuf, int d32) {
 385   cbuf.insts()->emit_int32(d32);
 386 }
 387 
 388 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 389 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,
 390         int format) {
 391   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 392   cbuf.insts()->emit_int32(d32);
 393 }
 394 
 395 // emit 32 bit value and construct relocation entry from RelocationHolder
 396 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,
 397         int format) {
 398 #ifdef ASSERT
 399   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {
 400     assert(oop(d32)->is_oop() && (ScavengeRootsInCode || !oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");
 401   }
 402 #endif
 403   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 404   cbuf.insts()->emit_int32(d32);
 405 }
 406 
 407 // Access stack slot for load or store
 408 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {
 409   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])
 410   if( -128 <= disp && disp <= 127 ) {
 411     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte
 412     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 413     emit_d8 (cbuf, disp);     // Displacement  // R/M byte
 414   } else {
 415     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte
 416     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 417     emit_d32(cbuf, disp);     // Displacement  // R/M byte
 418   }
 419 }
 420 
 421    // eRegI ereg, memory mem) %{    // emit_reg_mem
 422 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, bool displace_is_oop ) {
 423   // There is no index & no scale, use form without SIB byte
 424   if ((index == 0x4) &&
 425       (scale == 0) && (base != ESP_enc)) {
 426     // If no displacement, mode is 0x0; unless base is [EBP]
 427     if ( (displace == 0) && (base != EBP_enc) ) {
 428       emit_rm(cbuf, 0x0, reg_encoding, base);
 429     }
 430     else {                    // If 8-bit displacement, mode 0x1
 431       if ((displace >= -128) && (displace <= 127)
 432           && !(displace_is_oop) ) {
 433         emit_rm(cbuf, 0x1, reg_encoding, base);
 434         emit_d8(cbuf, displace);
 435       }
 436       else {                  // If 32-bit displacement
 437         if (base == -1) { // Special flag for absolute address
 438           emit_rm(cbuf, 0x0, reg_encoding, 0x5);
 439           // (manual lies; no SIB needed here)
 440           if ( displace_is_oop ) {
 441             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 442           } else {
 443             emit_d32      (cbuf, displace);
 444           }
 445         }
 446         else {                // Normal base + offset
 447           emit_rm(cbuf, 0x2, reg_encoding, base);
 448           if ( displace_is_oop ) {
 449             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 450           } else {
 451             emit_d32      (cbuf, displace);
 452           }
 453         }
 454       }
 455     }
 456   }
 457   else {                      // Else, encode with the SIB byte
 458     // If no displacement, mode is 0x0; unless base is [EBP]
 459     if (displace == 0 && (base != EBP_enc)) {  // If no displacement
 460       emit_rm(cbuf, 0x0, reg_encoding, 0x4);
 461       emit_rm(cbuf, scale, index, base);
 462     }
 463     else {                    // If 8-bit displacement, mode 0x1
 464       if ((displace >= -128) && (displace <= 127)
 465           && !(displace_is_oop) ) {
 466         emit_rm(cbuf, 0x1, reg_encoding, 0x4);
 467         emit_rm(cbuf, scale, index, base);
 468         emit_d8(cbuf, displace);
 469       }
 470       else {                  // If 32-bit displacement
 471         if (base == 0x04 ) {
 472           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 473           emit_rm(cbuf, scale, index, 0x04);
 474         } else {
 475           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 476           emit_rm(cbuf, scale, index, base);
 477         }
 478         if ( displace_is_oop ) {
 479           emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 480         } else {
 481           emit_d32      (cbuf, displace);
 482         }
 483       }
 484     }
 485   }
 486 }
 487 
 488 
 489 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 490   if( dst_encoding == src_encoding ) {
 491     // reg-reg copy, use an empty encoding
 492   } else {
 493     emit_opcode( cbuf, 0x8B );
 494     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );
 495   }
 496 }
 497 
 498 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 499   if( dst_encoding == src_encoding ) {
 500     // reg-reg copy, use an empty encoding
 501   } else {
 502     MacroAssembler _masm(&cbuf);
 503 
 504     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
 505   }
 506 }
 507 
 508 
 509 //=============================================================================
 510 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 511 
 512 int Compile::ConstantTable::calculate_table_base_offset() const {
 513   return 0;  // absolute addressing, no offset
 514 }
 515 
 516 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 517   // Empty encoding
 518 }
 519 
 520 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 521   return 0;
 522 }
 523 
 524 #ifndef PRODUCT
 525 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 526   st->print("# MachConstantBaseNode (empty encoding)");
 527 }
 528 #endif
 529 
 530 
 531 //=============================================================================
 532 #ifndef PRODUCT
 533 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 534   Compile* C = ra_->C;
 535   if( C->in_24_bit_fp_mode() ) {
 536     st->print("FLDCW  24 bit fpu control word");
 537     st->print_cr(""); st->print("\t");
 538   }
 539 
 540   int framesize = C->frame_slots() << LogBytesPerInt;
 541   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 542   // Remove two words for return addr and rbp,
 543   framesize -= 2*wordSize;
 544 
 545   // Calls to C2R adapters often do not accept exceptional returns.
 546   // We require that their callers must bang for them.  But be careful, because
 547   // some VM calls (such as call site linkage) can use several kilobytes of
 548   // stack.  But the stack safety zone should account for that.
 549   // See bugs 4446381, 4468289, 4497237.
 550   if (C->need_stack_bang(framesize)) {
 551     st->print_cr("# stack bang"); st->print("\t");
 552   }
 553   st->print_cr("PUSHL  EBP"); st->print("\t");
 554 
 555   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
 556     st->print("PUSH   0xBADB100D\t# Majik cookie for stack depth check");
 557     st->print_cr(""); st->print("\t");
 558     framesize -= wordSize;
 559   }
 560 
 561   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
 562     if (framesize) {
 563       st->print("SUB    ESP,%d\t# Create frame",framesize);
 564     }
 565   } else {
 566     st->print("SUB    ESP,%d\t# Create frame",framesize);
 567   }
 568 }
 569 #endif
 570 
 571 
 572 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 573   Compile* C = ra_->C;
 574 
 575   if (UseSSE >= 2 && VerifyFPU) {
 576     MacroAssembler masm(&cbuf);
 577     masm.verify_FPU(0, "FPU stack must be clean on entry");
 578   }
 579 
 580   // WARNING: Initial instruction MUST be 5 bytes or longer so that
 581   // NativeJump::patch_verified_entry will be able to patch out the entry
 582   // code safely. The fldcw is ok at 6 bytes, the push to verify stack
 583   // depth is ok at 5 bytes, the frame allocation can be either 3 or
 584   // 6 bytes. So if we don't do the fldcw or the push then we must
 585   // use the 6 byte frame allocation even if we have no frame. :-(
 586   // If method sets FPU control word do it now
 587   if( C->in_24_bit_fp_mode() ) {
 588     MacroAssembler masm(&cbuf);
 589     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 590   }
 591 
 592   int framesize = C->frame_slots() << LogBytesPerInt;
 593   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 594   // Remove two words for return addr and rbp,
 595   framesize -= 2*wordSize;
 596 
 597   // Calls to C2R adapters often do not accept exceptional returns.
 598   // We require that their callers must bang for them.  But be careful, because
 599   // some VM calls (such as call site linkage) can use several kilobytes of
 600   // stack.  But the stack safety zone should account for that.
 601   // See bugs 4446381, 4468289, 4497237.
 602   if (C->need_stack_bang(framesize)) {
 603     MacroAssembler masm(&cbuf);
 604     masm.generate_stack_overflow_check(framesize);
 605   }
 606 
 607   // We always push rbp, so that on return to interpreter rbp, will be
 608   // restored correctly and we can correct the stack.
 609   emit_opcode(cbuf, 0x50 | EBP_enc);
 610 
 611   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
 612     emit_opcode(cbuf, 0x68); // push 0xbadb100d
 613     emit_d32(cbuf, 0xbadb100d);
 614     framesize -= wordSize;
 615   }
 616 
 617   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
 618     if (framesize) {
 619       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
 620       emit_rm(cbuf, 0x3, 0x05, ESP_enc);
 621       emit_d8(cbuf, framesize);
 622     }
 623   } else {
 624     emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
 625     emit_rm(cbuf, 0x3, 0x05, ESP_enc);
 626     emit_d32(cbuf, framesize);
 627   }
 628   C->set_frame_complete(cbuf.insts_size());
 629 
 630 #ifdef ASSERT
 631   if (VerifyStackAtCalls) {
 632     Label L;
 633     MacroAssembler masm(&cbuf);
 634     masm.push(rax);
 635     masm.mov(rax, rsp);
 636     masm.andptr(rax, StackAlignmentInBytes-1);
 637     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
 638     masm.pop(rax);
 639     masm.jcc(Assembler::equal, L);
 640     masm.stop("Stack is not properly aligned!");
 641     masm.bind(L);
 642   }
 643 #endif
 644 
 645   if (C->has_mach_constant_base_node()) {
 646     // NOTE: We set the table base offset here because users might be
 647     // emitted before MachConstantBaseNode.
 648     Compile::ConstantTable& constant_table = C->constant_table();
 649     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 650   }
 651 }
 652 
 653 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 654   return MachNode::size(ra_); // too many variables; just compute it the hard way
 655 }
 656 
 657 int MachPrologNode::reloc() const {
 658   return 0; // a large enough number
 659 }
 660 
 661 //=============================================================================
 662 #ifndef PRODUCT
 663 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 664   Compile *C = ra_->C;
 665   int framesize = C->frame_slots() << LogBytesPerInt;
 666   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 667   // Remove two words for return addr and rbp,
 668   framesize -= 2*wordSize;
 669 
 670   if( C->in_24_bit_fp_mode() ) {
 671     st->print("FLDCW  standard control word");
 672     st->cr(); st->print("\t");
 673   }
 674   if( framesize ) {
 675     st->print("ADD    ESP,%d\t# Destroy frame",framesize);
 676     st->cr(); st->print("\t");
 677   }
 678   st->print_cr("POPL   EBP"); st->print("\t");
 679   if( do_polling() && C->is_method_compilation() ) {
 680     st->print("TEST   PollPage,EAX\t! Poll Safepoint");
 681     st->cr(); st->print("\t");
 682   }
 683 }
 684 #endif
 685 
 686 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 687   Compile *C = ra_->C;
 688 
 689   // If method set FPU control word, restore to standard control word
 690   if( C->in_24_bit_fp_mode() ) {
 691     MacroAssembler masm(&cbuf);
 692     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 693   }
 694 
 695   int framesize = C->frame_slots() << LogBytesPerInt;
 696   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 697   // Remove two words for return addr and rbp,
 698   framesize -= 2*wordSize;
 699 
 700   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 701 
 702   if( framesize >= 128 ) {
 703     emit_opcode(cbuf, 0x81); // add  SP, #framesize
 704     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 705     emit_d32(cbuf, framesize);
 706   }
 707   else if( framesize ) {
 708     emit_opcode(cbuf, 0x83); // add  SP, #framesize
 709     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 710     emit_d8(cbuf, framesize);
 711   }
 712 
 713   emit_opcode(cbuf, 0x58 | EBP_enc);
 714 
 715   if( do_polling() && C->is_method_compilation() ) {
 716     cbuf.relocate(cbuf.insts_end(), relocInfo::poll_return_type, 0);
 717     emit_opcode(cbuf,0x85);
 718     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX
 719     emit_d32(cbuf, (intptr_t)os::get_polling_page());
 720   }
 721 }
 722 
 723 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 724   Compile *C = ra_->C;
 725   // If method set FPU control word, restore to standard control word
 726   int size = C->in_24_bit_fp_mode() ? 6 : 0;
 727   if( do_polling() && C->is_method_compilation() ) size += 6;
 728 
 729   int framesize = C->frame_slots() << LogBytesPerInt;
 730   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 731   // Remove two words for return addr and rbp,
 732   framesize -= 2*wordSize;
 733 
 734   size++; // popl rbp,
 735 
 736   if( framesize >= 128 ) {
 737     size += 6;
 738   } else {
 739     size += framesize ? 3 : 0;
 740   }
 741   return size;
 742 }
 743 
 744 int MachEpilogNode::reloc() const {
 745   return 0; // a large enough number
 746 }
 747 
 748 const Pipeline * MachEpilogNode::pipeline() const {
 749   return MachNode::pipeline_class();
 750 }
 751 
 752 int MachEpilogNode::safepoint_offset() const { return 0; }
 753 
 754 //=============================================================================
 755 
 756 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };
 757 static enum RC rc_class( OptoReg::Name reg ) {
 758 
 759   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 760   if (OptoReg::is_stack(reg)) return rc_stack;
 761 
 762   VMReg r = OptoReg::as_VMReg(reg);
 763   if (r->is_Register()) return rc_int;
 764   if (r->is_FloatRegister()) {
 765     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");
 766     return rc_float;
 767   }
 768   assert(r->is_XMMRegister(), "must be");
 769   return rc_xmm;
 770 }
 771 
 772 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,
 773                         int opcode, const char *op_str, int size, outputStream* st ) {
 774   if( cbuf ) {
 775     emit_opcode  (*cbuf, opcode );
 776     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, false);
 777 #ifndef PRODUCT
 778   } else if( !do_size ) {
 779     if( size != 0 ) st->print("\n\t");
 780     if( opcode == 0x8B || opcode == 0x89 ) { // MOV
 781       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);
 782       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);
 783     } else { // FLD, FST, PUSH, POP
 784       st->print("%s [ESP + #%d]",op_str,offset);
 785     }
 786 #endif
 787   }
 788   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 789   return size+3+offset_size;
 790 }
 791 
 792 // Helper for XMM registers.  Extra opcode bits, limited syntax.
 793 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,
 794                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {
 795   if( cbuf ) {
 796     if( reg_lo+1 == reg_hi ) { // double move?
 797       if( is_load && !UseXmmLoadAndClearUpper )
 798         emit_opcode(*cbuf, 0x66 ); // use 'movlpd' for load
 799       else
 800         emit_opcode(*cbuf, 0xF2 ); // use 'movsd' otherwise
 801     } else {
 802       emit_opcode(*cbuf, 0xF3 );
 803     }
 804     emit_opcode(*cbuf, 0x0F );
 805     if( reg_lo+1 == reg_hi && is_load && !UseXmmLoadAndClearUpper )
 806       emit_opcode(*cbuf, 0x12 );   // use 'movlpd' for load
 807     else
 808       emit_opcode(*cbuf, is_load ? 0x10 : 0x11 );
 809     encode_RegMem(*cbuf, Matcher::_regEncode[reg_lo], ESP_enc, 0x4, 0, offset, false);
 810 #ifndef PRODUCT
 811   } else if( !do_size ) {
 812     if( size != 0 ) st->print("\n\t");
 813     if( reg_lo+1 == reg_hi ) { // double move?
 814       if( is_load ) st->print("%s %s,[ESP + #%d]",
 815                                UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",
 816                                Matcher::regName[reg_lo], offset);
 817       else          st->print("MOVSD  [ESP + #%d],%s",
 818                                offset, Matcher::regName[reg_lo]);
 819     } else {
 820       if( is_load ) st->print("MOVSS  %s,[ESP + #%d]",
 821                                Matcher::regName[reg_lo], offset);
 822       else          st->print("MOVSS  [ESP + #%d],%s",
 823                                offset, Matcher::regName[reg_lo]);
 824     }
 825 #endif
 826   }
 827   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 828   return size+5+offset_size;
 829 }
 830 
 831 
 832 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 833                             int src_hi, int dst_hi, int size, outputStream* st ) {
 834   if( UseXmmRegToRegMoveAll ) {//Use movaps,movapd to move between xmm registers
 835     if( cbuf ) {
 836       if( (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ) {
 837         emit_opcode(*cbuf, 0x66 );
 838       }
 839       emit_opcode(*cbuf, 0x0F );
 840       emit_opcode(*cbuf, 0x28 );
 841       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
 842 #ifndef PRODUCT
 843     } else if( !do_size ) {
 844       if( size != 0 ) st->print("\n\t");
 845       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
 846         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 847       } else {
 848         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 849       }
 850 #endif
 851     }
 852     return size + ((src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 4 : 3);
 853   } else {
 854     if( cbuf ) {
 855       emit_opcode(*cbuf, (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 0xF2 : 0xF3 );
 856       emit_opcode(*cbuf, 0x0F );
 857       emit_opcode(*cbuf, 0x10 );
 858       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
 859 #ifndef PRODUCT
 860     } else if( !do_size ) {
 861       if( size != 0 ) st->print("\n\t");
 862       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
 863         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 864       } else {
 865         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 866       }
 867 #endif
 868     }
 869     return size+4;
 870   }
 871 }
 872 
 873 static int impl_movgpr2x_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 874                             int src_hi, int dst_hi, int size, outputStream* st ) {
 875   // 32-bit
 876   if (cbuf) {
 877     emit_opcode(*cbuf, 0x66);
 878     emit_opcode(*cbuf, 0x0F);
 879     emit_opcode(*cbuf, 0x6E);
 880     emit_rm(*cbuf, 0x3, Matcher::_regEncode[dst_lo] & 7, Matcher::_regEncode[src_lo] & 7);
 881 #ifndef PRODUCT
 882   } else if (!do_size) {
 883     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
 884 #endif
 885   }
 886   return 4;
 887 }
 888 
 889 
 890 static int impl_movx2gpr_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 891                                  int src_hi, int dst_hi, int size, outputStream* st ) {
 892   // 32-bit
 893   if (cbuf) {
 894     emit_opcode(*cbuf, 0x66);
 895     emit_opcode(*cbuf, 0x0F);
 896     emit_opcode(*cbuf, 0x7E);
 897     emit_rm(*cbuf, 0x3, Matcher::_regEncode[src_lo] & 7, Matcher::_regEncode[dst_lo] & 7);
 898 #ifndef PRODUCT
 899   } else if (!do_size) {
 900     st->print("movdl   %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
 901 #endif
 902   }
 903   return 4;
 904 }
 905 
 906 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {
 907   if( cbuf ) {
 908     emit_opcode(*cbuf, 0x8B );
 909     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );
 910 #ifndef PRODUCT
 911   } else if( !do_size ) {
 912     if( size != 0 ) st->print("\n\t");
 913     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);
 914 #endif
 915   }
 916   return size+2;
 917 }
 918 
 919 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,
 920                                  int offset, int size, outputStream* st ) {
 921   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there
 922     if( cbuf ) {
 923       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)
 924       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );
 925 #ifndef PRODUCT
 926     } else if( !do_size ) {
 927       if( size != 0 ) st->print("\n\t");
 928       st->print("FLD    %s",Matcher::regName[src_lo]);
 929 #endif
 930     }
 931     size += 2;
 932   }
 933 
 934   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;
 935   const char *op_str;
 936   int op;
 937   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?
 938     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";
 939     op = 0xDD;
 940   } else {                   // 32-bit store
 941     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";
 942     op = 0xD9;
 943     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );
 944   }
 945 
 946   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);
 947 }
 948 
 949 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {
 950   // Get registers to move
 951   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 952   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 953   OptoReg::Name dst_second = ra_->get_reg_second(this );
 954   OptoReg::Name dst_first = ra_->get_reg_first(this );
 955 
 956   enum RC src_second_rc = rc_class(src_second);
 957   enum RC src_first_rc = rc_class(src_first);
 958   enum RC dst_second_rc = rc_class(dst_second);
 959   enum RC dst_first_rc = rc_class(dst_first);
 960 
 961   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
 962 
 963   // Generate spill code!
 964   int size = 0;
 965 
 966   if( src_first == dst_first && src_second == dst_second )
 967     return size;            // Self copy, no move
 968 
 969   // --------------------------------------
 970   // Check for mem-mem move.  push/pop to move.
 971   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 972     if( src_second == dst_first ) { // overlapping stack copy ranges
 973       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );
 974       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
 975       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
 976       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits
 977     }
 978     // move low bits
 979     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);
 980     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);
 981     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits
 982       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
 983       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
 984     }
 985     return size;
 986   }
 987 
 988   // --------------------------------------
 989   // Check for integer reg-reg copy
 990   if( src_first_rc == rc_int && dst_first_rc == rc_int )
 991     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);
 992 
 993   // Check for integer store
 994   if( src_first_rc == rc_int && dst_first_rc == rc_stack )
 995     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);
 996 
 997   // Check for integer load
 998   if( dst_first_rc == rc_int && src_first_rc == rc_stack )
 999     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);
1000 
1001   // Check for integer reg-xmm reg copy
1002   if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {
1003     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
1004             "no 64 bit integer-float reg moves" );
1005     return impl_movgpr2x_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1006   }
1007   // --------------------------------------
1008   // Check for float reg-reg copy
1009   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {
1010     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
1011             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );
1012     if( cbuf ) {
1013 
1014       // Note the mucking with the register encode to compensate for the 0/1
1015       // indexing issue mentioned in a comment in the reg_def sections
1016       // for FPR registers many lines above here.
1017 
1018       if( src_first != FPR1L_num ) {
1019         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)
1020         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );
1021         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
1022         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
1023      } else {
1024         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)
1025         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );
1026      }
1027 #ifndef PRODUCT
1028     } else if( !do_size ) {
1029       if( size != 0 ) st->print("\n\t");
1030       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);
1031       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);
1032 #endif
1033     }
1034     return size + ((src_first != FPR1L_num) ? 2+2 : 2);
1035   }
1036 
1037   // Check for float store
1038   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1039     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);
1040   }
1041 
1042   // Check for float load
1043   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {
1044     int offset = ra_->reg2offset(src_first);
1045     const char *op_str;
1046     int op;
1047     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?
1048       op_str = "FLD_D";
1049       op = 0xDD;
1050     } else {                   // 32-bit load
1051       op_str = "FLD_S";
1052       op = 0xD9;
1053       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );
1054     }
1055     if( cbuf ) {
1056       emit_opcode  (*cbuf, op );
1057       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, false);
1058       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
1059       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
1060 #ifndef PRODUCT
1061     } else if( !do_size ) {
1062       if( size != 0 ) st->print("\n\t");
1063       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);
1064 #endif
1065     }
1066     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
1067     return size + 3+offset_size+2;
1068   }
1069 
1070   // Check for xmm reg-reg copy
1071   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
1072     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
1073             (src_first+1 == src_second && dst_first+1 == dst_second),
1074             "no non-adjacent float-moves" );
1075     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1076   }
1077 
1078   // Check for xmm reg-integer reg copy
1079   if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {
1080     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
1081             "no 64 bit float-integer reg moves" );
1082     return impl_movx2gpr_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1083   }
1084 
1085   // Check for xmm store
1086   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
1087     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);
1088   }
1089 
1090   // Check for float xmm load
1091   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {
1092     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);
1093   }
1094 
1095   // Copy from float reg to xmm reg
1096   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {
1097     // copy to the top of stack from floating point reg
1098     // and use LEA to preserve flags
1099     if( cbuf ) {
1100       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]
1101       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1102       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1103       emit_d8(*cbuf,0xF8);
1104 #ifndef PRODUCT
1105     } else if( !do_size ) {
1106       if( size != 0 ) st->print("\n\t");
1107       st->print("LEA    ESP,[ESP-8]");
1108 #endif
1109     }
1110     size += 4;
1111 
1112     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);
1113 
1114     // Copy from the temp memory to the xmm reg.
1115     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);
1116 
1117     if( cbuf ) {
1118       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]
1119       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1120       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1121       emit_d8(*cbuf,0x08);
1122 #ifndef PRODUCT
1123     } else if( !do_size ) {
1124       if( size != 0 ) st->print("\n\t");
1125       st->print("LEA    ESP,[ESP+8]");
1126 #endif
1127     }
1128     size += 4;
1129     return size;
1130   }
1131 
1132   assert( size > 0, "missed a case" );
1133 
1134   // --------------------------------------------------------------------
1135   // Check for second bits still needing moving.
1136   if( src_second == dst_second )
1137     return size;               // Self copy; no move
1138   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
1139 
1140   // Check for second word int-int move
1141   if( src_second_rc == rc_int && dst_second_rc == rc_int )
1142     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);
1143 
1144   // Check for second word integer store
1145   if( src_second_rc == rc_int && dst_second_rc == rc_stack )
1146     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);
1147 
1148   // Check for second word integer load
1149   if( dst_second_rc == rc_int && src_second_rc == rc_stack )
1150     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);
1151 
1152 
1153   Unimplemented();
1154 }
1155 
1156 #ifndef PRODUCT
1157 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1158   implementation( NULL, ra_, false, st );
1159 }
1160 #endif
1161 
1162 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1163   implementation( &cbuf, ra_, false, NULL );
1164 }
1165 
1166 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1167   return implementation( NULL, ra_, true, NULL );
1168 }
1169 
1170 //=============================================================================
1171 #ifndef PRODUCT
1172 void MachNopNode::format( PhaseRegAlloc *, outputStream* st ) const {
1173   st->print("NOP \t# %d bytes pad for loops and calls", _count);
1174 }
1175 #endif
1176 
1177 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
1178   MacroAssembler _masm(&cbuf);
1179   __ nop(_count);
1180 }
1181 
1182 uint MachNopNode::size(PhaseRegAlloc *) const {
1183   return _count;
1184 }
1185 
1186 
1187 //=============================================================================
1188 #ifndef PRODUCT
1189 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1190   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1191   int reg = ra_->get_reg_first(this);
1192   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);
1193 }
1194 #endif
1195 
1196 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1197   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1198   int reg = ra_->get_encode(this);
1199   if( offset >= 128 ) {
1200     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1201     emit_rm(cbuf, 0x2, reg, 0x04);
1202     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1203     emit_d32(cbuf, offset);
1204   }
1205   else {
1206     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1207     emit_rm(cbuf, 0x1, reg, 0x04);
1208     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1209     emit_d8(cbuf, offset);
1210   }
1211 }
1212 
1213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1214   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1215   if( offset >= 128 ) {
1216     return 7;
1217   }
1218   else {
1219     return 4;
1220   }
1221 }
1222 
1223 //=============================================================================
1224 
1225 // emit call stub, compiled java to interpreter
1226 void emit_java_to_interp(CodeBuffer &cbuf ) {
1227   // Stub is fixed up when the corresponding call is converted from calling
1228   // compiled code to calling interpreted code.
1229   // mov rbx,0
1230   // jmp -1
1231 
1232   address mark = cbuf.insts_mark();  // get mark within main instrs section
1233 
1234   // Note that the code buffer's insts_mark is always relative to insts.
1235   // That's why we must use the macroassembler to generate a stub.
1236   MacroAssembler _masm(&cbuf);
1237 
1238   address base =
1239   __ start_a_stub(Compile::MAX_stubs_size);
1240   if (base == NULL)  return;  // CodeBuffer::expand failed
1241   // static stub relocation stores the instruction address of the call
1242   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32);
1243   // static stub relocation also tags the methodOop in the code-stream.
1244   __ movoop(rbx, (jobject)NULL);  // method is zapped till fixup time
1245   // This is recognized as unresolved by relocs/nativeInst/ic code
1246   __ jump(RuntimeAddress(__ pc()));
1247 
1248   __ end_a_stub();
1249   // Update current stubs pointer and restore insts_end.
1250 }
1251 // size of call stub, compiled java to interpretor
1252 uint size_java_to_interp() {
1253   return 10;  // movl; jmp
1254 }
1255 // relocation entries for call stub, compiled java to interpretor
1256 uint reloc_java_to_interp() {
1257   return 4;  // 3 in emit_java_to_interp + 1 in Java_Static_Call
1258 }
1259 
1260 //=============================================================================
1261 #ifndef PRODUCT
1262 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1263   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");
1264   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");
1265   st->print_cr("\tNOP");
1266   st->print_cr("\tNOP");
1267   if( !OptoBreakpoint )
1268     st->print_cr("\tNOP");
1269 }
1270 #endif
1271 
1272 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1273   MacroAssembler masm(&cbuf);
1274 #ifdef ASSERT
1275   uint insts_size = cbuf.insts_size();
1276 #endif
1277   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
1278   masm.jump_cc(Assembler::notEqual,
1279                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1280   /* WARNING these NOPs are critical so that verified entry point is properly
1281      aligned for patching by NativeJump::patch_verified_entry() */
1282   int nops_cnt = 2;
1283   if( !OptoBreakpoint ) // Leave space for int3
1284      nops_cnt += 1;
1285   masm.nop(nops_cnt);
1286 
1287   assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
1288 }
1289 
1290 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
1291   return OptoBreakpoint ? 11 : 12;
1292 }
1293 
1294 
1295 //=============================================================================
1296 uint size_exception_handler() {
1297   // NativeCall instruction size is the same as NativeJump.
1298   // exception handler starts out as jump and can be patched to
1299   // a call be deoptimization.  (4932387)
1300   // Note that this value is also credited (in output.cpp) to
1301   // the size of the code section.
1302   return NativeJump::instruction_size;
1303 }
1304 
1305 // Emit exception handler code.  Stuff framesize into a register
1306 // and call a VM stub routine.
1307 int emit_exception_handler(CodeBuffer& cbuf) {
1308 
1309   // Note that the code buffer's insts_mark is always relative to insts.
1310   // That's why we must use the macroassembler to generate a handler.
1311   MacroAssembler _masm(&cbuf);
1312   address base =
1313   __ start_a_stub(size_exception_handler());
1314   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1315   int offset = __ offset();
1316   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1317   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1318   __ end_a_stub();
1319   return offset;
1320 }
1321 
1322 uint size_deopt_handler() {
1323   // NativeCall instruction size is the same as NativeJump.
1324   // exception handler starts out as jump and can be patched to
1325   // a call be deoptimization.  (4932387)
1326   // Note that this value is also credited (in output.cpp) to
1327   // the size of the code section.
1328   return 5 + NativeJump::instruction_size; // pushl(); jmp;
1329 }
1330 
1331 // Emit deopt handler code.
1332 int emit_deopt_handler(CodeBuffer& cbuf) {
1333 
1334   // Note that the code buffer's insts_mark is always relative to insts.
1335   // That's why we must use the macroassembler to generate a handler.
1336   MacroAssembler _masm(&cbuf);
1337   address base =
1338   __ start_a_stub(size_exception_handler());
1339   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1340   int offset = __ offset();
1341   InternalAddress here(__ pc());
1342   __ pushptr(here.addr());
1343 
1344   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1345   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1346   __ end_a_stub();
1347   return offset;
1348 }
1349 
1350 
1351 const bool Matcher::match_rule_supported(int opcode) {
1352   if (!has_match_rule(opcode))
1353     return false;
1354 
1355   return true;  // Per default match rules are supported.
1356 }
1357 
1358 int Matcher::regnum_to_fpu_offset(int regnum) {
1359   return regnum - 32; // The FP registers are in the second chunk
1360 }
1361 
1362 // This is UltraSparc specific, true just means we have fast l2f conversion
1363 const bool Matcher::convL2FSupported(void) {
1364   return true;
1365 }
1366 
1367 // Vector width in bytes
1368 const uint Matcher::vector_width_in_bytes(void) {
1369   return UseSSE >= 2 ? 8 : 0;
1370 }
1371 
1372 // Vector ideal reg
1373 const uint Matcher::vector_ideal_reg(void) {
1374   return Op_RegD;
1375 }
1376 
1377 // Is this branch offset short enough that a short branch can be used?
1378 //
1379 // NOTE: If the platform does not provide any short branch variants, then
1380 //       this method should return false for offset 0.
1381 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1382   // The passed offset is relative to address of the branch.
1383   // On 86 a branch displacement is calculated relative to address
1384   // of a next instruction.
1385   offset -= br_size;
1386 
1387   // the short version of jmpConUCF2 contains multiple branches,
1388   // making the reach slightly less
1389   if (rule == jmpConUCF2_rule)
1390     return (-126 <= offset && offset <= 125);
1391   return (-128 <= offset && offset <= 127);
1392 }
1393 
1394 const bool Matcher::isSimpleConstant64(jlong value) {
1395   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1396   return false;
1397 }
1398 
1399 // The ecx parameter to rep stos for the ClearArray node is in dwords.
1400 const bool Matcher::init_array_count_is_in_bytes = false;
1401 
1402 // Threshold size for cleararray.
1403 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1404 
1405 // Needs 2 CMOV's for longs.
1406 const int Matcher::long_cmove_cost() { return 1; }
1407 
1408 // No CMOVF/CMOVD with SSE/SSE2
1409 const int Matcher::float_cmove_cost() { return (UseSSE>=1) ? ConditionalMoveLimit : 0; }
1410 
1411 // Should the Matcher clone shifts on addressing modes, expecting them to
1412 // be subsumed into complex addressing expressions or compute them into
1413 // registers?  True for Intel but false for most RISCs
1414 const bool Matcher::clone_shift_expressions = true;
1415 
1416 // Do we need to mask the count passed to shift instructions or does
1417 // the cpu only look at the lower 5/6 bits anyway?
1418 const bool Matcher::need_masked_shift_count = false;
1419 
1420 bool Matcher::narrow_oop_use_complex_address() {
1421   ShouldNotCallThis();
1422   return true;
1423 }
1424 
1425 
1426 // Is it better to copy float constants, or load them directly from memory?
1427 // Intel can load a float constant from a direct address, requiring no
1428 // extra registers.  Most RISCs will have to materialize an address into a
1429 // register first, so they would do better to copy the constant from stack.
1430 const bool Matcher::rematerialize_float_constants = true;
1431 
1432 // If CPU can load and store mis-aligned doubles directly then no fixup is
1433 // needed.  Else we split the double into 2 integer pieces and move it
1434 // piece-by-piece.  Only happens when passing doubles into C code as the
1435 // Java calling convention forces doubles to be aligned.
1436 const bool Matcher::misaligned_doubles_ok = true;
1437 
1438 
1439 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1440   // Get the memory operand from the node
1441   uint numopnds = node->num_opnds();        // Virtual call for number of operands
1442   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far
1443   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );
1444   uint opcnt     = 1;                 // First operand
1445   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand
1446   while( idx >= skipped+num_edges ) {
1447     skipped += num_edges;
1448     opcnt++;                          // Bump operand count
1449     assert( opcnt < numopnds, "Accessing non-existent operand" );
1450     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand
1451   }
1452 
1453   MachOper *memory = node->_opnds[opcnt];
1454   MachOper *new_memory = NULL;
1455   switch (memory->opcode()) {
1456   case DIRECT:
1457   case INDOFFSET32X:
1458     // No transformation necessary.
1459     return;
1460   case INDIRECT:
1461     new_memory = new (C) indirect_win95_safeOper( );
1462     break;
1463   case INDOFFSET8:
1464     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));
1465     break;
1466   case INDOFFSET32:
1467     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));
1468     break;
1469   case INDINDEXOFFSET:
1470     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));
1471     break;
1472   case INDINDEXSCALE:
1473     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());
1474     break;
1475   case INDINDEXSCALEOFFSET:
1476     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));
1477     break;
1478   case LOAD_LONG_INDIRECT:
1479   case LOAD_LONG_INDOFFSET32:
1480     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}
1481     return;
1482   default:
1483     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");
1484     return;
1485   }
1486   node->_opnds[opcnt] = new_memory;
1487 }
1488 
1489 // Advertise here if the CPU requires explicit rounding operations
1490 // to implement the UseStrictFP mode.
1491 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1492 
1493 // Are floats conerted to double when stored to stack during deoptimization?
1494 // On x32 it is stored with convertion only when FPU is used for floats.
1495 bool Matcher::float_in_double() { return (UseSSE == 0); }
1496 
1497 // Do ints take an entire long register or just half?
1498 const bool Matcher::int_in_long = false;
1499 
1500 // Return whether or not this register is ever used as an argument.  This
1501 // function is used on startup to build the trampoline stubs in generateOptoStub.
1502 // Registers not mentioned will be killed by the VM call in the trampoline, and
1503 // arguments in those registers not be available to the callee.
1504 bool Matcher::can_be_java_arg( int reg ) {
1505   if(  reg == ECX_num   || reg == EDX_num   ) return true;
1506   if( (reg == XMM0a_num || reg == XMM1a_num) && UseSSE>=1 ) return true;
1507   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;
1508   return false;
1509 }
1510 
1511 bool Matcher::is_spillable_arg( int reg ) {
1512   return can_be_java_arg(reg);
1513 }
1514 
1515 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1516   // Use hardware integer DIV instruction when
1517   // it is faster than a code which use multiply.
1518   // Only when constant divisor fits into 32 bit
1519   // (min_jint is excluded to get only correct
1520   // positive 32 bit values from negative).
1521   return VM_Version::has_fast_idiv() &&
1522          (divisor == (int)divisor && divisor != min_jint);
1523 }
1524 
1525 // Register for DIVI projection of divmodI
1526 RegMask Matcher::divI_proj_mask() {
1527   return EAX_REG_mask();
1528 }
1529 
1530 // Register for MODI projection of divmodI
1531 RegMask Matcher::modI_proj_mask() {
1532   return EDX_REG_mask();
1533 }
1534 
1535 // Register for DIVL projection of divmodL
1536 RegMask Matcher::divL_proj_mask() {
1537   ShouldNotReachHere();
1538   return RegMask();
1539 }
1540 
1541 // Register for MODL projection of divmodL
1542 RegMask Matcher::modL_proj_mask() {
1543   ShouldNotReachHere();
1544   return RegMask();
1545 }
1546 
1547 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1548   return EBP_REG_mask();
1549 }
1550 
1551 // Returns true if the high 32 bits of the value is known to be zero.
1552 bool is_operand_hi32_zero(Node* n) {
1553   int opc = n->Opcode();
1554   if (opc == Op_LoadUI2L) {
1555     return true;
1556   }
1557   if (opc == Op_AndL) {
1558     Node* o2 = n->in(2);
1559     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
1560       return true;
1561     }
1562   }
1563   if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
1564     return true;
1565   }
1566   return false;
1567 }
1568 
1569 %}
1570 
1571 //----------ENCODING BLOCK-----------------------------------------------------
1572 // This block specifies the encoding classes used by the compiler to output
1573 // byte streams.  Encoding classes generate functions which are called by
1574 // Machine Instruction Nodes in order to generate the bit encoding of the
1575 // instruction.  Operands specify their base encoding interface with the
1576 // interface keyword.  There are currently supported four interfaces,
1577 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1578 // operand to generate a function which returns its register number when
1579 // queried.   CONST_INTER causes an operand to generate a function which
1580 // returns the value of the constant when queried.  MEMORY_INTER causes an
1581 // operand to generate four functions which return the Base Register, the
1582 // Index Register, the Scale Value, and the Offset Value of the operand when
1583 // queried.  COND_INTER causes an operand to generate six functions which
1584 // return the encoding code (ie - encoding bits for the instruction)
1585 // associated with each basic boolean condition for a conditional instruction.
1586 // Instructions specify two basic values for encoding.  They use the
1587 // ins_encode keyword to specify their encoding class (which must be one of
1588 // the class names specified in the encoding block), and they use the
1589 // opcode keyword to specify, in order, their primary, secondary, and
1590 // tertiary opcode.  Only the opcode sections which a particular instruction
1591 // needs for encoding need to be specified.
1592 encode %{
1593   // Build emit functions for each basic byte or larger field in the intel
1594   // encoding scheme (opcode, rm, sib, immediate), and call them from C++
1595   // code in the enc_class source block.  Emit functions will live in the
1596   // main source block for now.  In future, we can generalize this by
1597   // adding a syntax that specifies the sizes of fields in an order,
1598   // so that the adlc can build the emit functions automagically
1599 
1600   // Emit primary opcode
1601   enc_class OpcP %{
1602     emit_opcode(cbuf, $primary);
1603   %}
1604 
1605   // Emit secondary opcode
1606   enc_class OpcS %{
1607     emit_opcode(cbuf, $secondary);
1608   %}
1609 
1610   // Emit opcode directly
1611   enc_class Opcode(immI d8) %{
1612     emit_opcode(cbuf, $d8$$constant);
1613   %}
1614 
1615   enc_class SizePrefix %{
1616     emit_opcode(cbuf,0x66);
1617   %}
1618 
1619   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
1620     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1621   %}
1622 
1623   enc_class OpcRegReg (immI opcode, eRegI dst, eRegI src) %{    // OpcRegReg(Many)
1624     emit_opcode(cbuf,$opcode$$constant);
1625     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1626   %}
1627 
1628   enc_class mov_r32_imm0( eRegI dst ) %{
1629     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32
1630     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0
1631   %}
1632 
1633   enc_class cdq_enc %{
1634     // Full implementation of Java idiv and irem; checks for
1635     // special case as described in JVM spec., p.243 & p.271.
1636     //
1637     //         normal case                           special case
1638     //
1639     // input : rax,: dividend                         min_int
1640     //         reg: divisor                          -1
1641     //
1642     // output: rax,: quotient  (= rax, idiv reg)       min_int
1643     //         rdx: remainder (= rax, irem reg)       0
1644     //
1645     //  Code sequnce:
1646     //
1647     //  81 F8 00 00 00 80    cmp         rax,80000000h
1648     //  0F 85 0B 00 00 00    jne         normal_case
1649     //  33 D2                xor         rdx,edx
1650     //  83 F9 FF             cmp         rcx,0FFh
1651     //  0F 84 03 00 00 00    je          done
1652     //                  normal_case:
1653     //  99                   cdq
1654     //  F7 F9                idiv        rax,ecx
1655     //                  done:
1656     //
1657     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);
1658     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);
1659     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h
1660     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);
1661     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);
1662     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case
1663     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx
1664     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh
1665     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);
1666     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);
1667     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done
1668     // normal_case:
1669     emit_opcode(cbuf,0x99);                                         // cdq
1670     // idiv (note: must be emitted by the user of this rule)
1671     // normal:
1672   %}
1673 
1674   // Dense encoding for older common ops
1675   enc_class Opc_plus(immI opcode, eRegI reg) %{
1676     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);
1677   %}
1678 
1679 
1680   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1681   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit
1682     // Check for 8-bit immediate, and set sign extend bit in opcode
1683     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1684       emit_opcode(cbuf, $primary | 0x02);
1685     }
1686     else {                          // If 32-bit immediate
1687       emit_opcode(cbuf, $primary);
1688     }
1689   %}
1690 
1691   enc_class OpcSErm (eRegI dst, immI imm) %{    // OpcSEr/m
1692     // Emit primary opcode and set sign-extend bit
1693     // Check for 8-bit immediate, and set sign extend bit in opcode
1694     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1695       emit_opcode(cbuf, $primary | 0x02);    }
1696     else {                          // If 32-bit immediate
1697       emit_opcode(cbuf, $primary);
1698     }
1699     // Emit r/m byte with secondary opcode, after primary opcode.
1700     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1701   %}
1702 
1703   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits
1704     // Check for 8-bit immediate, and set sign extend bit in opcode
1705     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1706       $$$emit8$imm$$constant;
1707     }
1708     else {                          // If 32-bit immediate
1709       // Output immediate
1710       $$$emit32$imm$$constant;
1711     }
1712   %}
1713 
1714   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{
1715     // Emit primary opcode and set sign-extend bit
1716     // Check for 8-bit immediate, and set sign extend bit in opcode
1717     int con = (int)$imm$$constant; // Throw away top bits
1718     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1719     // Emit r/m byte with secondary opcode, after primary opcode.
1720     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1721     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1722     else                               emit_d32(cbuf,con);
1723   %}
1724 
1725   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{
1726     // Emit primary opcode and set sign-extend bit
1727     // Check for 8-bit immediate, and set sign extend bit in opcode
1728     int con = (int)($imm$$constant >> 32); // Throw away bottom bits
1729     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1730     // Emit r/m byte with tertiary opcode, after primary opcode.
1731     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));
1732     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1733     else                               emit_d32(cbuf,con);
1734   %}
1735 
1736   enc_class OpcSReg (eRegI dst) %{    // BSWAP
1737     emit_cc(cbuf, $secondary, $dst$$reg );
1738   %}
1739 
1740   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP
1741     int destlo = $dst$$reg;
1742     int desthi = HIGH_FROM_LOW(destlo);
1743     // bswap lo
1744     emit_opcode(cbuf, 0x0F);
1745     emit_cc(cbuf, 0xC8, destlo);
1746     // bswap hi
1747     emit_opcode(cbuf, 0x0F);
1748     emit_cc(cbuf, 0xC8, desthi);
1749     // xchg lo and hi
1750     emit_opcode(cbuf, 0x87);
1751     emit_rm(cbuf, 0x3, destlo, desthi);
1752   %}
1753 
1754   enc_class RegOpc (eRegI div) %{    // IDIV, IMOD, JMP indirect, ...
1755     emit_rm(cbuf, 0x3, $secondary, $div$$reg );
1756   %}
1757 
1758   enc_class enc_cmov(cmpOp cop ) %{ // CMOV
1759     $$$emit8$primary;
1760     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1761   %}
1762 
1763   enc_class enc_cmov_d(cmpOp cop, regD src ) %{ // CMOV
1764     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);
1765     emit_d8(cbuf, op >> 8 );
1766     emit_d8(cbuf, op & 255);
1767   %}
1768 
1769   // emulate a CMOV with a conditional branch around a MOV
1770   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV
1771     // Invert sense of branch from sense of CMOV
1772     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );
1773     emit_d8( cbuf, $brOffs$$constant );
1774   %}
1775 
1776   enc_class enc_PartialSubtypeCheck( ) %{
1777     Register Redi = as_Register(EDI_enc); // result register
1778     Register Reax = as_Register(EAX_enc); // super class
1779     Register Recx = as_Register(ECX_enc); // killed
1780     Register Resi = as_Register(ESI_enc); // sub class
1781     Label miss;
1782 
1783     MacroAssembler _masm(&cbuf);
1784     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
1785                                      NULL, &miss,
1786                                      /*set_cond_codes:*/ true);
1787     if ($primary) {
1788       __ xorptr(Redi, Redi);
1789     }
1790     __ bind(miss);
1791   %}
1792 
1793   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All
1794     MacroAssembler masm(&cbuf);
1795     int start = masm.offset();
1796     if (UseSSE >= 2) {
1797       if (VerifyFPU) {
1798         masm.verify_FPU(0, "must be empty in SSE2+ mode");
1799       }
1800     } else {
1801       // External c_calling_convention expects the FPU stack to be 'clean'.
1802       // Compiled code leaves it dirty.  Do cleanup now.
1803       masm.empty_FPU_stack();
1804     }
1805     if (sizeof_FFree_Float_Stack_All == -1) {
1806       sizeof_FFree_Float_Stack_All = masm.offset() - start;
1807     } else {
1808       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");
1809     }
1810   %}
1811 
1812   enc_class Verify_FPU_For_Leaf %{
1813     if( VerifyFPU ) {
1814       MacroAssembler masm(&cbuf);
1815       masm.verify_FPU( -3, "Returning from Runtime Leaf call");
1816     }
1817   %}
1818 
1819   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf
1820     // This is the instruction starting address for relocation info.
1821     cbuf.set_insts_mark();
1822     $$$emit8$primary;
1823     // CALL directly to the runtime
1824     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1825                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1826 
1827     if (UseSSE >= 2) {
1828       MacroAssembler _masm(&cbuf);
1829       BasicType rt = tf()->return_type();
1830 
1831       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {
1832         // A C runtime call where the return value is unused.  In SSE2+
1833         // mode the result needs to be removed from the FPU stack.  It's
1834         // likely that this function call could be removed by the
1835         // optimizer if the C function is a pure function.
1836         __ ffree(0);
1837       } else if (rt == T_FLOAT) {
1838         __ lea(rsp, Address(rsp, -4));
1839         __ fstp_s(Address(rsp, 0));
1840         __ movflt(xmm0, Address(rsp, 0));
1841         __ lea(rsp, Address(rsp,  4));
1842       } else if (rt == T_DOUBLE) {
1843         __ lea(rsp, Address(rsp, -8));
1844         __ fstp_d(Address(rsp, 0));
1845         __ movdbl(xmm0, Address(rsp, 0));
1846         __ lea(rsp, Address(rsp,  8));
1847       }
1848     }
1849   %}
1850 
1851 
1852   enc_class pre_call_FPU %{
1853     // If method sets FPU control word restore it here
1854     debug_only(int off0 = cbuf.insts_size());
1855     if( Compile::current()->in_24_bit_fp_mode() ) {
1856       MacroAssembler masm(&cbuf);
1857       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1858     }
1859     debug_only(int off1 = cbuf.insts_size());
1860     assert(off1 - off0 == pre_call_FPU_size(), "correct size prediction");
1861   %}
1862 
1863   enc_class post_call_FPU %{
1864     // If method sets FPU control word do it here also
1865     if( Compile::current()->in_24_bit_fp_mode() ) {
1866       MacroAssembler masm(&cbuf);
1867       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
1868     }
1869   %}
1870 
1871   enc_class preserve_SP %{
1872     debug_only(int off0 = cbuf.insts_size());
1873     MacroAssembler _masm(&cbuf);
1874     // RBP is preserved across all calls, even compiled calls.
1875     // Use it to preserve RSP in places where the callee might change the SP.
1876     __ movptr(rbp_mh_SP_save, rsp);
1877     debug_only(int off1 = cbuf.insts_size());
1878     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
1879   %}
1880 
1881   enc_class restore_SP %{
1882     MacroAssembler _masm(&cbuf);
1883     __ movptr(rsp, rbp_mh_SP_save);
1884   %}
1885 
1886   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL
1887     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1888     // who we intended to call.
1889     cbuf.set_insts_mark();
1890     $$$emit8$primary;
1891     if ( !_method ) {
1892       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1893                      runtime_call_Relocation::spec(), RELOC_IMM32 );
1894     } else if(_optimized_virtual) {
1895       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1896                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );
1897     } else {
1898       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1899                      static_call_Relocation::spec(), RELOC_IMM32 );
1900     }
1901     if( _method ) {  // Emit stub for static call
1902       emit_java_to_interp(cbuf);
1903     }
1904   %}
1905 
1906   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL
1907     // !!!!!
1908     // Generate  "Mov EAX,0x00", placeholder instruction to load oop-info
1909     // emit_call_dynamic_prologue( cbuf );
1910     cbuf.set_insts_mark();
1911     emit_opcode(cbuf, 0xB8 + EAX_enc);        // mov    EAX,-1
1912     emit_d32_reloc(cbuf, (int)Universe::non_oop_word(), oop_Relocation::spec_for_immediate(), RELOC_IMM32);
1913     address  virtual_call_oop_addr = cbuf.insts_mark();
1914     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1915     // who we intended to call.
1916     cbuf.set_insts_mark();
1917     $$$emit8$primary;
1918     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.insts_end()) - 4),
1919                 virtual_call_Relocation::spec(virtual_call_oop_addr), RELOC_IMM32 );
1920   %}
1921 
1922   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL
1923     int disp = in_bytes(methodOopDesc::from_compiled_offset());
1924     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");
1925 
1926     // CALL *[EAX+in_bytes(methodOopDesc::from_compiled_code_entry_point_offset())]
1927     cbuf.set_insts_mark();
1928     $$$emit8$primary;
1929     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte
1930     emit_d8(cbuf, disp);             // Displacement
1931 
1932   %}
1933 
1934   enc_class Xor_Reg (eRegI dst) %{
1935     emit_opcode(cbuf, 0x33);
1936     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
1937   %}
1938 
1939 //   Following encoding is no longer used, but may be restored if calling
1940 //   convention changes significantly.
1941 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )
1942 //
1943 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL
1944 //     // int ic_reg     = Matcher::inline_cache_reg();
1945 //     // int ic_encode  = Matcher::_regEncode[ic_reg];
1946 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();
1947 //     // int imo_encode = Matcher::_regEncode[imo_reg];
1948 //
1949 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,
1950 //     // // so we load it immediately before the call
1951 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop
1952 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte
1953 //
1954 //     // xor rbp,ebp
1955 //     emit_opcode(cbuf, 0x33);
1956 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);
1957 //
1958 //     // CALL to interpreter.
1959 //     cbuf.set_insts_mark();
1960 //     $$$emit8$primary;
1961 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.insts_end()) - 4),
1962 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1963 //   %}
1964 
1965   enc_class RegOpcImm (eRegI dst, immI8 shift) %{    // SHL, SAR, SHR
1966     $$$emit8$primary;
1967     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1968     $$$emit8$shift$$constant;
1969   %}
1970 
1971   enc_class LdImmI (eRegI dst, immI src) %{    // Load Immediate
1972     // Load immediate does not have a zero or sign extended version
1973     // for 8-bit immediates
1974     emit_opcode(cbuf, 0xB8 + $dst$$reg);
1975     $$$emit32$src$$constant;
1976   %}
1977 
1978   enc_class LdImmP (eRegI dst, immI src) %{    // Load Immediate
1979     // Load immediate does not have a zero or sign extended version
1980     // for 8-bit immediates
1981     emit_opcode(cbuf, $primary + $dst$$reg);
1982     $$$emit32$src$$constant;
1983   %}
1984 
1985   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate
1986     // Load immediate does not have a zero or sign extended version
1987     // for 8-bit immediates
1988     int dst_enc = $dst$$reg;
1989     int src_con = $src$$constant & 0x0FFFFFFFFL;
1990     if (src_con == 0) {
1991       // xor dst, dst
1992       emit_opcode(cbuf, 0x33);
1993       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
1994     } else {
1995       emit_opcode(cbuf, $primary + dst_enc);
1996       emit_d32(cbuf, src_con);
1997     }
1998   %}
1999 
2000   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate
2001     // Load immediate does not have a zero or sign extended version
2002     // for 8-bit immediates
2003     int dst_enc = $dst$$reg + 2;
2004     int src_con = ((julong)($src$$constant)) >> 32;
2005     if (src_con == 0) {
2006       // xor dst, dst
2007       emit_opcode(cbuf, 0x33);
2008       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
2009     } else {
2010       emit_opcode(cbuf, $primary + dst_enc);
2011       emit_d32(cbuf, src_con);
2012     }
2013   %}
2014 
2015 
2016   enc_class MovI2X_reg(regX dst, eRegI src) %{
2017     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
2018     emit_opcode(cbuf, 0x0F );
2019     emit_opcode(cbuf, 0x6E );
2020     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2021   %}
2022 
2023   enc_class MovX2I_reg(eRegI dst, regX src) %{
2024     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
2025     emit_opcode(cbuf, 0x0F );
2026     emit_opcode(cbuf, 0x7E );
2027     emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
2028   %}
2029 
2030   enc_class MovL2XD_reg(regXD dst, eRegL src, regXD tmp) %{
2031     { // MOVD $dst,$src.lo
2032       emit_opcode(cbuf,0x66);
2033       emit_opcode(cbuf,0x0F);
2034       emit_opcode(cbuf,0x6E);
2035       emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2036     }
2037     { // MOVD $tmp,$src.hi
2038       emit_opcode(cbuf,0x66);
2039       emit_opcode(cbuf,0x0F);
2040       emit_opcode(cbuf,0x6E);
2041       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
2042     }
2043     { // PUNPCKLDQ $dst,$tmp
2044       emit_opcode(cbuf,0x66);
2045       emit_opcode(cbuf,0x0F);
2046       emit_opcode(cbuf,0x62);
2047       emit_rm(cbuf, 0x3, $dst$$reg, $tmp$$reg);
2048      }
2049   %}
2050 
2051   enc_class MovXD2L_reg(eRegL dst, regXD src, regXD tmp) %{
2052     { // MOVD $dst.lo,$src
2053       emit_opcode(cbuf,0x66);
2054       emit_opcode(cbuf,0x0F);
2055       emit_opcode(cbuf,0x7E);
2056       emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
2057     }
2058     { // PSHUFLW $tmp,$src,0x4E  (01001110b)
2059       emit_opcode(cbuf,0xF2);
2060       emit_opcode(cbuf,0x0F);
2061       emit_opcode(cbuf,0x70);
2062       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
2063       emit_d8(cbuf, 0x4E);
2064     }
2065     { // MOVD $dst.hi,$tmp
2066       emit_opcode(cbuf,0x66);
2067       emit_opcode(cbuf,0x0F);
2068       emit_opcode(cbuf,0x7E);
2069       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
2070     }
2071   %}
2072 
2073 
2074   // Encode a reg-reg copy.  If it is useless, then empty encoding.
2075   enc_class enc_Copy( eRegI dst, eRegI src ) %{
2076     encode_Copy( cbuf, $dst$$reg, $src$$reg );
2077   %}
2078 
2079   enc_class enc_CopyL_Lo( eRegI dst, eRegL src ) %{
2080     encode_Copy( cbuf, $dst$$reg, $src$$reg );
2081   %}
2082 
2083   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.
2084   enc_class enc_CopyXD( RegXD dst, RegXD src ) %{
2085     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
2086   %}
2087 
2088   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
2089     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2090   %}
2091 
2092   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)
2093     $$$emit8$primary;
2094     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2095   %}
2096 
2097   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)
2098     $$$emit8$secondary;
2099     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2100   %}
2101 
2102   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)
2103     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2104   %}
2105 
2106   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)
2107     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2108   %}
2109 
2110   enc_class RegReg_HiLo( eRegL src, eRegI dst ) %{
2111     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));
2112   %}
2113 
2114   enc_class Con32 (immI src) %{    // Con32(storeImmI)
2115     // Output immediate
2116     $$$emit32$src$$constant;
2117   %}
2118 
2119   enc_class Con32F_as_bits(immF src) %{        // storeF_imm
2120     // Output Float immediate bits
2121     jfloat jf = $src$$constant;
2122     int    jf_as_bits = jint_cast( jf );
2123     emit_d32(cbuf, jf_as_bits);
2124   %}
2125 
2126   enc_class Con32XF_as_bits(immXF src) %{      // storeX_imm
2127     // Output Float immediate bits
2128     jfloat jf = $src$$constant;
2129     int    jf_as_bits = jint_cast( jf );
2130     emit_d32(cbuf, jf_as_bits);
2131   %}
2132 
2133   enc_class Con16 (immI src) %{    // Con16(storeImmI)
2134     // Output immediate
2135     $$$emit16$src$$constant;
2136   %}
2137 
2138   enc_class Con_d32(immI src) %{
2139     emit_d32(cbuf,$src$$constant);
2140   %}
2141 
2142   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)
2143     // Output immediate memory reference
2144     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2145     emit_d32(cbuf, 0x00);
2146   %}
2147 
2148   enc_class lock_prefix( ) %{
2149     if( os::is_MP() )
2150       emit_opcode(cbuf,0xF0);         // [Lock]
2151   %}
2152 
2153   // Cmp-xchg long value.
2154   // Note: we need to swap rbx, and rcx before and after the
2155   //       cmpxchg8 instruction because the instruction uses
2156   //       rcx as the high order word of the new value to store but
2157   //       our register encoding uses rbx,.
2158   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{
2159 
2160     // XCHG  rbx,ecx
2161     emit_opcode(cbuf,0x87);
2162     emit_opcode(cbuf,0xD9);
2163     // [Lock]
2164     if( os::is_MP() )
2165       emit_opcode(cbuf,0xF0);
2166     // CMPXCHG8 [Eptr]
2167     emit_opcode(cbuf,0x0F);
2168     emit_opcode(cbuf,0xC7);
2169     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2170     // XCHG  rbx,ecx
2171     emit_opcode(cbuf,0x87);
2172     emit_opcode(cbuf,0xD9);
2173   %}
2174 
2175   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{
2176     // [Lock]
2177     if( os::is_MP() )
2178       emit_opcode(cbuf,0xF0);
2179 
2180     // CMPXCHG [Eptr]
2181     emit_opcode(cbuf,0x0F);
2182     emit_opcode(cbuf,0xB1);
2183     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2184   %}
2185 
2186   enc_class enc_flags_ne_to_boolean( iRegI res ) %{
2187     int res_encoding = $res$$reg;
2188 
2189     // MOV  res,0
2190     emit_opcode( cbuf, 0xB8 + res_encoding);
2191     emit_d32( cbuf, 0 );
2192     // JNE,s  fail
2193     emit_opcode(cbuf,0x75);
2194     emit_d8(cbuf, 5 );
2195     // MOV  res,1
2196     emit_opcode( cbuf, 0xB8 + res_encoding);
2197     emit_d32( cbuf, 1 );
2198     // fail:
2199   %}
2200 
2201   enc_class set_instruction_start( ) %{
2202     cbuf.set_insts_mark();            // Mark start of opcode for reloc info in mem operand
2203   %}
2204 
2205   enc_class RegMem (eRegI ereg, memory mem) %{    // emit_reg_mem
2206     int reg_encoding = $ereg$$reg;
2207     int base  = $mem$$base;
2208     int index = $mem$$index;
2209     int scale = $mem$$scale;
2210     int displace = $mem$$disp;
2211     bool disp_is_oop = $mem->disp_is_oop();
2212     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2213   %}
2214 
2215   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem
2216     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo
2217     int base  = $mem$$base;
2218     int index = $mem$$index;
2219     int scale = $mem$$scale;
2220     int displace = $mem$$disp + 4;      // Offset is 4 further in memory
2221     assert( !$mem->disp_is_oop(), "Cannot add 4 to oop" );
2222     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, false/*disp_is_oop*/);
2223   %}
2224 
2225   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{
2226     int r1, r2;
2227     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2228     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2229     emit_opcode(cbuf,0x0F);
2230     emit_opcode(cbuf,$tertiary);
2231     emit_rm(cbuf, 0x3, r1, r2);
2232     emit_d8(cbuf,$cnt$$constant);
2233     emit_d8(cbuf,$primary);
2234     emit_rm(cbuf, 0x3, $secondary, r1);
2235     emit_d8(cbuf,$cnt$$constant);
2236   %}
2237 
2238   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{
2239     emit_opcode( cbuf, 0x8B ); // Move
2240     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2241     if( $cnt$$constant > 32 ) { // Shift, if not by zero
2242       emit_d8(cbuf,$primary);
2243       emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
2244       emit_d8(cbuf,$cnt$$constant-32);
2245     }
2246     emit_d8(cbuf,$primary);
2247     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));
2248     emit_d8(cbuf,31);
2249   %}
2250 
2251   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{
2252     int r1, r2;
2253     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2254     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2255 
2256     emit_opcode( cbuf, 0x8B ); // Move r1,r2
2257     emit_rm(cbuf, 0x3, r1, r2);
2258     if( $cnt$$constant > 32 ) { // Shift, if not by zero
2259       emit_opcode(cbuf,$primary);
2260       emit_rm(cbuf, 0x3, $secondary, r1);
2261       emit_d8(cbuf,$cnt$$constant-32);
2262     }
2263     emit_opcode(cbuf,0x33);  // XOR r2,r2
2264     emit_rm(cbuf, 0x3, r2, r2);
2265   %}
2266 
2267   // Clone of RegMem but accepts an extra parameter to access each
2268   // half of a double in memory; it never needs relocation info.
2269   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, eRegI rm_reg) %{
2270     emit_opcode(cbuf,$opcode$$constant);
2271     int reg_encoding = $rm_reg$$reg;
2272     int base     = $mem$$base;
2273     int index    = $mem$$index;
2274     int scale    = $mem$$scale;
2275     int displace = $mem$$disp + $disp_for_half$$constant;
2276     bool disp_is_oop = false;
2277     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2278   %}
2279 
2280   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!
2281   //
2282   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant
2283   // and it never needs relocation information.
2284   // Frequently used to move data between FPU's Stack Top and memory.
2285   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{
2286     int rm_byte_opcode = $rm_opcode$$constant;
2287     int base     = $mem$$base;
2288     int index    = $mem$$index;
2289     int scale    = $mem$$scale;
2290     int displace = $mem$$disp;
2291     assert( !$mem->disp_is_oop(), "No oops here because no relo info allowed" );
2292     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, false);
2293   %}
2294 
2295   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{
2296     int rm_byte_opcode = $rm_opcode$$constant;
2297     int base     = $mem$$base;
2298     int index    = $mem$$index;
2299     int scale    = $mem$$scale;
2300     int displace = $mem$$disp;
2301     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
2302     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
2303   %}
2304 
2305   enc_class RegLea (eRegI dst, eRegI src0, immI src1 ) %{    // emit_reg_lea
2306     int reg_encoding = $dst$$reg;
2307     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2308     int index        = 0x04;            // 0x04 indicates no index
2309     int scale        = 0x00;            // 0x00 indicates no scale
2310     int displace     = $src1$$constant; // 0x00 indicates no displacement
2311     bool disp_is_oop = false;
2312     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2313   %}
2314 
2315   enc_class min_enc (eRegI dst, eRegI src) %{    // MIN
2316     // Compare dst,src
2317     emit_opcode(cbuf,0x3B);
2318     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2319     // jmp dst < src around move
2320     emit_opcode(cbuf,0x7C);
2321     emit_d8(cbuf,2);
2322     // move dst,src
2323     emit_opcode(cbuf,0x8B);
2324     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2325   %}
2326 
2327   enc_class max_enc (eRegI dst, eRegI src) %{    // MAX
2328     // Compare dst,src
2329     emit_opcode(cbuf,0x3B);
2330     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2331     // jmp dst > src around move
2332     emit_opcode(cbuf,0x7F);
2333     emit_d8(cbuf,2);
2334     // move dst,src
2335     emit_opcode(cbuf,0x8B);
2336     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2337   %}
2338 
2339   enc_class enc_FP_store(memory mem, regD src) %{
2340     // If src is FPR1, we can just FST to store it.
2341     // Else we need to FLD it to FPR1, then FSTP to store/pop it.
2342     int reg_encoding = 0x2; // Just store
2343     int base  = $mem$$base;
2344     int index = $mem$$index;
2345     int scale = $mem$$scale;
2346     int displace = $mem$$disp;
2347     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
2348     if( $src$$reg != FPR1L_enc ) {
2349       reg_encoding = 0x3;  // Store & pop
2350       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)
2351       emit_d8( cbuf, 0xC0-1+$src$$reg );
2352     }
2353     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
2354     emit_opcode(cbuf,$primary);
2355     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2356   %}
2357 
2358   enc_class neg_reg(eRegI dst) %{
2359     // NEG $dst
2360     emit_opcode(cbuf,0xF7);
2361     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );
2362   %}
2363 
2364   enc_class setLT_reg(eCXRegI dst) %{
2365     // SETLT $dst
2366     emit_opcode(cbuf,0x0F);
2367     emit_opcode(cbuf,0x9C);
2368     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );
2369   %}
2370 
2371   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT
2372     int tmpReg = $tmp$$reg;
2373 
2374     // SUB $p,$q
2375     emit_opcode(cbuf,0x2B);
2376     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
2377     // SBB $tmp,$tmp
2378     emit_opcode(cbuf,0x1B);
2379     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
2380     // AND $tmp,$y
2381     emit_opcode(cbuf,0x23);
2382     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);
2383     // ADD $p,$tmp
2384     emit_opcode(cbuf,0x03);
2385     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
2386   %}
2387 
2388   enc_class enc_cmpLTP_mem(eRegI p, eRegI q, memory mem, eCXRegI tmp) %{    // cadd_cmpLT
2389     int tmpReg = $tmp$$reg;
2390 
2391     // SUB $p,$q
2392     emit_opcode(cbuf,0x2B);
2393     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
2394     // SBB $tmp,$tmp
2395     emit_opcode(cbuf,0x1B);
2396     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
2397     // AND $tmp,$y
2398     cbuf.set_insts_mark();       // Mark start of opcode for reloc info in mem operand
2399     emit_opcode(cbuf,0x23);
2400     int reg_encoding = tmpReg;
2401     int base  = $mem$$base;
2402     int index = $mem$$index;
2403     int scale = $mem$$scale;
2404     int displace = $mem$$disp;
2405     bool disp_is_oop = $mem->disp_is_oop();
2406     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2407     // ADD $p,$tmp
2408     emit_opcode(cbuf,0x03);
2409     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
2410   %}
2411 
2412   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{
2413     // TEST shift,32
2414     emit_opcode(cbuf,0xF7);
2415     emit_rm(cbuf, 0x3, 0, ECX_enc);
2416     emit_d32(cbuf,0x20);
2417     // JEQ,s small
2418     emit_opcode(cbuf, 0x74);
2419     emit_d8(cbuf, 0x04);
2420     // MOV    $dst.hi,$dst.lo
2421     emit_opcode( cbuf, 0x8B );
2422     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
2423     // CLR    $dst.lo
2424     emit_opcode(cbuf, 0x33);
2425     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
2426 // small:
2427     // SHLD   $dst.hi,$dst.lo,$shift
2428     emit_opcode(cbuf,0x0F);
2429     emit_opcode(cbuf,0xA5);
2430     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2431     // SHL    $dst.lo,$shift"
2432     emit_opcode(cbuf,0xD3);
2433     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );
2434   %}
2435 
2436   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{
2437     // TEST shift,32
2438     emit_opcode(cbuf,0xF7);
2439     emit_rm(cbuf, 0x3, 0, ECX_enc);
2440     emit_d32(cbuf,0x20);
2441     // JEQ,s small
2442     emit_opcode(cbuf, 0x74);
2443     emit_d8(cbuf, 0x04);
2444     // MOV    $dst.lo,$dst.hi
2445     emit_opcode( cbuf, 0x8B );
2446     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2447     // CLR    $dst.hi
2448     emit_opcode(cbuf, 0x33);
2449     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));
2450 // small:
2451     // SHRD   $dst.lo,$dst.hi,$shift
2452     emit_opcode(cbuf,0x0F);
2453     emit_opcode(cbuf,0xAD);
2454     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2455     // SHR    $dst.hi,$shift"
2456     emit_opcode(cbuf,0xD3);
2457     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );
2458   %}
2459 
2460   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{
2461     // TEST shift,32
2462     emit_opcode(cbuf,0xF7);
2463     emit_rm(cbuf, 0x3, 0, ECX_enc);
2464     emit_d32(cbuf,0x20);
2465     // JEQ,s small
2466     emit_opcode(cbuf, 0x74);
2467     emit_d8(cbuf, 0x05);
2468     // MOV    $dst.lo,$dst.hi
2469     emit_opcode( cbuf, 0x8B );
2470     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2471     // SAR    $dst.hi,31
2472     emit_opcode(cbuf, 0xC1);
2473     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );
2474     emit_d8(cbuf, 0x1F );
2475 // small:
2476     // SHRD   $dst.lo,$dst.hi,$shift
2477     emit_opcode(cbuf,0x0F);
2478     emit_opcode(cbuf,0xAD);
2479     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2480     // SAR    $dst.hi,$shift"
2481     emit_opcode(cbuf,0xD3);
2482     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );
2483   %}
2484 
2485 
2486   // ----------------- Encodings for floating point unit -----------------
2487   // May leave result in FPU-TOS or FPU reg depending on opcodes
2488   enc_class OpcReg_F (regF src) %{    // FMUL, FDIV
2489     $$$emit8$primary;
2490     emit_rm(cbuf, 0x3, $secondary, $src$$reg );
2491   %}
2492 
2493   // Pop argument in FPR0 with FSTP ST(0)
2494   enc_class PopFPU() %{
2495     emit_opcode( cbuf, 0xDD );
2496     emit_d8( cbuf, 0xD8 );
2497   %}
2498 
2499   // !!!!! equivalent to Pop_Reg_F
2500   enc_class Pop_Reg_D( regD dst ) %{
2501     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2502     emit_d8( cbuf, 0xD8+$dst$$reg );
2503   %}
2504 
2505   enc_class Push_Reg_D( regD dst ) %{
2506     emit_opcode( cbuf, 0xD9 );
2507     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)
2508   %}
2509 
2510   enc_class strictfp_bias1( regD dst ) %{
2511     emit_opcode( cbuf, 0xDB );           // FLD m80real
2512     emit_opcode( cbuf, 0x2D );
2513     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );
2514     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2515     emit_opcode( cbuf, 0xC8+$dst$$reg );
2516   %}
2517 
2518   enc_class strictfp_bias2( regD dst ) %{
2519     emit_opcode( cbuf, 0xDB );           // FLD m80real
2520     emit_opcode( cbuf, 0x2D );
2521     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );
2522     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2523     emit_opcode( cbuf, 0xC8+$dst$$reg );
2524   %}
2525 
2526   // Special case for moving an integer register to a stack slot.
2527   enc_class OpcPRegSS( stackSlotI dst, eRegI src ) %{ // RegSS
2528     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );
2529   %}
2530 
2531   // Special case for moving a register to a stack slot.
2532   enc_class RegSS( stackSlotI dst, eRegI src ) %{ // RegSS
2533     // Opcode already emitted
2534     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte
2535     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte
2536     emit_d32(cbuf, $dst$$disp);   // Displacement
2537   %}
2538 
2539   // Push the integer in stackSlot 'src' onto FP-stack
2540   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]
2541     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );
2542   %}
2543 
2544   // Push the float in stackSlot 'src' onto FP-stack
2545   enc_class Push_Mem_F( memory src ) %{    // FLD_S   [ESP+src]
2546     store_to_stackslot( cbuf, 0xD9, 0x00, $src$$disp );
2547   %}
2548 
2549   // Push the double in stackSlot 'src' onto FP-stack
2550   enc_class Push_Mem_D( memory src ) %{    // FLD_D   [ESP+src]
2551     store_to_stackslot( cbuf, 0xDD, 0x00, $src$$disp );
2552   %}
2553 
2554   // Push FPU's TOS float to a stack-slot, and pop FPU-stack
2555   enc_class Pop_Mem_F( stackSlotF dst ) %{ // FSTP_S [ESP+dst]
2556     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );
2557   %}
2558 
2559   // Same as Pop_Mem_F except for opcode
2560   // Push FPU's TOS double to a stack-slot, and pop FPU-stack
2561   enc_class Pop_Mem_D( stackSlotD dst ) %{ // FSTP_D [ESP+dst]
2562     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );
2563   %}
2564 
2565   enc_class Pop_Reg_F( regF dst ) %{
2566     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2567     emit_d8( cbuf, 0xD8+$dst$$reg );
2568   %}
2569 
2570   enc_class Push_Reg_F( regF dst ) %{
2571     emit_opcode( cbuf, 0xD9 );           // FLD    ST(i-1)
2572     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2573   %}
2574 
2575   // Push FPU's float to a stack-slot, and pop FPU-stack
2576   enc_class Pop_Mem_Reg_F( stackSlotF dst, regF src ) %{
2577     int pop = 0x02;
2578     if ($src$$reg != FPR1L_enc) {
2579       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2580       emit_d8( cbuf, 0xC0-1+$src$$reg );
2581       pop = 0x03;
2582     }
2583     store_to_stackslot( cbuf, 0xD9, pop, $dst$$disp ); // FST<P>_S  [ESP+dst]
2584   %}
2585 
2586   // Push FPU's double to a stack-slot, and pop FPU-stack
2587   enc_class Pop_Mem_Reg_D( stackSlotD dst, regD src ) %{
2588     int pop = 0x02;
2589     if ($src$$reg != FPR1L_enc) {
2590       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2591       emit_d8( cbuf, 0xC0-1+$src$$reg );
2592       pop = 0x03;
2593     }
2594     store_to_stackslot( cbuf, 0xDD, pop, $dst$$disp ); // FST<P>_D  [ESP+dst]
2595   %}
2596 
2597   // Push FPU's double to a FPU-stack-slot, and pop FPU-stack
2598   enc_class Pop_Reg_Reg_D( regD dst, regF src ) %{
2599     int pop = 0xD0 - 1; // -1 since we skip FLD
2600     if ($src$$reg != FPR1L_enc) {
2601       emit_opcode( cbuf, 0xD9 );         // FLD    ST(src-1)
2602       emit_d8( cbuf, 0xC0-1+$src$$reg );
2603       pop = 0xD8;
2604     }
2605     emit_opcode( cbuf, 0xDD );
2606     emit_d8( cbuf, pop+$dst$$reg );      // FST<P> ST(i)
2607   %}
2608 
2609 
2610   enc_class Mul_Add_F( regF dst, regF src, regF src1, regF src2 ) %{
2611     MacroAssembler masm(&cbuf);
2612     masm.fld_s(  $src1$$reg-1);   // nothing at TOS, load TOS from src1.reg
2613     masm.fmul(   $src2$$reg+0);   // value at TOS
2614     masm.fadd(   $src$$reg+0);    // value at TOS
2615     masm.fstp_d( $dst$$reg+0);    // value at TOS, popped off after store
2616   %}
2617 
2618 
2619   enc_class Push_Reg_Mod_D( regD dst, regD src) %{
2620     // load dst in FPR0
2621     emit_opcode( cbuf, 0xD9 );
2622     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2623     if ($src$$reg != FPR1L_enc) {
2624       // fincstp
2625       emit_opcode (cbuf, 0xD9);
2626       emit_opcode (cbuf, 0xF7);
2627       // swap src with FPR1:
2628       // FXCH FPR1 with src
2629       emit_opcode(cbuf, 0xD9);
2630       emit_d8(cbuf, 0xC8-1+$src$$reg );
2631       // fdecstp
2632       emit_opcode (cbuf, 0xD9);
2633       emit_opcode (cbuf, 0xF6);
2634     }
2635   %}
2636 
2637   enc_class Push_ModD_encoding( regXD src0, regXD src1) %{
2638     // Allocate a word
2639     emit_opcode(cbuf,0x83);            // SUB ESP,8
2640     emit_opcode(cbuf,0xEC);
2641     emit_d8(cbuf,0x08);
2642 
2643     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src1
2644     emit_opcode  (cbuf, 0x0F );
2645     emit_opcode  (cbuf, 0x11 );
2646     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
2647 
2648     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2649     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2650 
2651     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src0
2652     emit_opcode  (cbuf, 0x0F );
2653     emit_opcode  (cbuf, 0x11 );
2654     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
2655 
2656     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2657     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2658 
2659   %}
2660 
2661   enc_class Push_ModX_encoding( regX src0, regX src1) %{
2662     // Allocate a word
2663     emit_opcode(cbuf,0x83);            // SUB ESP,4
2664     emit_opcode(cbuf,0xEC);
2665     emit_d8(cbuf,0x04);
2666 
2667     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src1
2668     emit_opcode  (cbuf, 0x0F );
2669     emit_opcode  (cbuf, 0x11 );
2670     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
2671 
2672     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
2673     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2674 
2675     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src0
2676     emit_opcode  (cbuf, 0x0F );
2677     emit_opcode  (cbuf, 0x11 );
2678     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
2679 
2680     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
2681     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2682 
2683   %}
2684 
2685   enc_class Push_ResultXD(regXD dst) %{
2686     store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [ESP]
2687 
2688     // UseXmmLoadAndClearUpper ? movsd dst,[esp] : movlpd dst,[esp]
2689     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
2690     emit_opcode  (cbuf, 0x0F );
2691     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
2692     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
2693 
2694     emit_opcode(cbuf,0x83);    // ADD ESP,8
2695     emit_opcode(cbuf,0xC4);
2696     emit_d8(cbuf,0x08);
2697   %}
2698 
2699   enc_class Push_ResultX(regX dst, immI d8) %{
2700     store_to_stackslot( cbuf, 0xD9, 0x03, 0 ); //FSTP_S [ESP]
2701 
2702     emit_opcode  (cbuf, 0xF3 );     // MOVSS dst(xmm), [ESP]
2703     emit_opcode  (cbuf, 0x0F );
2704     emit_opcode  (cbuf, 0x10 );
2705     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
2706 
2707     emit_opcode(cbuf,0x83);    // ADD ESP,d8 (4 or 8)
2708     emit_opcode(cbuf,0xC4);
2709     emit_d8(cbuf,$d8$$constant);
2710   %}
2711 
2712   enc_class Push_SrcXD(regXD src) %{
2713     // Allocate a word
2714     emit_opcode(cbuf,0x83);            // SUB ESP,8
2715     emit_opcode(cbuf,0xEC);
2716     emit_d8(cbuf,0x08);
2717 
2718     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src
2719     emit_opcode  (cbuf, 0x0F );
2720     emit_opcode  (cbuf, 0x11 );
2721     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
2722 
2723     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2724     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2725   %}
2726 
2727   enc_class push_stack_temp_qword() %{
2728     emit_opcode(cbuf,0x83);     // SUB ESP,8
2729     emit_opcode(cbuf,0xEC);
2730     emit_d8    (cbuf,0x08);
2731   %}
2732 
2733   enc_class pop_stack_temp_qword() %{
2734     emit_opcode(cbuf,0x83);     // ADD ESP,8
2735     emit_opcode(cbuf,0xC4);
2736     emit_d8    (cbuf,0x08);
2737   %}
2738 
2739   enc_class push_xmm_to_fpr1( regXD xmm_src ) %{
2740     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], xmm_src
2741     emit_opcode  (cbuf, 0x0F );
2742     emit_opcode  (cbuf, 0x11 );
2743     encode_RegMem(cbuf, $xmm_src$$reg, ESP_enc, 0x4, 0, 0, false);
2744 
2745     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2746     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2747   %}
2748 
2749   // Compute X^Y using Intel's fast hardware instructions, if possible.
2750   // Otherwise return a NaN.
2751   enc_class pow_exp_core_encoding %{
2752     // FPR1 holds Y*ln2(X).  Compute FPR1 = 2^(Y*ln2(X))
2753     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xC0);  // fdup = fld st(0)          Q       Q
2754     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xFC);  // frndint               int(Q)      Q
2755     emit_opcode(cbuf,0xDC); emit_opcode(cbuf,0xE9);  // fsub st(1) -= st(0);  int(Q) frac(Q)
2756     emit_opcode(cbuf,0xDB);                          // FISTP [ESP]           frac(Q)
2757     emit_opcode(cbuf,0x1C);
2758     emit_d8(cbuf,0x24);
2759     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xF0);  // f2xm1                 2^frac(Q)-1
2760     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xE8);  // fld1                  1 2^frac(Q)-1
2761     emit_opcode(cbuf,0xDE); emit_opcode(cbuf,0xC1);  // faddp                 2^frac(Q)
2762     emit_opcode(cbuf,0x8B);                          // mov rax,[esp+0]=int(Q)
2763     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 0, false);
2764     emit_opcode(cbuf,0xC7);                          // mov rcx,0xFFFFF800 - overflow mask
2765     emit_rm(cbuf, 0x3, 0x0, ECX_enc);
2766     emit_d32(cbuf,0xFFFFF800);
2767     emit_opcode(cbuf,0x81);                          // add rax,1023 - the double exponent bias
2768     emit_rm(cbuf, 0x3, 0x0, EAX_enc);
2769     emit_d32(cbuf,1023);
2770     emit_opcode(cbuf,0x8B);                          // mov rbx,eax
2771     emit_rm(cbuf, 0x3, EBX_enc, EAX_enc);
2772     emit_opcode(cbuf,0xC1);                          // shl rax,20 - Slide to exponent position
2773     emit_rm(cbuf,0x3,0x4,EAX_enc);
2774     emit_d8(cbuf,20);
2775     emit_opcode(cbuf,0x85);                          // test rbx,ecx - check for overflow
2776     emit_rm(cbuf, 0x3, EBX_enc, ECX_enc);
2777     emit_opcode(cbuf,0x0F); emit_opcode(cbuf,0x45);  // CMOVne rax,ecx - overflow; stuff NAN into EAX
2778     emit_rm(cbuf, 0x3, EAX_enc, ECX_enc);
2779     emit_opcode(cbuf,0x89);                          // mov [esp+4],eax - Store as part of double word
2780     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 4, false);
2781     emit_opcode(cbuf,0xC7);                          // mov [esp+0],0   - [ESP] = (double)(1<<int(Q)) = 2^int(Q)
2782     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2783     emit_d32(cbuf,0);
2784     emit_opcode(cbuf,0xDC);                          // fmul dword st(0),[esp+0]; FPR1 = 2^int(Q)*2^frac(Q) = 2^Q
2785     encode_RegMem(cbuf, 0x1, ESP_enc, 0x4, 0, 0, false);
2786   %}
2787 
2788 //   enc_class Pop_Reg_Mod_D( regD dst, regD src)
2789 //   was replaced by Push_Result_Mod_D followed by Pop_Reg_X() or Pop_Mem_X()
2790 
2791   enc_class Push_Result_Mod_D( regD src) %{
2792     if ($src$$reg != FPR1L_enc) {
2793       // fincstp
2794       emit_opcode (cbuf, 0xD9);
2795       emit_opcode (cbuf, 0xF7);
2796       // FXCH FPR1 with src
2797       emit_opcode(cbuf, 0xD9);
2798       emit_d8(cbuf, 0xC8-1+$src$$reg );
2799       // fdecstp
2800       emit_opcode (cbuf, 0xD9);
2801       emit_opcode (cbuf, 0xF6);
2802     }
2803     // // following asm replaced with Pop_Reg_F or Pop_Mem_F
2804     // // FSTP   FPR$dst$$reg
2805     // emit_opcode( cbuf, 0xDD );
2806     // emit_d8( cbuf, 0xD8+$dst$$reg );
2807   %}
2808 
2809   enc_class fnstsw_sahf_skip_parity() %{
2810     // fnstsw ax
2811     emit_opcode( cbuf, 0xDF );
2812     emit_opcode( cbuf, 0xE0 );
2813     // sahf
2814     emit_opcode( cbuf, 0x9E );
2815     // jnp  ::skip
2816     emit_opcode( cbuf, 0x7B );
2817     emit_opcode( cbuf, 0x05 );
2818   %}
2819 
2820   enc_class emitModD() %{
2821     // fprem must be iterative
2822     // :: loop
2823     // fprem
2824     emit_opcode( cbuf, 0xD9 );
2825     emit_opcode( cbuf, 0xF8 );
2826     // wait
2827     emit_opcode( cbuf, 0x9b );
2828     // fnstsw ax
2829     emit_opcode( cbuf, 0xDF );
2830     emit_opcode( cbuf, 0xE0 );
2831     // sahf
2832     emit_opcode( cbuf, 0x9E );
2833     // jp  ::loop
2834     emit_opcode( cbuf, 0x0F );
2835     emit_opcode( cbuf, 0x8A );
2836     emit_opcode( cbuf, 0xF4 );
2837     emit_opcode( cbuf, 0xFF );
2838     emit_opcode( cbuf, 0xFF );
2839     emit_opcode( cbuf, 0xFF );
2840   %}
2841 
2842   enc_class fpu_flags() %{
2843     // fnstsw_ax
2844     emit_opcode( cbuf, 0xDF);
2845     emit_opcode( cbuf, 0xE0);
2846     // test ax,0x0400
2847     emit_opcode( cbuf, 0x66 );   // operand-size prefix for 16-bit immediate
2848     emit_opcode( cbuf, 0xA9 );
2849     emit_d16   ( cbuf, 0x0400 );
2850     // // // This sequence works, but stalls for 12-16 cycles on PPro
2851     // // test rax,0x0400
2852     // emit_opcode( cbuf, 0xA9 );
2853     // emit_d32   ( cbuf, 0x00000400 );
2854     //
2855     // jz exit (no unordered comparison)
2856     emit_opcode( cbuf, 0x74 );
2857     emit_d8    ( cbuf, 0x02 );
2858     // mov ah,1 - treat as LT case (set carry flag)
2859     emit_opcode( cbuf, 0xB4 );
2860     emit_d8    ( cbuf, 0x01 );
2861     // sahf
2862     emit_opcode( cbuf, 0x9E);
2863   %}
2864 
2865   enc_class cmpF_P6_fixup() %{
2866     // Fixup the integer flags in case comparison involved a NaN
2867     //
2868     // JNP exit (no unordered comparison, P-flag is set by NaN)
2869     emit_opcode( cbuf, 0x7B );
2870     emit_d8    ( cbuf, 0x03 );
2871     // MOV AH,1 - treat as LT case (set carry flag)
2872     emit_opcode( cbuf, 0xB4 );
2873     emit_d8    ( cbuf, 0x01 );
2874     // SAHF
2875     emit_opcode( cbuf, 0x9E);
2876     // NOP     // target for branch to avoid branch to branch
2877     emit_opcode( cbuf, 0x90);
2878   %}
2879 
2880 //     fnstsw_ax();
2881 //     sahf();
2882 //     movl(dst, nan_result);
2883 //     jcc(Assembler::parity, exit);
2884 //     movl(dst, less_result);
2885 //     jcc(Assembler::below, exit);
2886 //     movl(dst, equal_result);
2887 //     jcc(Assembler::equal, exit);
2888 //     movl(dst, greater_result);
2889 
2890 // less_result     =  1;
2891 // greater_result  = -1;
2892 // equal_result    = 0;
2893 // nan_result      = -1;
2894 
2895   enc_class CmpF_Result(eRegI dst) %{
2896     // fnstsw_ax();
2897     emit_opcode( cbuf, 0xDF);
2898     emit_opcode( cbuf, 0xE0);
2899     // sahf
2900     emit_opcode( cbuf, 0x9E);
2901     // movl(dst, nan_result);
2902     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2903     emit_d32( cbuf, -1 );
2904     // jcc(Assembler::parity, exit);
2905     emit_opcode( cbuf, 0x7A );
2906     emit_d8    ( cbuf, 0x13 );
2907     // movl(dst, less_result);
2908     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2909     emit_d32( cbuf, -1 );
2910     // jcc(Assembler::below, exit);
2911     emit_opcode( cbuf, 0x72 );
2912     emit_d8    ( cbuf, 0x0C );
2913     // movl(dst, equal_result);
2914     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2915     emit_d32( cbuf, 0 );
2916     // jcc(Assembler::equal, exit);
2917     emit_opcode( cbuf, 0x74 );
2918     emit_d8    ( cbuf, 0x05 );
2919     // movl(dst, greater_result);
2920     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2921     emit_d32( cbuf, 1 );
2922   %}
2923 
2924 
2925   // XMM version of CmpF_Result. Because the XMM compare
2926   // instructions set the EFLAGS directly. It becomes simpler than
2927   // the float version above.
2928   enc_class CmpX_Result(eRegI dst) %{
2929     MacroAssembler _masm(&cbuf);
2930     Label nan, inc, done;
2931 
2932     __ jccb(Assembler::parity, nan);
2933     __ jccb(Assembler::equal,  done);
2934     __ jccb(Assembler::above,  inc);
2935     __ bind(nan);
2936     __ decrement(as_Register($dst$$reg)); // NO L qqq
2937     __ jmpb(done);
2938     __ bind(inc);
2939     __ increment(as_Register($dst$$reg)); // NO L qqq
2940     __ bind(done);
2941   %}
2942 
2943   // Compare the longs and set flags
2944   // BROKEN!  Do Not use as-is
2945   enc_class cmpl_test( eRegL src1, eRegL src2 ) %{
2946     // CMP    $src1.hi,$src2.hi
2947     emit_opcode( cbuf, 0x3B );
2948     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
2949     // JNE,s  done
2950     emit_opcode(cbuf,0x75);
2951     emit_d8(cbuf, 2 );
2952     // CMP    $src1.lo,$src2.lo
2953     emit_opcode( cbuf, 0x3B );
2954     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
2955 // done:
2956   %}
2957 
2958   enc_class convert_int_long( regL dst, eRegI src ) %{
2959     // mov $dst.lo,$src
2960     int dst_encoding = $dst$$reg;
2961     int src_encoding = $src$$reg;
2962     encode_Copy( cbuf, dst_encoding  , src_encoding );
2963     // mov $dst.hi,$src
2964     encode_Copy( cbuf, HIGH_FROM_LOW(dst_encoding), src_encoding );
2965     // sar $dst.hi,31
2966     emit_opcode( cbuf, 0xC1 );
2967     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW(dst_encoding) );
2968     emit_d8(cbuf, 0x1F );
2969   %}
2970 
2971   enc_class convert_long_double( eRegL src ) %{
2972     // push $src.hi
2973     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
2974     // push $src.lo
2975     emit_opcode(cbuf, 0x50+$src$$reg  );
2976     // fild 64-bits at [SP]
2977     emit_opcode(cbuf,0xdf);
2978     emit_d8(cbuf, 0x6C);
2979     emit_d8(cbuf, 0x24);
2980     emit_d8(cbuf, 0x00);
2981     // pop stack
2982     emit_opcode(cbuf, 0x83); // add  SP, #8
2983     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
2984     emit_d8(cbuf, 0x8);
2985   %}
2986 
2987   enc_class multiply_con_and_shift_high( eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr ) %{
2988     // IMUL   EDX:EAX,$src1
2989     emit_opcode( cbuf, 0xF7 );
2990     emit_rm( cbuf, 0x3, 0x5, $src1$$reg );
2991     // SAR    EDX,$cnt-32
2992     int shift_count = ((int)$cnt$$constant) - 32;
2993     if (shift_count > 0) {
2994       emit_opcode(cbuf, 0xC1);
2995       emit_rm(cbuf, 0x3, 7, $dst$$reg );
2996       emit_d8(cbuf, shift_count);
2997     }
2998   %}
2999 
3000   // this version doesn't have add sp, 8
3001   enc_class convert_long_double2( eRegL src ) %{
3002     // push $src.hi
3003     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
3004     // push $src.lo
3005     emit_opcode(cbuf, 0x50+$src$$reg  );
3006     // fild 64-bits at [SP]
3007     emit_opcode(cbuf,0xdf);
3008     emit_d8(cbuf, 0x6C);
3009     emit_d8(cbuf, 0x24);
3010     emit_d8(cbuf, 0x00);
3011   %}
3012 
3013   enc_class long_int_multiply( eADXRegL dst, nadxRegI src) %{
3014     // Basic idea: long = (long)int * (long)int
3015     // IMUL EDX:EAX, src
3016     emit_opcode( cbuf, 0xF7 );
3017     emit_rm( cbuf, 0x3, 0x5, $src$$reg);
3018   %}
3019 
3020   enc_class long_uint_multiply( eADXRegL dst, nadxRegI src) %{
3021     // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
3022     // MUL EDX:EAX, src
3023     emit_opcode( cbuf, 0xF7 );
3024     emit_rm( cbuf, 0x3, 0x4, $src$$reg);
3025   %}
3026 
3027   enc_class long_multiply( eADXRegL dst, eRegL src, eRegI tmp ) %{
3028     // Basic idea: lo(result) = lo(x_lo * y_lo)
3029     //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
3030     // MOV    $tmp,$src.lo
3031     encode_Copy( cbuf, $tmp$$reg, $src$$reg );
3032     // IMUL   $tmp,EDX
3033     emit_opcode( cbuf, 0x0F );
3034     emit_opcode( cbuf, 0xAF );
3035     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3036     // MOV    EDX,$src.hi
3037     encode_Copy( cbuf, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg) );
3038     // IMUL   EDX,EAX
3039     emit_opcode( cbuf, 0x0F );
3040     emit_opcode( cbuf, 0xAF );
3041     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
3042     // ADD    $tmp,EDX
3043     emit_opcode( cbuf, 0x03 );
3044     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3045     // MUL   EDX:EAX,$src.lo
3046     emit_opcode( cbuf, 0xF7 );
3047     emit_rm( cbuf, 0x3, 0x4, $src$$reg );
3048     // ADD    EDX,ESI
3049     emit_opcode( cbuf, 0x03 );
3050     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $tmp$$reg );
3051   %}
3052 
3053   enc_class long_multiply_con( eADXRegL dst, immL_127 src, eRegI tmp ) %{
3054     // Basic idea: lo(result) = lo(src * y_lo)
3055     //             hi(result) = hi(src * y_lo) + lo(src * y_hi)
3056     // IMUL   $tmp,EDX,$src
3057     emit_opcode( cbuf, 0x6B );
3058     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3059     emit_d8( cbuf, (int)$src$$constant );
3060     // MOV    EDX,$src
3061     emit_opcode(cbuf, 0xB8 + EDX_enc);
3062     emit_d32( cbuf, (int)$src$$constant );
3063     // MUL   EDX:EAX,EDX
3064     emit_opcode( cbuf, 0xF7 );
3065     emit_rm( cbuf, 0x3, 0x4, EDX_enc );
3066     // ADD    EDX,ESI
3067     emit_opcode( cbuf, 0x03 );
3068     emit_rm( cbuf, 0x3, EDX_enc, $tmp$$reg );
3069   %}
3070 
3071   enc_class long_div( eRegL src1, eRegL src2 ) %{
3072     // PUSH src1.hi
3073     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
3074     // PUSH src1.lo
3075     emit_opcode(cbuf,               0x50+$src1$$reg  );
3076     // PUSH src2.hi
3077     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
3078     // PUSH src2.lo
3079     emit_opcode(cbuf,               0x50+$src2$$reg  );
3080     // CALL directly to the runtime
3081     cbuf.set_insts_mark();
3082     emit_opcode(cbuf,0xE8);       // Call into runtime
3083     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3084     // Restore stack
3085     emit_opcode(cbuf, 0x83); // add  SP, #framesize
3086     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
3087     emit_d8(cbuf, 4*4);
3088   %}
3089 
3090   enc_class long_mod( eRegL src1, eRegL src2 ) %{
3091     // PUSH src1.hi
3092     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
3093     // PUSH src1.lo
3094     emit_opcode(cbuf,               0x50+$src1$$reg  );
3095     // PUSH src2.hi
3096     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
3097     // PUSH src2.lo
3098     emit_opcode(cbuf,               0x50+$src2$$reg  );
3099     // CALL directly to the runtime
3100     cbuf.set_insts_mark();
3101     emit_opcode(cbuf,0xE8);       // Call into runtime
3102     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3103     // Restore stack
3104     emit_opcode(cbuf, 0x83); // add  SP, #framesize
3105     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
3106     emit_d8(cbuf, 4*4);
3107   %}
3108 
3109   enc_class long_cmp_flags0( eRegL src, eRegI tmp ) %{
3110     // MOV   $tmp,$src.lo
3111     emit_opcode(cbuf, 0x8B);
3112     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
3113     // OR    $tmp,$src.hi
3114     emit_opcode(cbuf, 0x0B);
3115     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
3116   %}
3117 
3118   enc_class long_cmp_flags1( eRegL src1, eRegL src2 ) %{
3119     // CMP    $src1.lo,$src2.lo
3120     emit_opcode( cbuf, 0x3B );
3121     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
3122     // JNE,s  skip
3123     emit_cc(cbuf, 0x70, 0x5);
3124     emit_d8(cbuf,2);
3125     // CMP    $src1.hi,$src2.hi
3126     emit_opcode( cbuf, 0x3B );
3127     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
3128   %}
3129 
3130   enc_class long_cmp_flags2( eRegL src1, eRegL src2, eRegI tmp ) %{
3131     // CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits
3132     emit_opcode( cbuf, 0x3B );
3133     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
3134     // MOV    $tmp,$src1.hi
3135     emit_opcode( cbuf, 0x8B );
3136     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src1$$reg) );
3137     // SBB   $tmp,$src2.hi\t! Compute flags for long compare
3138     emit_opcode( cbuf, 0x1B );
3139     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src2$$reg) );
3140   %}
3141 
3142   enc_class long_cmp_flags3( eRegL src, eRegI tmp ) %{
3143     // XOR    $tmp,$tmp
3144     emit_opcode(cbuf,0x33);  // XOR
3145     emit_rm(cbuf,0x3, $tmp$$reg, $tmp$$reg);
3146     // CMP    $tmp,$src.lo
3147     emit_opcode( cbuf, 0x3B );
3148     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg );
3149     // SBB    $tmp,$src.hi
3150     emit_opcode( cbuf, 0x1B );
3151     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg) );
3152   %}
3153 
3154  // Sniff, sniff... smells like Gnu Superoptimizer
3155   enc_class neg_long( eRegL dst ) %{
3156     emit_opcode(cbuf,0xF7);    // NEG hi
3157     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
3158     emit_opcode(cbuf,0xF7);    // NEG lo
3159     emit_rm    (cbuf,0x3, 0x3,               $dst$$reg );
3160     emit_opcode(cbuf,0x83);    // SBB hi,0
3161     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
3162     emit_d8    (cbuf,0 );
3163   %}
3164 
3165   enc_class movq_ld(regXD dst, memory mem) %{
3166     MacroAssembler _masm(&cbuf);
3167     __ movq($dst$$XMMRegister, $mem$$Address);
3168   %}
3169 
3170   enc_class movq_st(memory mem, regXD src) %{
3171     MacroAssembler _masm(&cbuf);
3172     __ movq($mem$$Address, $src$$XMMRegister);
3173   %}
3174 
3175   enc_class pshufd_8x8(regX dst, regX src) %{
3176     MacroAssembler _masm(&cbuf);
3177 
3178     encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
3179     __ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
3180     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
3181   %}
3182 
3183   enc_class pshufd_4x16(regX dst, regX src) %{
3184     MacroAssembler _masm(&cbuf);
3185 
3186     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
3187   %}
3188 
3189   enc_class pshufd(regXD dst, regXD src, int mode) %{
3190     MacroAssembler _masm(&cbuf);
3191 
3192     __ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
3193   %}
3194 
3195   enc_class pxor(regXD dst, regXD src) %{
3196     MacroAssembler _masm(&cbuf);
3197 
3198     __ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
3199   %}
3200 
3201   enc_class mov_i2x(regXD dst, eRegI src) %{
3202     MacroAssembler _masm(&cbuf);
3203 
3204     __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
3205   %}
3206 
3207 
3208   // Because the transitions from emitted code to the runtime
3209   // monitorenter/exit helper stubs are so slow it's critical that
3210   // we inline both the stack-locking fast-path and the inflated fast path.
3211   //
3212   // See also: cmpFastLock and cmpFastUnlock.
3213   //
3214   // What follows is a specialized inline transliteration of the code
3215   // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
3216   // another option would be to emit TrySlowEnter and TrySlowExit methods
3217   // at startup-time.  These methods would accept arguments as
3218   // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
3219   // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
3220   // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
3221   // In practice, however, the # of lock sites is bounded and is usually small.
3222   // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
3223   // if the processor uses simple bimodal branch predictors keyed by EIP
3224   // Since the helper routines would be called from multiple synchronization
3225   // sites.
3226   //
3227   // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
3228   // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
3229   // to those specialized methods.  That'd give us a mostly platform-independent
3230   // implementation that the JITs could optimize and inline at their pleasure.
3231   // Done correctly, the only time we'd need to cross to native could would be
3232   // to park() or unpark() threads.  We'd also need a few more unsafe operators
3233   // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
3234   // (b) explicit barriers or fence operations.
3235   //
3236   // TODO:
3237   //
3238   // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
3239   //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
3240   //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
3241   //    the lock operators would typically be faster than reifying Self.
3242   //
3243   // *  Ideally I'd define the primitives as:
3244   //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
3245   //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
3246   //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
3247   //    Instead, we're stuck with a rather awkward and brittle register assignments below.
3248   //    Furthermore the register assignments are overconstrained, possibly resulting in
3249   //    sub-optimal code near the synchronization site.
3250   //
3251   // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
3252   //    Alternately, use a better sp-proximity test.
3253   //
3254   // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
3255   //    Either one is sufficient to uniquely identify a thread.
3256   //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
3257   //
3258   // *  Intrinsify notify() and notifyAll() for the common cases where the
3259   //    object is locked by the calling thread but the waitlist is empty.
3260   //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
3261   //
3262   // *  use jccb and jmpb instead of jcc and jmp to improve code density.
3263   //    But beware of excessive branch density on AMD Opterons.
3264   //
3265   // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
3266   //    or failure of the fast-path.  If the fast-path fails then we pass
3267   //    control to the slow-path, typically in C.  In Fast_Lock and
3268   //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
3269   //    will emit a conditional branch immediately after the node.
3270   //    So we have branches to branches and lots of ICC.ZF games.
3271   //    Instead, it might be better to have C2 pass a "FailureLabel"
3272   //    into Fast_Lock and Fast_Unlock.  In the case of success, control
3273   //    will drop through the node.  ICC.ZF is undefined at exit.
3274   //    In the case of failure, the node will branch directly to the
3275   //    FailureLabel
3276 
3277 
3278   // obj: object to lock
3279   // box: on-stack box address (displaced header location) - KILLED
3280   // rax,: tmp -- KILLED
3281   // scr: tmp -- KILLED
3282   enc_class Fast_Lock( eRegP obj, eRegP box, eAXRegI tmp, eRegP scr ) %{
3283 
3284     Register objReg = as_Register($obj$$reg);
3285     Register boxReg = as_Register($box$$reg);
3286     Register tmpReg = as_Register($tmp$$reg);
3287     Register scrReg = as_Register($scr$$reg);
3288 
3289     // Ensure the register assignents are disjoint
3290     guarantee (objReg != boxReg, "") ;
3291     guarantee (objReg != tmpReg, "") ;
3292     guarantee (objReg != scrReg, "") ;
3293     guarantee (boxReg != tmpReg, "") ;
3294     guarantee (boxReg != scrReg, "") ;
3295     guarantee (tmpReg == as_Register(EAX_enc), "") ;
3296 
3297     MacroAssembler masm(&cbuf);
3298 
3299     if (_counters != NULL) {
3300       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
3301     }
3302     if (EmitSync & 1) {
3303         // set box->dhw = unused_mark (3)
3304         // Force all sync thru slow-path: slow_enter() and slow_exit() 
3305         masm.movptr (Address(boxReg, 0), int32_t(markOopDesc::unused_mark())) ;             
3306         masm.cmpptr (rsp, (int32_t)0) ;                        
3307     } else 
3308     if (EmitSync & 2) { 
3309         Label DONE_LABEL ;           
3310         if (UseBiasedLocking) {
3311            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
3312            masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3313         }
3314 
3315         masm.movptr(tmpReg, Address(objReg, 0)) ;          // fetch markword 
3316         masm.orptr (tmpReg, 0x1);
3317         masm.movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS 
3318         if (os::is_MP()) { masm.lock();  }
3319         masm.cmpxchgptr(boxReg, Address(objReg, 0));          // Updates tmpReg
3320         masm.jcc(Assembler::equal, DONE_LABEL);
3321         // Recursive locking
3322         masm.subptr(tmpReg, rsp);
3323         masm.andptr(tmpReg, (int32_t) 0xFFFFF003 );
3324         masm.movptr(Address(boxReg, 0), tmpReg);
3325         masm.bind(DONE_LABEL) ; 
3326     } else {  
3327       // Possible cases that we'll encounter in fast_lock 
3328       // ------------------------------------------------
3329       // * Inflated
3330       //    -- unlocked
3331       //    -- Locked
3332       //       = by self
3333       //       = by other
3334       // * biased
3335       //    -- by Self
3336       //    -- by other
3337       // * neutral
3338       // * stack-locked
3339       //    -- by self
3340       //       = sp-proximity test hits
3341       //       = sp-proximity test generates false-negative
3342       //    -- by other
3343       //
3344 
3345       Label IsInflated, DONE_LABEL, PopDone ;
3346 
3347       // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
3348       // order to reduce the number of conditional branches in the most common cases.
3349       // Beware -- there's a subtle invariant that fetch of the markword
3350       // at [FETCH], below, will never observe a biased encoding (*101b).
3351       // If this invariant is not held we risk exclusion (safety) failure.
3352       if (UseBiasedLocking && !UseOptoBiasInlining) {
3353         masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3354       }
3355 
3356       masm.movptr(tmpReg, Address(objReg, 0)) ;         // [FETCH]
3357       masm.testptr(tmpReg, 0x02) ;                      // Inflated v (Stack-locked or neutral)
3358       masm.jccb  (Assembler::notZero, IsInflated) ;
3359 
3360       // Attempt stack-locking ...
3361       masm.orptr (tmpReg, 0x1);
3362       masm.movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
3363       if (os::is_MP()) { masm.lock();  }
3364       masm.cmpxchgptr(boxReg, Address(objReg, 0));           // Updates tmpReg
3365       if (_counters != NULL) {
3366         masm.cond_inc32(Assembler::equal,
3367                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
3368       }
3369       masm.jccb (Assembler::equal, DONE_LABEL);
3370 
3371       // Recursive locking
3372       masm.subptr(tmpReg, rsp);
3373       masm.andptr(tmpReg, 0xFFFFF003 );
3374       masm.movptr(Address(boxReg, 0), tmpReg);
3375       if (_counters != NULL) {
3376         masm.cond_inc32(Assembler::equal,
3377                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
3378       }
3379       masm.jmp  (DONE_LABEL) ;
3380 
3381       masm.bind (IsInflated) ;
3382 
3383       // The object is inflated.
3384       //
3385       // TODO-FIXME: eliminate the ugly use of manifest constants:
3386       //   Use markOopDesc::monitor_value instead of "2".
3387       //   use markOop::unused_mark() instead of "3".
3388       // The tmpReg value is an objectMonitor reference ORed with
3389       // markOopDesc::monitor_value (2).   We can either convert tmpReg to an
3390       // objectmonitor pointer by masking off the "2" bit or we can just
3391       // use tmpReg as an objectmonitor pointer but bias the objectmonitor
3392       // field offsets with "-2" to compensate for and annul the low-order tag bit.
3393       //
3394       // I use the latter as it avoids AGI stalls.
3395       // As such, we write "mov r, [tmpReg+OFFSETOF(Owner)-2]"
3396       // instead of "mov r, [tmpReg+OFFSETOF(Owner)]".
3397       //
3398       #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)
3399 
3400       // boxReg refers to the on-stack BasicLock in the current frame.
3401       // We'd like to write:
3402       //   set box->_displaced_header = markOop::unused_mark().  Any non-0 value suffices.
3403       // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
3404       // additional latency as we have another ST in the store buffer that must drain.
3405 
3406       if (EmitSync & 8192) { 
3407          masm.movptr(Address(boxReg, 0), 3) ;            // results in ST-before-CAS penalty
3408          masm.get_thread (scrReg) ; 
3409          masm.movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2] 
3410          masm.movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
3411          if (os::is_MP()) { masm.lock(); } 
3412          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
3413       } else 
3414       if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
3415          masm.movptr(scrReg, boxReg) ; 
3416          masm.movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2] 
3417 
3418          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
3419          if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
3420             // prefetchw [eax + Offset(_owner)-2]
3421             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
3422          }
3423 
3424          if ((EmitSync & 64) == 0) {
3425            // Optimistic form: consider XORL tmpReg,tmpReg
3426            masm.movptr(tmpReg, NULL_WORD) ; 
3427          } else { 
3428            // Can suffer RTS->RTO upgrades on shared or cold $ lines
3429            // Test-And-CAS instead of CAS
3430            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
3431            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
3432            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
3433          }
3434 
3435          // Appears unlocked - try to swing _owner from null to non-null.
3436          // Ideally, I'd manifest "Self" with get_thread and then attempt
3437          // to CAS the register containing Self into m->Owner.
3438          // But we don't have enough registers, so instead we can either try to CAS
3439          // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
3440          // we later store "Self" into m->Owner.  Transiently storing a stack address
3441          // (rsp or the address of the box) into  m->owner is harmless.
3442          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
3443          if (os::is_MP()) { masm.lock();  }
3444          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
3445          masm.movptr(Address(scrReg, 0), 3) ;          // box->_displaced_header = 3
3446          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3447          masm.get_thread (scrReg) ;                    // beware: clobbers ICCs
3448          masm.movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; 
3449          masm.xorptr(boxReg, boxReg) ;                 // set icc.ZFlag = 1 to indicate success
3450                        
3451          // If the CAS fails we can either retry or pass control to the slow-path.  
3452          // We use the latter tactic.  
3453          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
3454          // If the CAS was successful ...
3455          //   Self has acquired the lock
3456          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
3457          // Intentional fall-through into DONE_LABEL ...
3458       } else {
3459          masm.movptr(Address(boxReg, 0), 3) ;       // results in ST-before-CAS penalty
3460          masm.movptr(boxReg, tmpReg) ; 
3461 
3462          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
3463          if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
3464             // prefetchw [eax + Offset(_owner)-2]
3465             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
3466          }
3467 
3468          if ((EmitSync & 64) == 0) {
3469            // Optimistic form
3470            masm.xorptr  (tmpReg, tmpReg) ; 
3471          } else { 
3472            // Can suffer RTS->RTO upgrades on shared or cold $ lines
3473            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
3474            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
3475            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
3476          }
3477 
3478          // Appears unlocked - try to swing _owner from null to non-null.
3479          // Use either "Self" (in scr) or rsp as thread identity in _owner.
3480          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
3481          masm.get_thread (scrReg) ;
3482          if (os::is_MP()) { masm.lock(); }
3483          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3484 
3485          // If the CAS fails we can either retry or pass control to the slow-path.
3486          // We use the latter tactic.
3487          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
3488          // If the CAS was successful ...
3489          //   Self has acquired the lock
3490          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
3491          // Intentional fall-through into DONE_LABEL ...
3492       }
3493 
3494       // DONE_LABEL is a hot target - we'd really like to place it at the
3495       // start of cache line by padding with NOPs.
3496       // See the AMD and Intel software optimization manuals for the
3497       // most efficient "long" NOP encodings.
3498       // Unfortunately none of our alignment mechanisms suffice.
3499       masm.bind(DONE_LABEL);
3500 
3501       // Avoid branch-to-branch on AMD processors
3502       // This appears to be superstition.
3503       if (EmitSync & 32) masm.nop() ;
3504 
3505 
3506       // At DONE_LABEL the icc ZFlag is set as follows ...
3507       // Fast_Unlock uses the same protocol.
3508       // ZFlag == 1 -> Success
3509       // ZFlag == 0 -> Failure - force control through the slow-path
3510     }
3511   %}
3512 
3513   // obj: object to unlock
3514   // box: box address (displaced header location), killed.  Must be EAX.
3515   // rbx,: killed tmp; cannot be obj nor box.
3516   //
3517   // Some commentary on balanced locking:
3518   //
3519   // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
3520   // Methods that don't have provably balanced locking are forced to run in the
3521   // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
3522   // The interpreter provides two properties:
3523   // I1:  At return-time the interpreter automatically and quietly unlocks any
3524   //      objects acquired the current activation (frame).  Recall that the
3525   //      interpreter maintains an on-stack list of locks currently held by
3526   //      a frame.
3527   // I2:  If a method attempts to unlock an object that is not held by the
3528   //      the frame the interpreter throws IMSX.
3529   //
3530   // Lets say A(), which has provably balanced locking, acquires O and then calls B().
3531   // B() doesn't have provably balanced locking so it runs in the interpreter.
3532   // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
3533   // is still locked by A().
3534   //
3535   // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
3536   // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
3537   // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
3538   // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
3539 
3540   enc_class Fast_Unlock( nabxRegP obj, eAXRegP box, eRegP tmp) %{
3541 
3542     Register objReg = as_Register($obj$$reg);
3543     Register boxReg = as_Register($box$$reg);
3544     Register tmpReg = as_Register($tmp$$reg);
3545 
3546     guarantee (objReg != boxReg, "") ;
3547     guarantee (objReg != tmpReg, "") ;
3548     guarantee (boxReg != tmpReg, "") ;
3549     guarantee (boxReg == as_Register(EAX_enc), "") ;
3550     MacroAssembler masm(&cbuf);
3551 
3552     if (EmitSync & 4) {
3553       // Disable - inhibit all inlining.  Force control through the slow-path
3554       masm.cmpptr (rsp, 0) ; 
3555     } else 
3556     if (EmitSync & 8) {
3557       Label DONE_LABEL ;
3558       if (UseBiasedLocking) {
3559          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3560       }
3561       // classic stack-locking code ...
3562       masm.movptr(tmpReg, Address(boxReg, 0)) ;
3563       masm.testptr(tmpReg, tmpReg) ;
3564       masm.jcc   (Assembler::zero, DONE_LABEL) ;
3565       if (os::is_MP()) { masm.lock(); }
3566       masm.cmpxchgptr(tmpReg, Address(objReg, 0));          // Uses EAX which is box
3567       masm.bind(DONE_LABEL);
3568     } else {
3569       Label DONE_LABEL, Stacked, CheckSucc, Inflated ;
3570 
3571       // Critically, the biased locking test must have precedence over
3572       // and appear before the (box->dhw == 0) recursive stack-lock test.
3573       if (UseBiasedLocking && !UseOptoBiasInlining) {
3574          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3575       }
3576       
3577       masm.cmpptr(Address(boxReg, 0), 0) ;            // Examine the displaced header
3578       masm.movptr(tmpReg, Address(objReg, 0)) ;       // Examine the object's markword
3579       masm.jccb  (Assembler::zero, DONE_LABEL) ;      // 0 indicates recursive stack-lock
3580 
3581       masm.testptr(tmpReg, 0x02) ;                     // Inflated? 
3582       masm.jccb  (Assembler::zero, Stacked) ;
3583 
3584       masm.bind  (Inflated) ;
3585       // It's inflated.
3586       // Despite our balanced locking property we still check that m->_owner == Self
3587       // as java routines or native JNI code called by this thread might
3588       // have released the lock.
3589       // Refer to the comments in synchronizer.cpp for how we might encode extra
3590       // state in _succ so we can avoid fetching EntryList|cxq.
3591       //
3592       // I'd like to add more cases in fast_lock() and fast_unlock() --
3593       // such as recursive enter and exit -- but we have to be wary of
3594       // I$ bloat, T$ effects and BP$ effects.
3595       //
3596       // If there's no contention try a 1-0 exit.  That is, exit without
3597       // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
3598       // we detect and recover from the race that the 1-0 exit admits.
3599       //
3600       // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
3601       // before it STs null into _owner, releasing the lock.  Updates
3602       // to data protected by the critical section must be visible before
3603       // we drop the lock (and thus before any other thread could acquire
3604       // the lock and observe the fields protected by the lock).
3605       // IA32's memory-model is SPO, so STs are ordered with respect to
3606       // each other and there's no need for an explicit barrier (fence).
3607       // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
3608 
3609       masm.get_thread (boxReg) ;
3610       if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
3611         // prefetchw [ebx + Offset(_owner)-2]
3612         masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2));
3613       }
3614 
3615       // Note that we could employ various encoding schemes to reduce
3616       // the number of loads below (currently 4) to just 2 or 3.
3617       // Refer to the comments in synchronizer.cpp.
3618       // In practice the chain of fetches doesn't seem to impact performance, however.
3619       if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
3620          // Attempt to reduce branch density - AMD's branch predictor.
3621          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
3622          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3623          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
3624          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
3625          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3626          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3627          masm.jmpb  (DONE_LABEL) ; 
3628       } else { 
3629          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
3630          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3631          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3632          masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
3633          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
3634          masm.jccb  (Assembler::notZero, CheckSucc) ; 
3635          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3636          masm.jmpb  (DONE_LABEL) ; 
3637       }
3638 
3639       // The Following code fragment (EmitSync & 65536) improves the performance of
3640       // contended applications and contended synchronization microbenchmarks.
3641       // Unfortunately the emission of the code - even though not executed - causes regressions
3642       // in scimark and jetstream, evidently because of $ effects.  Replacing the code
3643       // with an equal number of never-executed NOPs results in the same regression.
3644       // We leave it off by default.
3645 
3646       if ((EmitSync & 65536) != 0) {
3647          Label LSuccess, LGoSlowPath ;
3648 
3649          masm.bind  (CheckSucc) ;
3650 
3651          // Optional pre-test ... it's safe to elide this
3652          if ((EmitSync & 16) == 0) { 
3653             masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
3654             masm.jccb  (Assembler::zero, LGoSlowPath) ; 
3655          }
3656 
3657          // We have a classic Dekker-style idiom:
3658          //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
3659          // There are a number of ways to implement the barrier:
3660          // (1) lock:andl &m->_owner, 0
3661          //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
3662          //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
3663          //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
3664          // (2) If supported, an explicit MFENCE is appealing.
3665          //     In older IA32 processors MFENCE is slower than lock:add or xchg
3666          //     particularly if the write-buffer is full as might be the case if
3667          //     if stores closely precede the fence or fence-equivalent instruction.
3668          //     In more modern implementations MFENCE appears faster, however.
3669          // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
3670          //     The $lines underlying the top-of-stack should be in M-state.
3671          //     The locked add instruction is serializing, of course.
3672          // (4) Use xchg, which is serializing
3673          //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
3674          // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
3675          //     The integer condition codes will tell us if succ was 0.
3676          //     Since _succ and _owner should reside in the same $line and
3677          //     we just stored into _owner, it's likely that the $line
3678          //     remains in M-state for the lock:orl.
3679          //
3680          // We currently use (3), although it's likely that switching to (2)
3681          // is correct for the future.
3682             
3683          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3684          if (os::is_MP()) { 
3685             if (VM_Version::supports_sse2() && 1 == FenceInstruction) { 
3686               masm.mfence();
3687             } else { 
3688               masm.lock () ; masm.addptr(Address(rsp, 0), 0) ; 
3689             }
3690          }
3691          // Ratify _succ remains non-null
3692          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
3693          masm.jccb  (Assembler::notZero, LSuccess) ; 
3694 
3695          masm.xorptr(boxReg, boxReg) ;                  // box is really EAX
3696          if (os::is_MP()) { masm.lock(); }
3697          masm.cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
3698          masm.jccb  (Assembler::notEqual, LSuccess) ;
3699          // Since we're low on registers we installed rsp as a placeholding in _owner.
3700          // Now install Self over rsp.  This is safe as we're transitioning from
3701          // non-null to non=null
3702          masm.get_thread (boxReg) ;
3703          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ;
3704          // Intentional fall-through into LGoSlowPath ...
3705 
3706          masm.bind  (LGoSlowPath) ; 
3707          masm.orptr(boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
3708          masm.jmpb  (DONE_LABEL) ; 
3709 
3710          masm.bind  (LSuccess) ; 
3711          masm.xorptr(boxReg, boxReg) ;                 // set ICC.ZF=1 to indicate success
3712          masm.jmpb  (DONE_LABEL) ; 
3713       }
3714 
3715       masm.bind (Stacked) ;
3716       // It's not inflated and it's not recursively stack-locked and it's not biased.
3717       // It must be stack-locked.
3718       // Try to reset the header to displaced header.
3719       // The "box" value on the stack is stable, so we can reload
3720       // and be assured we observe the same value as above.
3721       masm.movptr(tmpReg, Address(boxReg, 0)) ;
3722       if (os::is_MP()) {   masm.lock();    }
3723       masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box
3724       // Intention fall-thru into DONE_LABEL
3725 
3726 
3727       // DONE_LABEL is a hot target - we'd really like to place it at the
3728       // start of cache line by padding with NOPs.
3729       // See the AMD and Intel software optimization manuals for the
3730       // most efficient "long" NOP encodings.
3731       // Unfortunately none of our alignment mechanisms suffice.
3732       if ((EmitSync & 65536) == 0) {
3733          masm.bind (CheckSucc) ;
3734       }
3735       masm.bind(DONE_LABEL);
3736 
3737       // Avoid branch to branch on AMD processors
3738       if (EmitSync & 32768) { masm.nop() ; }
3739     }
3740   %}
3741 
3742 
3743   enc_class enc_pop_rdx() %{
3744     emit_opcode(cbuf,0x5A);
3745   %}
3746 
3747   enc_class enc_rethrow() %{
3748     cbuf.set_insts_mark();
3749     emit_opcode(cbuf, 0xE9);        // jmp    entry
3750     emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.insts_end())-4,
3751                    runtime_call_Relocation::spec(), RELOC_IMM32 );
3752   %}
3753 
3754 
3755   // Convert a double to an int.  Java semantics require we do complex
3756   // manglelations in the corner cases.  So we set the rounding mode to
3757   // 'zero', store the darned double down as an int, and reset the
3758   // rounding mode to 'nearest'.  The hardware throws an exception which
3759   // patches up the correct value directly to the stack.
3760   enc_class D2I_encoding( regD src ) %{
3761     // Flip to round-to-zero mode.  We attempted to allow invalid-op
3762     // exceptions here, so that a NAN or other corner-case value will
3763     // thrown an exception (but normal values get converted at full speed).
3764     // However, I2C adapters and other float-stack manglers leave pending
3765     // invalid-op exceptions hanging.  We would have to clear them before
3766     // enabling them and that is more expensive than just testing for the
3767     // invalid value Intel stores down in the corner cases.
3768     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
3769     emit_opcode(cbuf,0x2D);
3770     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3771     // Allocate a word
3772     emit_opcode(cbuf,0x83);            // SUB ESP,4
3773     emit_opcode(cbuf,0xEC);
3774     emit_d8(cbuf,0x04);
3775     // Encoding assumes a double has been pushed into FPR0.
3776     // Store down the double as an int, popping the FPU stack
3777     emit_opcode(cbuf,0xDB);            // FISTP [ESP]
3778     emit_opcode(cbuf,0x1C);
3779     emit_d8(cbuf,0x24);
3780     // Restore the rounding mode; mask the exception
3781     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
3782     emit_opcode(cbuf,0x2D);
3783     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3784         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3785         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3786 
3787     // Load the converted int; adjust CPU stack
3788     emit_opcode(cbuf,0x58);       // POP EAX
3789     emit_opcode(cbuf,0x3D);       // CMP EAX,imm
3790     emit_d32   (cbuf,0x80000000); //         0x80000000
3791     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3792     emit_d8    (cbuf,0x07);       // Size of slow_call
3793     // Push src onto stack slow-path
3794     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
3795     emit_d8    (cbuf,0xC0-1+$src$$reg );
3796     // CALL directly to the runtime
3797     cbuf.set_insts_mark();
3798     emit_opcode(cbuf,0xE8);       // Call into runtime
3799     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3800     // Carry on here...
3801   %}
3802 
3803   enc_class D2L_encoding( regD src ) %{
3804     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
3805     emit_opcode(cbuf,0x2D);
3806     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3807     // Allocate a word
3808     emit_opcode(cbuf,0x83);            // SUB ESP,8
3809     emit_opcode(cbuf,0xEC);
3810     emit_d8(cbuf,0x08);
3811     // Encoding assumes a double has been pushed into FPR0.
3812     // Store down the double as a long, popping the FPU stack
3813     emit_opcode(cbuf,0xDF);            // FISTP [ESP]
3814     emit_opcode(cbuf,0x3C);
3815     emit_d8(cbuf,0x24);
3816     // Restore the rounding mode; mask the exception
3817     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
3818     emit_opcode(cbuf,0x2D);
3819     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3820         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3821         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3822 
3823     // Load the converted int; adjust CPU stack
3824     emit_opcode(cbuf,0x58);       // POP EAX
3825     emit_opcode(cbuf,0x5A);       // POP EDX
3826     emit_opcode(cbuf,0x81);       // CMP EDX,imm
3827     emit_d8    (cbuf,0xFA);       // rdx
3828     emit_d32   (cbuf,0x80000000); //         0x80000000
3829     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3830     emit_d8    (cbuf,0x07+4);     // Size of slow_call
3831     emit_opcode(cbuf,0x85);       // TEST EAX,EAX
3832     emit_opcode(cbuf,0xC0);       // 2/rax,/rax,
3833     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3834     emit_d8    (cbuf,0x07);       // Size of slow_call
3835     // Push src onto stack slow-path
3836     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
3837     emit_d8    (cbuf,0xC0-1+$src$$reg );
3838     // CALL directly to the runtime
3839     cbuf.set_insts_mark();
3840     emit_opcode(cbuf,0xE8);       // Call into runtime
3841     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3842     // Carry on here...
3843   %}
3844 
3845   enc_class X2L_encoding( regX src ) %{
3846     // Allocate a word
3847     emit_opcode(cbuf,0x83);      // SUB ESP,8
3848     emit_opcode(cbuf,0xEC);
3849     emit_d8(cbuf,0x08);
3850 
3851     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
3852     emit_opcode  (cbuf, 0x0F );
3853     emit_opcode  (cbuf, 0x11 );
3854     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3855 
3856     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
3857     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3858 
3859     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
3860     emit_opcode(cbuf,0x2D);
3861     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3862 
3863     // Encoding assumes a double has been pushed into FPR0.
3864     // Store down the double as a long, popping the FPU stack
3865     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
3866     emit_opcode(cbuf,0x3C);
3867     emit_d8(cbuf,0x24);
3868 
3869     // Restore the rounding mode; mask the exception
3870     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
3871     emit_opcode(cbuf,0x2D);
3872     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3873       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3874       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3875 
3876     // Load the converted int; adjust CPU stack
3877     emit_opcode(cbuf,0x58);      // POP EAX
3878 
3879     emit_opcode(cbuf,0x5A);      // POP EDX
3880 
3881     emit_opcode(cbuf,0x81);      // CMP EDX,imm
3882     emit_d8    (cbuf,0xFA);      // rdx
3883     emit_d32   (cbuf,0x80000000);//         0x80000000
3884 
3885     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3886     emit_d8    (cbuf,0x13+4);    // Size of slow_call
3887 
3888     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
3889     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
3890 
3891     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3892     emit_d8    (cbuf,0x13);      // Size of slow_call
3893 
3894     // Allocate a word
3895     emit_opcode(cbuf,0x83);      // SUB ESP,4
3896     emit_opcode(cbuf,0xEC);
3897     emit_d8(cbuf,0x04);
3898 
3899     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
3900     emit_opcode  (cbuf, 0x0F );
3901     emit_opcode  (cbuf, 0x11 );
3902     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3903 
3904     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
3905     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3906 
3907     emit_opcode(cbuf,0x83);      // ADD ESP,4
3908     emit_opcode(cbuf,0xC4);
3909     emit_d8(cbuf,0x04);
3910 
3911     // CALL directly to the runtime
3912     cbuf.set_insts_mark();
3913     emit_opcode(cbuf,0xE8);       // Call into runtime
3914     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3915     // Carry on here...
3916   %}
3917 
3918   enc_class XD2L_encoding( regXD src ) %{
3919     // Allocate a word
3920     emit_opcode(cbuf,0x83);      // SUB ESP,8
3921     emit_opcode(cbuf,0xEC);
3922     emit_d8(cbuf,0x08);
3923 
3924     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
3925     emit_opcode  (cbuf, 0x0F );
3926     emit_opcode  (cbuf, 0x11 );
3927     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3928 
3929     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
3930     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3931 
3932     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
3933     emit_opcode(cbuf,0x2D);
3934     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3935 
3936     // Encoding assumes a double has been pushed into FPR0.
3937     // Store down the double as a long, popping the FPU stack
3938     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
3939     emit_opcode(cbuf,0x3C);
3940     emit_d8(cbuf,0x24);
3941 
3942     // Restore the rounding mode; mask the exception
3943     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
3944     emit_opcode(cbuf,0x2D);
3945     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3946       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3947       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3948 
3949     // Load the converted int; adjust CPU stack
3950     emit_opcode(cbuf,0x58);      // POP EAX
3951 
3952     emit_opcode(cbuf,0x5A);      // POP EDX
3953 
3954     emit_opcode(cbuf,0x81);      // CMP EDX,imm
3955     emit_d8    (cbuf,0xFA);      // rdx
3956     emit_d32   (cbuf,0x80000000); //         0x80000000
3957 
3958     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3959     emit_d8    (cbuf,0x13+4);    // Size of slow_call
3960 
3961     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
3962     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
3963 
3964     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3965     emit_d8    (cbuf,0x13);      // Size of slow_call
3966 
3967     // Push src onto stack slow-path
3968     // Allocate a word
3969     emit_opcode(cbuf,0x83);      // SUB ESP,8
3970     emit_opcode(cbuf,0xEC);
3971     emit_d8(cbuf,0x08);
3972 
3973     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
3974     emit_opcode  (cbuf, 0x0F );
3975     emit_opcode  (cbuf, 0x11 );
3976     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3977 
3978     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
3979     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3980 
3981     emit_opcode(cbuf,0x83);      // ADD ESP,8
3982     emit_opcode(cbuf,0xC4);
3983     emit_d8(cbuf,0x08);
3984 
3985     // CALL directly to the runtime
3986     cbuf.set_insts_mark();
3987     emit_opcode(cbuf,0xE8);      // Call into runtime
3988     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3989     // Carry on here...
3990   %}
3991 
3992   enc_class D2X_encoding( regX dst, regD src ) %{
3993     // Allocate a word
3994     emit_opcode(cbuf,0x83);            // SUB ESP,4
3995     emit_opcode(cbuf,0xEC);
3996     emit_d8(cbuf,0x04);
3997     int pop = 0x02;
3998     if ($src$$reg != FPR1L_enc) {
3999       emit_opcode( cbuf, 0xD9 );       // FLD    ST(i-1)
4000       emit_d8( cbuf, 0xC0-1+$src$$reg );
4001       pop = 0x03;
4002     }
4003     store_to_stackslot( cbuf, 0xD9, pop, 0 ); // FST<P>_S  [ESP]
4004 
4005     emit_opcode  (cbuf, 0xF3 );        // MOVSS dst(xmm), [ESP]
4006     emit_opcode  (cbuf, 0x0F );
4007     emit_opcode  (cbuf, 0x10 );
4008     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
4009 
4010     emit_opcode(cbuf,0x83);            // ADD ESP,4
4011     emit_opcode(cbuf,0xC4);
4012     emit_d8(cbuf,0x04);
4013     // Carry on here...
4014   %}
4015 
4016   enc_class FX2I_encoding( regX src, eRegI dst ) %{
4017     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
4018 
4019     // Compare the result to see if we need to go to the slow path
4020     emit_opcode(cbuf,0x81);       // CMP dst,imm
4021     emit_rm    (cbuf,0x3,0x7,$dst$$reg);
4022     emit_d32   (cbuf,0x80000000); //         0x80000000
4023 
4024     emit_opcode(cbuf,0x75);       // JNE around_slow_call
4025     emit_d8    (cbuf,0x13);       // Size of slow_call
4026     // Store xmm to a temp memory
4027     // location and push it onto stack.
4028 
4029     emit_opcode(cbuf,0x83);  // SUB ESP,4
4030     emit_opcode(cbuf,0xEC);
4031     emit_d8(cbuf, $primary ? 0x8 : 0x4);
4032 
4033     emit_opcode  (cbuf, $primary ? 0xF2 : 0xF3 );   // MOVSS [ESP], xmm
4034     emit_opcode  (cbuf, 0x0F );
4035     emit_opcode  (cbuf, 0x11 );
4036     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
4037 
4038     emit_opcode(cbuf, $primary ? 0xDD : 0xD9 );      // FLD [ESP]
4039     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
4040 
4041     emit_opcode(cbuf,0x83);    // ADD ESP,4
4042     emit_opcode(cbuf,0xC4);
4043     emit_d8(cbuf, $primary ? 0x8 : 0x4);
4044 
4045     // CALL directly to the runtime
4046     cbuf.set_insts_mark();
4047     emit_opcode(cbuf,0xE8);       // Call into runtime
4048     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.insts_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
4049 
4050     // Carry on here...
4051   %}
4052 
4053   enc_class X2D_encoding( regD dst, regX src ) %{
4054     // Allocate a word
4055     emit_opcode(cbuf,0x83);     // SUB ESP,4
4056     emit_opcode(cbuf,0xEC);
4057     emit_d8(cbuf,0x04);
4058 
4059     emit_opcode  (cbuf, 0xF3 ); // MOVSS [ESP], xmm
4060     emit_opcode  (cbuf, 0x0F );
4061     emit_opcode  (cbuf, 0x11 );
4062     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
4063 
4064     emit_opcode(cbuf,0xD9 );    // FLD_S [ESP]
4065     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
4066 
4067     emit_opcode(cbuf,0x83);     // ADD ESP,4
4068     emit_opcode(cbuf,0xC4);
4069     emit_d8(cbuf,0x04);
4070 
4071     // Carry on here...
4072   %}
4073 
4074   enc_class AbsXF_encoding(regX dst) %{
4075     address signmask_address=(address)float_signmask_pool;
4076     // andpd:\tANDPS  $dst,[signconst]
4077     emit_opcode(cbuf, 0x0F);
4078     emit_opcode(cbuf, 0x54);
4079     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4080     emit_d32(cbuf, (int)signmask_address);
4081   %}
4082 
4083   enc_class AbsXD_encoding(regXD dst) %{
4084     address signmask_address=(address)double_signmask_pool;
4085     // andpd:\tANDPD  $dst,[signconst]
4086     emit_opcode(cbuf, 0x66);
4087     emit_opcode(cbuf, 0x0F);
4088     emit_opcode(cbuf, 0x54);
4089     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4090     emit_d32(cbuf, (int)signmask_address);
4091   %}
4092 
4093   enc_class NegXF_encoding(regX dst) %{
4094     address signmask_address=(address)float_signflip_pool;
4095     // andpd:\tXORPS  $dst,[signconst]
4096     emit_opcode(cbuf, 0x0F);
4097     emit_opcode(cbuf, 0x57);
4098     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4099     emit_d32(cbuf, (int)signmask_address);
4100   %}
4101 
4102   enc_class NegXD_encoding(regXD dst) %{
4103     address signmask_address=(address)double_signflip_pool;
4104     // andpd:\tXORPD  $dst,[signconst]
4105     emit_opcode(cbuf, 0x66);
4106     emit_opcode(cbuf, 0x0F);
4107     emit_opcode(cbuf, 0x57);
4108     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4109     emit_d32(cbuf, (int)signmask_address);
4110   %}
4111 
4112   enc_class FMul_ST_reg( eRegF src1 ) %{
4113     // Operand was loaded from memory into fp ST (stack top)
4114     // FMUL   ST,$src  /* D8 C8+i */
4115     emit_opcode(cbuf, 0xD8);
4116     emit_opcode(cbuf, 0xC8 + $src1$$reg);
4117   %}
4118 
4119   enc_class FAdd_ST_reg( eRegF src2 ) %{
4120     // FADDP  ST,src2  /* D8 C0+i */
4121     emit_opcode(cbuf, 0xD8);
4122     emit_opcode(cbuf, 0xC0 + $src2$$reg);
4123     //could use FADDP  src2,fpST  /* DE C0+i */
4124   %}
4125 
4126   enc_class FAddP_reg_ST( eRegF src2 ) %{
4127     // FADDP  src2,ST  /* DE C0+i */
4128     emit_opcode(cbuf, 0xDE);
4129     emit_opcode(cbuf, 0xC0 + $src2$$reg);
4130   %}
4131 
4132   enc_class subF_divF_encode( eRegF src1, eRegF src2) %{
4133     // Operand has been loaded into fp ST (stack top)
4134       // FSUB   ST,$src1
4135       emit_opcode(cbuf, 0xD8);
4136       emit_opcode(cbuf, 0xE0 + $src1$$reg);
4137 
4138       // FDIV
4139       emit_opcode(cbuf, 0xD8);
4140       emit_opcode(cbuf, 0xF0 + $src2$$reg);
4141   %}
4142 
4143   enc_class MulFAddF (eRegF src1, eRegF src2) %{
4144     // Operand was loaded from memory into fp ST (stack top)
4145     // FADD   ST,$src  /* D8 C0+i */
4146     emit_opcode(cbuf, 0xD8);
4147     emit_opcode(cbuf, 0xC0 + $src1$$reg);
4148 
4149     // FMUL  ST,src2  /* D8 C*+i */
4150     emit_opcode(cbuf, 0xD8);
4151     emit_opcode(cbuf, 0xC8 + $src2$$reg);
4152   %}
4153 
4154 
4155   enc_class MulFAddFreverse (eRegF src1, eRegF src2) %{
4156     // Operand was loaded from memory into fp ST (stack top)
4157     // FADD   ST,$src  /* D8 C0+i */
4158     emit_opcode(cbuf, 0xD8);
4159     emit_opcode(cbuf, 0xC0 + $src1$$reg);
4160 
4161     // FMULP  src2,ST  /* DE C8+i */
4162     emit_opcode(cbuf, 0xDE);
4163     emit_opcode(cbuf, 0xC8 + $src2$$reg);
4164   %}
4165 
4166   // Atomically load the volatile long
4167   enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{
4168     emit_opcode(cbuf,0xDF);
4169     int rm_byte_opcode = 0x05;
4170     int base     = $mem$$base;
4171     int index    = $mem$$index;
4172     int scale    = $mem$$scale;
4173     int displace = $mem$$disp;
4174     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4175     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
4176     store_to_stackslot( cbuf, 0x0DF, 0x07, $dst$$disp );
4177   %}
4178 
4179   enc_class enc_loadLX_volatile( memory mem, stackSlotL dst, regXD tmp ) %{
4180     { // Atomic long load
4181       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
4182       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4183       emit_opcode(cbuf,0x0F);
4184       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4185       int base     = $mem$$base;
4186       int index    = $mem$$index;
4187       int scale    = $mem$$scale;
4188       int displace = $mem$$disp;
4189       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4190       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4191     }
4192     { // MOVSD $dst,$tmp ! atomic long store
4193       emit_opcode(cbuf,0xF2);
4194       emit_opcode(cbuf,0x0F);
4195       emit_opcode(cbuf,0x11);
4196       int base     = $dst$$base;
4197       int index    = $dst$$index;
4198       int scale    = $dst$$scale;
4199       int displace = $dst$$disp;
4200       bool disp_is_oop = $dst->disp_is_oop(); // disp-as-oop when working with static globals
4201       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4202     }
4203   %}
4204 
4205   enc_class enc_loadLX_reg_volatile( memory mem, eRegL dst, regXD tmp ) %{
4206     { // Atomic long load
4207       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
4208       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4209       emit_opcode(cbuf,0x0F);
4210       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4211       int base     = $mem$$base;
4212       int index    = $mem$$index;
4213       int scale    = $mem$$scale;
4214       int displace = $mem$$disp;
4215       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4216       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4217     }
4218     { // MOVD $dst.lo,$tmp
4219       emit_opcode(cbuf,0x66);
4220       emit_opcode(cbuf,0x0F);
4221       emit_opcode(cbuf,0x7E);
4222       emit_rm(cbuf, 0x3, $tmp$$reg, $dst$$reg);
4223     }
4224     { // PSRLQ $tmp,32
4225       emit_opcode(cbuf,0x66);
4226       emit_opcode(cbuf,0x0F);
4227       emit_opcode(cbuf,0x73);
4228       emit_rm(cbuf, 0x3, 0x02, $tmp$$reg);
4229       emit_d8(cbuf, 0x20);
4230     }
4231     { // MOVD $dst.hi,$tmp
4232       emit_opcode(cbuf,0x66);
4233       emit_opcode(cbuf,0x0F);
4234       emit_opcode(cbuf,0x7E);
4235       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
4236     }
4237   %}
4238 
4239   // Volatile Store Long.  Must be atomic, so move it into
4240   // the FP TOS and then do a 64-bit FIST.  Has to probe the
4241   // target address before the store (for null-ptr checks)
4242   // so the memory operand is used twice in the encoding.
4243   enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
4244     store_to_stackslot( cbuf, 0x0DF, 0x05, $src$$disp );
4245     cbuf.set_insts_mark();            // Mark start of FIST in case $mem has an oop
4246     emit_opcode(cbuf,0xDF);
4247     int rm_byte_opcode = 0x07;
4248     int base     = $mem$$base;
4249     int index    = $mem$$index;
4250     int scale    = $mem$$scale;
4251     int displace = $mem$$disp;
4252     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4253     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
4254   %}
4255 
4256   enc_class enc_storeLX_volatile( memory mem, stackSlotL src, regXD tmp) %{
4257     { // Atomic long load
4258       // UseXmmLoadAndClearUpper ? movsd $tmp,[$src] : movlpd $tmp,[$src]
4259       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4260       emit_opcode(cbuf,0x0F);
4261       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4262       int base     = $src$$base;
4263       int index    = $src$$index;
4264       int scale    = $src$$scale;
4265       int displace = $src$$disp;
4266       bool disp_is_oop = $src->disp_is_oop(); // disp-as-oop when working with static globals
4267       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4268     }
4269     cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
4270     { // MOVSD $mem,$tmp ! atomic long store
4271       emit_opcode(cbuf,0xF2);
4272       emit_opcode(cbuf,0x0F);
4273       emit_opcode(cbuf,0x11);
4274       int base     = $mem$$base;
4275       int index    = $mem$$index;
4276       int scale    = $mem$$scale;
4277       int displace = $mem$$disp;
4278       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4279       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4280     }
4281   %}
4282 
4283   enc_class enc_storeLX_reg_volatile( memory mem, eRegL src, regXD tmp, regXD tmp2) %{
4284     { // MOVD $tmp,$src.lo
4285       emit_opcode(cbuf,0x66);
4286       emit_opcode(cbuf,0x0F);
4287       emit_opcode(cbuf,0x6E);
4288       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
4289     }
4290     { // MOVD $tmp2,$src.hi
4291       emit_opcode(cbuf,0x66);
4292       emit_opcode(cbuf,0x0F);
4293       emit_opcode(cbuf,0x6E);
4294       emit_rm(cbuf, 0x3, $tmp2$$reg, HIGH_FROM_LOW($src$$reg));
4295     }
4296     { // PUNPCKLDQ $tmp,$tmp2
4297       emit_opcode(cbuf,0x66);
4298       emit_opcode(cbuf,0x0F);
4299       emit_opcode(cbuf,0x62);
4300       emit_rm(cbuf, 0x3, $tmp$$reg, $tmp2$$reg);
4301     }
4302     cbuf.set_insts_mark();            // Mark start of MOVSD in case $mem has an oop
4303     { // MOVSD $mem,$tmp ! atomic long store
4304       emit_opcode(cbuf,0xF2);
4305       emit_opcode(cbuf,0x0F);
4306       emit_opcode(cbuf,0x11);
4307       int base     = $mem$$base;
4308       int index    = $mem$$index;
4309       int scale    = $mem$$scale;
4310       int displace = $mem$$disp;
4311       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4312       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4313     }
4314   %}
4315 
4316   // Safepoint Poll.  This polls the safepoint page, and causes an
4317   // exception if it is not readable. Unfortunately, it kills the condition code
4318   // in the process
4319   // We current use TESTL [spp],EDI
4320   // A better choice might be TESTB [spp + pagesize() - CacheLineSize()],0
4321 
4322   enc_class Safepoint_Poll() %{
4323     cbuf.relocate(cbuf.insts_mark(), relocInfo::poll_type, 0);
4324     emit_opcode(cbuf,0x85);
4325     emit_rm (cbuf, 0x0, 0x7, 0x5);
4326     emit_d32(cbuf, (intptr_t)os::get_polling_page());
4327   %}
4328 %}
4329 
4330 
4331 //----------FRAME--------------------------------------------------------------
4332 // Definition of frame structure and management information.
4333 //
4334 //  S T A C K   L A Y O U T    Allocators stack-slot number
4335 //                             |   (to get allocators register number
4336 //  G  Owned by    |        |  v    add OptoReg::stack0())
4337 //  r   CALLER     |        |
4338 //  o     |        +--------+      pad to even-align allocators stack-slot
4339 //  w     V        |  pad0  |        numbers; owned by CALLER
4340 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
4341 //  h     ^        |   in   |  5
4342 //        |        |  args  |  4   Holes in incoming args owned by SELF
4343 //  |     |        |        |  3
4344 //  |     |        +--------+
4345 //  V     |        | old out|      Empty on Intel, window on Sparc
4346 //        |    old |preserve|      Must be even aligned.
4347 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
4348 //        |        |   in   |  3   area for Intel ret address
4349 //     Owned by    |preserve|      Empty on Sparc.
4350 //       SELF      +--------+
4351 //        |        |  pad2  |  2   pad to align old SP
4352 //        |        +--------+  1
4353 //        |        | locks  |  0
4354 //        |        +--------+----> OptoReg::stack0(), even aligned
4355 //        |        |  pad1  | 11   pad to align new SP
4356 //        |        +--------+
4357 //        |        |        | 10
4358 //        |        | spills |  9   spills
4359 //        V        |        |  8   (pad0 slot for callee)
4360 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
4361 //        ^        |  out   |  7
4362 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
4363 //     Owned by    +--------+
4364 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
4365 //        |    new |preserve|      Must be even-aligned.
4366 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
4367 //        |        |        |
4368 //
4369 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
4370 //         known from SELF's arguments and the Java calling convention.
4371 //         Region 6-7 is determined per call site.
4372 // Note 2: If the calling convention leaves holes in the incoming argument
4373 //         area, those holes are owned by SELF.  Holes in the outgoing area
4374 //         are owned by the CALLEE.  Holes should not be nessecary in the
4375 //         incoming area, as the Java calling convention is completely under
4376 //         the control of the AD file.  Doubles can be sorted and packed to
4377 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
4378 //         varargs C calling conventions.
4379 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
4380 //         even aligned with pad0 as needed.
4381 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
4382 //         region 6-11 is even aligned; it may be padded out more so that
4383 //         the region from SP to FP meets the minimum stack alignment.
4384 
4385 frame %{
4386   // What direction does stack grow in (assumed to be same for C & Java)
4387   stack_direction(TOWARDS_LOW);
4388 
4389   // These three registers define part of the calling convention
4390   // between compiled code and the interpreter.
4391   inline_cache_reg(EAX);                // Inline Cache Register
4392   interpreter_method_oop_reg(EBX);      // Method Oop Register when calling interpreter
4393 
4394   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
4395   cisc_spilling_operand_name(indOffset32);
4396 
4397   // Number of stack slots consumed by locking an object
4398   sync_stack_slots(1);
4399 
4400   // Compiled code's Frame Pointer
4401   frame_pointer(ESP);
4402   // Interpreter stores its frame pointer in a register which is
4403   // stored to the stack by I2CAdaptors.
4404   // I2CAdaptors convert from interpreted java to compiled java.
4405   interpreter_frame_pointer(EBP);
4406 
4407   // Stack alignment requirement
4408   // Alignment size in bytes (128-bit -> 16 bytes)
4409   stack_alignment(StackAlignmentInBytes);
4410 
4411   // Number of stack slots between incoming argument block and the start of
4412   // a new frame.  The PROLOG must add this many slots to the stack.  The
4413   // EPILOG must remove this many slots.  Intel needs one slot for
4414   // return address and one for rbp, (must save rbp)
4415   in_preserve_stack_slots(2+VerifyStackAtCalls);
4416 
4417   // Number of outgoing stack slots killed above the out_preserve_stack_slots
4418   // for calls to C.  Supports the var-args backing area for register parms.
4419   varargs_C_out_slots_killed(0);
4420 
4421   // The after-PROLOG location of the return address.  Location of
4422   // return address specifies a type (REG or STACK) and a number
4423   // representing the register number (i.e. - use a register name) or
4424   // stack slot.
4425   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4426   // Otherwise, it is above the locks and verification slot and alignment word
4427   return_addr(STACK - 1 +
4428               round_to(1+VerifyStackAtCalls+
4429               Compile::current()->fixed_slots(),
4430               (StackAlignmentInBytes/wordSize)));
4431 
4432   // Body of function which returns an integer array locating
4433   // arguments either in registers or in stack slots.  Passed an array
4434   // of ideal registers called "sig" and a "length" count.  Stack-slot
4435   // offsets are based on outgoing arguments, i.e. a CALLER setting up
4436   // arguments for a CALLEE.  Incoming stack arguments are
4437   // automatically biased by the preserve_stack_slots field above.
4438   calling_convention %{
4439     // No difference between ingoing/outgoing just pass false
4440     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
4441   %}
4442 
4443 
4444   // Body of function which returns an integer array locating
4445   // arguments either in registers or in stack slots.  Passed an array
4446   // of ideal registers called "sig" and a "length" count.  Stack-slot
4447   // offsets are based on outgoing arguments, i.e. a CALLER setting up
4448   // arguments for a CALLEE.  Incoming stack arguments are
4449   // automatically biased by the preserve_stack_slots field above.
4450   c_calling_convention %{
4451     // This is obviously always outgoing
4452     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
4453   %}
4454 
4455   // Location of C & interpreter return values
4456   c_return_value %{
4457     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
4458     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
4459     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
4460 
4461     // in SSE2+ mode we want to keep the FPU stack clean so pretend
4462     // that C functions return float and double results in XMM0.
4463     if( ideal_reg == Op_RegD && UseSSE>=2 )
4464       return OptoRegPair(XMM0b_num,XMM0a_num);
4465     if( ideal_reg == Op_RegF && UseSSE>=2 )
4466       return OptoRegPair(OptoReg::Bad,XMM0a_num);
4467 
4468     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
4469   %}
4470 
4471   // Location of return values
4472   return_value %{
4473     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
4474     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
4475     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
4476     if( ideal_reg == Op_RegD && UseSSE>=2 )
4477       return OptoRegPair(XMM0b_num,XMM0a_num);
4478     if( ideal_reg == Op_RegF && UseSSE>=1 )
4479       return OptoRegPair(OptoReg::Bad,XMM0a_num);
4480     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
4481   %}
4482 
4483 %}
4484 
4485 //----------ATTRIBUTES---------------------------------------------------------
4486 //----------Operand Attributes-------------------------------------------------
4487 op_attrib op_cost(0);        // Required cost attribute
4488 
4489 //----------Instruction Attributes---------------------------------------------
4490 ins_attrib ins_cost(100);       // Required cost attribute
4491 ins_attrib ins_size(8);         // Required size attribute (in bits)
4492 ins_attrib ins_short_branch(0); // Required flag: is this instruction a
4493                                 // non-matching short branch variant of some
4494                                                             // long branch?
4495 ins_attrib ins_alignment(1);    // Required alignment attribute (must be a power of 2)
4496                                 // specifies the alignment that some part of the instruction (not
4497                                 // necessarily the start) requires.  If > 1, a compute_padding()
4498                                 // function must be provided for the instruction
4499 
4500 //----------OPERANDS-----------------------------------------------------------
4501 // Operand definitions must precede instruction definitions for correct parsing
4502 // in the ADLC because operands constitute user defined types which are used in
4503 // instruction definitions.
4504 
4505 //----------Simple Operands----------------------------------------------------
4506 // Immediate Operands
4507 // Integer Immediate
4508 operand immI() %{
4509   match(ConI);
4510 
4511   op_cost(10);
4512   format %{ %}
4513   interface(CONST_INTER);
4514 %}
4515 
4516 // Constant for test vs zero
4517 operand immI0() %{
4518   predicate(n->get_int() == 0);
4519   match(ConI);
4520 
4521   op_cost(0);
4522   format %{ %}
4523   interface(CONST_INTER);
4524 %}
4525 
4526 // Constant for increment
4527 operand immI1() %{
4528   predicate(n->get_int() == 1);
4529   match(ConI);
4530 
4531   op_cost(0);
4532   format %{ %}
4533   interface(CONST_INTER);
4534 %}
4535 
4536 // Constant for decrement
4537 operand immI_M1() %{
4538   predicate(n->get_int() == -1);
4539   match(ConI);
4540 
4541   op_cost(0);
4542   format %{ %}
4543   interface(CONST_INTER);
4544 %}
4545 
4546 // Valid scale values for addressing modes
4547 operand immI2() %{
4548   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4549   match(ConI);
4550 
4551   format %{ %}
4552   interface(CONST_INTER);
4553 %}
4554 
4555 operand immI8() %{
4556   predicate((-128 <= n->get_int()) && (n->get_int() <= 127));
4557   match(ConI);
4558 
4559   op_cost(5);
4560   format %{ %}
4561   interface(CONST_INTER);
4562 %}
4563 
4564 operand immI16() %{
4565   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4566   match(ConI);
4567 
4568   op_cost(10);
4569   format %{ %}
4570   interface(CONST_INTER);
4571 %}
4572 
4573 // Constant for long shifts
4574 operand immI_32() %{
4575   predicate( n->get_int() == 32 );
4576   match(ConI);
4577 
4578   op_cost(0);
4579   format %{ %}
4580   interface(CONST_INTER);
4581 %}
4582 
4583 operand immI_1_31() %{
4584   predicate( n->get_int() >= 1 && n->get_int() <= 31 );
4585   match(ConI);
4586 
4587   op_cost(0);
4588   format %{ %}
4589   interface(CONST_INTER);
4590 %}
4591 
4592 operand immI_32_63() %{
4593   predicate( n->get_int() >= 32 && n->get_int() <= 63 );
4594   match(ConI);
4595   op_cost(0);
4596 
4597   format %{ %}
4598   interface(CONST_INTER);
4599 %}
4600 
4601 operand immI_1() %{
4602   predicate( n->get_int() == 1 );
4603   match(ConI);
4604 
4605   op_cost(0);
4606   format %{ %}
4607   interface(CONST_INTER);
4608 %}
4609 
4610 operand immI_2() %{
4611   predicate( n->get_int() == 2 );
4612   match(ConI);
4613 
4614   op_cost(0);
4615   format %{ %}
4616   interface(CONST_INTER);
4617 %}
4618 
4619 operand immI_3() %{
4620   predicate( n->get_int() == 3 );
4621   match(ConI);
4622 
4623   op_cost(0);
4624   format %{ %}
4625   interface(CONST_INTER);
4626 %}
4627 
4628 // Pointer Immediate
4629 operand immP() %{
4630   match(ConP);
4631 
4632   op_cost(10);
4633   format %{ %}
4634   interface(CONST_INTER);
4635 %}
4636 
4637 // NULL Pointer Immediate
4638 operand immP0() %{
4639   predicate( n->get_ptr() == 0 );
4640   match(ConP);
4641   op_cost(0);
4642 
4643   format %{ %}
4644   interface(CONST_INTER);
4645 %}
4646 
4647 // Long Immediate
4648 operand immL() %{
4649   match(ConL);
4650 
4651   op_cost(20);
4652   format %{ %}
4653   interface(CONST_INTER);
4654 %}
4655 
4656 // Long Immediate zero
4657 operand immL0() %{
4658   predicate( n->get_long() == 0L );
4659   match(ConL);
4660   op_cost(0);
4661 
4662   format %{ %}
4663   interface(CONST_INTER);
4664 %}
4665 
4666 // Long Immediate zero
4667 operand immL_M1() %{
4668   predicate( n->get_long() == -1L );
4669   match(ConL);
4670   op_cost(0);
4671 
4672   format %{ %}
4673   interface(CONST_INTER);
4674 %}
4675 
4676 // Long immediate from 0 to 127.
4677 // Used for a shorter form of long mul by 10.
4678 operand immL_127() %{
4679   predicate((0 <= n->get_long()) && (n->get_long() <= 127));
4680   match(ConL);
4681   op_cost(0);
4682 
4683   format %{ %}
4684   interface(CONST_INTER);
4685 %}
4686 
4687 // Long Immediate: low 32-bit mask
4688 operand immL_32bits() %{
4689   predicate(n->get_long() == 0xFFFFFFFFL);
4690   match(ConL);
4691   op_cost(0);
4692 
4693   format %{ %}
4694   interface(CONST_INTER);
4695 %}
4696 
4697 // Long Immediate: low 32-bit mask
4698 operand immL32() %{
4699   predicate(n->get_long() == (int)(n->get_long()));
4700   match(ConL);
4701   op_cost(20);
4702 
4703   format %{ %}
4704   interface(CONST_INTER);
4705 %}
4706 
4707 //Double Immediate zero
4708 operand immD0() %{
4709   // Do additional (and counter-intuitive) test against NaN to work around VC++
4710   // bug that generates code such that NaNs compare equal to 0.0
4711   predicate( UseSSE<=1 && n->getd() == 0.0 && !g_isnan(n->getd()) );
4712   match(ConD);
4713 
4714   op_cost(5);
4715   format %{ %}
4716   interface(CONST_INTER);
4717 %}
4718 
4719 // Double Immediate one
4720 operand immD1() %{
4721   predicate( UseSSE<=1 && n->getd() == 1.0 );
4722   match(ConD);
4723 
4724   op_cost(5);
4725   format %{ %}
4726   interface(CONST_INTER);
4727 %}
4728 
4729 // Double Immediate
4730 operand immD() %{
4731   predicate(UseSSE<=1);
4732   match(ConD);
4733 
4734   op_cost(5);
4735   format %{ %}
4736   interface(CONST_INTER);
4737 %}
4738 
4739 operand immXD() %{
4740   predicate(UseSSE>=2);
4741   match(ConD);
4742 
4743   op_cost(5);
4744   format %{ %}
4745   interface(CONST_INTER);
4746 %}
4747 
4748 // Double Immediate zero
4749 operand immXD0() %{
4750   // Do additional (and counter-intuitive) test against NaN to work around VC++
4751   // bug that generates code such that NaNs compare equal to 0.0 AND do not
4752   // compare equal to -0.0.
4753   predicate( UseSSE>=2 && jlong_cast(n->getd()) == 0 );
4754   match(ConD);
4755 
4756   format %{ %}
4757   interface(CONST_INTER);
4758 %}
4759 
4760 // Float Immediate zero
4761 operand immF0() %{
4762   predicate(UseSSE == 0 && n->getf() == 0.0F);
4763   match(ConF);
4764 
4765   op_cost(5);
4766   format %{ %}
4767   interface(CONST_INTER);
4768 %}
4769 
4770 // Float Immediate one
4771 operand immF1() %{
4772   predicate(UseSSE == 0 && n->getf() == 1.0F);
4773   match(ConF);
4774 
4775   op_cost(5);
4776   format %{ %}
4777   interface(CONST_INTER);
4778 %}
4779 
4780 // Float Immediate
4781 operand immF() %{
4782   predicate( UseSSE == 0 );
4783   match(ConF);
4784 
4785   op_cost(5);
4786   format %{ %}
4787   interface(CONST_INTER);
4788 %}
4789 
4790 // Float Immediate
4791 operand immXF() %{
4792   predicate(UseSSE >= 1);
4793   match(ConF);
4794 
4795   op_cost(5);
4796   format %{ %}
4797   interface(CONST_INTER);
4798 %}
4799 
4800 // Float Immediate zero.  Zero and not -0.0
4801 operand immXF0() %{
4802   predicate( UseSSE >= 1 && jint_cast(n->getf()) == 0 );
4803   match(ConF);
4804 
4805   op_cost(5);
4806   format %{ %}
4807   interface(CONST_INTER);
4808 %}
4809 
4810 // Immediates for special shifts (sign extend)
4811 
4812 // Constants for increment
4813 operand immI_16() %{
4814   predicate( n->get_int() == 16 );
4815   match(ConI);
4816 
4817   format %{ %}
4818   interface(CONST_INTER);
4819 %}
4820 
4821 operand immI_24() %{
4822   predicate( n->get_int() == 24 );
4823   match(ConI);
4824 
4825   format %{ %}
4826   interface(CONST_INTER);
4827 %}
4828 
4829 // Constant for byte-wide masking
4830 operand immI_255() %{
4831   predicate( n->get_int() == 255 );
4832   match(ConI);
4833 
4834   format %{ %}
4835   interface(CONST_INTER);
4836 %}
4837 
4838 // Constant for short-wide masking
4839 operand immI_65535() %{
4840   predicate(n->get_int() == 65535);
4841   match(ConI);
4842 
4843   format %{ %}
4844   interface(CONST_INTER);
4845 %}
4846 
4847 // Register Operands
4848 // Integer Register
4849 operand eRegI() %{
4850   constraint(ALLOC_IN_RC(e_reg));
4851   match(RegI);
4852   match(xRegI);
4853   match(eAXRegI);
4854   match(eBXRegI);
4855   match(eCXRegI);
4856   match(eDXRegI);
4857   match(eDIRegI);
4858   match(eSIRegI);
4859 
4860   format %{ %}
4861   interface(REG_INTER);
4862 %}
4863 
4864 // Subset of Integer Register
4865 operand xRegI(eRegI reg) %{
4866   constraint(ALLOC_IN_RC(x_reg));
4867   match(reg);
4868   match(eAXRegI);
4869   match(eBXRegI);
4870   match(eCXRegI);
4871   match(eDXRegI);
4872 
4873   format %{ %}
4874   interface(REG_INTER);
4875 %}
4876 
4877 // Special Registers
4878 operand eAXRegI(xRegI reg) %{
4879   constraint(ALLOC_IN_RC(eax_reg));
4880   match(reg);
4881   match(eRegI);
4882 
4883   format %{ "EAX" %}
4884   interface(REG_INTER);
4885 %}
4886 
4887 // Special Registers
4888 operand eBXRegI(xRegI reg) %{
4889   constraint(ALLOC_IN_RC(ebx_reg));
4890   match(reg);
4891   match(eRegI);
4892 
4893   format %{ "EBX" %}
4894   interface(REG_INTER);
4895 %}
4896 
4897 operand eCXRegI(xRegI reg) %{
4898   constraint(ALLOC_IN_RC(ecx_reg));
4899   match(reg);
4900   match(eRegI);
4901 
4902   format %{ "ECX" %}
4903   interface(REG_INTER);
4904 %}
4905 
4906 operand eDXRegI(xRegI reg) %{
4907   constraint(ALLOC_IN_RC(edx_reg));
4908   match(reg);
4909   match(eRegI);
4910 
4911   format %{ "EDX" %}
4912   interface(REG_INTER);
4913 %}
4914 
4915 operand eDIRegI(xRegI reg) %{
4916   constraint(ALLOC_IN_RC(edi_reg));
4917   match(reg);
4918   match(eRegI);
4919 
4920   format %{ "EDI" %}
4921   interface(REG_INTER);
4922 %}
4923 
4924 operand naxRegI() %{
4925   constraint(ALLOC_IN_RC(nax_reg));
4926   match(RegI);
4927   match(eCXRegI);
4928   match(eDXRegI);
4929   match(eSIRegI);
4930   match(eDIRegI);
4931 
4932   format %{ %}
4933   interface(REG_INTER);
4934 %}
4935 
4936 operand nadxRegI() %{
4937   constraint(ALLOC_IN_RC(nadx_reg));
4938   match(RegI);
4939   match(eBXRegI);
4940   match(eCXRegI);
4941   match(eSIRegI);
4942   match(eDIRegI);
4943 
4944   format %{ %}
4945   interface(REG_INTER);
4946 %}
4947 
4948 operand ncxRegI() %{
4949   constraint(ALLOC_IN_RC(ncx_reg));
4950   match(RegI);
4951   match(eAXRegI);
4952   match(eDXRegI);
4953   match(eSIRegI);
4954   match(eDIRegI);
4955 
4956   format %{ %}
4957   interface(REG_INTER);
4958 %}
4959 
4960 // // This operand was used by cmpFastUnlock, but conflicted with 'object' reg
4961 // //
4962 operand eSIRegI(xRegI reg) %{
4963    constraint(ALLOC_IN_RC(esi_reg));
4964    match(reg);
4965    match(eRegI);
4966 
4967    format %{ "ESI" %}
4968    interface(REG_INTER);
4969 %}
4970 
4971 // Pointer Register
4972 operand anyRegP() %{
4973   constraint(ALLOC_IN_RC(any_reg));
4974   match(RegP);
4975   match(eAXRegP);
4976   match(eBXRegP);
4977   match(eCXRegP);
4978   match(eDIRegP);
4979   match(eRegP);
4980 
4981   format %{ %}
4982   interface(REG_INTER);
4983 %}
4984 
4985 operand eRegP() %{
4986   constraint(ALLOC_IN_RC(e_reg));
4987   match(RegP);
4988   match(eAXRegP);
4989   match(eBXRegP);
4990   match(eCXRegP);
4991   match(eDIRegP);
4992 
4993   format %{ %}
4994   interface(REG_INTER);
4995 %}
4996 
4997 // On windows95, EBP is not safe to use for implicit null tests.
4998 operand eRegP_no_EBP() %{
4999   constraint(ALLOC_IN_RC(e_reg_no_rbp));
5000   match(RegP);
5001   match(eAXRegP);
5002   match(eBXRegP);
5003   match(eCXRegP);
5004   match(eDIRegP);
5005 
5006   op_cost(100);
5007   format %{ %}
5008   interface(REG_INTER);
5009 %}
5010 
5011 operand naxRegP() %{
5012   constraint(ALLOC_IN_RC(nax_reg));
5013   match(RegP);
5014   match(eBXRegP);
5015   match(eDXRegP);
5016   match(eCXRegP);
5017   match(eSIRegP);
5018   match(eDIRegP);
5019 
5020   format %{ %}
5021   interface(REG_INTER);
5022 %}
5023 
5024 operand nabxRegP() %{
5025   constraint(ALLOC_IN_RC(nabx_reg));
5026   match(RegP);
5027   match(eCXRegP);
5028   match(eDXRegP);
5029   match(eSIRegP);
5030   match(eDIRegP);
5031 
5032   format %{ %}
5033   interface(REG_INTER);
5034 %}
5035 
5036 operand pRegP() %{
5037   constraint(ALLOC_IN_RC(p_reg));
5038   match(RegP);
5039   match(eBXRegP);
5040   match(eDXRegP);
5041   match(eSIRegP);
5042   match(eDIRegP);
5043 
5044   format %{ %}
5045   interface(REG_INTER);
5046 %}
5047 
5048 // Special Registers
5049 // Return a pointer value
5050 operand eAXRegP(eRegP reg) %{
5051   constraint(ALLOC_IN_RC(eax_reg));
5052   match(reg);
5053   format %{ "EAX" %}
5054   interface(REG_INTER);
5055 %}
5056 
5057 // Used in AtomicAdd
5058 operand eBXRegP(eRegP reg) %{
5059   constraint(ALLOC_IN_RC(ebx_reg));
5060   match(reg);
5061   format %{ "EBX" %}
5062   interface(REG_INTER);
5063 %}
5064 
5065 // Tail-call (interprocedural jump) to interpreter
5066 operand eCXRegP(eRegP reg) %{
5067   constraint(ALLOC_IN_RC(ecx_reg));
5068   match(reg);
5069   format %{ "ECX" %}
5070   interface(REG_INTER);
5071 %}
5072 
5073 operand eSIRegP(eRegP reg) %{
5074   constraint(ALLOC_IN_RC(esi_reg));
5075   match(reg);
5076   format %{ "ESI" %}
5077   interface(REG_INTER);
5078 %}
5079 
5080 // Used in rep stosw
5081 operand eDIRegP(eRegP reg) %{
5082   constraint(ALLOC_IN_RC(edi_reg));
5083   match(reg);
5084   format %{ "EDI" %}
5085   interface(REG_INTER);
5086 %}
5087 
5088 operand eBPRegP() %{
5089   constraint(ALLOC_IN_RC(ebp_reg));
5090   match(RegP);
5091   format %{ "EBP" %}
5092   interface(REG_INTER);
5093 %}
5094 
5095 operand eRegL() %{
5096   constraint(ALLOC_IN_RC(long_reg));
5097   match(RegL);
5098   match(eADXRegL);
5099 
5100   format %{ %}
5101   interface(REG_INTER);
5102 %}
5103 
5104 operand eADXRegL( eRegL reg ) %{
5105   constraint(ALLOC_IN_RC(eadx_reg));
5106   match(reg);
5107 
5108   format %{ "EDX:EAX" %}
5109   interface(REG_INTER);
5110 %}
5111 
5112 operand eBCXRegL( eRegL reg ) %{
5113   constraint(ALLOC_IN_RC(ebcx_reg));
5114   match(reg);
5115 
5116   format %{ "EBX:ECX" %}
5117   interface(REG_INTER);
5118 %}
5119 
5120 // Special case for integer high multiply
5121 operand eADXRegL_low_only() %{
5122   constraint(ALLOC_IN_RC(eadx_reg));
5123   match(RegL);
5124 
5125   format %{ "EAX" %}
5126   interface(REG_INTER);
5127 %}
5128 
5129 // Flags register, used as output of compare instructions
5130 operand eFlagsReg() %{
5131   constraint(ALLOC_IN_RC(int_flags));
5132   match(RegFlags);
5133 
5134   format %{ "EFLAGS" %}
5135   interface(REG_INTER);
5136 %}
5137 
5138 // Flags register, used as output of FLOATING POINT compare instructions
5139 operand eFlagsRegU() %{
5140   constraint(ALLOC_IN_RC(int_flags));
5141   match(RegFlags);
5142 
5143   format %{ "EFLAGS_U" %}
5144   interface(REG_INTER);
5145 %}
5146 
5147 operand eFlagsRegUCF() %{
5148   constraint(ALLOC_IN_RC(int_flags));
5149   match(RegFlags);
5150   predicate(false);
5151 
5152   format %{ "EFLAGS_U_CF" %}
5153   interface(REG_INTER);
5154 %}
5155 
5156 // Condition Code Register used by long compare
5157 operand flagsReg_long_LTGE() %{
5158   constraint(ALLOC_IN_RC(int_flags));
5159   match(RegFlags);
5160   format %{ "FLAGS_LTGE" %}
5161   interface(REG_INTER);
5162 %}
5163 operand flagsReg_long_EQNE() %{
5164   constraint(ALLOC_IN_RC(int_flags));
5165   match(RegFlags);
5166   format %{ "FLAGS_EQNE" %}
5167   interface(REG_INTER);
5168 %}
5169 operand flagsReg_long_LEGT() %{
5170   constraint(ALLOC_IN_RC(int_flags));
5171   match(RegFlags);
5172   format %{ "FLAGS_LEGT" %}
5173   interface(REG_INTER);
5174 %}
5175 
5176 // Float register operands
5177 operand regD() %{
5178   predicate( UseSSE < 2 );
5179   constraint(ALLOC_IN_RC(dbl_reg));
5180   match(RegD);
5181   match(regDPR1);
5182   match(regDPR2);
5183   format %{ %}
5184   interface(REG_INTER);
5185 %}
5186 
5187 operand regDPR1(regD reg) %{
5188   predicate( UseSSE < 2 );
5189   constraint(ALLOC_IN_RC(dbl_reg0));
5190   match(reg);
5191   format %{ "FPR1" %}
5192   interface(REG_INTER);
5193 %}
5194 
5195 operand regDPR2(regD reg) %{
5196   predicate( UseSSE < 2 );
5197   constraint(ALLOC_IN_RC(dbl_reg1));
5198   match(reg);
5199   format %{ "FPR2" %}
5200   interface(REG_INTER);
5201 %}
5202 
5203 operand regnotDPR1(regD reg) %{
5204   predicate( UseSSE < 2 );
5205   constraint(ALLOC_IN_RC(dbl_notreg0));
5206   match(reg);
5207   format %{ %}
5208   interface(REG_INTER);
5209 %}
5210 
5211 // XMM Double register operands
5212 operand regXD() %{
5213   predicate( UseSSE>=2 );
5214   constraint(ALLOC_IN_RC(xdb_reg));
5215   match(RegD);
5216   match(regXD6);
5217   match(regXD7);
5218   format %{ %}
5219   interface(REG_INTER);
5220 %}
5221 
5222 // XMM6 double register operands
5223 operand regXD6(regXD reg) %{
5224   predicate( UseSSE>=2 );
5225   constraint(ALLOC_IN_RC(xdb_reg6));
5226   match(reg);
5227   format %{ "XMM6" %}
5228   interface(REG_INTER);
5229 %}
5230 
5231 // XMM7 double register operands
5232 operand regXD7(regXD reg) %{
5233   predicate( UseSSE>=2 );
5234   constraint(ALLOC_IN_RC(xdb_reg7));
5235   match(reg);
5236   format %{ "XMM7" %}
5237   interface(REG_INTER);
5238 %}
5239 
5240 // Float register operands
5241 operand regF() %{
5242   predicate( UseSSE < 2 );
5243   constraint(ALLOC_IN_RC(flt_reg));
5244   match(RegF);
5245   match(regFPR1);
5246   format %{ %}
5247   interface(REG_INTER);
5248 %}
5249 
5250 // Float register operands
5251 operand regFPR1(regF reg) %{
5252   predicate( UseSSE < 2 );
5253   constraint(ALLOC_IN_RC(flt_reg0));
5254   match(reg);
5255   format %{ "FPR1" %}
5256   interface(REG_INTER);
5257 %}
5258 
5259 // XMM register operands
5260 operand regX() %{
5261   predicate( UseSSE>=1 );
5262   constraint(ALLOC_IN_RC(xmm_reg));
5263   match(RegF);
5264   format %{ %}
5265   interface(REG_INTER);
5266 %}
5267 
5268 
5269 //----------Memory Operands----------------------------------------------------
5270 // Direct Memory Operand
5271 operand direct(immP addr) %{
5272   match(addr);
5273 
5274   format %{ "[$addr]" %}
5275   interface(MEMORY_INTER) %{
5276     base(0xFFFFFFFF);
5277     index(0x4);
5278     scale(0x0);
5279     disp($addr);
5280   %}
5281 %}
5282 
5283 // Indirect Memory Operand
5284 operand indirect(eRegP reg) %{
5285   constraint(ALLOC_IN_RC(e_reg));
5286   match(reg);
5287 
5288   format %{ "[$reg]" %}
5289   interface(MEMORY_INTER) %{
5290     base($reg);
5291     index(0x4);
5292     scale(0x0);
5293     disp(0x0);
5294   %}
5295 %}
5296 
5297 // Indirect Memory Plus Short Offset Operand
5298 operand indOffset8(eRegP reg, immI8 off) %{
5299   match(AddP reg off);
5300 
5301   format %{ "[$reg + $off]" %}
5302   interface(MEMORY_INTER) %{
5303     base($reg);
5304     index(0x4);
5305     scale(0x0);
5306     disp($off);
5307   %}
5308 %}
5309 
5310 // Indirect Memory Plus Long Offset Operand
5311 operand indOffset32(eRegP reg, immI off) %{
5312   match(AddP reg off);
5313 
5314   format %{ "[$reg + $off]" %}
5315   interface(MEMORY_INTER) %{
5316     base($reg);
5317     index(0x4);
5318     scale(0x0);
5319     disp($off);
5320   %}
5321 %}
5322 
5323 // Indirect Memory Plus Long Offset Operand
5324 operand indOffset32X(eRegI reg, immP off) %{
5325   match(AddP off reg);
5326 
5327   format %{ "[$reg + $off]" %}
5328   interface(MEMORY_INTER) %{
5329     base($reg);
5330     index(0x4);
5331     scale(0x0);
5332     disp($off);
5333   %}
5334 %}
5335 
5336 // Indirect Memory Plus Index Register Plus Offset Operand
5337 operand indIndexOffset(eRegP reg, eRegI ireg, immI off) %{
5338   match(AddP (AddP reg ireg) off);
5339 
5340   op_cost(10);
5341   format %{"[$reg + $off + $ireg]" %}
5342   interface(MEMORY_INTER) %{
5343     base($reg);
5344     index($ireg);
5345     scale(0x0);
5346     disp($off);
5347   %}
5348 %}
5349 
5350 // Indirect Memory Plus Index Register Plus Offset Operand
5351 operand indIndex(eRegP reg, eRegI ireg) %{
5352   match(AddP reg ireg);
5353 
5354   op_cost(10);
5355   format %{"[$reg + $ireg]" %}
5356   interface(MEMORY_INTER) %{
5357     base($reg);
5358     index($ireg);
5359     scale(0x0);
5360     disp(0x0);
5361   %}
5362 %}
5363 
5364 // // -------------------------------------------------------------------------
5365 // // 486 architecture doesn't support "scale * index + offset" with out a base
5366 // // -------------------------------------------------------------------------
5367 // // Scaled Memory Operands
5368 // // Indirect Memory Times Scale Plus Offset Operand
5369 // operand indScaleOffset(immP off, eRegI ireg, immI2 scale) %{
5370 //   match(AddP off (LShiftI ireg scale));
5371 //
5372 //   op_cost(10);
5373 //   format %{"[$off + $ireg << $scale]" %}
5374 //   interface(MEMORY_INTER) %{
5375 //     base(0x4);
5376 //     index($ireg);
5377 //     scale($scale);
5378 //     disp($off);
5379 //   %}
5380 // %}
5381 
5382 // Indirect Memory Times Scale Plus Index Register
5383 operand indIndexScale(eRegP reg, eRegI ireg, immI2 scale) %{
5384   match(AddP reg (LShiftI ireg scale));
5385 
5386   op_cost(10);
5387   format %{"[$reg + $ireg << $scale]" %}
5388   interface(MEMORY_INTER) %{
5389     base($reg);
5390     index($ireg);
5391     scale($scale);
5392     disp(0x0);
5393   %}
5394 %}
5395 
5396 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5397 operand indIndexScaleOffset(eRegP reg, immI off, eRegI ireg, immI2 scale) %{
5398   match(AddP (AddP reg (LShiftI ireg scale)) off);
5399 
5400   op_cost(10);
5401   format %{"[$reg + $off + $ireg << $scale]" %}
5402   interface(MEMORY_INTER) %{
5403     base($reg);
5404     index($ireg);
5405     scale($scale);
5406     disp($off);
5407   %}
5408 %}
5409 
5410 //----------Load Long Memory Operands------------------------------------------
5411 // The load-long idiom will use it's address expression again after loading
5412 // the first word of the long.  If the load-long destination overlaps with
5413 // registers used in the addressing expression, the 2nd half will be loaded
5414 // from a clobbered address.  Fix this by requiring that load-long use
5415 // address registers that do not overlap with the load-long target.
5416 
5417 // load-long support
5418 operand load_long_RegP() %{
5419   constraint(ALLOC_IN_RC(esi_reg));
5420   match(RegP);
5421   match(eSIRegP);
5422   op_cost(100);
5423   format %{  %}
5424   interface(REG_INTER);
5425 %}
5426 
5427 // Indirect Memory Operand Long
5428 operand load_long_indirect(load_long_RegP reg) %{
5429   constraint(ALLOC_IN_RC(esi_reg));
5430   match(reg);
5431 
5432   format %{ "[$reg]" %}
5433   interface(MEMORY_INTER) %{
5434     base($reg);
5435     index(0x4);
5436     scale(0x0);
5437     disp(0x0);
5438   %}
5439 %}
5440 
5441 // Indirect Memory Plus Long Offset Operand
5442 operand load_long_indOffset32(load_long_RegP reg, immI off) %{
5443   match(AddP reg off);
5444 
5445   format %{ "[$reg + $off]" %}
5446   interface(MEMORY_INTER) %{
5447     base($reg);
5448     index(0x4);
5449     scale(0x0);
5450     disp($off);
5451   %}
5452 %}
5453 
5454 opclass load_long_memory(load_long_indirect, load_long_indOffset32);
5455 
5456 
5457 //----------Special Memory Operands--------------------------------------------
5458 // Stack Slot Operand - This operand is used for loading and storing temporary
5459 //                      values on the stack where a match requires a value to
5460 //                      flow through memory.
5461 operand stackSlotP(sRegP reg) %{
5462   constraint(ALLOC_IN_RC(stack_slots));
5463   // No match rule because this operand is only generated in matching
5464   format %{ "[$reg]" %}
5465   interface(MEMORY_INTER) %{
5466     base(0x4);   // ESP
5467     index(0x4);  // No Index
5468     scale(0x0);  // No Scale
5469     disp($reg);  // Stack Offset
5470   %}
5471 %}
5472 
5473 operand stackSlotI(sRegI reg) %{
5474   constraint(ALLOC_IN_RC(stack_slots));
5475   // No match rule because this operand is only generated in matching
5476   format %{ "[$reg]" %}
5477   interface(MEMORY_INTER) %{
5478     base(0x4);   // ESP
5479     index(0x4);  // No Index
5480     scale(0x0);  // No Scale
5481     disp($reg);  // Stack Offset
5482   %}
5483 %}
5484 
5485 operand stackSlotF(sRegF reg) %{
5486   constraint(ALLOC_IN_RC(stack_slots));
5487   // No match rule because this operand is only generated in matching
5488   format %{ "[$reg]" %}
5489   interface(MEMORY_INTER) %{
5490     base(0x4);   // ESP
5491     index(0x4);  // No Index
5492     scale(0x0);  // No Scale
5493     disp($reg);  // Stack Offset
5494   %}
5495 %}
5496 
5497 operand stackSlotD(sRegD reg) %{
5498   constraint(ALLOC_IN_RC(stack_slots));
5499   // No match rule because this operand is only generated in matching
5500   format %{ "[$reg]" %}
5501   interface(MEMORY_INTER) %{
5502     base(0x4);   // ESP
5503     index(0x4);  // No Index
5504     scale(0x0);  // No Scale
5505     disp($reg);  // Stack Offset
5506   %}
5507 %}
5508 
5509 operand stackSlotL(sRegL reg) %{
5510   constraint(ALLOC_IN_RC(stack_slots));
5511   // No match rule because this operand is only generated in matching
5512   format %{ "[$reg]" %}
5513   interface(MEMORY_INTER) %{
5514     base(0x4);   // ESP
5515     index(0x4);  // No Index
5516     scale(0x0);  // No Scale
5517     disp($reg);  // Stack Offset
5518   %}
5519 %}
5520 
5521 //----------Memory Operands - Win95 Implicit Null Variants----------------
5522 // Indirect Memory Operand
5523 operand indirect_win95_safe(eRegP_no_EBP reg)
5524 %{
5525   constraint(ALLOC_IN_RC(e_reg));
5526   match(reg);
5527 
5528   op_cost(100);
5529   format %{ "[$reg]" %}
5530   interface(MEMORY_INTER) %{
5531     base($reg);
5532     index(0x4);
5533     scale(0x0);
5534     disp(0x0);
5535   %}
5536 %}
5537 
5538 // Indirect Memory Plus Short Offset Operand
5539 operand indOffset8_win95_safe(eRegP_no_EBP reg, immI8 off)
5540 %{
5541   match(AddP reg off);
5542 
5543   op_cost(100);
5544   format %{ "[$reg + $off]" %}
5545   interface(MEMORY_INTER) %{
5546     base($reg);
5547     index(0x4);
5548     scale(0x0);
5549     disp($off);
5550   %}
5551 %}
5552 
5553 // Indirect Memory Plus Long Offset Operand
5554 operand indOffset32_win95_safe(eRegP_no_EBP reg, immI off)
5555 %{
5556   match(AddP reg off);
5557 
5558   op_cost(100);
5559   format %{ "[$reg + $off]" %}
5560   interface(MEMORY_INTER) %{
5561     base($reg);
5562     index(0x4);
5563     scale(0x0);
5564     disp($off);
5565   %}
5566 %}
5567 
5568 // Indirect Memory Plus Index Register Plus Offset Operand
5569 operand indIndexOffset_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI off)
5570 %{
5571   match(AddP (AddP reg ireg) off);
5572 
5573   op_cost(100);
5574   format %{"[$reg + $off + $ireg]" %}
5575   interface(MEMORY_INTER) %{
5576     base($reg);
5577     index($ireg);
5578     scale(0x0);
5579     disp($off);
5580   %}
5581 %}
5582 
5583 // Indirect Memory Times Scale Plus Index Register
5584 operand indIndexScale_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI2 scale)
5585 %{
5586   match(AddP reg (LShiftI ireg scale));
5587 
5588   op_cost(100);
5589   format %{"[$reg + $ireg << $scale]" %}
5590   interface(MEMORY_INTER) %{
5591     base($reg);
5592     index($ireg);
5593     scale($scale);
5594     disp(0x0);
5595   %}
5596 %}
5597 
5598 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5599 operand indIndexScaleOffset_win95_safe(eRegP_no_EBP reg, immI off, eRegI ireg, immI2 scale)
5600 %{
5601   match(AddP (AddP reg (LShiftI ireg scale)) off);
5602 
5603   op_cost(100);
5604   format %{"[$reg + $off + $ireg << $scale]" %}
5605   interface(MEMORY_INTER) %{
5606     base($reg);
5607     index($ireg);
5608     scale($scale);
5609     disp($off);
5610   %}
5611 %}
5612 
5613 //----------Conditional Branch Operands----------------------------------------
5614 // Comparison Op  - This is the operation of the comparison, and is limited to
5615 //                  the following set of codes:
5616 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5617 //
5618 // Other attributes of the comparison, such as unsignedness, are specified
5619 // by the comparison instruction that sets a condition code flags register.
5620 // That result is represented by a flags operand whose subtype is appropriate
5621 // to the unsignedness (etc.) of the comparison.
5622 //
5623 // Later, the instruction which matches both the Comparison Op (a Bool) and
5624 // the flags (produced by the Cmp) specifies the coding of the comparison op
5625 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5626 
5627 // Comparision Code
5628 operand cmpOp() %{
5629   match(Bool);
5630 
5631   format %{ "" %}
5632   interface(COND_INTER) %{
5633     equal(0x4, "e");
5634     not_equal(0x5, "ne");
5635     less(0xC, "l");
5636     greater_equal(0xD, "ge");
5637     less_equal(0xE, "le");
5638     greater(0xF, "g");
5639   %}
5640 %}
5641 
5642 // Comparison Code, unsigned compare.  Used by FP also, with
5643 // C2 (unordered) turned into GT or LT already.  The other bits
5644 // C0 and C3 are turned into Carry & Zero flags.
5645 operand cmpOpU() %{
5646   match(Bool);
5647 
5648   format %{ "" %}
5649   interface(COND_INTER) %{
5650     equal(0x4, "e");
5651     not_equal(0x5, "ne");
5652     less(0x2, "b");
5653     greater_equal(0x3, "nb");
5654     less_equal(0x6, "be");
5655     greater(0x7, "nbe");
5656   %}
5657 %}
5658 
5659 // Floating comparisons that don't require any fixup for the unordered case
5660 operand cmpOpUCF() %{
5661   match(Bool);
5662   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
5663             n->as_Bool()->_test._test == BoolTest::ge ||
5664             n->as_Bool()->_test._test == BoolTest::le ||
5665             n->as_Bool()->_test._test == BoolTest::gt);
5666   format %{ "" %}
5667   interface(COND_INTER) %{
5668     equal(0x4, "e");
5669     not_equal(0x5, "ne");
5670     less(0x2, "b");
5671     greater_equal(0x3, "nb");
5672     less_equal(0x6, "be");
5673     greater(0x7, "nbe");
5674   %}
5675 %}
5676 
5677 
5678 // Floating comparisons that can be fixed up with extra conditional jumps
5679 operand cmpOpUCF2() %{
5680   match(Bool);
5681   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
5682             n->as_Bool()->_test._test == BoolTest::eq);
5683   format %{ "" %}
5684   interface(COND_INTER) %{
5685     equal(0x4, "e");
5686     not_equal(0x5, "ne");
5687     less(0x2, "b");
5688     greater_equal(0x3, "nb");
5689     less_equal(0x6, "be");
5690     greater(0x7, "nbe");
5691   %}
5692 %}
5693 
5694 // Comparison Code for FP conditional move
5695 operand cmpOp_fcmov() %{
5696   match(Bool);
5697 
5698   format %{ "" %}
5699   interface(COND_INTER) %{
5700     equal        (0x0C8);
5701     not_equal    (0x1C8);
5702     less         (0x0C0);
5703     greater_equal(0x1C0);
5704     less_equal   (0x0D0);
5705     greater      (0x1D0);
5706   %}
5707 %}
5708 
5709 // Comparision Code used in long compares
5710 operand cmpOp_commute() %{
5711   match(Bool);
5712 
5713   format %{ "" %}
5714   interface(COND_INTER) %{
5715     equal(0x4, "e");
5716     not_equal(0x5, "ne");
5717     less(0xF, "g");
5718     greater_equal(0xE, "le");
5719     less_equal(0xD, "ge");
5720     greater(0xC, "l");
5721   %}
5722 %}
5723 
5724 //----------OPERAND CLASSES----------------------------------------------------
5725 // Operand Classes are groups of operands that are used as to simplify
5726 // instruction definitions by not requiring the AD writer to specify separate
5727 // instructions for every form of operand when the instruction accepts
5728 // multiple operand types with the same basic encoding and format.  The classic
5729 // case of this is memory operands.
5730 
5731 opclass memory(direct, indirect, indOffset8, indOffset32, indOffset32X, indIndexOffset,
5732                indIndex, indIndexScale, indIndexScaleOffset);
5733 
5734 // Long memory operations are encoded in 2 instructions and a +4 offset.
5735 // This means some kind of offset is always required and you cannot use
5736 // an oop as the offset (done when working on static globals).
5737 opclass long_memory(direct, indirect, indOffset8, indOffset32, indIndexOffset,
5738                     indIndex, indIndexScale, indIndexScaleOffset);
5739 
5740 
5741 //----------PIPELINE-----------------------------------------------------------
5742 // Rules which define the behavior of the target architectures pipeline.
5743 pipeline %{
5744 
5745 //----------ATTRIBUTES---------------------------------------------------------
5746 attributes %{
5747   variable_size_instructions;        // Fixed size instructions
5748   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
5749   instruction_unit_size = 1;         // An instruction is 1 bytes long
5750   instruction_fetch_unit_size = 16;  // The processor fetches one line
5751   instruction_fetch_units = 1;       // of 16 bytes
5752 
5753   // List of nop instructions
5754   nops( MachNop );
5755 %}
5756 
5757 //----------RESOURCES----------------------------------------------------------
5758 // Resources are the functional units available to the machine
5759 
5760 // Generic P2/P3 pipeline
5761 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
5762 // 3 instructions decoded per cycle.
5763 // 2 load/store ops per cycle, 1 branch, 1 FPU,
5764 // 2 ALU op, only ALU0 handles mul/div instructions.
5765 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
5766            MS0, MS1, MEM = MS0 | MS1,
5767            BR, FPU,
5768            ALU0, ALU1, ALU = ALU0 | ALU1 );
5769 
5770 //----------PIPELINE DESCRIPTION-----------------------------------------------
5771 // Pipeline Description specifies the stages in the machine's pipeline
5772 
5773 // Generic P2/P3 pipeline
5774 pipe_desc(S0, S1, S2, S3, S4, S5);
5775 
5776 //----------PIPELINE CLASSES---------------------------------------------------
5777 // Pipeline Classes describe the stages in which input and output are
5778 // referenced by the hardware pipeline.
5779 
5780 // Naming convention: ialu or fpu
5781 // Then: _reg
5782 // Then: _reg if there is a 2nd register
5783 // Then: _long if it's a pair of instructions implementing a long
5784 // Then: _fat if it requires the big decoder
5785 //   Or: _mem if it requires the big decoder and a memory unit.
5786 
5787 // Integer ALU reg operation
5788 pipe_class ialu_reg(eRegI dst) %{
5789     single_instruction;
5790     dst    : S4(write);
5791     dst    : S3(read);
5792     DECODE : S0;        // any decoder
5793     ALU    : S3;        // any alu
5794 %}
5795 
5796 // Long ALU reg operation
5797 pipe_class ialu_reg_long(eRegL dst) %{
5798     instruction_count(2);
5799     dst    : S4(write);
5800     dst    : S3(read);
5801     DECODE : S0(2);     // any 2 decoders
5802     ALU    : S3(2);     // both alus
5803 %}
5804 
5805 // Integer ALU reg operation using big decoder
5806 pipe_class ialu_reg_fat(eRegI dst) %{
5807     single_instruction;
5808     dst    : S4(write);
5809     dst    : S3(read);
5810     D0     : S0;        // big decoder only
5811     ALU    : S3;        // any alu
5812 %}
5813 
5814 // Long ALU reg operation using big decoder
5815 pipe_class ialu_reg_long_fat(eRegL dst) %{
5816     instruction_count(2);
5817     dst    : S4(write);
5818     dst    : S3(read);
5819     D0     : S0(2);     // big decoder only; twice
5820     ALU    : S3(2);     // any 2 alus
5821 %}
5822 
5823 // Integer ALU reg-reg operation
5824 pipe_class ialu_reg_reg(eRegI dst, eRegI src) %{
5825     single_instruction;
5826     dst    : S4(write);
5827     src    : S3(read);
5828     DECODE : S0;        // any decoder
5829     ALU    : S3;        // any alu
5830 %}
5831 
5832 // Long ALU reg-reg operation
5833 pipe_class ialu_reg_reg_long(eRegL dst, eRegL src) %{
5834     instruction_count(2);
5835     dst    : S4(write);
5836     src    : S3(read);
5837     DECODE : S0(2);     // any 2 decoders
5838     ALU    : S3(2);     // both alus
5839 %}
5840 
5841 // Integer ALU reg-reg operation
5842 pipe_class ialu_reg_reg_fat(eRegI dst, memory src) %{
5843     single_instruction;
5844     dst    : S4(write);
5845     src    : S3(read);
5846     D0     : S0;        // big decoder only
5847     ALU    : S3;        // any alu
5848 %}
5849 
5850 // Long ALU reg-reg operation
5851 pipe_class ialu_reg_reg_long_fat(eRegL dst, eRegL src) %{
5852     instruction_count(2);
5853     dst    : S4(write);
5854     src    : S3(read);
5855     D0     : S0(2);     // big decoder only; twice
5856     ALU    : S3(2);     // both alus
5857 %}
5858 
5859 // Integer ALU reg-mem operation
5860 pipe_class ialu_reg_mem(eRegI dst, memory mem) %{
5861     single_instruction;
5862     dst    : S5(write);
5863     mem    : S3(read);
5864     D0     : S0;        // big decoder only
5865     ALU    : S4;        // any alu
5866     MEM    : S3;        // any mem
5867 %}
5868 
5869 // Long ALU reg-mem operation
5870 pipe_class ialu_reg_long_mem(eRegL dst, load_long_memory mem) %{
5871     instruction_count(2);
5872     dst    : S5(write);
5873     mem    : S3(read);
5874     D0     : S0(2);     // big decoder only; twice
5875     ALU    : S4(2);     // any 2 alus
5876     MEM    : S3(2);     // both mems
5877 %}
5878 
5879 // Integer mem operation (prefetch)
5880 pipe_class ialu_mem(memory mem)
5881 %{
5882     single_instruction;
5883     mem    : S3(read);
5884     D0     : S0;        // big decoder only
5885     MEM    : S3;        // any mem
5886 %}
5887 
5888 // Integer Store to Memory
5889 pipe_class ialu_mem_reg(memory mem, eRegI src) %{
5890     single_instruction;
5891     mem    : S3(read);
5892     src    : S5(read);
5893     D0     : S0;        // big decoder only
5894     ALU    : S4;        // any alu
5895     MEM    : S3;
5896 %}
5897 
5898 // Long Store to Memory
5899 pipe_class ialu_mem_long_reg(memory mem, eRegL src) %{
5900     instruction_count(2);
5901     mem    : S3(read);
5902     src    : S5(read);
5903     D0     : S0(2);     // big decoder only; twice
5904     ALU    : S4(2);     // any 2 alus
5905     MEM    : S3(2);     // Both mems
5906 %}
5907 
5908 // Integer Store to Memory
5909 pipe_class ialu_mem_imm(memory mem) %{
5910     single_instruction;
5911     mem    : S3(read);
5912     D0     : S0;        // big decoder only
5913     ALU    : S4;        // any alu
5914     MEM    : S3;
5915 %}
5916 
5917 // Integer ALU0 reg-reg operation
5918 pipe_class ialu_reg_reg_alu0(eRegI dst, eRegI src) %{
5919     single_instruction;
5920     dst    : S4(write);
5921     src    : S3(read);
5922     D0     : S0;        // Big decoder only
5923     ALU0   : S3;        // only alu0
5924 %}
5925 
5926 // Integer ALU0 reg-mem operation
5927 pipe_class ialu_reg_mem_alu0(eRegI dst, memory mem) %{
5928     single_instruction;
5929     dst    : S5(write);
5930     mem    : S3(read);
5931     D0     : S0;        // big decoder only
5932     ALU0   : S4;        // ALU0 only
5933     MEM    : S3;        // any mem
5934 %}
5935 
5936 // Integer ALU reg-reg operation
5937 pipe_class ialu_cr_reg_reg(eFlagsReg cr, eRegI src1, eRegI src2) %{
5938     single_instruction;
5939     cr     : S4(write);
5940     src1   : S3(read);
5941     src2   : S3(read);
5942     DECODE : S0;        // any decoder
5943     ALU    : S3;        // any alu
5944 %}
5945 
5946 // Integer ALU reg-imm operation
5947 pipe_class ialu_cr_reg_imm(eFlagsReg cr, eRegI src1) %{
5948     single_instruction;
5949     cr     : S4(write);
5950     src1   : S3(read);
5951     DECODE : S0;        // any decoder
5952     ALU    : S3;        // any alu
5953 %}
5954 
5955 // Integer ALU reg-mem operation
5956 pipe_class ialu_cr_reg_mem(eFlagsReg cr, eRegI src1, memory src2) %{
5957     single_instruction;
5958     cr     : S4(write);
5959     src1   : S3(read);
5960     src2   : S3(read);
5961     D0     : S0;        // big decoder only
5962     ALU    : S4;        // any alu
5963     MEM    : S3;
5964 %}
5965 
5966 // Conditional move reg-reg
5967 pipe_class pipe_cmplt( eRegI p, eRegI q, eRegI y ) %{
5968     instruction_count(4);
5969     y      : S4(read);
5970     q      : S3(read);
5971     p      : S3(read);
5972     DECODE : S0(4);     // any decoder
5973 %}
5974 
5975 // Conditional move reg-reg
5976 pipe_class pipe_cmov_reg( eRegI dst, eRegI src, eFlagsReg cr ) %{
5977     single_instruction;
5978     dst    : S4(write);
5979     src    : S3(read);
5980     cr     : S3(read);
5981     DECODE : S0;        // any decoder
5982 %}
5983 
5984 // Conditional move reg-mem
5985 pipe_class pipe_cmov_mem( eFlagsReg cr, eRegI dst, memory src) %{
5986     single_instruction;
5987     dst    : S4(write);
5988     src    : S3(read);
5989     cr     : S3(read);
5990     DECODE : S0;        // any decoder
5991     MEM    : S3;
5992 %}
5993 
5994 // Conditional move reg-reg long
5995 pipe_class pipe_cmov_reg_long( eFlagsReg cr, eRegL dst, eRegL src) %{
5996     single_instruction;
5997     dst    : S4(write);
5998     src    : S3(read);
5999     cr     : S3(read);
6000     DECODE : S0(2);     // any 2 decoders
6001 %}
6002 
6003 // Conditional move double reg-reg
6004 pipe_class pipe_cmovD_reg( eFlagsReg cr, regDPR1 dst, regD src) %{
6005     single_instruction;
6006     dst    : S4(write);
6007     src    : S3(read);
6008     cr     : S3(read);
6009     DECODE : S0;        // any decoder
6010 %}
6011 
6012 // Float reg-reg operation
6013 pipe_class fpu_reg(regD dst) %{
6014     instruction_count(2);
6015     dst    : S3(read);
6016     DECODE : S0(2);     // any 2 decoders
6017     FPU    : S3;
6018 %}
6019 
6020 // Float reg-reg operation
6021 pipe_class fpu_reg_reg(regD dst, regD src) %{
6022     instruction_count(2);
6023     dst    : S4(write);
6024     src    : S3(read);
6025     DECODE : S0(2);     // any 2 decoders
6026     FPU    : S3;
6027 %}
6028 
6029 // Float reg-reg operation
6030 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2) %{
6031     instruction_count(3);
6032     dst    : S4(write);
6033     src1   : S3(read);
6034     src2   : S3(read);
6035     DECODE : S0(3);     // any 3 decoders
6036     FPU    : S3(2);
6037 %}
6038 
6039 // Float reg-reg operation
6040 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3) %{
6041     instruction_count(4);
6042     dst    : S4(write);
6043     src1   : S3(read);
6044     src2   : S3(read);
6045     src3   : S3(read);
6046     DECODE : S0(4);     // any 3 decoders
6047     FPU    : S3(2);
6048 %}
6049 
6050 // Float reg-reg operation
6051 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3) %{
6052     instruction_count(4);
6053     dst    : S4(write);
6054     src1   : S3(read);
6055     src2   : S3(read);
6056     src3   : S3(read);
6057     DECODE : S1(3);     // any 3 decoders
6058     D0     : S0;        // Big decoder only
6059     FPU    : S3(2);
6060     MEM    : S3;
6061 %}
6062 
6063 // Float reg-mem operation
6064 pipe_class fpu_reg_mem(regD dst, memory mem) %{
6065     instruction_count(2);
6066     dst    : S5(write);
6067     mem    : S3(read);
6068     D0     : S0;        // big decoder only
6069     DECODE : S1;        // any decoder for FPU POP
6070     FPU    : S4;
6071     MEM    : S3;        // any mem
6072 %}
6073 
6074 // Float reg-mem operation
6075 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem) %{
6076     instruction_count(3);
6077     dst    : S5(write);
6078     src1   : S3(read);
6079     mem    : S3(read);
6080     D0     : S0;        // big decoder only
6081     DECODE : S1(2);     // any decoder for FPU POP
6082     FPU    : S4;
6083     MEM    : S3;        // any mem
6084 %}
6085 
6086 // Float mem-reg operation
6087 pipe_class fpu_mem_reg(memory mem, regD src) %{
6088     instruction_count(2);
6089     src    : S5(read);
6090     mem    : S3(read);
6091     DECODE : S0;        // any decoder for FPU PUSH
6092     D0     : S1;        // big decoder only
6093     FPU    : S4;
6094     MEM    : S3;        // any mem
6095 %}
6096 
6097 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2) %{
6098     instruction_count(3);
6099     src1   : S3(read);
6100     src2   : S3(read);
6101     mem    : S3(read);
6102     DECODE : S0(2);     // any decoder for FPU PUSH
6103     D0     : S1;        // big decoder only
6104     FPU    : S4;
6105     MEM    : S3;        // any mem
6106 %}
6107 
6108 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2) %{
6109     instruction_count(3);
6110     src1   : S3(read);
6111     src2   : S3(read);
6112     mem    : S4(read);
6113     DECODE : S0;        // any decoder for FPU PUSH
6114     D0     : S0(2);     // big decoder only
6115     FPU    : S4;
6116     MEM    : S3(2);     // any mem
6117 %}
6118 
6119 pipe_class fpu_mem_mem(memory dst, memory src1) %{
6120     instruction_count(2);
6121     src1   : S3(read);
6122     dst    : S4(read);
6123     D0     : S0(2);     // big decoder only
6124     MEM    : S3(2);     // any mem
6125 %}
6126 
6127 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2) %{
6128     instruction_count(3);
6129     src1   : S3(read);
6130     src2   : S3(read);
6131     dst    : S4(read);
6132     D0     : S0(3);     // big decoder only
6133     FPU    : S4;
6134     MEM    : S3(3);     // any mem
6135 %}
6136 
6137 pipe_class fpu_mem_reg_con(memory mem, regD src1) %{
6138     instruction_count(3);
6139     src1   : S4(read);
6140     mem    : S4(read);
6141     DECODE : S0;        // any decoder for FPU PUSH
6142     D0     : S0(2);     // big decoder only
6143     FPU    : S4;
6144     MEM    : S3(2);     // any mem
6145 %}
6146 
6147 // Float load constant
6148 pipe_class fpu_reg_con(regD dst) %{
6149     instruction_count(2);
6150     dst    : S5(write);
6151     D0     : S0;        // big decoder only for the load
6152     DECODE : S1;        // any decoder for FPU POP
6153     FPU    : S4;
6154     MEM    : S3;        // any mem
6155 %}
6156 
6157 // Float load constant
6158 pipe_class fpu_reg_reg_con(regD dst, regD src) %{
6159     instruction_count(3);
6160     dst    : S5(write);
6161     src    : S3(read);
6162     D0     : S0;        // big decoder only for the load
6163     DECODE : S1(2);     // any decoder for FPU POP
6164     FPU    : S4;
6165     MEM    : S3;        // any mem
6166 %}
6167 
6168 // UnConditional branch
6169 pipe_class pipe_jmp( label labl ) %{
6170     single_instruction;
6171     BR   : S3;
6172 %}
6173 
6174 // Conditional branch
6175 pipe_class pipe_jcc( cmpOp cmp, eFlagsReg cr, label labl ) %{
6176     single_instruction;
6177     cr    : S1(read);
6178     BR    : S3;
6179 %}
6180 
6181 // Allocation idiom
6182 pipe_class pipe_cmpxchg( eRegP dst, eRegP heap_ptr ) %{
6183     instruction_count(1); force_serialization;
6184     fixed_latency(6);
6185     heap_ptr : S3(read);
6186     DECODE   : S0(3);
6187     D0       : S2;
6188     MEM      : S3;
6189     ALU      : S3(2);
6190     dst      : S5(write);
6191     BR       : S5;
6192 %}
6193 
6194 // Generic big/slow expanded idiom
6195 pipe_class pipe_slow(  ) %{
6196     instruction_count(10); multiple_bundles; force_serialization;
6197     fixed_latency(100);
6198     D0  : S0(2);
6199     MEM : S3(2);
6200 %}
6201 
6202 // The real do-nothing guy
6203 pipe_class empty( ) %{
6204     instruction_count(0);
6205 %}
6206 
6207 // Define the class for the Nop node
6208 define %{
6209    MachNop = empty;
6210 %}
6211 
6212 %}
6213 
6214 //----------INSTRUCTIONS-------------------------------------------------------
6215 //
6216 // match      -- States which machine-independent subtree may be replaced
6217 //               by this instruction.
6218 // ins_cost   -- The estimated cost of this instruction is used by instruction
6219 //               selection to identify a minimum cost tree of machine
6220 //               instructions that matches a tree of machine-independent
6221 //               instructions.
6222 // format     -- A string providing the disassembly for this instruction.
6223 //               The value of an instruction's operand may be inserted
6224 //               by referring to it with a '$' prefix.
6225 // opcode     -- Three instruction opcodes may be provided.  These are referred
6226 //               to within an encode class as $primary, $secondary, and $tertiary
6227 //               respectively.  The primary opcode is commonly used to
6228 //               indicate the type of machine instruction, while secondary
6229 //               and tertiary are often used for prefix options or addressing
6230 //               modes.
6231 // ins_encode -- A list of encode classes with parameters. The encode class
6232 //               name must have been defined in an 'enc_class' specification
6233 //               in the encode section of the architecture description.
6234 
6235 //----------BSWAP-Instruction--------------------------------------------------
6236 instruct bytes_reverse_int(eRegI dst) %{
6237   match(Set dst (ReverseBytesI dst));
6238 
6239   format %{ "BSWAP  $dst" %}
6240   opcode(0x0F, 0xC8);
6241   ins_encode( OpcP, OpcSReg(dst) );
6242   ins_pipe( ialu_reg );
6243 %}
6244 
6245 instruct bytes_reverse_long(eRegL dst) %{
6246   match(Set dst (ReverseBytesL dst));
6247 
6248   format %{ "BSWAP  $dst.lo\n\t"
6249             "BSWAP  $dst.hi\n\t"
6250             "XCHG   $dst.lo $dst.hi" %}
6251 
6252   ins_cost(125);
6253   ins_encode( bswap_long_bytes(dst) );
6254   ins_pipe( ialu_reg_reg);
6255 %}
6256 
6257 instruct bytes_reverse_unsigned_short(eRegI dst) %{
6258   match(Set dst (ReverseBytesUS dst));
6259 
6260   format %{ "BSWAP  $dst\n\t" 
6261             "SHR    $dst,16\n\t" %}
6262   ins_encode %{
6263     __ bswapl($dst$$Register);
6264     __ shrl($dst$$Register, 16); 
6265   %}
6266   ins_pipe( ialu_reg );
6267 %}
6268 
6269 instruct bytes_reverse_short(eRegI dst) %{
6270   match(Set dst (ReverseBytesS dst));
6271 
6272   format %{ "BSWAP  $dst\n\t" 
6273             "SAR    $dst,16\n\t" %}
6274   ins_encode %{
6275     __ bswapl($dst$$Register);
6276     __ sarl($dst$$Register, 16); 
6277   %}
6278   ins_pipe( ialu_reg );
6279 %}
6280 
6281 
6282 //---------- Zeros Count Instructions ------------------------------------------
6283 
6284 instruct countLeadingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
6285   predicate(UseCountLeadingZerosInstruction);
6286   match(Set dst (CountLeadingZerosI src));
6287   effect(KILL cr);
6288 
6289   format %{ "LZCNT  $dst, $src\t# count leading zeros (int)" %}
6290   ins_encode %{
6291     __ lzcntl($dst$$Register, $src$$Register);
6292   %}
6293   ins_pipe(ialu_reg);
6294 %}
6295 
6296 instruct countLeadingZerosI_bsr(eRegI dst, eRegI src, eFlagsReg cr) %{
6297   predicate(!UseCountLeadingZerosInstruction);
6298   match(Set dst (CountLeadingZerosI src));
6299   effect(KILL cr);
6300 
6301   format %{ "BSR    $dst, $src\t# count leading zeros (int)\n\t"
6302             "JNZ    skip\n\t"
6303             "MOV    $dst, -1\n"
6304       "skip:\n\t"
6305             "NEG    $dst\n\t"
6306             "ADD    $dst, 31" %}
6307   ins_encode %{
6308     Register Rdst = $dst$$Register;
6309     Register Rsrc = $src$$Register;
6310     Label skip;
6311     __ bsrl(Rdst, Rsrc);
6312     __ jccb(Assembler::notZero, skip);
6313     __ movl(Rdst, -1);
6314     __ bind(skip);
6315     __ negl(Rdst);
6316     __ addl(Rdst, BitsPerInt - 1);
6317   %}
6318   ins_pipe(ialu_reg);
6319 %}
6320 
6321 instruct countLeadingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
6322   predicate(UseCountLeadingZerosInstruction);
6323   match(Set dst (CountLeadingZerosL src));
6324   effect(TEMP dst, KILL cr);
6325 
6326   format %{ "LZCNT  $dst, $src.hi\t# count leading zeros (long)\n\t"
6327             "JNC    done\n\t"
6328             "LZCNT  $dst, $src.lo\n\t"
6329             "ADD    $dst, 32\n"
6330       "done:" %}
6331   ins_encode %{
6332     Register Rdst = $dst$$Register;
6333     Register Rsrc = $src$$Register;
6334     Label done;
6335     __ lzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
6336     __ jccb(Assembler::carryClear, done);
6337     __ lzcntl(Rdst, Rsrc);
6338     __ addl(Rdst, BitsPerInt);
6339     __ bind(done);
6340   %}
6341   ins_pipe(ialu_reg);
6342 %}
6343 
6344 instruct countLeadingZerosL_bsr(eRegI dst, eRegL src, eFlagsReg cr) %{
6345   predicate(!UseCountLeadingZerosInstruction);
6346   match(Set dst (CountLeadingZerosL src));
6347   effect(TEMP dst, KILL cr);
6348 
6349   format %{ "BSR    $dst, $src.hi\t# count leading zeros (long)\n\t"
6350             "JZ     msw_is_zero\n\t"
6351             "ADD    $dst, 32\n\t"
6352             "JMP    not_zero\n"
6353       "msw_is_zero:\n\t"
6354             "BSR    $dst, $src.lo\n\t"
6355             "JNZ    not_zero\n\t"
6356             "MOV    $dst, -1\n"
6357       "not_zero:\n\t"
6358             "NEG    $dst\n\t"
6359             "ADD    $dst, 63\n" %}
6360  ins_encode %{
6361     Register Rdst = $dst$$Register;
6362     Register Rsrc = $src$$Register;
6363     Label msw_is_zero;
6364     Label not_zero;
6365     __ bsrl(Rdst, HIGH_FROM_LOW(Rsrc));
6366     __ jccb(Assembler::zero, msw_is_zero);
6367     __ addl(Rdst, BitsPerInt);
6368     __ jmpb(not_zero);
6369     __ bind(msw_is_zero);
6370     __ bsrl(Rdst, Rsrc);
6371     __ jccb(Assembler::notZero, not_zero);
6372     __ movl(Rdst, -1);
6373     __ bind(not_zero);
6374     __ negl(Rdst);
6375     __ addl(Rdst, BitsPerLong - 1);
6376   %}
6377   ins_pipe(ialu_reg);
6378 %}
6379 
6380 instruct countTrailingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
6381   match(Set dst (CountTrailingZerosI src));
6382   effect(KILL cr);
6383 
6384   format %{ "BSF    $dst, $src\t# count trailing zeros (int)\n\t"
6385             "JNZ    done\n\t"
6386             "MOV    $dst, 32\n"
6387       "done:" %}
6388   ins_encode %{
6389     Register Rdst = $dst$$Register;
6390     Label done;
6391     __ bsfl(Rdst, $src$$Register);
6392     __ jccb(Assembler::notZero, done);
6393     __ movl(Rdst, BitsPerInt);
6394     __ bind(done);
6395   %}
6396   ins_pipe(ialu_reg);
6397 %}
6398 
6399 instruct countTrailingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
6400   match(Set dst (CountTrailingZerosL src));
6401   effect(TEMP dst, KILL cr);
6402 
6403   format %{ "BSF    $dst, $src.lo\t# count trailing zeros (long)\n\t"
6404             "JNZ    done\n\t"
6405             "BSF    $dst, $src.hi\n\t"
6406             "JNZ    msw_not_zero\n\t"
6407             "MOV    $dst, 32\n"
6408       "msw_not_zero:\n\t"
6409             "ADD    $dst, 32\n"
6410       "done:" %}
6411   ins_encode %{
6412     Register Rdst = $dst$$Register;
6413     Register Rsrc = $src$$Register;
6414     Label msw_not_zero;
6415     Label done;
6416     __ bsfl(Rdst, Rsrc);
6417     __ jccb(Assembler::notZero, done);
6418     __ bsfl(Rdst, HIGH_FROM_LOW(Rsrc));
6419     __ jccb(Assembler::notZero, msw_not_zero);
6420     __ movl(Rdst, BitsPerInt);
6421     __ bind(msw_not_zero);
6422     __ addl(Rdst, BitsPerInt);
6423     __ bind(done);
6424   %}
6425   ins_pipe(ialu_reg);
6426 %}
6427 
6428 
6429 //---------- Population Count Instructions -------------------------------------
6430 
6431 instruct popCountI(eRegI dst, eRegI src) %{
6432   predicate(UsePopCountInstruction);
6433   match(Set dst (PopCountI src));
6434 
6435   format %{ "POPCNT $dst, $src" %}
6436   ins_encode %{
6437     __ popcntl($dst$$Register, $src$$Register);
6438   %}
6439   ins_pipe(ialu_reg);
6440 %}
6441 
6442 instruct popCountI_mem(eRegI dst, memory mem) %{
6443   predicate(UsePopCountInstruction);
6444   match(Set dst (PopCountI (LoadI mem)));
6445 
6446   format %{ "POPCNT $dst, $mem" %}
6447   ins_encode %{
6448     __ popcntl($dst$$Register, $mem$$Address);
6449   %}
6450   ins_pipe(ialu_reg);
6451 %}
6452 
6453 // Note: Long.bitCount(long) returns an int.
6454 instruct popCountL(eRegI dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
6455   predicate(UsePopCountInstruction);
6456   match(Set dst (PopCountL src));
6457   effect(KILL cr, TEMP tmp, TEMP dst);
6458 
6459   format %{ "POPCNT $dst, $src.lo\n\t"
6460             "POPCNT $tmp, $src.hi\n\t"
6461             "ADD    $dst, $tmp" %}
6462   ins_encode %{
6463     __ popcntl($dst$$Register, $src$$Register);
6464     __ popcntl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
6465     __ addl($dst$$Register, $tmp$$Register);
6466   %}
6467   ins_pipe(ialu_reg);
6468 %}
6469 
6470 // Note: Long.bitCount(long) returns an int.
6471 instruct popCountL_mem(eRegI dst, memory mem, eRegI tmp, eFlagsReg cr) %{
6472   predicate(UsePopCountInstruction);
6473   match(Set dst (PopCountL (LoadL mem)));
6474   effect(KILL cr, TEMP tmp, TEMP dst);
6475 
6476   format %{ "POPCNT $dst, $mem\n\t"
6477             "POPCNT $tmp, $mem+4\n\t"
6478             "ADD    $dst, $tmp" %}
6479   ins_encode %{
6480     //__ popcntl($dst$$Register, $mem$$Address$$first);
6481     //__ popcntl($tmp$$Register, $mem$$Address$$second);
6482     __ popcntl($dst$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false));
6483     __ popcntl($tmp$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false));
6484     __ addl($dst$$Register, $tmp$$Register);
6485   %}
6486   ins_pipe(ialu_reg);
6487 %}
6488 
6489 
6490 //----------Load/Store/Move Instructions---------------------------------------
6491 //----------Load Instructions--------------------------------------------------
6492 // Load Byte (8bit signed)
6493 instruct loadB(xRegI dst, memory mem) %{
6494   match(Set dst (LoadB mem));
6495 
6496   ins_cost(125);
6497   format %{ "MOVSX8 $dst,$mem\t# byte" %}
6498 
6499   ins_encode %{
6500     __ movsbl($dst$$Register, $mem$$Address);
6501   %}
6502 
6503   ins_pipe(ialu_reg_mem);
6504 %}
6505 
6506 // Load Byte (8bit signed) into Long Register
6507 instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
6508   match(Set dst (ConvI2L (LoadB mem)));
6509   effect(KILL cr);
6510 
6511   ins_cost(375);
6512   format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
6513             "MOV    $dst.hi,$dst.lo\n\t"
6514             "SAR    $dst.hi,7" %}
6515 
6516   ins_encode %{
6517     __ movsbl($dst$$Register, $mem$$Address);
6518     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6519     __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
6520   %}
6521 
6522   ins_pipe(ialu_reg_mem);
6523 %}
6524 
6525 // Load Unsigned Byte (8bit UNsigned)
6526 instruct loadUB(xRegI dst, memory mem) %{
6527   match(Set dst (LoadUB mem));
6528 
6529   ins_cost(125);
6530   format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
6531 
6532   ins_encode %{
6533     __ movzbl($dst$$Register, $mem$$Address);
6534   %}
6535 
6536   ins_pipe(ialu_reg_mem);
6537 %}
6538 
6539 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6540 instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
6541   match(Set dst (ConvI2L (LoadUB mem)));
6542   effect(KILL cr);
6543 
6544   ins_cost(250);
6545   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
6546             "XOR    $dst.hi,$dst.hi" %}
6547 
6548   ins_encode %{
6549     Register Rdst = $dst$$Register;
6550     __ movzbl(Rdst, $mem$$Address);
6551     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6552   %}
6553 
6554   ins_pipe(ialu_reg_mem);
6555 %}
6556 
6557 // Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
6558 instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
6559   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
6560   effect(KILL cr);
6561 
6562   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
6563             "XOR    $dst.hi,$dst.hi\n\t"
6564             "AND    $dst.lo,$mask" %}
6565   ins_encode %{
6566     Register Rdst = $dst$$Register;
6567     __ movzbl(Rdst, $mem$$Address);
6568     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6569     __ andl(Rdst, $mask$$constant);
6570   %}
6571   ins_pipe(ialu_reg_mem);
6572 %}
6573 
6574 // Load Short (16bit signed)
6575 instruct loadS(eRegI dst, memory mem) %{
6576   match(Set dst (LoadS mem));
6577 
6578   ins_cost(125);
6579   format %{ "MOVSX  $dst,$mem\t# short" %}
6580 
6581   ins_encode %{
6582     __ movswl($dst$$Register, $mem$$Address);
6583   %}
6584 
6585   ins_pipe(ialu_reg_mem);
6586 %}
6587 
6588 // Load Short (16 bit signed) to Byte (8 bit signed)
6589 instruct loadS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6590   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
6591 
6592   ins_cost(125);
6593   format %{ "MOVSX  $dst, $mem\t# short -> byte" %}
6594   ins_encode %{
6595     __ movsbl($dst$$Register, $mem$$Address);
6596   %}
6597   ins_pipe(ialu_reg_mem);
6598 %}
6599 
6600 // Load Short (16bit signed) into Long Register
6601 instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
6602   match(Set dst (ConvI2L (LoadS mem)));
6603   effect(KILL cr);
6604 
6605   ins_cost(375);
6606   format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
6607             "MOV    $dst.hi,$dst.lo\n\t"
6608             "SAR    $dst.hi,15" %}
6609 
6610   ins_encode %{
6611     __ movswl($dst$$Register, $mem$$Address);
6612     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6613     __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
6614   %}
6615 
6616   ins_pipe(ialu_reg_mem);
6617 %}
6618 
6619 // Load Unsigned Short/Char (16bit unsigned)
6620 instruct loadUS(eRegI dst, memory mem) %{
6621   match(Set dst (LoadUS mem));
6622 
6623   ins_cost(125);
6624   format %{ "MOVZX  $dst,$mem\t# ushort/char -> int" %}
6625 
6626   ins_encode %{
6627     __ movzwl($dst$$Register, $mem$$Address);
6628   %}
6629 
6630   ins_pipe(ialu_reg_mem);
6631 %}
6632 
6633 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
6634 instruct loadUS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6635   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
6636 
6637   ins_cost(125);
6638   format %{ "MOVSX  $dst, $mem\t# ushort -> byte" %}
6639   ins_encode %{
6640     __ movsbl($dst$$Register, $mem$$Address);
6641   %}
6642   ins_pipe(ialu_reg_mem);
6643 %}
6644 
6645 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
6646 instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
6647   match(Set dst (ConvI2L (LoadUS mem)));
6648   effect(KILL cr);
6649 
6650   ins_cost(250);
6651   format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
6652             "XOR    $dst.hi,$dst.hi" %}
6653 
6654   ins_encode %{
6655     __ movzwl($dst$$Register, $mem$$Address);
6656     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
6657   %}
6658 
6659   ins_pipe(ialu_reg_mem);
6660 %}
6661 
6662 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
6663 instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
6664   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
6665   effect(KILL cr);
6666 
6667   format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
6668             "XOR    $dst.hi,$dst.hi" %}
6669   ins_encode %{
6670     Register Rdst = $dst$$Register;
6671     __ movzbl(Rdst, $mem$$Address);
6672     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6673   %}
6674   ins_pipe(ialu_reg_mem);
6675 %}
6676 
6677 // Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
6678 instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
6679   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
6680   effect(KILL cr);
6681 
6682   format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
6683             "XOR    $dst.hi,$dst.hi\n\t"
6684             "AND    $dst.lo,$mask" %}
6685   ins_encode %{
6686     Register Rdst = $dst$$Register;
6687     __ movzwl(Rdst, $mem$$Address);
6688     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6689     __ andl(Rdst, $mask$$constant);
6690   %}
6691   ins_pipe(ialu_reg_mem);
6692 %}
6693 
6694 // Load Integer
6695 instruct loadI(eRegI dst, memory mem) %{
6696   match(Set dst (LoadI mem));
6697 
6698   ins_cost(125);
6699   format %{ "MOV    $dst,$mem\t# int" %}
6700 
6701   ins_encode %{
6702     __ movl($dst$$Register, $mem$$Address);
6703   %}
6704 
6705   ins_pipe(ialu_reg_mem);
6706 %}
6707 
6708 // Load Integer (32 bit signed) to Byte (8 bit signed)
6709 instruct loadI2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6710   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
6711 
6712   ins_cost(125);
6713   format %{ "MOVSX  $dst, $mem\t# int -> byte" %}
6714   ins_encode %{
6715     __ movsbl($dst$$Register, $mem$$Address);
6716   %}
6717   ins_pipe(ialu_reg_mem);
6718 %}
6719 
6720 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
6721 instruct loadI2UB(eRegI dst, memory mem, immI_255 mask) %{
6722   match(Set dst (AndI (LoadI mem) mask));
6723 
6724   ins_cost(125);
6725   format %{ "MOVZX  $dst, $mem\t# int -> ubyte" %}
6726   ins_encode %{
6727     __ movzbl($dst$$Register, $mem$$Address);
6728   %}
6729   ins_pipe(ialu_reg_mem);
6730 %}
6731 
6732 // Load Integer (32 bit signed) to Short (16 bit signed)
6733 instruct loadI2S(eRegI dst, memory mem, immI_16 sixteen) %{
6734   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
6735 
6736   ins_cost(125);
6737   format %{ "MOVSX  $dst, $mem\t# int -> short" %}
6738   ins_encode %{
6739     __ movswl($dst$$Register, $mem$$Address);
6740   %}
6741   ins_pipe(ialu_reg_mem);
6742 %}
6743 
6744 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
6745 instruct loadI2US(eRegI dst, memory mem, immI_65535 mask) %{
6746   match(Set dst (AndI (LoadI mem) mask));
6747 
6748   ins_cost(125);
6749   format %{ "MOVZX  $dst, $mem\t# int -> ushort/char" %}
6750   ins_encode %{
6751     __ movzwl($dst$$Register, $mem$$Address);
6752   %}
6753   ins_pipe(ialu_reg_mem);
6754 %}
6755 
6756 // Load Integer into Long Register
6757 instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
6758   match(Set dst (ConvI2L (LoadI mem)));
6759   effect(KILL cr);
6760 
6761   ins_cost(375);
6762   format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
6763             "MOV    $dst.hi,$dst.lo\n\t"
6764             "SAR    $dst.hi,31" %}
6765 
6766   ins_encode %{
6767     __ movl($dst$$Register, $mem$$Address);
6768     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6769     __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
6770   %}
6771 
6772   ins_pipe(ialu_reg_mem);
6773 %}
6774 
6775 // Load Integer with mask 0xFF into Long Register
6776 instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
6777   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6778   effect(KILL cr);
6779 
6780   format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
6781             "XOR    $dst.hi,$dst.hi" %}
6782   ins_encode %{
6783     Register Rdst = $dst$$Register;
6784     __ movzbl(Rdst, $mem$$Address);
6785     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6786   %}
6787   ins_pipe(ialu_reg_mem);
6788 %}
6789 
6790 // Load Integer with mask 0xFFFF into Long Register
6791 instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
6792   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6793   effect(KILL cr);
6794 
6795   format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
6796             "XOR    $dst.hi,$dst.hi" %}
6797   ins_encode %{
6798     Register Rdst = $dst$$Register;
6799     __ movzwl(Rdst, $mem$$Address);
6800     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6801   %}
6802   ins_pipe(ialu_reg_mem);
6803 %}
6804 
6805 // Load Integer with 32-bit mask into Long Register
6806 instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
6807   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6808   effect(KILL cr);
6809 
6810   format %{ "MOV    $dst.lo,$mem\t# int & 32-bit mask -> long\n\t"
6811             "XOR    $dst.hi,$dst.hi\n\t"
6812             "AND    $dst.lo,$mask" %}
6813   ins_encode %{
6814     Register Rdst = $dst$$Register;
6815     __ movl(Rdst, $mem$$Address);
6816     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6817     __ andl(Rdst, $mask$$constant);
6818   %}
6819   ins_pipe(ialu_reg_mem);
6820 %}
6821 
6822 // Load Unsigned Integer into Long Register
6823 instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
6824   match(Set dst (LoadUI2L mem));
6825   effect(KILL cr);
6826 
6827   ins_cost(250);
6828   format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
6829             "XOR    $dst.hi,$dst.hi" %}
6830 
6831   ins_encode %{
6832     __ movl($dst$$Register, $mem$$Address);
6833     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
6834   %}
6835 
6836   ins_pipe(ialu_reg_mem);
6837 %}
6838 
6839 // Load Long.  Cannot clobber address while loading, so restrict address
6840 // register to ESI
6841 instruct loadL(eRegL dst, load_long_memory mem) %{
6842   predicate(!((LoadLNode*)n)->require_atomic_access());
6843   match(Set dst (LoadL mem));
6844 
6845   ins_cost(250);
6846   format %{ "MOV    $dst.lo,$mem\t# long\n\t"
6847             "MOV    $dst.hi,$mem+4" %}
6848 
6849   ins_encode %{
6850     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false);
6851     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false);
6852     __ movl($dst$$Register, Amemlo);
6853     __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
6854   %}
6855 
6856   ins_pipe(ialu_reg_long_mem);
6857 %}
6858 
6859 // Volatile Load Long.  Must be atomic, so do 64-bit FILD
6860 // then store it down to the stack and reload on the int
6861 // side.
6862 instruct loadL_volatile(stackSlotL dst, memory mem) %{
6863   predicate(UseSSE<=1 && ((LoadLNode*)n)->require_atomic_access());
6864   match(Set dst (LoadL mem));
6865 
6866   ins_cost(200);
6867   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
6868             "FISTp  $dst" %}
6869   ins_encode(enc_loadL_volatile(mem,dst));
6870   ins_pipe( fpu_reg_mem );
6871 %}
6872 
6873 instruct loadLX_volatile(stackSlotL dst, memory mem, regXD tmp) %{
6874   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
6875   match(Set dst (LoadL mem));
6876   effect(TEMP tmp);
6877   ins_cost(180);
6878   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
6879             "MOVSD  $dst,$tmp" %}
6880   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
6881   ins_pipe( pipe_slow );
6882 %}
6883 
6884 instruct loadLX_reg_volatile(eRegL dst, memory mem, regXD tmp) %{
6885   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
6886   match(Set dst (LoadL mem));
6887   effect(TEMP tmp);
6888   ins_cost(160);
6889   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
6890             "MOVD   $dst.lo,$tmp\n\t"
6891             "PSRLQ  $tmp,32\n\t"
6892             "MOVD   $dst.hi,$tmp" %}
6893   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
6894   ins_pipe( pipe_slow );
6895 %}
6896 
6897 // Load Range
6898 instruct loadRange(eRegI dst, memory mem) %{
6899   match(Set dst (LoadRange mem));
6900 
6901   ins_cost(125);
6902   format %{ "MOV    $dst,$mem" %}
6903   opcode(0x8B);
6904   ins_encode( OpcP, RegMem(dst,mem));
6905   ins_pipe( ialu_reg_mem );
6906 %}
6907 
6908 
6909 // Load Pointer
6910 instruct loadP(eRegP dst, memory mem) %{
6911   match(Set dst (LoadP mem));
6912 
6913   ins_cost(125);
6914   format %{ "MOV    $dst,$mem" %}
6915   opcode(0x8B);
6916   ins_encode( OpcP, RegMem(dst,mem));
6917   ins_pipe( ialu_reg_mem );
6918 %}
6919 
6920 // Load Klass Pointer
6921 instruct loadKlass(eRegP dst, memory mem) %{
6922   match(Set dst (LoadKlass mem));
6923 
6924   ins_cost(125);
6925   format %{ "MOV    $dst,$mem" %}
6926   opcode(0x8B);
6927   ins_encode( OpcP, RegMem(dst,mem));
6928   ins_pipe( ialu_reg_mem );
6929 %}
6930 
6931 // Load Double
6932 instruct loadD(regD dst, memory mem) %{
6933   predicate(UseSSE<=1);
6934   match(Set dst (LoadD mem));
6935 
6936   ins_cost(150);
6937   format %{ "FLD_D  ST,$mem\n\t"
6938             "FSTP   $dst" %}
6939   opcode(0xDD);               /* DD /0 */
6940   ins_encode( OpcP, RMopc_Mem(0x00,mem),
6941               Pop_Reg_D(dst) );
6942   ins_pipe( fpu_reg_mem );
6943 %}
6944 
6945 // Load Double to XMM
6946 instruct loadXD(regXD dst, memory mem) %{
6947   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
6948   match(Set dst (LoadD mem));
6949   ins_cost(145);
6950   format %{ "MOVSD  $dst,$mem" %}
6951   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
6952   ins_pipe( pipe_slow );
6953 %}
6954 
6955 instruct loadXD_partial(regXD dst, memory mem) %{
6956   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
6957   match(Set dst (LoadD mem));
6958   ins_cost(145);
6959   format %{ "MOVLPD $dst,$mem" %}
6960   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,mem));
6961   ins_pipe( pipe_slow );
6962 %}
6963 
6964 // Load to XMM register (single-precision floating point)
6965 // MOVSS instruction
6966 instruct loadX(regX dst, memory mem) %{
6967   predicate(UseSSE>=1);
6968   match(Set dst (LoadF mem));
6969   ins_cost(145);
6970   format %{ "MOVSS  $dst,$mem" %}
6971   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
6972   ins_pipe( pipe_slow );
6973 %}
6974 
6975 // Load Float
6976 instruct loadF(regF dst, memory mem) %{
6977   predicate(UseSSE==0);
6978   match(Set dst (LoadF mem));
6979 
6980   ins_cost(150);
6981   format %{ "FLD_S  ST,$mem\n\t"
6982             "FSTP   $dst" %}
6983   opcode(0xD9);               /* D9 /0 */
6984   ins_encode( OpcP, RMopc_Mem(0x00,mem),
6985               Pop_Reg_F(dst) );
6986   ins_pipe( fpu_reg_mem );
6987 %}
6988 
6989 // Load Aligned Packed Byte to XMM register
6990 instruct loadA8B(regXD dst, memory mem) %{
6991   predicate(UseSSE>=1);
6992   match(Set dst (Load8B mem));
6993   ins_cost(125);
6994   format %{ "MOVQ  $dst,$mem\t! packed8B" %}
6995   ins_encode( movq_ld(dst, mem));
6996   ins_pipe( pipe_slow );
6997 %}
6998 
6999 // Load Aligned Packed Short to XMM register
7000 instruct loadA4S(regXD dst, memory mem) %{
7001   predicate(UseSSE>=1);
7002   match(Set dst (Load4S mem));
7003   ins_cost(125);
7004   format %{ "MOVQ  $dst,$mem\t! packed4S" %}
7005   ins_encode( movq_ld(dst, mem));
7006   ins_pipe( pipe_slow );
7007 %}
7008 
7009 // Load Aligned Packed Char to XMM register
7010 instruct loadA4C(regXD dst, memory mem) %{
7011   predicate(UseSSE>=1);
7012   match(Set dst (Load4C mem));
7013   ins_cost(125);
7014   format %{ "MOVQ  $dst,$mem\t! packed4C" %}
7015   ins_encode( movq_ld(dst, mem));
7016   ins_pipe( pipe_slow );
7017 %}
7018 
7019 // Load Aligned Packed Integer to XMM register
7020 instruct load2IU(regXD dst, memory mem) %{
7021   predicate(UseSSE>=1);
7022   match(Set dst (Load2I mem));
7023   ins_cost(125);
7024   format %{ "MOVQ  $dst,$mem\t! packed2I" %}
7025   ins_encode( movq_ld(dst, mem));
7026   ins_pipe( pipe_slow );
7027 %}
7028 
7029 // Load Aligned Packed Single to XMM
7030 instruct loadA2F(regXD dst, memory mem) %{
7031   predicate(UseSSE>=1);
7032   match(Set dst (Load2F mem));
7033   ins_cost(145);
7034   format %{ "MOVQ  $dst,$mem\t! packed2F" %}
7035   ins_encode( movq_ld(dst, mem));
7036   ins_pipe( pipe_slow );
7037 %}
7038 
7039 // Load Effective Address
7040 instruct leaP8(eRegP dst, indOffset8 mem) %{
7041   match(Set dst mem);
7042 
7043   ins_cost(110);
7044   format %{ "LEA    $dst,$mem" %}
7045   opcode(0x8D);
7046   ins_encode( OpcP, RegMem(dst,mem));
7047   ins_pipe( ialu_reg_reg_fat );
7048 %}
7049 
7050 instruct leaP32(eRegP dst, indOffset32 mem) %{
7051   match(Set dst mem);
7052 
7053   ins_cost(110);
7054   format %{ "LEA    $dst,$mem" %}
7055   opcode(0x8D);
7056   ins_encode( OpcP, RegMem(dst,mem));
7057   ins_pipe( ialu_reg_reg_fat );
7058 %}
7059 
7060 instruct leaPIdxOff(eRegP dst, indIndexOffset mem) %{
7061   match(Set dst mem);
7062 
7063   ins_cost(110);
7064   format %{ "LEA    $dst,$mem" %}
7065   opcode(0x8D);
7066   ins_encode( OpcP, RegMem(dst,mem));
7067   ins_pipe( ialu_reg_reg_fat );
7068 %}
7069 
7070 instruct leaPIdxScale(eRegP dst, indIndexScale mem) %{
7071   match(Set dst mem);
7072 
7073   ins_cost(110);
7074   format %{ "LEA    $dst,$mem" %}
7075   opcode(0x8D);
7076   ins_encode( OpcP, RegMem(dst,mem));
7077   ins_pipe( ialu_reg_reg_fat );
7078 %}
7079 
7080 instruct leaPIdxScaleOff(eRegP dst, indIndexScaleOffset mem) %{
7081   match(Set dst mem);
7082 
7083   ins_cost(110);
7084   format %{ "LEA    $dst,$mem" %}
7085   opcode(0x8D);
7086   ins_encode( OpcP, RegMem(dst,mem));
7087   ins_pipe( ialu_reg_reg_fat );
7088 %}
7089 
7090 // Load Constant
7091 instruct loadConI(eRegI dst, immI src) %{
7092   match(Set dst src);
7093 
7094   format %{ "MOV    $dst,$src" %}
7095   ins_encode( LdImmI(dst, src) );
7096   ins_pipe( ialu_reg_fat );
7097 %}
7098 
7099 // Load Constant zero
7100 instruct loadConI0(eRegI dst, immI0 src, eFlagsReg cr) %{
7101   match(Set dst src);
7102   effect(KILL cr);
7103 
7104   ins_cost(50);
7105   format %{ "XOR    $dst,$dst" %}
7106   opcode(0x33);  /* + rd */
7107   ins_encode( OpcP, RegReg( dst, dst ) );
7108   ins_pipe( ialu_reg );
7109 %}
7110 
7111 instruct loadConP(eRegP dst, immP src) %{
7112   match(Set dst src);
7113 
7114   format %{ "MOV    $dst,$src" %}
7115   opcode(0xB8);  /* + rd */
7116   ins_encode( LdImmP(dst, src) );
7117   ins_pipe( ialu_reg_fat );
7118 %}
7119 
7120 instruct loadConL(eRegL dst, immL src, eFlagsReg cr) %{
7121   match(Set dst src);
7122   effect(KILL cr);
7123   ins_cost(200);
7124   format %{ "MOV    $dst.lo,$src.lo\n\t"
7125             "MOV    $dst.hi,$src.hi" %}
7126   opcode(0xB8);
7127   ins_encode( LdImmL_Lo(dst, src), LdImmL_Hi(dst, src) );
7128   ins_pipe( ialu_reg_long_fat );
7129 %}
7130 
7131 instruct loadConL0(eRegL dst, immL0 src, eFlagsReg cr) %{
7132   match(Set dst src);
7133   effect(KILL cr);
7134   ins_cost(150);
7135   format %{ "XOR    $dst.lo,$dst.lo\n\t"
7136             "XOR    $dst.hi,$dst.hi" %}
7137   opcode(0x33,0x33);
7138   ins_encode( RegReg_Lo(dst,dst), RegReg_Hi(dst, dst) );
7139   ins_pipe( ialu_reg_long );
7140 %}
7141 
7142 // The instruction usage is guarded by predicate in operand immF().
7143 instruct loadConF(regF dst, immF con) %{
7144   match(Set dst con);
7145   ins_cost(125);
7146   format %{ "FLD_S  ST,[$constantaddress]\t# load from constant table: float=$con\n\t"
7147             "FSTP   $dst" %}
7148   ins_encode %{
7149     __ fld_s($constantaddress($con));
7150     __ fstp_d($dst$$reg);
7151   %}
7152   ins_pipe(fpu_reg_con);
7153 %}
7154 
7155 // The instruction usage is guarded by predicate in operand immF0().
7156 instruct loadConF0(regF dst, immF0 con) %{
7157   match(Set dst con);
7158   ins_cost(125);
7159   format %{ "FLDZ   ST\n\t"
7160             "FSTP   $dst" %}
7161   ins_encode %{
7162     __ fldz();
7163     __ fstp_d($dst$$reg);
7164   %}
7165   ins_pipe(fpu_reg_con);
7166 %}
7167 
7168 // The instruction usage is guarded by predicate in operand immF1().
7169 instruct loadConF1(regF dst, immF1 con) %{
7170   match(Set dst con);
7171   ins_cost(125);
7172   format %{ "FLD1   ST\n\t"
7173             "FSTP   $dst" %}
7174   ins_encode %{
7175     __ fld1();
7176     __ fstp_d($dst$$reg);
7177   %}
7178   ins_pipe(fpu_reg_con);
7179 %}
7180 
7181 // The instruction usage is guarded by predicate in operand immXF().
7182 instruct loadConX(regX dst, immXF con) %{
7183   match(Set dst con);
7184   ins_cost(125);
7185   format %{ "MOVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
7186   ins_encode %{
7187     __ movflt($dst$$XMMRegister, $constantaddress($con));
7188   %}
7189   ins_pipe(pipe_slow);
7190 %}
7191 
7192 // The instruction usage is guarded by predicate in operand immXF0().
7193 instruct loadConX0(regX dst, immXF0 src) %{
7194   match(Set dst src);
7195   ins_cost(100);
7196   format %{ "XORPS  $dst,$dst\t# float 0.0" %}
7197   ins_encode %{
7198     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
7199   %}
7200   ins_pipe(pipe_slow);
7201 %}
7202 
7203 // The instruction usage is guarded by predicate in operand immD().
7204 instruct loadConD(regD dst, immD con) %{
7205   match(Set dst con);
7206   ins_cost(125);
7207 
7208   format %{ "FLD_D  ST,[$constantaddress]\t# load from constant table: double=$con\n\t"
7209             "FSTP   $dst" %}
7210   ins_encode %{
7211     __ fld_d($constantaddress($con));
7212     __ fstp_d($dst$$reg);
7213   %}
7214   ins_pipe(fpu_reg_con);
7215 %}
7216 
7217 // The instruction usage is guarded by predicate in operand immD0().
7218 instruct loadConD0(regD dst, immD0 con) %{
7219   match(Set dst con);
7220   ins_cost(125);
7221 
7222   format %{ "FLDZ   ST\n\t"
7223             "FSTP   $dst" %}
7224   ins_encode %{
7225     __ fldz();
7226     __ fstp_d($dst$$reg);
7227   %}
7228   ins_pipe(fpu_reg_con);
7229 %}
7230 
7231 // The instruction usage is guarded by predicate in operand immD1().
7232 instruct loadConD1(regD dst, immD1 con) %{
7233   match(Set dst con);
7234   ins_cost(125);
7235 
7236   format %{ "FLD1   ST\n\t"
7237             "FSTP   $dst" %}
7238   ins_encode %{
7239     __ fld1();
7240     __ fstp_d($dst$$reg);
7241   %}
7242   ins_pipe(fpu_reg_con);
7243 %}
7244 
7245 // The instruction usage is guarded by predicate in operand immXD().
7246 instruct loadConXD(regXD dst, immXD con) %{
7247   match(Set dst con);
7248   ins_cost(125);
7249   format %{ "MOVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
7250   ins_encode %{
7251     __ movdbl($dst$$XMMRegister, $constantaddress($con));
7252   %}
7253   ins_pipe(pipe_slow);
7254 %}
7255 
7256 // The instruction usage is guarded by predicate in operand immXD0().
7257 instruct loadConXD0(regXD dst, immXD0 src) %{
7258   match(Set dst src);
7259   ins_cost(100);
7260   format %{ "XORPD  $dst,$dst\t# double 0.0" %}
7261   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x57), RegReg(dst,dst));
7262   ins_pipe( pipe_slow );
7263 %}
7264 
7265 // Load Stack Slot
7266 instruct loadSSI(eRegI dst, stackSlotI src) %{
7267   match(Set dst src);
7268   ins_cost(125);
7269 
7270   format %{ "MOV    $dst,$src" %}
7271   opcode(0x8B);
7272   ins_encode( OpcP, RegMem(dst,src));
7273   ins_pipe( ialu_reg_mem );
7274 %}
7275 
7276 instruct loadSSL(eRegL dst, stackSlotL src) %{
7277   match(Set dst src);
7278 
7279   ins_cost(200);
7280   format %{ "MOV    $dst,$src.lo\n\t"
7281             "MOV    $dst+4,$src.hi" %}
7282   opcode(0x8B, 0x8B);
7283   ins_encode( OpcP, RegMem( dst, src ), OpcS, RegMem_Hi( dst, src ) );
7284   ins_pipe( ialu_mem_long_reg );
7285 %}
7286 
7287 // Load Stack Slot
7288 instruct loadSSP(eRegP dst, stackSlotP src) %{
7289   match(Set dst src);
7290   ins_cost(125);
7291 
7292   format %{ "MOV    $dst,$src" %}
7293   opcode(0x8B);
7294   ins_encode( OpcP, RegMem(dst,src));
7295   ins_pipe( ialu_reg_mem );
7296 %}
7297 
7298 // Load Stack Slot
7299 instruct loadSSF(regF dst, stackSlotF src) %{
7300   match(Set dst src);
7301   ins_cost(125);
7302 
7303   format %{ "FLD_S  $src\n\t"
7304             "FSTP   $dst" %}
7305   opcode(0xD9);               /* D9 /0, FLD m32real */
7306   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
7307               Pop_Reg_F(dst) );
7308   ins_pipe( fpu_reg_mem );
7309 %}
7310 
7311 // Load Stack Slot
7312 instruct loadSSD(regD dst, stackSlotD src) %{
7313   match(Set dst src);
7314   ins_cost(125);
7315 
7316   format %{ "FLD_D  $src\n\t"
7317             "FSTP   $dst" %}
7318   opcode(0xDD);               /* DD /0, FLD m64real */
7319   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
7320               Pop_Reg_D(dst) );
7321   ins_pipe( fpu_reg_mem );
7322 %}
7323 
7324 // Prefetch instructions.
7325 // Must be safe to execute with invalid address (cannot fault).
7326 
7327 instruct prefetchr0( memory mem ) %{
7328   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
7329   match(PrefetchRead mem);
7330   ins_cost(0);
7331   size(0);
7332   format %{ "PREFETCHR (non-SSE is empty encoding)" %}
7333   ins_encode();
7334   ins_pipe(empty);
7335 %}
7336 
7337 instruct prefetchr( memory mem ) %{
7338   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch() || ReadPrefetchInstr==3);
7339   match(PrefetchRead mem);
7340   ins_cost(100);
7341 
7342   format %{ "PREFETCHR $mem\t! Prefetch into level 1 cache for read" %}
7343   ins_encode %{
7344     __ prefetchr($mem$$Address);
7345   %}
7346   ins_pipe(ialu_mem);
7347 %}
7348 
7349 instruct prefetchrNTA( memory mem ) %{
7350   predicate(UseSSE>=1 && ReadPrefetchInstr==0);
7351   match(PrefetchRead mem);
7352   ins_cost(100);
7353 
7354   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for read" %}
7355   ins_encode %{
7356     __ prefetchnta($mem$$Address);
7357   %}
7358   ins_pipe(ialu_mem);
7359 %}
7360 
7361 instruct prefetchrT0( memory mem ) %{
7362   predicate(UseSSE>=1 && ReadPrefetchInstr==1);
7363   match(PrefetchRead mem);
7364   ins_cost(100);
7365 
7366   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for read" %}
7367   ins_encode %{
7368     __ prefetcht0($mem$$Address);
7369   %}
7370   ins_pipe(ialu_mem);
7371 %}
7372 
7373 instruct prefetchrT2( memory mem ) %{
7374   predicate(UseSSE>=1 && ReadPrefetchInstr==2);
7375   match(PrefetchRead mem);
7376   ins_cost(100);
7377 
7378   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for read" %}
7379   ins_encode %{
7380     __ prefetcht2($mem$$Address);
7381   %}
7382   ins_pipe(ialu_mem);
7383 %}
7384 
7385 instruct prefetchw0( memory mem ) %{
7386   predicate(UseSSE==0 && !VM_Version::supports_3dnow_prefetch());
7387   match(PrefetchWrite mem);
7388   ins_cost(0);
7389   size(0);
7390   format %{ "Prefetch (non-SSE is empty encoding)" %}
7391   ins_encode();
7392   ins_pipe(empty);
7393 %}
7394 
7395 instruct prefetchw( memory mem ) %{
7396   predicate(UseSSE==0 && VM_Version::supports_3dnow_prefetch());
7397   match( PrefetchWrite mem );
7398   ins_cost(100);
7399 
7400   format %{ "PREFETCHW $mem\t! Prefetch into L1 cache and mark modified" %}
7401   ins_encode %{
7402     __ prefetchw($mem$$Address);
7403   %}
7404   ins_pipe(ialu_mem);
7405 %}
7406 
7407 instruct prefetchwNTA( memory mem ) %{
7408   predicate(UseSSE>=1);
7409   match(PrefetchWrite mem);
7410   ins_cost(100);
7411 
7412   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for write" %}
7413   ins_encode %{
7414     __ prefetchnta($mem$$Address);
7415   %}
7416   ins_pipe(ialu_mem);
7417 %}
7418 
7419 // Prefetch instructions for allocation.
7420 
7421 instruct prefetchAlloc0( memory mem ) %{
7422   predicate(UseSSE==0 && AllocatePrefetchInstr!=3);
7423   match(PrefetchAllocation mem);
7424   ins_cost(0);
7425   size(0);
7426   format %{ "Prefetch allocation (non-SSE is empty encoding)" %}
7427   ins_encode();
7428   ins_pipe(empty);
7429 %}
7430 
7431 instruct prefetchAlloc( memory mem ) %{
7432   predicate(AllocatePrefetchInstr==3);
7433   match( PrefetchAllocation mem );
7434   ins_cost(100);
7435 
7436   format %{ "PREFETCHW $mem\t! Prefetch allocation into L1 cache and mark modified" %}
7437   ins_encode %{
7438     __ prefetchw($mem$$Address);
7439   %}
7440   ins_pipe(ialu_mem);
7441 %}
7442 
7443 instruct prefetchAllocNTA( memory mem ) %{
7444   predicate(UseSSE>=1 && AllocatePrefetchInstr==0);
7445   match(PrefetchAllocation mem);
7446   ins_cost(100);
7447 
7448   format %{ "PREFETCHNTA $mem\t! Prefetch allocation into non-temporal cache for write" %}
7449   ins_encode %{
7450     __ prefetchnta($mem$$Address);
7451   %}
7452   ins_pipe(ialu_mem);
7453 %}
7454 
7455 instruct prefetchAllocT0( memory mem ) %{
7456   predicate(UseSSE>=1 && AllocatePrefetchInstr==1);
7457   match(PrefetchAllocation mem);
7458   ins_cost(100);
7459 
7460   format %{ "PREFETCHT0 $mem\t! Prefetch allocation into L1 and L2 caches for write" %}
7461   ins_encode %{
7462     __ prefetcht0($mem$$Address);
7463   %}
7464   ins_pipe(ialu_mem);
7465 %}
7466 
7467 instruct prefetchAllocT2( memory mem ) %{
7468   predicate(UseSSE>=1 && AllocatePrefetchInstr==2);
7469   match(PrefetchAllocation mem);
7470   ins_cost(100);
7471 
7472   format %{ "PREFETCHT2 $mem\t! Prefetch allocation into L2 cache for write" %}
7473   ins_encode %{
7474     __ prefetcht2($mem$$Address);
7475   %}
7476   ins_pipe(ialu_mem);
7477 %}
7478 
7479 //----------Store Instructions-------------------------------------------------
7480 
7481 // Store Byte
7482 instruct storeB(memory mem, xRegI src) %{
7483   match(Set mem (StoreB mem src));
7484 
7485   ins_cost(125);
7486   format %{ "MOV8   $mem,$src" %}
7487   opcode(0x88);
7488   ins_encode( OpcP, RegMem( src, mem ) );
7489   ins_pipe( ialu_mem_reg );
7490 %}
7491 
7492 // Store Char/Short
7493 instruct storeC(memory mem, eRegI src) %{
7494   match(Set mem (StoreC mem src));
7495 
7496   ins_cost(125);
7497   format %{ "MOV16  $mem,$src" %}
7498   opcode(0x89, 0x66);
7499   ins_encode( OpcS, OpcP, RegMem( src, mem ) );
7500   ins_pipe( ialu_mem_reg );
7501 %}
7502 
7503 // Store Integer
7504 instruct storeI(memory mem, eRegI src) %{
7505   match(Set mem (StoreI mem src));
7506 
7507   ins_cost(125);
7508   format %{ "MOV    $mem,$src" %}
7509   opcode(0x89);
7510   ins_encode( OpcP, RegMem( src, mem ) );
7511   ins_pipe( ialu_mem_reg );
7512 %}
7513 
7514 // Store Long
7515 instruct storeL(long_memory mem, eRegL src) %{
7516   predicate(!((StoreLNode*)n)->require_atomic_access());
7517   match(Set mem (StoreL mem src));
7518 
7519   ins_cost(200);
7520   format %{ "MOV    $mem,$src.lo\n\t"
7521             "MOV    $mem+4,$src.hi" %}
7522   opcode(0x89, 0x89);
7523   ins_encode( OpcP, RegMem( src, mem ), OpcS, RegMem_Hi( src, mem ) );
7524   ins_pipe( ialu_mem_long_reg );
7525 %}
7526 
7527 // Store Long to Integer
7528 instruct storeL2I(memory mem, eRegL src) %{
7529   match(Set mem (StoreI mem (ConvL2I src)));
7530 
7531   format %{ "MOV    $mem,$src.lo\t# long -> int" %}
7532   ins_encode %{
7533     __ movl($mem$$Address, $src$$Register);
7534   %}
7535   ins_pipe(ialu_mem_reg);
7536 %}
7537 
7538 // Volatile Store Long.  Must be atomic, so move it into
7539 // the FP TOS and then do a 64-bit FIST.  Has to probe the
7540 // target address before the store (for null-ptr checks)
7541 // so the memory operand is used twice in the encoding.
7542 instruct storeL_volatile(memory mem, stackSlotL src, eFlagsReg cr ) %{
7543   predicate(UseSSE<=1 && ((StoreLNode*)n)->require_atomic_access());
7544   match(Set mem (StoreL mem src));
7545   effect( KILL cr );
7546   ins_cost(400);
7547   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7548             "FILD   $src\n\t"
7549             "FISTp  $mem\t # 64-bit atomic volatile long store" %}
7550   opcode(0x3B);
7551   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeL_volatile(mem,src));
7552   ins_pipe( fpu_reg_mem );
7553 %}
7554 
7555 instruct storeLX_volatile(memory mem, stackSlotL src, regXD tmp, eFlagsReg cr) %{
7556   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
7557   match(Set mem (StoreL mem src));
7558   effect( TEMP tmp, KILL cr );
7559   ins_cost(380);
7560   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7561             "MOVSD  $tmp,$src\n\t"
7562             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
7563   opcode(0x3B);
7564   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_volatile(mem, src, tmp));
7565   ins_pipe( pipe_slow );
7566 %}
7567 
7568 instruct storeLX_reg_volatile(memory mem, eRegL src, regXD tmp2, regXD tmp, eFlagsReg cr) %{
7569   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
7570   match(Set mem (StoreL mem src));
7571   effect( TEMP tmp2 , TEMP tmp, KILL cr );
7572   ins_cost(360);
7573   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7574             "MOVD   $tmp,$src.lo\n\t"
7575             "MOVD   $tmp2,$src.hi\n\t"
7576             "PUNPCKLDQ $tmp,$tmp2\n\t"
7577             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
7578   opcode(0x3B);
7579   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_reg_volatile(mem, src, tmp, tmp2));
7580   ins_pipe( pipe_slow );
7581 %}
7582 
7583 // Store Pointer; for storing unknown oops and raw pointers
7584 instruct storeP(memory mem, anyRegP src) %{
7585   match(Set mem (StoreP mem src));
7586 
7587   ins_cost(125);
7588   format %{ "MOV    $mem,$src" %}
7589   opcode(0x89);
7590   ins_encode( OpcP, RegMem( src, mem ) );
7591   ins_pipe( ialu_mem_reg );
7592 %}
7593 
7594 // Store Integer Immediate
7595 instruct storeImmI(memory mem, immI src) %{
7596   match(Set mem (StoreI mem src));
7597 
7598   ins_cost(150);
7599   format %{ "MOV    $mem,$src" %}
7600   opcode(0xC7);               /* C7 /0 */
7601   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
7602   ins_pipe( ialu_mem_imm );
7603 %}
7604 
7605 // Store Short/Char Immediate
7606 instruct storeImmI16(memory mem, immI16 src) %{
7607   predicate(UseStoreImmI16);
7608   match(Set mem (StoreC mem src));
7609 
7610   ins_cost(150);
7611   format %{ "MOV16  $mem,$src" %}
7612   opcode(0xC7);     /* C7 /0 Same as 32 store immediate with prefix */
7613   ins_encode( SizePrefix, OpcP, RMopc_Mem(0x00,mem),  Con16( src ));
7614   ins_pipe( ialu_mem_imm );
7615 %}
7616 
7617 // Store Pointer Immediate; null pointers or constant oops that do not
7618 // need card-mark barriers.
7619 instruct storeImmP(memory mem, immP src) %{
7620   match(Set mem (StoreP mem src));
7621 
7622   ins_cost(150);
7623   format %{ "MOV    $mem,$src" %}
7624   opcode(0xC7);               /* C7 /0 */
7625   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
7626   ins_pipe( ialu_mem_imm );
7627 %}
7628 
7629 // Store Byte Immediate
7630 instruct storeImmB(memory mem, immI8 src) %{
7631   match(Set mem (StoreB mem src));
7632 
7633   ins_cost(150);
7634   format %{ "MOV8   $mem,$src" %}
7635   opcode(0xC6);               /* C6 /0 */
7636   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
7637   ins_pipe( ialu_mem_imm );
7638 %}
7639 
7640 // Store Aligned Packed Byte XMM register to memory
7641 instruct storeA8B(memory mem, regXD src) %{
7642   predicate(UseSSE>=1);
7643   match(Set mem (Store8B mem src));
7644   ins_cost(145);
7645   format %{ "MOVQ  $mem,$src\t! packed8B" %}
7646   ins_encode( movq_st(mem, src));
7647   ins_pipe( pipe_slow );
7648 %}
7649 
7650 // Store Aligned Packed Char/Short XMM register to memory
7651 instruct storeA4C(memory mem, regXD src) %{
7652   predicate(UseSSE>=1);
7653   match(Set mem (Store4C mem src));
7654   ins_cost(145);
7655   format %{ "MOVQ  $mem,$src\t! packed4C" %}
7656   ins_encode( movq_st(mem, src));
7657   ins_pipe( pipe_slow );
7658 %}
7659 
7660 // Store Aligned Packed Integer XMM register to memory
7661 instruct storeA2I(memory mem, regXD src) %{
7662   predicate(UseSSE>=1);
7663   match(Set mem (Store2I mem src));
7664   ins_cost(145);
7665   format %{ "MOVQ  $mem,$src\t! packed2I" %}
7666   ins_encode( movq_st(mem, src));
7667   ins_pipe( pipe_slow );
7668 %}
7669 
7670 // Store CMS card-mark Immediate
7671 instruct storeImmCM(memory mem, immI8 src) %{
7672   match(Set mem (StoreCM mem src));
7673 
7674   ins_cost(150);
7675   format %{ "MOV8   $mem,$src\t! CMS card-mark imm0" %}
7676   opcode(0xC6);               /* C6 /0 */
7677   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
7678   ins_pipe( ialu_mem_imm );
7679 %}
7680 
7681 // Store Double
7682 instruct storeD( memory mem, regDPR1 src) %{
7683   predicate(UseSSE<=1);
7684   match(Set mem (StoreD mem src));
7685 
7686   ins_cost(100);
7687   format %{ "FST_D  $mem,$src" %}
7688   opcode(0xDD);       /* DD /2 */
7689   ins_encode( enc_FP_store(mem,src) );
7690   ins_pipe( fpu_mem_reg );
7691 %}
7692 
7693 // Store double does rounding on x86
7694 instruct storeD_rounded( memory mem, regDPR1 src) %{
7695   predicate(UseSSE<=1);
7696   match(Set mem (StoreD mem (RoundDouble src)));
7697 
7698   ins_cost(100);
7699   format %{ "FST_D  $mem,$src\t# round" %}
7700   opcode(0xDD);       /* DD /2 */
7701   ins_encode( enc_FP_store(mem,src) );
7702   ins_pipe( fpu_mem_reg );
7703 %}
7704 
7705 // Store XMM register to memory (double-precision floating points)
7706 // MOVSD instruction
7707 instruct storeXD(memory mem, regXD src) %{
7708   predicate(UseSSE>=2);
7709   match(Set mem (StoreD mem src));
7710   ins_cost(95);
7711   format %{ "MOVSD  $mem,$src" %}
7712   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
7713   ins_pipe( pipe_slow );
7714 %}
7715 
7716 // Store XMM register to memory (single-precision floating point)
7717 // MOVSS instruction
7718 instruct storeX(memory mem, regX src) %{
7719   predicate(UseSSE>=1);
7720   match(Set mem (StoreF mem src));
7721   ins_cost(95);
7722   format %{ "MOVSS  $mem,$src" %}
7723   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
7724   ins_pipe( pipe_slow );
7725 %}
7726 
7727 // Store Aligned Packed Single Float XMM register to memory
7728 instruct storeA2F(memory mem, regXD src) %{
7729   predicate(UseSSE>=1);
7730   match(Set mem (Store2F mem src));
7731   ins_cost(145);
7732   format %{ "MOVQ  $mem,$src\t! packed2F" %}
7733   ins_encode( movq_st(mem, src));
7734   ins_pipe( pipe_slow );
7735 %}
7736 
7737 // Store Float
7738 instruct storeF( memory mem, regFPR1 src) %{
7739   predicate(UseSSE==0);
7740   match(Set mem (StoreF mem src));
7741 
7742   ins_cost(100);
7743   format %{ "FST_S  $mem,$src" %}
7744   opcode(0xD9);       /* D9 /2 */
7745   ins_encode( enc_FP_store(mem,src) );
7746   ins_pipe( fpu_mem_reg );
7747 %}
7748 
7749 // Store Float does rounding on x86
7750 instruct storeF_rounded( memory mem, regFPR1 src) %{
7751   predicate(UseSSE==0);
7752   match(Set mem (StoreF mem (RoundFloat src)));
7753 
7754   ins_cost(100);
7755   format %{ "FST_S  $mem,$src\t# round" %}
7756   opcode(0xD9);       /* D9 /2 */
7757   ins_encode( enc_FP_store(mem,src) );
7758   ins_pipe( fpu_mem_reg );
7759 %}
7760 
7761 // Store Float does rounding on x86
7762 instruct storeF_Drounded( memory mem, regDPR1 src) %{
7763   predicate(UseSSE<=1);
7764   match(Set mem (StoreF mem (ConvD2F src)));
7765 
7766   ins_cost(100);
7767   format %{ "FST_S  $mem,$src\t# D-round" %}
7768   opcode(0xD9);       /* D9 /2 */
7769   ins_encode( enc_FP_store(mem,src) );
7770   ins_pipe( fpu_mem_reg );
7771 %}
7772 
7773 // Store immediate Float value (it is faster than store from FPU register)
7774 // The instruction usage is guarded by predicate in operand immF().
7775 instruct storeF_imm( memory mem, immF src) %{
7776   match(Set mem (StoreF mem src));
7777 
7778   ins_cost(50);
7779   format %{ "MOV    $mem,$src\t# store float" %}
7780   opcode(0xC7);               /* C7 /0 */
7781   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32F_as_bits( src ));
7782   ins_pipe( ialu_mem_imm );
7783 %}
7784 
7785 // Store immediate Float value (it is faster than store from XMM register)
7786 // The instruction usage is guarded by predicate in operand immXF().
7787 instruct storeX_imm( memory mem, immXF src) %{
7788   match(Set mem (StoreF mem src));
7789 
7790   ins_cost(50);
7791   format %{ "MOV    $mem,$src\t# store float" %}
7792   opcode(0xC7);               /* C7 /0 */
7793   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32XF_as_bits( src ));
7794   ins_pipe( ialu_mem_imm );
7795 %}
7796 
7797 // Store Integer to stack slot
7798 instruct storeSSI(stackSlotI dst, eRegI src) %{
7799   match(Set dst src);
7800 
7801   ins_cost(100);
7802   format %{ "MOV    $dst,$src" %}
7803   opcode(0x89);
7804   ins_encode( OpcPRegSS( dst, src ) );
7805   ins_pipe( ialu_mem_reg );
7806 %}
7807 
7808 // Store Integer to stack slot
7809 instruct storeSSP(stackSlotP dst, eRegP src) %{
7810   match(Set dst src);
7811 
7812   ins_cost(100);
7813   format %{ "MOV    $dst,$src" %}
7814   opcode(0x89);
7815   ins_encode( OpcPRegSS( dst, src ) );
7816   ins_pipe( ialu_mem_reg );
7817 %}
7818 
7819 // Store Long to stack slot
7820 instruct storeSSL(stackSlotL dst, eRegL src) %{
7821   match(Set dst src);
7822 
7823   ins_cost(200);
7824   format %{ "MOV    $dst,$src.lo\n\t"
7825             "MOV    $dst+4,$src.hi" %}
7826   opcode(0x89, 0x89);
7827   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
7828   ins_pipe( ialu_mem_long_reg );
7829 %}
7830 
7831 //----------MemBar Instructions-----------------------------------------------
7832 // Memory barrier flavors
7833 
7834 instruct membar_acquire() %{
7835   match(MemBarAcquire);
7836   ins_cost(400);
7837 
7838   size(0);
7839   format %{ "MEMBAR-acquire ! (empty encoding)" %}
7840   ins_encode();
7841   ins_pipe(empty);
7842 %}
7843 
7844 instruct membar_acquire_lock() %{
7845   match(MemBarAcquireLock);
7846   ins_cost(0);
7847 
7848   size(0);
7849   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
7850   ins_encode( );
7851   ins_pipe(empty);
7852 %}
7853 
7854 instruct membar_release() %{
7855   match(MemBarRelease);
7856   ins_cost(400);
7857 
7858   size(0);
7859   format %{ "MEMBAR-release ! (empty encoding)" %}
7860   ins_encode( );
7861   ins_pipe(empty);
7862 %}
7863 
7864 instruct membar_release_lock() %{
7865   match(MemBarReleaseLock);
7866   ins_cost(0);
7867 
7868   size(0);
7869   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
7870   ins_encode( );
7871   ins_pipe(empty);
7872 %}
7873 
7874 instruct membar_volatile(eFlagsReg cr) %{
7875   match(MemBarVolatile);
7876   effect(KILL cr);
7877   ins_cost(400);
7878 
7879   format %{ 
7880     $$template
7881     if (os::is_MP()) {
7882       $$emit$$"LOCK ADDL [ESP + #0], 0\t! membar_volatile"
7883     } else {
7884       $$emit$$"MEMBAR-volatile ! (empty encoding)"
7885     }
7886   %}
7887   ins_encode %{
7888     __ membar(Assembler::StoreLoad);
7889   %}
7890   ins_pipe(pipe_slow);
7891 %}
7892 
7893 instruct unnecessary_membar_volatile() %{
7894   match(MemBarVolatile);
7895   predicate(Matcher::post_store_load_barrier(n));
7896   ins_cost(0);
7897 
7898   size(0);
7899   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
7900   ins_encode( );
7901   ins_pipe(empty);
7902 %}
7903 
7904 //----------Move Instructions--------------------------------------------------
7905 instruct castX2P(eAXRegP dst, eAXRegI src) %{
7906   match(Set dst (CastX2P src));
7907   format %{ "# X2P  $dst, $src" %}
7908   ins_encode( /*empty encoding*/ );
7909   ins_cost(0);
7910   ins_pipe(empty);
7911 %}
7912 
7913 instruct castP2X(eRegI dst, eRegP src ) %{
7914   match(Set dst (CastP2X src));
7915   ins_cost(50);
7916   format %{ "MOV    $dst, $src\t# CastP2X" %}
7917   ins_encode( enc_Copy( dst, src) );
7918   ins_pipe( ialu_reg_reg );
7919 %}
7920 
7921 //----------Conditional Move---------------------------------------------------
7922 // Conditional move
7923 instruct jmovI_reg(cmpOp cop, eFlagsReg cr, eRegI dst, eRegI src) %{
7924   predicate(!VM_Version::supports_cmov() );
7925   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7926   ins_cost(200);
7927   format %{ "J$cop,us skip\t# signed cmove\n\t"
7928             "MOV    $dst,$src\n"
7929       "skip:" %}
7930   ins_encode %{
7931     Label Lskip;
7932     // Invert sense of branch from sense of CMOV
7933     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7934     __ movl($dst$$Register, $src$$Register);
7935     __ bind(Lskip);
7936   %}
7937   ins_pipe( pipe_cmov_reg );
7938 %}
7939 
7940 instruct jmovI_regU(cmpOpU cop, eFlagsRegU cr, eRegI dst, eRegI src) %{
7941   predicate(!VM_Version::supports_cmov() );
7942   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7943   ins_cost(200);
7944   format %{ "J$cop,us skip\t# unsigned cmove\n\t"
7945             "MOV    $dst,$src\n"
7946       "skip:" %}
7947   ins_encode %{
7948     Label Lskip;
7949     // Invert sense of branch from sense of CMOV
7950     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7951     __ movl($dst$$Register, $src$$Register);
7952     __ bind(Lskip);
7953   %}
7954   ins_pipe( pipe_cmov_reg );
7955 %}
7956 
7957 instruct cmovI_reg(eRegI dst, eRegI src, eFlagsReg cr, cmpOp cop ) %{
7958   predicate(VM_Version::supports_cmov() );
7959   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7960   ins_cost(200);
7961   format %{ "CMOV$cop $dst,$src" %}
7962   opcode(0x0F,0x40);
7963   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7964   ins_pipe( pipe_cmov_reg );
7965 %}
7966 
7967 instruct cmovI_regU( cmpOpU cop, eFlagsRegU cr, eRegI dst, eRegI src ) %{
7968   predicate(VM_Version::supports_cmov() );
7969   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7970   ins_cost(200);
7971   format %{ "CMOV$cop $dst,$src" %}
7972   opcode(0x0F,0x40);
7973   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7974   ins_pipe( pipe_cmov_reg );
7975 %}
7976 
7977 instruct cmovI_regUCF( cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, eRegI src ) %{
7978   predicate(VM_Version::supports_cmov() );
7979   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7980   ins_cost(200);
7981   expand %{
7982     cmovI_regU(cop, cr, dst, src);
7983   %}
7984 %}
7985 
7986 // Conditional move
7987 instruct cmovI_mem(cmpOp cop, eFlagsReg cr, eRegI dst, memory src) %{
7988   predicate(VM_Version::supports_cmov() );
7989   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7990   ins_cost(250);
7991   format %{ "CMOV$cop $dst,$src" %}
7992   opcode(0x0F,0x40);
7993   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
7994   ins_pipe( pipe_cmov_mem );
7995 %}
7996 
7997 // Conditional move
7998 instruct cmovI_memU(cmpOpU cop, eFlagsRegU cr, eRegI dst, memory src) %{
7999   predicate(VM_Version::supports_cmov() );
8000   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
8001   ins_cost(250);
8002   format %{ "CMOV$cop $dst,$src" %}
8003   opcode(0x0F,0x40);
8004   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
8005   ins_pipe( pipe_cmov_mem );
8006 %}
8007 
8008 instruct cmovI_memUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, memory src) %{
8009   predicate(VM_Version::supports_cmov() );
8010   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
8011   ins_cost(250);
8012   expand %{
8013     cmovI_memU(cop, cr, dst, src);
8014   %}
8015 %}
8016 
8017 // Conditional move
8018 instruct cmovP_reg(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
8019   predicate(VM_Version::supports_cmov() );
8020   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
8021   ins_cost(200);
8022   format %{ "CMOV$cop $dst,$src\t# ptr" %}
8023   opcode(0x0F,0x40);
8024   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
8025   ins_pipe( pipe_cmov_reg );
8026 %}
8027 
8028 // Conditional move (non-P6 version)
8029 // Note:  a CMoveP is generated for  stubs and native wrappers
8030 //        regardless of whether we are on a P6, so we
8031 //        emulate a cmov here
8032 instruct cmovP_reg_nonP6(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
8033   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
8034   ins_cost(300);
8035   format %{ "Jn$cop   skip\n\t"
8036           "MOV    $dst,$src\t# pointer\n"
8037       "skip:" %}
8038   opcode(0x8b);
8039   ins_encode( enc_cmov_branch(cop, 0x2), OpcP, RegReg(dst, src));
8040   ins_pipe( pipe_cmov_reg );
8041 %}
8042 
8043 // Conditional move
8044 instruct cmovP_regU(cmpOpU cop, eFlagsRegU cr, eRegP dst, eRegP src ) %{
8045   predicate(VM_Version::supports_cmov() );
8046   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
8047   ins_cost(200);
8048   format %{ "CMOV$cop $dst,$src\t# ptr" %}
8049   opcode(0x0F,0x40);
8050   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
8051   ins_pipe( pipe_cmov_reg );
8052 %}
8053 
8054 instruct cmovP_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegP dst, eRegP src ) %{
8055   predicate(VM_Version::supports_cmov() );
8056   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
8057   ins_cost(200);
8058   expand %{
8059     cmovP_regU(cop, cr, dst, src);
8060   %}
8061 %}
8062 
8063 // DISABLED: Requires the ADLC to emit a bottom_type call that
8064 // correctly meets the two pointer arguments; one is an incoming
8065 // register but the other is a memory operand.  ALSO appears to
8066 // be buggy with implicit null checks.
8067 //
8068 //// Conditional move
8069 //instruct cmovP_mem(cmpOp cop, eFlagsReg cr, eRegP dst, memory src) %{
8070 //  predicate(VM_Version::supports_cmov() );
8071 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
8072 //  ins_cost(250);
8073 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
8074 //  opcode(0x0F,0x40);
8075 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
8076 //  ins_pipe( pipe_cmov_mem );
8077 //%}
8078 //
8079 //// Conditional move
8080 //instruct cmovP_memU(cmpOpU cop, eFlagsRegU cr, eRegP dst, memory src) %{
8081 //  predicate(VM_Version::supports_cmov() );
8082 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
8083 //  ins_cost(250);
8084 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
8085 //  opcode(0x0F,0x40);
8086 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
8087 //  ins_pipe( pipe_cmov_mem );
8088 //%}
8089 
8090 // Conditional move
8091 instruct fcmovD_regU(cmpOp_fcmov cop, eFlagsRegU cr, regDPR1 dst, regD src) %{
8092   predicate(UseSSE<=1);
8093   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8094   ins_cost(200);
8095   format %{ "FCMOV$cop $dst,$src\t# double" %}
8096   opcode(0xDA);
8097   ins_encode( enc_cmov_d(cop,src) );
8098   ins_pipe( pipe_cmovD_reg );
8099 %}
8100 
8101 // Conditional move
8102 instruct fcmovF_regU(cmpOp_fcmov cop, eFlagsRegU cr, regFPR1 dst, regF src) %{
8103   predicate(UseSSE==0);
8104   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8105   ins_cost(200);
8106   format %{ "FCMOV$cop $dst,$src\t# float" %}
8107   opcode(0xDA);
8108   ins_encode( enc_cmov_d(cop,src) );
8109   ins_pipe( pipe_cmovD_reg );
8110 %}
8111 
8112 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
8113 instruct fcmovD_regS(cmpOp cop, eFlagsReg cr, regD dst, regD src) %{
8114   predicate(UseSSE<=1);
8115   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8116   ins_cost(200);
8117   format %{ "Jn$cop   skip\n\t"
8118             "MOV    $dst,$src\t# double\n"
8119       "skip:" %}
8120   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
8121   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_D(src), OpcP, RegOpc(dst) );
8122   ins_pipe( pipe_cmovD_reg );
8123 %}
8124 
8125 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
8126 instruct fcmovF_regS(cmpOp cop, eFlagsReg cr, regF dst, regF src) %{
8127   predicate(UseSSE==0);
8128   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8129   ins_cost(200);
8130   format %{ "Jn$cop    skip\n\t"
8131             "MOV    $dst,$src\t# float\n"
8132       "skip:" %}
8133   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
8134   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_F(src), OpcP, RegOpc(dst) );
8135   ins_pipe( pipe_cmovD_reg );
8136 %}
8137 
8138 // No CMOVE with SSE/SSE2
8139 instruct fcmovX_regS(cmpOp cop, eFlagsReg cr, regX dst, regX src) %{
8140   predicate (UseSSE>=1);
8141   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8142   ins_cost(200);
8143   format %{ "Jn$cop   skip\n\t"
8144             "MOVSS  $dst,$src\t# float\n"
8145       "skip:" %}
8146   ins_encode %{
8147     Label skip;
8148     // Invert sense of branch from sense of CMOV
8149     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8150     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
8151     __ bind(skip);
8152   %}
8153   ins_pipe( pipe_slow );
8154 %}
8155 
8156 // No CMOVE with SSE/SSE2
8157 instruct fcmovXD_regS(cmpOp cop, eFlagsReg cr, regXD dst, regXD src) %{
8158   predicate (UseSSE>=2);
8159   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8160   ins_cost(200);
8161   format %{ "Jn$cop   skip\n\t"
8162             "MOVSD  $dst,$src\t# float\n"
8163       "skip:" %}
8164   ins_encode %{
8165     Label skip;
8166     // Invert sense of branch from sense of CMOV
8167     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8168     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
8169     __ bind(skip);
8170   %}
8171   ins_pipe( pipe_slow );
8172 %}
8173 
8174 // unsigned version
8175 instruct fcmovX_regU(cmpOpU cop, eFlagsRegU cr, regX dst, regX src) %{
8176   predicate (UseSSE>=1);
8177   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8178   ins_cost(200);
8179   format %{ "Jn$cop   skip\n\t"
8180             "MOVSS  $dst,$src\t# float\n"
8181       "skip:" %}
8182   ins_encode %{
8183     Label skip;
8184     // Invert sense of branch from sense of CMOV
8185     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8186     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
8187     __ bind(skip);
8188   %}
8189   ins_pipe( pipe_slow );
8190 %}
8191 
8192 instruct fcmovX_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regX dst, regX src) %{
8193   predicate (UseSSE>=1);
8194   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8195   ins_cost(200);
8196   expand %{
8197     fcmovX_regU(cop, cr, dst, src);
8198   %}
8199 %}
8200 
8201 // unsigned version
8202 instruct fcmovXD_regU(cmpOpU cop, eFlagsRegU cr, regXD dst, regXD src) %{
8203   predicate (UseSSE>=2);
8204   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8205   ins_cost(200);
8206   format %{ "Jn$cop   skip\n\t"
8207             "MOVSD  $dst,$src\t# float\n"
8208       "skip:" %}
8209   ins_encode %{
8210     Label skip;
8211     // Invert sense of branch from sense of CMOV
8212     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8213     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
8214     __ bind(skip);
8215   %}
8216   ins_pipe( pipe_slow );
8217 %}
8218 
8219 instruct fcmovXD_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regXD dst, regXD src) %{
8220   predicate (UseSSE>=2);
8221   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8222   ins_cost(200);
8223   expand %{
8224     fcmovXD_regU(cop, cr, dst, src);
8225   %}
8226 %}
8227 
8228 instruct cmovL_reg(cmpOp cop, eFlagsReg cr, eRegL dst, eRegL src) %{
8229   predicate(VM_Version::supports_cmov() );
8230   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8231   ins_cost(200);
8232   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
8233             "CMOV$cop $dst.hi,$src.hi" %}
8234   opcode(0x0F,0x40);
8235   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
8236   ins_pipe( pipe_cmov_reg_long );
8237 %}
8238 
8239 instruct cmovL_regU(cmpOpU cop, eFlagsRegU cr, eRegL dst, eRegL src) %{
8240   predicate(VM_Version::supports_cmov() );
8241   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8242   ins_cost(200);
8243   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
8244             "CMOV$cop $dst.hi,$src.hi" %}
8245   opcode(0x0F,0x40);
8246   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
8247   ins_pipe( pipe_cmov_reg_long );
8248 %}
8249 
8250 instruct cmovL_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegL dst, eRegL src) %{
8251   predicate(VM_Version::supports_cmov() );
8252   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8253   ins_cost(200);
8254   expand %{
8255     cmovL_regU(cop, cr, dst, src);
8256   %}
8257 %}
8258 
8259 //----------Arithmetic Instructions--------------------------------------------
8260 //----------Addition Instructions----------------------------------------------
8261 // Integer Addition Instructions
8262 instruct addI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8263   match(Set dst (AddI dst src));
8264   effect(KILL cr);
8265 
8266   size(2);
8267   format %{ "ADD    $dst,$src" %}
8268   opcode(0x03);
8269   ins_encode( OpcP, RegReg( dst, src) );
8270   ins_pipe( ialu_reg_reg );
8271 %}
8272 
8273 instruct addI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
8274   match(Set dst (AddI dst src));
8275   effect(KILL cr);
8276 
8277   format %{ "ADD    $dst,$src" %}
8278   opcode(0x81, 0x00); /* /0 id */
8279   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8280   ins_pipe( ialu_reg );
8281 %}
8282 
8283 instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
8284   predicate(UseIncDec);
8285   match(Set dst (AddI dst src));
8286   effect(KILL cr);
8287 
8288   size(1);
8289   format %{ "INC    $dst" %}
8290   opcode(0x40); /*  */
8291   ins_encode( Opc_plus( primary, dst ) );
8292   ins_pipe( ialu_reg );
8293 %}
8294 
8295 instruct leaI_eReg_immI(eRegI dst, eRegI src0, immI src1) %{
8296   match(Set dst (AddI src0 src1));
8297   ins_cost(110);
8298 
8299   format %{ "LEA    $dst,[$src0 + $src1]" %}
8300   opcode(0x8D); /* 0x8D /r */
8301   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
8302   ins_pipe( ialu_reg_reg );
8303 %}
8304 
8305 instruct leaP_eReg_immI(eRegP dst, eRegP src0, immI src1) %{
8306   match(Set dst (AddP src0 src1));
8307   ins_cost(110);
8308 
8309   format %{ "LEA    $dst,[$src0 + $src1]\t# ptr" %}
8310   opcode(0x8D); /* 0x8D /r */
8311   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
8312   ins_pipe( ialu_reg_reg );
8313 %}
8314 
8315 instruct decI_eReg(eRegI dst, immI_M1 src, eFlagsReg cr) %{
8316   predicate(UseIncDec);
8317   match(Set dst (AddI dst src));
8318   effect(KILL cr);
8319 
8320   size(1);
8321   format %{ "DEC    $dst" %}
8322   opcode(0x48); /*  */
8323   ins_encode( Opc_plus( primary, dst ) );
8324   ins_pipe( ialu_reg );
8325 %}
8326 
8327 instruct addP_eReg(eRegP dst, eRegI src, eFlagsReg cr) %{
8328   match(Set dst (AddP dst src));
8329   effect(KILL cr);
8330 
8331   size(2);
8332   format %{ "ADD    $dst,$src" %}
8333   opcode(0x03);
8334   ins_encode( OpcP, RegReg( dst, src) );
8335   ins_pipe( ialu_reg_reg );
8336 %}
8337 
8338 instruct addP_eReg_imm(eRegP dst, immI src, eFlagsReg cr) %{
8339   match(Set dst (AddP dst src));
8340   effect(KILL cr);
8341 
8342   format %{ "ADD    $dst,$src" %}
8343   opcode(0x81,0x00); /* Opcode 81 /0 id */
8344   // ins_encode( RegImm( dst, src) );
8345   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8346   ins_pipe( ialu_reg );
8347 %}
8348 
8349 instruct addI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
8350   match(Set dst (AddI dst (LoadI src)));
8351   effect(KILL cr);
8352 
8353   ins_cost(125);
8354   format %{ "ADD    $dst,$src" %}
8355   opcode(0x03);
8356   ins_encode( OpcP, RegMem( dst, src) );
8357   ins_pipe( ialu_reg_mem );
8358 %}
8359 
8360 instruct addI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
8361   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8362   effect(KILL cr);
8363 
8364   ins_cost(150);
8365   format %{ "ADD    $dst,$src" %}
8366   opcode(0x01);  /* Opcode 01 /r */
8367   ins_encode( OpcP, RegMem( src, dst ) );
8368   ins_pipe( ialu_mem_reg );
8369 %}
8370 
8371 // Add Memory with Immediate
8372 instruct addI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
8373   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8374   effect(KILL cr);
8375 
8376   ins_cost(125);
8377   format %{ "ADD    $dst,$src" %}
8378   opcode(0x81);               /* Opcode 81 /0 id */
8379   ins_encode( OpcSE( src ), RMopc_Mem(0x00,dst), Con8or32( src ) );
8380   ins_pipe( ialu_mem_imm );
8381 %}
8382 
8383 instruct incI_mem(memory dst, immI1 src, eFlagsReg cr) %{
8384   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8385   effect(KILL cr);
8386 
8387   ins_cost(125);
8388   format %{ "INC    $dst" %}
8389   opcode(0xFF);               /* Opcode FF /0 */
8390   ins_encode( OpcP, RMopc_Mem(0x00,dst));
8391   ins_pipe( ialu_mem_imm );
8392 %}
8393 
8394 instruct decI_mem(memory dst, immI_M1 src, eFlagsReg cr) %{
8395   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8396   effect(KILL cr);
8397 
8398   ins_cost(125);
8399   format %{ "DEC    $dst" %}
8400   opcode(0xFF);               /* Opcode FF /1 */
8401   ins_encode( OpcP, RMopc_Mem(0x01,dst));
8402   ins_pipe( ialu_mem_imm );
8403 %}
8404 
8405 
8406 instruct checkCastPP( eRegP dst ) %{
8407   match(Set dst (CheckCastPP dst));
8408 
8409   size(0);
8410   format %{ "#checkcastPP of $dst" %}
8411   ins_encode( /*empty encoding*/ );
8412   ins_pipe( empty );
8413 %}
8414 
8415 instruct castPP( eRegP dst ) %{
8416   match(Set dst (CastPP dst));
8417   format %{ "#castPP of $dst" %}
8418   ins_encode( /*empty encoding*/ );
8419   ins_pipe( empty );
8420 %}
8421 
8422 instruct castII( eRegI dst ) %{
8423   match(Set dst (CastII dst));
8424   format %{ "#castII of $dst" %}
8425   ins_encode( /*empty encoding*/ );
8426   ins_cost(0);
8427   ins_pipe( empty );
8428 %}
8429 
8430 
8431 // Load-locked - same as a regular pointer load when used with compare-swap
8432 instruct loadPLocked(eRegP dst, memory mem) %{
8433   match(Set dst (LoadPLocked mem));
8434 
8435   ins_cost(125);
8436   format %{ "MOV    $dst,$mem\t# Load ptr. locked" %}
8437   opcode(0x8B);
8438   ins_encode( OpcP, RegMem(dst,mem));
8439   ins_pipe( ialu_reg_mem );
8440 %}
8441 
8442 // LoadLong-locked - same as a volatile long load when used with compare-swap
8443 instruct loadLLocked(stackSlotL dst, load_long_memory mem) %{
8444   predicate(UseSSE<=1);
8445   match(Set dst (LoadLLocked mem));
8446 
8447   ins_cost(200);
8448   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
8449             "FISTp  $dst" %}
8450   ins_encode(enc_loadL_volatile(mem,dst));
8451   ins_pipe( fpu_reg_mem );
8452 %}
8453 
8454 instruct loadLX_Locked(stackSlotL dst, load_long_memory mem, regXD tmp) %{
8455   predicate(UseSSE>=2);
8456   match(Set dst (LoadLLocked mem));
8457   effect(TEMP tmp);
8458   ins_cost(180);
8459   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
8460             "MOVSD  $dst,$tmp" %}
8461   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
8462   ins_pipe( pipe_slow );
8463 %}
8464 
8465 instruct loadLX_reg_Locked(eRegL dst, load_long_memory mem, regXD tmp) %{
8466   predicate(UseSSE>=2);
8467   match(Set dst (LoadLLocked mem));
8468   effect(TEMP tmp);
8469   ins_cost(160);
8470   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
8471             "MOVD   $dst.lo,$tmp\n\t"
8472             "PSRLQ  $tmp,32\n\t"
8473             "MOVD   $dst.hi,$tmp" %}
8474   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
8475   ins_pipe( pipe_slow );
8476 %}
8477 
8478 // Conditional-store of the updated heap-top.
8479 // Used during allocation of the shared heap.
8480 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
8481 instruct storePConditional( memory heap_top_ptr, eAXRegP oldval, eRegP newval, eFlagsReg cr ) %{
8482   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8483   // EAX is killed if there is contention, but then it's also unused.
8484   // In the common case of no contention, EAX holds the new oop address.
8485   format %{ "CMPXCHG $heap_top_ptr,$newval\t# If EAX==$heap_top_ptr Then store $newval into $heap_top_ptr" %}
8486   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval,heap_top_ptr) );
8487   ins_pipe( pipe_cmpxchg );
8488 %}
8489 
8490 // Conditional-store of an int value.
8491 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG on Intel.
8492 instruct storeIConditional( memory mem, eAXRegI oldval, eRegI newval, eFlagsReg cr ) %{
8493   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8494   effect(KILL oldval);
8495   format %{ "CMPXCHG $mem,$newval\t# If EAX==$mem Then store $newval into $mem" %}
8496   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval, mem) );
8497   ins_pipe( pipe_cmpxchg );
8498 %}
8499 
8500 // Conditional-store of a long value.
8501 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG8 on Intel.
8502 instruct storeLConditional( memory mem, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
8503   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8504   effect(KILL oldval);
8505   format %{ "XCHG   EBX,ECX\t# correct order for CMPXCHG8 instruction\n\t"
8506             "CMPXCHG8 $mem,ECX:EBX\t# If EDX:EAX==$mem Then store ECX:EBX into $mem\n\t"
8507             "XCHG   EBX,ECX"
8508   %}
8509   ins_encode %{
8510     // Note: we need to swap rbx, and rcx before and after the
8511     //       cmpxchg8 instruction because the instruction uses
8512     //       rcx as the high order word of the new value to store but
8513     //       our register encoding uses rbx.
8514     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
8515     if( os::is_MP() )
8516       __ lock();
8517     __ cmpxchg8($mem$$Address);
8518     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
8519   %}
8520   ins_pipe( pipe_cmpxchg );
8521 %}
8522 
8523 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
8524 
8525 instruct compareAndSwapL( eRegI res, eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
8526   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
8527   effect(KILL cr, KILL oldval);
8528   format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8529             "MOV    $res,0\n\t"
8530             "JNE,s  fail\n\t"
8531             "MOV    $res,1\n"
8532           "fail:" %}
8533   ins_encode( enc_cmpxchg8(mem_ptr),
8534               enc_flags_ne_to_boolean(res) );
8535   ins_pipe( pipe_cmpxchg );
8536 %}
8537 
8538 instruct compareAndSwapP( eRegI res,  pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
8539   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
8540   effect(KILL cr, KILL oldval);
8541   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8542             "MOV    $res,0\n\t"
8543             "JNE,s  fail\n\t"
8544             "MOV    $res,1\n"
8545           "fail:" %}
8546   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
8547   ins_pipe( pipe_cmpxchg );
8548 %}
8549 
8550 instruct compareAndSwapI( eRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
8551   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
8552   effect(KILL cr, KILL oldval);
8553   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8554             "MOV    $res,0\n\t"
8555             "JNE,s  fail\n\t"
8556             "MOV    $res,1\n"
8557           "fail:" %}
8558   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
8559   ins_pipe( pipe_cmpxchg );
8560 %}
8561 
8562 //----------Subtraction Instructions-------------------------------------------
8563 // Integer Subtraction Instructions
8564 instruct subI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8565   match(Set dst (SubI dst src));
8566   effect(KILL cr);
8567 
8568   size(2);
8569   format %{ "SUB    $dst,$src" %}
8570   opcode(0x2B);
8571   ins_encode( OpcP, RegReg( dst, src) );
8572   ins_pipe( ialu_reg_reg );
8573 %}
8574 
8575 instruct subI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
8576   match(Set dst (SubI dst src));
8577   effect(KILL cr);
8578 
8579   format %{ "SUB    $dst,$src" %}
8580   opcode(0x81,0x05);  /* Opcode 81 /5 */
8581   // ins_encode( RegImm( dst, src) );
8582   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8583   ins_pipe( ialu_reg );
8584 %}
8585 
8586 instruct subI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
8587   match(Set dst (SubI dst (LoadI src)));
8588   effect(KILL cr);
8589 
8590   ins_cost(125);
8591   format %{ "SUB    $dst,$src" %}
8592   opcode(0x2B);
8593   ins_encode( OpcP, RegMem( dst, src) );
8594   ins_pipe( ialu_reg_mem );
8595 %}
8596 
8597 instruct subI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
8598   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8599   effect(KILL cr);
8600 
8601   ins_cost(150);
8602   format %{ "SUB    $dst,$src" %}
8603   opcode(0x29);  /* Opcode 29 /r */
8604   ins_encode( OpcP, RegMem( src, dst ) );
8605   ins_pipe( ialu_mem_reg );
8606 %}
8607 
8608 // Subtract from a pointer
8609 instruct subP_eReg(eRegP dst, eRegI src, immI0 zero, eFlagsReg cr) %{
8610   match(Set dst (AddP dst (SubI zero src)));
8611   effect(KILL cr);
8612 
8613   size(2);
8614   format %{ "SUB    $dst,$src" %}
8615   opcode(0x2B);
8616   ins_encode( OpcP, RegReg( dst, src) );
8617   ins_pipe( ialu_reg_reg );
8618 %}
8619 
8620 instruct negI_eReg(eRegI dst, immI0 zero, eFlagsReg cr) %{
8621   match(Set dst (SubI zero dst));
8622   effect(KILL cr);
8623 
8624   size(2);
8625   format %{ "NEG    $dst" %}
8626   opcode(0xF7,0x03);  // Opcode F7 /3
8627   ins_encode( OpcP, RegOpc( dst ) );
8628   ins_pipe( ialu_reg );
8629 %}
8630 
8631 
8632 //----------Multiplication/Division Instructions-------------------------------
8633 // Integer Multiplication Instructions
8634 // Multiply Register
8635 instruct mulI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8636   match(Set dst (MulI dst src));
8637   effect(KILL cr);
8638 
8639   size(3);
8640   ins_cost(300);
8641   format %{ "IMUL   $dst,$src" %}
8642   opcode(0xAF, 0x0F);
8643   ins_encode( OpcS, OpcP, RegReg( dst, src) );
8644   ins_pipe( ialu_reg_reg_alu0 );
8645 %}
8646 
8647 // Multiply 32-bit Immediate
8648 instruct mulI_eReg_imm(eRegI dst, eRegI src, immI imm, eFlagsReg cr) %{
8649   match(Set dst (MulI src imm));
8650   effect(KILL cr);
8651 
8652   ins_cost(300);
8653   format %{ "IMUL   $dst,$src,$imm" %}
8654   opcode(0x69);  /* 69 /r id */
8655   ins_encode( OpcSE(imm), RegReg( dst, src ), Con8or32( imm ) );
8656   ins_pipe( ialu_reg_reg_alu0 );
8657 %}
8658 
8659 instruct loadConL_low_only(eADXRegL_low_only dst, immL32 src, eFlagsReg cr) %{
8660   match(Set dst src);
8661   effect(KILL cr);
8662 
8663   // Note that this is artificially increased to make it more expensive than loadConL
8664   ins_cost(250);
8665   format %{ "MOV    EAX,$src\t// low word only" %}
8666   opcode(0xB8);
8667   ins_encode( LdImmL_Lo(dst, src) );
8668   ins_pipe( ialu_reg_fat );
8669 %}
8670 
8671 // Multiply by 32-bit Immediate, taking the shifted high order results
8672 //  (special case for shift by 32)
8673 instruct mulI_imm_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32 cnt, eFlagsReg cr) %{
8674   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
8675   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
8676              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
8677              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
8678   effect(USE src1, KILL cr);
8679 
8680   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
8681   ins_cost(0*100 + 1*400 - 150);
8682   format %{ "IMUL   EDX:EAX,$src1" %}
8683   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
8684   ins_pipe( pipe_slow );
8685 %}
8686 
8687 // Multiply by 32-bit Immediate, taking the shifted high order results
8688 instruct mulI_imm_RShift_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr) %{
8689   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
8690   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
8691              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
8692              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
8693   effect(USE src1, KILL cr);
8694 
8695   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
8696   ins_cost(1*100 + 1*400 - 150);
8697   format %{ "IMUL   EDX:EAX,$src1\n\t"
8698             "SAR    EDX,$cnt-32" %}
8699   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
8700   ins_pipe( pipe_slow );
8701 %}
8702 
8703 // Multiply Memory 32-bit Immediate
8704 instruct mulI_mem_imm(eRegI dst, memory src, immI imm, eFlagsReg cr) %{
8705   match(Set dst (MulI (LoadI src) imm));
8706   effect(KILL cr);
8707 
8708   ins_cost(300);
8709   format %{ "IMUL   $dst,$src,$imm" %}
8710   opcode(0x69);  /* 69 /r id */
8711   ins_encode( OpcSE(imm), RegMem( dst, src ), Con8or32( imm ) );
8712   ins_pipe( ialu_reg_mem_alu0 );
8713 %}
8714 
8715 // Multiply Memory
8716 instruct mulI(eRegI dst, memory src, eFlagsReg cr) %{
8717   match(Set dst (MulI dst (LoadI src)));
8718   effect(KILL cr);
8719 
8720   ins_cost(350);
8721   format %{ "IMUL   $dst,$src" %}
8722   opcode(0xAF, 0x0F);
8723   ins_encode( OpcS, OpcP, RegMem( dst, src) );
8724   ins_pipe( ialu_reg_mem_alu0 );
8725 %}
8726 
8727 // Multiply Register Int to Long
8728 instruct mulI2L(eADXRegL dst, eAXRegI src, nadxRegI src1, eFlagsReg flags) %{
8729   // Basic Idea: long = (long)int * (long)int
8730   match(Set dst (MulL (ConvI2L src) (ConvI2L src1)));
8731   effect(DEF dst, USE src, USE src1, KILL flags);
8732 
8733   ins_cost(300);
8734   format %{ "IMUL   $dst,$src1" %}
8735 
8736   ins_encode( long_int_multiply( dst, src1 ) );
8737   ins_pipe( ialu_reg_reg_alu0 );
8738 %}
8739 
8740 instruct mulIS_eReg(eADXRegL dst, immL_32bits mask, eFlagsReg flags, eAXRegI src, nadxRegI src1) %{
8741   // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
8742   match(Set dst (MulL (AndL (ConvI2L src) mask) (AndL (ConvI2L src1) mask)));
8743   effect(KILL flags);
8744 
8745   ins_cost(300);
8746   format %{ "MUL    $dst,$src1" %}
8747 
8748   ins_encode( long_uint_multiply(dst, src1) );
8749   ins_pipe( ialu_reg_reg_alu0 );
8750 %}
8751 
8752 // Multiply Register Long
8753 instruct mulL_eReg(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8754   match(Set dst (MulL dst src));
8755   effect(KILL cr, TEMP tmp);
8756   ins_cost(4*100+3*400);
8757 // Basic idea: lo(result) = lo(x_lo * y_lo)
8758 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
8759   format %{ "MOV    $tmp,$src.lo\n\t"
8760             "IMUL   $tmp,EDX\n\t"
8761             "MOV    EDX,$src.hi\n\t"
8762             "IMUL   EDX,EAX\n\t"
8763             "ADD    $tmp,EDX\n\t"
8764             "MUL    EDX:EAX,$src.lo\n\t"
8765             "ADD    EDX,$tmp" %}
8766   ins_encode( long_multiply( dst, src, tmp ) );
8767   ins_pipe( pipe_slow );
8768 %}
8769 
8770 // Multiply Register Long where the left operand's high 32 bits are zero
8771 instruct mulL_eReg_lhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8772   predicate(is_operand_hi32_zero(n->in(1)));
8773   match(Set dst (MulL dst src));
8774   effect(KILL cr, TEMP tmp);
8775   ins_cost(2*100+2*400);
8776 // Basic idea: lo(result) = lo(x_lo * y_lo)
8777 //             hi(result) = hi(x_lo * y_lo) + lo(x_lo * y_hi) where lo(x_hi * y_lo) = 0 because x_hi = 0
8778   format %{ "MOV    $tmp,$src.hi\n\t"
8779             "IMUL   $tmp,EAX\n\t"
8780             "MUL    EDX:EAX,$src.lo\n\t"
8781             "ADD    EDX,$tmp" %}
8782   ins_encode %{
8783     __ movl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
8784     __ imull($tmp$$Register, rax);
8785     __ mull($src$$Register);
8786     __ addl(rdx, $tmp$$Register);
8787   %}
8788   ins_pipe( pipe_slow );
8789 %}
8790 
8791 // Multiply Register Long where the right operand's high 32 bits are zero
8792 instruct mulL_eReg_rhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8793   predicate(is_operand_hi32_zero(n->in(2)));
8794   match(Set dst (MulL dst src));
8795   effect(KILL cr, TEMP tmp);
8796   ins_cost(2*100+2*400);
8797 // Basic idea: lo(result) = lo(x_lo * y_lo)
8798 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) where lo(x_lo * y_hi) = 0 because y_hi = 0
8799   format %{ "MOV    $tmp,$src.lo\n\t"
8800             "IMUL   $tmp,EDX\n\t"
8801             "MUL    EDX:EAX,$src.lo\n\t"
8802             "ADD    EDX,$tmp" %}
8803   ins_encode %{
8804     __ movl($tmp$$Register, $src$$Register);
8805     __ imull($tmp$$Register, rdx);
8806     __ mull($src$$Register);
8807     __ addl(rdx, $tmp$$Register);
8808   %}
8809   ins_pipe( pipe_slow );
8810 %}
8811 
8812 // Multiply Register Long where the left and the right operands' high 32 bits are zero
8813 instruct mulL_eReg_hi0(eADXRegL dst, eRegL src, eFlagsReg cr) %{
8814   predicate(is_operand_hi32_zero(n->in(1)) && is_operand_hi32_zero(n->in(2)));
8815   match(Set dst (MulL dst src));
8816   effect(KILL cr);
8817   ins_cost(1*400);
8818 // Basic idea: lo(result) = lo(x_lo * y_lo)
8819 //             hi(result) = hi(x_lo * y_lo) where lo(x_hi * y_lo) = 0 and lo(x_lo * y_hi) = 0 because x_hi = 0 and y_hi = 0
8820   format %{ "MUL    EDX:EAX,$src.lo\n\t" %}
8821   ins_encode %{
8822     __ mull($src$$Register);
8823   %}
8824   ins_pipe( pipe_slow );
8825 %}
8826 
8827 // Multiply Register Long by small constant
8828 instruct mulL_eReg_con(eADXRegL dst, immL_127 src, eRegI tmp, eFlagsReg cr) %{
8829   match(Set dst (MulL dst src));
8830   effect(KILL cr, TEMP tmp);
8831   ins_cost(2*100+2*400);
8832   size(12);
8833 // Basic idea: lo(result) = lo(src * EAX)
8834 //             hi(result) = hi(src * EAX) + lo(src * EDX)
8835   format %{ "IMUL   $tmp,EDX,$src\n\t"
8836             "MOV    EDX,$src\n\t"
8837             "MUL    EDX\t# EDX*EAX -> EDX:EAX\n\t"
8838             "ADD    EDX,$tmp" %}
8839   ins_encode( long_multiply_con( dst, src, tmp ) );
8840   ins_pipe( pipe_slow );
8841 %}
8842 
8843 // Integer DIV with Register
8844 instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
8845   match(Set rax (DivI rax div));
8846   effect(KILL rdx, KILL cr);
8847   size(26);
8848   ins_cost(30*100+10*100);
8849   format %{ "CMP    EAX,0x80000000\n\t"
8850             "JNE,s  normal\n\t"
8851             "XOR    EDX,EDX\n\t"
8852             "CMP    ECX,-1\n\t"
8853             "JE,s   done\n"
8854     "normal: CDQ\n\t"
8855             "IDIV   $div\n\t"
8856     "done:"        %}
8857   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8858   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8859   ins_pipe( ialu_reg_reg_alu0 );
8860 %}
8861 
8862 // Divide Register Long
8863 instruct divL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
8864   match(Set dst (DivL src1 src2));
8865   effect( KILL cr, KILL cx, KILL bx );
8866   ins_cost(10000);
8867   format %{ "PUSH   $src1.hi\n\t"
8868             "PUSH   $src1.lo\n\t"
8869             "PUSH   $src2.hi\n\t"
8870             "PUSH   $src2.lo\n\t"
8871             "CALL   SharedRuntime::ldiv\n\t"
8872             "ADD    ESP,16" %}
8873   ins_encode( long_div(src1,src2) );
8874   ins_pipe( pipe_slow );
8875 %}
8876 
8877 // Integer DIVMOD with Register, both quotient and mod results
8878 instruct divModI_eReg_divmod(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
8879   match(DivModI rax div);
8880   effect(KILL cr);
8881   size(26);
8882   ins_cost(30*100+10*100);
8883   format %{ "CMP    EAX,0x80000000\n\t"
8884             "JNE,s  normal\n\t"
8885             "XOR    EDX,EDX\n\t"
8886             "CMP    ECX,-1\n\t"
8887             "JE,s   done\n"
8888     "normal: CDQ\n\t"
8889             "IDIV   $div\n\t"
8890     "done:"        %}
8891   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8892   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8893   ins_pipe( pipe_slow );
8894 %}
8895 
8896 // Integer MOD with Register
8897 instruct modI_eReg(eDXRegI rdx, eAXRegI rax, eCXRegI div, eFlagsReg cr) %{
8898   match(Set rdx (ModI rax div));
8899   effect(KILL rax, KILL cr);
8900 
8901   size(26);
8902   ins_cost(300);
8903   format %{ "CDQ\n\t"
8904             "IDIV   $div" %}
8905   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8906   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8907   ins_pipe( ialu_reg_reg_alu0 );
8908 %}
8909 
8910 // Remainder Register Long
8911 instruct modL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
8912   match(Set dst (ModL src1 src2));
8913   effect( KILL cr, KILL cx, KILL bx );
8914   ins_cost(10000);
8915   format %{ "PUSH   $src1.hi\n\t"
8916             "PUSH   $src1.lo\n\t"
8917             "PUSH   $src2.hi\n\t"
8918             "PUSH   $src2.lo\n\t"
8919             "CALL   SharedRuntime::lrem\n\t"
8920             "ADD    ESP,16" %}
8921   ins_encode( long_mod(src1,src2) );
8922   ins_pipe( pipe_slow );
8923 %}
8924 
8925 // Divide Register Long (no special case since divisor != -1)
8926 instruct divL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
8927   match(Set dst (DivL dst imm));
8928   effect( TEMP tmp, TEMP tmp2, KILL cr );
8929   ins_cost(1000);
8930   format %{ "MOV    $tmp,abs($imm) # ldiv EDX:EAX,$imm\n\t"
8931             "XOR    $tmp2,$tmp2\n\t"
8932             "CMP    $tmp,EDX\n\t"
8933             "JA,s   fast\n\t"
8934             "MOV    $tmp2,EAX\n\t"
8935             "MOV    EAX,EDX\n\t"
8936             "MOV    EDX,0\n\t"
8937             "JLE,s  pos\n\t"
8938             "LNEG   EAX : $tmp2\n\t"
8939             "DIV    $tmp # unsigned division\n\t"
8940             "XCHG   EAX,$tmp2\n\t"
8941             "DIV    $tmp\n\t"
8942             "LNEG   $tmp2 : EAX\n\t"
8943             "JMP,s  done\n"
8944     "pos:\n\t"
8945             "DIV    $tmp\n\t"
8946             "XCHG   EAX,$tmp2\n"
8947     "fast:\n\t"
8948             "DIV    $tmp\n"
8949     "done:\n\t"
8950             "MOV    EDX,$tmp2\n\t"
8951             "NEG    EDX:EAX # if $imm < 0" %}
8952   ins_encode %{
8953     int con = (int)$imm$$constant;
8954     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
8955     int pcon = (con > 0) ? con : -con;
8956     Label Lfast, Lpos, Ldone;
8957 
8958     __ movl($tmp$$Register, pcon);
8959     __ xorl($tmp2$$Register,$tmp2$$Register);
8960     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
8961     __ jccb(Assembler::above, Lfast); // result fits into 32 bit
8962 
8963     __ movl($tmp2$$Register, $dst$$Register); // save
8964     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
8965     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
8966     __ jccb(Assembler::lessEqual, Lpos); // result is positive
8967 
8968     // Negative dividend.
8969     // convert value to positive to use unsigned division
8970     __ lneg($dst$$Register, $tmp2$$Register);
8971     __ divl($tmp$$Register);
8972     __ xchgl($dst$$Register, $tmp2$$Register);
8973     __ divl($tmp$$Register);
8974     // revert result back to negative
8975     __ lneg($tmp2$$Register, $dst$$Register);
8976     __ jmpb(Ldone);
8977 
8978     __ bind(Lpos);
8979     __ divl($tmp$$Register); // Use unsigned division
8980     __ xchgl($dst$$Register, $tmp2$$Register);
8981     // Fallthrow for final divide, tmp2 has 32 bit hi result
8982 
8983     __ bind(Lfast);
8984     // fast path: src is positive
8985     __ divl($tmp$$Register); // Use unsigned division
8986 
8987     __ bind(Ldone);
8988     __ movl(HIGH_FROM_LOW($dst$$Register),$tmp2$$Register);
8989     if (con < 0) {
8990       __ lneg(HIGH_FROM_LOW($dst$$Register), $dst$$Register);
8991     }
8992   %}
8993   ins_pipe( pipe_slow );
8994 %}
8995 
8996 // Remainder Register Long (remainder fit into 32 bits)
8997 instruct modL_eReg_imm32( eADXRegL dst, immL32 imm, eRegI tmp, eRegI tmp2, eFlagsReg cr ) %{
8998   match(Set dst (ModL dst imm));
8999   effect( TEMP tmp, TEMP tmp2, KILL cr );
9000   ins_cost(1000);
9001   format %{ "MOV    $tmp,abs($imm) # lrem EDX:EAX,$imm\n\t"
9002             "CMP    $tmp,EDX\n\t"
9003             "JA,s   fast\n\t"
9004             "MOV    $tmp2,EAX\n\t"
9005             "MOV    EAX,EDX\n\t"
9006             "MOV    EDX,0\n\t"
9007             "JLE,s  pos\n\t"
9008             "LNEG   EAX : $tmp2\n\t"
9009             "DIV    $tmp # unsigned division\n\t"
9010             "MOV    EAX,$tmp2\n\t"
9011             "DIV    $tmp\n\t"
9012             "NEG    EDX\n\t"
9013             "JMP,s  done\n"
9014     "pos:\n\t"
9015             "DIV    $tmp\n\t"
9016             "MOV    EAX,$tmp2\n"
9017     "fast:\n\t"
9018             "DIV    $tmp\n"
9019     "done:\n\t"
9020             "MOV    EAX,EDX\n\t"
9021             "SAR    EDX,31\n\t" %}
9022   ins_encode %{
9023     int con = (int)$imm$$constant;
9024     assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
9025     int pcon = (con > 0) ? con : -con;
9026     Label  Lfast, Lpos, Ldone;
9027 
9028     __ movl($tmp$$Register, pcon);
9029     __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
9030     __ jccb(Assembler::above, Lfast); // src is positive and result fits into 32 bit
9031 
9032     __ movl($tmp2$$Register, $dst$$Register); // save
9033     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
9034     __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
9035     __ jccb(Assembler::lessEqual, Lpos); // result is positive
9036 
9037     // Negative dividend.
9038     // convert value to positive to use unsigned division
9039     __ lneg($dst$$Register, $tmp2$$Register);
9040     __ divl($tmp$$Register);
9041     __ movl($dst$$Register, $tmp2$$Register);
9042     __ divl($tmp$$Register);
9043     // revert remainder back to negative
9044     __ negl(HIGH_FROM_LOW($dst$$Register));
9045     __ jmpb(Ldone);
9046 
9047     __ bind(Lpos);
9048     __ divl($tmp$$Register);
9049     __ movl($dst$$Register, $tmp2$$Register);
9050 
9051     __ bind(Lfast);
9052     // fast path: src is positive
9053     __ divl($tmp$$Register);
9054 
9055     __ bind(Ldone);
9056     __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
9057     __ sarl(HIGH_FROM_LOW($dst$$Register), 31); // result sign
9058 
9059   %}
9060   ins_pipe( pipe_slow );
9061 %}
9062 
9063 // Integer Shift Instructions
9064 // Shift Left by one
9065 instruct shlI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9066   match(Set dst (LShiftI dst shift));
9067   effect(KILL cr);
9068 
9069   size(2);
9070   format %{ "SHL    $dst,$shift" %}
9071   opcode(0xD1, 0x4);  /* D1 /4 */
9072   ins_encode( OpcP, RegOpc( dst ) );
9073   ins_pipe( ialu_reg );
9074 %}
9075 
9076 // Shift Left by 8-bit immediate
9077 instruct salI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
9078   match(Set dst (LShiftI dst shift));
9079   effect(KILL cr);
9080 
9081   size(3);
9082   format %{ "SHL    $dst,$shift" %}
9083   opcode(0xC1, 0x4);  /* C1 /4 ib */
9084   ins_encode( RegOpcImm( dst, shift) );
9085   ins_pipe( ialu_reg );
9086 %}
9087 
9088 // Shift Left by variable
9089 instruct salI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
9090   match(Set dst (LShiftI dst shift));
9091   effect(KILL cr);
9092 
9093   size(2);
9094   format %{ "SHL    $dst,$shift" %}
9095   opcode(0xD3, 0x4);  /* D3 /4 */
9096   ins_encode( OpcP, RegOpc( dst ) );
9097   ins_pipe( ialu_reg_reg );
9098 %}
9099 
9100 // Arithmetic shift right by one
9101 instruct sarI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9102   match(Set dst (RShiftI dst shift));
9103   effect(KILL cr);
9104 
9105   size(2);
9106   format %{ "SAR    $dst,$shift" %}
9107   opcode(0xD1, 0x7);  /* D1 /7 */
9108   ins_encode( OpcP, RegOpc( dst ) );
9109   ins_pipe( ialu_reg );
9110 %}
9111 
9112 // Arithmetic shift right by one
9113 instruct sarI_mem_1(memory dst, immI1 shift, eFlagsReg cr) %{
9114   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
9115   effect(KILL cr);
9116   format %{ "SAR    $dst,$shift" %}
9117   opcode(0xD1, 0x7);  /* D1 /7 */
9118   ins_encode( OpcP, RMopc_Mem(secondary,dst) );
9119   ins_pipe( ialu_mem_imm );
9120 %}
9121 
9122 // Arithmetic Shift Right by 8-bit immediate
9123 instruct sarI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
9124   match(Set dst (RShiftI dst shift));
9125   effect(KILL cr);
9126 
9127   size(3);
9128   format %{ "SAR    $dst,$shift" %}
9129   opcode(0xC1, 0x7);  /* C1 /7 ib */
9130   ins_encode( RegOpcImm( dst, shift ) );
9131   ins_pipe( ialu_mem_imm );
9132 %}
9133 
9134 // Arithmetic Shift Right by 8-bit immediate
9135 instruct sarI_mem_imm(memory dst, immI8 shift, eFlagsReg cr) %{
9136   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
9137   effect(KILL cr);
9138 
9139   format %{ "SAR    $dst,$shift" %}
9140   opcode(0xC1, 0x7);  /* C1 /7 ib */
9141   ins_encode( OpcP, RMopc_Mem(secondary, dst ), Con8or32( shift ) );
9142   ins_pipe( ialu_mem_imm );
9143 %}
9144 
9145 // Arithmetic Shift Right by variable
9146 instruct sarI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
9147   match(Set dst (RShiftI dst shift));
9148   effect(KILL cr);
9149 
9150   size(2);
9151   format %{ "SAR    $dst,$shift" %}
9152   opcode(0xD3, 0x7);  /* D3 /7 */
9153   ins_encode( OpcP, RegOpc( dst ) );
9154   ins_pipe( ialu_reg_reg );
9155 %}
9156 
9157 // Logical shift right by one
9158 instruct shrI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9159   match(Set dst (URShiftI dst shift));
9160   effect(KILL cr);
9161 
9162   size(2);
9163   format %{ "SHR    $dst,$shift" %}
9164   opcode(0xD1, 0x5);  /* D1 /5 */
9165   ins_encode( OpcP, RegOpc( dst ) );
9166   ins_pipe( ialu_reg );
9167 %}
9168 
9169 // Logical Shift Right by 8-bit immediate
9170 instruct shrI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
9171   match(Set dst (URShiftI dst shift));
9172   effect(KILL cr);
9173 
9174   size(3);
9175   format %{ "SHR    $dst,$shift" %}
9176   opcode(0xC1, 0x5);  /* C1 /5 ib */
9177   ins_encode( RegOpcImm( dst, shift) );
9178   ins_pipe( ialu_reg );
9179 %}
9180 
9181 
9182 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9183 // This idiom is used by the compiler for the i2b bytecode.
9184 instruct i2b(eRegI dst, xRegI src, immI_24 twentyfour) %{
9185   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9186 
9187   size(3);
9188   format %{ "MOVSX  $dst,$src :8" %}
9189   ins_encode %{
9190     __ movsbl($dst$$Register, $src$$Register);
9191   %}
9192   ins_pipe(ialu_reg_reg);
9193 %}
9194 
9195 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9196 // This idiom is used by the compiler the i2s bytecode.
9197 instruct i2s(eRegI dst, xRegI src, immI_16 sixteen) %{
9198   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9199 
9200   size(3);
9201   format %{ "MOVSX  $dst,$src :16" %}
9202   ins_encode %{
9203     __ movswl($dst$$Register, $src$$Register);
9204   %}
9205   ins_pipe(ialu_reg_reg);
9206 %}
9207 
9208 
9209 // Logical Shift Right by variable
9210 instruct shrI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
9211   match(Set dst (URShiftI dst shift));
9212   effect(KILL cr);
9213 
9214   size(2);
9215   format %{ "SHR    $dst,$shift" %}
9216   opcode(0xD3, 0x5);  /* D3 /5 */
9217   ins_encode( OpcP, RegOpc( dst ) );
9218   ins_pipe( ialu_reg_reg );
9219 %}
9220 
9221 
9222 //----------Logical Instructions-----------------------------------------------
9223 //----------Integer Logical Instructions---------------------------------------
9224 // And Instructions
9225 // And Register with Register
9226 instruct andI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
9227   match(Set dst (AndI dst src));
9228   effect(KILL cr);
9229 
9230   size(2);
9231   format %{ "AND    $dst,$src" %}
9232   opcode(0x23);
9233   ins_encode( OpcP, RegReg( dst, src) );
9234   ins_pipe( ialu_reg_reg );
9235 %}
9236 
9237 // And Register with Immediate
9238 instruct andI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
9239   match(Set dst (AndI dst src));
9240   effect(KILL cr);
9241 
9242   format %{ "AND    $dst,$src" %}
9243   opcode(0x81,0x04);  /* Opcode 81 /4 */
9244   // ins_encode( RegImm( dst, src) );
9245   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
9246   ins_pipe( ialu_reg );
9247 %}
9248 
9249 // And Register with Memory
9250 instruct andI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
9251   match(Set dst (AndI dst (LoadI src)));
9252   effect(KILL cr);
9253 
9254   ins_cost(125);
9255   format %{ "AND    $dst,$src" %}
9256   opcode(0x23);
9257   ins_encode( OpcP, RegMem( dst, src) );
9258   ins_pipe( ialu_reg_mem );
9259 %}
9260 
9261 // And Memory with Register
9262 instruct andI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9263   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9264   effect(KILL cr);
9265 
9266   ins_cost(150);
9267   format %{ "AND    $dst,$src" %}
9268   opcode(0x21);  /* Opcode 21 /r */
9269   ins_encode( OpcP, RegMem( src, dst ) );
9270   ins_pipe( ialu_mem_reg );
9271 %}
9272 
9273 // And Memory with Immediate
9274 instruct andI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9275   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9276   effect(KILL cr);
9277 
9278   ins_cost(125);
9279   format %{ "AND    $dst,$src" %}
9280   opcode(0x81, 0x4);  /* Opcode 81 /4 id */
9281   // ins_encode( MemImm( dst, src) );
9282   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9283   ins_pipe( ialu_mem_imm );
9284 %}
9285 
9286 // Or Instructions
9287 // Or Register with Register
9288 instruct orI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
9289   match(Set dst (OrI dst src));
9290   effect(KILL cr);
9291 
9292   size(2);
9293   format %{ "OR     $dst,$src" %}
9294   opcode(0x0B);
9295   ins_encode( OpcP, RegReg( dst, src) );
9296   ins_pipe( ialu_reg_reg );
9297 %}
9298 
9299 instruct orI_eReg_castP2X(eRegI dst, eRegP src, eFlagsReg cr) %{
9300   match(Set dst (OrI dst (CastP2X src)));
9301   effect(KILL cr);
9302 
9303   size(2);
9304   format %{ "OR     $dst,$src" %}
9305   opcode(0x0B);
9306   ins_encode( OpcP, RegReg( dst, src) );
9307   ins_pipe( ialu_reg_reg );
9308 %}
9309 
9310 
9311 // Or Register with Immediate
9312 instruct orI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
9313   match(Set dst (OrI dst src));
9314   effect(KILL cr);
9315 
9316   format %{ "OR     $dst,$src" %}
9317   opcode(0x81,0x01);  /* Opcode 81 /1 id */
9318   // ins_encode( RegImm( dst, src) );
9319   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
9320   ins_pipe( ialu_reg );
9321 %}
9322 
9323 // Or Register with Memory
9324 instruct orI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
9325   match(Set dst (OrI dst (LoadI src)));
9326   effect(KILL cr);
9327 
9328   ins_cost(125);
9329   format %{ "OR     $dst,$src" %}
9330   opcode(0x0B);
9331   ins_encode( OpcP, RegMem( dst, src) );
9332   ins_pipe( ialu_reg_mem );
9333 %}
9334 
9335 // Or Memory with Register
9336 instruct orI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9337   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9338   effect(KILL cr);
9339 
9340   ins_cost(150);
9341   format %{ "OR     $dst,$src" %}
9342   opcode(0x09);  /* Opcode 09 /r */
9343   ins_encode( OpcP, RegMem( src, dst ) );
9344   ins_pipe( ialu_mem_reg );
9345 %}
9346 
9347 // Or Memory with Immediate
9348 instruct orI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9349   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9350   effect(KILL cr);
9351 
9352   ins_cost(125);
9353   format %{ "OR     $dst,$src" %}
9354   opcode(0x81,0x1);  /* Opcode 81 /1 id */
9355   // ins_encode( MemImm( dst, src) );
9356   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9357   ins_pipe( ialu_mem_imm );
9358 %}
9359 
9360 // ROL/ROR
9361 // ROL expand
9362 instruct rolI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9363   effect(USE_DEF dst, USE shift, KILL cr);
9364 
9365   format %{ "ROL    $dst, $shift" %}
9366   opcode(0xD1, 0x0); /* Opcode D1 /0 */
9367   ins_encode( OpcP, RegOpc( dst ));
9368   ins_pipe( ialu_reg );
9369 %}
9370 
9371 instruct rolI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
9372   effect(USE_DEF dst, USE shift, KILL cr);
9373 
9374   format %{ "ROL    $dst, $shift" %}
9375   opcode(0xC1, 0x0); /*Opcode /C1  /0  */
9376   ins_encode( RegOpcImm(dst, shift) );
9377   ins_pipe(ialu_reg);
9378 %}
9379 
9380 instruct rolI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr) %{
9381   effect(USE_DEF dst, USE shift, KILL cr);
9382 
9383   format %{ "ROL    $dst, $shift" %}
9384   opcode(0xD3, 0x0);    /* Opcode D3 /0 */
9385   ins_encode(OpcP, RegOpc(dst));
9386   ins_pipe( ialu_reg_reg );
9387 %}
9388 // end of ROL expand
9389 
9390 // ROL 32bit by one once
9391 instruct rolI_eReg_i1(eRegI dst, immI1 lshift, immI_M1 rshift, eFlagsReg cr) %{
9392   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9393 
9394   expand %{
9395     rolI_eReg_imm1(dst, lshift, cr);
9396   %}
9397 %}
9398 
9399 // ROL 32bit var by imm8 once
9400 instruct rolI_eReg_i8(eRegI dst, immI8 lshift, immI8 rshift, eFlagsReg cr) %{
9401   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9402   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9403 
9404   expand %{
9405     rolI_eReg_imm8(dst, lshift, cr);
9406   %}
9407 %}
9408 
9409 // ROL 32bit var by var once
9410 instruct rolI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
9411   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9412 
9413   expand %{
9414     rolI_eReg_CL(dst, shift, cr);
9415   %}
9416 %}
9417 
9418 // ROL 32bit var by var once
9419 instruct rolI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
9420   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9421 
9422   expand %{
9423     rolI_eReg_CL(dst, shift, cr);
9424   %}
9425 %}
9426 
9427 // ROR expand
9428 instruct rorI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9429   effect(USE_DEF dst, USE shift, KILL cr);
9430 
9431   format %{ "ROR    $dst, $shift" %}
9432   opcode(0xD1,0x1);  /* Opcode D1 /1 */
9433   ins_encode( OpcP, RegOpc( dst ) );
9434   ins_pipe( ialu_reg );
9435 %}
9436 
9437 instruct rorI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
9438   effect (USE_DEF dst, USE shift, KILL cr);
9439 
9440   format %{ "ROR    $dst, $shift" %}
9441   opcode(0xC1, 0x1); /* Opcode /C1 /1 ib */
9442   ins_encode( RegOpcImm(dst, shift) );
9443   ins_pipe( ialu_reg );
9444 %}
9445 
9446 instruct rorI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr)%{
9447   effect(USE_DEF dst, USE shift, KILL cr);
9448 
9449   format %{ "ROR    $dst, $shift" %}
9450   opcode(0xD3, 0x1);    /* Opcode D3 /1 */
9451   ins_encode(OpcP, RegOpc(dst));
9452   ins_pipe( ialu_reg_reg );
9453 %}
9454 // end of ROR expand
9455 
9456 // ROR right once
9457 instruct rorI_eReg_i1(eRegI dst, immI1 rshift, immI_M1 lshift, eFlagsReg cr) %{
9458   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9459 
9460   expand %{
9461     rorI_eReg_imm1(dst, rshift, cr);
9462   %}
9463 %}
9464 
9465 // ROR 32bit by immI8 once
9466 instruct rorI_eReg_i8(eRegI dst, immI8 rshift, immI8 lshift, eFlagsReg cr) %{
9467   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9468   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9469 
9470   expand %{
9471     rorI_eReg_imm8(dst, rshift, cr);
9472   %}
9473 %}
9474 
9475 // ROR 32bit var by var once
9476 instruct rorI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
9477   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9478 
9479   expand %{
9480     rorI_eReg_CL(dst, shift, cr);
9481   %}
9482 %}
9483 
9484 // ROR 32bit var by var once
9485 instruct rorI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
9486   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9487 
9488   expand %{
9489     rorI_eReg_CL(dst, shift, cr);
9490   %}
9491 %}
9492 
9493 // Xor Instructions
9494 // Xor Register with Register
9495 instruct xorI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
9496   match(Set dst (XorI dst src));
9497   effect(KILL cr);
9498 
9499   size(2);
9500   format %{ "XOR    $dst,$src" %}
9501   opcode(0x33);
9502   ins_encode( OpcP, RegReg( dst, src) );
9503   ins_pipe( ialu_reg_reg );
9504 %}
9505 
9506 // Xor Register with Immediate -1
9507 instruct xorI_eReg_im1(eRegI dst, immI_M1 imm) %{
9508   match(Set dst (XorI dst imm));  
9509 
9510   size(2);
9511   format %{ "NOT    $dst" %}  
9512   ins_encode %{
9513      __ notl($dst$$Register);
9514   %}
9515   ins_pipe( ialu_reg );
9516 %}
9517 
9518 // Xor Register with Immediate
9519 instruct xorI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
9520   match(Set dst (XorI dst src));
9521   effect(KILL cr);
9522 
9523   format %{ "XOR    $dst,$src" %}
9524   opcode(0x81,0x06);  /* Opcode 81 /6 id */
9525   // ins_encode( RegImm( dst, src) );
9526   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
9527   ins_pipe( ialu_reg );
9528 %}
9529 
9530 // Xor Register with Memory
9531 instruct xorI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
9532   match(Set dst (XorI dst (LoadI src)));
9533   effect(KILL cr);
9534 
9535   ins_cost(125);
9536   format %{ "XOR    $dst,$src" %}
9537   opcode(0x33);
9538   ins_encode( OpcP, RegMem(dst, src) );
9539   ins_pipe( ialu_reg_mem );
9540 %}
9541 
9542 // Xor Memory with Register
9543 instruct xorI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9544   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9545   effect(KILL cr);
9546 
9547   ins_cost(150);
9548   format %{ "XOR    $dst,$src" %}
9549   opcode(0x31);  /* Opcode 31 /r */
9550   ins_encode( OpcP, RegMem( src, dst ) );
9551   ins_pipe( ialu_mem_reg );
9552 %}
9553 
9554 // Xor Memory with Immediate
9555 instruct xorI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9556   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9557   effect(KILL cr);
9558 
9559   ins_cost(125);
9560   format %{ "XOR    $dst,$src" %}
9561   opcode(0x81,0x6);  /* Opcode 81 /6 id */
9562   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9563   ins_pipe( ialu_mem_imm );
9564 %}
9565 
9566 //----------Convert Int to Boolean---------------------------------------------
9567 
9568 instruct movI_nocopy(eRegI dst, eRegI src) %{
9569   effect( DEF dst, USE src );
9570   format %{ "MOV    $dst,$src" %}
9571   ins_encode( enc_Copy( dst, src) );
9572   ins_pipe( ialu_reg_reg );
9573 %}
9574 
9575 instruct ci2b( eRegI dst, eRegI src, eFlagsReg cr ) %{
9576   effect( USE_DEF dst, USE src, KILL cr );
9577 
9578   size(4);
9579   format %{ "NEG    $dst\n\t"
9580             "ADC    $dst,$src" %}
9581   ins_encode( neg_reg(dst),
9582               OpcRegReg(0x13,dst,src) );
9583   ins_pipe( ialu_reg_reg_long );
9584 %}
9585 
9586 instruct convI2B( eRegI dst, eRegI src, eFlagsReg cr ) %{
9587   match(Set dst (Conv2B src));
9588 
9589   expand %{
9590     movI_nocopy(dst,src);
9591     ci2b(dst,src,cr);
9592   %}
9593 %}
9594 
9595 instruct movP_nocopy(eRegI dst, eRegP src) %{
9596   effect( DEF dst, USE src );
9597   format %{ "MOV    $dst,$src" %}
9598   ins_encode( enc_Copy( dst, src) );
9599   ins_pipe( ialu_reg_reg );
9600 %}
9601 
9602 instruct cp2b( eRegI dst, eRegP src, eFlagsReg cr ) %{
9603   effect( USE_DEF dst, USE src, KILL cr );
9604   format %{ "NEG    $dst\n\t"
9605             "ADC    $dst,$src" %}
9606   ins_encode( neg_reg(dst),
9607               OpcRegReg(0x13,dst,src) );
9608   ins_pipe( ialu_reg_reg_long );
9609 %}
9610 
9611 instruct convP2B( eRegI dst, eRegP src, eFlagsReg cr ) %{
9612   match(Set dst (Conv2B src));
9613 
9614   expand %{
9615     movP_nocopy(dst,src);
9616     cp2b(dst,src,cr);
9617   %}
9618 %}
9619 
9620 instruct cmpLTMask( eCXRegI dst, ncxRegI p, ncxRegI q, eFlagsReg cr ) %{
9621   match(Set dst (CmpLTMask p q));
9622   effect( KILL cr );
9623   ins_cost(400);
9624 
9625   // SETlt can only use low byte of EAX,EBX, ECX, or EDX as destination
9626   format %{ "XOR    $dst,$dst\n\t"
9627             "CMP    $p,$q\n\t"
9628             "SETlt  $dst\n\t"
9629             "NEG    $dst" %}
9630   ins_encode( OpcRegReg(0x33,dst,dst),
9631               OpcRegReg(0x3B,p,q),
9632               setLT_reg(dst), neg_reg(dst) );
9633   ins_pipe( pipe_slow );
9634 %}
9635 
9636 instruct cmpLTMask0( eRegI dst, immI0 zero, eFlagsReg cr ) %{
9637   match(Set dst (CmpLTMask dst zero));
9638   effect( DEF dst, KILL cr );
9639   ins_cost(100);
9640 
9641   format %{ "SAR    $dst,31" %}
9642   opcode(0xC1, 0x7);  /* C1 /7 ib */
9643   ins_encode( RegOpcImm( dst, 0x1F ) );
9644   ins_pipe( ialu_reg );
9645 %}
9646 
9647 
9648 instruct cadd_cmpLTMask( ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp, eFlagsReg cr ) %{
9649   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9650   effect( KILL tmp, KILL cr );
9651   ins_cost(400);
9652   // annoyingly, $tmp has no edges so you cant ask for it in
9653   // any format or encoding
9654   format %{ "SUB    $p,$q\n\t"
9655             "SBB    ECX,ECX\n\t"
9656             "AND    ECX,$y\n\t"
9657             "ADD    $p,ECX" %}
9658   ins_encode( enc_cmpLTP(p,q,y,tmp) );
9659   ins_pipe( pipe_cmplt );
9660 %}
9661 
9662 /* If I enable this, I encourage spilling in the inner loop of compress.
9663 instruct cadd_cmpLTMask_mem( ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr ) %{
9664   match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
9665   effect( USE_KILL tmp, KILL cr );
9666   ins_cost(400);
9667 
9668   format %{ "SUB    $p,$q\n\t"
9669             "SBB    ECX,ECX\n\t"
9670             "AND    ECX,$y\n\t"
9671             "ADD    $p,ECX" %}
9672   ins_encode( enc_cmpLTP_mem(p,q,y,tmp) );
9673 %}
9674 */
9675 
9676 //----------Long Instructions------------------------------------------------
9677 // Add Long Register with Register
9678 instruct addL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9679   match(Set dst (AddL dst src));
9680   effect(KILL cr);
9681   ins_cost(200);
9682   format %{ "ADD    $dst.lo,$src.lo\n\t"
9683             "ADC    $dst.hi,$src.hi" %}
9684   opcode(0x03, 0x13);
9685   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
9686   ins_pipe( ialu_reg_reg_long );
9687 %}
9688 
9689 // Add Long Register with Immediate
9690 instruct addL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9691   match(Set dst (AddL dst src));
9692   effect(KILL cr);
9693   format %{ "ADD    $dst.lo,$src.lo\n\t"
9694             "ADC    $dst.hi,$src.hi" %}
9695   opcode(0x81,0x00,0x02);  /* Opcode 81 /0, 81 /2 */
9696   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9697   ins_pipe( ialu_reg_long );
9698 %}
9699 
9700 // Add Long Register with Memory
9701 instruct addL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9702   match(Set dst (AddL dst (LoadL mem)));
9703   effect(KILL cr);
9704   ins_cost(125);
9705   format %{ "ADD    $dst.lo,$mem\n\t"
9706             "ADC    $dst.hi,$mem+4" %}
9707   opcode(0x03, 0x13);
9708   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9709   ins_pipe( ialu_reg_long_mem );
9710 %}
9711 
9712 // Subtract Long Register with Register.
9713 instruct subL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9714   match(Set dst (SubL dst src));
9715   effect(KILL cr);
9716   ins_cost(200);
9717   format %{ "SUB    $dst.lo,$src.lo\n\t"
9718             "SBB    $dst.hi,$src.hi" %}
9719   opcode(0x2B, 0x1B);
9720   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
9721   ins_pipe( ialu_reg_reg_long );
9722 %}
9723 
9724 // Subtract Long Register with Immediate
9725 instruct subL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9726   match(Set dst (SubL dst src));
9727   effect(KILL cr);
9728   format %{ "SUB    $dst.lo,$src.lo\n\t"
9729             "SBB    $dst.hi,$src.hi" %}
9730   opcode(0x81,0x05,0x03);  /* Opcode 81 /5, 81 /3 */
9731   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9732   ins_pipe( ialu_reg_long );
9733 %}
9734 
9735 // Subtract Long Register with Memory
9736 instruct subL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9737   match(Set dst (SubL dst (LoadL mem)));
9738   effect(KILL cr);
9739   ins_cost(125);
9740   format %{ "SUB    $dst.lo,$mem\n\t"
9741             "SBB    $dst.hi,$mem+4" %}
9742   opcode(0x2B, 0x1B);
9743   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9744   ins_pipe( ialu_reg_long_mem );
9745 %}
9746 
9747 instruct negL_eReg(eRegL dst, immL0 zero, eFlagsReg cr) %{
9748   match(Set dst (SubL zero dst));
9749   effect(KILL cr);
9750   ins_cost(300);
9751   format %{ "NEG    $dst.hi\n\tNEG    $dst.lo\n\tSBB    $dst.hi,0" %}
9752   ins_encode( neg_long(dst) );
9753   ins_pipe( ialu_reg_reg_long );
9754 %}
9755 
9756 // And Long Register with Register
9757 instruct andL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9758   match(Set dst (AndL dst src));
9759   effect(KILL cr);
9760   format %{ "AND    $dst.lo,$src.lo\n\t"
9761             "AND    $dst.hi,$src.hi" %}
9762   opcode(0x23,0x23);
9763   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9764   ins_pipe( ialu_reg_reg_long );
9765 %}
9766 
9767 // And Long Register with Immediate
9768 instruct andL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9769   match(Set dst (AndL dst src));
9770   effect(KILL cr);
9771   format %{ "AND    $dst.lo,$src.lo\n\t"
9772             "AND    $dst.hi,$src.hi" %}
9773   opcode(0x81,0x04,0x04);  /* Opcode 81 /4, 81 /4 */
9774   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9775   ins_pipe( ialu_reg_long );
9776 %}
9777 
9778 // And Long Register with Memory
9779 instruct andL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9780   match(Set dst (AndL dst (LoadL mem)));
9781   effect(KILL cr);
9782   ins_cost(125);
9783   format %{ "AND    $dst.lo,$mem\n\t"
9784             "AND    $dst.hi,$mem+4" %}
9785   opcode(0x23, 0x23);
9786   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9787   ins_pipe( ialu_reg_long_mem );
9788 %}
9789 
9790 // Or Long Register with Register
9791 instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9792   match(Set dst (OrL dst src));
9793   effect(KILL cr);
9794   format %{ "OR     $dst.lo,$src.lo\n\t"
9795             "OR     $dst.hi,$src.hi" %}
9796   opcode(0x0B,0x0B);
9797   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9798   ins_pipe( ialu_reg_reg_long );
9799 %}
9800 
9801 // Or Long Register with Immediate
9802 instruct orl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9803   match(Set dst (OrL dst src));
9804   effect(KILL cr);
9805   format %{ "OR     $dst.lo,$src.lo\n\t"
9806             "OR     $dst.hi,$src.hi" %}
9807   opcode(0x81,0x01,0x01);  /* Opcode 81 /1, 81 /1 */
9808   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9809   ins_pipe( ialu_reg_long );
9810 %}
9811 
9812 // Or Long Register with Memory
9813 instruct orl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9814   match(Set dst (OrL dst (LoadL mem)));
9815   effect(KILL cr);
9816   ins_cost(125);
9817   format %{ "OR     $dst.lo,$mem\n\t"
9818             "OR     $dst.hi,$mem+4" %}
9819   opcode(0x0B,0x0B);
9820   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9821   ins_pipe( ialu_reg_long_mem );
9822 %}
9823 
9824 // Xor Long Register with Register
9825 instruct xorl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9826   match(Set dst (XorL dst src));
9827   effect(KILL cr);
9828   format %{ "XOR    $dst.lo,$src.lo\n\t"
9829             "XOR    $dst.hi,$src.hi" %}
9830   opcode(0x33,0x33);
9831   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9832   ins_pipe( ialu_reg_reg_long );
9833 %}
9834 
9835 // Xor Long Register with Immediate -1
9836 instruct xorl_eReg_im1(eRegL dst, immL_M1 imm) %{
9837   match(Set dst (XorL dst imm));  
9838   format %{ "NOT    $dst.lo\n\t"
9839             "NOT    $dst.hi" %}
9840   ins_encode %{
9841      __ notl($dst$$Register);
9842      __ notl(HIGH_FROM_LOW($dst$$Register));
9843   %}
9844   ins_pipe( ialu_reg_long );
9845 %}
9846 
9847 // Xor Long Register with Immediate
9848 instruct xorl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9849   match(Set dst (XorL dst src));
9850   effect(KILL cr);
9851   format %{ "XOR    $dst.lo,$src.lo\n\t"
9852             "XOR    $dst.hi,$src.hi" %}
9853   opcode(0x81,0x06,0x06);  /* Opcode 81 /6, 81 /6 */
9854   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9855   ins_pipe( ialu_reg_long );
9856 %}
9857 
9858 // Xor Long Register with Memory
9859 instruct xorl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9860   match(Set dst (XorL dst (LoadL mem)));
9861   effect(KILL cr);
9862   ins_cost(125);
9863   format %{ "XOR    $dst.lo,$mem\n\t"
9864             "XOR    $dst.hi,$mem+4" %}
9865   opcode(0x33,0x33);
9866   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9867   ins_pipe( ialu_reg_long_mem );
9868 %}
9869 
9870 // Shift Left Long by 1
9871 instruct shlL_eReg_1(eRegL dst, immI_1 cnt, eFlagsReg cr) %{
9872   predicate(UseNewLongLShift);
9873   match(Set dst (LShiftL dst cnt));
9874   effect(KILL cr);
9875   ins_cost(100);
9876   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9877             "ADC    $dst.hi,$dst.hi" %}
9878   ins_encode %{
9879     __ addl($dst$$Register,$dst$$Register);
9880     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9881   %}
9882   ins_pipe( ialu_reg_long );
9883 %}
9884 
9885 // Shift Left Long by 2
9886 instruct shlL_eReg_2(eRegL dst, immI_2 cnt, eFlagsReg cr) %{
9887   predicate(UseNewLongLShift);
9888   match(Set dst (LShiftL dst cnt));
9889   effect(KILL cr);
9890   ins_cost(100);
9891   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9892             "ADC    $dst.hi,$dst.hi\n\t" 
9893             "ADD    $dst.lo,$dst.lo\n\t"
9894             "ADC    $dst.hi,$dst.hi" %}
9895   ins_encode %{
9896     __ addl($dst$$Register,$dst$$Register);
9897     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9898     __ addl($dst$$Register,$dst$$Register);
9899     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9900   %}
9901   ins_pipe( ialu_reg_long );
9902 %}
9903 
9904 // Shift Left Long by 3
9905 instruct shlL_eReg_3(eRegL dst, immI_3 cnt, eFlagsReg cr) %{
9906   predicate(UseNewLongLShift);
9907   match(Set dst (LShiftL dst cnt));
9908   effect(KILL cr);
9909   ins_cost(100);
9910   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9911             "ADC    $dst.hi,$dst.hi\n\t" 
9912             "ADD    $dst.lo,$dst.lo\n\t"
9913             "ADC    $dst.hi,$dst.hi\n\t" 
9914             "ADD    $dst.lo,$dst.lo\n\t"
9915             "ADC    $dst.hi,$dst.hi" %}
9916   ins_encode %{
9917     __ addl($dst$$Register,$dst$$Register);
9918     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9919     __ addl($dst$$Register,$dst$$Register);
9920     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9921     __ addl($dst$$Register,$dst$$Register);
9922     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9923   %}
9924   ins_pipe( ialu_reg_long );
9925 %}
9926 
9927 // Shift Left Long by 1-31
9928 instruct shlL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9929   match(Set dst (LShiftL dst cnt));
9930   effect(KILL cr);
9931   ins_cost(200);
9932   format %{ "SHLD   $dst.hi,$dst.lo,$cnt\n\t"
9933             "SHL    $dst.lo,$cnt" %}
9934   opcode(0xC1, 0x4, 0xA4);  /* 0F/A4, then C1 /4 ib */
9935   ins_encode( move_long_small_shift(dst,cnt) );
9936   ins_pipe( ialu_reg_long );
9937 %}
9938 
9939 // Shift Left Long by 32-63
9940 instruct shlL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9941   match(Set dst (LShiftL dst cnt));
9942   effect(KILL cr);
9943   ins_cost(300);
9944   format %{ "MOV    $dst.hi,$dst.lo\n"
9945           "\tSHL    $dst.hi,$cnt-32\n"
9946           "\tXOR    $dst.lo,$dst.lo" %}
9947   opcode(0xC1, 0x4);  /* C1 /4 ib */
9948   ins_encode( move_long_big_shift_clr(dst,cnt) );
9949   ins_pipe( ialu_reg_long );
9950 %}
9951 
9952 // Shift Left Long by variable
9953 instruct salL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9954   match(Set dst (LShiftL dst shift));
9955   effect(KILL cr);
9956   ins_cost(500+200);
9957   size(17);
9958   format %{ "TEST   $shift,32\n\t"
9959             "JEQ,s  small\n\t"
9960             "MOV    $dst.hi,$dst.lo\n\t"
9961             "XOR    $dst.lo,$dst.lo\n"
9962     "small:\tSHLD   $dst.hi,$dst.lo,$shift\n\t"
9963             "SHL    $dst.lo,$shift" %}
9964   ins_encode( shift_left_long( dst, shift ) );
9965   ins_pipe( pipe_slow );
9966 %}
9967 
9968 // Shift Right Long by 1-31
9969 instruct shrL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9970   match(Set dst (URShiftL dst cnt));
9971   effect(KILL cr);
9972   ins_cost(200);
9973   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
9974             "SHR    $dst.hi,$cnt" %}
9975   opcode(0xC1, 0x5, 0xAC);  /* 0F/AC, then C1 /5 ib */
9976   ins_encode( move_long_small_shift(dst,cnt) );
9977   ins_pipe( ialu_reg_long );
9978 %}
9979 
9980 // Shift Right Long by 32-63
9981 instruct shrL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9982   match(Set dst (URShiftL dst cnt));
9983   effect(KILL cr);
9984   ins_cost(300);
9985   format %{ "MOV    $dst.lo,$dst.hi\n"
9986           "\tSHR    $dst.lo,$cnt-32\n"
9987           "\tXOR    $dst.hi,$dst.hi" %}
9988   opcode(0xC1, 0x5);  /* C1 /5 ib */
9989   ins_encode( move_long_big_shift_clr(dst,cnt) );
9990   ins_pipe( ialu_reg_long );
9991 %}
9992 
9993 // Shift Right Long by variable
9994 instruct shrL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9995   match(Set dst (URShiftL dst shift));
9996   effect(KILL cr);
9997   ins_cost(600);
9998   size(17);
9999   format %{ "TEST   $shift,32\n\t"
10000             "JEQ,s  small\n\t"
10001             "MOV    $dst.lo,$dst.hi\n\t"
10002             "XOR    $dst.hi,$dst.hi\n"
10003     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
10004             "SHR    $dst.hi,$shift" %}
10005   ins_encode( shift_right_long( dst, shift ) );
10006   ins_pipe( pipe_slow );
10007 %}
10008 
10009 // Shift Right Long by 1-31
10010 instruct sarL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
10011   match(Set dst (RShiftL dst cnt));
10012   effect(KILL cr);
10013   ins_cost(200);
10014   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
10015             "SAR    $dst.hi,$cnt" %}
10016   opcode(0xC1, 0x7, 0xAC);  /* 0F/AC, then C1 /7 ib */
10017   ins_encode( move_long_small_shift(dst,cnt) );
10018   ins_pipe( ialu_reg_long );
10019 %}
10020 
10021 // Shift Right Long by 32-63
10022 instruct sarL_eReg_32_63( eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
10023   match(Set dst (RShiftL dst cnt));
10024   effect(KILL cr);
10025   ins_cost(300);
10026   format %{ "MOV    $dst.lo,$dst.hi\n"
10027           "\tSAR    $dst.lo,$cnt-32\n"
10028           "\tSAR    $dst.hi,31" %}
10029   opcode(0xC1, 0x7);  /* C1 /7 ib */
10030   ins_encode( move_long_big_shift_sign(dst,cnt) );
10031   ins_pipe( ialu_reg_long );
10032 %}
10033 
10034 // Shift Right arithmetic Long by variable
10035 instruct sarL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
10036   match(Set dst (RShiftL dst shift));
10037   effect(KILL cr);
10038   ins_cost(600);
10039   size(18);
10040   format %{ "TEST   $shift,32\n\t"
10041             "JEQ,s  small\n\t"
10042             "MOV    $dst.lo,$dst.hi\n\t"
10043             "SAR    $dst.hi,31\n"
10044     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
10045             "SAR    $dst.hi,$shift" %}
10046   ins_encode( shift_right_arith_long( dst, shift ) );
10047   ins_pipe( pipe_slow );
10048 %}
10049 
10050 
10051 //----------Double Instructions------------------------------------------------
10052 // Double Math
10053 
10054 // Compare & branch
10055 
10056 // P6 version of float compare, sets condition codes in EFLAGS
10057 instruct cmpD_cc_P6(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
10058   predicate(VM_Version::supports_cmov() && UseSSE <=1);
10059   match(Set cr (CmpD src1 src2));
10060   effect(KILL rax);
10061   ins_cost(150);
10062   format %{ "FLD    $src1\n\t"
10063             "FUCOMIP ST,$src2  // P6 instruction\n\t"
10064             "JNP    exit\n\t"
10065             "MOV    ah,1       // saw a NaN, set CF\n\t"
10066             "SAHF\n"
10067      "exit:\tNOP               // avoid branch to branch" %}
10068   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10069   ins_encode( Push_Reg_D(src1),
10070               OpcP, RegOpc(src2),
10071               cmpF_P6_fixup );
10072   ins_pipe( pipe_slow );
10073 %}
10074 
10075 instruct cmpD_cc_P6CF(eFlagsRegUCF cr, regD src1, regD src2) %{
10076   predicate(VM_Version::supports_cmov() && UseSSE <=1);
10077   match(Set cr (CmpD src1 src2));
10078   ins_cost(150);
10079   format %{ "FLD    $src1\n\t"
10080             "FUCOMIP ST,$src2  // P6 instruction" %}
10081   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10082   ins_encode( Push_Reg_D(src1),
10083               OpcP, RegOpc(src2));
10084   ins_pipe( pipe_slow );
10085 %}
10086 
10087 // Compare & branch
10088 instruct cmpD_cc(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
10089   predicate(UseSSE<=1);
10090   match(Set cr (CmpD src1 src2));
10091   effect(KILL rax);
10092   ins_cost(200);
10093   format %{ "FLD    $src1\n\t"
10094             "FCOMp  $src2\n\t"
10095             "FNSTSW AX\n\t"
10096             "TEST   AX,0x400\n\t"
10097             "JZ,s   flags\n\t"
10098             "MOV    AH,1\t# unordered treat as LT\n"
10099     "flags:\tSAHF" %}
10100   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10101   ins_encode( Push_Reg_D(src1),
10102               OpcP, RegOpc(src2),
10103               fpu_flags);
10104   ins_pipe( pipe_slow );
10105 %}
10106 
10107 // Compare vs zero into -1,0,1
10108 instruct cmpD_0(eRegI dst, regD src1, immD0 zero, eAXRegI rax, eFlagsReg cr) %{
10109   predicate(UseSSE<=1);
10110   match(Set dst (CmpD3 src1 zero));
10111   effect(KILL cr, KILL rax);
10112   ins_cost(280);
10113   format %{ "FTSTD  $dst,$src1" %}
10114   opcode(0xE4, 0xD9);
10115   ins_encode( Push_Reg_D(src1),
10116               OpcS, OpcP, PopFPU,
10117               CmpF_Result(dst));
10118   ins_pipe( pipe_slow );
10119 %}
10120 
10121 // Compare into -1,0,1
10122 instruct cmpD_reg(eRegI dst, regD src1, regD src2, eAXRegI rax, eFlagsReg cr) %{
10123   predicate(UseSSE<=1);
10124   match(Set dst (CmpD3 src1 src2));
10125   effect(KILL cr, KILL rax);
10126   ins_cost(300);
10127   format %{ "FCMPD  $dst,$src1,$src2" %}
10128   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10129   ins_encode( Push_Reg_D(src1),
10130               OpcP, RegOpc(src2),
10131               CmpF_Result(dst));
10132   ins_pipe( pipe_slow );
10133 %}
10134 
10135 // float compare and set condition codes in EFLAGS by XMM regs
10136 instruct cmpXD_cc(eFlagsRegU cr, regXD dst, regXD src, eAXRegI rax) %{
10137   predicate(UseSSE>=2);
10138   match(Set cr (CmpD dst src));
10139   effect(KILL rax);
10140   ins_cost(125);
10141   format %{ "COMISD $dst,$src\n"
10142           "\tJNP    exit\n"
10143           "\tMOV    ah,1       // saw a NaN, set CF\n"
10144           "\tSAHF\n"
10145      "exit:\tNOP               // avoid branch to branch" %}
10146   opcode(0x66, 0x0F, 0x2F);
10147   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src), cmpF_P6_fixup);
10148   ins_pipe( pipe_slow );
10149 %}
10150 
10151 instruct cmpXD_ccCF(eFlagsRegUCF cr, regXD dst, regXD src) %{
10152   predicate(UseSSE>=2);
10153   match(Set cr (CmpD dst src));
10154   ins_cost(100);
10155   format %{ "COMISD $dst,$src" %}
10156   opcode(0x66, 0x0F, 0x2F);
10157   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
10158   ins_pipe( pipe_slow );
10159 %}
10160 
10161 // float compare and set condition codes in EFLAGS by XMM regs
10162 instruct cmpXD_ccmem(eFlagsRegU cr, regXD dst, memory src, eAXRegI rax) %{
10163   predicate(UseSSE>=2);
10164   match(Set cr (CmpD dst (LoadD src)));
10165   effect(KILL rax);
10166   ins_cost(145);
10167   format %{ "COMISD $dst,$src\n"
10168           "\tJNP    exit\n"
10169           "\tMOV    ah,1       // saw a NaN, set CF\n"
10170           "\tSAHF\n"
10171      "exit:\tNOP               // avoid branch to branch" %}
10172   opcode(0x66, 0x0F, 0x2F);
10173   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src), cmpF_P6_fixup);
10174   ins_pipe( pipe_slow );
10175 %}
10176 
10177 instruct cmpXD_ccmemCF(eFlagsRegUCF cr, regXD dst, memory src) %{
10178   predicate(UseSSE>=2);
10179   match(Set cr (CmpD dst (LoadD src)));
10180   ins_cost(100);
10181   format %{ "COMISD $dst,$src" %}
10182   opcode(0x66, 0x0F, 0x2F);
10183   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src));
10184   ins_pipe( pipe_slow );
10185 %}
10186 
10187 // Compare into -1,0,1 in XMM
10188 instruct cmpXD_reg(eRegI dst, regXD src1, regXD src2, eFlagsReg cr) %{
10189   predicate(UseSSE>=2);
10190   match(Set dst (CmpD3 src1 src2));
10191   effect(KILL cr);
10192   ins_cost(255);
10193   format %{ "XOR    $dst,$dst\n"
10194           "\tCOMISD $src1,$src2\n"
10195           "\tJP,s   nan\n"
10196           "\tJEQ,s  exit\n"
10197           "\tJA,s   inc\n"
10198       "nan:\tDEC    $dst\n"
10199           "\tJMP,s  exit\n"
10200       "inc:\tINC    $dst\n"
10201       "exit:"
10202                 %}
10203   opcode(0x66, 0x0F, 0x2F);
10204   ins_encode(Xor_Reg(dst), OpcP, OpcS, Opcode(tertiary), RegReg(src1, src2),
10205              CmpX_Result(dst));
10206   ins_pipe( pipe_slow );
10207 %}
10208 
10209 // Compare into -1,0,1 in XMM and memory
10210 instruct cmpXD_regmem(eRegI dst, regXD src1, memory mem, eFlagsReg cr) %{
10211   predicate(UseSSE>=2);
10212   match(Set dst (CmpD3 src1 (LoadD mem)));
10213   effect(KILL cr);
10214   ins_cost(275);
10215   format %{ "COMISD $src1,$mem\n"
10216           "\tMOV    $dst,0\t\t# do not blow flags\n"
10217           "\tJP,s   nan\n"
10218           "\tJEQ,s  exit\n"
10219           "\tJA,s   inc\n"
10220       "nan:\tDEC    $dst\n"
10221           "\tJMP,s  exit\n"
10222       "inc:\tINC    $dst\n"
10223       "exit:"
10224                 %}
10225   opcode(0x66, 0x0F, 0x2F);
10226   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(src1, mem),
10227              LdImmI(dst,0x0), CmpX_Result(dst));
10228   ins_pipe( pipe_slow );
10229 %}
10230 
10231 
10232 instruct subD_reg(regD dst, regD src) %{
10233   predicate (UseSSE <=1);
10234   match(Set dst (SubD dst src));
10235 
10236   format %{ "FLD    $src\n\t"
10237             "DSUBp  $dst,ST" %}
10238   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
10239   ins_cost(150);
10240   ins_encode( Push_Reg_D(src),
10241               OpcP, RegOpc(dst) );
10242   ins_pipe( fpu_reg_reg );
10243 %}
10244 
10245 instruct subD_reg_round(stackSlotD dst, regD src1, regD src2) %{
10246   predicate (UseSSE <=1);
10247   match(Set dst (RoundDouble (SubD src1 src2)));
10248   ins_cost(250);
10249 
10250   format %{ "FLD    $src2\n\t"
10251             "DSUB   ST,$src1\n\t"
10252             "FSTP_D $dst\t# D-round" %}
10253   opcode(0xD8, 0x5);
10254   ins_encode( Push_Reg_D(src2),
10255               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
10256   ins_pipe( fpu_mem_reg_reg );
10257 %}
10258 
10259 
10260 instruct subD_reg_mem(regD dst, memory src) %{
10261   predicate (UseSSE <=1);
10262   match(Set dst (SubD dst (LoadD src)));
10263   ins_cost(150);
10264 
10265   format %{ "FLD    $src\n\t"
10266             "DSUBp  $dst,ST" %}
10267   opcode(0xDE, 0x5, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
10268   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10269               OpcP, RegOpc(dst) );
10270   ins_pipe( fpu_reg_mem );
10271 %}
10272 
10273 instruct absD_reg(regDPR1 dst, regDPR1 src) %{
10274   predicate (UseSSE<=1);
10275   match(Set dst (AbsD src));
10276   ins_cost(100);
10277   format %{ "FABS" %}
10278   opcode(0xE1, 0xD9);
10279   ins_encode( OpcS, OpcP );
10280   ins_pipe( fpu_reg_reg );
10281 %}
10282 
10283 instruct absXD_reg( regXD dst ) %{
10284   predicate(UseSSE>=2);
10285   match(Set dst (AbsD dst));
10286   format %{ "ANDPD  $dst,[0x7FFFFFFFFFFFFFFF]\t# ABS D by sign masking" %}
10287   ins_encode( AbsXD_encoding(dst));
10288   ins_pipe( pipe_slow );
10289 %}
10290 
10291 instruct negD_reg(regDPR1 dst, regDPR1 src) %{
10292   predicate(UseSSE<=1);
10293   match(Set dst (NegD src));
10294   ins_cost(100);
10295   format %{ "FCHS" %}
10296   opcode(0xE0, 0xD9);
10297   ins_encode( OpcS, OpcP );
10298   ins_pipe( fpu_reg_reg );
10299 %}
10300 
10301 instruct negXD_reg( regXD dst ) %{
10302   predicate(UseSSE>=2);
10303   match(Set dst (NegD dst));
10304   format %{ "XORPD  $dst,[0x8000000000000000]\t# CHS D by sign flipping" %}
10305   ins_encode %{
10306      __ xorpd($dst$$XMMRegister,
10307               ExternalAddress((address)double_signflip_pool));
10308   %}
10309   ins_pipe( pipe_slow );
10310 %}
10311 
10312 instruct addD_reg(regD dst, regD src) %{
10313   predicate(UseSSE<=1);
10314   match(Set dst (AddD dst src));
10315   format %{ "FLD    $src\n\t"
10316             "DADD   $dst,ST" %}
10317   size(4);
10318   ins_cost(150);
10319   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
10320   ins_encode( Push_Reg_D(src),
10321               OpcP, RegOpc(dst) );
10322   ins_pipe( fpu_reg_reg );
10323 %}
10324 
10325 
10326 instruct addD_reg_round(stackSlotD dst, regD src1, regD src2) %{
10327   predicate(UseSSE<=1);
10328   match(Set dst (RoundDouble (AddD src1 src2)));
10329   ins_cost(250);
10330 
10331   format %{ "FLD    $src2\n\t"
10332             "DADD   ST,$src1\n\t"
10333             "FSTP_D $dst\t# D-round" %}
10334   opcode(0xD8, 0x0); /* D8 C0+i or D8 /0*/
10335   ins_encode( Push_Reg_D(src2),
10336               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
10337   ins_pipe( fpu_mem_reg_reg );
10338 %}
10339 
10340 
10341 instruct addD_reg_mem(regD dst, memory src) %{
10342   predicate(UseSSE<=1);
10343   match(Set dst (AddD dst (LoadD src)));
10344   ins_cost(150);
10345 
10346   format %{ "FLD    $src\n\t"
10347             "DADDp  $dst,ST" %}
10348   opcode(0xDE, 0x0, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
10349   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10350               OpcP, RegOpc(dst) );
10351   ins_pipe( fpu_reg_mem );
10352 %}
10353 
10354 // add-to-memory
10355 instruct addD_mem_reg(memory dst, regD src) %{
10356   predicate(UseSSE<=1);
10357   match(Set dst (StoreD dst (RoundDouble (AddD (LoadD dst) src))));
10358   ins_cost(150);
10359 
10360   format %{ "FLD_D  $dst\n\t"
10361             "DADD   ST,$src\n\t"
10362             "FST_D  $dst" %}
10363   opcode(0xDD, 0x0);
10364   ins_encode( Opcode(0xDD), RMopc_Mem(0x00,dst),
10365               Opcode(0xD8), RegOpc(src),
10366               set_instruction_start,
10367               Opcode(0xDD), RMopc_Mem(0x03,dst) );
10368   ins_pipe( fpu_reg_mem );
10369 %}
10370 
10371 instruct addD_reg_imm1(regD dst, immD1 con) %{
10372   predicate(UseSSE<=1);
10373   match(Set dst (AddD dst con));
10374   ins_cost(125);
10375   format %{ "FLD1\n\t"
10376             "DADDp  $dst,ST" %}
10377   ins_encode %{
10378     __ fld1();
10379     __ faddp($dst$$reg);
10380   %}
10381   ins_pipe(fpu_reg);
10382 %}
10383 
10384 instruct addD_reg_imm(regD dst, immD con) %{
10385   predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
10386   match(Set dst (AddD dst con));
10387   ins_cost(200);
10388   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
10389             "DADDp  $dst,ST" %}
10390   ins_encode %{
10391     __ fld_d($constantaddress($con));
10392     __ faddp($dst$$reg);
10393   %}
10394   ins_pipe(fpu_reg_mem);
10395 %}
10396 
10397 instruct addD_reg_imm_round(stackSlotD dst, regD src, immD con) %{
10398   predicate(UseSSE<=1 && _kids[0]->_kids[1]->_leaf->getd() != 0.0 && _kids[0]->_kids[1]->_leaf->getd() != 1.0 );
10399   match(Set dst (RoundDouble (AddD src con)));
10400   ins_cost(200);
10401   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
10402             "DADD   ST,$src\n\t"
10403             "FSTP_D $dst\t# D-round" %}
10404   ins_encode %{
10405     __ fld_d($constantaddress($con));
10406     __ fadd($src$$reg);
10407     __ fstp_d(Address(rsp, $dst$$disp));
10408   %}
10409   ins_pipe(fpu_mem_reg_con);
10410 %}
10411 
10412 // Add two double precision floating point values in xmm
10413 instruct addXD_reg(regXD dst, regXD src) %{
10414   predicate(UseSSE>=2);
10415   match(Set dst (AddD dst src));
10416   format %{ "ADDSD  $dst,$src" %}
10417   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
10418   ins_pipe( pipe_slow );
10419 %}
10420 
10421 instruct addXD_imm(regXD dst, immXD con) %{
10422   predicate(UseSSE>=2);
10423   match(Set dst (AddD dst con));
10424   format %{ "ADDSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
10425   ins_encode %{
10426     __ addsd($dst$$XMMRegister, $constantaddress($con));
10427   %}
10428   ins_pipe(pipe_slow);
10429 %}
10430 
10431 instruct addXD_mem(regXD dst, memory mem) %{
10432   predicate(UseSSE>=2);
10433   match(Set dst (AddD dst (LoadD mem)));
10434   format %{ "ADDSD  $dst,$mem" %}
10435   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegMem(dst,mem));
10436   ins_pipe( pipe_slow );
10437 %}
10438 
10439 // Sub two double precision floating point values in xmm
10440 instruct subXD_reg(regXD dst, regXD src) %{
10441   predicate(UseSSE>=2);
10442   match(Set dst (SubD dst src));
10443   format %{ "SUBSD  $dst,$src" %}
10444   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
10445   ins_pipe( pipe_slow );
10446 %}
10447 
10448 instruct subXD_imm(regXD dst, immXD con) %{
10449   predicate(UseSSE>=2);
10450   match(Set dst (SubD dst con));
10451   format %{ "SUBSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
10452   ins_encode %{
10453     __ subsd($dst$$XMMRegister, $constantaddress($con));
10454   %}
10455   ins_pipe(pipe_slow);
10456 %}
10457 
10458 instruct subXD_mem(regXD dst, memory mem) %{
10459   predicate(UseSSE>=2);
10460   match(Set dst (SubD dst (LoadD mem)));
10461   format %{ "SUBSD  $dst,$mem" %}
10462   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
10463   ins_pipe( pipe_slow );
10464 %}
10465 
10466 // Mul two double precision floating point values in xmm
10467 instruct mulXD_reg(regXD dst, regXD src) %{
10468   predicate(UseSSE>=2);
10469   match(Set dst (MulD dst src));
10470   format %{ "MULSD  $dst,$src" %}
10471   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
10472   ins_pipe( pipe_slow );
10473 %}
10474 
10475 instruct mulXD_imm(regXD dst, immXD con) %{
10476   predicate(UseSSE>=2);
10477   match(Set dst (MulD dst con));
10478   format %{ "MULSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
10479   ins_encode %{
10480     __ mulsd($dst$$XMMRegister, $constantaddress($con));
10481   %}
10482   ins_pipe(pipe_slow);
10483 %}
10484 
10485 instruct mulXD_mem(regXD dst, memory mem) %{
10486   predicate(UseSSE>=2);
10487   match(Set dst (MulD dst (LoadD mem)));
10488   format %{ "MULSD  $dst,$mem" %}
10489   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
10490   ins_pipe( pipe_slow );
10491 %}
10492 
10493 // Div two double precision floating point values in xmm
10494 instruct divXD_reg(regXD dst, regXD src) %{
10495   predicate(UseSSE>=2);
10496   match(Set dst (DivD dst src));
10497   format %{ "DIVSD  $dst,$src" %}
10498   opcode(0xF2, 0x0F, 0x5E);
10499   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
10500   ins_pipe( pipe_slow );
10501 %}
10502 
10503 instruct divXD_imm(regXD dst, immXD con) %{
10504   predicate(UseSSE>=2);
10505   match(Set dst (DivD dst con));
10506   format %{ "DIVSD  $dst,[$constantaddress]\t# load from constant table: double=$con" %}
10507   ins_encode %{
10508     __ divsd($dst$$XMMRegister, $constantaddress($con));
10509   %}
10510   ins_pipe(pipe_slow);
10511 %}
10512 
10513 instruct divXD_mem(regXD dst, memory mem) %{
10514   predicate(UseSSE>=2);
10515   match(Set dst (DivD dst (LoadD mem)));
10516   format %{ "DIVSD  $dst,$mem" %}
10517   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
10518   ins_pipe( pipe_slow );
10519 %}
10520 
10521 
10522 instruct mulD_reg(regD dst, regD src) %{
10523   predicate(UseSSE<=1);
10524   match(Set dst (MulD dst src));
10525   format %{ "FLD    $src\n\t"
10526             "DMULp  $dst,ST" %}
10527   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
10528   ins_cost(150);
10529   ins_encode( Push_Reg_D(src),
10530               OpcP, RegOpc(dst) );
10531   ins_pipe( fpu_reg_reg );
10532 %}
10533 
10534 // Strict FP instruction biases argument before multiply then
10535 // biases result to avoid double rounding of subnormals.
10536 //
10537 // scale arg1 by multiplying arg1 by 2^(-15360)
10538 // load arg2
10539 // multiply scaled arg1 by arg2
10540 // rescale product by 2^(15360)
10541 //
10542 instruct strictfp_mulD_reg(regDPR1 dst, regnotDPR1 src) %{
10543   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
10544   match(Set dst (MulD dst src));
10545   ins_cost(1);   // Select this instruction for all strict FP double multiplies
10546 
10547   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
10548             "DMULp  $dst,ST\n\t"
10549             "FLD    $src\n\t"
10550             "DMULp  $dst,ST\n\t"
10551             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
10552             "DMULp  $dst,ST\n\t" %}
10553   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
10554   ins_encode( strictfp_bias1(dst),
10555               Push_Reg_D(src),
10556               OpcP, RegOpc(dst),
10557               strictfp_bias2(dst) );
10558   ins_pipe( fpu_reg_reg );
10559 %}
10560 
10561 instruct mulD_reg_imm(regD dst, immD con) %{
10562   predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
10563   match(Set dst (MulD dst con));
10564   ins_cost(200);
10565   format %{ "FLD_D  [$constantaddress]\t# load from constant table: double=$con\n\t"
10566             "DMULp  $dst,ST" %}
10567   ins_encode %{
10568     __ fld_d($constantaddress($con));
10569     __ fmulp($dst$$reg);
10570   %}
10571   ins_pipe(fpu_reg_mem);
10572 %}
10573 
10574 
10575 instruct mulD_reg_mem(regD dst, memory src) %{
10576   predicate( UseSSE<=1 );
10577   match(Set dst (MulD dst (LoadD src)));
10578   ins_cost(200);
10579   format %{ "FLD_D  $src\n\t"
10580             "DMULp  $dst,ST" %}
10581   opcode(0xDE, 0x1, 0xDD); /* DE C8+i or DE /1*/  /* LoadD  DD /0 */
10582   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10583               OpcP, RegOpc(dst) );
10584   ins_pipe( fpu_reg_mem );
10585 %}
10586 
10587 //
10588 // Cisc-alternate to reg-reg multiply
10589 instruct mulD_reg_mem_cisc(regD dst, regD src, memory mem) %{
10590   predicate( UseSSE<=1 );
10591   match(Set dst (MulD src (LoadD mem)));
10592   ins_cost(250);
10593   format %{ "FLD_D  $mem\n\t"
10594             "DMUL   ST,$src\n\t"
10595             "FSTP_D $dst" %}
10596   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadD D9 /0 */
10597   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem),
10598               OpcReg_F(src),
10599               Pop_Reg_D(dst) );
10600   ins_pipe( fpu_reg_reg_mem );
10601 %}
10602 
10603 
10604 // MACRO3 -- addD a mulD
10605 // This instruction is a '2-address' instruction in that the result goes
10606 // back to src2.  This eliminates a move from the macro; possibly the
10607 // register allocator will have to add it back (and maybe not).
10608 instruct addD_mulD_reg(regD src2, regD src1, regD src0) %{
10609   predicate( UseSSE<=1 );
10610   match(Set src2 (AddD (MulD src0 src1) src2));
10611   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
10612             "DMUL   ST,$src1\n\t"
10613             "DADDp  $src2,ST" %}
10614   ins_cost(250);
10615   opcode(0xDD); /* LoadD DD /0 */
10616   ins_encode( Push_Reg_F(src0),
10617               FMul_ST_reg(src1),
10618               FAddP_reg_ST(src2) );
10619   ins_pipe( fpu_reg_reg_reg );
10620 %}
10621 
10622 
10623 // MACRO3 -- subD a mulD
10624 instruct subD_mulD_reg(regD src2, regD src1, regD src0) %{
10625   predicate( UseSSE<=1 );
10626   match(Set src2 (SubD (MulD src0 src1) src2));
10627   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
10628             "DMUL   ST,$src1\n\t"
10629             "DSUBRp $src2,ST" %}
10630   ins_cost(250);
10631   ins_encode( Push_Reg_F(src0),
10632               FMul_ST_reg(src1),
10633               Opcode(0xDE), Opc_plus(0xE0,src2));
10634   ins_pipe( fpu_reg_reg_reg );
10635 %}
10636 
10637 
10638 instruct divD_reg(regD dst, regD src) %{
10639   predicate( UseSSE<=1 );
10640   match(Set dst (DivD dst src));
10641 
10642   format %{ "FLD    $src\n\t"
10643             "FDIVp  $dst,ST" %}
10644   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10645   ins_cost(150);
10646   ins_encode( Push_Reg_D(src),
10647               OpcP, RegOpc(dst) );
10648   ins_pipe( fpu_reg_reg );
10649 %}
10650 
10651 // Strict FP instruction biases argument before division then
10652 // biases result, to avoid double rounding of subnormals.
10653 //
10654 // scale dividend by multiplying dividend by 2^(-15360)
10655 // load divisor
10656 // divide scaled dividend by divisor
10657 // rescale quotient by 2^(15360)
10658 //
10659 instruct strictfp_divD_reg(regDPR1 dst, regnotDPR1 src) %{
10660   predicate (UseSSE<=1);
10661   match(Set dst (DivD dst src));
10662   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
10663   ins_cost(01);
10664 
10665   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
10666             "DMULp  $dst,ST\n\t"
10667             "FLD    $src\n\t"
10668             "FDIVp  $dst,ST\n\t"
10669             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
10670             "DMULp  $dst,ST\n\t" %}
10671   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10672   ins_encode( strictfp_bias1(dst),
10673               Push_Reg_D(src),
10674               OpcP, RegOpc(dst),
10675               strictfp_bias2(dst) );
10676   ins_pipe( fpu_reg_reg );
10677 %}
10678 
10679 instruct divD_reg_round(stackSlotD dst, regD src1, regD src2) %{
10680   predicate( UseSSE<=1 && !(Compile::current()->has_method() && Compile::current()->method()->is_strict()) );
10681   match(Set dst (RoundDouble (DivD src1 src2)));
10682 
10683   format %{ "FLD    $src1\n\t"
10684             "FDIV   ST,$src2\n\t"
10685             "FSTP_D $dst\t# D-round" %}
10686   opcode(0xD8, 0x6); /* D8 F0+i or D8 /6 */
10687   ins_encode( Push_Reg_D(src1),
10688               OpcP, RegOpc(src2), Pop_Mem_D(dst) );
10689   ins_pipe( fpu_mem_reg_reg );
10690 %}
10691 
10692 
10693 instruct modD_reg(regD dst, regD src, eAXRegI rax, eFlagsReg cr) %{
10694   predicate(UseSSE<=1);
10695   match(Set dst (ModD dst src));
10696   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
10697 
10698   format %{ "DMOD   $dst,$src" %}
10699   ins_cost(250);
10700   ins_encode(Push_Reg_Mod_D(dst, src),
10701               emitModD(),
10702               Push_Result_Mod_D(src),
10703               Pop_Reg_D(dst));
10704   ins_pipe( pipe_slow );
10705 %}
10706 
10707 instruct modXD_reg(regXD dst, regXD src0, regXD src1, eAXRegI rax, eFlagsReg cr) %{
10708   predicate(UseSSE>=2);
10709   match(Set dst (ModD src0 src1));
10710   effect(KILL rax, KILL cr);
10711 
10712   format %{ "SUB    ESP,8\t # DMOD\n"
10713           "\tMOVSD  [ESP+0],$src1\n"
10714           "\tFLD_D  [ESP+0]\n"
10715           "\tMOVSD  [ESP+0],$src0\n"
10716           "\tFLD_D  [ESP+0]\n"
10717      "loop:\tFPREM\n"
10718           "\tFWAIT\n"
10719           "\tFNSTSW AX\n"
10720           "\tSAHF\n"
10721           "\tJP     loop\n"
10722           "\tFSTP_D [ESP+0]\n"
10723           "\tMOVSD  $dst,[ESP+0]\n"
10724           "\tADD    ESP,8\n"
10725           "\tFSTP   ST0\t # Restore FPU Stack"
10726     %}
10727   ins_cost(250);
10728   ins_encode( Push_ModD_encoding(src0, src1), emitModD(), Push_ResultXD(dst), PopFPU);
10729   ins_pipe( pipe_slow );
10730 %}
10731 
10732 instruct sinD_reg(regDPR1 dst, regDPR1 src) %{
10733   predicate (UseSSE<=1);
10734   match(Set dst (SinD src));
10735   ins_cost(1800);
10736   format %{ "DSIN   $dst" %}
10737   opcode(0xD9, 0xFE);
10738   ins_encode( OpcP, OpcS );
10739   ins_pipe( pipe_slow );
10740 %}
10741 
10742 instruct sinXD_reg(regXD dst, eFlagsReg cr) %{
10743   predicate (UseSSE>=2);
10744   match(Set dst (SinD dst));
10745   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10746   ins_cost(1800);
10747   format %{ "DSIN   $dst" %}
10748   opcode(0xD9, 0xFE);
10749   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10750   ins_pipe( pipe_slow );
10751 %}
10752 
10753 instruct cosD_reg(regDPR1 dst, regDPR1 src) %{
10754   predicate (UseSSE<=1);
10755   match(Set dst (CosD src));
10756   ins_cost(1800);
10757   format %{ "DCOS   $dst" %}
10758   opcode(0xD9, 0xFF);
10759   ins_encode( OpcP, OpcS );
10760   ins_pipe( pipe_slow );
10761 %}
10762 
10763 instruct cosXD_reg(regXD dst, eFlagsReg cr) %{
10764   predicate (UseSSE>=2);
10765   match(Set dst (CosD dst));
10766   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10767   ins_cost(1800);
10768   format %{ "DCOS   $dst" %}
10769   opcode(0xD9, 0xFF);
10770   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10771   ins_pipe( pipe_slow );
10772 %}
10773 
10774 instruct tanD_reg(regDPR1 dst, regDPR1 src) %{
10775   predicate (UseSSE<=1);
10776   match(Set dst(TanD src));
10777   format %{ "DTAN   $dst" %}
10778   ins_encode( Opcode(0xD9), Opcode(0xF2),    // fptan
10779               Opcode(0xDD), Opcode(0xD8));   // fstp st
10780   ins_pipe( pipe_slow );
10781 %}
10782 
10783 instruct tanXD_reg(regXD dst, eFlagsReg cr) %{
10784   predicate (UseSSE>=2);
10785   match(Set dst(TanD dst));
10786   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10787   format %{ "DTAN   $dst" %}
10788   ins_encode( Push_SrcXD(dst),
10789               Opcode(0xD9), Opcode(0xF2),    // fptan
10790               Opcode(0xDD), Opcode(0xD8),   // fstp st
10791               Push_ResultXD(dst) );
10792   ins_pipe( pipe_slow );
10793 %}
10794 
10795 instruct atanD_reg(regD dst, regD src) %{
10796   predicate (UseSSE<=1);
10797   match(Set dst(AtanD dst src));
10798   format %{ "DATA   $dst,$src" %}
10799   opcode(0xD9, 0xF3);
10800   ins_encode( Push_Reg_D(src),
10801               OpcP, OpcS, RegOpc(dst) );
10802   ins_pipe( pipe_slow );
10803 %}
10804 
10805 instruct atanXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
10806   predicate (UseSSE>=2);
10807   match(Set dst(AtanD dst src));
10808   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10809   format %{ "DATA   $dst,$src" %}
10810   opcode(0xD9, 0xF3);
10811   ins_encode( Push_SrcXD(src),
10812               OpcP, OpcS, Push_ResultXD(dst) );
10813   ins_pipe( pipe_slow );
10814 %}
10815 
10816 instruct sqrtD_reg(regD dst, regD src) %{
10817   predicate (UseSSE<=1);
10818   match(Set dst (SqrtD src));
10819   format %{ "DSQRT  $dst,$src" %}
10820   opcode(0xFA, 0xD9);
10821   ins_encode( Push_Reg_D(src),
10822               OpcS, OpcP, Pop_Reg_D(dst) );
10823   ins_pipe( pipe_slow );
10824 %}
10825 
10826 instruct powD_reg(regD X, regDPR1 Y, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10827   predicate (UseSSE<=1);
10828   match(Set Y (PowD X Y));  // Raise X to the Yth power
10829   effect(KILL rax, KILL rbx, KILL rcx);
10830   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
10831             "FLD_D  $X\n\t"
10832             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
10833 
10834             "FDUP   \t\t\t# Q Q\n\t"
10835             "FRNDINT\t\t\t# int(Q) Q\n\t"
10836             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10837             "FISTP  dword [ESP]\n\t"
10838             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10839             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10840             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10841             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10842             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10843             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10844             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10845             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10846             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10847             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10848             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10849             "MOV    [ESP+0],0\n\t"
10850             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10851 
10852             "ADD    ESP,8"
10853              %}
10854   ins_encode( push_stack_temp_qword,
10855               Push_Reg_D(X),
10856               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10857               pow_exp_core_encoding,
10858               pop_stack_temp_qword);
10859   ins_pipe( pipe_slow );
10860 %}
10861 
10862 instruct powXD_reg(regXD dst, regXD src0, regXD src1, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx ) %{
10863   predicate (UseSSE>=2);
10864   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
10865   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx );
10866   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
10867             "MOVSD  [ESP],$src1\n\t"
10868             "FLD    FPR1,$src1\n\t"
10869             "MOVSD  [ESP],$src0\n\t"
10870             "FLD    FPR1,$src0\n\t"
10871             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
10872 
10873             "FDUP   \t\t\t# Q Q\n\t"
10874             "FRNDINT\t\t\t# int(Q) Q\n\t"
10875             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10876             "FISTP  dword [ESP]\n\t"
10877             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10878             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10879             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10880             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10881             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10882             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10883             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10884             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10885             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10886             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10887             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10888             "MOV    [ESP+0],0\n\t"
10889             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10890 
10891             "FST_D  [ESP]\n\t"
10892             "MOVSD  $dst,[ESP]\n\t"
10893             "ADD    ESP,8"
10894              %}
10895   ins_encode( push_stack_temp_qword,
10896               push_xmm_to_fpr1(src1),
10897               push_xmm_to_fpr1(src0),
10898               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10899               pow_exp_core_encoding,
10900               Push_ResultXD(dst) );
10901   ins_pipe( pipe_slow );
10902 %}
10903 
10904 
10905 instruct expD_reg(regDPR1 dpr1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10906   predicate (UseSSE<=1);
10907   match(Set dpr1 (ExpD dpr1));
10908   effect(KILL rax, KILL rbx, KILL rcx);
10909   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding"
10910             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
10911             "FMULP  \t\t\t# Q=X*log2(e)\n\t"
10912 
10913             "FDUP   \t\t\t# Q Q\n\t"
10914             "FRNDINT\t\t\t# int(Q) Q\n\t"
10915             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10916             "FISTP  dword [ESP]\n\t"
10917             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10918             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10919             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10920             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10921             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10922             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10923             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10924             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10925             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10926             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10927             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10928             "MOV    [ESP+0],0\n\t"
10929             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10930 
10931             "ADD    ESP,8"
10932              %}
10933   ins_encode( push_stack_temp_qword,
10934               Opcode(0xD9), Opcode(0xEA),   // fldl2e
10935               Opcode(0xDE), Opcode(0xC9),   // fmulp
10936               pow_exp_core_encoding,
10937               pop_stack_temp_qword);
10938   ins_pipe( pipe_slow );
10939 %}
10940 
10941 instruct expXD_reg(regXD dst, regXD src, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10942   predicate (UseSSE>=2);
10943   match(Set dst (ExpD src));
10944   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx);
10945   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding\n\t"
10946             "MOVSD  [ESP],$src\n\t"
10947             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
10948             "FMULP  \t\t\t# Q=X*log2(e) X\n\t"
10949 
10950             "FDUP   \t\t\t# Q Q\n\t"
10951             "FRNDINT\t\t\t# int(Q) Q\n\t"
10952             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10953             "FISTP  dword [ESP]\n\t"
10954             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10955             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10956             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10957             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10958             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10959             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10960             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10961             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10962             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10963             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10964             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10965             "MOV    [ESP+0],0\n\t"
10966             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10967 
10968             "FST_D  [ESP]\n\t"
10969             "MOVSD  $dst,[ESP]\n\t"
10970             "ADD    ESP,8"
10971              %}
10972   ins_encode( Push_SrcXD(src),
10973               Opcode(0xD9), Opcode(0xEA),   // fldl2e
10974               Opcode(0xDE), Opcode(0xC9),   // fmulp
10975               pow_exp_core_encoding,
10976               Push_ResultXD(dst) );
10977   ins_pipe( pipe_slow );
10978 %}
10979 
10980 
10981 
10982 instruct log10D_reg(regDPR1 dst, regDPR1 src) %{
10983   predicate (UseSSE<=1);
10984   // The source Double operand on FPU stack
10985   match(Set dst (Log10D src));
10986   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10987   // fxch         ; swap ST(0) with ST(1)
10988   // fyl2x        ; compute log_10(2) * log_2(x)
10989   format %{ "FLDLG2 \t\t\t#Log10\n\t"
10990             "FXCH   \n\t"
10991             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
10992          %}
10993   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
10994               Opcode(0xD9), Opcode(0xC9),   // fxch
10995               Opcode(0xD9), Opcode(0xF1));  // fyl2x
10996 
10997   ins_pipe( pipe_slow );
10998 %}
10999 
11000 instruct log10XD_reg(regXD dst, regXD src, eFlagsReg cr) %{
11001   predicate (UseSSE>=2);
11002   effect(KILL cr);
11003   match(Set dst (Log10D src));
11004   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
11005   // fyl2x        ; compute log_10(2) * log_2(x)
11006   format %{ "FLDLG2 \t\t\t#Log10\n\t"
11007             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
11008          %}
11009   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
11010               Push_SrcXD(src),
11011               Opcode(0xD9), Opcode(0xF1),   // fyl2x
11012               Push_ResultXD(dst));
11013 
11014   ins_pipe( pipe_slow );
11015 %}
11016 
11017 instruct logD_reg(regDPR1 dst, regDPR1 src) %{
11018   predicate (UseSSE<=1);
11019   // The source Double operand on FPU stack
11020   match(Set dst (LogD src));
11021   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
11022   // fxch         ; swap ST(0) with ST(1)
11023   // fyl2x        ; compute log_e(2) * log_2(x)
11024   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
11025             "FXCH   \n\t"
11026             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
11027          %}
11028   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
11029               Opcode(0xD9), Opcode(0xC9),   // fxch
11030               Opcode(0xD9), Opcode(0xF1));  // fyl2x
11031 
11032   ins_pipe( pipe_slow );
11033 %}
11034 
11035 instruct logXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
11036   predicate (UseSSE>=2);
11037   effect(KILL cr);
11038   // The source and result Double operands in XMM registers
11039   match(Set dst (LogD src));
11040   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
11041   // fyl2x        ; compute log_e(2) * log_2(x)
11042   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
11043             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
11044          %}
11045   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
11046               Push_SrcXD(src),
11047               Opcode(0xD9), Opcode(0xF1),   // fyl2x
11048               Push_ResultXD(dst));
11049   ins_pipe( pipe_slow );
11050 %}
11051 
11052 //-------------Float Instructions-------------------------------
11053 // Float Math
11054 
11055 // Code for float compare:
11056 //     fcompp();
11057 //     fwait(); fnstsw_ax();
11058 //     sahf();
11059 //     movl(dst, unordered_result);
11060 //     jcc(Assembler::parity, exit);
11061 //     movl(dst, less_result);
11062 //     jcc(Assembler::below, exit);
11063 //     movl(dst, equal_result);
11064 //     jcc(Assembler::equal, exit);
11065 //     movl(dst, greater_result);
11066 //   exit:
11067 
11068 // P6 version of float compare, sets condition codes in EFLAGS
11069 instruct cmpF_cc_P6(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
11070   predicate(VM_Version::supports_cmov() && UseSSE == 0);
11071   match(Set cr (CmpF src1 src2));
11072   effect(KILL rax);
11073   ins_cost(150);
11074   format %{ "FLD    $src1\n\t"
11075             "FUCOMIP ST,$src2  // P6 instruction\n\t"
11076             "JNP    exit\n\t"
11077             "MOV    ah,1       // saw a NaN, set CF (treat as LT)\n\t"
11078             "SAHF\n"
11079      "exit:\tNOP               // avoid branch to branch" %}
11080   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
11081   ins_encode( Push_Reg_D(src1),
11082               OpcP, RegOpc(src2),
11083               cmpF_P6_fixup );
11084   ins_pipe( pipe_slow );
11085 %}
11086 
11087 instruct cmpF_cc_P6CF(eFlagsRegUCF cr, regF src1, regF src2) %{
11088   predicate(VM_Version::supports_cmov() && UseSSE == 0);
11089   match(Set cr (CmpF src1 src2));
11090   ins_cost(100);
11091   format %{ "FLD    $src1\n\t"
11092             "FUCOMIP ST,$src2  // P6 instruction" %}
11093   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
11094   ins_encode( Push_Reg_D(src1),
11095               OpcP, RegOpc(src2));
11096   ins_pipe( pipe_slow );
11097 %}
11098 
11099 
11100 // Compare & branch
11101 instruct cmpF_cc(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
11102   predicate(UseSSE == 0);
11103   match(Set cr (CmpF src1 src2));
11104   effect(KILL rax);
11105   ins_cost(200);
11106   format %{ "FLD    $src1\n\t"
11107             "FCOMp  $src2\n\t"
11108             "FNSTSW AX\n\t"
11109             "TEST   AX,0x400\n\t"
11110             "JZ,s   flags\n\t"
11111             "MOV    AH,1\t# unordered treat as LT\n"
11112     "flags:\tSAHF" %}
11113   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
11114   ins_encode( Push_Reg_D(src1),
11115               OpcP, RegOpc(src2),
11116               fpu_flags);
11117   ins_pipe( pipe_slow );
11118 %}
11119 
11120 // Compare vs zero into -1,0,1
11121 instruct cmpF_0(eRegI dst, regF src1, immF0 zero, eAXRegI rax, eFlagsReg cr) %{
11122   predicate(UseSSE == 0);
11123   match(Set dst (CmpF3 src1 zero));
11124   effect(KILL cr, KILL rax);
11125   ins_cost(280);
11126   format %{ "FTSTF  $dst,$src1" %}
11127   opcode(0xE4, 0xD9);
11128   ins_encode( Push_Reg_D(src1),
11129               OpcS, OpcP, PopFPU,
11130               CmpF_Result(dst));
11131   ins_pipe( pipe_slow );
11132 %}
11133 
11134 // Compare into -1,0,1
11135 instruct cmpF_reg(eRegI dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
11136   predicate(UseSSE == 0);
11137   match(Set dst (CmpF3 src1 src2));
11138   effect(KILL cr, KILL rax);
11139   ins_cost(300);
11140   format %{ "FCMPF  $dst,$src1,$src2" %}
11141   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
11142   ins_encode( Push_Reg_D(src1),
11143               OpcP, RegOpc(src2),
11144               CmpF_Result(dst));
11145   ins_pipe( pipe_slow );
11146 %}
11147 
11148 // float compare and set condition codes in EFLAGS by XMM regs
11149 instruct cmpX_cc(eFlagsRegU cr, regX dst, regX src, eAXRegI rax) %{
11150   predicate(UseSSE>=1);
11151   match(Set cr (CmpF dst src));
11152   effect(KILL rax);
11153   ins_cost(145);
11154   format %{ "COMISS $dst,$src\n"
11155           "\tJNP    exit\n"
11156           "\tMOV    ah,1       // saw a NaN, set CF\n"
11157           "\tSAHF\n"
11158      "exit:\tNOP               // avoid branch to branch" %}
11159   opcode(0x0F, 0x2F);
11160   ins_encode(OpcP, OpcS, RegReg(dst, src), cmpF_P6_fixup);
11161   ins_pipe( pipe_slow );
11162 %}
11163 
11164 instruct cmpX_ccCF(eFlagsRegUCF cr, regX dst, regX src) %{
11165   predicate(UseSSE>=1);
11166   match(Set cr (CmpF dst src));
11167   ins_cost(100);
11168   format %{ "COMISS $dst,$src" %}
11169   opcode(0x0F, 0x2F);
11170   ins_encode(OpcP, OpcS, RegReg(dst, src));
11171   ins_pipe( pipe_slow );
11172 %}
11173 
11174 // float compare and set condition codes in EFLAGS by XMM regs
11175 instruct cmpX_ccmem(eFlagsRegU cr, regX dst, memory src, eAXRegI rax) %{
11176   predicate(UseSSE>=1);
11177   match(Set cr (CmpF dst (LoadF src)));
11178   effect(KILL rax);
11179   ins_cost(165);
11180   format %{ "COMISS $dst,$src\n"
11181           "\tJNP    exit\n"
11182           "\tMOV    ah,1       // saw a NaN, set CF\n"
11183           "\tSAHF\n"
11184      "exit:\tNOP               // avoid branch to branch" %}
11185   opcode(0x0F, 0x2F);
11186   ins_encode(OpcP, OpcS, RegMem(dst, src), cmpF_P6_fixup);
11187   ins_pipe( pipe_slow );
11188 %}
11189 
11190 instruct cmpX_ccmemCF(eFlagsRegUCF cr, regX dst, memory src) %{
11191   predicate(UseSSE>=1);
11192   match(Set cr (CmpF dst (LoadF src)));
11193   ins_cost(100);
11194   format %{ "COMISS $dst,$src" %}
11195   opcode(0x0F, 0x2F);
11196   ins_encode(OpcP, OpcS, RegMem(dst, src));
11197   ins_pipe( pipe_slow );
11198 %}
11199 
11200 // Compare into -1,0,1 in XMM
11201 instruct cmpX_reg(eRegI dst, regX src1, regX src2, eFlagsReg cr) %{
11202   predicate(UseSSE>=1);
11203   match(Set dst (CmpF3 src1 src2));
11204   effect(KILL cr);
11205   ins_cost(255);
11206   format %{ "XOR    $dst,$dst\n"
11207           "\tCOMISS $src1,$src2\n"
11208           "\tJP,s   nan\n"
11209           "\tJEQ,s  exit\n"
11210           "\tJA,s   inc\n"
11211       "nan:\tDEC    $dst\n"
11212           "\tJMP,s  exit\n"
11213       "inc:\tINC    $dst\n"
11214       "exit:"
11215                 %}
11216   opcode(0x0F, 0x2F);
11217   ins_encode(Xor_Reg(dst), OpcP, OpcS, RegReg(src1, src2), CmpX_Result(dst));
11218   ins_pipe( pipe_slow );
11219 %}
11220 
11221 // Compare into -1,0,1 in XMM and memory
11222 instruct cmpX_regmem(eRegI dst, regX src1, memory mem, eFlagsReg cr) %{
11223   predicate(UseSSE>=1);
11224   match(Set dst (CmpF3 src1 (LoadF mem)));
11225   effect(KILL cr);
11226   ins_cost(275);
11227   format %{ "COMISS $src1,$mem\n"
11228           "\tMOV    $dst,0\t\t# do not blow flags\n"
11229           "\tJP,s   nan\n"
11230           "\tJEQ,s  exit\n"
11231           "\tJA,s   inc\n"
11232       "nan:\tDEC    $dst\n"
11233           "\tJMP,s  exit\n"
11234       "inc:\tINC    $dst\n"
11235       "exit:"
11236                 %}
11237   opcode(0x0F, 0x2F);
11238   ins_encode(OpcP, OpcS, RegMem(src1, mem), LdImmI(dst,0x0), CmpX_Result(dst));
11239   ins_pipe( pipe_slow );
11240 %}
11241 
11242 // Spill to obtain 24-bit precision
11243 instruct subF24_reg(stackSlotF dst, regF src1, regF src2) %{
11244   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11245   match(Set dst (SubF src1 src2));
11246 
11247   format %{ "FSUB   $dst,$src1 - $src2" %}
11248   opcode(0xD8, 0x4); /* D8 E0+i or D8 /4 mod==0x3 ;; result in TOS */
11249   ins_encode( Push_Reg_F(src1),
11250               OpcReg_F(src2),
11251               Pop_Mem_F(dst) );
11252   ins_pipe( fpu_mem_reg_reg );
11253 %}
11254 //
11255 // This instruction does not round to 24-bits
11256 instruct subF_reg(regF dst, regF src) %{
11257   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11258   match(Set dst (SubF dst src));
11259 
11260   format %{ "FSUB   $dst,$src" %}
11261   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
11262   ins_encode( Push_Reg_F(src),
11263               OpcP, RegOpc(dst) );
11264   ins_pipe( fpu_reg_reg );
11265 %}
11266 
11267 // Spill to obtain 24-bit precision
11268 instruct addF24_reg(stackSlotF dst, regF src1, regF src2) %{
11269   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11270   match(Set dst (AddF src1 src2));
11271 
11272   format %{ "FADD   $dst,$src1,$src2" %}
11273   opcode(0xD8, 0x0); /* D8 C0+i */
11274   ins_encode( Push_Reg_F(src2),
11275               OpcReg_F(src1),
11276               Pop_Mem_F(dst) );
11277   ins_pipe( fpu_mem_reg_reg );
11278 %}
11279 //
11280 // This instruction does not round to 24-bits
11281 instruct addF_reg(regF dst, regF src) %{
11282   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11283   match(Set dst (AddF dst src));
11284 
11285   format %{ "FLD    $src\n\t"
11286             "FADDp  $dst,ST" %}
11287   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
11288   ins_encode( Push_Reg_F(src),
11289               OpcP, RegOpc(dst) );
11290   ins_pipe( fpu_reg_reg );
11291 %}
11292 
11293 // Add two single precision floating point values in xmm
11294 instruct addX_reg(regX dst, regX src) %{
11295   predicate(UseSSE>=1);
11296   match(Set dst (AddF dst src));
11297   format %{ "ADDSS  $dst,$src" %}
11298   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
11299   ins_pipe( pipe_slow );
11300 %}
11301 
11302 instruct addX_imm(regX dst, immXF con) %{
11303   predicate(UseSSE>=1);
11304   match(Set dst (AddF dst con));
11305   format %{ "ADDSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
11306   ins_encode %{
11307     __ addss($dst$$XMMRegister, $constantaddress($con));
11308   %}
11309   ins_pipe(pipe_slow);
11310 %}
11311 
11312 instruct addX_mem(regX dst, memory mem) %{
11313   predicate(UseSSE>=1);
11314   match(Set dst (AddF dst (LoadF mem)));
11315   format %{ "ADDSS  $dst,$mem" %}
11316   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegMem(dst, mem));
11317   ins_pipe( pipe_slow );
11318 %}
11319 
11320 // Subtract two single precision floating point values in xmm
11321 instruct subX_reg(regX dst, regX src) %{
11322   predicate(UseSSE>=1);
11323   match(Set dst (SubF dst src));
11324   format %{ "SUBSS  $dst,$src" %}
11325   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
11326   ins_pipe( pipe_slow );
11327 %}
11328 
11329 instruct subX_imm(regX dst, immXF con) %{
11330   predicate(UseSSE>=1);
11331   match(Set dst (SubF dst con));
11332   format %{ "SUBSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
11333   ins_encode %{
11334     __ subss($dst$$XMMRegister, $constantaddress($con));
11335   %}
11336   ins_pipe(pipe_slow);
11337 %}
11338 
11339 instruct subX_mem(regX dst, memory mem) %{
11340   predicate(UseSSE>=1);
11341   match(Set dst (SubF dst (LoadF mem)));
11342   format %{ "SUBSS  $dst,$mem" %}
11343   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
11344   ins_pipe( pipe_slow );
11345 %}
11346 
11347 // Multiply two single precision floating point values in xmm
11348 instruct mulX_reg(regX dst, regX src) %{
11349   predicate(UseSSE>=1);
11350   match(Set dst (MulF dst src));
11351   format %{ "MULSS  $dst,$src" %}
11352   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
11353   ins_pipe( pipe_slow );
11354 %}
11355 
11356 instruct mulX_imm(regX dst, immXF con) %{
11357   predicate(UseSSE>=1);
11358   match(Set dst (MulF dst con));
11359   format %{ "MULSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
11360   ins_encode %{
11361     __ mulss($dst$$XMMRegister, $constantaddress($con));
11362   %}
11363   ins_pipe(pipe_slow);
11364 %}
11365 
11366 instruct mulX_mem(regX dst, memory mem) %{
11367   predicate(UseSSE>=1);
11368   match(Set dst (MulF dst (LoadF mem)));
11369   format %{ "MULSS  $dst,$mem" %}
11370   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
11371   ins_pipe( pipe_slow );
11372 %}
11373 
11374 // Divide two single precision floating point values in xmm
11375 instruct divX_reg(regX dst, regX src) %{
11376   predicate(UseSSE>=1);
11377   match(Set dst (DivF dst src));
11378   format %{ "DIVSS  $dst,$src" %}
11379   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
11380   ins_pipe( pipe_slow );
11381 %}
11382 
11383 instruct divX_imm(regX dst, immXF con) %{
11384   predicate(UseSSE>=1);
11385   match(Set dst (DivF dst con));
11386   format %{ "DIVSS  $dst,[$constantaddress]\t# load from constant table: float=$con" %}
11387   ins_encode %{
11388     __ divss($dst$$XMMRegister, $constantaddress($con));
11389   %}
11390   ins_pipe(pipe_slow);
11391 %}
11392 
11393 instruct divX_mem(regX dst, memory mem) %{
11394   predicate(UseSSE>=1);
11395   match(Set dst (DivF dst (LoadF mem)));
11396   format %{ "DIVSS  $dst,$mem" %}
11397   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
11398   ins_pipe( pipe_slow );
11399 %}
11400 
11401 // Get the square root of a single precision floating point values in xmm
11402 instruct sqrtX_reg(regX dst, regX src) %{
11403   predicate(UseSSE>=1);
11404   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
11405   format %{ "SQRTSS $dst,$src" %}
11406   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
11407   ins_pipe( pipe_slow );
11408 %}
11409 
11410 instruct sqrtX_mem(regX dst, memory mem) %{
11411   predicate(UseSSE>=1);
11412   match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF mem)))));
11413   format %{ "SQRTSS $dst,$mem" %}
11414   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
11415   ins_pipe( pipe_slow );
11416 %}
11417 
11418 // Get the square root of a double precision floating point values in xmm
11419 instruct sqrtXD_reg(regXD dst, regXD src) %{
11420   predicate(UseSSE>=2);
11421   match(Set dst (SqrtD src));
11422   format %{ "SQRTSD $dst,$src" %}
11423   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
11424   ins_pipe( pipe_slow );
11425 %}
11426 
11427 instruct sqrtXD_mem(regXD dst, memory mem) %{
11428   predicate(UseSSE>=2);
11429   match(Set dst (SqrtD (LoadD mem)));
11430   format %{ "SQRTSD $dst,$mem" %}
11431   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
11432   ins_pipe( pipe_slow );
11433 %}
11434 
11435 instruct absF_reg(regFPR1 dst, regFPR1 src) %{
11436   predicate(UseSSE==0);
11437   match(Set dst (AbsF src));
11438   ins_cost(100);
11439   format %{ "FABS" %}
11440   opcode(0xE1, 0xD9);
11441   ins_encode( OpcS, OpcP );
11442   ins_pipe( fpu_reg_reg );
11443 %}
11444 
11445 instruct absX_reg(regX dst ) %{
11446   predicate(UseSSE>=1);
11447   match(Set dst (AbsF dst));
11448   format %{ "ANDPS  $dst,[0x7FFFFFFF]\t# ABS F by sign masking" %}
11449   ins_encode( AbsXF_encoding(dst));
11450   ins_pipe( pipe_slow );
11451 %}
11452 
11453 instruct negF_reg(regFPR1 dst, regFPR1 src) %{
11454   predicate(UseSSE==0);
11455   match(Set dst (NegF src));
11456   ins_cost(100);
11457   format %{ "FCHS" %}
11458   opcode(0xE0, 0xD9);
11459   ins_encode( OpcS, OpcP );
11460   ins_pipe( fpu_reg_reg );
11461 %}
11462 
11463 instruct negX_reg( regX dst ) %{
11464   predicate(UseSSE>=1);
11465   match(Set dst (NegF dst));
11466   format %{ "XORPS  $dst,[0x80000000]\t# CHS F by sign flipping" %}
11467   ins_encode( NegXF_encoding(dst));
11468   ins_pipe( pipe_slow );
11469 %}
11470 
11471 // Cisc-alternate to addF_reg
11472 // Spill to obtain 24-bit precision
11473 instruct addF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
11474   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11475   match(Set dst (AddF src1 (LoadF src2)));
11476 
11477   format %{ "FLD    $src2\n\t"
11478             "FADD   ST,$src1\n\t"
11479             "FSTP_S $dst" %}
11480   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11481   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11482               OpcReg_F(src1),
11483               Pop_Mem_F(dst) );
11484   ins_pipe( fpu_mem_reg_mem );
11485 %}
11486 //
11487 // Cisc-alternate to addF_reg
11488 // This instruction does not round to 24-bits
11489 instruct addF_reg_mem(regF dst, memory src) %{
11490   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11491   match(Set dst (AddF dst (LoadF src)));
11492 
11493   format %{ "FADD   $dst,$src" %}
11494   opcode(0xDE, 0x0, 0xD9); /* DE C0+i or DE /0*/  /* LoadF  D9 /0 */
11495   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
11496               OpcP, RegOpc(dst) );
11497   ins_pipe( fpu_reg_mem );
11498 %}
11499 
11500 // // Following two instructions for _222_mpegaudio
11501 // Spill to obtain 24-bit precision
11502 instruct addF24_mem_reg(stackSlotF dst, regF src2, memory src1 ) %{
11503   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11504   match(Set dst (AddF src1 src2));
11505 
11506   format %{ "FADD   $dst,$src1,$src2" %}
11507   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11508   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src1),
11509               OpcReg_F(src2),
11510               Pop_Mem_F(dst) );
11511   ins_pipe( fpu_mem_reg_mem );
11512 %}
11513 
11514 // Cisc-spill variant
11515 // Spill to obtain 24-bit precision
11516 instruct addF24_mem_cisc(stackSlotF dst, memory src1, memory src2) %{
11517   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11518   match(Set dst (AddF src1 (LoadF src2)));
11519 
11520   format %{ "FADD   $dst,$src1,$src2 cisc" %}
11521   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11522   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11523               set_instruction_start,
11524               OpcP, RMopc_Mem(secondary,src1),
11525               Pop_Mem_F(dst) );
11526   ins_pipe( fpu_mem_mem_mem );
11527 %}
11528 
11529 // Spill to obtain 24-bit precision
11530 instruct addF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
11531   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11532   match(Set dst (AddF src1 src2));
11533 
11534   format %{ "FADD   $dst,$src1,$src2" %}
11535   opcode(0xD8, 0x0, 0xD9); /* D8 /0 */  /* LoadF  D9 /0 */
11536   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11537               set_instruction_start,
11538               OpcP, RMopc_Mem(secondary,src1),
11539               Pop_Mem_F(dst) );
11540   ins_pipe( fpu_mem_mem_mem );
11541 %}
11542 
11543 
11544 // Spill to obtain 24-bit precision
11545 instruct addF24_reg_imm(stackSlotF dst, regF src, immF con) %{
11546   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11547   match(Set dst (AddF src con));
11548   format %{ "FLD    $src\n\t"
11549             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
11550             "FSTP_S $dst"  %}
11551   ins_encode %{
11552     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
11553     __ fadd_s($constantaddress($con));
11554     __ fstp_s(Address(rsp, $dst$$disp));
11555   %}
11556   ins_pipe(fpu_mem_reg_con);
11557 %}
11558 //
11559 // This instruction does not round to 24-bits
11560 instruct addF_reg_imm(regF dst, regF src, immF con) %{
11561   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11562   match(Set dst (AddF src con));
11563   format %{ "FLD    $src\n\t"
11564             "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
11565             "FSTP   $dst"  %}
11566   ins_encode %{
11567     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
11568     __ fadd_s($constantaddress($con));
11569     __ fstp_d($dst$$reg);
11570   %}
11571   ins_pipe(fpu_reg_reg_con);
11572 %}
11573 
11574 // Spill to obtain 24-bit precision
11575 instruct mulF24_reg(stackSlotF dst, regF src1, regF src2) %{
11576   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11577   match(Set dst (MulF src1 src2));
11578 
11579   format %{ "FLD    $src1\n\t"
11580             "FMUL   $src2\n\t"
11581             "FSTP_S $dst"  %}
11582   opcode(0xD8, 0x1); /* D8 C8+i or D8 /1 ;; result in TOS */
11583   ins_encode( Push_Reg_F(src1),
11584               OpcReg_F(src2),
11585               Pop_Mem_F(dst) );
11586   ins_pipe( fpu_mem_reg_reg );
11587 %}
11588 //
11589 // This instruction does not round to 24-bits
11590 instruct mulF_reg(regF dst, regF src1, regF src2) %{
11591   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11592   match(Set dst (MulF src1 src2));
11593 
11594   format %{ "FLD    $src1\n\t"
11595             "FMUL   $src2\n\t"
11596             "FSTP_S $dst"  %}
11597   opcode(0xD8, 0x1); /* D8 C8+i */
11598   ins_encode( Push_Reg_F(src2),
11599               OpcReg_F(src1),
11600               Pop_Reg_F(dst) );
11601   ins_pipe( fpu_reg_reg_reg );
11602 %}
11603 
11604 
11605 // Spill to obtain 24-bit precision
11606 // Cisc-alternate to reg-reg multiply
11607 instruct mulF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
11608   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11609   match(Set dst (MulF src1 (LoadF src2)));
11610 
11611   format %{ "FLD_S  $src2\n\t"
11612             "FMUL   $src1\n\t"
11613             "FSTP_S $dst"  %}
11614   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or DE /1*/  /* LoadF D9 /0 */
11615   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11616               OpcReg_F(src1),
11617               Pop_Mem_F(dst) );
11618   ins_pipe( fpu_mem_reg_mem );
11619 %}
11620 //
11621 // This instruction does not round to 24-bits
11622 // Cisc-alternate to reg-reg multiply
11623 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
11624   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11625   match(Set dst (MulF src1 (LoadF src2)));
11626 
11627   format %{ "FMUL   $dst,$src1,$src2" %}
11628   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadF D9 /0 */
11629   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11630               OpcReg_F(src1),
11631               Pop_Reg_F(dst) );
11632   ins_pipe( fpu_reg_reg_mem );
11633 %}
11634 
11635 // Spill to obtain 24-bit precision
11636 instruct mulF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
11637   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11638   match(Set dst (MulF src1 src2));
11639 
11640   format %{ "FMUL   $dst,$src1,$src2" %}
11641   opcode(0xD8, 0x1, 0xD9); /* D8 /1 */  /* LoadF D9 /0 */
11642   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11643               set_instruction_start,
11644               OpcP, RMopc_Mem(secondary,src1),
11645               Pop_Mem_F(dst) );
11646   ins_pipe( fpu_mem_mem_mem );
11647 %}
11648 
11649 // Spill to obtain 24-bit precision
11650 instruct mulF24_reg_imm(stackSlotF dst, regF src, immF con) %{
11651   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11652   match(Set dst (MulF src con));
11653 
11654   format %{ "FLD    $src\n\t"
11655             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
11656             "FSTP_S $dst"  %}
11657   ins_encode %{
11658     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
11659     __ fmul_s($constantaddress($con));
11660     __ fstp_s(Address(rsp, $dst$$disp));
11661   %}
11662   ins_pipe(fpu_mem_reg_con);
11663 %}
11664 //
11665 // This instruction does not round to 24-bits
11666 instruct mulF_reg_imm(regF dst, regF src, immF con) %{
11667   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11668   match(Set dst (MulF src con));
11669 
11670   format %{ "FLD    $src\n\t"
11671             "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
11672             "FSTP   $dst"  %}
11673   ins_encode %{
11674     __ fld_s($src$$reg - 1);  // FLD ST(i-1)
11675     __ fmul_s($constantaddress($con));
11676     __ fstp_d($dst$$reg);
11677   %}
11678   ins_pipe(fpu_reg_reg_con);
11679 %}
11680 
11681 
11682 //
11683 // MACRO1 -- subsume unshared load into mulF
11684 // This instruction does not round to 24-bits
11685 instruct mulF_reg_load1(regF dst, regF src, memory mem1 ) %{
11686   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11687   match(Set dst (MulF (LoadF mem1) src));
11688 
11689   format %{ "FLD    $mem1    ===MACRO1===\n\t"
11690             "FMUL   ST,$src\n\t"
11691             "FSTP   $dst" %}
11692   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or D8 /1 */  /* LoadF D9 /0 */
11693   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem1),
11694               OpcReg_F(src),
11695               Pop_Reg_F(dst) );
11696   ins_pipe( fpu_reg_reg_mem );
11697 %}
11698 //
11699 // MACRO2 -- addF a mulF which subsumed an unshared load
11700 // This instruction does not round to 24-bits
11701 instruct addF_mulF_reg_load1(regF dst, memory mem1, regF src1, regF src2) %{
11702   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11703   match(Set dst (AddF (MulF (LoadF mem1) src1) src2));
11704   ins_cost(95);
11705 
11706   format %{ "FLD    $mem1     ===MACRO2===\n\t"
11707             "FMUL   ST,$src1  subsume mulF left load\n\t"
11708             "FADD   ST,$src2\n\t"
11709             "FSTP   $dst" %}
11710   opcode(0xD9); /* LoadF D9 /0 */
11711   ins_encode( OpcP, RMopc_Mem(0x00,mem1),
11712               FMul_ST_reg(src1),
11713               FAdd_ST_reg(src2),
11714               Pop_Reg_F(dst) );
11715   ins_pipe( fpu_reg_mem_reg_reg );
11716 %}
11717 
11718 // MACRO3 -- addF a mulF
11719 // This instruction does not round to 24-bits.  It is a '2-address'
11720 // instruction in that the result goes back to src2.  This eliminates
11721 // a move from the macro; possibly the register allocator will have
11722 // to add it back (and maybe not).
11723 instruct addF_mulF_reg(regF src2, regF src1, regF src0) %{
11724   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11725   match(Set src2 (AddF (MulF src0 src1) src2));
11726 
11727   format %{ "FLD    $src0     ===MACRO3===\n\t"
11728             "FMUL   ST,$src1\n\t"
11729             "FADDP  $src2,ST" %}
11730   opcode(0xD9); /* LoadF D9 /0 */
11731   ins_encode( Push_Reg_F(src0),
11732               FMul_ST_reg(src1),
11733               FAddP_reg_ST(src2) );
11734   ins_pipe( fpu_reg_reg_reg );
11735 %}
11736 
11737 // MACRO4 -- divF subF
11738 // This instruction does not round to 24-bits
11739 instruct subF_divF_reg(regF dst, regF src1, regF src2, regF src3) %{
11740   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11741   match(Set dst (DivF (SubF src2 src1) src3));
11742 
11743   format %{ "FLD    $src2   ===MACRO4===\n\t"
11744             "FSUB   ST,$src1\n\t"
11745             "FDIV   ST,$src3\n\t"
11746             "FSTP  $dst" %}
11747   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
11748   ins_encode( Push_Reg_F(src2),
11749               subF_divF_encode(src1,src3),
11750               Pop_Reg_F(dst) );
11751   ins_pipe( fpu_reg_reg_reg_reg );
11752 %}
11753 
11754 // Spill to obtain 24-bit precision
11755 instruct divF24_reg(stackSlotF dst, regF src1, regF src2) %{
11756   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11757   match(Set dst (DivF src1 src2));
11758 
11759   format %{ "FDIV   $dst,$src1,$src2" %}
11760   opcode(0xD8, 0x6); /* D8 F0+i or DE /6*/
11761   ins_encode( Push_Reg_F(src1),
11762               OpcReg_F(src2),
11763               Pop_Mem_F(dst) );
11764   ins_pipe( fpu_mem_reg_reg );
11765 %}
11766 //
11767 // This instruction does not round to 24-bits
11768 instruct divF_reg(regF dst, regF src) %{
11769   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11770   match(Set dst (DivF dst src));
11771 
11772   format %{ "FDIV   $dst,$src" %}
11773   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
11774   ins_encode( Push_Reg_F(src),
11775               OpcP, RegOpc(dst) );
11776   ins_pipe( fpu_reg_reg );
11777 %}
11778 
11779 
11780 // Spill to obtain 24-bit precision
11781 instruct modF24_reg(stackSlotF dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
11782   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11783   match(Set dst (ModF src1 src2));
11784   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
11785 
11786   format %{ "FMOD   $dst,$src1,$src2" %}
11787   ins_encode( Push_Reg_Mod_D(src1, src2),
11788               emitModD(),
11789               Push_Result_Mod_D(src2),
11790               Pop_Mem_F(dst));
11791   ins_pipe( pipe_slow );
11792 %}
11793 //
11794 // This instruction does not round to 24-bits
11795 instruct modF_reg(regF dst, regF src, eAXRegI rax, eFlagsReg cr) %{
11796   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11797   match(Set dst (ModF dst src));
11798   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
11799 
11800   format %{ "FMOD   $dst,$src" %}
11801   ins_encode(Push_Reg_Mod_D(dst, src),
11802               emitModD(),
11803               Push_Result_Mod_D(src),
11804               Pop_Reg_F(dst));
11805   ins_pipe( pipe_slow );
11806 %}
11807 
11808 instruct modX_reg(regX dst, regX src0, regX src1, eAXRegI rax, eFlagsReg cr) %{
11809   predicate(UseSSE>=1);
11810   match(Set dst (ModF src0 src1));
11811   effect(KILL rax, KILL cr);
11812   format %{ "SUB    ESP,4\t # FMOD\n"
11813           "\tMOVSS  [ESP+0],$src1\n"
11814           "\tFLD_S  [ESP+0]\n"
11815           "\tMOVSS  [ESP+0],$src0\n"
11816           "\tFLD_S  [ESP+0]\n"
11817      "loop:\tFPREM\n"
11818           "\tFWAIT\n"
11819           "\tFNSTSW AX\n"
11820           "\tSAHF\n"
11821           "\tJP     loop\n"
11822           "\tFSTP_S [ESP+0]\n"
11823           "\tMOVSS  $dst,[ESP+0]\n"
11824           "\tADD    ESP,4\n"
11825           "\tFSTP   ST0\t # Restore FPU Stack"
11826     %}
11827   ins_cost(250);
11828   ins_encode( Push_ModX_encoding(src0, src1), emitModD(), Push_ResultX(dst,0x4), PopFPU);
11829   ins_pipe( pipe_slow );
11830 %}
11831 
11832 
11833 //----------Arithmetic Conversion Instructions---------------------------------
11834 // The conversions operations are all Alpha sorted.  Please keep it that way!
11835 
11836 instruct roundFloat_mem_reg(stackSlotF dst, regF src) %{
11837   predicate(UseSSE==0);
11838   match(Set dst (RoundFloat src));
11839   ins_cost(125);
11840   format %{ "FST_S  $dst,$src\t# F-round" %}
11841   ins_encode( Pop_Mem_Reg_F(dst, src) );
11842   ins_pipe( fpu_mem_reg );
11843 %}
11844 
11845 instruct roundDouble_mem_reg(stackSlotD dst, regD src) %{
11846   predicate(UseSSE<=1);
11847   match(Set dst (RoundDouble src));
11848   ins_cost(125);
11849   format %{ "FST_D  $dst,$src\t# D-round" %}
11850   ins_encode( Pop_Mem_Reg_D(dst, src) );
11851   ins_pipe( fpu_mem_reg );
11852 %}
11853 
11854 // Force rounding to 24-bit precision and 6-bit exponent
11855 instruct convD2F_reg(stackSlotF dst, regD src) %{
11856   predicate(UseSSE==0);
11857   match(Set dst (ConvD2F src));
11858   format %{ "FST_S  $dst,$src\t# F-round" %}
11859   expand %{
11860     roundFloat_mem_reg(dst,src);
11861   %}
11862 %}
11863 
11864 // Force rounding to 24-bit precision and 6-bit exponent
11865 instruct convD2X_reg(regX dst, regD src, eFlagsReg cr) %{
11866   predicate(UseSSE==1);
11867   match(Set dst (ConvD2F src));
11868   effect( KILL cr );
11869   format %{ "SUB    ESP,4\n\t"
11870             "FST_S  [ESP],$src\t# F-round\n\t"
11871             "MOVSS  $dst,[ESP]\n\t"
11872             "ADD ESP,4" %}
11873   ins_encode( D2X_encoding(dst, src) );
11874   ins_pipe( pipe_slow );
11875 %}
11876 
11877 // Force rounding double precision to single precision
11878 instruct convXD2X_reg(regX dst, regXD src) %{
11879   predicate(UseSSE>=2);
11880   match(Set dst (ConvD2F src));
11881   format %{ "CVTSD2SS $dst,$src\t# F-round" %}
11882   opcode(0xF2, 0x0F, 0x5A);
11883   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11884   ins_pipe( pipe_slow );
11885 %}
11886 
11887 instruct convF2D_reg_reg(regD dst, regF src) %{
11888   predicate(UseSSE==0);
11889   match(Set dst (ConvF2D src));
11890   format %{ "FST_S  $dst,$src\t# D-round" %}
11891   ins_encode( Pop_Reg_Reg_D(dst, src));
11892   ins_pipe( fpu_reg_reg );
11893 %}
11894 
11895 instruct convF2D_reg(stackSlotD dst, regF src) %{
11896   predicate(UseSSE==1);
11897   match(Set dst (ConvF2D src));
11898   format %{ "FST_D  $dst,$src\t# D-round" %}
11899   expand %{
11900     roundDouble_mem_reg(dst,src);
11901   %}
11902 %}
11903 
11904 instruct convX2D_reg(regD dst, regX src, eFlagsReg cr) %{
11905   predicate(UseSSE==1);
11906   match(Set dst (ConvF2D src));
11907   effect( KILL cr );
11908   format %{ "SUB    ESP,4\n\t"
11909             "MOVSS  [ESP] $src\n\t"
11910             "FLD_S  [ESP]\n\t"
11911             "ADD    ESP,4\n\t"
11912             "FSTP   $dst\t# D-round" %}
11913   ins_encode( X2D_encoding(dst, src), Pop_Reg_D(dst));
11914   ins_pipe( pipe_slow );
11915 %}
11916 
11917 instruct convX2XD_reg(regXD dst, regX src) %{
11918   predicate(UseSSE>=2);
11919   match(Set dst (ConvF2D src));
11920   format %{ "CVTSS2SD $dst,$src\t# D-round" %}
11921   opcode(0xF3, 0x0F, 0x5A);
11922   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11923   ins_pipe( pipe_slow );
11924 %}
11925 
11926 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
11927 instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{
11928   predicate(UseSSE<=1);
11929   match(Set dst (ConvD2I src));
11930   effect( KILL tmp, KILL cr );
11931   format %{ "FLD    $src\t# Convert double to int \n\t"
11932             "FLDCW  trunc mode\n\t"
11933             "SUB    ESP,4\n\t"
11934             "FISTp  [ESP + #0]\n\t"
11935             "FLDCW  std/24-bit mode\n\t"
11936             "POP    EAX\n\t"
11937             "CMP    EAX,0x80000000\n\t"
11938             "JNE,s  fast\n\t"
11939             "FLD_D  $src\n\t"
11940             "CALL   d2i_wrapper\n"
11941       "fast:" %}
11942   ins_encode( Push_Reg_D(src), D2I_encoding(src) );
11943   ins_pipe( pipe_slow );
11944 %}
11945 
11946 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
11947 instruct convXD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regXD src, eFlagsReg cr ) %{
11948   predicate(UseSSE>=2);
11949   match(Set dst (ConvD2I src));
11950   effect( KILL tmp, KILL cr );
11951   format %{ "CVTTSD2SI $dst, $src\n\t"
11952             "CMP    $dst,0x80000000\n\t"
11953             "JNE,s  fast\n\t"
11954             "SUB    ESP, 8\n\t"
11955             "MOVSD  [ESP], $src\n\t"
11956             "FLD_D  [ESP]\n\t"
11957             "ADD    ESP, 8\n\t"
11958             "CALL   d2i_wrapper\n"
11959       "fast:" %}
11960   opcode(0x1); // double-precision conversion
11961   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
11962   ins_pipe( pipe_slow );
11963 %}
11964 
11965 instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{
11966   predicate(UseSSE<=1);
11967   match(Set dst (ConvD2L src));
11968   effect( KILL cr );
11969   format %{ "FLD    $src\t# Convert double to long\n\t"
11970             "FLDCW  trunc mode\n\t"
11971             "SUB    ESP,8\n\t"
11972             "FISTp  [ESP + #0]\n\t"
11973             "FLDCW  std/24-bit mode\n\t"
11974             "POP    EAX\n\t"
11975             "POP    EDX\n\t"
11976             "CMP    EDX,0x80000000\n\t"
11977             "JNE,s  fast\n\t"
11978             "TEST   EAX,EAX\n\t"
11979             "JNE,s  fast\n\t"
11980             "FLD    $src\n\t"
11981             "CALL   d2l_wrapper\n"
11982       "fast:" %}
11983   ins_encode( Push_Reg_D(src),  D2L_encoding(src) );
11984   ins_pipe( pipe_slow );
11985 %}
11986 
11987 // XMM lacks a float/double->long conversion, so use the old FPU stack.
11988 instruct convXD2L_reg_reg( eADXRegL dst, regXD src, eFlagsReg cr ) %{
11989   predicate (UseSSE>=2);
11990   match(Set dst (ConvD2L src));
11991   effect( KILL cr );
11992   format %{ "SUB    ESP,8\t# Convert double to long\n\t"
11993             "MOVSD  [ESP],$src\n\t"
11994             "FLD_D  [ESP]\n\t"
11995             "FLDCW  trunc mode\n\t"
11996             "FISTp  [ESP + #0]\n\t"
11997             "FLDCW  std/24-bit mode\n\t"
11998             "POP    EAX\n\t"
11999             "POP    EDX\n\t"
12000             "CMP    EDX,0x80000000\n\t"
12001             "JNE,s  fast\n\t"
12002             "TEST   EAX,EAX\n\t"
12003             "JNE,s  fast\n\t"
12004             "SUB    ESP,8\n\t"
12005             "MOVSD  [ESP],$src\n\t"
12006             "FLD_D  [ESP]\n\t"
12007             "CALL   d2l_wrapper\n"
12008       "fast:" %}
12009   ins_encode( XD2L_encoding(src) );
12010   ins_pipe( pipe_slow );
12011 %}
12012 
12013 // Convert a double to an int.  Java semantics require we do complex
12014 // manglations in the corner cases.  So we set the rounding mode to
12015 // 'zero', store the darned double down as an int, and reset the
12016 // rounding mode to 'nearest'.  The hardware stores a flag value down
12017 // if we would overflow or converted a NAN; we check for this and
12018 // and go the slow path if needed.
12019 instruct convF2I_reg_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{
12020   predicate(UseSSE==0);
12021   match(Set dst (ConvF2I src));
12022   effect( KILL tmp, KILL cr );
12023   format %{ "FLD    $src\t# Convert float to int \n\t"
12024             "FLDCW  trunc mode\n\t"
12025             "SUB    ESP,4\n\t"
12026             "FISTp  [ESP + #0]\n\t"
12027             "FLDCW  std/24-bit mode\n\t"
12028             "POP    EAX\n\t"
12029             "CMP    EAX,0x80000000\n\t"
12030             "JNE,s  fast\n\t"
12031             "FLD    $src\n\t"
12032             "CALL   d2i_wrapper\n"
12033       "fast:" %}
12034   // D2I_encoding works for F2I
12035   ins_encode( Push_Reg_F(src), D2I_encoding(src) );
12036   ins_pipe( pipe_slow );
12037 %}
12038 
12039 // Convert a float in xmm to an int reg.
12040 instruct convX2I_reg(eAXRegI dst, eDXRegI tmp, regX src, eFlagsReg cr ) %{
12041   predicate(UseSSE>=1);
12042   match(Set dst (ConvF2I src));
12043   effect( KILL tmp, KILL cr );
12044   format %{ "CVTTSS2SI $dst, $src\n\t"
12045             "CMP    $dst,0x80000000\n\t"
12046             "JNE,s  fast\n\t"
12047             "SUB    ESP, 4\n\t"
12048             "MOVSS  [ESP], $src\n\t"
12049             "FLD    [ESP]\n\t"
12050             "ADD    ESP, 4\n\t"
12051             "CALL   d2i_wrapper\n"
12052       "fast:" %}
12053   opcode(0x0); // single-precision conversion
12054   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
12055   ins_pipe( pipe_slow );
12056 %}
12057 
12058 instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{
12059   predicate(UseSSE==0);
12060   match(Set dst (ConvF2L src));
12061   effect( KILL cr );
12062   format %{ "FLD    $src\t# Convert float to long\n\t"
12063             "FLDCW  trunc mode\n\t"
12064             "SUB    ESP,8\n\t"
12065             "FISTp  [ESP + #0]\n\t"
12066             "FLDCW  std/24-bit mode\n\t"
12067             "POP    EAX\n\t"
12068             "POP    EDX\n\t"
12069             "CMP    EDX,0x80000000\n\t"
12070             "JNE,s  fast\n\t"
12071             "TEST   EAX,EAX\n\t"
12072             "JNE,s  fast\n\t"
12073             "FLD    $src\n\t"
12074             "CALL   d2l_wrapper\n"
12075       "fast:" %}
12076   // D2L_encoding works for F2L
12077   ins_encode( Push_Reg_F(src), D2L_encoding(src) );
12078   ins_pipe( pipe_slow );
12079 %}
12080 
12081 // XMM lacks a float/double->long conversion, so use the old FPU stack.
12082 instruct convX2L_reg_reg( eADXRegL dst, regX src, eFlagsReg cr ) %{
12083   predicate (UseSSE>=1);
12084   match(Set dst (ConvF2L src));
12085   effect( KILL cr );
12086   format %{ "SUB    ESP,8\t# Convert float to long\n\t"
12087             "MOVSS  [ESP],$src\n\t"
12088             "FLD_S  [ESP]\n\t"
12089             "FLDCW  trunc mode\n\t"
12090             "FISTp  [ESP + #0]\n\t"
12091             "FLDCW  std/24-bit mode\n\t"
12092             "POP    EAX\n\t"
12093             "POP    EDX\n\t"
12094             "CMP    EDX,0x80000000\n\t"
12095             "JNE,s  fast\n\t"
12096             "TEST   EAX,EAX\n\t"
12097             "JNE,s  fast\n\t"
12098             "SUB    ESP,4\t# Convert float to long\n\t"
12099             "MOVSS  [ESP],$src\n\t"
12100             "FLD_S  [ESP]\n\t"
12101             "ADD    ESP,4\n\t"
12102             "CALL   d2l_wrapper\n"
12103       "fast:" %}
12104   ins_encode( X2L_encoding(src) );
12105   ins_pipe( pipe_slow );
12106 %}
12107 
12108 instruct convI2D_reg(regD dst, stackSlotI src) %{
12109   predicate( UseSSE<=1 );
12110   match(Set dst (ConvI2D src));
12111   format %{ "FILD   $src\n\t"
12112             "FSTP   $dst" %}
12113   opcode(0xDB, 0x0);  /* DB /0 */
12114   ins_encode(Push_Mem_I(src), Pop_Reg_D(dst));
12115   ins_pipe( fpu_reg_mem );
12116 %}
12117 
12118 instruct convI2XD_reg(regXD dst, eRegI src) %{
12119   predicate( UseSSE>=2 && !UseXmmI2D );
12120   match(Set dst (ConvI2D src));
12121   format %{ "CVTSI2SD $dst,$src" %}
12122   opcode(0xF2, 0x0F, 0x2A);
12123   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
12124   ins_pipe( pipe_slow );
12125 %}
12126 
12127 instruct convI2XD_mem(regXD dst, memory mem) %{
12128   predicate( UseSSE>=2 );
12129   match(Set dst (ConvI2D (LoadI mem)));
12130   format %{ "CVTSI2SD $dst,$mem" %}
12131   opcode(0xF2, 0x0F, 0x2A);
12132   ins_encode( OpcP, OpcS, Opcode(tertiary), RegMem(dst, mem));
12133   ins_pipe( pipe_slow );
12134 %}
12135 
12136 instruct convXI2XD_reg(regXD dst, eRegI src)
12137 %{
12138   predicate( UseSSE>=2 && UseXmmI2D );
12139   match(Set dst (ConvI2D src));
12140 
12141   format %{ "MOVD  $dst,$src\n\t"
12142             "CVTDQ2PD $dst,$dst\t# i2d" %}
12143   ins_encode %{
12144     __ movdl($dst$$XMMRegister, $src$$Register);
12145     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
12146   %}
12147   ins_pipe(pipe_slow); // XXX
12148 %}
12149 
12150 instruct convI2D_mem(regD dst, memory mem) %{
12151   predicate( UseSSE<=1 && !Compile::current()->select_24_bit_instr());
12152   match(Set dst (ConvI2D (LoadI mem)));
12153   format %{ "FILD   $mem\n\t"
12154             "FSTP   $dst" %}
12155   opcode(0xDB);      /* DB /0 */
12156   ins_encode( OpcP, RMopc_Mem(0x00,mem),
12157               Pop_Reg_D(dst));
12158   ins_pipe( fpu_reg_mem );
12159 %}
12160 
12161 // Convert a byte to a float; no rounding step needed.
12162 instruct conv24I2F_reg(regF dst, stackSlotI src) %{
12163   predicate( UseSSE==0 && n->in(1)->Opcode() == Op_AndI && n->in(1)->in(2)->is_Con() && n->in(1)->in(2)->get_int() == 255 );
12164   match(Set dst (ConvI2F src));
12165   format %{ "FILD   $src\n\t"
12166             "FSTP   $dst" %}
12167 
12168   opcode(0xDB, 0x0);  /* DB /0 */
12169   ins_encode(Push_Mem_I(src), Pop_Reg_F(dst));
12170   ins_pipe( fpu_reg_mem );
12171 %}
12172 
12173 // In 24-bit mode, force exponent rounding by storing back out
12174 instruct convI2F_SSF(stackSlotF dst, stackSlotI src) %{
12175   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
12176   match(Set dst (ConvI2F src));
12177   ins_cost(200);
12178   format %{ "FILD   $src\n\t"
12179             "FSTP_S $dst" %}
12180   opcode(0xDB, 0x0);  /* DB /0 */
12181   ins_encode( Push_Mem_I(src),
12182               Pop_Mem_F(dst));
12183   ins_pipe( fpu_mem_mem );
12184 %}
12185 
12186 // In 24-bit mode, force exponent rounding by storing back out
12187 instruct convI2F_SSF_mem(stackSlotF dst, memory mem) %{
12188   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
12189   match(Set dst (ConvI2F (LoadI mem)));
12190   ins_cost(200);
12191   format %{ "FILD   $mem\n\t"
12192             "FSTP_S $dst" %}
12193   opcode(0xDB);  /* DB /0 */
12194   ins_encode( OpcP, RMopc_Mem(0x00,mem),
12195               Pop_Mem_F(dst));
12196   ins_pipe( fpu_mem_mem );
12197 %}
12198 
12199 // This instruction does not round to 24-bits
12200 instruct convI2F_reg(regF dst, stackSlotI src) %{
12201   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
12202   match(Set dst (ConvI2F src));
12203   format %{ "FILD   $src\n\t"
12204             "FSTP   $dst" %}
12205   opcode(0xDB, 0x0);  /* DB /0 */
12206   ins_encode( Push_Mem_I(src),
12207               Pop_Reg_F(dst));
12208   ins_pipe( fpu_reg_mem );
12209 %}
12210 
12211 // This instruction does not round to 24-bits
12212 instruct convI2F_mem(regF dst, memory mem) %{
12213   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
12214   match(Set dst (ConvI2F (LoadI mem)));
12215   format %{ "FILD   $mem\n\t"
12216             "FSTP   $dst" %}
12217   opcode(0xDB);      /* DB /0 */
12218   ins_encode( OpcP, RMopc_Mem(0x00,mem),
12219               Pop_Reg_F(dst));
12220   ins_pipe( fpu_reg_mem );
12221 %}
12222 
12223 // Convert an int to a float in xmm; no rounding step needed.
12224 instruct convI2X_reg(regX dst, eRegI src) %{
12225   predicate( UseSSE==1 || UseSSE>=2 && !UseXmmI2F );
12226   match(Set dst (ConvI2F src));
12227   format %{ "CVTSI2SS $dst, $src" %}
12228 
12229   opcode(0xF3, 0x0F, 0x2A);  /* F3 0F 2A /r */
12230   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
12231   ins_pipe( pipe_slow );
12232 %}
12233 
12234  instruct convXI2X_reg(regX dst, eRegI src)
12235 %{
12236   predicate( UseSSE>=2 && UseXmmI2F );
12237   match(Set dst (ConvI2F src));
12238 
12239   format %{ "MOVD  $dst,$src\n\t"
12240             "CVTDQ2PS $dst,$dst\t# i2f" %}
12241   ins_encode %{
12242     __ movdl($dst$$XMMRegister, $src$$Register);
12243     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
12244   %}
12245   ins_pipe(pipe_slow); // XXX
12246 %}
12247 
12248 instruct convI2L_reg( eRegL dst, eRegI src, eFlagsReg cr) %{
12249   match(Set dst (ConvI2L src));
12250   effect(KILL cr);
12251   ins_cost(375);
12252   format %{ "MOV    $dst.lo,$src\n\t"
12253             "MOV    $dst.hi,$src\n\t"
12254             "SAR    $dst.hi,31" %}
12255   ins_encode(convert_int_long(dst,src));
12256   ins_pipe( ialu_reg_reg_long );
12257 %}
12258 
12259 // Zero-extend convert int to long
12260 instruct convI2L_reg_zex(eRegL dst, eRegI src, immL_32bits mask, eFlagsReg flags ) %{
12261   match(Set dst (AndL (ConvI2L src) mask) );
12262   effect( KILL flags );
12263   ins_cost(250);
12264   format %{ "MOV    $dst.lo,$src\n\t"
12265             "XOR    $dst.hi,$dst.hi" %}
12266   opcode(0x33); // XOR
12267   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
12268   ins_pipe( ialu_reg_reg_long );
12269 %}
12270 
12271 // Zero-extend long
12272 instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
12273   match(Set dst (AndL src mask) );
12274   effect( KILL flags );
12275   ins_cost(250);
12276   format %{ "MOV    $dst.lo,$src.lo\n\t"
12277             "XOR    $dst.hi,$dst.hi\n\t" %}
12278   opcode(0x33); // XOR
12279   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
12280   ins_pipe( ialu_reg_reg_long );
12281 %}
12282 
12283 instruct convL2D_reg( stackSlotD dst, eRegL src, eFlagsReg cr) %{
12284   predicate (UseSSE<=1);
12285   match(Set dst (ConvL2D src));
12286   effect( KILL cr );
12287   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
12288             "PUSH   $src.lo\n\t"
12289             "FILD   ST,[ESP + #0]\n\t"
12290             "ADD    ESP,8\n\t"
12291             "FSTP_D $dst\t# D-round" %}
12292   opcode(0xDF, 0x5);  /* DF /5 */
12293   ins_encode(convert_long_double(src), Pop_Mem_D(dst));
12294   ins_pipe( pipe_slow );
12295 %}
12296 
12297 instruct convL2XD_reg( regXD dst, eRegL src, eFlagsReg cr) %{
12298   predicate (UseSSE>=2);
12299   match(Set dst (ConvL2D src));
12300   effect( KILL cr );
12301   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
12302             "PUSH   $src.lo\n\t"
12303             "FILD_D [ESP]\n\t"
12304             "FSTP_D [ESP]\n\t"
12305             "MOVSD  $dst,[ESP]\n\t"
12306             "ADD    ESP,8" %}
12307   opcode(0xDF, 0x5);  /* DF /5 */
12308   ins_encode(convert_long_double2(src), Push_ResultXD(dst));
12309   ins_pipe( pipe_slow );
12310 %}
12311 
12312 instruct convL2X_reg( regX dst, eRegL src, eFlagsReg cr) %{
12313   predicate (UseSSE>=1);
12314   match(Set dst (ConvL2F src));
12315   effect( KILL cr );
12316   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
12317             "PUSH   $src.lo\n\t"
12318             "FILD_D [ESP]\n\t"
12319             "FSTP_S [ESP]\n\t"
12320             "MOVSS  $dst,[ESP]\n\t"
12321             "ADD    ESP,8" %}
12322   opcode(0xDF, 0x5);  /* DF /5 */
12323   ins_encode(convert_long_double2(src), Push_ResultX(dst,0x8));
12324   ins_pipe( pipe_slow );
12325 %}
12326 
12327 instruct convL2F_reg( stackSlotF dst, eRegL src, eFlagsReg cr) %{
12328   match(Set dst (ConvL2F src));
12329   effect( KILL cr );
12330   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
12331             "PUSH   $src.lo\n\t"
12332             "FILD   ST,[ESP + #0]\n\t"
12333             "ADD    ESP,8\n\t"
12334             "FSTP_S $dst\t# F-round" %}
12335   opcode(0xDF, 0x5);  /* DF /5 */
12336   ins_encode(convert_long_double(src), Pop_Mem_F(dst));
12337   ins_pipe( pipe_slow );
12338 %}
12339 
12340 instruct convL2I_reg( eRegI dst, eRegL src ) %{
12341   match(Set dst (ConvL2I src));
12342   effect( DEF dst, USE src );
12343   format %{ "MOV    $dst,$src.lo" %}
12344   ins_encode(enc_CopyL_Lo(dst,src));
12345   ins_pipe( ialu_reg_reg );
12346 %}
12347 
12348 
12349 instruct MoveF2I_stack_reg(eRegI dst, stackSlotF src) %{
12350   match(Set dst (MoveF2I src));
12351   effect( DEF dst, USE src );
12352   ins_cost(100);
12353   format %{ "MOV    $dst,$src\t# MoveF2I_stack_reg" %}
12354   opcode(0x8B);
12355   ins_encode( OpcP, RegMem(dst,src));
12356   ins_pipe( ialu_reg_mem );
12357 %}
12358 
12359 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12360   predicate(UseSSE==0);
12361   match(Set dst (MoveF2I src));
12362   effect( DEF dst, USE src );
12363 
12364   ins_cost(125);
12365   format %{ "FST_S  $dst,$src\t# MoveF2I_reg_stack" %}
12366   ins_encode( Pop_Mem_Reg_F(dst, src) );
12367   ins_pipe( fpu_mem_reg );
12368 %}
12369 
12370 instruct MoveF2I_reg_stack_sse(stackSlotI dst, regX src) %{
12371   predicate(UseSSE>=1);
12372   match(Set dst (MoveF2I src));
12373   effect( DEF dst, USE src );
12374 
12375   ins_cost(95);
12376   format %{ "MOVSS  $dst,$src\t# MoveF2I_reg_stack_sse" %}
12377   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, dst));
12378   ins_pipe( pipe_slow );
12379 %}
12380 
12381 instruct MoveF2I_reg_reg_sse(eRegI dst, regX src) %{
12382   predicate(UseSSE>=2);
12383   match(Set dst (MoveF2I src));
12384   effect( DEF dst, USE src );
12385   ins_cost(85);
12386   format %{ "MOVD   $dst,$src\t# MoveF2I_reg_reg_sse" %}
12387   ins_encode( MovX2I_reg(dst, src));
12388   ins_pipe( pipe_slow );
12389 %}
12390 
12391 instruct MoveI2F_reg_stack(stackSlotF dst, eRegI src) %{
12392   match(Set dst (MoveI2F src));
12393   effect( DEF dst, USE src );
12394 
12395   ins_cost(100);
12396   format %{ "MOV    $dst,$src\t# MoveI2F_reg_stack" %}
12397   opcode(0x89);
12398   ins_encode( OpcPRegSS( dst, src ) );
12399   ins_pipe( ialu_mem_reg );
12400 %}
12401 
12402 
12403 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12404   predicate(UseSSE==0);
12405   match(Set dst (MoveI2F src));
12406   effect(DEF dst, USE src);
12407 
12408   ins_cost(125);
12409   format %{ "FLD_S  $src\n\t"
12410             "FSTP   $dst\t# MoveI2F_stack_reg" %}
12411   opcode(0xD9);               /* D9 /0, FLD m32real */
12412   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
12413               Pop_Reg_F(dst) );
12414   ins_pipe( fpu_reg_mem );
12415 %}
12416 
12417 instruct MoveI2F_stack_reg_sse(regX dst, stackSlotI src) %{
12418   predicate(UseSSE>=1);
12419   match(Set dst (MoveI2F src));
12420   effect( DEF dst, USE src );
12421 
12422   ins_cost(95);
12423   format %{ "MOVSS  $dst,$src\t# MoveI2F_stack_reg_sse" %}
12424   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
12425   ins_pipe( pipe_slow );
12426 %}
12427 
12428 instruct MoveI2F_reg_reg_sse(regX dst, eRegI src) %{
12429   predicate(UseSSE>=2);
12430   match(Set dst (MoveI2F src));
12431   effect( DEF dst, USE src );
12432 
12433   ins_cost(85);
12434   format %{ "MOVD   $dst,$src\t# MoveI2F_reg_reg_sse" %}
12435   ins_encode( MovI2X_reg(dst, src) );
12436   ins_pipe( pipe_slow );
12437 %}
12438 
12439 instruct MoveD2L_stack_reg(eRegL dst, stackSlotD src) %{
12440   match(Set dst (MoveD2L src));
12441   effect(DEF dst, USE src);
12442 
12443   ins_cost(250);
12444   format %{ "MOV    $dst.lo,$src\n\t"
12445             "MOV    $dst.hi,$src+4\t# MoveD2L_stack_reg" %}
12446   opcode(0x8B, 0x8B);
12447   ins_encode( OpcP, RegMem(dst,src), OpcS, RegMem_Hi(dst,src));
12448   ins_pipe( ialu_mem_long_reg );
12449 %}
12450 
12451 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12452   predicate(UseSSE<=1);
12453   match(Set dst (MoveD2L src));
12454   effect(DEF dst, USE src);
12455 
12456   ins_cost(125);
12457   format %{ "FST_D  $dst,$src\t# MoveD2L_reg_stack" %}
12458   ins_encode( Pop_Mem_Reg_D(dst, src) );
12459   ins_pipe( fpu_mem_reg );
12460 %}
12461 
12462 instruct MoveD2L_reg_stack_sse(stackSlotL dst, regXD src) %{
12463   predicate(UseSSE>=2);
12464   match(Set dst (MoveD2L src));
12465   effect(DEF dst, USE src);
12466   ins_cost(95);
12467 
12468   format %{ "MOVSD  $dst,$src\t# MoveD2L_reg_stack_sse" %}
12469   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src,dst));
12470   ins_pipe( pipe_slow );
12471 %}
12472 
12473 instruct MoveD2L_reg_reg_sse(eRegL dst, regXD src, regXD tmp) %{
12474   predicate(UseSSE>=2);
12475   match(Set dst (MoveD2L src));
12476   effect(DEF dst, USE src, TEMP tmp);
12477   ins_cost(85);
12478   format %{ "MOVD   $dst.lo,$src\n\t"
12479             "PSHUFLW $tmp,$src,0x4E\n\t"
12480             "MOVD   $dst.hi,$tmp\t# MoveD2L_reg_reg_sse" %}
12481   ins_encode( MovXD2L_reg(dst, src, tmp) );
12482   ins_pipe( pipe_slow );
12483 %}
12484 
12485 instruct MoveL2D_reg_stack(stackSlotD dst, eRegL src) %{
12486   match(Set dst (MoveL2D src));
12487   effect(DEF dst, USE src);
12488 
12489   ins_cost(200);
12490   format %{ "MOV    $dst,$src.lo\n\t"
12491             "MOV    $dst+4,$src.hi\t# MoveL2D_reg_stack" %}
12492   opcode(0x89, 0x89);
12493   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
12494   ins_pipe( ialu_mem_long_reg );
12495 %}
12496 
12497 
12498 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12499   predicate(UseSSE<=1);
12500   match(Set dst (MoveL2D src));
12501   effect(DEF dst, USE src);
12502   ins_cost(125);
12503 
12504   format %{ "FLD_D  $src\n\t"
12505             "FSTP   $dst\t# MoveL2D_stack_reg" %}
12506   opcode(0xDD);               /* DD /0, FLD m64real */
12507   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
12508               Pop_Reg_D(dst) );
12509   ins_pipe( fpu_reg_mem );
12510 %}
12511 
12512 
12513 instruct MoveL2D_stack_reg_sse(regXD dst, stackSlotL src) %{
12514   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
12515   match(Set dst (MoveL2D src));
12516   effect(DEF dst, USE src);
12517 
12518   ins_cost(95);
12519   format %{ "MOVSD  $dst,$src\t# MoveL2D_stack_reg_sse" %}
12520   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
12521   ins_pipe( pipe_slow );
12522 %}
12523 
12524 instruct MoveL2D_stack_reg_sse_partial(regXD dst, stackSlotL src) %{
12525   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
12526   match(Set dst (MoveL2D src));
12527   effect(DEF dst, USE src);
12528 
12529   ins_cost(95);
12530   format %{ "MOVLPD $dst,$src\t# MoveL2D_stack_reg_sse" %}
12531   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,src));
12532   ins_pipe( pipe_slow );
12533 %}
12534 
12535 instruct MoveL2D_reg_reg_sse(regXD dst, eRegL src, regXD tmp) %{
12536   predicate(UseSSE>=2);
12537   match(Set dst (MoveL2D src));
12538   effect(TEMP dst, USE src, TEMP tmp);
12539   ins_cost(85);
12540   format %{ "MOVD   $dst,$src.lo\n\t"
12541             "MOVD   $tmp,$src.hi\n\t"
12542             "PUNPCKLDQ $dst,$tmp\t# MoveL2D_reg_reg_sse" %}
12543   ins_encode( MovL2XD_reg(dst, src, tmp) );
12544   ins_pipe( pipe_slow );
12545 %}
12546 
12547 // Replicate scalar to packed byte (1 byte) values in xmm
12548 instruct Repl8B_reg(regXD dst, regXD src) %{
12549   predicate(UseSSE>=2);
12550   match(Set dst (Replicate8B src));
12551   format %{ "MOVDQA  $dst,$src\n\t"
12552             "PUNPCKLBW $dst,$dst\n\t"
12553             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
12554   ins_encode( pshufd_8x8(dst, src));
12555   ins_pipe( pipe_slow );
12556 %}
12557 
12558 // Replicate scalar to packed byte (1 byte) values in xmm
12559 instruct Repl8B_eRegI(regXD dst, eRegI src) %{
12560   predicate(UseSSE>=2);
12561   match(Set dst (Replicate8B src));
12562   format %{ "MOVD    $dst,$src\n\t"
12563             "PUNPCKLBW $dst,$dst\n\t"
12564             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
12565   ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
12566   ins_pipe( pipe_slow );
12567 %}
12568 
12569 // Replicate scalar zero to packed byte (1 byte) values in xmm
12570 instruct Repl8B_immI0(regXD dst, immI0 zero) %{
12571   predicate(UseSSE>=2);
12572   match(Set dst (Replicate8B zero));
12573   format %{ "PXOR  $dst,$dst\t! replicate8B" %}
12574   ins_encode( pxor(dst, dst));
12575   ins_pipe( fpu_reg_reg );
12576 %}
12577 
12578 // Replicate scalar to packed shore (2 byte) values in xmm
12579 instruct Repl4S_reg(regXD dst, regXD src) %{
12580   predicate(UseSSE>=2);
12581   match(Set dst (Replicate4S src));
12582   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
12583   ins_encode( pshufd_4x16(dst, src));
12584   ins_pipe( fpu_reg_reg );
12585 %}
12586 
12587 // Replicate scalar to packed shore (2 byte) values in xmm
12588 instruct Repl4S_eRegI(regXD dst, eRegI src) %{
12589   predicate(UseSSE>=2);
12590   match(Set dst (Replicate4S src));
12591   format %{ "MOVD    $dst,$src\n\t"
12592             "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
12593   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
12594   ins_pipe( fpu_reg_reg );
12595 %}
12596 
12597 // Replicate scalar zero to packed short (2 byte) values in xmm
12598 instruct Repl4S_immI0(regXD dst, immI0 zero) %{
12599   predicate(UseSSE>=2);
12600   match(Set dst (Replicate4S zero));
12601   format %{ "PXOR  $dst,$dst\t! replicate4S" %}
12602   ins_encode( pxor(dst, dst));
12603   ins_pipe( fpu_reg_reg );
12604 %}
12605 
12606 // Replicate scalar to packed char (2 byte) values in xmm
12607 instruct Repl4C_reg(regXD dst, regXD src) %{
12608   predicate(UseSSE>=2);
12609   match(Set dst (Replicate4C src));
12610   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
12611   ins_encode( pshufd_4x16(dst, src));
12612   ins_pipe( fpu_reg_reg );
12613 %}
12614 
12615 // Replicate scalar to packed char (2 byte) values in xmm
12616 instruct Repl4C_eRegI(regXD dst, eRegI src) %{
12617   predicate(UseSSE>=2);
12618   match(Set dst (Replicate4C src));
12619   format %{ "MOVD    $dst,$src\n\t"
12620             "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
12621   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
12622   ins_pipe( fpu_reg_reg );
12623 %}
12624 
12625 // Replicate scalar zero to packed char (2 byte) values in xmm
12626 instruct Repl4C_immI0(regXD dst, immI0 zero) %{
12627   predicate(UseSSE>=2);
12628   match(Set dst (Replicate4C zero));
12629   format %{ "PXOR  $dst,$dst\t! replicate4C" %}
12630   ins_encode( pxor(dst, dst));
12631   ins_pipe( fpu_reg_reg );
12632 %}
12633 
12634 // Replicate scalar to packed integer (4 byte) values in xmm
12635 instruct Repl2I_reg(regXD dst, regXD src) %{
12636   predicate(UseSSE>=2);
12637   match(Set dst (Replicate2I src));
12638   format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
12639   ins_encode( pshufd(dst, src, 0x00));
12640   ins_pipe( fpu_reg_reg );
12641 %}
12642 
12643 // Replicate scalar to packed integer (4 byte) values in xmm
12644 instruct Repl2I_eRegI(regXD dst, eRegI src) %{
12645   predicate(UseSSE>=2);
12646   match(Set dst (Replicate2I src));
12647   format %{ "MOVD   $dst,$src\n\t"
12648             "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
12649   ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
12650   ins_pipe( fpu_reg_reg );
12651 %}
12652 
12653 // Replicate scalar zero to packed integer (2 byte) values in xmm
12654 instruct Repl2I_immI0(regXD dst, immI0 zero) %{
12655   predicate(UseSSE>=2);
12656   match(Set dst (Replicate2I zero));
12657   format %{ "PXOR  $dst,$dst\t! replicate2I" %}
12658   ins_encode( pxor(dst, dst));
12659   ins_pipe( fpu_reg_reg );
12660 %}
12661 
12662 // Replicate scalar to packed single precision floating point values in xmm
12663 instruct Repl2F_reg(regXD dst, regXD src) %{
12664   predicate(UseSSE>=2);
12665   match(Set dst (Replicate2F src));
12666   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
12667   ins_encode( pshufd(dst, src, 0xe0));
12668   ins_pipe( fpu_reg_reg );
12669 %}
12670 
12671 // Replicate scalar to packed single precision floating point values in xmm
12672 instruct Repl2F_regX(regXD dst, regX src) %{
12673   predicate(UseSSE>=2);
12674   match(Set dst (Replicate2F src));
12675   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
12676   ins_encode( pshufd(dst, src, 0xe0));
12677   ins_pipe( fpu_reg_reg );
12678 %}
12679 
12680 // Replicate scalar to packed single precision floating point values in xmm
12681 instruct Repl2F_immXF0(regXD dst, immXF0 zero) %{
12682   predicate(UseSSE>=2);
12683   match(Set dst (Replicate2F zero));
12684   format %{ "PXOR  $dst,$dst\t! replicate2F" %}
12685   ins_encode( pxor(dst, dst));
12686   ins_pipe( fpu_reg_reg );
12687 %}
12688 
12689 // =======================================================================
12690 // fast clearing of an array
12691 instruct rep_stos(eCXRegI cnt, eDIRegP base, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
12692   match(Set dummy (ClearArray cnt base));
12693   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
12694   format %{ "SHL    ECX,1\t# Convert doublewords to words\n\t"
12695             "XOR    EAX,EAX\n\t"
12696             "REP STOS\t# store EAX into [EDI++] while ECX--" %}
12697   opcode(0,0x4);
12698   ins_encode( Opcode(0xD1), RegOpc(ECX),
12699               OpcRegReg(0x33,EAX,EAX),
12700               Opcode(0xF3), Opcode(0xAB) );
12701   ins_pipe( pipe_slow );
12702 %}
12703 
12704 instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
12705                         eAXRegI result, regXD tmp1, eFlagsReg cr) %{
12706   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12707   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12708 
12709   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
12710   ins_encode %{
12711     __ string_compare($str1$$Register, $str2$$Register,
12712                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12713                       $tmp1$$XMMRegister);
12714   %}
12715   ins_pipe( pipe_slow );
12716 %}
12717 
12718 // fast string equals
12719 instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
12720                        regXD tmp1, regXD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
12721   match(Set result (StrEquals (Binary str1 str2) cnt));
12722   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12723 
12724   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12725   ins_encode %{
12726     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
12727                           $cnt$$Register, $result$$Register, $tmp3$$Register,
12728                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
12729   %}
12730   ins_pipe( pipe_slow );
12731 %}
12732 
12733 // fast search of substring with known size.
12734 instruct string_indexof_con(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
12735                             eBXRegI result, regXD vec, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
12736   predicate(UseSSE42Intrinsics);
12737   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
12738   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
12739 
12740   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
12741   ins_encode %{
12742     int icnt2 = (int)$int_cnt2$$constant;
12743     if (icnt2 >= 8) {
12744       // IndexOf for constant substrings with size >= 8 elements
12745       // which don't need to be loaded through stack.
12746       __ string_indexofC8($str1$$Register, $str2$$Register,
12747                           $cnt1$$Register, $cnt2$$Register,
12748                           icnt2, $result$$Register,
12749                           $vec$$XMMRegister, $tmp$$Register);
12750     } else {
12751       // Small strings are loaded through stack if they cross page boundary.
12752       __ string_indexof($str1$$Register, $str2$$Register,
12753                         $cnt1$$Register, $cnt2$$Register,
12754                         icnt2, $result$$Register,
12755                         $vec$$XMMRegister, $tmp$$Register);
12756     }
12757   %}
12758   ins_pipe( pipe_slow );
12759 %}
12760 
12761 instruct string_indexof(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
12762                         eBXRegI result, regXD vec, eCXRegI tmp, eFlagsReg cr) %{
12763   predicate(UseSSE42Intrinsics);
12764   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12765   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
12766 
12767   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
12768   ins_encode %{
12769     __ string_indexof($str1$$Register, $str2$$Register,
12770                       $cnt1$$Register, $cnt2$$Register,
12771                       (-1), $result$$Register,
12772                       $vec$$XMMRegister, $tmp$$Register);
12773   %}
12774   ins_pipe( pipe_slow );
12775 %}
12776 
12777 // fast array equals
12778 instruct array_equals(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
12779                       regXD tmp1, regXD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
12780 %{
12781   match(Set result (AryEq ary1 ary2));
12782   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12783   //ins_cost(300);
12784 
12785   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12786   ins_encode %{
12787     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
12788                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
12789                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
12790   %}
12791   ins_pipe( pipe_slow );
12792 %}
12793 
12794 //----------Control Flow Instructions------------------------------------------
12795 // Signed compare Instructions
12796 instruct compI_eReg(eFlagsReg cr, eRegI op1, eRegI op2) %{
12797   match(Set cr (CmpI op1 op2));
12798   effect( DEF cr, USE op1, USE op2 );
12799   format %{ "CMP    $op1,$op2" %}
12800   opcode(0x3B);  /* Opcode 3B /r */
12801   ins_encode( OpcP, RegReg( op1, op2) );
12802   ins_pipe( ialu_cr_reg_reg );
12803 %}
12804 
12805 instruct compI_eReg_imm(eFlagsReg cr, eRegI op1, immI op2) %{
12806   match(Set cr (CmpI op1 op2));
12807   effect( DEF cr, USE op1 );
12808   format %{ "CMP    $op1,$op2" %}
12809   opcode(0x81,0x07);  /* Opcode 81 /7 */
12810   // ins_encode( RegImm( op1, op2) );  /* Was CmpImm */
12811   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12812   ins_pipe( ialu_cr_reg_imm );
12813 %}
12814 
12815 // Cisc-spilled version of cmpI_eReg
12816 instruct compI_eReg_mem(eFlagsReg cr, eRegI op1, memory op2) %{
12817   match(Set cr (CmpI op1 (LoadI op2)));
12818 
12819   format %{ "CMP    $op1,$op2" %}
12820   ins_cost(500);
12821   opcode(0x3B);  /* Opcode 3B /r */
12822   ins_encode( OpcP, RegMem( op1, op2) );
12823   ins_pipe( ialu_cr_reg_mem );
12824 %}
12825 
12826 instruct testI_reg( eFlagsReg cr, eRegI src, immI0 zero ) %{
12827   match(Set cr (CmpI src zero));
12828   effect( DEF cr, USE src );
12829 
12830   format %{ "TEST   $src,$src" %}
12831   opcode(0x85);
12832   ins_encode( OpcP, RegReg( src, src ) );
12833   ins_pipe( ialu_cr_reg_imm );
12834 %}
12835 
12836 instruct testI_reg_imm( eFlagsReg cr, eRegI src, immI con, immI0 zero ) %{
12837   match(Set cr (CmpI (AndI src con) zero));
12838 
12839   format %{ "TEST   $src,$con" %}
12840   opcode(0xF7,0x00);
12841   ins_encode( OpcP, RegOpc(src), Con32(con) );
12842   ins_pipe( ialu_cr_reg_imm );
12843 %}
12844 
12845 instruct testI_reg_mem( eFlagsReg cr, eRegI src, memory mem, immI0 zero ) %{
12846   match(Set cr (CmpI (AndI src mem) zero));
12847 
12848   format %{ "TEST   $src,$mem" %}
12849   opcode(0x85);
12850   ins_encode( OpcP, RegMem( src, mem ) );
12851   ins_pipe( ialu_cr_reg_mem );
12852 %}
12853 
12854 // Unsigned compare Instructions; really, same as signed except they
12855 // produce an eFlagsRegU instead of eFlagsReg.
12856 instruct compU_eReg(eFlagsRegU cr, eRegI op1, eRegI op2) %{
12857   match(Set cr (CmpU op1 op2));
12858 
12859   format %{ "CMPu   $op1,$op2" %}
12860   opcode(0x3B);  /* Opcode 3B /r */
12861   ins_encode( OpcP, RegReg( op1, op2) );
12862   ins_pipe( ialu_cr_reg_reg );
12863 %}
12864 
12865 instruct compU_eReg_imm(eFlagsRegU cr, eRegI op1, immI op2) %{
12866   match(Set cr (CmpU op1 op2));
12867 
12868   format %{ "CMPu   $op1,$op2" %}
12869   opcode(0x81,0x07);  /* Opcode 81 /7 */
12870   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12871   ins_pipe( ialu_cr_reg_imm );
12872 %}
12873 
12874 // // Cisc-spilled version of cmpU_eReg
12875 instruct compU_eReg_mem(eFlagsRegU cr, eRegI op1, memory op2) %{
12876   match(Set cr (CmpU op1 (LoadI op2)));
12877 
12878   format %{ "CMPu   $op1,$op2" %}
12879   ins_cost(500);
12880   opcode(0x3B);  /* Opcode 3B /r */
12881   ins_encode( OpcP, RegMem( op1, op2) );
12882   ins_pipe( ialu_cr_reg_mem );
12883 %}
12884 
12885 // // Cisc-spilled version of cmpU_eReg
12886 //instruct compU_mem_eReg(eFlagsRegU cr, memory op1, eRegI op2) %{
12887 //  match(Set cr (CmpU (LoadI op1) op2));
12888 //
12889 //  format %{ "CMPu   $op1,$op2" %}
12890 //  ins_cost(500);
12891 //  opcode(0x39);  /* Opcode 39 /r */
12892 //  ins_encode( OpcP, RegMem( op1, op2) );
12893 //%}
12894 
12895 instruct testU_reg( eFlagsRegU cr, eRegI src, immI0 zero ) %{
12896   match(Set cr (CmpU src zero));
12897 
12898   format %{ "TESTu  $src,$src" %}
12899   opcode(0x85);
12900   ins_encode( OpcP, RegReg( src, src ) );
12901   ins_pipe( ialu_cr_reg_imm );
12902 %}
12903 
12904 // Unsigned pointer compare Instructions
12905 instruct compP_eReg(eFlagsRegU cr, eRegP op1, eRegP op2) %{
12906   match(Set cr (CmpP op1 op2));
12907 
12908   format %{ "CMPu   $op1,$op2" %}
12909   opcode(0x3B);  /* Opcode 3B /r */
12910   ins_encode( OpcP, RegReg( op1, op2) );
12911   ins_pipe( ialu_cr_reg_reg );
12912 %}
12913 
12914 instruct compP_eReg_imm(eFlagsRegU cr, eRegP op1, immP op2) %{
12915   match(Set cr (CmpP op1 op2));
12916 
12917   format %{ "CMPu   $op1,$op2" %}
12918   opcode(0x81,0x07);  /* Opcode 81 /7 */
12919   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12920   ins_pipe( ialu_cr_reg_imm );
12921 %}
12922 
12923 // // Cisc-spilled version of cmpP_eReg
12924 instruct compP_eReg_mem(eFlagsRegU cr, eRegP op1, memory op2) %{
12925   match(Set cr (CmpP op1 (LoadP op2)));
12926 
12927   format %{ "CMPu   $op1,$op2" %}
12928   ins_cost(500);
12929   opcode(0x3B);  /* Opcode 3B /r */
12930   ins_encode( OpcP, RegMem( op1, op2) );
12931   ins_pipe( ialu_cr_reg_mem );
12932 %}
12933 
12934 // // Cisc-spilled version of cmpP_eReg
12935 //instruct compP_mem_eReg(eFlagsRegU cr, memory op1, eRegP op2) %{
12936 //  match(Set cr (CmpP (LoadP op1) op2));
12937 //
12938 //  format %{ "CMPu   $op1,$op2" %}
12939 //  ins_cost(500);
12940 //  opcode(0x39);  /* Opcode 39 /r */
12941 //  ins_encode( OpcP, RegMem( op1, op2) );
12942 //%}
12943 
12944 // Compare raw pointer (used in out-of-heap check).
12945 // Only works because non-oop pointers must be raw pointers
12946 // and raw pointers have no anti-dependencies.
12947 instruct compP_mem_eReg( eFlagsRegU cr, eRegP op1, memory op2 ) %{
12948   predicate( !n->in(2)->in(2)->bottom_type()->isa_oop_ptr() );
12949   match(Set cr (CmpP op1 (LoadP op2)));
12950 
12951   format %{ "CMPu   $op1,$op2" %}
12952   opcode(0x3B);  /* Opcode 3B /r */
12953   ins_encode( OpcP, RegMem( op1, op2) );
12954   ins_pipe( ialu_cr_reg_mem );
12955 %}
12956 
12957 //
12958 // This will generate a signed flags result. This should be ok
12959 // since any compare to a zero should be eq/neq.
12960 instruct testP_reg( eFlagsReg cr, eRegP src, immP0 zero ) %{
12961   match(Set cr (CmpP src zero));
12962 
12963   format %{ "TEST   $src,$src" %}
12964   opcode(0x85);
12965   ins_encode( OpcP, RegReg( src, src ) );
12966   ins_pipe( ialu_cr_reg_imm );
12967 %}
12968 
12969 // Cisc-spilled version of testP_reg
12970 // This will generate a signed flags result. This should be ok
12971 // since any compare to a zero should be eq/neq.
12972 instruct testP_Reg_mem( eFlagsReg cr, memory op, immI0 zero ) %{
12973   match(Set cr (CmpP (LoadP op) zero));
12974 
12975   format %{ "TEST   $op,0xFFFFFFFF" %}
12976   ins_cost(500);
12977   opcode(0xF7);               /* Opcode F7 /0 */
12978   ins_encode( OpcP, RMopc_Mem(0x00,op), Con_d32(0xFFFFFFFF) );
12979   ins_pipe( ialu_cr_reg_imm );
12980 %}
12981 
12982 // Yanked all unsigned pointer compare operations.
12983 // Pointer compares are done with CmpP which is already unsigned.
12984 
12985 //----------Max and Min--------------------------------------------------------
12986 // Min Instructions
12987 ////
12988 //   *** Min and Max using the conditional move are slower than the
12989 //   *** branch version on a Pentium III.
12990 // // Conditional move for min
12991 //instruct cmovI_reg_lt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
12992 //  effect( USE_DEF op2, USE op1, USE cr );
12993 //  format %{ "CMOVlt $op2,$op1\t! min" %}
12994 //  opcode(0x4C,0x0F);
12995 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
12996 //  ins_pipe( pipe_cmov_reg );
12997 //%}
12998 //
12999 //// Min Register with Register (P6 version)
13000 //instruct minI_eReg_p6( eRegI op1, eRegI op2 ) %{
13001 //  predicate(VM_Version::supports_cmov() );
13002 //  match(Set op2 (MinI op1 op2));
13003 //  ins_cost(200);
13004 //  expand %{
13005 //    eFlagsReg cr;
13006 //    compI_eReg(cr,op1,op2);
13007 //    cmovI_reg_lt(op2,op1,cr);
13008 //  %}
13009 //%}
13010 
13011 // Min Register with Register (generic version)
13012 instruct minI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
13013   match(Set dst (MinI dst src));
13014   effect(KILL flags);
13015   ins_cost(300);
13016 
13017   format %{ "MIN    $dst,$src" %}
13018   opcode(0xCC);
13019   ins_encode( min_enc(dst,src) );
13020   ins_pipe( pipe_slow );
13021 %}
13022 
13023 // Max Register with Register
13024 //   *** Min and Max using the conditional move are slower than the
13025 //   *** branch version on a Pentium III.
13026 // // Conditional move for max
13027 //instruct cmovI_reg_gt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
13028 //  effect( USE_DEF op2, USE op1, USE cr );
13029 //  format %{ "CMOVgt $op2,$op1\t! max" %}
13030 //  opcode(0x4F,0x0F);
13031 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
13032 //  ins_pipe( pipe_cmov_reg );
13033 //%}
13034 //
13035 // // Max Register with Register (P6 version)
13036 //instruct maxI_eReg_p6( eRegI op1, eRegI op2 ) %{
13037 //  predicate(VM_Version::supports_cmov() );
13038 //  match(Set op2 (MaxI op1 op2));
13039 //  ins_cost(200);
13040 //  expand %{
13041 //    eFlagsReg cr;
13042 //    compI_eReg(cr,op1,op2);
13043 //    cmovI_reg_gt(op2,op1,cr);
13044 //  %}
13045 //%}
13046 
13047 // Max Register with Register (generic version)
13048 instruct maxI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
13049   match(Set dst (MaxI dst src));
13050   effect(KILL flags);
13051   ins_cost(300);
13052 
13053   format %{ "MAX    $dst,$src" %}
13054   opcode(0xCC);
13055   ins_encode( max_enc(dst,src) );
13056   ins_pipe( pipe_slow );
13057 %}
13058 
13059 // ============================================================================
13060 // Counted Loop limit node which represents exact final iterator value.
13061 // Note: the resulting value should fit into integer range since
13062 // counted loops have limit check on overflow.
13063 instruct loopLimit_eReg(eAXRegI limit, nadxRegI init, immI stride, eDXRegI limit_hi, nadxRegI tmp, eFlagsReg flags) %{
13064   match(Set limit (LoopLimit (Binary init limit) stride));
13065   effect(TEMP limit_hi, TEMP tmp, KILL flags);
13066   ins_cost(300);
13067 
13068   format %{ "loopLimit $init,$limit,$stride  # $limit = $init + $stride *( $limit - $init + $stride -1)/ $stride, kills $limit_hi" %}
13069   ins_encode %{
13070     int strd = (int)$stride$$constant;
13071     assert(strd != 1 && strd != -1, "sanity");
13072     int m1 = (strd > 0) ? 1 : -1;
13073     // Convert limit to long (EAX:EDX)
13074     __ cdql();
13075     // Convert init to long (init:tmp)
13076     __ movl($tmp$$Register, $init$$Register);
13077     __ sarl($tmp$$Register, 31);
13078     // $limit - $init
13079     __ subl($limit$$Register, $init$$Register);
13080     __ sbbl($limit_hi$$Register, $tmp$$Register);
13081     // + ($stride - 1)
13082     if (strd > 0) {
13083       __ addl($limit$$Register, (strd - 1));
13084       __ adcl($limit_hi$$Register, 0);
13085       __ movl($tmp$$Register, strd);
13086     } else {
13087       __ addl($limit$$Register, (strd + 1));
13088       __ adcl($limit_hi$$Register, -1);
13089       __ lneg($limit_hi$$Register, $limit$$Register);
13090       __ movl($tmp$$Register, -strd);
13091     }
13092     // signed devision: (EAX:EDX) / pos_stride
13093     __ idivl($tmp$$Register);
13094     if (strd < 0) {
13095       // restore sign
13096       __ negl($tmp$$Register);
13097     }
13098     // (EAX) * stride
13099     __ mull($tmp$$Register);
13100     // + init (ignore upper bits)
13101     __ addl($limit$$Register, $init$$Register);
13102   %}
13103   ins_pipe( pipe_slow );
13104 %}
13105 
13106 // ============================================================================
13107 // Branch Instructions
13108 // Jump Table
13109 instruct jumpXtnd(eRegI switch_val) %{
13110   match(Jump switch_val);
13111   ins_cost(350);
13112   format %{  "JMP    [$constantaddress](,$switch_val,1)\n\t" %}
13113   ins_encode %{
13114     // Jump to Address(table_base + switch_reg)
13115     Address index(noreg, $switch_val$$Register, Address::times_1);
13116     __ jump(ArrayAddress($constantaddress, index));
13117   %}
13118   ins_pipe(pipe_jmp);
13119 %}
13120 
13121 // Jump Direct - Label defines a relative address from JMP+1
13122 instruct jmpDir(label labl) %{
13123   match(Goto);
13124   effect(USE labl);
13125 
13126   ins_cost(300);
13127   format %{ "JMP    $labl" %}
13128   size(5);
13129   ins_encode %{
13130     Label* L = $labl$$label;
13131     __ jmp(*L, false); // Always long jump
13132   %}
13133   ins_pipe( pipe_jmp );
13134 %}
13135 
13136 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13137 instruct jmpCon(cmpOp cop, eFlagsReg cr, label labl) %{
13138   match(If cop cr);
13139   effect(USE labl);
13140 
13141   ins_cost(300);
13142   format %{ "J$cop    $labl" %}
13143   size(6);
13144   ins_encode %{
13145     Label* L = $labl$$label;
13146     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13147   %}
13148   ins_pipe( pipe_jcc );
13149 %}
13150 
13151 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13152 instruct jmpLoopEnd(cmpOp cop, eFlagsReg cr, label labl) %{
13153   match(CountedLoopEnd cop cr);
13154   effect(USE labl);
13155 
13156   ins_cost(300);
13157   format %{ "J$cop    $labl\t# Loop end" %}
13158   size(6);
13159   ins_encode %{
13160     Label* L = $labl$$label;
13161     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13162   %}
13163   ins_pipe( pipe_jcc );
13164 %}
13165 
13166 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13167 instruct jmpLoopEndU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13168   match(CountedLoopEnd cop cmp);
13169   effect(USE labl);
13170 
13171   ins_cost(300);
13172   format %{ "J$cop,u  $labl\t# Loop end" %}
13173   size(6);
13174   ins_encode %{
13175     Label* L = $labl$$label;
13176     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13177   %}
13178   ins_pipe( pipe_jcc );
13179 %}
13180 
13181 instruct jmpLoopEndUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13182   match(CountedLoopEnd cop cmp);
13183   effect(USE labl);
13184 
13185   ins_cost(200);
13186   format %{ "J$cop,u  $labl\t# Loop end" %}
13187   size(6);
13188   ins_encode %{
13189     Label* L = $labl$$label;
13190     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13191   %}
13192   ins_pipe( pipe_jcc );
13193 %}
13194 
13195 // Jump Direct Conditional - using unsigned comparison
13196 instruct jmpConU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13197   match(If cop cmp);
13198   effect(USE labl);
13199 
13200   ins_cost(300);
13201   format %{ "J$cop,u  $labl" %}
13202   size(6);
13203   ins_encode %{
13204     Label* L = $labl$$label;
13205     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13206   %}
13207   ins_pipe(pipe_jcc);
13208 %}
13209 
13210 instruct jmpConUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13211   match(If cop cmp);
13212   effect(USE labl);
13213 
13214   ins_cost(200);
13215   format %{ "J$cop,u  $labl" %}
13216   size(6);
13217   ins_encode %{
13218     Label* L = $labl$$label;
13219     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
13220   %}
13221   ins_pipe(pipe_jcc);
13222 %}
13223 
13224 instruct jmpConUCF2(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
13225   match(If cop cmp);
13226   effect(USE labl);
13227 
13228   ins_cost(200);
13229   format %{ $$template
13230     if ($cop$$cmpcode == Assembler::notEqual) {
13231       $$emit$$"JP,u   $labl\n\t"
13232       $$emit$$"J$cop,u   $labl"
13233     } else {
13234       $$emit$$"JP,u   done\n\t"
13235       $$emit$$"J$cop,u   $labl\n\t"
13236       $$emit$$"done:"
13237     }
13238   %}
13239   ins_encode %{
13240     Label* l = $labl$$label;
13241     if ($cop$$cmpcode == Assembler::notEqual) {
13242       __ jcc(Assembler::parity, *l, false);
13243       __ jcc(Assembler::notEqual, *l, false);
13244     } else if ($cop$$cmpcode == Assembler::equal) {
13245       Label done;
13246       __ jccb(Assembler::parity, done);
13247       __ jcc(Assembler::equal, *l, false);
13248       __ bind(done);
13249     } else {
13250        ShouldNotReachHere();
13251     }
13252   %}
13253   ins_pipe(pipe_jcc);
13254 %}
13255 
13256 // ============================================================================
13257 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
13258 // array for an instance of the superklass.  Set a hidden internal cache on a
13259 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
13260 // NZ for a miss or zero for a hit.  The encoding ALSO sets flags.
13261 instruct partialSubtypeCheck( eDIRegP result, eSIRegP sub, eAXRegP super, eCXRegI rcx, eFlagsReg cr ) %{
13262   match(Set result (PartialSubtypeCheck sub super));
13263   effect( KILL rcx, KILL cr );
13264 
13265   ins_cost(1100);  // slightly larger than the next version
13266   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
13267             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
13268             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
13269             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
13270             "JNE,s  miss\t\t# Missed: EDI not-zero\n\t"
13271             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache\n\t"
13272             "XOR    $result,$result\t\t Hit: EDI zero\n\t"
13273      "miss:\t" %}
13274 
13275   opcode(0x1); // Force a XOR of EDI
13276   ins_encode( enc_PartialSubtypeCheck() );
13277   ins_pipe( pipe_slow );
13278 %}
13279 
13280 instruct partialSubtypeCheck_vs_Zero( eFlagsReg cr, eSIRegP sub, eAXRegP super, eCXRegI rcx, eDIRegP result, immP0 zero ) %{
13281   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13282   effect( KILL rcx, KILL result );
13283 
13284   ins_cost(1000);
13285   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
13286             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
13287             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
13288             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
13289             "JNE,s  miss\t\t# Missed: flags NZ\n\t"
13290             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache, flags Z\n\t"
13291      "miss:\t" %}
13292 
13293   opcode(0x0);  // No need to XOR EDI
13294   ins_encode( enc_PartialSubtypeCheck() );
13295   ins_pipe( pipe_slow );
13296 %}
13297 
13298 // ============================================================================
13299 // Branch Instructions -- short offset versions
13300 //
13301 // These instructions are used to replace jumps of a long offset (the default
13302 // match) with jumps of a shorter offset.  These instructions are all tagged
13303 // with the ins_short_branch attribute, which causes the ADLC to suppress the
13304 // match rules in general matching.  Instead, the ADLC generates a conversion
13305 // method in the MachNode which can be used to do in-place replacement of the
13306 // long variant with the shorter variant.  The compiler will determine if a
13307 // branch can be taken by the is_short_branch_offset() predicate in the machine
13308 // specific code section of the file.
13309 
13310 // Jump Direct - Label defines a relative address from JMP+1
13311 instruct jmpDir_short(label labl) %{
13312   match(Goto);
13313   effect(USE labl);
13314 
13315   ins_cost(300);
13316   format %{ "JMP,s  $labl" %}
13317   size(2);
13318   ins_encode %{
13319     Label* L = $labl$$label;
13320     __ jmpb(*L);
13321   %}
13322   ins_pipe( pipe_jmp );
13323   ins_short_branch(1);
13324 %}
13325 
13326 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13327 instruct jmpCon_short(cmpOp cop, eFlagsReg cr, label labl) %{
13328   match(If cop cr);
13329   effect(USE labl);
13330 
13331   ins_cost(300);
13332   format %{ "J$cop,s  $labl" %}
13333   size(2);
13334   ins_encode %{
13335     Label* L = $labl$$label;
13336     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13337   %}
13338   ins_pipe( pipe_jcc );
13339   ins_short_branch(1);
13340 %}
13341 
13342 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13343 instruct jmpLoopEnd_short(cmpOp cop, eFlagsReg cr, label labl) %{
13344   match(CountedLoopEnd cop cr);
13345   effect(USE labl);
13346 
13347   ins_cost(300);
13348   format %{ "J$cop,s  $labl\t# Loop end" %}
13349   size(2);
13350   ins_encode %{
13351     Label* L = $labl$$label;
13352     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13353   %}
13354   ins_pipe( pipe_jcc );
13355   ins_short_branch(1);
13356 %}
13357 
13358 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13359 instruct jmpLoopEndU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13360   match(CountedLoopEnd cop cmp);
13361   effect(USE labl);
13362 
13363   ins_cost(300);
13364   format %{ "J$cop,us $labl\t# Loop end" %}
13365   size(2);
13366   ins_encode %{
13367     Label* L = $labl$$label;
13368     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13369   %}
13370   ins_pipe( pipe_jcc );
13371   ins_short_branch(1);
13372 %}
13373 
13374 instruct jmpLoopEndUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13375   match(CountedLoopEnd cop cmp);
13376   effect(USE labl);
13377 
13378   ins_cost(300);
13379   format %{ "J$cop,us $labl\t# Loop end" %}
13380   size(2);
13381   ins_encode %{
13382     Label* L = $labl$$label;
13383     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13384   %}
13385   ins_pipe( pipe_jcc );
13386   ins_short_branch(1);
13387 %}
13388 
13389 // Jump Direct Conditional - using unsigned comparison
13390 instruct jmpConU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13391   match(If cop cmp);
13392   effect(USE labl);
13393 
13394   ins_cost(300);
13395   format %{ "J$cop,us $labl" %}
13396   size(2);
13397   ins_encode %{
13398     Label* L = $labl$$label;
13399     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13400   %}
13401   ins_pipe( pipe_jcc );
13402   ins_short_branch(1);
13403 %}
13404 
13405 instruct jmpConUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13406   match(If cop cmp);
13407   effect(USE labl);
13408 
13409   ins_cost(300);
13410   format %{ "J$cop,us $labl" %}
13411   size(2);
13412   ins_encode %{
13413     Label* L = $labl$$label;
13414     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
13415   %}
13416   ins_pipe( pipe_jcc );
13417   ins_short_branch(1);
13418 %}
13419 
13420 instruct jmpConUCF2_short(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
13421   match(If cop cmp);
13422   effect(USE labl);
13423 
13424   ins_cost(300);
13425   format %{ $$template
13426     if ($cop$$cmpcode == Assembler::notEqual) {
13427       $$emit$$"JP,u,s   $labl\n\t"
13428       $$emit$$"J$cop,u,s   $labl"
13429     } else {
13430       $$emit$$"JP,u,s   done\n\t"
13431       $$emit$$"J$cop,u,s  $labl\n\t"
13432       $$emit$$"done:"
13433     }
13434   %}
13435   size(4);
13436   ins_encode %{
13437     Label* l = $labl$$label;
13438     if ($cop$$cmpcode == Assembler::notEqual) {
13439       __ jccb(Assembler::parity, *l);
13440       __ jccb(Assembler::notEqual, *l);
13441     } else if ($cop$$cmpcode == Assembler::equal) {
13442       Label done;
13443       __ jccb(Assembler::parity, done);
13444       __ jccb(Assembler::equal, *l);
13445       __ bind(done);
13446     } else {
13447        ShouldNotReachHere();
13448     }
13449   %}
13450   ins_pipe(pipe_jcc);
13451   ins_short_branch(1);
13452 %}
13453 
13454 // ============================================================================
13455 // Long Compare
13456 //
13457 // Currently we hold longs in 2 registers.  Comparing such values efficiently
13458 // is tricky.  The flavor of compare used depends on whether we are testing
13459 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
13460 // The GE test is the negated LT test.  The LE test can be had by commuting
13461 // the operands (yielding a GE test) and then negating; negate again for the
13462 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
13463 // NE test is negated from that.
13464 
13465 // Due to a shortcoming in the ADLC, it mixes up expressions like:
13466 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
13467 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
13468 // are collapsed internally in the ADLC's dfa-gen code.  The match for
13469 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
13470 // foo match ends up with the wrong leaf.  One fix is to not match both
13471 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
13472 // both forms beat the trinary form of long-compare and both are very useful
13473 // on Intel which has so few registers.
13474 
13475 // Manifest a CmpL result in an integer register.  Very painful.
13476 // This is the test to avoid.
13477 instruct cmpL3_reg_reg(eSIRegI dst, eRegL src1, eRegL src2, eFlagsReg flags ) %{
13478   match(Set dst (CmpL3 src1 src2));
13479   effect( KILL flags );
13480   ins_cost(1000);
13481   format %{ "XOR    $dst,$dst\n\t"
13482             "CMP    $src1.hi,$src2.hi\n\t"
13483             "JLT,s  m_one\n\t"
13484             "JGT,s  p_one\n\t"
13485             "CMP    $src1.lo,$src2.lo\n\t"
13486             "JB,s   m_one\n\t"
13487             "JEQ,s  done\n"
13488     "p_one:\tINC    $dst\n\t"
13489             "JMP,s  done\n"
13490     "m_one:\tDEC    $dst\n"
13491      "done:" %}
13492   ins_encode %{
13493     Label p_one, m_one, done;
13494     __ xorptr($dst$$Register, $dst$$Register);
13495     __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register));
13496     __ jccb(Assembler::less,    m_one);
13497     __ jccb(Assembler::greater, p_one);
13498     __ cmpl($src1$$Register, $src2$$Register);
13499     __ jccb(Assembler::below,   m_one);
13500     __ jccb(Assembler::equal,   done);
13501     __ bind(p_one);
13502     __ incrementl($dst$$Register);
13503     __ jmpb(done);
13504     __ bind(m_one);
13505     __ decrementl($dst$$Register);
13506     __ bind(done);
13507   %}
13508   ins_pipe( pipe_slow );
13509 %}
13510 
13511 //======
13512 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
13513 // compares.  Can be used for LE or GT compares by reversing arguments.
13514 // NOT GOOD FOR EQ/NE tests.
13515 instruct cmpL_zero_flags_LTGE( flagsReg_long_LTGE flags, eRegL src, immL0 zero ) %{
13516   match( Set flags (CmpL src zero ));
13517   ins_cost(100);
13518   format %{ "TEST   $src.hi,$src.hi" %}
13519   opcode(0x85);
13520   ins_encode( OpcP, RegReg_Hi2( src, src ) );
13521   ins_pipe( ialu_cr_reg_reg );
13522 %}
13523 
13524 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
13525 // compares.  Can be used for LE or GT compares by reversing arguments.
13526 // NOT GOOD FOR EQ/NE tests.
13527 instruct cmpL_reg_flags_LTGE( flagsReg_long_LTGE flags, eRegL src1, eRegL src2, eRegI tmp ) %{
13528   match( Set flags (CmpL src1 src2 ));
13529   effect( TEMP tmp );
13530   ins_cost(300);
13531   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
13532             "MOV    $tmp,$src1.hi\n\t"
13533             "SBB    $tmp,$src2.hi\t! Compute flags for long compare" %}
13534   ins_encode( long_cmp_flags2( src1, src2, tmp ) );
13535   ins_pipe( ialu_cr_reg_reg );
13536 %}
13537 
13538 // Long compares reg < zero/req OR reg >= zero/req.
13539 // Just a wrapper for a normal branch, plus the predicate test.
13540 instruct cmpL_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, label labl) %{
13541   match(If cmp flags);
13542   effect(USE labl);
13543   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13544   expand %{
13545     jmpCon(cmp,flags,labl);    // JLT or JGE...
13546   %}
13547 %}
13548 
13549 // Compare 2 longs and CMOVE longs.
13550 instruct cmovLL_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, eRegL src) %{
13551   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13552   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13553   ins_cost(400);
13554   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13555             "CMOV$cmp $dst.hi,$src.hi" %}
13556   opcode(0x0F,0x40);
13557   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13558   ins_pipe( pipe_cmov_reg_long );
13559 %}
13560 
13561 instruct cmovLL_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, load_long_memory src) %{
13562   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13563   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13564   ins_cost(500);
13565   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13566             "CMOV$cmp $dst.hi,$src.hi" %}
13567   opcode(0x0F,0x40);
13568   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13569   ins_pipe( pipe_cmov_reg_long );
13570 %}
13571 
13572 // Compare 2 longs and CMOVE ints.
13573 instruct cmovII_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, eRegI src) %{
13574   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13575   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13576   ins_cost(200);
13577   format %{ "CMOV$cmp $dst,$src" %}
13578   opcode(0x0F,0x40);
13579   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13580   ins_pipe( pipe_cmov_reg );
13581 %}
13582 
13583 instruct cmovII_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, memory src) %{
13584   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13585   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13586   ins_cost(250);
13587   format %{ "CMOV$cmp $dst,$src" %}
13588   opcode(0x0F,0x40);
13589   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13590   ins_pipe( pipe_cmov_mem );
13591 %}
13592 
13593 // Compare 2 longs and CMOVE ints.
13594 instruct cmovPP_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegP dst, eRegP src) %{
13595   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13596   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13597   ins_cost(200);
13598   format %{ "CMOV$cmp $dst,$src" %}
13599   opcode(0x0F,0x40);
13600   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13601   ins_pipe( pipe_cmov_reg );
13602 %}
13603 
13604 // Compare 2 longs and CMOVE doubles
13605 instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
13606   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13607   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13608   ins_cost(200);
13609   expand %{
13610     fcmovD_regS(cmp,flags,dst,src);
13611   %}
13612 %}
13613 
13614 // Compare 2 longs and CMOVE doubles
13615 instruct cmovXDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regXD dst, regXD src) %{
13616   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13617   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13618   ins_cost(200);
13619   expand %{
13620     fcmovXD_regS(cmp,flags,dst,src);
13621   %}
13622 %}
13623 
13624 instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
13625   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13626   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13627   ins_cost(200);
13628   expand %{
13629     fcmovF_regS(cmp,flags,dst,src);
13630   %}
13631 %}
13632 
13633 instruct cmovXX_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regX dst, regX src) %{
13634   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13635   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13636   ins_cost(200);
13637   expand %{
13638     fcmovX_regS(cmp,flags,dst,src);
13639   %}
13640 %}
13641 
13642 //======
13643 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
13644 instruct cmpL_zero_flags_EQNE( flagsReg_long_EQNE flags, eRegL src, immL0 zero, eRegI tmp ) %{
13645   match( Set flags (CmpL src zero ));
13646   effect(TEMP tmp);
13647   ins_cost(200);
13648   format %{ "MOV    $tmp,$src.lo\n\t"
13649             "OR     $tmp,$src.hi\t! Long is EQ/NE 0?" %}
13650   ins_encode( long_cmp_flags0( src, tmp ) );
13651   ins_pipe( ialu_reg_reg_long );
13652 %}
13653 
13654 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
13655 instruct cmpL_reg_flags_EQNE( flagsReg_long_EQNE flags, eRegL src1, eRegL src2 ) %{
13656   match( Set flags (CmpL src1 src2 ));
13657   ins_cost(200+300);
13658   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
13659             "JNE,s  skip\n\t"
13660             "CMP    $src1.hi,$src2.hi\n\t"
13661      "skip:\t" %}
13662   ins_encode( long_cmp_flags1( src1, src2 ) );
13663   ins_pipe( ialu_cr_reg_reg );
13664 %}
13665 
13666 // Long compare reg == zero/reg OR reg != zero/reg
13667 // Just a wrapper for a normal branch, plus the predicate test.
13668 instruct cmpL_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, label labl) %{
13669   match(If cmp flags);
13670   effect(USE labl);
13671   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13672   expand %{
13673     jmpCon(cmp,flags,labl);    // JEQ or JNE...
13674   %}
13675 %}
13676 
13677 // Compare 2 longs and CMOVE longs.
13678 instruct cmovLL_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, eRegL src) %{
13679   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13680   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13681   ins_cost(400);
13682   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13683             "CMOV$cmp $dst.hi,$src.hi" %}
13684   opcode(0x0F,0x40);
13685   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13686   ins_pipe( pipe_cmov_reg_long );
13687 %}
13688 
13689 instruct cmovLL_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, load_long_memory src) %{
13690   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13691   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13692   ins_cost(500);
13693   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13694             "CMOV$cmp $dst.hi,$src.hi" %}
13695   opcode(0x0F,0x40);
13696   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13697   ins_pipe( pipe_cmov_reg_long );
13698 %}
13699 
13700 // Compare 2 longs and CMOVE ints.
13701 instruct cmovII_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, eRegI src) %{
13702   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13703   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13704   ins_cost(200);
13705   format %{ "CMOV$cmp $dst,$src" %}
13706   opcode(0x0F,0x40);
13707   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13708   ins_pipe( pipe_cmov_reg );
13709 %}
13710 
13711 instruct cmovII_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, memory src) %{
13712   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13713   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13714   ins_cost(250);
13715   format %{ "CMOV$cmp $dst,$src" %}
13716   opcode(0x0F,0x40);
13717   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13718   ins_pipe( pipe_cmov_mem );
13719 %}
13720 
13721 // Compare 2 longs and CMOVE ints.
13722 instruct cmovPP_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegP dst, eRegP src) %{
13723   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13724   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13725   ins_cost(200);
13726   format %{ "CMOV$cmp $dst,$src" %}
13727   opcode(0x0F,0x40);
13728   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13729   ins_pipe( pipe_cmov_reg );
13730 %}
13731 
13732 // Compare 2 longs and CMOVE doubles
13733 instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
13734   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13735   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13736   ins_cost(200);
13737   expand %{
13738     fcmovD_regS(cmp,flags,dst,src);
13739   %}
13740 %}
13741 
13742 // Compare 2 longs and CMOVE doubles
13743 instruct cmovXDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regXD dst, regXD src) %{
13744   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13745   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13746   ins_cost(200);
13747   expand %{
13748     fcmovXD_regS(cmp,flags,dst,src);
13749   %}
13750 %}
13751 
13752 instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
13753   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13754   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13755   ins_cost(200);
13756   expand %{
13757     fcmovF_regS(cmp,flags,dst,src);
13758   %}
13759 %}
13760 
13761 instruct cmovXX_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regX dst, regX src) %{
13762   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13763   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13764   ins_cost(200);
13765   expand %{
13766     fcmovX_regS(cmp,flags,dst,src);
13767   %}
13768 %}
13769 
13770 //======
13771 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
13772 // Same as cmpL_reg_flags_LEGT except must negate src
13773 instruct cmpL_zero_flags_LEGT( flagsReg_long_LEGT flags, eRegL src, immL0 zero, eRegI tmp ) %{
13774   match( Set flags (CmpL src zero ));
13775   effect( TEMP tmp );
13776   ins_cost(300);
13777   format %{ "XOR    $tmp,$tmp\t# Long compare for -$src < 0, use commuted test\n\t"
13778             "CMP    $tmp,$src.lo\n\t"
13779             "SBB    $tmp,$src.hi\n\t" %}
13780   ins_encode( long_cmp_flags3(src, tmp) );
13781   ins_pipe( ialu_reg_reg_long );
13782 %}
13783 
13784 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
13785 // Same as cmpL_reg_flags_LTGE except operands swapped.  Swapping operands
13786 // requires a commuted test to get the same result.
13787 instruct cmpL_reg_flags_LEGT( flagsReg_long_LEGT flags, eRegL src1, eRegL src2, eRegI tmp ) %{
13788   match( Set flags (CmpL src1 src2 ));
13789   effect( TEMP tmp );
13790   ins_cost(300);
13791   format %{ "CMP    $src2.lo,$src1.lo\t! Long compare, swapped operands, use with commuted test\n\t"
13792             "MOV    $tmp,$src2.hi\n\t"
13793             "SBB    $tmp,$src1.hi\t! Compute flags for long compare" %}
13794   ins_encode( long_cmp_flags2( src2, src1, tmp ) );
13795   ins_pipe( ialu_cr_reg_reg );
13796 %}
13797 
13798 // Long compares reg < zero/req OR reg >= zero/req.
13799 // Just a wrapper for a normal branch, plus the predicate test
13800 instruct cmpL_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, label labl) %{
13801   match(If cmp flags);
13802   effect(USE labl);
13803   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
13804   ins_cost(300);
13805   expand %{
13806     jmpCon(cmp,flags,labl);    // JGT or JLE...
13807   %}
13808 %}
13809 
13810 // Compare 2 longs and CMOVE longs.
13811 instruct cmovLL_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, eRegL src) %{
13812   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13813   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13814   ins_cost(400);
13815   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13816             "CMOV$cmp $dst.hi,$src.hi" %}
13817   opcode(0x0F,0x40);
13818   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13819   ins_pipe( pipe_cmov_reg_long );
13820 %}
13821 
13822 instruct cmovLL_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, load_long_memory src) %{
13823   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13824   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13825   ins_cost(500);
13826   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13827             "CMOV$cmp $dst.hi,$src.hi+4" %}
13828   opcode(0x0F,0x40);
13829   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13830   ins_pipe( pipe_cmov_reg_long );
13831 %}
13832 
13833 // Compare 2 longs and CMOVE ints.
13834 instruct cmovII_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, eRegI src) %{
13835   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13836   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13837   ins_cost(200);
13838   format %{ "CMOV$cmp $dst,$src" %}
13839   opcode(0x0F,0x40);
13840   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13841   ins_pipe( pipe_cmov_reg );
13842 %}
13843 
13844 instruct cmovII_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, memory src) %{
13845   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13846   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13847   ins_cost(250);
13848   format %{ "CMOV$cmp $dst,$src" %}
13849   opcode(0x0F,0x40);
13850   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13851   ins_pipe( pipe_cmov_mem );
13852 %}
13853 
13854 // Compare 2 longs and CMOVE ptrs.
13855 instruct cmovPP_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegP dst, eRegP src) %{
13856   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13857   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13858   ins_cost(200);
13859   format %{ "CMOV$cmp $dst,$src" %}
13860   opcode(0x0F,0x40);
13861   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13862   ins_pipe( pipe_cmov_reg );
13863 %}
13864 
13865 // Compare 2 longs and CMOVE doubles
13866 instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
13867   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13868   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13869   ins_cost(200);
13870   expand %{
13871     fcmovD_regS(cmp,flags,dst,src);
13872   %}
13873 %}
13874 
13875 // Compare 2 longs and CMOVE doubles
13876 instruct cmovXDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regXD dst, regXD src) %{
13877   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13878   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13879   ins_cost(200);
13880   expand %{
13881     fcmovXD_regS(cmp,flags,dst,src);
13882   %}
13883 %}
13884 
13885 instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
13886   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13887   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13888   ins_cost(200);
13889   expand %{
13890     fcmovF_regS(cmp,flags,dst,src);
13891   %}
13892 %}
13893 
13894 
13895 instruct cmovXX_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regX dst, regX src) %{
13896   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13897   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13898   ins_cost(200);
13899   expand %{
13900     fcmovX_regS(cmp,flags,dst,src);
13901   %}
13902 %}
13903 
13904 
13905 // ============================================================================
13906 // Procedure Call/Return Instructions
13907 // Call Java Static Instruction
13908 // Note: If this code changes, the corresponding ret_addr_offset() and
13909 //       compute_padding() functions will have to be adjusted.
13910 instruct CallStaticJavaDirect(method meth) %{
13911   match(CallStaticJava);
13912   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
13913   effect(USE meth);
13914 
13915   ins_cost(300);
13916   format %{ "CALL,static " %}
13917   opcode(0xE8); /* E8 cd */
13918   ins_encode( pre_call_FPU,
13919               Java_Static_Call( meth ),
13920               call_epilog,
13921               post_call_FPU );
13922   ins_pipe( pipe_slow );
13923   ins_alignment(4);
13924 %}
13925 
13926 // Call Java Static Instruction (method handle version)
13927 // Note: If this code changes, the corresponding ret_addr_offset() and
13928 //       compute_padding() functions will have to be adjusted.
13929 instruct CallStaticJavaHandle(method meth, eBPRegP ebp_mh_SP_save) %{
13930   match(CallStaticJava);
13931   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
13932   effect(USE meth);
13933   // EBP is saved by all callees (for interpreter stack correction).
13934   // We use it here for a similar purpose, in {preserve,restore}_SP.
13935 
13936   ins_cost(300);
13937   format %{ "CALL,static/MethodHandle " %}
13938   opcode(0xE8); /* E8 cd */
13939   ins_encode( pre_call_FPU,
13940               preserve_SP,
13941               Java_Static_Call( meth ),
13942               restore_SP,
13943               call_epilog,
13944               post_call_FPU );
13945   ins_pipe( pipe_slow );
13946   ins_alignment(4);
13947 %}
13948 
13949 // Call Java Dynamic Instruction
13950 // Note: If this code changes, the corresponding ret_addr_offset() and
13951 //       compute_padding() functions will have to be adjusted.
13952 instruct CallDynamicJavaDirect(method meth) %{
13953   match(CallDynamicJava);
13954   effect(USE meth);
13955 
13956   ins_cost(300);
13957   format %{ "MOV    EAX,(oop)-1\n\t"
13958             "CALL,dynamic" %}
13959   opcode(0xE8); /* E8 cd */
13960   ins_encode( pre_call_FPU,
13961               Java_Dynamic_Call( meth ),
13962               call_epilog,
13963               post_call_FPU );
13964   ins_pipe( pipe_slow );
13965   ins_alignment(4);
13966 %}
13967 
13968 // Call Runtime Instruction
13969 instruct CallRuntimeDirect(method meth) %{
13970   match(CallRuntime );
13971   effect(USE meth);
13972 
13973   ins_cost(300);
13974   format %{ "CALL,runtime " %}
13975   opcode(0xE8); /* E8 cd */
13976   // Use FFREEs to clear entries in float stack
13977   ins_encode( pre_call_FPU,
13978               FFree_Float_Stack_All,
13979               Java_To_Runtime( meth ),
13980               post_call_FPU );
13981   ins_pipe( pipe_slow );
13982 %}
13983 
13984 // Call runtime without safepoint
13985 instruct CallLeafDirect(method meth) %{
13986   match(CallLeaf);
13987   effect(USE meth);
13988 
13989   ins_cost(300);
13990   format %{ "CALL_LEAF,runtime " %}
13991   opcode(0xE8); /* E8 cd */
13992   ins_encode( pre_call_FPU,
13993               FFree_Float_Stack_All,
13994               Java_To_Runtime( meth ),
13995               Verify_FPU_For_Leaf, post_call_FPU );
13996   ins_pipe( pipe_slow );
13997 %}
13998 
13999 instruct CallLeafNoFPDirect(method meth) %{
14000   match(CallLeafNoFP);
14001   effect(USE meth);
14002 
14003   ins_cost(300);
14004   format %{ "CALL_LEAF_NOFP,runtime " %}
14005   opcode(0xE8); /* E8 cd */
14006   ins_encode(Java_To_Runtime(meth));
14007   ins_pipe( pipe_slow );
14008 %}
14009 
14010 
14011 // Return Instruction
14012 // Remove the return address & jump to it.
14013 instruct Ret() %{
14014   match(Return);
14015   format %{ "RET" %}
14016   opcode(0xC3);
14017   ins_encode(OpcP);
14018   ins_pipe( pipe_jmp );
14019 %}
14020 
14021 // Tail Call; Jump from runtime stub to Java code.
14022 // Also known as an 'interprocedural jump'.
14023 // Target of jump will eventually return to caller.
14024 // TailJump below removes the return address.
14025 instruct TailCalljmpInd(eRegP_no_EBP jump_target, eBXRegP method_oop) %{
14026   match(TailCall jump_target method_oop );
14027   ins_cost(300);
14028   format %{ "JMP    $jump_target \t# EBX holds method oop" %}
14029   opcode(0xFF, 0x4);  /* Opcode FF /4 */
14030   ins_encode( OpcP, RegOpc(jump_target) );
14031   ins_pipe( pipe_jmp );
14032 %}
14033 
14034 
14035 // Tail Jump; remove the return address; jump to target.
14036 // TailCall above leaves the return address around.
14037 instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
14038   match( TailJump jump_target ex_oop );
14039   ins_cost(300);
14040   format %{ "POP    EDX\t# pop return address into dummy\n\t"
14041             "JMP    $jump_target " %}
14042   opcode(0xFF, 0x4);  /* Opcode FF /4 */
14043   ins_encode( enc_pop_rdx,
14044               OpcP, RegOpc(jump_target) );
14045   ins_pipe( pipe_jmp );
14046 %}
14047 
14048 // Create exception oop: created by stack-crawling runtime code.
14049 // Created exception is now available to this handler, and is setup
14050 // just prior to jumping to this handler.  No code emitted.
14051 instruct CreateException( eAXRegP ex_oop )
14052 %{
14053   match(Set ex_oop (CreateEx));
14054 
14055   size(0);
14056   // use the following format syntax
14057   format %{ "# exception oop is in EAX; no code emitted" %}
14058   ins_encode();
14059   ins_pipe( empty );
14060 %}
14061 
14062 
14063 // Rethrow exception:
14064 // The exception oop will come in the first argument position.
14065 // Then JUMP (not call) to the rethrow stub code.
14066 instruct RethrowException()
14067 %{
14068   match(Rethrow);
14069 
14070   // use the following format syntax
14071   format %{ "JMP    rethrow_stub" %}
14072   ins_encode(enc_rethrow);
14073   ins_pipe( pipe_jmp );
14074 %}
14075 
14076 // inlined locking and unlocking
14077 
14078 
14079 instruct cmpFastLock( eFlagsReg cr, eRegP object, eRegP box, eAXRegI tmp, eRegP scr) %{
14080   match( Set cr (FastLock object box) );
14081   effect( TEMP tmp, TEMP scr );
14082   ins_cost(300);
14083   format %{ "FASTLOCK $object, $box KILLS $tmp,$scr" %}
14084   ins_encode( Fast_Lock(object,box,tmp,scr) );
14085   ins_pipe( pipe_slow );
14086 %}
14087 
14088 instruct cmpFastUnlock( eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
14089   match( Set cr (FastUnlock object box) );
14090   effect( TEMP tmp );
14091   ins_cost(300);
14092   format %{ "FASTUNLOCK $object, $box, $tmp" %}
14093   ins_encode( Fast_Unlock(object,box,tmp) );
14094   ins_pipe( pipe_slow );
14095 %}
14096 
14097 
14098 
14099 // ============================================================================
14100 // Safepoint Instruction
14101 instruct safePoint_poll(eFlagsReg cr) %{
14102   match(SafePoint);
14103   effect(KILL cr);
14104 
14105   // TODO-FIXME: we currently poll at offset 0 of the safepoint polling page.
14106   // On SPARC that might be acceptable as we can generate the address with
14107   // just a sethi, saving an or.  By polling at offset 0 we can end up
14108   // putting additional pressure on the index-0 in the D$.  Because of
14109   // alignment (just like the situation at hand) the lower indices tend
14110   // to see more traffic.  It'd be better to change the polling address
14111   // to offset 0 of the last $line in the polling page.
14112 
14113   format %{ "TSTL   #polladdr,EAX\t! Safepoint: poll for GC" %}
14114   ins_cost(125);
14115   size(6) ;
14116   ins_encode( Safepoint_Poll() );
14117   ins_pipe( ialu_reg_mem );
14118 %}
14119 
14120 //----------PEEPHOLE RULES-----------------------------------------------------
14121 // These must follow all instruction definitions as they use the names
14122 // defined in the instructions definitions.
14123 //
14124 // peepmatch ( root_instr_name [preceding_instruction]* );
14125 //
14126 // peepconstraint %{
14127 // (instruction_number.operand_name relational_op instruction_number.operand_name
14128 //  [, ...] );
14129 // // instruction numbers are zero-based using left to right order in peepmatch
14130 //
14131 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
14132 // // provide an instruction_number.operand_name for each operand that appears
14133 // // in the replacement instruction's match rule
14134 //
14135 // ---------VM FLAGS---------------------------------------------------------
14136 //
14137 // All peephole optimizations can be turned off using -XX:-OptoPeephole
14138 //
14139 // Each peephole rule is given an identifying number starting with zero and
14140 // increasing by one in the order seen by the parser.  An individual peephole
14141 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
14142 // on the command-line.
14143 //
14144 // ---------CURRENT LIMITATIONS----------------------------------------------
14145 //
14146 // Only match adjacent instructions in same basic block
14147 // Only equality constraints
14148 // Only constraints between operands, not (0.dest_reg == EAX_enc)
14149 // Only one replacement instruction
14150 //
14151 // ---------EXAMPLE----------------------------------------------------------
14152 //
14153 // // pertinent parts of existing instructions in architecture description
14154 // instruct movI(eRegI dst, eRegI src) %{
14155 //   match(Set dst (CopyI src));
14156 // %}
14157 //
14158 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
14159 //   match(Set dst (AddI dst src));
14160 //   effect(KILL cr);
14161 // %}
14162 //
14163 // // Change (inc mov) to lea
14164 // peephole %{
14165 //   // increment preceeded by register-register move
14166 //   peepmatch ( incI_eReg movI );
14167 //   // require that the destination register of the increment
14168 //   // match the destination register of the move
14169 //   peepconstraint ( 0.dst == 1.dst );
14170 //   // construct a replacement instruction that sets
14171 //   // the destination to ( move's source register + one )
14172 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
14173 // %}
14174 //
14175 // Implementation no longer uses movX instructions since
14176 // machine-independent system no longer uses CopyX nodes.
14177 //
14178 // peephole %{
14179 //   peepmatch ( incI_eReg movI );
14180 //   peepconstraint ( 0.dst == 1.dst );
14181 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
14182 // %}
14183 //
14184 // peephole %{
14185 //   peepmatch ( decI_eReg movI );
14186 //   peepconstraint ( 0.dst == 1.dst );
14187 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
14188 // %}
14189 //
14190 // peephole %{
14191 //   peepmatch ( addI_eReg_imm movI );
14192 //   peepconstraint ( 0.dst == 1.dst );
14193 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
14194 // %}
14195 //
14196 // peephole %{
14197 //   peepmatch ( addP_eReg_imm movP );
14198 //   peepconstraint ( 0.dst == 1.dst );
14199 //   peepreplace ( leaP_eReg_immI( 0.dst 1.src 0.src ) );
14200 // %}
14201 
14202 // // Change load of spilled value to only a spill
14203 // instruct storeI(memory mem, eRegI src) %{
14204 //   match(Set mem (StoreI mem src));
14205 // %}
14206 //
14207 // instruct loadI(eRegI dst, memory mem) %{
14208 //   match(Set dst (LoadI mem));
14209 // %}
14210 //
14211 peephole %{
14212   peepmatch ( loadI storeI );
14213   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
14214   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
14215 %}
14216 
14217 //----------SMARTSPILL RULES---------------------------------------------------
14218 // These must follow all instruction definitions as they use the names
14219 // defined in the instructions definitions.