1 //
   2 // Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20 // CA 95054 USA or visit www.sun.com if you need additional information or
  21 // have any questions.
  22 //
  23 //
  24 
  25 // X86 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // Previously set EBX, ESI, and EDI as save-on-entry for java code
  64 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  65 // Now that allocator is better, turn on ESI and EDI as SOE registers.
  66 
  67 reg_def EBX(SOC, SOE, Op_RegI, 3, rbx->as_VMReg());
  68 reg_def ECX(SOC, SOC, Op_RegI, 1, rcx->as_VMReg());
  69 reg_def ESI(SOC, SOE, Op_RegI, 6, rsi->as_VMReg());
  70 reg_def EDI(SOC, SOE, Op_RegI, 7, rdi->as_VMReg());
  71 // now that adapter frames are gone EBP is always saved and restored by the prolog/epilog code
  72 reg_def EBP(NS, SOE, Op_RegI, 5, rbp->as_VMReg());
  73 reg_def EDX(SOC, SOC, Op_RegI, 2, rdx->as_VMReg());
  74 reg_def EAX(SOC, SOC, Op_RegI, 0, rax->as_VMReg());
  75 reg_def ESP( NS,  NS, Op_RegI, 4, rsp->as_VMReg());
  76 
  77 // Special Registers
  78 reg_def EFLAGS(SOC, SOC, 0, 8, VMRegImpl::Bad());
  79 
  80 // Float registers.  We treat TOS/FPR0 special.  It is invisible to the
  81 // allocator, and only shows up in the encodings.
  82 reg_def FPR0L( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  83 reg_def FPR0H( SOC, SOC, Op_RegF, 0, VMRegImpl::Bad());
  84 // Ok so here's the trick FPR1 is really st(0) except in the midst
  85 // of emission of assembly for a machnode. During the emission the fpu stack
  86 // is pushed making FPR1 == st(1) temporarily. However at any safepoint
  87 // the stack will not have this element so FPR1 == st(0) from the
  88 // oopMap viewpoint. This same weirdness with numbering causes
  89 // instruction encoding to have to play games with the register
  90 // encode to correct for this 0/1 issue. See MachSpillCopyNode::implementation
  91 // where it does flt->flt moves to see an example
  92 //
  93 reg_def FPR1L( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg());
  94 reg_def FPR1H( SOC, SOC, Op_RegF, 1, as_FloatRegister(0)->as_VMReg()->next());
  95 reg_def FPR2L( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg());
  96 reg_def FPR2H( SOC, SOC, Op_RegF, 2, as_FloatRegister(1)->as_VMReg()->next());
  97 reg_def FPR3L( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg());
  98 reg_def FPR3H( SOC, SOC, Op_RegF, 3, as_FloatRegister(2)->as_VMReg()->next());
  99 reg_def FPR4L( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg());
 100 reg_def FPR4H( SOC, SOC, Op_RegF, 4, as_FloatRegister(3)->as_VMReg()->next());
 101 reg_def FPR5L( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg());
 102 reg_def FPR5H( SOC, SOC, Op_RegF, 5, as_FloatRegister(4)->as_VMReg()->next());
 103 reg_def FPR6L( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg());
 104 reg_def FPR6H( SOC, SOC, Op_RegF, 6, as_FloatRegister(5)->as_VMReg()->next());
 105 reg_def FPR7L( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg());
 106 reg_def FPR7H( SOC, SOC, Op_RegF, 7, as_FloatRegister(6)->as_VMReg()->next());
 107 
 108 // XMM registers.  128-bit registers or 4 words each, labeled a-d.
 109 // Word a in each register holds a Float, words ab hold a Double.
 110 // We currently do not use the SIMD capabilities, so registers cd
 111 // are unused at the moment.
 112 reg_def XMM0a( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg());
 113 reg_def XMM0b( SOC, SOC, Op_RegF, 0, xmm0->as_VMReg()->next());
 114 reg_def XMM1a( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg());
 115 reg_def XMM1b( SOC, SOC, Op_RegF, 1, xmm1->as_VMReg()->next());
 116 reg_def XMM2a( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg());
 117 reg_def XMM2b( SOC, SOC, Op_RegF, 2, xmm2->as_VMReg()->next());
 118 reg_def XMM3a( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg());
 119 reg_def XMM3b( SOC, SOC, Op_RegF, 3, xmm3->as_VMReg()->next());
 120 reg_def XMM4a( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg());
 121 reg_def XMM4b( SOC, SOC, Op_RegF, 4, xmm4->as_VMReg()->next());
 122 reg_def XMM5a( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg());
 123 reg_def XMM5b( SOC, SOC, Op_RegF, 5, xmm5->as_VMReg()->next());
 124 reg_def XMM6a( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg());
 125 reg_def XMM6b( SOC, SOC, Op_RegF, 6, xmm6->as_VMReg()->next());
 126 reg_def XMM7a( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg());
 127 reg_def XMM7b( SOC, SOC, Op_RegF, 7, xmm7->as_VMReg()->next());
 128 
 129 // Specify priority of register selection within phases of register
 130 // allocation.  Highest priority is first.  A useful heuristic is to
 131 // give registers a low priority when they are required by machine
 132 // instructions, like EAX and EDX.  Registers which are used as
 133 // pairs must fall on an even boundary (witness the FPR#L's in this list).
 134 // For the Intel integer registers, the equivalent Long pairs are
 135 // EDX:EAX, EBX:ECX, and EDI:EBP.
 136 alloc_class chunk0( ECX,   EBX,   EBP,   EDI,   EAX,   EDX,   ESI, ESP,
 137                     FPR0L, FPR0H, FPR1L, FPR1H, FPR2L, FPR2H,
 138                     FPR3L, FPR3H, FPR4L, FPR4H, FPR5L, FPR5H,
 139                     FPR6L, FPR6H, FPR7L, FPR7H );
 140 
 141 alloc_class chunk1( XMM0a, XMM0b,
 142                     XMM1a, XMM1b,
 143                     XMM2a, XMM2b,
 144                     XMM3a, XMM3b,
 145                     XMM4a, XMM4b,
 146                     XMM5a, XMM5b,
 147                     XMM6a, XMM6b,
 148                     XMM7a, XMM7b, EFLAGS);
 149 
 150 
 151 //----------Architecture Description Register Classes--------------------------
 152 // Several register classes are automatically defined based upon information in
 153 // this architecture description.
 154 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 155 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 156 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 157 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 158 //
 159 // Class for all registers
 160 reg_class any_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX, ESP);
 161 // Class for general registers
 162 reg_class e_reg(EAX, EDX, EBP, EDI, ESI, ECX, EBX);
 163 // Class for general registers which may be used for implicit null checks on win95
 164 // Also safe for use by tailjump. We don't want to allocate in rbp,
 165 reg_class e_reg_no_rbp(EAX, EDX, EDI, ESI, ECX, EBX);
 166 // Class of "X" registers
 167 reg_class x_reg(EBX, ECX, EDX, EAX);
 168 // Class of registers that can appear in an address with no offset.
 169 // EBP and ESP require an extra instruction byte for zero offset.
 170 // Used in fast-unlock
 171 reg_class p_reg(EDX, EDI, ESI, EBX);
 172 // Class for general registers not including ECX
 173 reg_class ncx_reg(EAX, EDX, EBP, EDI, ESI, EBX);
 174 // Class for general registers not including EAX
 175 reg_class nax_reg(EDX, EDI, ESI, ECX, EBX);
 176 // Class for general registers not including EAX or EBX.
 177 reg_class nabx_reg(EDX, EDI, ESI, ECX, EBP);
 178 // Class of EAX (for multiply and divide operations)
 179 reg_class eax_reg(EAX);
 180 // Class of EBX (for atomic add)
 181 reg_class ebx_reg(EBX);
 182 // Class of ECX (for shift and JCXZ operations and cmpLTMask)
 183 reg_class ecx_reg(ECX);
 184 // Class of EDX (for multiply and divide operations)
 185 reg_class edx_reg(EDX);
 186 // Class of EDI (for synchronization)
 187 reg_class edi_reg(EDI);
 188 // Class of ESI (for synchronization)
 189 reg_class esi_reg(ESI);
 190 // Singleton class for interpreter's stack pointer
 191 reg_class ebp_reg(EBP);
 192 // Singleton class for stack pointer
 193 reg_class sp_reg(ESP);
 194 // Singleton class for instruction pointer
 195 // reg_class ip_reg(EIP);
 196 // Singleton class for condition codes
 197 reg_class int_flags(EFLAGS);
 198 // Class of integer register pairs
 199 reg_class long_reg( EAX,EDX, ECX,EBX, EBP,EDI );
 200 // Class of integer register pairs that aligns with calling convention
 201 reg_class eadx_reg( EAX,EDX );
 202 reg_class ebcx_reg( ECX,EBX );
 203 // Not AX or DX, used in divides
 204 reg_class nadx_reg( EBX,ECX,ESI,EDI,EBP );
 205 
 206 // Floating point registers.  Notice FPR0 is not a choice.
 207 // FPR0 is not ever allocated; we use clever encodings to fake
 208 // a 2-address instructions out of Intels FP stack.
 209 reg_class flt_reg( FPR1L,FPR2L,FPR3L,FPR4L,FPR5L,FPR6L,FPR7L );
 210 
 211 // make a register class for SSE registers
 212 reg_class xmm_reg(XMM0a, XMM1a, XMM2a, XMM3a, XMM4a, XMM5a, XMM6a, XMM7a);
 213 
 214 // make a double register class for SSE2 registers
 215 reg_class xdb_reg(XMM0a,XMM0b, XMM1a,XMM1b, XMM2a,XMM2b, XMM3a,XMM3b,
 216                   XMM4a,XMM4b, XMM5a,XMM5b, XMM6a,XMM6b, XMM7a,XMM7b );
 217 
 218 reg_class dbl_reg( FPR1L,FPR1H, FPR2L,FPR2H, FPR3L,FPR3H,
 219                    FPR4L,FPR4H, FPR5L,FPR5H, FPR6L,FPR6H,
 220                    FPR7L,FPR7H );
 221 
 222 reg_class flt_reg0( FPR1L );
 223 reg_class dbl_reg0( FPR1L,FPR1H );
 224 reg_class dbl_reg1( FPR2L,FPR2H );
 225 reg_class dbl_notreg0( FPR2L,FPR2H, FPR3L,FPR3H, FPR4L,FPR4H,
 226                        FPR5L,FPR5H, FPR6L,FPR6H, FPR7L,FPR7H );
 227 
 228 // XMM6 and XMM7 could be used as temporary registers for long, float and
 229 // double values for SSE2.
 230 reg_class xdb_reg6( XMM6a,XMM6b );
 231 reg_class xdb_reg7( XMM7a,XMM7b );
 232 %}
 233 
 234 
 235 //----------SOURCE BLOCK-------------------------------------------------------
 236 // This is a block of C++ code which provides values, functions, and
 237 // definitions necessary in the rest of the architecture description
 238 source_hpp %{
 239 // Must be visible to the DFA in dfa_x86_32.cpp
 240 extern bool is_operand_hi32_zero(Node* n);
 241 %}
 242 
 243 source %{
 244 #define   RELOC_IMM32    Assembler::imm_operand
 245 #define   RELOC_DISP32   Assembler::disp32_operand
 246 
 247 #define __ _masm.
 248 
 249 // How to find the high register of a Long pair, given the low register
 250 #define   HIGH_FROM_LOW(x) ((x)+2)
 251 
 252 // These masks are used to provide 128-bit aligned bitmasks to the XMM
 253 // instructions, to allow sign-masking or sign-bit flipping.  They allow
 254 // fast versions of NegF/NegD and AbsF/AbsD.
 255 
 256 // Note: 'double' and 'long long' have 32-bits alignment on x86.
 257 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
 258   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
 259   // of 128-bits operands for SSE instructions.
 260   jlong *operand = (jlong*)(((uintptr_t)adr)&((uintptr_t)(~0xF)));
 261   // Store the value to a 128-bits operand.
 262   operand[0] = lo;
 263   operand[1] = hi;
 264   return operand;
 265 }
 266 
 267 // Buffer for 128-bits masks used by SSE instructions.
 268 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
 269 
 270 // Static initialization during VM startup.
 271 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2], CONST64(0x7FFFFFFF7FFFFFFF), CONST64(0x7FFFFFFF7FFFFFFF));
 272 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2], CONST64(0x7FFFFFFFFFFFFFFF), CONST64(0x7FFFFFFFFFFFFFFF));
 273 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
 274 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
 275 
 276 // Offset hacking within calls.
 277 static int pre_call_FPU_size() {
 278   if (Compile::current()->in_24_bit_fp_mode())
 279     return 6; // fldcw
 280   return 0;
 281 }
 282 
 283 static int preserve_SP_size() {
 284   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
 285 }
 286 
 287 // !!!!! Special hack to get all type of calls to specify the byte offset
 288 //       from the start of the call to the point where the return address
 289 //       will point.
 290 int MachCallStaticJavaNode::ret_addr_offset() {
 291   int offset = 5 + pre_call_FPU_size();  // 5 bytes from start of call to where return address points
 292   if (_method_handle_invoke)
 293     offset += preserve_SP_size();
 294   return offset;
 295 }
 296 
 297 int MachCallDynamicJavaNode::ret_addr_offset() {
 298   return 10 + pre_call_FPU_size();  // 10 bytes from start of call to where return address points
 299 }
 300 
 301 static int sizeof_FFree_Float_Stack_All = -1;
 302 
 303 int MachCallRuntimeNode::ret_addr_offset() {
 304   assert(sizeof_FFree_Float_Stack_All != -1, "must have been emitted already");
 305   return sizeof_FFree_Float_Stack_All + 5 + pre_call_FPU_size();
 306 }
 307 
 308 // Indicate if the safepoint node needs the polling page as an input.
 309 // Since x86 does have absolute addressing, it doesn't.
 310 bool SafePointNode::needs_polling_address_input() {
 311   return false;
 312 }
 313 
 314 //
 315 // Compute padding required for nodes which need alignment
 316 //
 317 
 318 // The address of the call instruction needs to be 4-byte aligned to
 319 // ensure that it does not span a cache line so that it can be patched.
 320 int CallStaticJavaDirectNode::compute_padding(int current_offset) const {
 321   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 322   current_offset += 1;      // skip call opcode byte
 323   return round_to(current_offset, alignment_required()) - current_offset;
 324 }
 325 
 326 // The address of the call instruction needs to be 4-byte aligned to
 327 // ensure that it does not span a cache line so that it can be patched.
 328 int CallStaticJavaHandleNode::compute_padding(int current_offset) const {
 329   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 330   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 331   current_offset += 1;      // skip call opcode byte
 332   return round_to(current_offset, alignment_required()) - current_offset;
 333 }
 334 
 335 // The address of the call instruction needs to be 4-byte aligned to
 336 // ensure that it does not span a cache line so that it can be patched.
 337 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const {
 338   current_offset += pre_call_FPU_size();  // skip fldcw, if any
 339   current_offset += 5;      // skip MOV instruction
 340   current_offset += 1;      // skip call opcode byte
 341   return round_to(current_offset, alignment_required()) - current_offset;
 342 }
 343 
 344 #ifndef PRODUCT
 345 void MachBreakpointNode::format( PhaseRegAlloc *, outputStream* st ) const {
 346   st->print("INT3");
 347 }
 348 #endif
 349 
 350 // EMIT_RM()
 351 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 352   unsigned char c = (unsigned char)((f1 << 6) | (f2 << 3) | f3);
 353   *(cbuf.code_end()) = c;
 354   cbuf.set_code_end(cbuf.code_end() + 1);
 355 }
 356 
 357 // EMIT_CC()
 358 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 359   unsigned char c = (unsigned char)( f1 | f2 );
 360   *(cbuf.code_end()) = c;
 361   cbuf.set_code_end(cbuf.code_end() + 1);
 362 }
 363 
 364 // EMIT_OPCODE()
 365 void emit_opcode(CodeBuffer &cbuf, int code) {
 366   *(cbuf.code_end()) = (unsigned char)code;
 367   cbuf.set_code_end(cbuf.code_end() + 1);
 368 }
 369 
 370 // EMIT_OPCODE() w/ relocation information
 371 void emit_opcode(CodeBuffer &cbuf, int code, relocInfo::relocType reloc, int offset = 0) {
 372   cbuf.relocate(cbuf.inst_mark() + offset, reloc);
 373   emit_opcode(cbuf, code);
 374 }
 375 
 376 // EMIT_D8()
 377 void emit_d8(CodeBuffer &cbuf, int d8) {
 378   *(cbuf.code_end()) = (unsigned char)d8;
 379   cbuf.set_code_end(cbuf.code_end() + 1);
 380 }
 381 
 382 // EMIT_D16()
 383 void emit_d16(CodeBuffer &cbuf, int d16) {
 384   *((short *)(cbuf.code_end())) = d16;
 385   cbuf.set_code_end(cbuf.code_end() + 2);
 386 }
 387 
 388 // EMIT_D32()
 389 void emit_d32(CodeBuffer &cbuf, int d32) {
 390   *((int *)(cbuf.code_end())) = d32;
 391   cbuf.set_code_end(cbuf.code_end() + 4);
 392 }
 393 
 394 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 395 void emit_d32_reloc(CodeBuffer &cbuf, int d32, relocInfo::relocType reloc,
 396         int format) {
 397   cbuf.relocate(cbuf.inst_mark(), reloc, format);
 398 
 399   *((int *)(cbuf.code_end())) = d32;
 400   cbuf.set_code_end(cbuf.code_end() + 4);
 401 }
 402 
 403 // emit 32 bit value and construct relocation entry from RelocationHolder
 404 void emit_d32_reloc(CodeBuffer &cbuf, int d32, RelocationHolder const& rspec,
 405         int format) {
 406 #ifdef ASSERT
 407   if (rspec.reloc()->type() == relocInfo::oop_type && d32 != 0 && d32 != (int)Universe::non_oop_word()) {
 408     assert(oop(d32)->is_oop() && (ScavengeRootsInCode || !oop(d32)->is_scavengable()), "cannot embed scavengable oops in code");
 409   }
 410 #endif
 411   cbuf.relocate(cbuf.inst_mark(), rspec, format);
 412 
 413   *((int *)(cbuf.code_end())) = d32;
 414   cbuf.set_code_end(cbuf.code_end() + 4);
 415 }
 416 
 417 // Access stack slot for load or store
 418 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp) {
 419   emit_opcode( cbuf, opcode );               // (e.g., FILD   [ESP+src])
 420   if( -128 <= disp && disp <= 127 ) {
 421     emit_rm( cbuf, 0x01, rm_field, ESP_enc );  // R/M byte
 422     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 423     emit_d8 (cbuf, disp);     // Displacement  // R/M byte
 424   } else {
 425     emit_rm( cbuf, 0x02, rm_field, ESP_enc );  // R/M byte
 426     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);    // SIB byte
 427     emit_d32(cbuf, disp);     // Displacement  // R/M byte
 428   }
 429 }
 430 
 431    // eRegI ereg, memory mem) %{    // emit_reg_mem
 432 void encode_RegMem( CodeBuffer &cbuf, int reg_encoding, int base, int index, int scale, int displace, bool displace_is_oop ) {
 433   // There is no index & no scale, use form without SIB byte
 434   if ((index == 0x4) &&
 435       (scale == 0) && (base != ESP_enc)) {
 436     // If no displacement, mode is 0x0; unless base is [EBP]
 437     if ( (displace == 0) && (base != EBP_enc) ) {
 438       emit_rm(cbuf, 0x0, reg_encoding, base);
 439     }
 440     else {                    // If 8-bit displacement, mode 0x1
 441       if ((displace >= -128) && (displace <= 127)
 442           && !(displace_is_oop) ) {
 443         emit_rm(cbuf, 0x1, reg_encoding, base);
 444         emit_d8(cbuf, displace);
 445       }
 446       else {                  // If 32-bit displacement
 447         if (base == -1) { // Special flag for absolute address
 448           emit_rm(cbuf, 0x0, reg_encoding, 0x5);
 449           // (manual lies; no SIB needed here)
 450           if ( displace_is_oop ) {
 451             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 452           } else {
 453             emit_d32      (cbuf, displace);
 454           }
 455         }
 456         else {                // Normal base + offset
 457           emit_rm(cbuf, 0x2, reg_encoding, base);
 458           if ( displace_is_oop ) {
 459             emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 460           } else {
 461             emit_d32      (cbuf, displace);
 462           }
 463         }
 464       }
 465     }
 466   }
 467   else {                      // Else, encode with the SIB byte
 468     // If no displacement, mode is 0x0; unless base is [EBP]
 469     if (displace == 0 && (base != EBP_enc)) {  // If no displacement
 470       emit_rm(cbuf, 0x0, reg_encoding, 0x4);
 471       emit_rm(cbuf, scale, index, base);
 472     }
 473     else {                    // If 8-bit displacement, mode 0x1
 474       if ((displace >= -128) && (displace <= 127)
 475           && !(displace_is_oop) ) {
 476         emit_rm(cbuf, 0x1, reg_encoding, 0x4);
 477         emit_rm(cbuf, scale, index, base);
 478         emit_d8(cbuf, displace);
 479       }
 480       else {                  // If 32-bit displacement
 481         if (base == 0x04 ) {
 482           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 483           emit_rm(cbuf, scale, index, 0x04);
 484         } else {
 485           emit_rm(cbuf, 0x2, reg_encoding, 0x4);
 486           emit_rm(cbuf, scale, index, base);
 487         }
 488         if ( displace_is_oop ) {
 489           emit_d32_reloc(cbuf, displace, relocInfo::oop_type, 1);
 490         } else {
 491           emit_d32      (cbuf, displace);
 492         }
 493       }
 494     }
 495   }
 496 }
 497 
 498 
 499 void encode_Copy( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 500   if( dst_encoding == src_encoding ) {
 501     // reg-reg copy, use an empty encoding
 502   } else {
 503     emit_opcode( cbuf, 0x8B );
 504     emit_rm(cbuf, 0x3, dst_encoding, src_encoding );
 505   }
 506 }
 507 
 508 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 509   if( dst_encoding == src_encoding ) {
 510     // reg-reg copy, use an empty encoding
 511   } else {
 512     MacroAssembler _masm(&cbuf);
 513 
 514     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
 515   }
 516 }
 517 
 518 
 519 //=============================================================================
 520 #ifndef PRODUCT
 521 void MachPrologNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 522   Compile* C = ra_->C;
 523   if( C->in_24_bit_fp_mode() ) {
 524     st->print("FLDCW  24 bit fpu control word");
 525     st->print_cr(""); st->print("\t");
 526   }
 527 
 528   int framesize = C->frame_slots() << LogBytesPerInt;
 529   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 530   // Remove two words for return addr and rbp,
 531   framesize -= 2*wordSize;
 532 
 533   // Calls to C2R adapters often do not accept exceptional returns.
 534   // We require that their callers must bang for them.  But be careful, because
 535   // some VM calls (such as call site linkage) can use several kilobytes of
 536   // stack.  But the stack safety zone should account for that.
 537   // See bugs 4446381, 4468289, 4497237.
 538   if (C->need_stack_bang(framesize)) {
 539     st->print_cr("# stack bang"); st->print("\t");
 540   }
 541   st->print_cr("PUSHL  EBP"); st->print("\t");
 542 
 543   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
 544     st->print("PUSH   0xBADB100D\t# Majik cookie for stack depth check");
 545     st->print_cr(""); st->print("\t");
 546     framesize -= wordSize;
 547   }
 548 
 549   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
 550     if (framesize) {
 551       st->print("SUB    ESP,%d\t# Create frame",framesize);
 552     }
 553   } else {
 554     st->print("SUB    ESP,%d\t# Create frame",framesize);
 555   }
 556 }
 557 #endif
 558 
 559 
 560 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 561   Compile* C = ra_->C;
 562 
 563   if (UseSSE >= 2 && VerifyFPU) {
 564     MacroAssembler masm(&cbuf);
 565     masm.verify_FPU(0, "FPU stack must be clean on entry");
 566   }
 567 
 568   // WARNING: Initial instruction MUST be 5 bytes or longer so that
 569   // NativeJump::patch_verified_entry will be able to patch out the entry
 570   // code safely. The fldcw is ok at 6 bytes, the push to verify stack
 571   // depth is ok at 5 bytes, the frame allocation can be either 3 or
 572   // 6 bytes. So if we don't do the fldcw or the push then we must
 573   // use the 6 byte frame allocation even if we have no frame. :-(
 574   // If method sets FPU control word do it now
 575   if( C->in_24_bit_fp_mode() ) {
 576     MacroAssembler masm(&cbuf);
 577     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 578   }
 579 
 580   int framesize = C->frame_slots() << LogBytesPerInt;
 581   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 582   // Remove two words for return addr and rbp,
 583   framesize -= 2*wordSize;
 584 
 585   // Calls to C2R adapters often do not accept exceptional returns.
 586   // We require that their callers must bang for them.  But be careful, because
 587   // some VM calls (such as call site linkage) can use several kilobytes of
 588   // stack.  But the stack safety zone should account for that.
 589   // See bugs 4446381, 4468289, 4497237.
 590   if (C->need_stack_bang(framesize)) {
 591     MacroAssembler masm(&cbuf);
 592     masm.generate_stack_overflow_check(framesize);
 593   }
 594 
 595   // We always push rbp, so that on return to interpreter rbp, will be
 596   // restored correctly and we can correct the stack.
 597   emit_opcode(cbuf, 0x50 | EBP_enc);
 598 
 599   if( VerifyStackAtCalls ) { // Majik cookie to verify stack depth
 600     emit_opcode(cbuf, 0x68); // push 0xbadb100d
 601     emit_d32(cbuf, 0xbadb100d);
 602     framesize -= wordSize;
 603   }
 604 
 605   if ((C->in_24_bit_fp_mode() || VerifyStackAtCalls ) && framesize < 128 ) {
 606     if (framesize) {
 607       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
 608       emit_rm(cbuf, 0x3, 0x05, ESP_enc);
 609       emit_d8(cbuf, framesize);
 610     }
 611   } else {
 612     emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
 613     emit_rm(cbuf, 0x3, 0x05, ESP_enc);
 614     emit_d32(cbuf, framesize);
 615   }
 616   C->set_frame_complete(cbuf.code_end() - cbuf.code_begin());
 617 
 618 #ifdef ASSERT
 619   if (VerifyStackAtCalls) {
 620     Label L;
 621     MacroAssembler masm(&cbuf);
 622     masm.push(rax);
 623     masm.mov(rax, rsp);
 624     masm.andptr(rax, StackAlignmentInBytes-1);
 625     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
 626     masm.pop(rax);
 627     masm.jcc(Assembler::equal, L);
 628     masm.stop("Stack is not properly aligned!");
 629     masm.bind(L);
 630   }
 631 #endif
 632 
 633 }
 634 
 635 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
 636   return MachNode::size(ra_); // too many variables; just compute it the hard way
 637 }
 638 
 639 int MachPrologNode::reloc() const {
 640   return 0; // a large enough number
 641 }
 642 
 643 //=============================================================================
 644 #ifndef PRODUCT
 645 void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
 646   Compile *C = ra_->C;
 647   int framesize = C->frame_slots() << LogBytesPerInt;
 648   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 649   // Remove two words for return addr and rbp,
 650   framesize -= 2*wordSize;
 651 
 652   if( C->in_24_bit_fp_mode() ) {
 653     st->print("FLDCW  standard control word");
 654     st->cr(); st->print("\t");
 655   }
 656   if( framesize ) {
 657     st->print("ADD    ESP,%d\t# Destroy frame",framesize);
 658     st->cr(); st->print("\t");
 659   }
 660   st->print_cr("POPL   EBP"); st->print("\t");
 661   if( do_polling() && C->is_method_compilation() ) {
 662     st->print("TEST   PollPage,EAX\t! Poll Safepoint");
 663     st->cr(); st->print("\t");
 664   }
 665 }
 666 #endif
 667 
 668 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 669   Compile *C = ra_->C;
 670 
 671   // If method set FPU control word, restore to standard control word
 672   if( C->in_24_bit_fp_mode() ) {
 673     MacroAssembler masm(&cbuf);
 674     masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 675   }
 676 
 677   int framesize = C->frame_slots() << LogBytesPerInt;
 678   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 679   // Remove two words for return addr and rbp,
 680   framesize -= 2*wordSize;
 681 
 682   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 683 
 684   if( framesize >= 128 ) {
 685     emit_opcode(cbuf, 0x81); // add  SP, #framesize
 686     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 687     emit_d32(cbuf, framesize);
 688   }
 689   else if( framesize ) {
 690     emit_opcode(cbuf, 0x83); // add  SP, #framesize
 691     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
 692     emit_d8(cbuf, framesize);
 693   }
 694 
 695   emit_opcode(cbuf, 0x58 | EBP_enc);
 696 
 697   if( do_polling() && C->is_method_compilation() ) {
 698     cbuf.relocate(cbuf.code_end(), relocInfo::poll_return_type, 0);
 699     emit_opcode(cbuf,0x85);
 700     emit_rm(cbuf, 0x0, EAX_enc, 0x5); // EAX
 701     emit_d32(cbuf, (intptr_t)os::get_polling_page());
 702   }
 703 }
 704 
 705 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
 706   Compile *C = ra_->C;
 707   // If method set FPU control word, restore to standard control word
 708   int size = C->in_24_bit_fp_mode() ? 6 : 0;
 709   if( do_polling() && C->is_method_compilation() ) size += 6;
 710 
 711   int framesize = C->frame_slots() << LogBytesPerInt;
 712   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 713   // Remove two words for return addr and rbp,
 714   framesize -= 2*wordSize;
 715 
 716   size++; // popl rbp,
 717 
 718   if( framesize >= 128 ) {
 719     size += 6;
 720   } else {
 721     size += framesize ? 3 : 0;
 722   }
 723   return size;
 724 }
 725 
 726 int MachEpilogNode::reloc() const {
 727   return 0; // a large enough number
 728 }
 729 
 730 const Pipeline * MachEpilogNode::pipeline() const {
 731   return MachNode::pipeline_class();
 732 }
 733 
 734 int MachEpilogNode::safepoint_offset() const { return 0; }
 735 
 736 //=============================================================================
 737 
 738 enum RC { rc_bad, rc_int, rc_float, rc_xmm, rc_stack };
 739 static enum RC rc_class( OptoReg::Name reg ) {
 740 
 741   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 742   if (OptoReg::is_stack(reg)) return rc_stack;
 743 
 744   VMReg r = OptoReg::as_VMReg(reg);
 745   if (r->is_Register()) return rc_int;
 746   if (r->is_FloatRegister()) {
 747     assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");
 748     return rc_float;
 749   }
 750   assert(r->is_XMMRegister(), "must be");
 751   return rc_xmm;
 752 }
 753 
 754 static int impl_helper( CodeBuffer *cbuf, bool do_size, bool is_load, int offset, int reg,
 755                         int opcode, const char *op_str, int size, outputStream* st ) {
 756   if( cbuf ) {
 757     emit_opcode  (*cbuf, opcode );
 758     encode_RegMem(*cbuf, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, false);
 759 #ifndef PRODUCT
 760   } else if( !do_size ) {
 761     if( size != 0 ) st->print("\n\t");
 762     if( opcode == 0x8B || opcode == 0x89 ) { // MOV
 763       if( is_load ) st->print("%s   %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);
 764       else          st->print("%s   [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);
 765     } else { // FLD, FST, PUSH, POP
 766       st->print("%s [ESP + #%d]",op_str,offset);
 767     }
 768 #endif
 769   }
 770   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 771   return size+3+offset_size;
 772 }
 773 
 774 // Helper for XMM registers.  Extra opcode bits, limited syntax.
 775 static int impl_x_helper( CodeBuffer *cbuf, bool do_size, bool is_load,
 776                          int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {
 777   if( cbuf ) {
 778     if( reg_lo+1 == reg_hi ) { // double move?
 779       if( is_load && !UseXmmLoadAndClearUpper )
 780         emit_opcode(*cbuf, 0x66 ); // use 'movlpd' for load
 781       else
 782         emit_opcode(*cbuf, 0xF2 ); // use 'movsd' otherwise
 783     } else {
 784       emit_opcode(*cbuf, 0xF3 );
 785     }
 786     emit_opcode(*cbuf, 0x0F );
 787     if( reg_lo+1 == reg_hi && is_load && !UseXmmLoadAndClearUpper )
 788       emit_opcode(*cbuf, 0x12 );   // use 'movlpd' for load
 789     else
 790       emit_opcode(*cbuf, is_load ? 0x10 : 0x11 );
 791     encode_RegMem(*cbuf, Matcher::_regEncode[reg_lo], ESP_enc, 0x4, 0, offset, false);
 792 #ifndef PRODUCT
 793   } else if( !do_size ) {
 794     if( size != 0 ) st->print("\n\t");
 795     if( reg_lo+1 == reg_hi ) { // double move?
 796       if( is_load ) st->print("%s %s,[ESP + #%d]",
 797                                UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",
 798                                Matcher::regName[reg_lo], offset);
 799       else          st->print("MOVSD  [ESP + #%d],%s",
 800                                offset, Matcher::regName[reg_lo]);
 801     } else {
 802       if( is_load ) st->print("MOVSS  %s,[ESP + #%d]",
 803                                Matcher::regName[reg_lo], offset);
 804       else          st->print("MOVSS  [ESP + #%d],%s",
 805                                offset, Matcher::regName[reg_lo]);
 806     }
 807 #endif
 808   }
 809   int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
 810   return size+5+offset_size;
 811 }
 812 
 813 
 814 static int impl_movx_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 815                             int src_hi, int dst_hi, int size, outputStream* st ) {
 816   if( UseXmmRegToRegMoveAll ) {//Use movaps,movapd to move between xmm registers
 817     if( cbuf ) {
 818       if( (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ) {
 819         emit_opcode(*cbuf, 0x66 );
 820       }
 821       emit_opcode(*cbuf, 0x0F );
 822       emit_opcode(*cbuf, 0x28 );
 823       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
 824 #ifndef PRODUCT
 825     } else if( !do_size ) {
 826       if( size != 0 ) st->print("\n\t");
 827       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
 828         st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 829       } else {
 830         st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 831       }
 832 #endif
 833     }
 834     return size + ((src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 4 : 3);
 835   } else {
 836     if( cbuf ) {
 837       emit_opcode(*cbuf, (src_lo+1 == src_hi && dst_lo+1 == dst_hi) ? 0xF2 : 0xF3 );
 838       emit_opcode(*cbuf, 0x0F );
 839       emit_opcode(*cbuf, 0x10 );
 840       emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst_lo], Matcher::_regEncode[src_lo] );
 841 #ifndef PRODUCT
 842     } else if( !do_size ) {
 843       if( size != 0 ) st->print("\n\t");
 844       if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
 845         st->print("MOVSD  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 846       } else {
 847         st->print("MOVSS  %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
 848       }
 849 #endif
 850     }
 851     return size+4;
 852   }
 853 }
 854 
 855 static int impl_mov_helper( CodeBuffer *cbuf, bool do_size, int src, int dst, int size, outputStream* st ) {
 856   if( cbuf ) {
 857     emit_opcode(*cbuf, 0x8B );
 858     emit_rm    (*cbuf, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );
 859 #ifndef PRODUCT
 860   } else if( !do_size ) {
 861     if( size != 0 ) st->print("\n\t");
 862     st->print("MOV    %s,%s",Matcher::regName[dst],Matcher::regName[src]);
 863 #endif
 864   }
 865   return size+2;
 866 }
 867 
 868 static int impl_fp_store_helper( CodeBuffer *cbuf, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,
 869                                  int offset, int size, outputStream* st ) {
 870   if( src_lo != FPR1L_num ) {      // Move value to top of FP stack, if not already there
 871     if( cbuf ) {
 872       emit_opcode( *cbuf, 0xD9 );  // FLD (i.e., push it)
 873       emit_d8( *cbuf, 0xC0-1+Matcher::_regEncode[src_lo] );
 874 #ifndef PRODUCT
 875     } else if( !do_size ) {
 876       if( size != 0 ) st->print("\n\t");
 877       st->print("FLD    %s",Matcher::regName[src_lo]);
 878 #endif
 879     }
 880     size += 2;
 881   }
 882 
 883   int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;
 884   const char *op_str;
 885   int op;
 886   if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?
 887     op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";
 888     op = 0xDD;
 889   } else {                   // 32-bit store
 890     op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";
 891     op = 0xD9;
 892     assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );
 893   }
 894 
 895   return impl_helper(cbuf,do_size,false,offset,st_op,op,op_str,size, st);
 896 }
 897 
 898 uint MachSpillCopyNode::implementation( CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {
 899   // Get registers to move
 900   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 901   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 902   OptoReg::Name dst_second = ra_->get_reg_second(this );
 903   OptoReg::Name dst_first = ra_->get_reg_first(this );
 904 
 905   enum RC src_second_rc = rc_class(src_second);
 906   enum RC src_first_rc = rc_class(src_first);
 907   enum RC dst_second_rc = rc_class(dst_second);
 908   enum RC dst_first_rc = rc_class(dst_first);
 909 
 910   assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
 911 
 912   // Generate spill code!
 913   int size = 0;
 914 
 915   if( src_first == dst_first && src_second == dst_second )
 916     return size;            // Self copy, no move
 917 
 918   // --------------------------------------
 919   // Check for mem-mem move.  push/pop to move.
 920   if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
 921     if( src_second == dst_first ) { // overlapping stack copy ranges
 922       assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );
 923       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
 924       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
 925       src_second_rc = dst_second_rc = rc_bad;  // flag as already moved the second bits
 926     }
 927     // move low bits
 928     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH  ",size, st);
 929     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP   ",size, st);
 930     if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits
 931       size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH  ",size, st);
 932       size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP   ",size, st);
 933     }
 934     return size;
 935   }
 936 
 937   // --------------------------------------
 938   // Check for integer reg-reg copy
 939   if( src_first_rc == rc_int && dst_first_rc == rc_int )
 940     size = impl_mov_helper(cbuf,do_size,src_first,dst_first,size, st);
 941 
 942   // Check for integer store
 943   if( src_first_rc == rc_int && dst_first_rc == rc_stack )
 944     size = impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);
 945 
 946   // Check for integer load
 947   if( dst_first_rc == rc_int && src_first_rc == rc_stack )
 948     size = impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);
 949 
 950   // --------------------------------------
 951   // Check for float reg-reg copy
 952   if( src_first_rc == rc_float && dst_first_rc == rc_float ) {
 953     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
 954             (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );
 955     if( cbuf ) {
 956 
 957       // Note the mucking with the register encode to compensate for the 0/1
 958       // indexing issue mentioned in a comment in the reg_def sections
 959       // for FPR registers many lines above here.
 960 
 961       if( src_first != FPR1L_num ) {
 962         emit_opcode  (*cbuf, 0xD9 );           // FLD    ST(i)
 963         emit_d8      (*cbuf, 0xC0+Matcher::_regEncode[src_first]-1 );
 964         emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
 965         emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
 966      } else {
 967         emit_opcode  (*cbuf, 0xDD );           // FST    ST(i)
 968         emit_d8      (*cbuf, 0xD0+Matcher::_regEncode[dst_first]-1 );
 969      }
 970 #ifndef PRODUCT
 971     } else if( !do_size ) {
 972       if( size != 0 ) st->print("\n\t");
 973       if( src_first != FPR1L_num ) st->print("FLD    %s\n\tFSTP   %s",Matcher::regName[src_first],Matcher::regName[dst_first]);
 974       else                      st->print(             "FST    %s",                            Matcher::regName[dst_first]);
 975 #endif
 976     }
 977     return size + ((src_first != FPR1L_num) ? 2+2 : 2);
 978   }
 979 
 980   // Check for float store
 981   if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {
 982     return impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);
 983   }
 984 
 985   // Check for float load
 986   if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {
 987     int offset = ra_->reg2offset(src_first);
 988     const char *op_str;
 989     int op;
 990     if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?
 991       op_str = "FLD_D";
 992       op = 0xDD;
 993     } else {                   // 32-bit load
 994       op_str = "FLD_S";
 995       op = 0xD9;
 996       assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );
 997     }
 998     if( cbuf ) {
 999       emit_opcode  (*cbuf, op );
1000       encode_RegMem(*cbuf, 0x0, ESP_enc, 0x4, 0, offset, false);
1001       emit_opcode  (*cbuf, 0xDD );           // FSTP   ST(i)
1002       emit_d8      (*cbuf, 0xD8+Matcher::_regEncode[dst_first] );
1003 #ifndef PRODUCT
1004     } else if( !do_size ) {
1005       if( size != 0 ) st->print("\n\t");
1006       st->print("%s  ST,[ESP + #%d]\n\tFSTP   %s",op_str, offset,Matcher::regName[dst_first]);
1007 #endif
1008     }
1009     int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
1010     return size + 3+offset_size+2;
1011   }
1012 
1013   // Check for xmm reg-reg copy
1014   if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
1015     assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
1016             (src_first+1 == src_second && dst_first+1 == dst_second),
1017             "no non-adjacent float-moves" );
1018     return impl_movx_helper(cbuf,do_size,src_first,dst_first,src_second, dst_second, size, st);
1019   }
1020 
1021   // Check for xmm store
1022   if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
1023     return impl_x_helper(cbuf,do_size,false,ra_->reg2offset(dst_first),src_first, src_second, size, st);
1024   }
1025 
1026   // Check for float xmm load
1027   if( dst_first_rc == rc_xmm && src_first_rc == rc_stack ) {
1028     return impl_x_helper(cbuf,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);
1029   }
1030 
1031   // Copy from float reg to xmm reg
1032   if( dst_first_rc == rc_xmm && src_first_rc == rc_float ) {
1033     // copy to the top of stack from floating point reg
1034     // and use LEA to preserve flags
1035     if( cbuf ) {
1036       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP-8]
1037       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1038       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1039       emit_d8(*cbuf,0xF8);
1040 #ifndef PRODUCT
1041     } else if( !do_size ) {
1042       if( size != 0 ) st->print("\n\t");
1043       st->print("LEA    ESP,[ESP-8]");
1044 #endif
1045     }
1046     size += 4;
1047 
1048     size = impl_fp_store_helper(cbuf,do_size,src_first,src_second,dst_first,dst_second,0,size, st);
1049 
1050     // Copy from the temp memory to the xmm reg.
1051     size = impl_x_helper(cbuf,do_size,true ,0,dst_first, dst_second, size, st);
1052 
1053     if( cbuf ) {
1054       emit_opcode(*cbuf,0x8D);  // LEA  ESP,[ESP+8]
1055       emit_rm(*cbuf, 0x1, ESP_enc, 0x04);
1056       emit_rm(*cbuf, 0x0, 0x04, ESP_enc);
1057       emit_d8(*cbuf,0x08);
1058 #ifndef PRODUCT
1059     } else if( !do_size ) {
1060       if( size != 0 ) st->print("\n\t");
1061       st->print("LEA    ESP,[ESP+8]");
1062 #endif
1063     }
1064     size += 4;
1065     return size;
1066   }
1067 
1068   assert( size > 0, "missed a case" );
1069 
1070   // --------------------------------------------------------------------
1071   // Check for second bits still needing moving.
1072   if( src_second == dst_second )
1073     return size;               // Self copy; no move
1074   assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
1075 
1076   // Check for second word int-int move
1077   if( src_second_rc == rc_int && dst_second_rc == rc_int )
1078     return impl_mov_helper(cbuf,do_size,src_second,dst_second,size, st);
1079 
1080   // Check for second word integer store
1081   if( src_second_rc == rc_int && dst_second_rc == rc_stack )
1082     return impl_helper(cbuf,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);
1083 
1084   // Check for second word integer load
1085   if( dst_second_rc == rc_int && src_second_rc == rc_stack )
1086     return impl_helper(cbuf,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);
1087 
1088 
1089   Unimplemented();
1090 }
1091 
1092 #ifndef PRODUCT
1093 void MachSpillCopyNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1094   implementation( NULL, ra_, false, st );
1095 }
1096 #endif
1097 
1098 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1099   implementation( &cbuf, ra_, false, NULL );
1100 }
1101 
1102 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1103   return implementation( NULL, ra_, true, NULL );
1104 }
1105 
1106 //=============================================================================
1107 #ifndef PRODUCT
1108 void MachNopNode::format( PhaseRegAlloc *, outputStream* st ) const {
1109   st->print("NOP \t# %d bytes pad for loops and calls", _count);
1110 }
1111 #endif
1112 
1113 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc * ) const {
1114   MacroAssembler _masm(&cbuf);
1115   __ nop(_count);
1116 }
1117 
1118 uint MachNopNode::size(PhaseRegAlloc *) const {
1119   return _count;
1120 }
1121 
1122 
1123 //=============================================================================
1124 #ifndef PRODUCT
1125 void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1126   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1127   int reg = ra_->get_reg_first(this);
1128   st->print("LEA    %s,[ESP + #%d]",Matcher::regName[reg],offset);
1129 }
1130 #endif
1131 
1132 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1133   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1134   int reg = ra_->get_encode(this);
1135   if( offset >= 128 ) {
1136     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1137     emit_rm(cbuf, 0x2, reg, 0x04);
1138     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1139     emit_d32(cbuf, offset);
1140   }
1141   else {
1142     emit_opcode(cbuf, 0x8D);      // LEA  reg,[SP+offset]
1143     emit_rm(cbuf, 0x1, reg, 0x04);
1144     emit_rm(cbuf, 0x0, 0x04, ESP_enc);
1145     emit_d8(cbuf, offset);
1146   }
1147 }
1148 
1149 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1150   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1151   if( offset >= 128 ) {
1152     return 7;
1153   }
1154   else {
1155     return 4;
1156   }
1157 }
1158 
1159 //=============================================================================
1160 
1161 // emit call stub, compiled java to interpreter
1162 void emit_java_to_interp(CodeBuffer &cbuf ) {
1163   // Stub is fixed up when the corresponding call is converted from calling
1164   // compiled code to calling interpreted code.
1165   // mov rbx,0
1166   // jmp -1
1167 
1168   address mark = cbuf.inst_mark();  // get mark within main instrs section
1169 
1170   // Note that the code buffer's inst_mark is always relative to insts.
1171   // That's why we must use the macroassembler to generate a stub.
1172   MacroAssembler _masm(&cbuf);
1173 
1174   address base =
1175   __ start_a_stub(Compile::MAX_stubs_size);
1176   if (base == NULL)  return;  // CodeBuffer::expand failed
1177   // static stub relocation stores the instruction address of the call
1178   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32);
1179   // static stub relocation also tags the methodOop in the code-stream.
1180   __ movoop(rbx, (jobject)NULL);  // method is zapped till fixup time
1181   // This is recognized as unresolved by relocs/nativeInst/ic code
1182   __ jump(RuntimeAddress(__ pc()));
1183 
1184   __ end_a_stub();
1185   // Update current stubs pointer and restore code_end.
1186 }
1187 // size of call stub, compiled java to interpretor
1188 uint size_java_to_interp() {
1189   return 10;  // movl; jmp
1190 }
1191 // relocation entries for call stub, compiled java to interpretor
1192 uint reloc_java_to_interp() {
1193   return 4;  // 3 in emit_java_to_interp + 1 in Java_Static_Call
1194 }
1195 
1196 //=============================================================================
1197 #ifndef PRODUCT
1198 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
1199   st->print_cr(  "CMP    EAX,[ECX+4]\t# Inline cache check");
1200   st->print_cr("\tJNE    SharedRuntime::handle_ic_miss_stub");
1201   st->print_cr("\tNOP");
1202   st->print_cr("\tNOP");
1203   if( !OptoBreakpoint )
1204     st->print_cr("\tNOP");
1205 }
1206 #endif
1207 
1208 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1209   MacroAssembler masm(&cbuf);
1210 #ifdef ASSERT
1211   uint code_size = cbuf.code_size();
1212 #endif
1213   masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
1214   masm.jump_cc(Assembler::notEqual,
1215                RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1216   /* WARNING these NOPs are critical so that verified entry point is properly
1217      aligned for patching by NativeJump::patch_verified_entry() */
1218   int nops_cnt = 2;
1219   if( !OptoBreakpoint ) // Leave space for int3
1220      nops_cnt += 1;
1221   masm.nop(nops_cnt);
1222 
1223   assert(cbuf.code_size() - code_size == size(ra_), "checking code size of inline cache node");
1224 }
1225 
1226 uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
1227   return OptoBreakpoint ? 11 : 12;
1228 }
1229 
1230 
1231 //=============================================================================
1232 uint size_exception_handler() {
1233   // NativeCall instruction size is the same as NativeJump.
1234   // exception handler starts out as jump and can be patched to
1235   // a call be deoptimization.  (4932387)
1236   // Note that this value is also credited (in output.cpp) to
1237   // the size of the code section.
1238   return NativeJump::instruction_size;
1239 }
1240 
1241 // Emit exception handler code.  Stuff framesize into a register
1242 // and call a VM stub routine.
1243 int emit_exception_handler(CodeBuffer& cbuf) {
1244 
1245   // Note that the code buffer's inst_mark is always relative to insts.
1246   // That's why we must use the macroassembler to generate a handler.
1247   MacroAssembler _masm(&cbuf);
1248   address base =
1249   __ start_a_stub(size_exception_handler());
1250   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1251   int offset = __ offset();
1252   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->instructions_begin()));
1253   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1254   __ end_a_stub();
1255   return offset;
1256 }
1257 
1258 uint size_deopt_handler() {
1259   // NativeCall instruction size is the same as NativeJump.
1260   // exception handler starts out as jump and can be patched to
1261   // a call be deoptimization.  (4932387)
1262   // Note that this value is also credited (in output.cpp) to
1263   // the size of the code section.
1264   return 5 + NativeJump::instruction_size; // pushl(); jmp;
1265 }
1266 
1267 // Emit deopt handler code.
1268 int emit_deopt_handler(CodeBuffer& cbuf) {
1269 
1270   // Note that the code buffer's inst_mark is always relative to insts.
1271   // That's why we must use the macroassembler to generate a handler.
1272   MacroAssembler _masm(&cbuf);
1273   address base =
1274   __ start_a_stub(size_exception_handler());
1275   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1276   int offset = __ offset();
1277   InternalAddress here(__ pc());
1278   __ pushptr(here.addr());
1279 
1280   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1281   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1282   __ end_a_stub();
1283   return offset;
1284 }
1285 
1286 
1287 static void emit_double_constant(CodeBuffer& cbuf, double x) {
1288   int mark = cbuf.insts()->mark_off();
1289   MacroAssembler _masm(&cbuf);
1290   address double_address = __ double_constant(x);
1291   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
1292   emit_d32_reloc(cbuf,
1293                  (int)double_address,
1294                  internal_word_Relocation::spec(double_address),
1295                  RELOC_DISP32);
1296 }
1297 
1298 static void emit_float_constant(CodeBuffer& cbuf, float x) {
1299   int mark = cbuf.insts()->mark_off();
1300   MacroAssembler _masm(&cbuf);
1301   address float_address = __ float_constant(x);
1302   cbuf.insts()->set_mark_off(mark);  // preserve mark across masm shift
1303   emit_d32_reloc(cbuf,
1304                  (int)float_address,
1305                  internal_word_Relocation::spec(float_address),
1306                  RELOC_DISP32);
1307 }
1308 
1309 
1310 const bool Matcher::match_rule_supported(int opcode) {
1311   if (!has_match_rule(opcode))
1312     return false;
1313 
1314   return true;  // Per default match rules are supported.
1315 }
1316 
1317 int Matcher::regnum_to_fpu_offset(int regnum) {
1318   return regnum - 32; // The FP registers are in the second chunk
1319 }
1320 
1321 bool is_positive_zero_float(jfloat f) {
1322   return jint_cast(f) == jint_cast(0.0F);
1323 }
1324 
1325 bool is_positive_one_float(jfloat f) {
1326   return jint_cast(f) == jint_cast(1.0F);
1327 }
1328 
1329 bool is_positive_zero_double(jdouble d) {
1330   return jlong_cast(d) == jlong_cast(0.0);
1331 }
1332 
1333 bool is_positive_one_double(jdouble d) {
1334   return jlong_cast(d) == jlong_cast(1.0);
1335 }
1336 
1337 // This is UltraSparc specific, true just means we have fast l2f conversion
1338 const bool Matcher::convL2FSupported(void) {
1339   return true;
1340 }
1341 
1342 // Vector width in bytes
1343 const uint Matcher::vector_width_in_bytes(void) {
1344   return UseSSE >= 2 ? 8 : 0;
1345 }
1346 
1347 // Vector ideal reg
1348 const uint Matcher::vector_ideal_reg(void) {
1349   return Op_RegD;
1350 }
1351 
1352 // Is this branch offset short enough that a short branch can be used?
1353 //
1354 // NOTE: If the platform does not provide any short branch variants, then
1355 //       this method should return false for offset 0.
1356 bool Matcher::is_short_branch_offset(int rule, int offset) {
1357   // the short version of jmpConUCF2 contains multiple branches,
1358   // making the reach slightly less
1359   if (rule == jmpConUCF2_rule)
1360     return (-126 <= offset && offset <= 125);
1361   return (-128 <= offset && offset <= 127);
1362 }
1363 
1364 const bool Matcher::isSimpleConstant64(jlong value) {
1365   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1366   return false;
1367 }
1368 
1369 // The ecx parameter to rep stos for the ClearArray node is in dwords.
1370 const bool Matcher::init_array_count_is_in_bytes = false;
1371 
1372 // Threshold size for cleararray.
1373 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1374 
1375 // Should the Matcher clone shifts on addressing modes, expecting them to
1376 // be subsumed into complex addressing expressions or compute them into
1377 // registers?  True for Intel but false for most RISCs
1378 const bool Matcher::clone_shift_expressions = true;
1379 
1380 // Is it better to copy float constants, or load them directly from memory?
1381 // Intel can load a float constant from a direct address, requiring no
1382 // extra registers.  Most RISCs will have to materialize an address into a
1383 // register first, so they would do better to copy the constant from stack.
1384 const bool Matcher::rematerialize_float_constants = true;
1385 
1386 // If CPU can load and store mis-aligned doubles directly then no fixup is
1387 // needed.  Else we split the double into 2 integer pieces and move it
1388 // piece-by-piece.  Only happens when passing doubles into C code as the
1389 // Java calling convention forces doubles to be aligned.
1390 const bool Matcher::misaligned_doubles_ok = true;
1391 
1392 
1393 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
1394   // Get the memory operand from the node
1395   uint numopnds = node->num_opnds();        // Virtual call for number of operands
1396   uint skipped  = node->oper_input_base();  // Sum of leaves skipped so far
1397   assert( idx >= skipped, "idx too low in pd_implicit_null_fixup" );
1398   uint opcnt     = 1;                 // First operand
1399   uint num_edges = node->_opnds[1]->num_edges(); // leaves for first operand
1400   while( idx >= skipped+num_edges ) {
1401     skipped += num_edges;
1402     opcnt++;                          // Bump operand count
1403     assert( opcnt < numopnds, "Accessing non-existent operand" );
1404     num_edges = node->_opnds[opcnt]->num_edges(); // leaves for next operand
1405   }
1406 
1407   MachOper *memory = node->_opnds[opcnt];
1408   MachOper *new_memory = NULL;
1409   switch (memory->opcode()) {
1410   case DIRECT:
1411   case INDOFFSET32X:
1412     // No transformation necessary.
1413     return;
1414   case INDIRECT:
1415     new_memory = new (C) indirect_win95_safeOper( );
1416     break;
1417   case INDOFFSET8:
1418     new_memory = new (C) indOffset8_win95_safeOper(memory->disp(NULL, NULL, 0));
1419     break;
1420   case INDOFFSET32:
1421     new_memory = new (C) indOffset32_win95_safeOper(memory->disp(NULL, NULL, 0));
1422     break;
1423   case INDINDEXOFFSET:
1424     new_memory = new (C) indIndexOffset_win95_safeOper(memory->disp(NULL, NULL, 0));
1425     break;
1426   case INDINDEXSCALE:
1427     new_memory = new (C) indIndexScale_win95_safeOper(memory->scale());
1428     break;
1429   case INDINDEXSCALEOFFSET:
1430     new_memory = new (C) indIndexScaleOffset_win95_safeOper(memory->scale(), memory->disp(NULL, NULL, 0));
1431     break;
1432   case LOAD_LONG_INDIRECT:
1433   case LOAD_LONG_INDOFFSET32:
1434     // Does not use EBP as address register, use { EDX, EBX, EDI, ESI}
1435     return;
1436   default:
1437     assert(false, "unexpected memory operand in pd_implicit_null_fixup()");
1438     return;
1439   }
1440   node->_opnds[opcnt] = new_memory;
1441 }
1442 
1443 // Advertise here if the CPU requires explicit rounding operations
1444 // to implement the UseStrictFP mode.
1445 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1446 
1447 // Are floats conerted to double when stored to stack during deoptimization?
1448 // On x32 it is stored with convertion only when FPU is used for floats.
1449 bool Matcher::float_in_double() { return (UseSSE == 0); }
1450 
1451 // Do ints take an entire long register or just half?
1452 const bool Matcher::int_in_long = false;
1453 
1454 // Return whether or not this register is ever used as an argument.  This
1455 // function is used on startup to build the trampoline stubs in generateOptoStub.
1456 // Registers not mentioned will be killed by the VM call in the trampoline, and
1457 // arguments in those registers not be available to the callee.
1458 bool Matcher::can_be_java_arg( int reg ) {
1459   if(  reg == ECX_num   || reg == EDX_num   ) return true;
1460   if( (reg == XMM0a_num || reg == XMM1a_num) && UseSSE>=1 ) return true;
1461   if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;
1462   return false;
1463 }
1464 
1465 bool Matcher::is_spillable_arg( int reg ) {
1466   return can_be_java_arg(reg);
1467 }
1468 
1469 // Register for DIVI projection of divmodI
1470 RegMask Matcher::divI_proj_mask() {
1471   return EAX_REG_mask;
1472 }
1473 
1474 // Register for MODI projection of divmodI
1475 RegMask Matcher::modI_proj_mask() {
1476   return EDX_REG_mask;
1477 }
1478 
1479 // Register for DIVL projection of divmodL
1480 RegMask Matcher::divL_proj_mask() {
1481   ShouldNotReachHere();
1482   return RegMask();
1483 }
1484 
1485 // Register for MODL projection of divmodL
1486 RegMask Matcher::modL_proj_mask() {
1487   ShouldNotReachHere();
1488   return RegMask();
1489 }
1490 
1491 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1492   return EBP_REG_mask;
1493 }
1494 
1495 // Returns true if the high 32 bits of the value is known to be zero.
1496 bool is_operand_hi32_zero(Node* n) {
1497   int opc = n->Opcode();
1498   if (opc == Op_LoadUI2L) {
1499     return true;
1500   }
1501   if (opc == Op_AndL) {
1502     Node* o2 = n->in(2);
1503     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
1504       return true;
1505     }
1506   }
1507   return false;
1508 }
1509 
1510 %}
1511 
1512 //----------ENCODING BLOCK-----------------------------------------------------
1513 // This block specifies the encoding classes used by the compiler to output
1514 // byte streams.  Encoding classes generate functions which are called by
1515 // Machine Instruction Nodes in order to generate the bit encoding of the
1516 // instruction.  Operands specify their base encoding interface with the
1517 // interface keyword.  There are currently supported four interfaces,
1518 // REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER.  REG_INTER causes an
1519 // operand to generate a function which returns its register number when
1520 // queried.   CONST_INTER causes an operand to generate a function which
1521 // returns the value of the constant when queried.  MEMORY_INTER causes an
1522 // operand to generate four functions which return the Base Register, the
1523 // Index Register, the Scale Value, and the Offset Value of the operand when
1524 // queried.  COND_INTER causes an operand to generate six functions which
1525 // return the encoding code (ie - encoding bits for the instruction)
1526 // associated with each basic boolean condition for a conditional instruction.
1527 // Instructions specify two basic values for encoding.  They use the
1528 // ins_encode keyword to specify their encoding class (which must be one of
1529 // the class names specified in the encoding block), and they use the
1530 // opcode keyword to specify, in order, their primary, secondary, and
1531 // tertiary opcode.  Only the opcode sections which a particular instruction
1532 // needs for encoding need to be specified.
1533 encode %{
1534   // Build emit functions for each basic byte or larger field in the intel
1535   // encoding scheme (opcode, rm, sib, immediate), and call them from C++
1536   // code in the enc_class source block.  Emit functions will live in the
1537   // main source block for now.  In future, we can generalize this by
1538   // adding a syntax that specifies the sizes of fields in an order,
1539   // so that the adlc can build the emit functions automagically
1540 
1541   // Emit primary opcode
1542   enc_class OpcP %{
1543     emit_opcode(cbuf, $primary);
1544   %}
1545 
1546   // Emit secondary opcode
1547   enc_class OpcS %{
1548     emit_opcode(cbuf, $secondary);
1549   %}
1550 
1551   // Emit opcode directly
1552   enc_class Opcode(immI d8) %{
1553     emit_opcode(cbuf, $d8$$constant);
1554   %}
1555 
1556   enc_class SizePrefix %{
1557     emit_opcode(cbuf,0x66);
1558   %}
1559 
1560   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
1561     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1562   %}
1563 
1564   enc_class OpcRegReg (immI opcode, eRegI dst, eRegI src) %{    // OpcRegReg(Many)
1565     emit_opcode(cbuf,$opcode$$constant);
1566     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
1567   %}
1568 
1569   enc_class mov_r32_imm0( eRegI dst ) %{
1570     emit_opcode( cbuf, 0xB8 + $dst$$reg ); // 0xB8+ rd   -- MOV r32  ,imm32
1571     emit_d32   ( cbuf, 0x0  );             //                         imm32==0x0
1572   %}
1573 
1574   enc_class cdq_enc %{
1575     // Full implementation of Java idiv and irem; checks for
1576     // special case as described in JVM spec., p.243 & p.271.
1577     //
1578     //         normal case                           special case
1579     //
1580     // input : rax,: dividend                         min_int
1581     //         reg: divisor                          -1
1582     //
1583     // output: rax,: quotient  (= rax, idiv reg)       min_int
1584     //         rdx: remainder (= rax, irem reg)       0
1585     //
1586     //  Code sequnce:
1587     //
1588     //  81 F8 00 00 00 80    cmp         rax,80000000h
1589     //  0F 85 0B 00 00 00    jne         normal_case
1590     //  33 D2                xor         rdx,edx
1591     //  83 F9 FF             cmp         rcx,0FFh
1592     //  0F 84 03 00 00 00    je          done
1593     //                  normal_case:
1594     //  99                   cdq
1595     //  F7 F9                idiv        rax,ecx
1596     //                  done:
1597     //
1598     emit_opcode(cbuf,0x81); emit_d8(cbuf,0xF8);
1599     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);
1600     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x80);                     // cmp rax,80000000h
1601     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x85);
1602     emit_opcode(cbuf,0x0B); emit_d8(cbuf,0x00);
1603     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // jne normal_case
1604     emit_opcode(cbuf,0x33); emit_d8(cbuf,0xD2);                     // xor rdx,edx
1605     emit_opcode(cbuf,0x83); emit_d8(cbuf,0xF9); emit_d8(cbuf,0xFF); // cmp rcx,0FFh
1606     emit_opcode(cbuf,0x0F); emit_d8(cbuf,0x84);
1607     emit_opcode(cbuf,0x03); emit_d8(cbuf,0x00);
1608     emit_opcode(cbuf,0x00); emit_d8(cbuf,0x00);                     // je done
1609     // normal_case:
1610     emit_opcode(cbuf,0x99);                                         // cdq
1611     // idiv (note: must be emitted by the user of this rule)
1612     // normal:
1613   %}
1614 
1615   // Dense encoding for older common ops
1616   enc_class Opc_plus(immI opcode, eRegI reg) %{
1617     emit_opcode(cbuf, $opcode$$constant + $reg$$reg);
1618   %}
1619 
1620 
1621   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1622   enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit
1623     // Check for 8-bit immediate, and set sign extend bit in opcode
1624     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1625       emit_opcode(cbuf, $primary | 0x02);
1626     }
1627     else {                          // If 32-bit immediate
1628       emit_opcode(cbuf, $primary);
1629     }
1630   %}
1631 
1632   enc_class OpcSErm (eRegI dst, immI imm) %{    // OpcSEr/m
1633     // Emit primary opcode and set sign-extend bit
1634     // Check for 8-bit immediate, and set sign extend bit in opcode
1635     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1636       emit_opcode(cbuf, $primary | 0x02);    }
1637     else {                          // If 32-bit immediate
1638       emit_opcode(cbuf, $primary);
1639     }
1640     // Emit r/m byte with secondary opcode, after primary opcode.
1641     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1642   %}
1643 
1644   enc_class Con8or32 (immI imm) %{    // Con8or32(storeImmI), 8 or 32 bits
1645     // Check for 8-bit immediate, and set sign extend bit in opcode
1646     if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
1647       $$$emit8$imm$$constant;
1648     }
1649     else {                          // If 32-bit immediate
1650       // Output immediate
1651       $$$emit32$imm$$constant;
1652     }
1653   %}
1654 
1655   enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{
1656     // Emit primary opcode and set sign-extend bit
1657     // Check for 8-bit immediate, and set sign extend bit in opcode
1658     int con = (int)$imm$$constant; // Throw away top bits
1659     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1660     // Emit r/m byte with secondary opcode, after primary opcode.
1661     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1662     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1663     else                               emit_d32(cbuf,con);
1664   %}
1665 
1666   enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{
1667     // Emit primary opcode and set sign-extend bit
1668     // Check for 8-bit immediate, and set sign extend bit in opcode
1669     int con = (int)($imm$$constant >> 32); // Throw away bottom bits
1670     emit_opcode(cbuf, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
1671     // Emit r/m byte with tertiary opcode, after primary opcode.
1672     emit_rm(cbuf, 0x3, $tertiary, HIGH_FROM_LOW($dst$$reg));
1673     if ((con >= -128) && (con <= 127)) emit_d8 (cbuf,con);
1674     else                               emit_d32(cbuf,con);
1675   %}
1676 
1677   enc_class Lbl (label labl) %{ // JMP, CALL
1678     Label *l = $labl$$label;
1679     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);
1680   %}
1681 
1682   enc_class LblShort (label labl) %{ // JMP, CALL
1683     Label *l = $labl$$label;
1684     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;
1685     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
1686     emit_d8(cbuf, disp);
1687   %}
1688 
1689   enc_class OpcSReg (eRegI dst) %{    // BSWAP
1690     emit_cc(cbuf, $secondary, $dst$$reg );
1691   %}
1692 
1693   enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP
1694     int destlo = $dst$$reg;
1695     int desthi = HIGH_FROM_LOW(destlo);
1696     // bswap lo
1697     emit_opcode(cbuf, 0x0F);
1698     emit_cc(cbuf, 0xC8, destlo);
1699     // bswap hi
1700     emit_opcode(cbuf, 0x0F);
1701     emit_cc(cbuf, 0xC8, desthi);
1702     // xchg lo and hi
1703     emit_opcode(cbuf, 0x87);
1704     emit_rm(cbuf, 0x3, destlo, desthi);
1705   %}
1706 
1707   enc_class RegOpc (eRegI div) %{    // IDIV, IMOD, JMP indirect, ...
1708     emit_rm(cbuf, 0x3, $secondary, $div$$reg );
1709   %}
1710 
1711   enc_class Jcc (cmpOp cop, label labl) %{    // JCC
1712     Label *l = $labl$$label;
1713     $$$emit8$primary;
1714     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1715     emit_d32(cbuf, l ? (l->loc_pos() - (cbuf.code_size()+4)) : 0);
1716   %}
1717 
1718   enc_class JccShort (cmpOp cop, label labl) %{    // JCC
1719     Label *l = $labl$$label;
1720     emit_cc(cbuf, $primary, $cop$$cmpcode);
1721     int disp = l ? (l->loc_pos() - (cbuf.code_size()+1)) : 0;
1722     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
1723     emit_d8(cbuf, disp);
1724   %}
1725 
1726   enc_class enc_cmov(cmpOp cop ) %{ // CMOV
1727     $$$emit8$primary;
1728     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1729   %}
1730 
1731   enc_class enc_cmov_d(cmpOp cop, regD src ) %{ // CMOV
1732     int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);
1733     emit_d8(cbuf, op >> 8 );
1734     emit_d8(cbuf, op & 255);
1735   %}
1736 
1737   // emulate a CMOV with a conditional branch around a MOV
1738   enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV
1739     // Invert sense of branch from sense of CMOV
1740     emit_cc( cbuf, 0x70, ($cop$$cmpcode^1) );
1741     emit_d8( cbuf, $brOffs$$constant );
1742   %}
1743 
1744   enc_class enc_PartialSubtypeCheck( ) %{
1745     Register Redi = as_Register(EDI_enc); // result register
1746     Register Reax = as_Register(EAX_enc); // super class
1747     Register Recx = as_Register(ECX_enc); // killed
1748     Register Resi = as_Register(ESI_enc); // sub class
1749     Label miss;
1750 
1751     MacroAssembler _masm(&cbuf);
1752     __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
1753                                      NULL, &miss,
1754                                      /*set_cond_codes:*/ true);
1755     if ($primary) {
1756       __ xorptr(Redi, Redi);
1757     }
1758     __ bind(miss);
1759   %}
1760 
1761   enc_class FFree_Float_Stack_All %{    // Free_Float_Stack_All
1762     MacroAssembler masm(&cbuf);
1763     int start = masm.offset();
1764     if (UseSSE >= 2) {
1765       if (VerifyFPU) {
1766         masm.verify_FPU(0, "must be empty in SSE2+ mode");
1767       }
1768     } else {
1769       // External c_calling_convention expects the FPU stack to be 'clean'.
1770       // Compiled code leaves it dirty.  Do cleanup now.
1771       masm.empty_FPU_stack();
1772     }
1773     if (sizeof_FFree_Float_Stack_All == -1) {
1774       sizeof_FFree_Float_Stack_All = masm.offset() - start;
1775     } else {
1776       assert(masm.offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");
1777     }
1778   %}
1779 
1780   enc_class Verify_FPU_For_Leaf %{
1781     if( VerifyFPU ) {
1782       MacroAssembler masm(&cbuf);
1783       masm.verify_FPU( -3, "Returning from Runtime Leaf call");
1784     }
1785   %}
1786 
1787   enc_class Java_To_Runtime (method meth) %{    // CALL Java_To_Runtime, Java_To_Runtime_Leaf
1788     // This is the instruction starting address for relocation info.
1789     cbuf.set_inst_mark();
1790     $$$emit8$primary;
1791     // CALL directly to the runtime
1792     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
1793                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1794 
1795     if (UseSSE >= 2) {
1796       MacroAssembler _masm(&cbuf);
1797       BasicType rt = tf()->return_type();
1798 
1799       if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {
1800         // A C runtime call where the return value is unused.  In SSE2+
1801         // mode the result needs to be removed from the FPU stack.  It's
1802         // likely that this function call could be removed by the
1803         // optimizer if the C function is a pure function.
1804         __ ffree(0);
1805       } else if (rt == T_FLOAT) {
1806         __ lea(rsp, Address(rsp, -4));
1807         __ fstp_s(Address(rsp, 0));
1808         __ movflt(xmm0, Address(rsp, 0));
1809         __ lea(rsp, Address(rsp,  4));
1810       } else if (rt == T_DOUBLE) {
1811         __ lea(rsp, Address(rsp, -8));
1812         __ fstp_d(Address(rsp, 0));
1813         __ movdbl(xmm0, Address(rsp, 0));
1814         __ lea(rsp, Address(rsp,  8));
1815       }
1816     }
1817   %}
1818 
1819 
1820   enc_class pre_call_FPU %{
1821     // If method sets FPU control word restore it here
1822     debug_only(int off0 = cbuf.code_size());
1823     if( Compile::current()->in_24_bit_fp_mode() ) {
1824       MacroAssembler masm(&cbuf);
1825       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1826     }
1827     debug_only(int off1 = cbuf.code_size());
1828     assert(off1 - off0 == pre_call_FPU_size(), "correct size prediction");
1829   %}
1830 
1831   enc_class post_call_FPU %{
1832     // If method sets FPU control word do it here also
1833     if( Compile::current()->in_24_bit_fp_mode() ) {
1834       MacroAssembler masm(&cbuf);
1835       masm.fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
1836     }
1837   %}
1838 
1839   enc_class preserve_SP %{
1840     debug_only(int off0 = cbuf.code_size());
1841     MacroAssembler _masm(&cbuf);
1842     // RBP is preserved across all calls, even compiled calls.
1843     // Use it to preserve RSP in places where the callee might change the SP.
1844     __ movptr(rbp, rsp);
1845     debug_only(int off1 = cbuf.code_size());
1846     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
1847   %}
1848 
1849   enc_class restore_SP %{
1850     MacroAssembler _masm(&cbuf);
1851     __ movptr(rsp, rbp);
1852   %}
1853 
1854   enc_class Java_Static_Call (method meth) %{    // JAVA STATIC CALL
1855     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1856     // who we intended to call.
1857     cbuf.set_inst_mark();
1858     $$$emit8$primary;
1859     if ( !_method ) {
1860       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
1861                      runtime_call_Relocation::spec(), RELOC_IMM32 );
1862     } else if(_optimized_virtual) {
1863       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
1864                      opt_virtual_call_Relocation::spec(), RELOC_IMM32 );
1865     } else {
1866       emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
1867                      static_call_Relocation::spec(), RELOC_IMM32 );
1868     }
1869     if( _method ) {  // Emit stub for static call
1870       emit_java_to_interp(cbuf);
1871     }
1872   %}
1873 
1874   enc_class Java_Dynamic_Call (method meth) %{    // JAVA DYNAMIC CALL
1875     // !!!!!
1876     // Generate  "Mov EAX,0x00", placeholder instruction to load oop-info
1877     // emit_call_dynamic_prologue( cbuf );
1878     cbuf.set_inst_mark();
1879     emit_opcode(cbuf, 0xB8 + EAX_enc);        // mov    EAX,-1
1880     emit_d32_reloc(cbuf, (int)Universe::non_oop_word(), oop_Relocation::spec_for_immediate(), RELOC_IMM32);
1881     address  virtual_call_oop_addr = cbuf.inst_mark();
1882     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
1883     // who we intended to call.
1884     cbuf.set_inst_mark();
1885     $$$emit8$primary;
1886     emit_d32_reloc(cbuf, ($meth$$method - (int)(cbuf.code_end()) - 4),
1887                 virtual_call_Relocation::spec(virtual_call_oop_addr), RELOC_IMM32 );
1888   %}
1889 
1890   enc_class Java_Compiled_Call (method meth) %{    // JAVA COMPILED CALL
1891     int disp = in_bytes(methodOopDesc::from_compiled_offset());
1892     assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");
1893 
1894     // CALL *[EAX+in_bytes(methodOopDesc::from_compiled_code_entry_point_offset())]
1895     cbuf.set_inst_mark();
1896     $$$emit8$primary;
1897     emit_rm(cbuf, 0x01, $secondary, EAX_enc );  // R/M byte
1898     emit_d8(cbuf, disp);             // Displacement
1899 
1900   %}
1901 
1902   enc_class Xor_Reg (eRegI dst) %{
1903     emit_opcode(cbuf, 0x33);
1904     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
1905   %}
1906 
1907 //   Following encoding is no longer used, but may be restored if calling
1908 //   convention changes significantly.
1909 //   Became: Xor_Reg(EBP), Java_To_Runtime( labl )
1910 //
1911 //   enc_class Java_Interpreter_Call (label labl) %{    // JAVA INTERPRETER CALL
1912 //     // int ic_reg     = Matcher::inline_cache_reg();
1913 //     // int ic_encode  = Matcher::_regEncode[ic_reg];
1914 //     // int imo_reg    = Matcher::interpreter_method_oop_reg();
1915 //     // int imo_encode = Matcher::_regEncode[imo_reg];
1916 //
1917 //     // // Interpreter expects method_oop in EBX, currently a callee-saved register,
1918 //     // // so we load it immediately before the call
1919 //     // emit_opcode(cbuf, 0x8B);                     // MOV    imo_reg,ic_reg  # method_oop
1920 //     // emit_rm(cbuf, 0x03, imo_encode, ic_encode ); // R/M byte
1921 //
1922 //     // xor rbp,ebp
1923 //     emit_opcode(cbuf, 0x33);
1924 //     emit_rm(cbuf, 0x3, EBP_enc, EBP_enc);
1925 //
1926 //     // CALL to interpreter.
1927 //     cbuf.set_inst_mark();
1928 //     $$$emit8$primary;
1929 //     emit_d32_reloc(cbuf, ($labl$$label - (int)(cbuf.code_end()) - 4),
1930 //                 runtime_call_Relocation::spec(), RELOC_IMM32 );
1931 //   %}
1932 
1933   enc_class RegOpcImm (eRegI dst, immI8 shift) %{    // SHL, SAR, SHR
1934     $$$emit8$primary;
1935     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
1936     $$$emit8$shift$$constant;
1937   %}
1938 
1939   enc_class LdImmI (eRegI dst, immI src) %{    // Load Immediate
1940     // Load immediate does not have a zero or sign extended version
1941     // for 8-bit immediates
1942     emit_opcode(cbuf, 0xB8 + $dst$$reg);
1943     $$$emit32$src$$constant;
1944   %}
1945 
1946   enc_class LdImmP (eRegI dst, immI src) %{    // Load Immediate
1947     // Load immediate does not have a zero or sign extended version
1948     // for 8-bit immediates
1949     emit_opcode(cbuf, $primary + $dst$$reg);
1950     $$$emit32$src$$constant;
1951   %}
1952 
1953   enc_class LdImmL_Lo( eRegL dst, immL src) %{    // Load Immediate
1954     // Load immediate does not have a zero or sign extended version
1955     // for 8-bit immediates
1956     int dst_enc = $dst$$reg;
1957     int src_con = $src$$constant & 0x0FFFFFFFFL;
1958     if (src_con == 0) {
1959       // xor dst, dst
1960       emit_opcode(cbuf, 0x33);
1961       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
1962     } else {
1963       emit_opcode(cbuf, $primary + dst_enc);
1964       emit_d32(cbuf, src_con);
1965     }
1966   %}
1967 
1968   enc_class LdImmL_Hi( eRegL dst, immL src) %{    // Load Immediate
1969     // Load immediate does not have a zero or sign extended version
1970     // for 8-bit immediates
1971     int dst_enc = $dst$$reg + 2;
1972     int src_con = ((julong)($src$$constant)) >> 32;
1973     if (src_con == 0) {
1974       // xor dst, dst
1975       emit_opcode(cbuf, 0x33);
1976       emit_rm(cbuf, 0x3, dst_enc, dst_enc);
1977     } else {
1978       emit_opcode(cbuf, $primary + dst_enc);
1979       emit_d32(cbuf, src_con);
1980     }
1981   %}
1982 
1983 
1984   enc_class LdImmD (immD src) %{    // Load Immediate
1985     if( is_positive_zero_double($src$$constant)) {
1986       // FLDZ
1987       emit_opcode(cbuf,0xD9);
1988       emit_opcode(cbuf,0xEE);
1989     } else if( is_positive_one_double($src$$constant)) {
1990       // FLD1
1991       emit_opcode(cbuf,0xD9);
1992       emit_opcode(cbuf,0xE8);
1993     } else {
1994       emit_opcode(cbuf,0xDD);
1995       emit_rm(cbuf, 0x0, 0x0, 0x5);
1996       emit_double_constant(cbuf, $src$$constant);
1997     }
1998   %}
1999 
2000 
2001   enc_class LdImmF (immF src) %{    // Load Immediate
2002     if( is_positive_zero_float($src$$constant)) {
2003       emit_opcode(cbuf,0xD9);
2004       emit_opcode(cbuf,0xEE);
2005     } else if( is_positive_one_float($src$$constant)) {
2006       emit_opcode(cbuf,0xD9);
2007       emit_opcode(cbuf,0xE8);
2008     } else {
2009       $$$emit8$primary;
2010       // Load immediate does not have a zero or sign extended version
2011       // for 8-bit immediates
2012       // First load to TOS, then move to dst
2013       emit_rm(cbuf, 0x0, 0x0, 0x5);
2014       emit_float_constant(cbuf, $src$$constant);
2015     }
2016   %}
2017 
2018   enc_class LdImmX (regX dst, immXF con) %{    // Load Immediate
2019     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
2020     emit_float_constant(cbuf, $con$$constant);
2021   %}
2022 
2023   enc_class LdImmXD (regXD dst, immXD con) %{    // Load Immediate
2024     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
2025     emit_double_constant(cbuf, $con$$constant);
2026   %}
2027 
2028   enc_class load_conXD (regXD dst, immXD con) %{ // Load double constant
2029     // UseXmmLoadAndClearUpper ? movsd(dst, con) : movlpd(dst, con)
2030     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
2031     emit_opcode(cbuf, 0x0F);
2032     emit_opcode(cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
2033     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
2034     emit_double_constant(cbuf, $con$$constant);
2035   %}
2036 
2037   enc_class Opc_MemImm_F(immF src) %{
2038     cbuf.set_inst_mark();
2039     $$$emit8$primary;
2040     emit_rm(cbuf, 0x0, $secondary, 0x5);
2041     emit_float_constant(cbuf, $src$$constant);
2042   %}
2043 
2044 
2045   enc_class MovI2X_reg(regX dst, eRegI src) %{
2046     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
2047     emit_opcode(cbuf, 0x0F );
2048     emit_opcode(cbuf, 0x6E );
2049     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2050   %}
2051 
2052   enc_class MovX2I_reg(eRegI dst, regX src) %{
2053     emit_opcode(cbuf, 0x66 );     // MOVD dst,src
2054     emit_opcode(cbuf, 0x0F );
2055     emit_opcode(cbuf, 0x7E );
2056     emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
2057   %}
2058 
2059   enc_class MovL2XD_reg(regXD dst, eRegL src, regXD tmp) %{
2060     { // MOVD $dst,$src.lo
2061       emit_opcode(cbuf,0x66);
2062       emit_opcode(cbuf,0x0F);
2063       emit_opcode(cbuf,0x6E);
2064       emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2065     }
2066     { // MOVD $tmp,$src.hi
2067       emit_opcode(cbuf,0x66);
2068       emit_opcode(cbuf,0x0F);
2069       emit_opcode(cbuf,0x6E);
2070       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
2071     }
2072     { // PUNPCKLDQ $dst,$tmp
2073       emit_opcode(cbuf,0x66);
2074       emit_opcode(cbuf,0x0F);
2075       emit_opcode(cbuf,0x62);
2076       emit_rm(cbuf, 0x3, $dst$$reg, $tmp$$reg);
2077      }
2078   %}
2079 
2080   enc_class MovXD2L_reg(eRegL dst, regXD src, regXD tmp) %{
2081     { // MOVD $dst.lo,$src
2082       emit_opcode(cbuf,0x66);
2083       emit_opcode(cbuf,0x0F);
2084       emit_opcode(cbuf,0x7E);
2085       emit_rm(cbuf, 0x3, $src$$reg, $dst$$reg);
2086     }
2087     { // PSHUFLW $tmp,$src,0x4E  (01001110b)
2088       emit_opcode(cbuf,0xF2);
2089       emit_opcode(cbuf,0x0F);
2090       emit_opcode(cbuf,0x70);
2091       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
2092       emit_d8(cbuf, 0x4E);
2093     }
2094     { // MOVD $dst.hi,$tmp
2095       emit_opcode(cbuf,0x66);
2096       emit_opcode(cbuf,0x0F);
2097       emit_opcode(cbuf,0x7E);
2098       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
2099     }
2100   %}
2101 
2102 
2103   // Encode a reg-reg copy.  If it is useless, then empty encoding.
2104   enc_class enc_Copy( eRegI dst, eRegI src ) %{
2105     encode_Copy( cbuf, $dst$$reg, $src$$reg );
2106   %}
2107 
2108   enc_class enc_CopyL_Lo( eRegI dst, eRegL src ) %{
2109     encode_Copy( cbuf, $dst$$reg, $src$$reg );
2110   %}
2111 
2112   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.
2113   enc_class enc_CopyXD( RegXD dst, RegXD src ) %{
2114     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
2115   %}
2116 
2117   enc_class RegReg (eRegI dst, eRegI src) %{    // RegReg(Many)
2118     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2119   %}
2120 
2121   enc_class RegReg_Lo(eRegL dst, eRegL src) %{    // RegReg(Many)
2122     $$$emit8$primary;
2123     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2124   %}
2125 
2126   enc_class RegReg_Hi(eRegL dst, eRegL src) %{    // RegReg(Many)
2127     $$$emit8$secondary;
2128     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2129   %}
2130 
2131   enc_class RegReg_Lo2(eRegL dst, eRegL src) %{    // RegReg(Many)
2132     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2133   %}
2134 
2135   enc_class RegReg_Hi2(eRegL dst, eRegL src) %{    // RegReg(Many)
2136     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg));
2137   %}
2138 
2139   enc_class RegReg_HiLo( eRegL src, eRegI dst ) %{
2140     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($src$$reg));
2141   %}
2142 
2143   enc_class Con32 (immI src) %{    // Con32(storeImmI)
2144     // Output immediate
2145     $$$emit32$src$$constant;
2146   %}
2147 
2148   enc_class Con32F_as_bits(immF src) %{        // storeF_imm
2149     // Output Float immediate bits
2150     jfloat jf = $src$$constant;
2151     int    jf_as_bits = jint_cast( jf );
2152     emit_d32(cbuf, jf_as_bits);
2153   %}
2154 
2155   enc_class Con32XF_as_bits(immXF src) %{      // storeX_imm
2156     // Output Float immediate bits
2157     jfloat jf = $src$$constant;
2158     int    jf_as_bits = jint_cast( jf );
2159     emit_d32(cbuf, jf_as_bits);
2160   %}
2161 
2162   enc_class Con16 (immI src) %{    // Con16(storeImmI)
2163     // Output immediate
2164     $$$emit16$src$$constant;
2165   %}
2166 
2167   enc_class Con_d32(immI src) %{
2168     emit_d32(cbuf,$src$$constant);
2169   %}
2170 
2171   enc_class conmemref (eRegP t1) %{    // Con32(storeImmI)
2172     // Output immediate memory reference
2173     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2174     emit_d32(cbuf, 0x00);
2175   %}
2176 
2177   enc_class lock_prefix( ) %{
2178     if( os::is_MP() )
2179       emit_opcode(cbuf,0xF0);         // [Lock]
2180   %}
2181 
2182   // Cmp-xchg long value.
2183   // Note: we need to swap rbx, and rcx before and after the
2184   //       cmpxchg8 instruction because the instruction uses
2185   //       rcx as the high order word of the new value to store but
2186   //       our register encoding uses rbx,.
2187   enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{
2188 
2189     // XCHG  rbx,ecx
2190     emit_opcode(cbuf,0x87);
2191     emit_opcode(cbuf,0xD9);
2192     // [Lock]
2193     if( os::is_MP() )
2194       emit_opcode(cbuf,0xF0);
2195     // CMPXCHG8 [Eptr]
2196     emit_opcode(cbuf,0x0F);
2197     emit_opcode(cbuf,0xC7);
2198     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2199     // XCHG  rbx,ecx
2200     emit_opcode(cbuf,0x87);
2201     emit_opcode(cbuf,0xD9);
2202   %}
2203 
2204   enc_class enc_cmpxchg(eSIRegP mem_ptr) %{
2205     // [Lock]
2206     if( os::is_MP() )
2207       emit_opcode(cbuf,0xF0);
2208 
2209     // CMPXCHG [Eptr]
2210     emit_opcode(cbuf,0x0F);
2211     emit_opcode(cbuf,0xB1);
2212     emit_rm( cbuf, 0x0, 1, $mem_ptr$$reg );
2213   %}
2214 
2215   enc_class enc_flags_ne_to_boolean( iRegI res ) %{
2216     int res_encoding = $res$$reg;
2217 
2218     // MOV  res,0
2219     emit_opcode( cbuf, 0xB8 + res_encoding);
2220     emit_d32( cbuf, 0 );
2221     // JNE,s  fail
2222     emit_opcode(cbuf,0x75);
2223     emit_d8(cbuf, 5 );
2224     // MOV  res,1
2225     emit_opcode( cbuf, 0xB8 + res_encoding);
2226     emit_d32( cbuf, 1 );
2227     // fail:
2228   %}
2229 
2230   enc_class set_instruction_start( ) %{
2231     cbuf.set_inst_mark();            // Mark start of opcode for reloc info in mem operand
2232   %}
2233 
2234   enc_class RegMem (eRegI ereg, memory mem) %{    // emit_reg_mem
2235     int reg_encoding = $ereg$$reg;
2236     int base  = $mem$$base;
2237     int index = $mem$$index;
2238     int scale = $mem$$scale;
2239     int displace = $mem$$disp;
2240     bool disp_is_oop = $mem->disp_is_oop();
2241     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2242   %}
2243 
2244   enc_class RegMem_Hi(eRegL ereg, memory mem) %{    // emit_reg_mem
2245     int reg_encoding = HIGH_FROM_LOW($ereg$$reg);  // Hi register of pair, computed from lo
2246     int base  = $mem$$base;
2247     int index = $mem$$index;
2248     int scale = $mem$$scale;
2249     int displace = $mem$$disp + 4;      // Offset is 4 further in memory
2250     assert( !$mem->disp_is_oop(), "Cannot add 4 to oop" );
2251     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, false/*disp_is_oop*/);
2252   %}
2253 
2254   enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{
2255     int r1, r2;
2256     if( $tertiary == 0xA4 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2257     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2258     emit_opcode(cbuf,0x0F);
2259     emit_opcode(cbuf,$tertiary);
2260     emit_rm(cbuf, 0x3, r1, r2);
2261     emit_d8(cbuf,$cnt$$constant);
2262     emit_d8(cbuf,$primary);
2263     emit_rm(cbuf, 0x3, $secondary, r1);
2264     emit_d8(cbuf,$cnt$$constant);
2265   %}
2266 
2267   enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{
2268     emit_opcode( cbuf, 0x8B ); // Move
2269     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2270     emit_d8(cbuf,$primary);
2271     emit_rm(cbuf, 0x3, $secondary, $dst$$reg);
2272     emit_d8(cbuf,$cnt$$constant-32);
2273     emit_d8(cbuf,$primary);
2274     emit_rm(cbuf, 0x3, $secondary, HIGH_FROM_LOW($dst$$reg));
2275     emit_d8(cbuf,31);
2276   %}
2277 
2278   enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{
2279     int r1, r2;
2280     if( $secondary == 0x5 ) { r1 = $dst$$reg;  r2 = HIGH_FROM_LOW($dst$$reg); }
2281     else                    { r2 = $dst$$reg;  r1 = HIGH_FROM_LOW($dst$$reg); }
2282 
2283     emit_opcode( cbuf, 0x8B ); // Move r1,r2
2284     emit_rm(cbuf, 0x3, r1, r2);
2285     if( $cnt$$constant > 32 ) { // Shift, if not by zero
2286       emit_opcode(cbuf,$primary);
2287       emit_rm(cbuf, 0x3, $secondary, r1);
2288       emit_d8(cbuf,$cnt$$constant-32);
2289     }
2290     emit_opcode(cbuf,0x33);  // XOR r2,r2
2291     emit_rm(cbuf, 0x3, r2, r2);
2292   %}
2293 
2294   // Clone of RegMem but accepts an extra parameter to access each
2295   // half of a double in memory; it never needs relocation info.
2296   enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, eRegI rm_reg) %{
2297     emit_opcode(cbuf,$opcode$$constant);
2298     int reg_encoding = $rm_reg$$reg;
2299     int base     = $mem$$base;
2300     int index    = $mem$$index;
2301     int scale    = $mem$$scale;
2302     int displace = $mem$$disp + $disp_for_half$$constant;
2303     bool disp_is_oop = false;
2304     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2305   %}
2306 
2307   // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!
2308   //
2309   // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant
2310   // and it never needs relocation information.
2311   // Frequently used to move data between FPU's Stack Top and memory.
2312   enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{
2313     int rm_byte_opcode = $rm_opcode$$constant;
2314     int base     = $mem$$base;
2315     int index    = $mem$$index;
2316     int scale    = $mem$$scale;
2317     int displace = $mem$$disp;
2318     assert( !$mem->disp_is_oop(), "No oops here because no relo info allowed" );
2319     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, false);
2320   %}
2321 
2322   enc_class RMopc_Mem (immI rm_opcode, memory mem) %{
2323     int rm_byte_opcode = $rm_opcode$$constant;
2324     int base     = $mem$$base;
2325     int index    = $mem$$index;
2326     int scale    = $mem$$scale;
2327     int displace = $mem$$disp;
2328     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
2329     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
2330   %}
2331 
2332   enc_class RegLea (eRegI dst, eRegI src0, immI src1 ) %{    // emit_reg_lea
2333     int reg_encoding = $dst$$reg;
2334     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2335     int index        = 0x04;            // 0x04 indicates no index
2336     int scale        = 0x00;            // 0x00 indicates no scale
2337     int displace     = $src1$$constant; // 0x00 indicates no displacement
2338     bool disp_is_oop = false;
2339     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2340   %}
2341 
2342   enc_class min_enc (eRegI dst, eRegI src) %{    // MIN
2343     // Compare dst,src
2344     emit_opcode(cbuf,0x3B);
2345     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2346     // jmp dst < src around move
2347     emit_opcode(cbuf,0x7C);
2348     emit_d8(cbuf,2);
2349     // move dst,src
2350     emit_opcode(cbuf,0x8B);
2351     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2352   %}
2353 
2354   enc_class max_enc (eRegI dst, eRegI src) %{    // MAX
2355     // Compare dst,src
2356     emit_opcode(cbuf,0x3B);
2357     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2358     // jmp dst > src around move
2359     emit_opcode(cbuf,0x7F);
2360     emit_d8(cbuf,2);
2361     // move dst,src
2362     emit_opcode(cbuf,0x8B);
2363     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
2364   %}
2365 
2366   enc_class enc_FP_store(memory mem, regD src) %{
2367     // If src is FPR1, we can just FST to store it.
2368     // Else we need to FLD it to FPR1, then FSTP to store/pop it.
2369     int reg_encoding = 0x2; // Just store
2370     int base  = $mem$$base;
2371     int index = $mem$$index;
2372     int scale = $mem$$scale;
2373     int displace = $mem$$disp;
2374     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
2375     if( $src$$reg != FPR1L_enc ) {
2376       reg_encoding = 0x3;  // Store & pop
2377       emit_opcode( cbuf, 0xD9 ); // FLD (i.e., push it)
2378       emit_d8( cbuf, 0xC0-1+$src$$reg );
2379     }
2380     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand
2381     emit_opcode(cbuf,$primary);
2382     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2383   %}
2384 
2385   enc_class neg_reg(eRegI dst) %{
2386     // NEG $dst
2387     emit_opcode(cbuf,0xF7);
2388     emit_rm(cbuf, 0x3, 0x03, $dst$$reg );
2389   %}
2390 
2391   enc_class setLT_reg(eCXRegI dst) %{
2392     // SETLT $dst
2393     emit_opcode(cbuf,0x0F);
2394     emit_opcode(cbuf,0x9C);
2395     emit_rm( cbuf, 0x3, 0x4, $dst$$reg );
2396   %}
2397 
2398   enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{    // cadd_cmpLT
2399     int tmpReg = $tmp$$reg;
2400 
2401     // SUB $p,$q
2402     emit_opcode(cbuf,0x2B);
2403     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
2404     // SBB $tmp,$tmp
2405     emit_opcode(cbuf,0x1B);
2406     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
2407     // AND $tmp,$y
2408     emit_opcode(cbuf,0x23);
2409     emit_rm(cbuf, 0x3, tmpReg, $y$$reg);
2410     // ADD $p,$tmp
2411     emit_opcode(cbuf,0x03);
2412     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
2413   %}
2414 
2415   enc_class enc_cmpLTP_mem(eRegI p, eRegI q, memory mem, eCXRegI tmp) %{    // cadd_cmpLT
2416     int tmpReg = $tmp$$reg;
2417 
2418     // SUB $p,$q
2419     emit_opcode(cbuf,0x2B);
2420     emit_rm(cbuf, 0x3, $p$$reg, $q$$reg);
2421     // SBB $tmp,$tmp
2422     emit_opcode(cbuf,0x1B);
2423     emit_rm(cbuf, 0x3, tmpReg, tmpReg);
2424     // AND $tmp,$y
2425     cbuf.set_inst_mark();       // Mark start of opcode for reloc info in mem operand
2426     emit_opcode(cbuf,0x23);
2427     int reg_encoding = tmpReg;
2428     int base  = $mem$$base;
2429     int index = $mem$$index;
2430     int scale = $mem$$scale;
2431     int displace = $mem$$disp;
2432     bool disp_is_oop = $mem->disp_is_oop();
2433     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace, disp_is_oop);
2434     // ADD $p,$tmp
2435     emit_opcode(cbuf,0x03);
2436     emit_rm(cbuf, 0x3, $p$$reg, tmpReg);
2437   %}
2438 
2439   enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{
2440     // TEST shift,32
2441     emit_opcode(cbuf,0xF7);
2442     emit_rm(cbuf, 0x3, 0, ECX_enc);
2443     emit_d32(cbuf,0x20);
2444     // JEQ,s small
2445     emit_opcode(cbuf, 0x74);
2446     emit_d8(cbuf, 0x04);
2447     // MOV    $dst.hi,$dst.lo
2448     emit_opcode( cbuf, 0x8B );
2449     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
2450     // CLR    $dst.lo
2451     emit_opcode(cbuf, 0x33);
2452     emit_rm(cbuf, 0x3, $dst$$reg, $dst$$reg);
2453 // small:
2454     // SHLD   $dst.hi,$dst.lo,$shift
2455     emit_opcode(cbuf,0x0F);
2456     emit_opcode(cbuf,0xA5);
2457     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg));
2458     // SHL    $dst.lo,$shift"
2459     emit_opcode(cbuf,0xD3);
2460     emit_rm(cbuf, 0x3, 0x4, $dst$$reg );
2461   %}
2462 
2463   enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{
2464     // TEST shift,32
2465     emit_opcode(cbuf,0xF7);
2466     emit_rm(cbuf, 0x3, 0, ECX_enc);
2467     emit_d32(cbuf,0x20);
2468     // JEQ,s small
2469     emit_opcode(cbuf, 0x74);
2470     emit_d8(cbuf, 0x04);
2471     // MOV    $dst.lo,$dst.hi
2472     emit_opcode( cbuf, 0x8B );
2473     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2474     // CLR    $dst.hi
2475     emit_opcode(cbuf, 0x33);
2476     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($dst$$reg));
2477 // small:
2478     // SHRD   $dst.lo,$dst.hi,$shift
2479     emit_opcode(cbuf,0x0F);
2480     emit_opcode(cbuf,0xAD);
2481     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2482     // SHR    $dst.hi,$shift"
2483     emit_opcode(cbuf,0xD3);
2484     emit_rm(cbuf, 0x3, 0x5, HIGH_FROM_LOW($dst$$reg) );
2485   %}
2486 
2487   enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{
2488     // TEST shift,32
2489     emit_opcode(cbuf,0xF7);
2490     emit_rm(cbuf, 0x3, 0, ECX_enc);
2491     emit_d32(cbuf,0x20);
2492     // JEQ,s small
2493     emit_opcode(cbuf, 0x74);
2494     emit_d8(cbuf, 0x05);
2495     // MOV    $dst.lo,$dst.hi
2496     emit_opcode( cbuf, 0x8B );
2497     emit_rm(cbuf, 0x3, $dst$$reg, HIGH_FROM_LOW($dst$$reg) );
2498     // SAR    $dst.hi,31
2499     emit_opcode(cbuf, 0xC1);
2500     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW($dst$$reg) );
2501     emit_d8(cbuf, 0x1F );
2502 // small:
2503     // SHRD   $dst.lo,$dst.hi,$shift
2504     emit_opcode(cbuf,0x0F);
2505     emit_opcode(cbuf,0xAD);
2506     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg);
2507     // SAR    $dst.hi,$shift"
2508     emit_opcode(cbuf,0xD3);
2509     emit_rm(cbuf, 0x3, 0x7, HIGH_FROM_LOW($dst$$reg) );
2510   %}
2511 
2512 
2513   // ----------------- Encodings for floating point unit -----------------
2514   // May leave result in FPU-TOS or FPU reg depending on opcodes
2515   enc_class OpcReg_F (regF src) %{    // FMUL, FDIV
2516     $$$emit8$primary;
2517     emit_rm(cbuf, 0x3, $secondary, $src$$reg );
2518   %}
2519 
2520   // Pop argument in FPR0 with FSTP ST(0)
2521   enc_class PopFPU() %{
2522     emit_opcode( cbuf, 0xDD );
2523     emit_d8( cbuf, 0xD8 );
2524   %}
2525 
2526   // !!!!! equivalent to Pop_Reg_F
2527   enc_class Pop_Reg_D( regD dst ) %{
2528     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2529     emit_d8( cbuf, 0xD8+$dst$$reg );
2530   %}
2531 
2532   enc_class Push_Reg_D( regD dst ) %{
2533     emit_opcode( cbuf, 0xD9 );
2534     emit_d8( cbuf, 0xC0-1+$dst$$reg );   // FLD ST(i-1)
2535   %}
2536 
2537   enc_class strictfp_bias1( regD dst ) %{
2538     emit_opcode( cbuf, 0xDB );           // FLD m80real
2539     emit_opcode( cbuf, 0x2D );
2540     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias1() );
2541     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2542     emit_opcode( cbuf, 0xC8+$dst$$reg );
2543   %}
2544 
2545   enc_class strictfp_bias2( regD dst ) %{
2546     emit_opcode( cbuf, 0xDB );           // FLD m80real
2547     emit_opcode( cbuf, 0x2D );
2548     emit_d32( cbuf, (int)StubRoutines::addr_fpu_subnormal_bias2() );
2549     emit_opcode( cbuf, 0xDE );           // FMULP ST(dst), ST0
2550     emit_opcode( cbuf, 0xC8+$dst$$reg );
2551   %}
2552 
2553   // Special case for moving an integer register to a stack slot.
2554   enc_class OpcPRegSS( stackSlotI dst, eRegI src ) %{ // RegSS
2555     store_to_stackslot( cbuf, $primary, $src$$reg, $dst$$disp );
2556   %}
2557 
2558   // Special case for moving a register to a stack slot.
2559   enc_class RegSS( stackSlotI dst, eRegI src ) %{ // RegSS
2560     // Opcode already emitted
2561     emit_rm( cbuf, 0x02, $src$$reg, ESP_enc );   // R/M byte
2562     emit_rm( cbuf, 0x00, ESP_enc, ESP_enc);          // SIB byte
2563     emit_d32(cbuf, $dst$$disp);   // Displacement
2564   %}
2565 
2566   // Push the integer in stackSlot 'src' onto FP-stack
2567   enc_class Push_Mem_I( memory src ) %{    // FILD   [ESP+src]
2568     store_to_stackslot( cbuf, $primary, $secondary, $src$$disp );
2569   %}
2570 
2571   // Push the float in stackSlot 'src' onto FP-stack
2572   enc_class Push_Mem_F( memory src ) %{    // FLD_S   [ESP+src]
2573     store_to_stackslot( cbuf, 0xD9, 0x00, $src$$disp );
2574   %}
2575 
2576   // Push the double in stackSlot 'src' onto FP-stack
2577   enc_class Push_Mem_D( memory src ) %{    // FLD_D   [ESP+src]
2578     store_to_stackslot( cbuf, 0xDD, 0x00, $src$$disp );
2579   %}
2580 
2581   // Push FPU's TOS float to a stack-slot, and pop FPU-stack
2582   enc_class Pop_Mem_F( stackSlotF dst ) %{ // FSTP_S [ESP+dst]
2583     store_to_stackslot( cbuf, 0xD9, 0x03, $dst$$disp );
2584   %}
2585 
2586   // Same as Pop_Mem_F except for opcode
2587   // Push FPU's TOS double to a stack-slot, and pop FPU-stack
2588   enc_class Pop_Mem_D( stackSlotD dst ) %{ // FSTP_D [ESP+dst]
2589     store_to_stackslot( cbuf, 0xDD, 0x03, $dst$$disp );
2590   %}
2591 
2592   enc_class Pop_Reg_F( regF dst ) %{
2593     emit_opcode( cbuf, 0xDD );           // FSTP   ST(i)
2594     emit_d8( cbuf, 0xD8+$dst$$reg );
2595   %}
2596 
2597   enc_class Push_Reg_F( regF dst ) %{
2598     emit_opcode( cbuf, 0xD9 );           // FLD    ST(i-1)
2599     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2600   %}
2601 
2602   // Push FPU's float to a stack-slot, and pop FPU-stack
2603   enc_class Pop_Mem_Reg_F( stackSlotF dst, regF src ) %{
2604     int pop = 0x02;
2605     if ($src$$reg != FPR1L_enc) {
2606       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2607       emit_d8( cbuf, 0xC0-1+$src$$reg );
2608       pop = 0x03;
2609     }
2610     store_to_stackslot( cbuf, 0xD9, pop, $dst$$disp ); // FST<P>_S  [ESP+dst]
2611   %}
2612 
2613   // Push FPU's double to a stack-slot, and pop FPU-stack
2614   enc_class Pop_Mem_Reg_D( stackSlotD dst, regD src ) %{
2615     int pop = 0x02;
2616     if ($src$$reg != FPR1L_enc) {
2617       emit_opcode( cbuf, 0xD9 );         // FLD    ST(i-1)
2618       emit_d8( cbuf, 0xC0-1+$src$$reg );
2619       pop = 0x03;
2620     }
2621     store_to_stackslot( cbuf, 0xDD, pop, $dst$$disp ); // FST<P>_D  [ESP+dst]
2622   %}
2623 
2624   // Push FPU's double to a FPU-stack-slot, and pop FPU-stack
2625   enc_class Pop_Reg_Reg_D( regD dst, regF src ) %{
2626     int pop = 0xD0 - 1; // -1 since we skip FLD
2627     if ($src$$reg != FPR1L_enc) {
2628       emit_opcode( cbuf, 0xD9 );         // FLD    ST(src-1)
2629       emit_d8( cbuf, 0xC0-1+$src$$reg );
2630       pop = 0xD8;
2631     }
2632     emit_opcode( cbuf, 0xDD );
2633     emit_d8( cbuf, pop+$dst$$reg );      // FST<P> ST(i)
2634   %}
2635 
2636 
2637   enc_class Mul_Add_F( regF dst, regF src, regF src1, regF src2 ) %{
2638     MacroAssembler masm(&cbuf);
2639     masm.fld_s(  $src1$$reg-1);   // nothing at TOS, load TOS from src1.reg
2640     masm.fmul(   $src2$$reg+0);   // value at TOS
2641     masm.fadd(   $src$$reg+0);    // value at TOS
2642     masm.fstp_d( $dst$$reg+0);    // value at TOS, popped off after store
2643   %}
2644 
2645 
2646   enc_class Push_Reg_Mod_D( regD dst, regD src) %{
2647     // load dst in FPR0
2648     emit_opcode( cbuf, 0xD9 );
2649     emit_d8( cbuf, 0xC0-1+$dst$$reg );
2650     if ($src$$reg != FPR1L_enc) {
2651       // fincstp
2652       emit_opcode (cbuf, 0xD9);
2653       emit_opcode (cbuf, 0xF7);
2654       // swap src with FPR1:
2655       // FXCH FPR1 with src
2656       emit_opcode(cbuf, 0xD9);
2657       emit_d8(cbuf, 0xC8-1+$src$$reg );
2658       // fdecstp
2659       emit_opcode (cbuf, 0xD9);
2660       emit_opcode (cbuf, 0xF6);
2661     }
2662   %}
2663 
2664   enc_class Push_ModD_encoding( regXD src0, regXD src1) %{
2665     // Allocate a word
2666     emit_opcode(cbuf,0x83);            // SUB ESP,8
2667     emit_opcode(cbuf,0xEC);
2668     emit_d8(cbuf,0x08);
2669 
2670     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src1
2671     emit_opcode  (cbuf, 0x0F );
2672     emit_opcode  (cbuf, 0x11 );
2673     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
2674 
2675     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2676     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2677 
2678     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src0
2679     emit_opcode  (cbuf, 0x0F );
2680     emit_opcode  (cbuf, 0x11 );
2681     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
2682 
2683     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2684     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2685 
2686   %}
2687 
2688   enc_class Push_ModX_encoding( regX src0, regX src1) %{
2689     // Allocate a word
2690     emit_opcode(cbuf,0x83);            // SUB ESP,4
2691     emit_opcode(cbuf,0xEC);
2692     emit_d8(cbuf,0x04);
2693 
2694     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src1
2695     emit_opcode  (cbuf, 0x0F );
2696     emit_opcode  (cbuf, 0x11 );
2697     encode_RegMem(cbuf, $src1$$reg, ESP_enc, 0x4, 0, 0, false);
2698 
2699     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
2700     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2701 
2702     emit_opcode  (cbuf, 0xF3 );     // MOVSS [ESP], src0
2703     emit_opcode  (cbuf, 0x0F );
2704     emit_opcode  (cbuf, 0x11 );
2705     encode_RegMem(cbuf, $src0$$reg, ESP_enc, 0x4, 0, 0, false);
2706 
2707     emit_opcode(cbuf,0xD9 );      // FLD [ESP]
2708     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2709 
2710   %}
2711 
2712   enc_class Push_ResultXD(regXD dst) %{
2713     store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [ESP]
2714 
2715     // UseXmmLoadAndClearUpper ? movsd dst,[esp] : movlpd dst,[esp]
2716     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
2717     emit_opcode  (cbuf, 0x0F );
2718     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
2719     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
2720 
2721     emit_opcode(cbuf,0x83);    // ADD ESP,8
2722     emit_opcode(cbuf,0xC4);
2723     emit_d8(cbuf,0x08);
2724   %}
2725 
2726   enc_class Push_ResultX(regX dst, immI d8) %{
2727     store_to_stackslot( cbuf, 0xD9, 0x03, 0 ); //FSTP_S [ESP]
2728 
2729     emit_opcode  (cbuf, 0xF3 );     // MOVSS dst(xmm), [ESP]
2730     emit_opcode  (cbuf, 0x0F );
2731     emit_opcode  (cbuf, 0x10 );
2732     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
2733 
2734     emit_opcode(cbuf,0x83);    // ADD ESP,d8 (4 or 8)
2735     emit_opcode(cbuf,0xC4);
2736     emit_d8(cbuf,$d8$$constant);
2737   %}
2738 
2739   enc_class Push_SrcXD(regXD src) %{
2740     // Allocate a word
2741     emit_opcode(cbuf,0x83);            // SUB ESP,8
2742     emit_opcode(cbuf,0xEC);
2743     emit_d8(cbuf,0x08);
2744 
2745     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], src
2746     emit_opcode  (cbuf, 0x0F );
2747     emit_opcode  (cbuf, 0x11 );
2748     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
2749 
2750     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2751     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2752   %}
2753 
2754   enc_class push_stack_temp_qword() %{
2755     emit_opcode(cbuf,0x83);     // SUB ESP,8
2756     emit_opcode(cbuf,0xEC);
2757     emit_d8    (cbuf,0x08);
2758   %}
2759 
2760   enc_class pop_stack_temp_qword() %{
2761     emit_opcode(cbuf,0x83);     // ADD ESP,8
2762     emit_opcode(cbuf,0xC4);
2763     emit_d8    (cbuf,0x08);
2764   %}
2765 
2766   enc_class push_xmm_to_fpr1( regXD xmm_src ) %{
2767     emit_opcode  (cbuf, 0xF2 );     // MOVSD [ESP], xmm_src
2768     emit_opcode  (cbuf, 0x0F );
2769     emit_opcode  (cbuf, 0x11 );
2770     encode_RegMem(cbuf, $xmm_src$$reg, ESP_enc, 0x4, 0, 0, false);
2771 
2772     emit_opcode(cbuf,0xDD );      // FLD_D [ESP]
2773     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2774   %}
2775 
2776   // Compute X^Y using Intel's fast hardware instructions, if possible.
2777   // Otherwise return a NaN.
2778   enc_class pow_exp_core_encoding %{
2779     // FPR1 holds Y*ln2(X).  Compute FPR1 = 2^(Y*ln2(X))
2780     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xC0);  // fdup = fld st(0)          Q       Q
2781     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xFC);  // frndint               int(Q)      Q
2782     emit_opcode(cbuf,0xDC); emit_opcode(cbuf,0xE9);  // fsub st(1) -= st(0);  int(Q) frac(Q)
2783     emit_opcode(cbuf,0xDB);                          // FISTP [ESP]           frac(Q)
2784     emit_opcode(cbuf,0x1C);
2785     emit_d8(cbuf,0x24);
2786     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xF0);  // f2xm1                 2^frac(Q)-1
2787     emit_opcode(cbuf,0xD9); emit_opcode(cbuf,0xE8);  // fld1                  1 2^frac(Q)-1
2788     emit_opcode(cbuf,0xDE); emit_opcode(cbuf,0xC1);  // faddp                 2^frac(Q)
2789     emit_opcode(cbuf,0x8B);                          // mov rax,[esp+0]=int(Q)
2790     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 0, false);
2791     emit_opcode(cbuf,0xC7);                          // mov rcx,0xFFFFF800 - overflow mask
2792     emit_rm(cbuf, 0x3, 0x0, ECX_enc);
2793     emit_d32(cbuf,0xFFFFF800);
2794     emit_opcode(cbuf,0x81);                          // add rax,1023 - the double exponent bias
2795     emit_rm(cbuf, 0x3, 0x0, EAX_enc);
2796     emit_d32(cbuf,1023);
2797     emit_opcode(cbuf,0x8B);                          // mov rbx,eax
2798     emit_rm(cbuf, 0x3, EBX_enc, EAX_enc);
2799     emit_opcode(cbuf,0xC1);                          // shl rax,20 - Slide to exponent position
2800     emit_rm(cbuf,0x3,0x4,EAX_enc);
2801     emit_d8(cbuf,20);
2802     emit_opcode(cbuf,0x85);                          // test rbx,ecx - check for overflow
2803     emit_rm(cbuf, 0x3, EBX_enc, ECX_enc);
2804     emit_opcode(cbuf,0x0F); emit_opcode(cbuf,0x45);  // CMOVne rax,ecx - overflow; stuff NAN into EAX
2805     emit_rm(cbuf, 0x3, EAX_enc, ECX_enc);
2806     emit_opcode(cbuf,0x89);                          // mov [esp+4],eax - Store as part of double word
2807     encode_RegMem(cbuf, EAX_enc, ESP_enc, 0x4, 0, 4, false);
2808     emit_opcode(cbuf,0xC7);                          // mov [esp+0],0   - [ESP] = (double)(1<<int(Q)) = 2^int(Q)
2809     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
2810     emit_d32(cbuf,0);
2811     emit_opcode(cbuf,0xDC);                          // fmul dword st(0),[esp+0]; FPR1 = 2^int(Q)*2^frac(Q) = 2^Q
2812     encode_RegMem(cbuf, 0x1, ESP_enc, 0x4, 0, 0, false);
2813   %}
2814 
2815 //   enc_class Pop_Reg_Mod_D( regD dst, regD src)
2816 //   was replaced by Push_Result_Mod_D followed by Pop_Reg_X() or Pop_Mem_X()
2817 
2818   enc_class Push_Result_Mod_D( regD src) %{
2819     if ($src$$reg != FPR1L_enc) {
2820       // fincstp
2821       emit_opcode (cbuf, 0xD9);
2822       emit_opcode (cbuf, 0xF7);
2823       // FXCH FPR1 with src
2824       emit_opcode(cbuf, 0xD9);
2825       emit_d8(cbuf, 0xC8-1+$src$$reg );
2826       // fdecstp
2827       emit_opcode (cbuf, 0xD9);
2828       emit_opcode (cbuf, 0xF6);
2829     }
2830     // // following asm replaced with Pop_Reg_F or Pop_Mem_F
2831     // // FSTP   FPR$dst$$reg
2832     // emit_opcode( cbuf, 0xDD );
2833     // emit_d8( cbuf, 0xD8+$dst$$reg );
2834   %}
2835 
2836   enc_class fnstsw_sahf_skip_parity() %{
2837     // fnstsw ax
2838     emit_opcode( cbuf, 0xDF );
2839     emit_opcode( cbuf, 0xE0 );
2840     // sahf
2841     emit_opcode( cbuf, 0x9E );
2842     // jnp  ::skip
2843     emit_opcode( cbuf, 0x7B );
2844     emit_opcode( cbuf, 0x05 );
2845   %}
2846 
2847   enc_class emitModD() %{
2848     // fprem must be iterative
2849     // :: loop
2850     // fprem
2851     emit_opcode( cbuf, 0xD9 );
2852     emit_opcode( cbuf, 0xF8 );
2853     // wait
2854     emit_opcode( cbuf, 0x9b );
2855     // fnstsw ax
2856     emit_opcode( cbuf, 0xDF );
2857     emit_opcode( cbuf, 0xE0 );
2858     // sahf
2859     emit_opcode( cbuf, 0x9E );
2860     // jp  ::loop
2861     emit_opcode( cbuf, 0x0F );
2862     emit_opcode( cbuf, 0x8A );
2863     emit_opcode( cbuf, 0xF4 );
2864     emit_opcode( cbuf, 0xFF );
2865     emit_opcode( cbuf, 0xFF );
2866     emit_opcode( cbuf, 0xFF );
2867   %}
2868 
2869   enc_class fpu_flags() %{
2870     // fnstsw_ax
2871     emit_opcode( cbuf, 0xDF);
2872     emit_opcode( cbuf, 0xE0);
2873     // test ax,0x0400
2874     emit_opcode( cbuf, 0x66 );   // operand-size prefix for 16-bit immediate
2875     emit_opcode( cbuf, 0xA9 );
2876     emit_d16   ( cbuf, 0x0400 );
2877     // // // This sequence works, but stalls for 12-16 cycles on PPro
2878     // // test rax,0x0400
2879     // emit_opcode( cbuf, 0xA9 );
2880     // emit_d32   ( cbuf, 0x00000400 );
2881     //
2882     // jz exit (no unordered comparison)
2883     emit_opcode( cbuf, 0x74 );
2884     emit_d8    ( cbuf, 0x02 );
2885     // mov ah,1 - treat as LT case (set carry flag)
2886     emit_opcode( cbuf, 0xB4 );
2887     emit_d8    ( cbuf, 0x01 );
2888     // sahf
2889     emit_opcode( cbuf, 0x9E);
2890   %}
2891 
2892   enc_class cmpF_P6_fixup() %{
2893     // Fixup the integer flags in case comparison involved a NaN
2894     //
2895     // JNP exit (no unordered comparison, P-flag is set by NaN)
2896     emit_opcode( cbuf, 0x7B );
2897     emit_d8    ( cbuf, 0x03 );
2898     // MOV AH,1 - treat as LT case (set carry flag)
2899     emit_opcode( cbuf, 0xB4 );
2900     emit_d8    ( cbuf, 0x01 );
2901     // SAHF
2902     emit_opcode( cbuf, 0x9E);
2903     // NOP     // target for branch to avoid branch to branch
2904     emit_opcode( cbuf, 0x90);
2905   %}
2906 
2907 //     fnstsw_ax();
2908 //     sahf();
2909 //     movl(dst, nan_result);
2910 //     jcc(Assembler::parity, exit);
2911 //     movl(dst, less_result);
2912 //     jcc(Assembler::below, exit);
2913 //     movl(dst, equal_result);
2914 //     jcc(Assembler::equal, exit);
2915 //     movl(dst, greater_result);
2916 
2917 // less_result     =  1;
2918 // greater_result  = -1;
2919 // equal_result    = 0;
2920 // nan_result      = -1;
2921 
2922   enc_class CmpF_Result(eRegI dst) %{
2923     // fnstsw_ax();
2924     emit_opcode( cbuf, 0xDF);
2925     emit_opcode( cbuf, 0xE0);
2926     // sahf
2927     emit_opcode( cbuf, 0x9E);
2928     // movl(dst, nan_result);
2929     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2930     emit_d32( cbuf, -1 );
2931     // jcc(Assembler::parity, exit);
2932     emit_opcode( cbuf, 0x7A );
2933     emit_d8    ( cbuf, 0x13 );
2934     // movl(dst, less_result);
2935     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2936     emit_d32( cbuf, -1 );
2937     // jcc(Assembler::below, exit);
2938     emit_opcode( cbuf, 0x72 );
2939     emit_d8    ( cbuf, 0x0C );
2940     // movl(dst, equal_result);
2941     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2942     emit_d32( cbuf, 0 );
2943     // jcc(Assembler::equal, exit);
2944     emit_opcode( cbuf, 0x74 );
2945     emit_d8    ( cbuf, 0x05 );
2946     // movl(dst, greater_result);
2947     emit_opcode( cbuf, 0xB8 + $dst$$reg);
2948     emit_d32( cbuf, 1 );
2949   %}
2950 
2951 
2952   // XMM version of CmpF_Result. Because the XMM compare
2953   // instructions set the EFLAGS directly. It becomes simpler than
2954   // the float version above.
2955   enc_class CmpX_Result(eRegI dst) %{
2956     MacroAssembler _masm(&cbuf);
2957     Label nan, inc, done;
2958 
2959     __ jccb(Assembler::parity, nan);
2960     __ jccb(Assembler::equal,  done);
2961     __ jccb(Assembler::above,  inc);
2962     __ bind(nan);
2963     __ decrement(as_Register($dst$$reg)); // NO L qqq
2964     __ jmpb(done);
2965     __ bind(inc);
2966     __ increment(as_Register($dst$$reg)); // NO L qqq
2967     __ bind(done);
2968   %}
2969 
2970   // Compare the longs and set flags
2971   // BROKEN!  Do Not use as-is
2972   enc_class cmpl_test( eRegL src1, eRegL src2 ) %{
2973     // CMP    $src1.hi,$src2.hi
2974     emit_opcode( cbuf, 0x3B );
2975     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
2976     // JNE,s  done
2977     emit_opcode(cbuf,0x75);
2978     emit_d8(cbuf, 2 );
2979     // CMP    $src1.lo,$src2.lo
2980     emit_opcode( cbuf, 0x3B );
2981     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
2982 // done:
2983   %}
2984 
2985   enc_class convert_int_long( regL dst, eRegI src ) %{
2986     // mov $dst.lo,$src
2987     int dst_encoding = $dst$$reg;
2988     int src_encoding = $src$$reg;
2989     encode_Copy( cbuf, dst_encoding  , src_encoding );
2990     // mov $dst.hi,$src
2991     encode_Copy( cbuf, HIGH_FROM_LOW(dst_encoding), src_encoding );
2992     // sar $dst.hi,31
2993     emit_opcode( cbuf, 0xC1 );
2994     emit_rm(cbuf, 0x3, 7, HIGH_FROM_LOW(dst_encoding) );
2995     emit_d8(cbuf, 0x1F );
2996   %}
2997 
2998   enc_class convert_long_double( eRegL src ) %{
2999     // push $src.hi
3000     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
3001     // push $src.lo
3002     emit_opcode(cbuf, 0x50+$src$$reg  );
3003     // fild 64-bits at [SP]
3004     emit_opcode(cbuf,0xdf);
3005     emit_d8(cbuf, 0x6C);
3006     emit_d8(cbuf, 0x24);
3007     emit_d8(cbuf, 0x00);
3008     // pop stack
3009     emit_opcode(cbuf, 0x83); // add  SP, #8
3010     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
3011     emit_d8(cbuf, 0x8);
3012   %}
3013 
3014   enc_class multiply_con_and_shift_high( eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr ) %{
3015     // IMUL   EDX:EAX,$src1
3016     emit_opcode( cbuf, 0xF7 );
3017     emit_rm( cbuf, 0x3, 0x5, $src1$$reg );
3018     // SAR    EDX,$cnt-32
3019     int shift_count = ((int)$cnt$$constant) - 32;
3020     if (shift_count > 0) {
3021       emit_opcode(cbuf, 0xC1);
3022       emit_rm(cbuf, 0x3, 7, $dst$$reg );
3023       emit_d8(cbuf, shift_count);
3024     }
3025   %}
3026 
3027   // this version doesn't have add sp, 8
3028   enc_class convert_long_double2( eRegL src ) %{
3029     // push $src.hi
3030     emit_opcode(cbuf, 0x50+HIGH_FROM_LOW($src$$reg));
3031     // push $src.lo
3032     emit_opcode(cbuf, 0x50+$src$$reg  );
3033     // fild 64-bits at [SP]
3034     emit_opcode(cbuf,0xdf);
3035     emit_d8(cbuf, 0x6C);
3036     emit_d8(cbuf, 0x24);
3037     emit_d8(cbuf, 0x00);
3038   %}
3039 
3040   enc_class long_int_multiply( eADXRegL dst, nadxRegI src) %{
3041     // Basic idea: long = (long)int * (long)int
3042     // IMUL EDX:EAX, src
3043     emit_opcode( cbuf, 0xF7 );
3044     emit_rm( cbuf, 0x3, 0x5, $src$$reg);
3045   %}
3046 
3047   enc_class long_uint_multiply( eADXRegL dst, nadxRegI src) %{
3048     // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
3049     // MUL EDX:EAX, src
3050     emit_opcode( cbuf, 0xF7 );
3051     emit_rm( cbuf, 0x3, 0x4, $src$$reg);
3052   %}
3053 
3054   enc_class long_multiply( eADXRegL dst, eRegL src, eRegI tmp ) %{
3055     // Basic idea: lo(result) = lo(x_lo * y_lo)
3056     //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
3057     // MOV    $tmp,$src.lo
3058     encode_Copy( cbuf, $tmp$$reg, $src$$reg );
3059     // IMUL   $tmp,EDX
3060     emit_opcode( cbuf, 0x0F );
3061     emit_opcode( cbuf, 0xAF );
3062     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3063     // MOV    EDX,$src.hi
3064     encode_Copy( cbuf, HIGH_FROM_LOW($dst$$reg), HIGH_FROM_LOW($src$$reg) );
3065     // IMUL   EDX,EAX
3066     emit_opcode( cbuf, 0x0F );
3067     emit_opcode( cbuf, 0xAF );
3068     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $dst$$reg );
3069     // ADD    $tmp,EDX
3070     emit_opcode( cbuf, 0x03 );
3071     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3072     // MUL   EDX:EAX,$src.lo
3073     emit_opcode( cbuf, 0xF7 );
3074     emit_rm( cbuf, 0x3, 0x4, $src$$reg );
3075     // ADD    EDX,ESI
3076     emit_opcode( cbuf, 0x03 );
3077     emit_rm( cbuf, 0x3, HIGH_FROM_LOW($dst$$reg), $tmp$$reg );
3078   %}
3079 
3080   enc_class long_multiply_con( eADXRegL dst, immL_127 src, eRegI tmp ) %{
3081     // Basic idea: lo(result) = lo(src * y_lo)
3082     //             hi(result) = hi(src * y_lo) + lo(src * y_hi)
3083     // IMUL   $tmp,EDX,$src
3084     emit_opcode( cbuf, 0x6B );
3085     emit_rm( cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg) );
3086     emit_d8( cbuf, (int)$src$$constant );
3087     // MOV    EDX,$src
3088     emit_opcode(cbuf, 0xB8 + EDX_enc);
3089     emit_d32( cbuf, (int)$src$$constant );
3090     // MUL   EDX:EAX,EDX
3091     emit_opcode( cbuf, 0xF7 );
3092     emit_rm( cbuf, 0x3, 0x4, EDX_enc );
3093     // ADD    EDX,ESI
3094     emit_opcode( cbuf, 0x03 );
3095     emit_rm( cbuf, 0x3, EDX_enc, $tmp$$reg );
3096   %}
3097 
3098   enc_class long_div( eRegL src1, eRegL src2 ) %{
3099     // PUSH src1.hi
3100     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
3101     // PUSH src1.lo
3102     emit_opcode(cbuf,               0x50+$src1$$reg  );
3103     // PUSH src2.hi
3104     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
3105     // PUSH src2.lo
3106     emit_opcode(cbuf,               0x50+$src2$$reg  );
3107     // CALL directly to the runtime
3108     cbuf.set_inst_mark();
3109     emit_opcode(cbuf,0xE8);       // Call into runtime
3110     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3111     // Restore stack
3112     emit_opcode(cbuf, 0x83); // add  SP, #framesize
3113     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
3114     emit_d8(cbuf, 4*4);
3115   %}
3116 
3117   enc_class long_mod( eRegL src1, eRegL src2 ) %{
3118     // PUSH src1.hi
3119     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src1$$reg) );
3120     // PUSH src1.lo
3121     emit_opcode(cbuf,               0x50+$src1$$reg  );
3122     // PUSH src2.hi
3123     emit_opcode(cbuf, HIGH_FROM_LOW(0x50+$src2$$reg) );
3124     // PUSH src2.lo
3125     emit_opcode(cbuf,               0x50+$src2$$reg  );
3126     // CALL directly to the runtime
3127     cbuf.set_inst_mark();
3128     emit_opcode(cbuf,0xE8);       // Call into runtime
3129     emit_d32_reloc(cbuf, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3130     // Restore stack
3131     emit_opcode(cbuf, 0x83); // add  SP, #framesize
3132     emit_rm(cbuf, 0x3, 0x00, ESP_enc);
3133     emit_d8(cbuf, 4*4);
3134   %}
3135 
3136   enc_class long_cmp_flags0( eRegL src, eRegI tmp ) %{
3137     // MOV   $tmp,$src.lo
3138     emit_opcode(cbuf, 0x8B);
3139     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
3140     // OR    $tmp,$src.hi
3141     emit_opcode(cbuf, 0x0B);
3142     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg));
3143   %}
3144 
3145   enc_class long_cmp_flags1( eRegL src1, eRegL src2 ) %{
3146     // CMP    $src1.lo,$src2.lo
3147     emit_opcode( cbuf, 0x3B );
3148     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
3149     // JNE,s  skip
3150     emit_cc(cbuf, 0x70, 0x5);
3151     emit_d8(cbuf,2);
3152     // CMP    $src1.hi,$src2.hi
3153     emit_opcode( cbuf, 0x3B );
3154     emit_rm(cbuf, 0x3, HIGH_FROM_LOW($src1$$reg), HIGH_FROM_LOW($src2$$reg) );
3155   %}
3156 
3157   enc_class long_cmp_flags2( eRegL src1, eRegL src2, eRegI tmp ) %{
3158     // CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits
3159     emit_opcode( cbuf, 0x3B );
3160     emit_rm(cbuf, 0x3, $src1$$reg, $src2$$reg );
3161     // MOV    $tmp,$src1.hi
3162     emit_opcode( cbuf, 0x8B );
3163     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src1$$reg) );
3164     // SBB   $tmp,$src2.hi\t! Compute flags for long compare
3165     emit_opcode( cbuf, 0x1B );
3166     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src2$$reg) );
3167   %}
3168 
3169   enc_class long_cmp_flags3( eRegL src, eRegI tmp ) %{
3170     // XOR    $tmp,$tmp
3171     emit_opcode(cbuf,0x33);  // XOR
3172     emit_rm(cbuf,0x3, $tmp$$reg, $tmp$$reg);
3173     // CMP    $tmp,$src.lo
3174     emit_opcode( cbuf, 0x3B );
3175     emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg );
3176     // SBB    $tmp,$src.hi
3177     emit_opcode( cbuf, 0x1B );
3178     emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($src$$reg) );
3179   %}
3180 
3181  // Sniff, sniff... smells like Gnu Superoptimizer
3182   enc_class neg_long( eRegL dst ) %{
3183     emit_opcode(cbuf,0xF7);    // NEG hi
3184     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
3185     emit_opcode(cbuf,0xF7);    // NEG lo
3186     emit_rm    (cbuf,0x3, 0x3,               $dst$$reg );
3187     emit_opcode(cbuf,0x83);    // SBB hi,0
3188     emit_rm    (cbuf,0x3, 0x3, HIGH_FROM_LOW($dst$$reg));
3189     emit_d8    (cbuf,0 );
3190   %}
3191 
3192   enc_class movq_ld(regXD dst, memory mem) %{
3193     MacroAssembler _masm(&cbuf);
3194     __ movq($dst$$XMMRegister, $mem$$Address);
3195   %}
3196 
3197   enc_class movq_st(memory mem, regXD src) %{
3198     MacroAssembler _masm(&cbuf);
3199     __ movq($mem$$Address, $src$$XMMRegister);
3200   %}
3201 
3202   enc_class pshufd_8x8(regX dst, regX src) %{
3203     MacroAssembler _masm(&cbuf);
3204 
3205     encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
3206     __ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
3207     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
3208   %}
3209 
3210   enc_class pshufd_4x16(regX dst, regX src) %{
3211     MacroAssembler _masm(&cbuf);
3212 
3213     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
3214   %}
3215 
3216   enc_class pshufd(regXD dst, regXD src, int mode) %{
3217     MacroAssembler _masm(&cbuf);
3218 
3219     __ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
3220   %}
3221 
3222   enc_class pxor(regXD dst, regXD src) %{
3223     MacroAssembler _masm(&cbuf);
3224 
3225     __ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
3226   %}
3227 
3228   enc_class mov_i2x(regXD dst, eRegI src) %{
3229     MacroAssembler _masm(&cbuf);
3230 
3231     __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
3232   %}
3233 
3234 
3235   // Because the transitions from emitted code to the runtime
3236   // monitorenter/exit helper stubs are so slow it's critical that
3237   // we inline both the stack-locking fast-path and the inflated fast path.
3238   //
3239   // See also: cmpFastLock and cmpFastUnlock.
3240   //
3241   // What follows is a specialized inline transliteration of the code
3242   // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
3243   // another option would be to emit TrySlowEnter and TrySlowExit methods
3244   // at startup-time.  These methods would accept arguments as
3245   // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
3246   // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
3247   // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
3248   // In practice, however, the # of lock sites is bounded and is usually small.
3249   // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
3250   // if the processor uses simple bimodal branch predictors keyed by EIP
3251   // Since the helper routines would be called from multiple synchronization
3252   // sites.
3253   //
3254   // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
3255   // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
3256   // to those specialized methods.  That'd give us a mostly platform-independent
3257   // implementation that the JITs could optimize and inline at their pleasure.
3258   // Done correctly, the only time we'd need to cross to native could would be
3259   // to park() or unpark() threads.  We'd also need a few more unsafe operators
3260   // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
3261   // (b) explicit barriers or fence operations.
3262   //
3263   // TODO:
3264   //
3265   // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
3266   //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
3267   //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
3268   //    the lock operators would typically be faster than reifying Self.
3269   //
3270   // *  Ideally I'd define the primitives as:
3271   //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
3272   //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
3273   //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
3274   //    Instead, we're stuck with a rather awkward and brittle register assignments below.
3275   //    Furthermore the register assignments are overconstrained, possibly resulting in
3276   //    sub-optimal code near the synchronization site.
3277   //
3278   // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
3279   //    Alternately, use a better sp-proximity test.
3280   //
3281   // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
3282   //    Either one is sufficient to uniquely identify a thread.
3283   //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
3284   //
3285   // *  Intrinsify notify() and notifyAll() for the common cases where the
3286   //    object is locked by the calling thread but the waitlist is empty.
3287   //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
3288   //
3289   // *  use jccb and jmpb instead of jcc and jmp to improve code density.
3290   //    But beware of excessive branch density on AMD Opterons.
3291   //
3292   // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
3293   //    or failure of the fast-path.  If the fast-path fails then we pass
3294   //    control to the slow-path, typically in C.  In Fast_Lock and
3295   //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
3296   //    will emit a conditional branch immediately after the node.
3297   //    So we have branches to branches and lots of ICC.ZF games.
3298   //    Instead, it might be better to have C2 pass a "FailureLabel"
3299   //    into Fast_Lock and Fast_Unlock.  In the case of success, control
3300   //    will drop through the node.  ICC.ZF is undefined at exit.
3301   //    In the case of failure, the node will branch directly to the
3302   //    FailureLabel
3303 
3304 
3305   // obj: object to lock
3306   // box: on-stack box address (displaced header location) - KILLED
3307   // rax,: tmp -- KILLED
3308   // scr: tmp -- KILLED
3309   enc_class Fast_Lock( eRegP obj, eRegP box, eAXRegI tmp, eRegP scr ) %{
3310 
3311     Register objReg = as_Register($obj$$reg);
3312     Register boxReg = as_Register($box$$reg);
3313     Register tmpReg = as_Register($tmp$$reg);
3314     Register scrReg = as_Register($scr$$reg);
3315 
3316     // Ensure the register assignents are disjoint
3317     guarantee (objReg != boxReg, "") ;
3318     guarantee (objReg != tmpReg, "") ;
3319     guarantee (objReg != scrReg, "") ;
3320     guarantee (boxReg != tmpReg, "") ;
3321     guarantee (boxReg != scrReg, "") ;
3322     guarantee (tmpReg == as_Register(EAX_enc), "") ;
3323 
3324     MacroAssembler masm(&cbuf);
3325 
3326     if (_counters != NULL) {
3327       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
3328     }
3329     if (EmitSync & 1) {
3330         // set box->dhw = unused_mark (3)
3331         // Force all sync thru slow-path: slow_enter() and slow_exit() 
3332         masm.movptr (Address(boxReg, 0), int32_t(markOopDesc::unused_mark())) ;             
3333         masm.cmpptr (rsp, (int32_t)0) ;                        
3334     } else 
3335     if (EmitSync & 2) { 
3336         Label DONE_LABEL ;           
3337         if (UseBiasedLocking) {
3338            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
3339            masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3340         }
3341 
3342         masm.movptr(tmpReg, Address(objReg, 0)) ;          // fetch markword 
3343         masm.orptr (tmpReg, 0x1);
3344         masm.movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS 
3345         if (os::is_MP()) { masm.lock();  }
3346         masm.cmpxchgptr(boxReg, Address(objReg, 0));          // Updates tmpReg
3347         masm.jcc(Assembler::equal, DONE_LABEL);
3348         // Recursive locking
3349         masm.subptr(tmpReg, rsp);
3350         masm.andptr(tmpReg, (int32_t) 0xFFFFF003 );
3351         masm.movptr(Address(boxReg, 0), tmpReg);
3352         masm.bind(DONE_LABEL) ; 
3353     } else {  
3354       // Possible cases that we'll encounter in fast_lock 
3355       // ------------------------------------------------
3356       // * Inflated
3357       //    -- unlocked
3358       //    -- Locked
3359       //       = by self
3360       //       = by other
3361       // * biased
3362       //    -- by Self
3363       //    -- by other
3364       // * neutral
3365       // * stack-locked
3366       //    -- by self
3367       //       = sp-proximity test hits
3368       //       = sp-proximity test generates false-negative
3369       //    -- by other
3370       //
3371 
3372       Label IsInflated, DONE_LABEL, PopDone ;
3373 
3374       // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
3375       // order to reduce the number of conditional branches in the most common cases.
3376       // Beware -- there's a subtle invariant that fetch of the markword
3377       // at [FETCH], below, will never observe a biased encoding (*101b).
3378       // If this invariant is not held we risk exclusion (safety) failure.
3379       if (UseBiasedLocking && !UseOptoBiasInlining) {
3380         masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3381       }
3382 
3383       masm.movptr(tmpReg, Address(objReg, 0)) ;         // [FETCH]
3384       masm.testptr(tmpReg, 0x02) ;                      // Inflated v (Stack-locked or neutral)
3385       masm.jccb  (Assembler::notZero, IsInflated) ;
3386 
3387       // Attempt stack-locking ...
3388       masm.orptr (tmpReg, 0x1);
3389       masm.movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
3390       if (os::is_MP()) { masm.lock();  }
3391       masm.cmpxchgptr(boxReg, Address(objReg, 0));           // Updates tmpReg
3392       if (_counters != NULL) {
3393         masm.cond_inc32(Assembler::equal,
3394                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
3395       }
3396       masm.jccb (Assembler::equal, DONE_LABEL);
3397 
3398       // Recursive locking
3399       masm.subptr(tmpReg, rsp);
3400       masm.andptr(tmpReg, 0xFFFFF003 );
3401       masm.movptr(Address(boxReg, 0), tmpReg);
3402       if (_counters != NULL) {
3403         masm.cond_inc32(Assembler::equal,
3404                         ExternalAddress((address)_counters->fast_path_entry_count_addr()));
3405       }
3406       masm.jmp  (DONE_LABEL) ;
3407 
3408       masm.bind (IsInflated) ;
3409 
3410       // The object is inflated.
3411       //
3412       // TODO-FIXME: eliminate the ugly use of manifest constants:
3413       //   Use markOopDesc::monitor_value instead of "2".
3414       //   use markOop::unused_mark() instead of "3".
3415       // The tmpReg value is an objectMonitor reference ORed with
3416       // markOopDesc::monitor_value (2).   We can either convert tmpReg to an
3417       // objectmonitor pointer by masking off the "2" bit or we can just
3418       // use tmpReg as an objectmonitor pointer but bias the objectmonitor
3419       // field offsets with "-2" to compensate for and annul the low-order tag bit.
3420       //
3421       // I use the latter as it avoids AGI stalls.
3422       // As such, we write "mov r, [tmpReg+OFFSETOF(Owner)-2]"
3423       // instead of "mov r, [tmpReg+OFFSETOF(Owner)]".
3424       //
3425       #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)
3426 
3427       // boxReg refers to the on-stack BasicLock in the current frame.
3428       // We'd like to write:
3429       //   set box->_displaced_header = markOop::unused_mark().  Any non-0 value suffices.
3430       // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
3431       // additional latency as we have another ST in the store buffer that must drain.
3432 
3433       if (EmitSync & 8192) { 
3434          masm.movptr(Address(boxReg, 0), 3) ;            // results in ST-before-CAS penalty
3435          masm.get_thread (scrReg) ; 
3436          masm.movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2] 
3437          masm.movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
3438          if (os::is_MP()) { masm.lock(); } 
3439          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
3440       } else 
3441       if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
3442          masm.movptr(scrReg, boxReg) ; 
3443          masm.movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2] 
3444 
3445          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
3446          if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) {
3447             // prefetchw [eax + Offset(_owner)-2]
3448             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
3449          }
3450 
3451          if ((EmitSync & 64) == 0) {
3452            // Optimistic form: consider XORL tmpReg,tmpReg
3453            masm.movptr(tmpReg, NULL_WORD) ; 
3454          } else { 
3455            // Can suffer RTS->RTO upgrades on shared or cold $ lines
3456            // Test-And-CAS instead of CAS
3457            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
3458            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
3459            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
3460          }
3461 
3462          // Appears unlocked - try to swing _owner from null to non-null.
3463          // Ideally, I'd manifest "Self" with get_thread and then attempt
3464          // to CAS the register containing Self into m->Owner.
3465          // But we don't have enough registers, so instead we can either try to CAS
3466          // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
3467          // we later store "Self" into m->Owner.  Transiently storing a stack address
3468          // (rsp or the address of the box) into  m->owner is harmless.
3469          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
3470          if (os::is_MP()) { masm.lock();  }
3471          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 
3472          masm.movptr(Address(scrReg, 0), 3) ;          // box->_displaced_header = 3
3473          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3474          masm.get_thread (scrReg) ;                    // beware: clobbers ICCs
3475          masm.movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; 
3476          masm.xorptr(boxReg, boxReg) ;                 // set icc.ZFlag = 1 to indicate success
3477                        
3478          // If the CAS fails we can either retry or pass control to the slow-path.  
3479          // We use the latter tactic.  
3480          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
3481          // If the CAS was successful ...
3482          //   Self has acquired the lock
3483          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
3484          // Intentional fall-through into DONE_LABEL ...
3485       } else {
3486          masm.movptr(Address(boxReg, 0), 3) ;       // results in ST-before-CAS penalty
3487          masm.movptr(boxReg, tmpReg) ; 
3488 
3489          // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
3490          if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) {
3491             // prefetchw [eax + Offset(_owner)-2]
3492             masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2));
3493          }
3494 
3495          if ((EmitSync & 64) == 0) {
3496            // Optimistic form
3497            masm.xorptr  (tmpReg, tmpReg) ; 
3498          } else { 
3499            // Can suffer RTS->RTO upgrades on shared or cold $ lines
3500            masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;   // rax, = m->_owner
3501            masm.testptr(tmpReg, tmpReg) ;                   // Locked ? 
3502            masm.jccb  (Assembler::notZero, DONE_LABEL) ;                   
3503          }
3504 
3505          // Appears unlocked - try to swing _owner from null to non-null.
3506          // Use either "Self" (in scr) or rsp as thread identity in _owner.
3507          // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
3508          masm.get_thread (scrReg) ;
3509          if (os::is_MP()) { masm.lock(); }
3510          masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3511 
3512          // If the CAS fails we can either retry or pass control to the slow-path.
3513          // We use the latter tactic.
3514          // Pass the CAS result in the icc.ZFlag into DONE_LABEL
3515          // If the CAS was successful ...
3516          //   Self has acquired the lock
3517          //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
3518          // Intentional fall-through into DONE_LABEL ...
3519       }
3520 
3521       // DONE_LABEL is a hot target - we'd really like to place it at the
3522       // start of cache line by padding with NOPs.
3523       // See the AMD and Intel software optimization manuals for the
3524       // most efficient "long" NOP encodings.
3525       // Unfortunately none of our alignment mechanisms suffice.
3526       masm.bind(DONE_LABEL);
3527 
3528       // Avoid branch-to-branch on AMD processors
3529       // This appears to be superstition.
3530       if (EmitSync & 32) masm.nop() ;
3531 
3532 
3533       // At DONE_LABEL the icc ZFlag is set as follows ...
3534       // Fast_Unlock uses the same protocol.
3535       // ZFlag == 1 -> Success
3536       // ZFlag == 0 -> Failure - force control through the slow-path
3537     }
3538   %}
3539 
3540   // obj: object to unlock
3541   // box: box address (displaced header location), killed.  Must be EAX.
3542   // rbx,: killed tmp; cannot be obj nor box.
3543   //
3544   // Some commentary on balanced locking:
3545   //
3546   // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
3547   // Methods that don't have provably balanced locking are forced to run in the
3548   // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
3549   // The interpreter provides two properties:
3550   // I1:  At return-time the interpreter automatically and quietly unlocks any
3551   //      objects acquired the current activation (frame).  Recall that the
3552   //      interpreter maintains an on-stack list of locks currently held by
3553   //      a frame.
3554   // I2:  If a method attempts to unlock an object that is not held by the
3555   //      the frame the interpreter throws IMSX.
3556   //
3557   // Lets say A(), which has provably balanced locking, acquires O and then calls B().
3558   // B() doesn't have provably balanced locking so it runs in the interpreter.
3559   // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
3560   // is still locked by A().
3561   //
3562   // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
3563   // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
3564   // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
3565   // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
3566 
3567   enc_class Fast_Unlock( nabxRegP obj, eAXRegP box, eRegP tmp) %{
3568 
3569     Register objReg = as_Register($obj$$reg);
3570     Register boxReg = as_Register($box$$reg);
3571     Register tmpReg = as_Register($tmp$$reg);
3572 
3573     guarantee (objReg != boxReg, "") ;
3574     guarantee (objReg != tmpReg, "") ;
3575     guarantee (boxReg != tmpReg, "") ;
3576     guarantee (boxReg == as_Register(EAX_enc), "") ;
3577     MacroAssembler masm(&cbuf);
3578 
3579     if (EmitSync & 4) {
3580       // Disable - inhibit all inlining.  Force control through the slow-path
3581       masm.cmpptr (rsp, 0) ; 
3582     } else 
3583     if (EmitSync & 8) {
3584       Label DONE_LABEL ;
3585       if (UseBiasedLocking) {
3586          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3587       }
3588       // classic stack-locking code ...
3589       masm.movptr(tmpReg, Address(boxReg, 0)) ;
3590       masm.testptr(tmpReg, tmpReg) ;
3591       masm.jcc   (Assembler::zero, DONE_LABEL) ;
3592       if (os::is_MP()) { masm.lock(); }
3593       masm.cmpxchgptr(tmpReg, Address(objReg, 0));          // Uses EAX which is box
3594       masm.bind(DONE_LABEL);
3595     } else {
3596       Label DONE_LABEL, Stacked, CheckSucc, Inflated ;
3597 
3598       // Critically, the biased locking test must have precedence over
3599       // and appear before the (box->dhw == 0) recursive stack-lock test.
3600       if (UseBiasedLocking && !UseOptoBiasInlining) {
3601          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3602       }
3603       
3604       masm.cmpptr(Address(boxReg, 0), 0) ;            // Examine the displaced header
3605       masm.movptr(tmpReg, Address(objReg, 0)) ;       // Examine the object's markword
3606       masm.jccb  (Assembler::zero, DONE_LABEL) ;      // 0 indicates recursive stack-lock
3607 
3608       masm.testptr(tmpReg, 0x02) ;                     // Inflated? 
3609       masm.jccb  (Assembler::zero, Stacked) ;
3610 
3611       masm.bind  (Inflated) ;
3612       // It's inflated.
3613       // Despite our balanced locking property we still check that m->_owner == Self
3614       // as java routines or native JNI code called by this thread might
3615       // have released the lock.
3616       // Refer to the comments in synchronizer.cpp for how we might encode extra
3617       // state in _succ so we can avoid fetching EntryList|cxq.
3618       //
3619       // I'd like to add more cases in fast_lock() and fast_unlock() --
3620       // such as recursive enter and exit -- but we have to be wary of
3621       // I$ bloat, T$ effects and BP$ effects.
3622       //
3623       // If there's no contention try a 1-0 exit.  That is, exit without
3624       // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
3625       // we detect and recover from the race that the 1-0 exit admits.
3626       //
3627       // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
3628       // before it STs null into _owner, releasing the lock.  Updates
3629       // to data protected by the critical section must be visible before
3630       // we drop the lock (and thus before any other thread could acquire
3631       // the lock and observe the fields protected by the lock).
3632       // IA32's memory-model is SPO, so STs are ordered with respect to
3633       // each other and there's no need for an explicit barrier (fence).
3634       // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
3635 
3636       masm.get_thread (boxReg) ;
3637       if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) {
3638         // prefetchw [ebx + Offset(_owner)-2]
3639         masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2));
3640       }
3641 
3642       // Note that we could employ various encoding schemes to reduce
3643       // the number of loads below (currently 4) to just 2 or 3.
3644       // Refer to the comments in synchronizer.cpp.
3645       // In practice the chain of fetches doesn't seem to impact performance, however.
3646       if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
3647          // Attempt to reduce branch density - AMD's branch predictor.
3648          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
3649          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3650          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
3651          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
3652          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3653          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3654          masm.jmpb  (DONE_LABEL) ; 
3655       } else { 
3656          masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;  
3657          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3658          masm.jccb  (Assembler::notZero, DONE_LABEL) ; 
3659          masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 
3660          masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 
3661          masm.jccb  (Assembler::notZero, CheckSucc) ; 
3662          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3663          masm.jmpb  (DONE_LABEL) ; 
3664       }
3665 
3666       // The Following code fragment (EmitSync & 65536) improves the performance of
3667       // contended applications and contended synchronization microbenchmarks.
3668       // Unfortunately the emission of the code - even though not executed - causes regressions
3669       // in scimark and jetstream, evidently because of $ effects.  Replacing the code
3670       // with an equal number of never-executed NOPs results in the same regression.
3671       // We leave it off by default.
3672 
3673       if ((EmitSync & 65536) != 0) {
3674          Label LSuccess, LGoSlowPath ;
3675 
3676          masm.bind  (CheckSucc) ;
3677 
3678          // Optional pre-test ... it's safe to elide this
3679          if ((EmitSync & 16) == 0) { 
3680             masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
3681             masm.jccb  (Assembler::zero, LGoSlowPath) ; 
3682          }
3683 
3684          // We have a classic Dekker-style idiom:
3685          //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
3686          // There are a number of ways to implement the barrier:
3687          // (1) lock:andl &m->_owner, 0
3688          //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
3689          //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
3690          //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
3691          // (2) If supported, an explicit MFENCE is appealing.
3692          //     In older IA32 processors MFENCE is slower than lock:add or xchg
3693          //     particularly if the write-buffer is full as might be the case if
3694          //     if stores closely precede the fence or fence-equivalent instruction.
3695          //     In more modern implementations MFENCE appears faster, however.
3696          // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
3697          //     The $lines underlying the top-of-stack should be in M-state.
3698          //     The locked add instruction is serializing, of course.
3699          // (4) Use xchg, which is serializing
3700          //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
3701          // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
3702          //     The integer condition codes will tell us if succ was 0.
3703          //     Since _succ and _owner should reside in the same $line and
3704          //     we just stored into _owner, it's likely that the $line
3705          //     remains in M-state for the lock:orl.
3706          //
3707          // We currently use (3), although it's likely that switching to (2)
3708          // is correct for the future.
3709             
3710          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD) ; 
3711          if (os::is_MP()) { 
3712             if (VM_Version::supports_sse2() && 1 == FenceInstruction) { 
3713               masm.mfence();
3714             } else { 
3715               masm.lock () ; masm.addptr(Address(rsp, 0), 0) ; 
3716             }
3717          }
3718          // Ratify _succ remains non-null
3719          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; 
3720          masm.jccb  (Assembler::notZero, LSuccess) ; 
3721 
3722          masm.xorptr(boxReg, boxReg) ;                  // box is really EAX
3723          if (os::is_MP()) { masm.lock(); }
3724          masm.cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
3725          masm.jccb  (Assembler::notEqual, LSuccess) ;
3726          // Since we're low on registers we installed rsp as a placeholding in _owner.
3727          // Now install Self over rsp.  This is safe as we're transitioning from
3728          // non-null to non=null
3729          masm.get_thread (boxReg) ;
3730          masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ;
3731          // Intentional fall-through into LGoSlowPath ...
3732 
3733          masm.bind  (LGoSlowPath) ; 
3734          masm.orptr(boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
3735          masm.jmpb  (DONE_LABEL) ; 
3736 
3737          masm.bind  (LSuccess) ; 
3738          masm.xorptr(boxReg, boxReg) ;                 // set ICC.ZF=1 to indicate success
3739          masm.jmpb  (DONE_LABEL) ; 
3740       }
3741 
3742       masm.bind (Stacked) ;
3743       // It's not inflated and it's not recursively stack-locked and it's not biased.
3744       // It must be stack-locked.
3745       // Try to reset the header to displaced header.
3746       // The "box" value on the stack is stable, so we can reload
3747       // and be assured we observe the same value as above.
3748       masm.movptr(tmpReg, Address(boxReg, 0)) ;
3749       if (os::is_MP()) {   masm.lock();    }
3750       masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box
3751       // Intention fall-thru into DONE_LABEL
3752 
3753 
3754       // DONE_LABEL is a hot target - we'd really like to place it at the
3755       // start of cache line by padding with NOPs.
3756       // See the AMD and Intel software optimization manuals for the
3757       // most efficient "long" NOP encodings.
3758       // Unfortunately none of our alignment mechanisms suffice.
3759       if ((EmitSync & 65536) == 0) {
3760          masm.bind (CheckSucc) ;
3761       }
3762       masm.bind(DONE_LABEL);
3763 
3764       // Avoid branch to branch on AMD processors
3765       if (EmitSync & 32768) { masm.nop() ; }
3766     }
3767   %}
3768 
3769 
3770   enc_class enc_pop_rdx() %{
3771     emit_opcode(cbuf,0x5A);
3772   %}
3773 
3774   enc_class enc_rethrow() %{
3775     cbuf.set_inst_mark();
3776     emit_opcode(cbuf, 0xE9);        // jmp    entry
3777     emit_d32_reloc(cbuf, (int)OptoRuntime::rethrow_stub() - ((int)cbuf.code_end())-4,
3778                    runtime_call_Relocation::spec(), RELOC_IMM32 );
3779   %}
3780 
3781 
3782   // Convert a double to an int.  Java semantics require we do complex
3783   // manglelations in the corner cases.  So we set the rounding mode to
3784   // 'zero', store the darned double down as an int, and reset the
3785   // rounding mode to 'nearest'.  The hardware throws an exception which
3786   // patches up the correct value directly to the stack.
3787   enc_class D2I_encoding( regD src ) %{
3788     // Flip to round-to-zero mode.  We attempted to allow invalid-op
3789     // exceptions here, so that a NAN or other corner-case value will
3790     // thrown an exception (but normal values get converted at full speed).
3791     // However, I2C adapters and other float-stack manglers leave pending
3792     // invalid-op exceptions hanging.  We would have to clear them before
3793     // enabling them and that is more expensive than just testing for the
3794     // invalid value Intel stores down in the corner cases.
3795     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
3796     emit_opcode(cbuf,0x2D);
3797     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3798     // Allocate a word
3799     emit_opcode(cbuf,0x83);            // SUB ESP,4
3800     emit_opcode(cbuf,0xEC);
3801     emit_d8(cbuf,0x04);
3802     // Encoding assumes a double has been pushed into FPR0.
3803     // Store down the double as an int, popping the FPU stack
3804     emit_opcode(cbuf,0xDB);            // FISTP [ESP]
3805     emit_opcode(cbuf,0x1C);
3806     emit_d8(cbuf,0x24);
3807     // Restore the rounding mode; mask the exception
3808     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
3809     emit_opcode(cbuf,0x2D);
3810     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3811         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3812         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3813 
3814     // Load the converted int; adjust CPU stack
3815     emit_opcode(cbuf,0x58);       // POP EAX
3816     emit_opcode(cbuf,0x3D);       // CMP EAX,imm
3817     emit_d32   (cbuf,0x80000000); //         0x80000000
3818     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3819     emit_d8    (cbuf,0x07);       // Size of slow_call
3820     // Push src onto stack slow-path
3821     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
3822     emit_d8    (cbuf,0xC0-1+$src$$reg );
3823     // CALL directly to the runtime
3824     cbuf.set_inst_mark();
3825     emit_opcode(cbuf,0xE8);       // Call into runtime
3826     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3827     // Carry on here...
3828   %}
3829 
3830   enc_class D2L_encoding( regD src ) %{
3831     emit_opcode(cbuf,0xD9);            // FLDCW  trunc
3832     emit_opcode(cbuf,0x2D);
3833     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3834     // Allocate a word
3835     emit_opcode(cbuf,0x83);            // SUB ESP,8
3836     emit_opcode(cbuf,0xEC);
3837     emit_d8(cbuf,0x08);
3838     // Encoding assumes a double has been pushed into FPR0.
3839     // Store down the double as a long, popping the FPU stack
3840     emit_opcode(cbuf,0xDF);            // FISTP [ESP]
3841     emit_opcode(cbuf,0x3C);
3842     emit_d8(cbuf,0x24);
3843     // Restore the rounding mode; mask the exception
3844     emit_opcode(cbuf,0xD9);            // FLDCW   std/24-bit mode
3845     emit_opcode(cbuf,0x2D);
3846     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3847         ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3848         : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3849 
3850     // Load the converted int; adjust CPU stack
3851     emit_opcode(cbuf,0x58);       // POP EAX
3852     emit_opcode(cbuf,0x5A);       // POP EDX
3853     emit_opcode(cbuf,0x81);       // CMP EDX,imm
3854     emit_d8    (cbuf,0xFA);       // rdx
3855     emit_d32   (cbuf,0x80000000); //         0x80000000
3856     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3857     emit_d8    (cbuf,0x07+4);     // Size of slow_call
3858     emit_opcode(cbuf,0x85);       // TEST EAX,EAX
3859     emit_opcode(cbuf,0xC0);       // 2/rax,/rax,
3860     emit_opcode(cbuf,0x75);       // JNE around_slow_call
3861     emit_d8    (cbuf,0x07);       // Size of slow_call
3862     // Push src onto stack slow-path
3863     emit_opcode(cbuf,0xD9 );      // FLD     ST(i)
3864     emit_d8    (cbuf,0xC0-1+$src$$reg );
3865     // CALL directly to the runtime
3866     cbuf.set_inst_mark();
3867     emit_opcode(cbuf,0xE8);       // Call into runtime
3868     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3869     // Carry on here...
3870   %}
3871 
3872   enc_class X2L_encoding( regX src ) %{
3873     // Allocate a word
3874     emit_opcode(cbuf,0x83);      // SUB ESP,8
3875     emit_opcode(cbuf,0xEC);
3876     emit_d8(cbuf,0x08);
3877 
3878     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
3879     emit_opcode  (cbuf, 0x0F );
3880     emit_opcode  (cbuf, 0x11 );
3881     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3882 
3883     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
3884     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3885 
3886     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
3887     emit_opcode(cbuf,0x2D);
3888     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3889 
3890     // Encoding assumes a double has been pushed into FPR0.
3891     // Store down the double as a long, popping the FPU stack
3892     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
3893     emit_opcode(cbuf,0x3C);
3894     emit_d8(cbuf,0x24);
3895 
3896     // Restore the rounding mode; mask the exception
3897     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
3898     emit_opcode(cbuf,0x2D);
3899     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3900       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3901       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3902 
3903     // Load the converted int; adjust CPU stack
3904     emit_opcode(cbuf,0x58);      // POP EAX
3905 
3906     emit_opcode(cbuf,0x5A);      // POP EDX
3907 
3908     emit_opcode(cbuf,0x81);      // CMP EDX,imm
3909     emit_d8    (cbuf,0xFA);      // rdx
3910     emit_d32   (cbuf,0x80000000);//         0x80000000
3911 
3912     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3913     emit_d8    (cbuf,0x13+4);    // Size of slow_call
3914 
3915     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
3916     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
3917 
3918     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3919     emit_d8    (cbuf,0x13);      // Size of slow_call
3920 
3921     // Allocate a word
3922     emit_opcode(cbuf,0x83);      // SUB ESP,4
3923     emit_opcode(cbuf,0xEC);
3924     emit_d8(cbuf,0x04);
3925 
3926     emit_opcode  (cbuf, 0xF3 );  // MOVSS [ESP], src
3927     emit_opcode  (cbuf, 0x0F );
3928     emit_opcode  (cbuf, 0x11 );
3929     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3930 
3931     emit_opcode(cbuf,0xD9 );     // FLD_S [ESP]
3932     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3933 
3934     emit_opcode(cbuf,0x83);      // ADD ESP,4
3935     emit_opcode(cbuf,0xC4);
3936     emit_d8(cbuf,0x04);
3937 
3938     // CALL directly to the runtime
3939     cbuf.set_inst_mark();
3940     emit_opcode(cbuf,0xE8);       // Call into runtime
3941     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
3942     // Carry on here...
3943   %}
3944 
3945   enc_class XD2L_encoding( regXD src ) %{
3946     // Allocate a word
3947     emit_opcode(cbuf,0x83);      // SUB ESP,8
3948     emit_opcode(cbuf,0xEC);
3949     emit_d8(cbuf,0x08);
3950 
3951     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
3952     emit_opcode  (cbuf, 0x0F );
3953     emit_opcode  (cbuf, 0x11 );
3954     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
3955 
3956     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
3957     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
3958 
3959     emit_opcode(cbuf,0xD9);      // FLDCW  trunc
3960     emit_opcode(cbuf,0x2D);
3961     emit_d32(cbuf,(int)StubRoutines::addr_fpu_cntrl_wrd_trunc());
3962 
3963     // Encoding assumes a double has been pushed into FPR0.
3964     // Store down the double as a long, popping the FPU stack
3965     emit_opcode(cbuf,0xDF);      // FISTP [ESP]
3966     emit_opcode(cbuf,0x3C);
3967     emit_d8(cbuf,0x24);
3968 
3969     // Restore the rounding mode; mask the exception
3970     emit_opcode(cbuf,0xD9);      // FLDCW   std/24-bit mode
3971     emit_opcode(cbuf,0x2D);
3972     emit_d32( cbuf, Compile::current()->in_24_bit_fp_mode()
3973       ? (int)StubRoutines::addr_fpu_cntrl_wrd_24()
3974       : (int)StubRoutines::addr_fpu_cntrl_wrd_std());
3975 
3976     // Load the converted int; adjust CPU stack
3977     emit_opcode(cbuf,0x58);      // POP EAX
3978 
3979     emit_opcode(cbuf,0x5A);      // POP EDX
3980 
3981     emit_opcode(cbuf,0x81);      // CMP EDX,imm
3982     emit_d8    (cbuf,0xFA);      // rdx
3983     emit_d32   (cbuf,0x80000000); //         0x80000000
3984 
3985     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3986     emit_d8    (cbuf,0x13+4);    // Size of slow_call
3987 
3988     emit_opcode(cbuf,0x85);      // TEST EAX,EAX
3989     emit_opcode(cbuf,0xC0);      // 2/rax,/rax,
3990 
3991     emit_opcode(cbuf,0x75);      // JNE around_slow_call
3992     emit_d8    (cbuf,0x13);      // Size of slow_call
3993 
3994     // Push src onto stack slow-path
3995     // Allocate a word
3996     emit_opcode(cbuf,0x83);      // SUB ESP,8
3997     emit_opcode(cbuf,0xEC);
3998     emit_d8(cbuf,0x08);
3999 
4000     emit_opcode  (cbuf, 0xF2 );  // MOVSD [ESP], src
4001     emit_opcode  (cbuf, 0x0F );
4002     emit_opcode  (cbuf, 0x11 );
4003     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
4004 
4005     emit_opcode(cbuf,0xDD );     // FLD_D [ESP]
4006     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
4007 
4008     emit_opcode(cbuf,0x83);      // ADD ESP,8
4009     emit_opcode(cbuf,0xC4);
4010     emit_d8(cbuf,0x08);
4011 
4012     // CALL directly to the runtime
4013     cbuf.set_inst_mark();
4014     emit_opcode(cbuf,0xE8);      // Call into runtime
4015     emit_d32_reloc(cbuf, (StubRoutines::d2l_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
4016     // Carry on here...
4017   %}
4018 
4019   enc_class D2X_encoding( regX dst, regD src ) %{
4020     // Allocate a word
4021     emit_opcode(cbuf,0x83);            // SUB ESP,4
4022     emit_opcode(cbuf,0xEC);
4023     emit_d8(cbuf,0x04);
4024     int pop = 0x02;
4025     if ($src$$reg != FPR1L_enc) {
4026       emit_opcode( cbuf, 0xD9 );       // FLD    ST(i-1)
4027       emit_d8( cbuf, 0xC0-1+$src$$reg );
4028       pop = 0x03;
4029     }
4030     store_to_stackslot( cbuf, 0xD9, pop, 0 ); // FST<P>_S  [ESP]
4031 
4032     emit_opcode  (cbuf, 0xF3 );        // MOVSS dst(xmm), [ESP]
4033     emit_opcode  (cbuf, 0x0F );
4034     emit_opcode  (cbuf, 0x10 );
4035     encode_RegMem(cbuf, $dst$$reg, ESP_enc, 0x4, 0, 0, false);
4036 
4037     emit_opcode(cbuf,0x83);            // ADD ESP,4
4038     emit_opcode(cbuf,0xC4);
4039     emit_d8(cbuf,0x04);
4040     // Carry on here...
4041   %}
4042 
4043   enc_class FX2I_encoding( regX src, eRegI dst ) %{
4044     emit_rm(cbuf, 0x3, $dst$$reg, $src$$reg);
4045 
4046     // Compare the result to see if we need to go to the slow path
4047     emit_opcode(cbuf,0x81);       // CMP dst,imm
4048     emit_rm    (cbuf,0x3,0x7,$dst$$reg);
4049     emit_d32   (cbuf,0x80000000); //         0x80000000
4050 
4051     emit_opcode(cbuf,0x75);       // JNE around_slow_call
4052     emit_d8    (cbuf,0x13);       // Size of slow_call
4053     // Store xmm to a temp memory
4054     // location and push it onto stack.
4055 
4056     emit_opcode(cbuf,0x83);  // SUB ESP,4
4057     emit_opcode(cbuf,0xEC);
4058     emit_d8(cbuf, $primary ? 0x8 : 0x4);
4059 
4060     emit_opcode  (cbuf, $primary ? 0xF2 : 0xF3 );   // MOVSS [ESP], xmm
4061     emit_opcode  (cbuf, 0x0F );
4062     emit_opcode  (cbuf, 0x11 );
4063     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
4064 
4065     emit_opcode(cbuf, $primary ? 0xDD : 0xD9 );      // FLD [ESP]
4066     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
4067 
4068     emit_opcode(cbuf,0x83);    // ADD ESP,4
4069     emit_opcode(cbuf,0xC4);
4070     emit_d8(cbuf, $primary ? 0x8 : 0x4);
4071 
4072     // CALL directly to the runtime
4073     cbuf.set_inst_mark();
4074     emit_opcode(cbuf,0xE8);       // Call into runtime
4075     emit_d32_reloc(cbuf, (StubRoutines::d2i_wrapper() - cbuf.code_end()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
4076 
4077     // Carry on here...
4078   %}
4079 
4080   enc_class X2D_encoding( regD dst, regX src ) %{
4081     // Allocate a word
4082     emit_opcode(cbuf,0x83);     // SUB ESP,4
4083     emit_opcode(cbuf,0xEC);
4084     emit_d8(cbuf,0x04);
4085 
4086     emit_opcode  (cbuf, 0xF3 ); // MOVSS [ESP], xmm
4087     emit_opcode  (cbuf, 0x0F );
4088     emit_opcode  (cbuf, 0x11 );
4089     encode_RegMem(cbuf, $src$$reg, ESP_enc, 0x4, 0, 0, false);
4090 
4091     emit_opcode(cbuf,0xD9 );    // FLD_S [ESP]
4092     encode_RegMem(cbuf, 0x0, ESP_enc, 0x4, 0, 0, false);
4093 
4094     emit_opcode(cbuf,0x83);     // ADD ESP,4
4095     emit_opcode(cbuf,0xC4);
4096     emit_d8(cbuf,0x04);
4097 
4098     // Carry on here...
4099   %}
4100 
4101   enc_class AbsXF_encoding(regX dst) %{
4102     address signmask_address=(address)float_signmask_pool;
4103     // andpd:\tANDPS  $dst,[signconst]
4104     emit_opcode(cbuf, 0x0F);
4105     emit_opcode(cbuf, 0x54);
4106     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4107     emit_d32(cbuf, (int)signmask_address);
4108   %}
4109 
4110   enc_class AbsXD_encoding(regXD dst) %{
4111     address signmask_address=(address)double_signmask_pool;
4112     // andpd:\tANDPD  $dst,[signconst]
4113     emit_opcode(cbuf, 0x66);
4114     emit_opcode(cbuf, 0x0F);
4115     emit_opcode(cbuf, 0x54);
4116     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4117     emit_d32(cbuf, (int)signmask_address);
4118   %}
4119 
4120   enc_class NegXF_encoding(regX dst) %{
4121     address signmask_address=(address)float_signflip_pool;
4122     // andpd:\tXORPS  $dst,[signconst]
4123     emit_opcode(cbuf, 0x0F);
4124     emit_opcode(cbuf, 0x57);
4125     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4126     emit_d32(cbuf, (int)signmask_address);
4127   %}
4128 
4129   enc_class NegXD_encoding(regXD dst) %{
4130     address signmask_address=(address)double_signflip_pool;
4131     // andpd:\tXORPD  $dst,[signconst]
4132     emit_opcode(cbuf, 0x66);
4133     emit_opcode(cbuf, 0x0F);
4134     emit_opcode(cbuf, 0x57);
4135     emit_rm(cbuf, 0x0, $dst$$reg, 0x5);
4136     emit_d32(cbuf, (int)signmask_address);
4137   %}
4138 
4139   enc_class FMul_ST_reg( eRegF src1 ) %{
4140     // Operand was loaded from memory into fp ST (stack top)
4141     // FMUL   ST,$src  /* D8 C8+i */
4142     emit_opcode(cbuf, 0xD8);
4143     emit_opcode(cbuf, 0xC8 + $src1$$reg);
4144   %}
4145 
4146   enc_class FAdd_ST_reg( eRegF src2 ) %{
4147     // FADDP  ST,src2  /* D8 C0+i */
4148     emit_opcode(cbuf, 0xD8);
4149     emit_opcode(cbuf, 0xC0 + $src2$$reg);
4150     //could use FADDP  src2,fpST  /* DE C0+i */
4151   %}
4152 
4153   enc_class FAddP_reg_ST( eRegF src2 ) %{
4154     // FADDP  src2,ST  /* DE C0+i */
4155     emit_opcode(cbuf, 0xDE);
4156     emit_opcode(cbuf, 0xC0 + $src2$$reg);
4157   %}
4158 
4159   enc_class subF_divF_encode( eRegF src1, eRegF src2) %{
4160     // Operand has been loaded into fp ST (stack top)
4161       // FSUB   ST,$src1
4162       emit_opcode(cbuf, 0xD8);
4163       emit_opcode(cbuf, 0xE0 + $src1$$reg);
4164 
4165       // FDIV
4166       emit_opcode(cbuf, 0xD8);
4167       emit_opcode(cbuf, 0xF0 + $src2$$reg);
4168   %}
4169 
4170   enc_class MulFAddF (eRegF src1, eRegF src2) %{
4171     // Operand was loaded from memory into fp ST (stack top)
4172     // FADD   ST,$src  /* D8 C0+i */
4173     emit_opcode(cbuf, 0xD8);
4174     emit_opcode(cbuf, 0xC0 + $src1$$reg);
4175 
4176     // FMUL  ST,src2  /* D8 C*+i */
4177     emit_opcode(cbuf, 0xD8);
4178     emit_opcode(cbuf, 0xC8 + $src2$$reg);
4179   %}
4180 
4181 
4182   enc_class MulFAddFreverse (eRegF src1, eRegF src2) %{
4183     // Operand was loaded from memory into fp ST (stack top)
4184     // FADD   ST,$src  /* D8 C0+i */
4185     emit_opcode(cbuf, 0xD8);
4186     emit_opcode(cbuf, 0xC0 + $src1$$reg);
4187 
4188     // FMULP  src2,ST  /* DE C8+i */
4189     emit_opcode(cbuf, 0xDE);
4190     emit_opcode(cbuf, 0xC8 + $src2$$reg);
4191   %}
4192 
4193   // Atomically load the volatile long
4194   enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{
4195     emit_opcode(cbuf,0xDF);
4196     int rm_byte_opcode = 0x05;
4197     int base     = $mem$$base;
4198     int index    = $mem$$index;
4199     int scale    = $mem$$scale;
4200     int displace = $mem$$disp;
4201     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4202     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
4203     store_to_stackslot( cbuf, 0x0DF, 0x07, $dst$$disp );
4204   %}
4205 
4206   enc_class enc_loadLX_volatile( memory mem, stackSlotL dst, regXD tmp ) %{
4207     { // Atomic long load
4208       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
4209       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4210       emit_opcode(cbuf,0x0F);
4211       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4212       int base     = $mem$$base;
4213       int index    = $mem$$index;
4214       int scale    = $mem$$scale;
4215       int displace = $mem$$disp;
4216       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4217       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4218     }
4219     { // MOVSD $dst,$tmp ! atomic long store
4220       emit_opcode(cbuf,0xF2);
4221       emit_opcode(cbuf,0x0F);
4222       emit_opcode(cbuf,0x11);
4223       int base     = $dst$$base;
4224       int index    = $dst$$index;
4225       int scale    = $dst$$scale;
4226       int displace = $dst$$disp;
4227       bool disp_is_oop = $dst->disp_is_oop(); // disp-as-oop when working with static globals
4228       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4229     }
4230   %}
4231 
4232   enc_class enc_loadLX_reg_volatile( memory mem, eRegL dst, regXD tmp ) %{
4233     { // Atomic long load
4234       // UseXmmLoadAndClearUpper ? movsd $tmp,$mem : movlpd $tmp,$mem
4235       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4236       emit_opcode(cbuf,0x0F);
4237       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4238       int base     = $mem$$base;
4239       int index    = $mem$$index;
4240       int scale    = $mem$$scale;
4241       int displace = $mem$$disp;
4242       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4243       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4244     }
4245     { // MOVD $dst.lo,$tmp
4246       emit_opcode(cbuf,0x66);
4247       emit_opcode(cbuf,0x0F);
4248       emit_opcode(cbuf,0x7E);
4249       emit_rm(cbuf, 0x3, $tmp$$reg, $dst$$reg);
4250     }
4251     { // PSRLQ $tmp,32
4252       emit_opcode(cbuf,0x66);
4253       emit_opcode(cbuf,0x0F);
4254       emit_opcode(cbuf,0x73);
4255       emit_rm(cbuf, 0x3, 0x02, $tmp$$reg);
4256       emit_d8(cbuf, 0x20);
4257     }
4258     { // MOVD $dst.hi,$tmp
4259       emit_opcode(cbuf,0x66);
4260       emit_opcode(cbuf,0x0F);
4261       emit_opcode(cbuf,0x7E);
4262       emit_rm(cbuf, 0x3, $tmp$$reg, HIGH_FROM_LOW($dst$$reg));
4263     }
4264   %}
4265 
4266   // Volatile Store Long.  Must be atomic, so move it into
4267   // the FP TOS and then do a 64-bit FIST.  Has to probe the
4268   // target address before the store (for null-ptr checks)
4269   // so the memory operand is used twice in the encoding.
4270   enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
4271     store_to_stackslot( cbuf, 0x0DF, 0x05, $src$$disp );
4272     cbuf.set_inst_mark();            // Mark start of FIST in case $mem has an oop
4273     emit_opcode(cbuf,0xDF);
4274     int rm_byte_opcode = 0x07;
4275     int base     = $mem$$base;
4276     int index    = $mem$$index;
4277     int scale    = $mem$$scale;
4278     int displace = $mem$$disp;
4279     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4280     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace, disp_is_oop);
4281   %}
4282 
4283   enc_class enc_storeLX_volatile( memory mem, stackSlotL src, regXD tmp) %{
4284     { // Atomic long load
4285       // UseXmmLoadAndClearUpper ? movsd $tmp,[$src] : movlpd $tmp,[$src]
4286       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
4287       emit_opcode(cbuf,0x0F);
4288       emit_opcode(cbuf,UseXmmLoadAndClearUpper ? 0x10 : 0x12);
4289       int base     = $src$$base;
4290       int index    = $src$$index;
4291       int scale    = $src$$scale;
4292       int displace = $src$$disp;
4293       bool disp_is_oop = $src->disp_is_oop(); // disp-as-oop when working with static globals
4294       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4295     }
4296     cbuf.set_inst_mark();            // Mark start of MOVSD in case $mem has an oop
4297     { // MOVSD $mem,$tmp ! atomic long store
4298       emit_opcode(cbuf,0xF2);
4299       emit_opcode(cbuf,0x0F);
4300       emit_opcode(cbuf,0x11);
4301       int base     = $mem$$base;
4302       int index    = $mem$$index;
4303       int scale    = $mem$$scale;
4304       int displace = $mem$$disp;
4305       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4306       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4307     }
4308   %}
4309 
4310   enc_class enc_storeLX_reg_volatile( memory mem, eRegL src, regXD tmp, regXD tmp2) %{
4311     { // MOVD $tmp,$src.lo
4312       emit_opcode(cbuf,0x66);
4313       emit_opcode(cbuf,0x0F);
4314       emit_opcode(cbuf,0x6E);
4315       emit_rm(cbuf, 0x3, $tmp$$reg, $src$$reg);
4316     }
4317     { // MOVD $tmp2,$src.hi
4318       emit_opcode(cbuf,0x66);
4319       emit_opcode(cbuf,0x0F);
4320       emit_opcode(cbuf,0x6E);
4321       emit_rm(cbuf, 0x3, $tmp2$$reg, HIGH_FROM_LOW($src$$reg));
4322     }
4323     { // PUNPCKLDQ $tmp,$tmp2
4324       emit_opcode(cbuf,0x66);
4325       emit_opcode(cbuf,0x0F);
4326       emit_opcode(cbuf,0x62);
4327       emit_rm(cbuf, 0x3, $tmp$$reg, $tmp2$$reg);
4328     }
4329     cbuf.set_inst_mark();            // Mark start of MOVSD in case $mem has an oop
4330     { // MOVSD $mem,$tmp ! atomic long store
4331       emit_opcode(cbuf,0xF2);
4332       emit_opcode(cbuf,0x0F);
4333       emit_opcode(cbuf,0x11);
4334       int base     = $mem$$base;
4335       int index    = $mem$$index;
4336       int scale    = $mem$$scale;
4337       int displace = $mem$$disp;
4338       bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when working with static globals
4339       encode_RegMem(cbuf, $tmp$$reg, base, index, scale, displace, disp_is_oop);
4340     }
4341   %}
4342 
4343   // Safepoint Poll.  This polls the safepoint page, and causes an
4344   // exception if it is not readable. Unfortunately, it kills the condition code
4345   // in the process
4346   // We current use TESTL [spp],EDI
4347   // A better choice might be TESTB [spp + pagesize() - CacheLineSize()],0
4348 
4349   enc_class Safepoint_Poll() %{
4350     cbuf.relocate(cbuf.inst_mark(), relocInfo::poll_type, 0);
4351     emit_opcode(cbuf,0x85);
4352     emit_rm (cbuf, 0x0, 0x7, 0x5);
4353     emit_d32(cbuf, (intptr_t)os::get_polling_page());
4354   %}
4355 %}
4356 
4357 
4358 //----------FRAME--------------------------------------------------------------
4359 // Definition of frame structure and management information.
4360 //
4361 //  S T A C K   L A Y O U T    Allocators stack-slot number
4362 //                             |   (to get allocators register number
4363 //  G  Owned by    |        |  v    add OptoReg::stack0())
4364 //  r   CALLER     |        |
4365 //  o     |        +--------+      pad to even-align allocators stack-slot
4366 //  w     V        |  pad0  |        numbers; owned by CALLER
4367 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
4368 //  h     ^        |   in   |  5
4369 //        |        |  args  |  4   Holes in incoming args owned by SELF
4370 //  |     |        |        |  3
4371 //  |     |        +--------+
4372 //  V     |        | old out|      Empty on Intel, window on Sparc
4373 //        |    old |preserve|      Must be even aligned.
4374 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
4375 //        |        |   in   |  3   area for Intel ret address
4376 //     Owned by    |preserve|      Empty on Sparc.
4377 //       SELF      +--------+
4378 //        |        |  pad2  |  2   pad to align old SP
4379 //        |        +--------+  1
4380 //        |        | locks  |  0
4381 //        |        +--------+----> OptoReg::stack0(), even aligned
4382 //        |        |  pad1  | 11   pad to align new SP
4383 //        |        +--------+
4384 //        |        |        | 10
4385 //        |        | spills |  9   spills
4386 //        V        |        |  8   (pad0 slot for callee)
4387 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
4388 //        ^        |  out   |  7
4389 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
4390 //     Owned by    +--------+
4391 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
4392 //        |    new |preserve|      Must be even-aligned.
4393 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
4394 //        |        |        |
4395 //
4396 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
4397 //         known from SELF's arguments and the Java calling convention.
4398 //         Region 6-7 is determined per call site.
4399 // Note 2: If the calling convention leaves holes in the incoming argument
4400 //         area, those holes are owned by SELF.  Holes in the outgoing area
4401 //         are owned by the CALLEE.  Holes should not be nessecary in the
4402 //         incoming area, as the Java calling convention is completely under
4403 //         the control of the AD file.  Doubles can be sorted and packed to
4404 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
4405 //         varargs C calling conventions.
4406 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
4407 //         even aligned with pad0 as needed.
4408 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
4409 //         region 6-11 is even aligned; it may be padded out more so that
4410 //         the region from SP to FP meets the minimum stack alignment.
4411 
4412 frame %{
4413   // What direction does stack grow in (assumed to be same for C & Java)
4414   stack_direction(TOWARDS_LOW);
4415 
4416   // These three registers define part of the calling convention
4417   // between compiled code and the interpreter.
4418   inline_cache_reg(EAX);                // Inline Cache Register
4419   interpreter_method_oop_reg(EBX);      // Method Oop Register when calling interpreter
4420 
4421   // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
4422   cisc_spilling_operand_name(indOffset32);
4423 
4424   // Number of stack slots consumed by locking an object
4425   sync_stack_slots(1);
4426 
4427   // Compiled code's Frame Pointer
4428   frame_pointer(ESP);
4429   // Interpreter stores its frame pointer in a register which is
4430   // stored to the stack by I2CAdaptors.
4431   // I2CAdaptors convert from interpreted java to compiled java.
4432   interpreter_frame_pointer(EBP);
4433 
4434   // Stack alignment requirement
4435   // Alignment size in bytes (128-bit -> 16 bytes)
4436   stack_alignment(StackAlignmentInBytes);
4437 
4438   // Number of stack slots between incoming argument block and the start of
4439   // a new frame.  The PROLOG must add this many slots to the stack.  The
4440   // EPILOG must remove this many slots.  Intel needs one slot for
4441   // return address and one for rbp, (must save rbp)
4442   in_preserve_stack_slots(2+VerifyStackAtCalls);
4443 
4444   // Number of outgoing stack slots killed above the out_preserve_stack_slots
4445   // for calls to C.  Supports the var-args backing area for register parms.
4446   varargs_C_out_slots_killed(0);
4447 
4448   // The after-PROLOG location of the return address.  Location of
4449   // return address specifies a type (REG or STACK) and a number
4450   // representing the register number (i.e. - use a register name) or
4451   // stack slot.
4452   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4453   // Otherwise, it is above the locks and verification slot and alignment word
4454   return_addr(STACK - 1 +
4455               round_to(1+VerifyStackAtCalls+
4456               Compile::current()->fixed_slots(),
4457               (StackAlignmentInBytes/wordSize)));
4458 
4459   // Body of function which returns an integer array locating
4460   // arguments either in registers or in stack slots.  Passed an array
4461   // of ideal registers called "sig" and a "length" count.  Stack-slot
4462   // offsets are based on outgoing arguments, i.e. a CALLER setting up
4463   // arguments for a CALLEE.  Incoming stack arguments are
4464   // automatically biased by the preserve_stack_slots field above.
4465   calling_convention %{
4466     // No difference between ingoing/outgoing just pass false
4467     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
4468   %}
4469 
4470 
4471   // Body of function which returns an integer array locating
4472   // arguments either in registers or in stack slots.  Passed an array
4473   // of ideal registers called "sig" and a "length" count.  Stack-slot
4474   // offsets are based on outgoing arguments, i.e. a CALLER setting up
4475   // arguments for a CALLEE.  Incoming stack arguments are
4476   // automatically biased by the preserve_stack_slots field above.
4477   c_calling_convention %{
4478     // This is obviously always outgoing
4479     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
4480   %}
4481 
4482   // Location of C & interpreter return values
4483   c_return_value %{
4484     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
4485     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
4486     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
4487 
4488     // in SSE2+ mode we want to keep the FPU stack clean so pretend
4489     // that C functions return float and double results in XMM0.
4490     if( ideal_reg == Op_RegD && UseSSE>=2 )
4491       return OptoRegPair(XMM0b_num,XMM0a_num);
4492     if( ideal_reg == Op_RegF && UseSSE>=2 )
4493       return OptoRegPair(OptoReg::Bad,XMM0a_num);
4494 
4495     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
4496   %}
4497 
4498   // Location of return values
4499   return_value %{
4500     assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
4501     static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num,      EAX_num,      FPR1L_num,    FPR1L_num, EAX_num };
4502     static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
4503     if( ideal_reg == Op_RegD && UseSSE>=2 )
4504       return OptoRegPair(XMM0b_num,XMM0a_num);
4505     if( ideal_reg == Op_RegF && UseSSE>=1 )
4506       return OptoRegPair(OptoReg::Bad,XMM0a_num);
4507     return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
4508   %}
4509 
4510 %}
4511 
4512 //----------ATTRIBUTES---------------------------------------------------------
4513 //----------Operand Attributes-------------------------------------------------
4514 op_attrib op_cost(0);        // Required cost attribute
4515 
4516 //----------Instruction Attributes---------------------------------------------
4517 ins_attrib ins_cost(100);       // Required cost attribute
4518 ins_attrib ins_size(8);         // Required size attribute (in bits)
4519 ins_attrib ins_pc_relative(0);  // Required PC Relative flag
4520 ins_attrib ins_short_branch(0); // Required flag: is this instruction a
4521                                 // non-matching short branch variant of some
4522                                                             // long branch?
4523 ins_attrib ins_alignment(1);    // Required alignment attribute (must be a power of 2)
4524                                 // specifies the alignment that some part of the instruction (not
4525                                 // necessarily the start) requires.  If > 1, a compute_padding()
4526                                 // function must be provided for the instruction
4527 
4528 //----------OPERANDS-----------------------------------------------------------
4529 // Operand definitions must precede instruction definitions for correct parsing
4530 // in the ADLC because operands constitute user defined types which are used in
4531 // instruction definitions.
4532 
4533 //----------Simple Operands----------------------------------------------------
4534 // Immediate Operands
4535 // Integer Immediate
4536 operand immI() %{
4537   match(ConI);
4538 
4539   op_cost(10);
4540   format %{ %}
4541   interface(CONST_INTER);
4542 %}
4543 
4544 // Constant for test vs zero
4545 operand immI0() %{
4546   predicate(n->get_int() == 0);
4547   match(ConI);
4548 
4549   op_cost(0);
4550   format %{ %}
4551   interface(CONST_INTER);
4552 %}
4553 
4554 // Constant for increment
4555 operand immI1() %{
4556   predicate(n->get_int() == 1);
4557   match(ConI);
4558 
4559   op_cost(0);
4560   format %{ %}
4561   interface(CONST_INTER);
4562 %}
4563 
4564 // Constant for decrement
4565 operand immI_M1() %{
4566   predicate(n->get_int() == -1);
4567   match(ConI);
4568 
4569   op_cost(0);
4570   format %{ %}
4571   interface(CONST_INTER);
4572 %}
4573 
4574 // Valid scale values for addressing modes
4575 operand immI2() %{
4576   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4577   match(ConI);
4578 
4579   format %{ %}
4580   interface(CONST_INTER);
4581 %}
4582 
4583 operand immI8() %{
4584   predicate((-128 <= n->get_int()) && (n->get_int() <= 127));
4585   match(ConI);
4586 
4587   op_cost(5);
4588   format %{ %}
4589   interface(CONST_INTER);
4590 %}
4591 
4592 operand immI16() %{
4593   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4594   match(ConI);
4595 
4596   op_cost(10);
4597   format %{ %}
4598   interface(CONST_INTER);
4599 %}
4600 
4601 // Constant for long shifts
4602 operand immI_32() %{
4603   predicate( n->get_int() == 32 );
4604   match(ConI);
4605 
4606   op_cost(0);
4607   format %{ %}
4608   interface(CONST_INTER);
4609 %}
4610 
4611 operand immI_1_31() %{
4612   predicate( n->get_int() >= 1 && n->get_int() <= 31 );
4613   match(ConI);
4614 
4615   op_cost(0);
4616   format %{ %}
4617   interface(CONST_INTER);
4618 %}
4619 
4620 operand immI_32_63() %{
4621   predicate( n->get_int() >= 32 && n->get_int() <= 63 );
4622   match(ConI);
4623   op_cost(0);
4624 
4625   format %{ %}
4626   interface(CONST_INTER);
4627 %}
4628 
4629 operand immI_1() %{
4630   predicate( n->get_int() == 1 );
4631   match(ConI);
4632 
4633   op_cost(0);
4634   format %{ %}
4635   interface(CONST_INTER);
4636 %}
4637 
4638 operand immI_2() %{
4639   predicate( n->get_int() == 2 );
4640   match(ConI);
4641 
4642   op_cost(0);
4643   format %{ %}
4644   interface(CONST_INTER);
4645 %}
4646 
4647 operand immI_3() %{
4648   predicate( n->get_int() == 3 );
4649   match(ConI);
4650 
4651   op_cost(0);
4652   format %{ %}
4653   interface(CONST_INTER);
4654 %}
4655 
4656 // Pointer Immediate
4657 operand immP() %{
4658   match(ConP);
4659 
4660   op_cost(10);
4661   format %{ %}
4662   interface(CONST_INTER);
4663 %}
4664 
4665 // NULL Pointer Immediate
4666 operand immP0() %{
4667   predicate( n->get_ptr() == 0 );
4668   match(ConP);
4669   op_cost(0);
4670 
4671   format %{ %}
4672   interface(CONST_INTER);
4673 %}
4674 
4675 // Long Immediate
4676 operand immL() %{
4677   match(ConL);
4678 
4679   op_cost(20);
4680   format %{ %}
4681   interface(CONST_INTER);
4682 %}
4683 
4684 // Long Immediate zero
4685 operand immL0() %{
4686   predicate( n->get_long() == 0L );
4687   match(ConL);
4688   op_cost(0);
4689 
4690   format %{ %}
4691   interface(CONST_INTER);
4692 %}
4693 
4694 // Long Immediate zero
4695 operand immL_M1() %{
4696   predicate( n->get_long() == -1L );
4697   match(ConL);
4698   op_cost(0);
4699 
4700   format %{ %}
4701   interface(CONST_INTER);
4702 %}
4703 
4704 // Long immediate from 0 to 127.
4705 // Used for a shorter form of long mul by 10.
4706 operand immL_127() %{
4707   predicate((0 <= n->get_long()) && (n->get_long() <= 127));
4708   match(ConL);
4709   op_cost(0);
4710 
4711   format %{ %}
4712   interface(CONST_INTER);
4713 %}
4714 
4715 // Long Immediate: low 32-bit mask
4716 operand immL_32bits() %{
4717   predicate(n->get_long() == 0xFFFFFFFFL);
4718   match(ConL);
4719   op_cost(0);
4720 
4721   format %{ %}
4722   interface(CONST_INTER);
4723 %}
4724 
4725 // Long Immediate: low 32-bit mask
4726 operand immL32() %{
4727   predicate(n->get_long() == (int)(n->get_long()));
4728   match(ConL);
4729   op_cost(20);
4730 
4731   format %{ %}
4732   interface(CONST_INTER);
4733 %}
4734 
4735 //Double Immediate zero
4736 operand immD0() %{
4737   // Do additional (and counter-intuitive) test against NaN to work around VC++
4738   // bug that generates code such that NaNs compare equal to 0.0
4739   predicate( UseSSE<=1 && n->getd() == 0.0 && !g_isnan(n->getd()) );
4740   match(ConD);
4741 
4742   op_cost(5);
4743   format %{ %}
4744   interface(CONST_INTER);
4745 %}
4746 
4747 // Double Immediate
4748 operand immD1() %{
4749   predicate( UseSSE<=1 && n->getd() == 1.0 );
4750   match(ConD);
4751 
4752   op_cost(5);
4753   format %{ %}
4754   interface(CONST_INTER);
4755 %}
4756 
4757 // Double Immediate
4758 operand immD() %{
4759   predicate(UseSSE<=1);
4760   match(ConD);
4761 
4762   op_cost(5);
4763   format %{ %}
4764   interface(CONST_INTER);
4765 %}
4766 
4767 operand immXD() %{
4768   predicate(UseSSE>=2);
4769   match(ConD);
4770 
4771   op_cost(5);
4772   format %{ %}
4773   interface(CONST_INTER);
4774 %}
4775 
4776 // Double Immediate zero
4777 operand immXD0() %{
4778   // Do additional (and counter-intuitive) test against NaN to work around VC++
4779   // bug that generates code such that NaNs compare equal to 0.0 AND do not
4780   // compare equal to -0.0.
4781   predicate( UseSSE>=2 && jlong_cast(n->getd()) == 0 );
4782   match(ConD);
4783 
4784   format %{ %}
4785   interface(CONST_INTER);
4786 %}
4787 
4788 // Float Immediate zero
4789 operand immF0() %{
4790   predicate( UseSSE == 0 && n->getf() == 0.0 );
4791   match(ConF);
4792 
4793   op_cost(5);
4794   format %{ %}
4795   interface(CONST_INTER);
4796 %}
4797 
4798 // Float Immediate
4799 operand immF() %{
4800   predicate( UseSSE == 0 );
4801   match(ConF);
4802 
4803   op_cost(5);
4804   format %{ %}
4805   interface(CONST_INTER);
4806 %}
4807 
4808 // Float Immediate
4809 operand immXF() %{
4810   predicate(UseSSE >= 1);
4811   match(ConF);
4812 
4813   op_cost(5);
4814   format %{ %}
4815   interface(CONST_INTER);
4816 %}
4817 
4818 // Float Immediate zero.  Zero and not -0.0
4819 operand immXF0() %{
4820   predicate( UseSSE >= 1 && jint_cast(n->getf()) == 0 );
4821   match(ConF);
4822 
4823   op_cost(5);
4824   format %{ %}
4825   interface(CONST_INTER);
4826 %}
4827 
4828 // Immediates for special shifts (sign extend)
4829 
4830 // Constants for increment
4831 operand immI_16() %{
4832   predicate( n->get_int() == 16 );
4833   match(ConI);
4834 
4835   format %{ %}
4836   interface(CONST_INTER);
4837 %}
4838 
4839 operand immI_24() %{
4840   predicate( n->get_int() == 24 );
4841   match(ConI);
4842 
4843   format %{ %}
4844   interface(CONST_INTER);
4845 %}
4846 
4847 // Constant for byte-wide masking
4848 operand immI_255() %{
4849   predicate( n->get_int() == 255 );
4850   match(ConI);
4851 
4852   format %{ %}
4853   interface(CONST_INTER);
4854 %}
4855 
4856 // Constant for short-wide masking
4857 operand immI_65535() %{
4858   predicate(n->get_int() == 65535);
4859   match(ConI);
4860 
4861   format %{ %}
4862   interface(CONST_INTER);
4863 %}
4864 
4865 // Register Operands
4866 // Integer Register
4867 operand eRegI() %{
4868   constraint(ALLOC_IN_RC(e_reg));
4869   match(RegI);
4870   match(xRegI);
4871   match(eAXRegI);
4872   match(eBXRegI);
4873   match(eCXRegI);
4874   match(eDXRegI);
4875   match(eDIRegI);
4876   match(eSIRegI);
4877 
4878   format %{ %}
4879   interface(REG_INTER);
4880 %}
4881 
4882 // Subset of Integer Register
4883 operand xRegI(eRegI reg) %{
4884   constraint(ALLOC_IN_RC(x_reg));
4885   match(reg);
4886   match(eAXRegI);
4887   match(eBXRegI);
4888   match(eCXRegI);
4889   match(eDXRegI);
4890 
4891   format %{ %}
4892   interface(REG_INTER);
4893 %}
4894 
4895 // Special Registers
4896 operand eAXRegI(xRegI reg) %{
4897   constraint(ALLOC_IN_RC(eax_reg));
4898   match(reg);
4899   match(eRegI);
4900 
4901   format %{ "EAX" %}
4902   interface(REG_INTER);
4903 %}
4904 
4905 // Special Registers
4906 operand eBXRegI(xRegI reg) %{
4907   constraint(ALLOC_IN_RC(ebx_reg));
4908   match(reg);
4909   match(eRegI);
4910 
4911   format %{ "EBX" %}
4912   interface(REG_INTER);
4913 %}
4914 
4915 operand eCXRegI(xRegI reg) %{
4916   constraint(ALLOC_IN_RC(ecx_reg));
4917   match(reg);
4918   match(eRegI);
4919 
4920   format %{ "ECX" %}
4921   interface(REG_INTER);
4922 %}
4923 
4924 operand eDXRegI(xRegI reg) %{
4925   constraint(ALLOC_IN_RC(edx_reg));
4926   match(reg);
4927   match(eRegI);
4928 
4929   format %{ "EDX" %}
4930   interface(REG_INTER);
4931 %}
4932 
4933 operand eDIRegI(xRegI reg) %{
4934   constraint(ALLOC_IN_RC(edi_reg));
4935   match(reg);
4936   match(eRegI);
4937 
4938   format %{ "EDI" %}
4939   interface(REG_INTER);
4940 %}
4941 
4942 operand naxRegI() %{
4943   constraint(ALLOC_IN_RC(nax_reg));
4944   match(RegI);
4945   match(eCXRegI);
4946   match(eDXRegI);
4947   match(eSIRegI);
4948   match(eDIRegI);
4949 
4950   format %{ %}
4951   interface(REG_INTER);
4952 %}
4953 
4954 operand nadxRegI() %{
4955   constraint(ALLOC_IN_RC(nadx_reg));
4956   match(RegI);
4957   match(eBXRegI);
4958   match(eCXRegI);
4959   match(eSIRegI);
4960   match(eDIRegI);
4961 
4962   format %{ %}
4963   interface(REG_INTER);
4964 %}
4965 
4966 operand ncxRegI() %{
4967   constraint(ALLOC_IN_RC(ncx_reg));
4968   match(RegI);
4969   match(eAXRegI);
4970   match(eDXRegI);
4971   match(eSIRegI);
4972   match(eDIRegI);
4973 
4974   format %{ %}
4975   interface(REG_INTER);
4976 %}
4977 
4978 // // This operand was used by cmpFastUnlock, but conflicted with 'object' reg
4979 // //
4980 operand eSIRegI(xRegI reg) %{
4981    constraint(ALLOC_IN_RC(esi_reg));
4982    match(reg);
4983    match(eRegI);
4984 
4985    format %{ "ESI" %}
4986    interface(REG_INTER);
4987 %}
4988 
4989 // Pointer Register
4990 operand anyRegP() %{
4991   constraint(ALLOC_IN_RC(any_reg));
4992   match(RegP);
4993   match(eAXRegP);
4994   match(eBXRegP);
4995   match(eCXRegP);
4996   match(eDIRegP);
4997   match(eRegP);
4998 
4999   format %{ %}
5000   interface(REG_INTER);
5001 %}
5002 
5003 operand eRegP() %{
5004   constraint(ALLOC_IN_RC(e_reg));
5005   match(RegP);
5006   match(eAXRegP);
5007   match(eBXRegP);
5008   match(eCXRegP);
5009   match(eDIRegP);
5010 
5011   format %{ %}
5012   interface(REG_INTER);
5013 %}
5014 
5015 // On windows95, EBP is not safe to use for implicit null tests.
5016 operand eRegP_no_EBP() %{
5017   constraint(ALLOC_IN_RC(e_reg_no_rbp));
5018   match(RegP);
5019   match(eAXRegP);
5020   match(eBXRegP);
5021   match(eCXRegP);
5022   match(eDIRegP);
5023 
5024   op_cost(100);
5025   format %{ %}
5026   interface(REG_INTER);
5027 %}
5028 
5029 operand naxRegP() %{
5030   constraint(ALLOC_IN_RC(nax_reg));
5031   match(RegP);
5032   match(eBXRegP);
5033   match(eDXRegP);
5034   match(eCXRegP);
5035   match(eSIRegP);
5036   match(eDIRegP);
5037 
5038   format %{ %}
5039   interface(REG_INTER);
5040 %}
5041 
5042 operand nabxRegP() %{
5043   constraint(ALLOC_IN_RC(nabx_reg));
5044   match(RegP);
5045   match(eCXRegP);
5046   match(eDXRegP);
5047   match(eSIRegP);
5048   match(eDIRegP);
5049 
5050   format %{ %}
5051   interface(REG_INTER);
5052 %}
5053 
5054 operand pRegP() %{
5055   constraint(ALLOC_IN_RC(p_reg));
5056   match(RegP);
5057   match(eBXRegP);
5058   match(eDXRegP);
5059   match(eSIRegP);
5060   match(eDIRegP);
5061 
5062   format %{ %}
5063   interface(REG_INTER);
5064 %}
5065 
5066 // Special Registers
5067 // Return a pointer value
5068 operand eAXRegP(eRegP reg) %{
5069   constraint(ALLOC_IN_RC(eax_reg));
5070   match(reg);
5071   format %{ "EAX" %}
5072   interface(REG_INTER);
5073 %}
5074 
5075 // Used in AtomicAdd
5076 operand eBXRegP(eRegP reg) %{
5077   constraint(ALLOC_IN_RC(ebx_reg));
5078   match(reg);
5079   format %{ "EBX" %}
5080   interface(REG_INTER);
5081 %}
5082 
5083 // Tail-call (interprocedural jump) to interpreter
5084 operand eCXRegP(eRegP reg) %{
5085   constraint(ALLOC_IN_RC(ecx_reg));
5086   match(reg);
5087   format %{ "ECX" %}
5088   interface(REG_INTER);
5089 %}
5090 
5091 operand eSIRegP(eRegP reg) %{
5092   constraint(ALLOC_IN_RC(esi_reg));
5093   match(reg);
5094   format %{ "ESI" %}
5095   interface(REG_INTER);
5096 %}
5097 
5098 // Used in rep stosw
5099 operand eDIRegP(eRegP reg) %{
5100   constraint(ALLOC_IN_RC(edi_reg));
5101   match(reg);
5102   format %{ "EDI" %}
5103   interface(REG_INTER);
5104 %}
5105 
5106 operand eBPRegP() %{
5107   constraint(ALLOC_IN_RC(ebp_reg));
5108   match(RegP);
5109   format %{ "EBP" %}
5110   interface(REG_INTER);
5111 %}
5112 
5113 operand eRegL() %{
5114   constraint(ALLOC_IN_RC(long_reg));
5115   match(RegL);
5116   match(eADXRegL);
5117 
5118   format %{ %}
5119   interface(REG_INTER);
5120 %}
5121 
5122 operand eADXRegL( eRegL reg ) %{
5123   constraint(ALLOC_IN_RC(eadx_reg));
5124   match(reg);
5125 
5126   format %{ "EDX:EAX" %}
5127   interface(REG_INTER);
5128 %}
5129 
5130 operand eBCXRegL( eRegL reg ) %{
5131   constraint(ALLOC_IN_RC(ebcx_reg));
5132   match(reg);
5133 
5134   format %{ "EBX:ECX" %}
5135   interface(REG_INTER);
5136 %}
5137 
5138 // Special case for integer high multiply
5139 operand eADXRegL_low_only() %{
5140   constraint(ALLOC_IN_RC(eadx_reg));
5141   match(RegL);
5142 
5143   format %{ "EAX" %}
5144   interface(REG_INTER);
5145 %}
5146 
5147 // Flags register, used as output of compare instructions
5148 operand eFlagsReg() %{
5149   constraint(ALLOC_IN_RC(int_flags));
5150   match(RegFlags);
5151 
5152   format %{ "EFLAGS" %}
5153   interface(REG_INTER);
5154 %}
5155 
5156 // Flags register, used as output of FLOATING POINT compare instructions
5157 operand eFlagsRegU() %{
5158   constraint(ALLOC_IN_RC(int_flags));
5159   match(RegFlags);
5160 
5161   format %{ "EFLAGS_U" %}
5162   interface(REG_INTER);
5163 %}
5164 
5165 operand eFlagsRegUCF() %{
5166   constraint(ALLOC_IN_RC(int_flags));
5167   match(RegFlags);
5168   predicate(false);
5169 
5170   format %{ "EFLAGS_U_CF" %}
5171   interface(REG_INTER);
5172 %}
5173 
5174 // Condition Code Register used by long compare
5175 operand flagsReg_long_LTGE() %{
5176   constraint(ALLOC_IN_RC(int_flags));
5177   match(RegFlags);
5178   format %{ "FLAGS_LTGE" %}
5179   interface(REG_INTER);
5180 %}
5181 operand flagsReg_long_EQNE() %{
5182   constraint(ALLOC_IN_RC(int_flags));
5183   match(RegFlags);
5184   format %{ "FLAGS_EQNE" %}
5185   interface(REG_INTER);
5186 %}
5187 operand flagsReg_long_LEGT() %{
5188   constraint(ALLOC_IN_RC(int_flags));
5189   match(RegFlags);
5190   format %{ "FLAGS_LEGT" %}
5191   interface(REG_INTER);
5192 %}
5193 
5194 // Float register operands
5195 operand regD() %{
5196   predicate( UseSSE < 2 );
5197   constraint(ALLOC_IN_RC(dbl_reg));
5198   match(RegD);
5199   match(regDPR1);
5200   match(regDPR2);
5201   format %{ %}
5202   interface(REG_INTER);
5203 %}
5204 
5205 operand regDPR1(regD reg) %{
5206   predicate( UseSSE < 2 );
5207   constraint(ALLOC_IN_RC(dbl_reg0));
5208   match(reg);
5209   format %{ "FPR1" %}
5210   interface(REG_INTER);
5211 %}
5212 
5213 operand regDPR2(regD reg) %{
5214   predicate( UseSSE < 2 );
5215   constraint(ALLOC_IN_RC(dbl_reg1));
5216   match(reg);
5217   format %{ "FPR2" %}
5218   interface(REG_INTER);
5219 %}
5220 
5221 operand regnotDPR1(regD reg) %{
5222   predicate( UseSSE < 2 );
5223   constraint(ALLOC_IN_RC(dbl_notreg0));
5224   match(reg);
5225   format %{ %}
5226   interface(REG_INTER);
5227 %}
5228 
5229 // XMM Double register operands
5230 operand regXD() %{
5231   predicate( UseSSE>=2 );
5232   constraint(ALLOC_IN_RC(xdb_reg));
5233   match(RegD);
5234   match(regXD6);
5235   match(regXD7);
5236   format %{ %}
5237   interface(REG_INTER);
5238 %}
5239 
5240 // XMM6 double register operands
5241 operand regXD6(regXD reg) %{
5242   predicate( UseSSE>=2 );
5243   constraint(ALLOC_IN_RC(xdb_reg6));
5244   match(reg);
5245   format %{ "XMM6" %}
5246   interface(REG_INTER);
5247 %}
5248 
5249 // XMM7 double register operands
5250 operand regXD7(regXD reg) %{
5251   predicate( UseSSE>=2 );
5252   constraint(ALLOC_IN_RC(xdb_reg7));
5253   match(reg);
5254   format %{ "XMM7" %}
5255   interface(REG_INTER);
5256 %}
5257 
5258 // Float register operands
5259 operand regF() %{
5260   predicate( UseSSE < 2 );
5261   constraint(ALLOC_IN_RC(flt_reg));
5262   match(RegF);
5263   match(regFPR1);
5264   format %{ %}
5265   interface(REG_INTER);
5266 %}
5267 
5268 // Float register operands
5269 operand regFPR1(regF reg) %{
5270   predicate( UseSSE < 2 );
5271   constraint(ALLOC_IN_RC(flt_reg0));
5272   match(reg);
5273   format %{ "FPR1" %}
5274   interface(REG_INTER);
5275 %}
5276 
5277 // XMM register operands
5278 operand regX() %{
5279   predicate( UseSSE>=1 );
5280   constraint(ALLOC_IN_RC(xmm_reg));
5281   match(RegF);
5282   format %{ %}
5283   interface(REG_INTER);
5284 %}
5285 
5286 
5287 //----------Memory Operands----------------------------------------------------
5288 // Direct Memory Operand
5289 operand direct(immP addr) %{
5290   match(addr);
5291 
5292   format %{ "[$addr]" %}
5293   interface(MEMORY_INTER) %{
5294     base(0xFFFFFFFF);
5295     index(0x4);
5296     scale(0x0);
5297     disp($addr);
5298   %}
5299 %}
5300 
5301 // Indirect Memory Operand
5302 operand indirect(eRegP reg) %{
5303   constraint(ALLOC_IN_RC(e_reg));
5304   match(reg);
5305 
5306   format %{ "[$reg]" %}
5307   interface(MEMORY_INTER) %{
5308     base($reg);
5309     index(0x4);
5310     scale(0x0);
5311     disp(0x0);
5312   %}
5313 %}
5314 
5315 // Indirect Memory Plus Short Offset Operand
5316 operand indOffset8(eRegP reg, immI8 off) %{
5317   match(AddP reg off);
5318 
5319   format %{ "[$reg + $off]" %}
5320   interface(MEMORY_INTER) %{
5321     base($reg);
5322     index(0x4);
5323     scale(0x0);
5324     disp($off);
5325   %}
5326 %}
5327 
5328 // Indirect Memory Plus Long Offset Operand
5329 operand indOffset32(eRegP reg, immI off) %{
5330   match(AddP reg off);
5331 
5332   format %{ "[$reg + $off]" %}
5333   interface(MEMORY_INTER) %{
5334     base($reg);
5335     index(0x4);
5336     scale(0x0);
5337     disp($off);
5338   %}
5339 %}
5340 
5341 // Indirect Memory Plus Long Offset Operand
5342 operand indOffset32X(eRegI reg, immP off) %{
5343   match(AddP off reg);
5344 
5345   format %{ "[$reg + $off]" %}
5346   interface(MEMORY_INTER) %{
5347     base($reg);
5348     index(0x4);
5349     scale(0x0);
5350     disp($off);
5351   %}
5352 %}
5353 
5354 // Indirect Memory Plus Index Register Plus Offset Operand
5355 operand indIndexOffset(eRegP reg, eRegI ireg, immI off) %{
5356   match(AddP (AddP reg ireg) off);
5357 
5358   op_cost(10);
5359   format %{"[$reg + $off + $ireg]" %}
5360   interface(MEMORY_INTER) %{
5361     base($reg);
5362     index($ireg);
5363     scale(0x0);
5364     disp($off);
5365   %}
5366 %}
5367 
5368 // Indirect Memory Plus Index Register Plus Offset Operand
5369 operand indIndex(eRegP reg, eRegI ireg) %{
5370   match(AddP reg ireg);
5371 
5372   op_cost(10);
5373   format %{"[$reg + $ireg]" %}
5374   interface(MEMORY_INTER) %{
5375     base($reg);
5376     index($ireg);
5377     scale(0x0);
5378     disp(0x0);
5379   %}
5380 %}
5381 
5382 // // -------------------------------------------------------------------------
5383 // // 486 architecture doesn't support "scale * index + offset" with out a base
5384 // // -------------------------------------------------------------------------
5385 // // Scaled Memory Operands
5386 // // Indirect Memory Times Scale Plus Offset Operand
5387 // operand indScaleOffset(immP off, eRegI ireg, immI2 scale) %{
5388 //   match(AddP off (LShiftI ireg scale));
5389 //
5390 //   op_cost(10);
5391 //   format %{"[$off + $ireg << $scale]" %}
5392 //   interface(MEMORY_INTER) %{
5393 //     base(0x4);
5394 //     index($ireg);
5395 //     scale($scale);
5396 //     disp($off);
5397 //   %}
5398 // %}
5399 
5400 // Indirect Memory Times Scale Plus Index Register
5401 operand indIndexScale(eRegP reg, eRegI ireg, immI2 scale) %{
5402   match(AddP reg (LShiftI ireg scale));
5403 
5404   op_cost(10);
5405   format %{"[$reg + $ireg << $scale]" %}
5406   interface(MEMORY_INTER) %{
5407     base($reg);
5408     index($ireg);
5409     scale($scale);
5410     disp(0x0);
5411   %}
5412 %}
5413 
5414 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5415 operand indIndexScaleOffset(eRegP reg, immI off, eRegI ireg, immI2 scale) %{
5416   match(AddP (AddP reg (LShiftI ireg scale)) off);
5417 
5418   op_cost(10);
5419   format %{"[$reg + $off + $ireg << $scale]" %}
5420   interface(MEMORY_INTER) %{
5421     base($reg);
5422     index($ireg);
5423     scale($scale);
5424     disp($off);
5425   %}
5426 %}
5427 
5428 //----------Load Long Memory Operands------------------------------------------
5429 // The load-long idiom will use it's address expression again after loading
5430 // the first word of the long.  If the load-long destination overlaps with
5431 // registers used in the addressing expression, the 2nd half will be loaded
5432 // from a clobbered address.  Fix this by requiring that load-long use
5433 // address registers that do not overlap with the load-long target.
5434 
5435 // load-long support
5436 operand load_long_RegP() %{
5437   constraint(ALLOC_IN_RC(esi_reg));
5438   match(RegP);
5439   match(eSIRegP);
5440   op_cost(100);
5441   format %{  %}
5442   interface(REG_INTER);
5443 %}
5444 
5445 // Indirect Memory Operand Long
5446 operand load_long_indirect(load_long_RegP reg) %{
5447   constraint(ALLOC_IN_RC(esi_reg));
5448   match(reg);
5449 
5450   format %{ "[$reg]" %}
5451   interface(MEMORY_INTER) %{
5452     base($reg);
5453     index(0x4);
5454     scale(0x0);
5455     disp(0x0);
5456   %}
5457 %}
5458 
5459 // Indirect Memory Plus Long Offset Operand
5460 operand load_long_indOffset32(load_long_RegP reg, immI off) %{
5461   match(AddP reg off);
5462 
5463   format %{ "[$reg + $off]" %}
5464   interface(MEMORY_INTER) %{
5465     base($reg);
5466     index(0x4);
5467     scale(0x0);
5468     disp($off);
5469   %}
5470 %}
5471 
5472 opclass load_long_memory(load_long_indirect, load_long_indOffset32);
5473 
5474 
5475 //----------Special Memory Operands--------------------------------------------
5476 // Stack Slot Operand - This operand is used for loading and storing temporary
5477 //                      values on the stack where a match requires a value to
5478 //                      flow through memory.
5479 operand stackSlotP(sRegP reg) %{
5480   constraint(ALLOC_IN_RC(stack_slots));
5481   // No match rule because this operand is only generated in matching
5482   format %{ "[$reg]" %}
5483   interface(MEMORY_INTER) %{
5484     base(0x4);   // ESP
5485     index(0x4);  // No Index
5486     scale(0x0);  // No Scale
5487     disp($reg);  // Stack Offset
5488   %}
5489 %}
5490 
5491 operand stackSlotI(sRegI reg) %{
5492   constraint(ALLOC_IN_RC(stack_slots));
5493   // No match rule because this operand is only generated in matching
5494   format %{ "[$reg]" %}
5495   interface(MEMORY_INTER) %{
5496     base(0x4);   // ESP
5497     index(0x4);  // No Index
5498     scale(0x0);  // No Scale
5499     disp($reg);  // Stack Offset
5500   %}
5501 %}
5502 
5503 operand stackSlotF(sRegF reg) %{
5504   constraint(ALLOC_IN_RC(stack_slots));
5505   // No match rule because this operand is only generated in matching
5506   format %{ "[$reg]" %}
5507   interface(MEMORY_INTER) %{
5508     base(0x4);   // ESP
5509     index(0x4);  // No Index
5510     scale(0x0);  // No Scale
5511     disp($reg);  // Stack Offset
5512   %}
5513 %}
5514 
5515 operand stackSlotD(sRegD reg) %{
5516   constraint(ALLOC_IN_RC(stack_slots));
5517   // No match rule because this operand is only generated in matching
5518   format %{ "[$reg]" %}
5519   interface(MEMORY_INTER) %{
5520     base(0x4);   // ESP
5521     index(0x4);  // No Index
5522     scale(0x0);  // No Scale
5523     disp($reg);  // Stack Offset
5524   %}
5525 %}
5526 
5527 operand stackSlotL(sRegL reg) %{
5528   constraint(ALLOC_IN_RC(stack_slots));
5529   // No match rule because this operand is only generated in matching
5530   format %{ "[$reg]" %}
5531   interface(MEMORY_INTER) %{
5532     base(0x4);   // ESP
5533     index(0x4);  // No Index
5534     scale(0x0);  // No Scale
5535     disp($reg);  // Stack Offset
5536   %}
5537 %}
5538 
5539 //----------Memory Operands - Win95 Implicit Null Variants----------------
5540 // Indirect Memory Operand
5541 operand indirect_win95_safe(eRegP_no_EBP reg)
5542 %{
5543   constraint(ALLOC_IN_RC(e_reg));
5544   match(reg);
5545 
5546   op_cost(100);
5547   format %{ "[$reg]" %}
5548   interface(MEMORY_INTER) %{
5549     base($reg);
5550     index(0x4);
5551     scale(0x0);
5552     disp(0x0);
5553   %}
5554 %}
5555 
5556 // Indirect Memory Plus Short Offset Operand
5557 operand indOffset8_win95_safe(eRegP_no_EBP reg, immI8 off)
5558 %{
5559   match(AddP reg off);
5560 
5561   op_cost(100);
5562   format %{ "[$reg + $off]" %}
5563   interface(MEMORY_INTER) %{
5564     base($reg);
5565     index(0x4);
5566     scale(0x0);
5567     disp($off);
5568   %}
5569 %}
5570 
5571 // Indirect Memory Plus Long Offset Operand
5572 operand indOffset32_win95_safe(eRegP_no_EBP reg, immI off)
5573 %{
5574   match(AddP reg off);
5575 
5576   op_cost(100);
5577   format %{ "[$reg + $off]" %}
5578   interface(MEMORY_INTER) %{
5579     base($reg);
5580     index(0x4);
5581     scale(0x0);
5582     disp($off);
5583   %}
5584 %}
5585 
5586 // Indirect Memory Plus Index Register Plus Offset Operand
5587 operand indIndexOffset_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI off)
5588 %{
5589   match(AddP (AddP reg ireg) off);
5590 
5591   op_cost(100);
5592   format %{"[$reg + $off + $ireg]" %}
5593   interface(MEMORY_INTER) %{
5594     base($reg);
5595     index($ireg);
5596     scale(0x0);
5597     disp($off);
5598   %}
5599 %}
5600 
5601 // Indirect Memory Times Scale Plus Index Register
5602 operand indIndexScale_win95_safe(eRegP_no_EBP reg, eRegI ireg, immI2 scale)
5603 %{
5604   match(AddP reg (LShiftI ireg scale));
5605 
5606   op_cost(100);
5607   format %{"[$reg + $ireg << $scale]" %}
5608   interface(MEMORY_INTER) %{
5609     base($reg);
5610     index($ireg);
5611     scale($scale);
5612     disp(0x0);
5613   %}
5614 %}
5615 
5616 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5617 operand indIndexScaleOffset_win95_safe(eRegP_no_EBP reg, immI off, eRegI ireg, immI2 scale)
5618 %{
5619   match(AddP (AddP reg (LShiftI ireg scale)) off);
5620 
5621   op_cost(100);
5622   format %{"[$reg + $off + $ireg << $scale]" %}
5623   interface(MEMORY_INTER) %{
5624     base($reg);
5625     index($ireg);
5626     scale($scale);
5627     disp($off);
5628   %}
5629 %}
5630 
5631 //----------Conditional Branch Operands----------------------------------------
5632 // Comparison Op  - This is the operation of the comparison, and is limited to
5633 //                  the following set of codes:
5634 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5635 //
5636 // Other attributes of the comparison, such as unsignedness, are specified
5637 // by the comparison instruction that sets a condition code flags register.
5638 // That result is represented by a flags operand whose subtype is appropriate
5639 // to the unsignedness (etc.) of the comparison.
5640 //
5641 // Later, the instruction which matches both the Comparison Op (a Bool) and
5642 // the flags (produced by the Cmp) specifies the coding of the comparison op
5643 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5644 
5645 // Comparision Code
5646 operand cmpOp() %{
5647   match(Bool);
5648 
5649   format %{ "" %}
5650   interface(COND_INTER) %{
5651     equal(0x4, "e");
5652     not_equal(0x5, "ne");
5653     less(0xC, "l");
5654     greater_equal(0xD, "ge");
5655     less_equal(0xE, "le");
5656     greater(0xF, "g");
5657   %}
5658 %}
5659 
5660 // Comparison Code, unsigned compare.  Used by FP also, with
5661 // C2 (unordered) turned into GT or LT already.  The other bits
5662 // C0 and C3 are turned into Carry & Zero flags.
5663 operand cmpOpU() %{
5664   match(Bool);
5665 
5666   format %{ "" %}
5667   interface(COND_INTER) %{
5668     equal(0x4, "e");
5669     not_equal(0x5, "ne");
5670     less(0x2, "b");
5671     greater_equal(0x3, "nb");
5672     less_equal(0x6, "be");
5673     greater(0x7, "nbe");
5674   %}
5675 %}
5676 
5677 // Floating comparisons that don't require any fixup for the unordered case
5678 operand cmpOpUCF() %{
5679   match(Bool);
5680   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
5681             n->as_Bool()->_test._test == BoolTest::ge ||
5682             n->as_Bool()->_test._test == BoolTest::le ||
5683             n->as_Bool()->_test._test == BoolTest::gt);
5684   format %{ "" %}
5685   interface(COND_INTER) %{
5686     equal(0x4, "e");
5687     not_equal(0x5, "ne");
5688     less(0x2, "b");
5689     greater_equal(0x3, "nb");
5690     less_equal(0x6, "be");
5691     greater(0x7, "nbe");
5692   %}
5693 %}
5694 
5695 
5696 // Floating comparisons that can be fixed up with extra conditional jumps
5697 operand cmpOpUCF2() %{
5698   match(Bool);
5699   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
5700             n->as_Bool()->_test._test == BoolTest::eq);
5701   format %{ "" %}
5702   interface(COND_INTER) %{
5703     equal(0x4, "e");
5704     not_equal(0x5, "ne");
5705     less(0x2, "b");
5706     greater_equal(0x3, "nb");
5707     less_equal(0x6, "be");
5708     greater(0x7, "nbe");
5709   %}
5710 %}
5711 
5712 // Comparison Code for FP conditional move
5713 operand cmpOp_fcmov() %{
5714   match(Bool);
5715 
5716   format %{ "" %}
5717   interface(COND_INTER) %{
5718     equal        (0x0C8);
5719     not_equal    (0x1C8);
5720     less         (0x0C0);
5721     greater_equal(0x1C0);
5722     less_equal   (0x0D0);
5723     greater      (0x1D0);
5724   %}
5725 %}
5726 
5727 // Comparision Code used in long compares
5728 operand cmpOp_commute() %{
5729   match(Bool);
5730 
5731   format %{ "" %}
5732   interface(COND_INTER) %{
5733     equal(0x4, "e");
5734     not_equal(0x5, "ne");
5735     less(0xF, "g");
5736     greater_equal(0xE, "le");
5737     less_equal(0xD, "ge");
5738     greater(0xC, "l");
5739   %}
5740 %}
5741 
5742 //----------OPERAND CLASSES----------------------------------------------------
5743 // Operand Classes are groups of operands that are used as to simplify
5744 // instruction definitions by not requiring the AD writer to specify separate
5745 // instructions for every form of operand when the instruction accepts
5746 // multiple operand types with the same basic encoding and format.  The classic
5747 // case of this is memory operands.
5748 
5749 opclass memory(direct, indirect, indOffset8, indOffset32, indOffset32X, indIndexOffset,
5750                indIndex, indIndexScale, indIndexScaleOffset);
5751 
5752 // Long memory operations are encoded in 2 instructions and a +4 offset.
5753 // This means some kind of offset is always required and you cannot use
5754 // an oop as the offset (done when working on static globals).
5755 opclass long_memory(direct, indirect, indOffset8, indOffset32, indIndexOffset,
5756                     indIndex, indIndexScale, indIndexScaleOffset);
5757 
5758 
5759 //----------PIPELINE-----------------------------------------------------------
5760 // Rules which define the behavior of the target architectures pipeline.
5761 pipeline %{
5762 
5763 //----------ATTRIBUTES---------------------------------------------------------
5764 attributes %{
5765   variable_size_instructions;        // Fixed size instructions
5766   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
5767   instruction_unit_size = 1;         // An instruction is 1 bytes long
5768   instruction_fetch_unit_size = 16;  // The processor fetches one line
5769   instruction_fetch_units = 1;       // of 16 bytes
5770 
5771   // List of nop instructions
5772   nops( MachNop );
5773 %}
5774 
5775 //----------RESOURCES----------------------------------------------------------
5776 // Resources are the functional units available to the machine
5777 
5778 // Generic P2/P3 pipeline
5779 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
5780 // 3 instructions decoded per cycle.
5781 // 2 load/store ops per cycle, 1 branch, 1 FPU,
5782 // 2 ALU op, only ALU0 handles mul/div instructions.
5783 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
5784            MS0, MS1, MEM = MS0 | MS1,
5785            BR, FPU,
5786            ALU0, ALU1, ALU = ALU0 | ALU1 );
5787 
5788 //----------PIPELINE DESCRIPTION-----------------------------------------------
5789 // Pipeline Description specifies the stages in the machine's pipeline
5790 
5791 // Generic P2/P3 pipeline
5792 pipe_desc(S0, S1, S2, S3, S4, S5);
5793 
5794 //----------PIPELINE CLASSES---------------------------------------------------
5795 // Pipeline Classes describe the stages in which input and output are
5796 // referenced by the hardware pipeline.
5797 
5798 // Naming convention: ialu or fpu
5799 // Then: _reg
5800 // Then: _reg if there is a 2nd register
5801 // Then: _long if it's a pair of instructions implementing a long
5802 // Then: _fat if it requires the big decoder
5803 //   Or: _mem if it requires the big decoder and a memory unit.
5804 
5805 // Integer ALU reg operation
5806 pipe_class ialu_reg(eRegI dst) %{
5807     single_instruction;
5808     dst    : S4(write);
5809     dst    : S3(read);
5810     DECODE : S0;        // any decoder
5811     ALU    : S3;        // any alu
5812 %}
5813 
5814 // Long ALU reg operation
5815 pipe_class ialu_reg_long(eRegL dst) %{
5816     instruction_count(2);
5817     dst    : S4(write);
5818     dst    : S3(read);
5819     DECODE : S0(2);     // any 2 decoders
5820     ALU    : S3(2);     // both alus
5821 %}
5822 
5823 // Integer ALU reg operation using big decoder
5824 pipe_class ialu_reg_fat(eRegI dst) %{
5825     single_instruction;
5826     dst    : S4(write);
5827     dst    : S3(read);
5828     D0     : S0;        // big decoder only
5829     ALU    : S3;        // any alu
5830 %}
5831 
5832 // Long ALU reg operation using big decoder
5833 pipe_class ialu_reg_long_fat(eRegL dst) %{
5834     instruction_count(2);
5835     dst    : S4(write);
5836     dst    : S3(read);
5837     D0     : S0(2);     // big decoder only; twice
5838     ALU    : S3(2);     // any 2 alus
5839 %}
5840 
5841 // Integer ALU reg-reg operation
5842 pipe_class ialu_reg_reg(eRegI dst, eRegI src) %{
5843     single_instruction;
5844     dst    : S4(write);
5845     src    : S3(read);
5846     DECODE : S0;        // any decoder
5847     ALU    : S3;        // any alu
5848 %}
5849 
5850 // Long ALU reg-reg operation
5851 pipe_class ialu_reg_reg_long(eRegL dst, eRegL src) %{
5852     instruction_count(2);
5853     dst    : S4(write);
5854     src    : S3(read);
5855     DECODE : S0(2);     // any 2 decoders
5856     ALU    : S3(2);     // both alus
5857 %}
5858 
5859 // Integer ALU reg-reg operation
5860 pipe_class ialu_reg_reg_fat(eRegI dst, memory src) %{
5861     single_instruction;
5862     dst    : S4(write);
5863     src    : S3(read);
5864     D0     : S0;        // big decoder only
5865     ALU    : S3;        // any alu
5866 %}
5867 
5868 // Long ALU reg-reg operation
5869 pipe_class ialu_reg_reg_long_fat(eRegL dst, eRegL src) %{
5870     instruction_count(2);
5871     dst    : S4(write);
5872     src    : S3(read);
5873     D0     : S0(2);     // big decoder only; twice
5874     ALU    : S3(2);     // both alus
5875 %}
5876 
5877 // Integer ALU reg-mem operation
5878 pipe_class ialu_reg_mem(eRegI dst, memory mem) %{
5879     single_instruction;
5880     dst    : S5(write);
5881     mem    : S3(read);
5882     D0     : S0;        // big decoder only
5883     ALU    : S4;        // any alu
5884     MEM    : S3;        // any mem
5885 %}
5886 
5887 // Long ALU reg-mem operation
5888 pipe_class ialu_reg_long_mem(eRegL dst, load_long_memory mem) %{
5889     instruction_count(2);
5890     dst    : S5(write);
5891     mem    : S3(read);
5892     D0     : S0(2);     // big decoder only; twice
5893     ALU    : S4(2);     // any 2 alus
5894     MEM    : S3(2);     // both mems
5895 %}
5896 
5897 // Integer mem operation (prefetch)
5898 pipe_class ialu_mem(memory mem)
5899 %{
5900     single_instruction;
5901     mem    : S3(read);
5902     D0     : S0;        // big decoder only
5903     MEM    : S3;        // any mem
5904 %}
5905 
5906 // Integer Store to Memory
5907 pipe_class ialu_mem_reg(memory mem, eRegI src) %{
5908     single_instruction;
5909     mem    : S3(read);
5910     src    : S5(read);
5911     D0     : S0;        // big decoder only
5912     ALU    : S4;        // any alu
5913     MEM    : S3;
5914 %}
5915 
5916 // Long Store to Memory
5917 pipe_class ialu_mem_long_reg(memory mem, eRegL src) %{
5918     instruction_count(2);
5919     mem    : S3(read);
5920     src    : S5(read);
5921     D0     : S0(2);     // big decoder only; twice
5922     ALU    : S4(2);     // any 2 alus
5923     MEM    : S3(2);     // Both mems
5924 %}
5925 
5926 // Integer Store to Memory
5927 pipe_class ialu_mem_imm(memory mem) %{
5928     single_instruction;
5929     mem    : S3(read);
5930     D0     : S0;        // big decoder only
5931     ALU    : S4;        // any alu
5932     MEM    : S3;
5933 %}
5934 
5935 // Integer ALU0 reg-reg operation
5936 pipe_class ialu_reg_reg_alu0(eRegI dst, eRegI src) %{
5937     single_instruction;
5938     dst    : S4(write);
5939     src    : S3(read);
5940     D0     : S0;        // Big decoder only
5941     ALU0   : S3;        // only alu0
5942 %}
5943 
5944 // Integer ALU0 reg-mem operation
5945 pipe_class ialu_reg_mem_alu0(eRegI dst, memory mem) %{
5946     single_instruction;
5947     dst    : S5(write);
5948     mem    : S3(read);
5949     D0     : S0;        // big decoder only
5950     ALU0   : S4;        // ALU0 only
5951     MEM    : S3;        // any mem
5952 %}
5953 
5954 // Integer ALU reg-reg operation
5955 pipe_class ialu_cr_reg_reg(eFlagsReg cr, eRegI src1, eRegI src2) %{
5956     single_instruction;
5957     cr     : S4(write);
5958     src1   : S3(read);
5959     src2   : S3(read);
5960     DECODE : S0;        // any decoder
5961     ALU    : S3;        // any alu
5962 %}
5963 
5964 // Integer ALU reg-imm operation
5965 pipe_class ialu_cr_reg_imm(eFlagsReg cr, eRegI src1) %{
5966     single_instruction;
5967     cr     : S4(write);
5968     src1   : S3(read);
5969     DECODE : S0;        // any decoder
5970     ALU    : S3;        // any alu
5971 %}
5972 
5973 // Integer ALU reg-mem operation
5974 pipe_class ialu_cr_reg_mem(eFlagsReg cr, eRegI src1, memory src2) %{
5975     single_instruction;
5976     cr     : S4(write);
5977     src1   : S3(read);
5978     src2   : S3(read);
5979     D0     : S0;        // big decoder only
5980     ALU    : S4;        // any alu
5981     MEM    : S3;
5982 %}
5983 
5984 // Conditional move reg-reg
5985 pipe_class pipe_cmplt( eRegI p, eRegI q, eRegI y ) %{
5986     instruction_count(4);
5987     y      : S4(read);
5988     q      : S3(read);
5989     p      : S3(read);
5990     DECODE : S0(4);     // any decoder
5991 %}
5992 
5993 // Conditional move reg-reg
5994 pipe_class pipe_cmov_reg( eRegI dst, eRegI src, eFlagsReg cr ) %{
5995     single_instruction;
5996     dst    : S4(write);
5997     src    : S3(read);
5998     cr     : S3(read);
5999     DECODE : S0;        // any decoder
6000 %}
6001 
6002 // Conditional move reg-mem
6003 pipe_class pipe_cmov_mem( eFlagsReg cr, eRegI dst, memory src) %{
6004     single_instruction;
6005     dst    : S4(write);
6006     src    : S3(read);
6007     cr     : S3(read);
6008     DECODE : S0;        // any decoder
6009     MEM    : S3;
6010 %}
6011 
6012 // Conditional move reg-reg long
6013 pipe_class pipe_cmov_reg_long( eFlagsReg cr, eRegL dst, eRegL src) %{
6014     single_instruction;
6015     dst    : S4(write);
6016     src    : S3(read);
6017     cr     : S3(read);
6018     DECODE : S0(2);     // any 2 decoders
6019 %}
6020 
6021 // Conditional move double reg-reg
6022 pipe_class pipe_cmovD_reg( eFlagsReg cr, regDPR1 dst, regD src) %{
6023     single_instruction;
6024     dst    : S4(write);
6025     src    : S3(read);
6026     cr     : S3(read);
6027     DECODE : S0;        // any decoder
6028 %}
6029 
6030 // Float reg-reg operation
6031 pipe_class fpu_reg(regD dst) %{
6032     instruction_count(2);
6033     dst    : S3(read);
6034     DECODE : S0(2);     // any 2 decoders
6035     FPU    : S3;
6036 %}
6037 
6038 // Float reg-reg operation
6039 pipe_class fpu_reg_reg(regD dst, regD src) %{
6040     instruction_count(2);
6041     dst    : S4(write);
6042     src    : S3(read);
6043     DECODE : S0(2);     // any 2 decoders
6044     FPU    : S3;
6045 %}
6046 
6047 // Float reg-reg operation
6048 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2) %{
6049     instruction_count(3);
6050     dst    : S4(write);
6051     src1   : S3(read);
6052     src2   : S3(read);
6053     DECODE : S0(3);     // any 3 decoders
6054     FPU    : S3(2);
6055 %}
6056 
6057 // Float reg-reg operation
6058 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3) %{
6059     instruction_count(4);
6060     dst    : S4(write);
6061     src1   : S3(read);
6062     src2   : S3(read);
6063     src3   : S3(read);
6064     DECODE : S0(4);     // any 3 decoders
6065     FPU    : S3(2);
6066 %}
6067 
6068 // Float reg-reg operation
6069 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3) %{
6070     instruction_count(4);
6071     dst    : S4(write);
6072     src1   : S3(read);
6073     src2   : S3(read);
6074     src3   : S3(read);
6075     DECODE : S1(3);     // any 3 decoders
6076     D0     : S0;        // Big decoder only
6077     FPU    : S3(2);
6078     MEM    : S3;
6079 %}
6080 
6081 // Float reg-mem operation
6082 pipe_class fpu_reg_mem(regD dst, memory mem) %{
6083     instruction_count(2);
6084     dst    : S5(write);
6085     mem    : S3(read);
6086     D0     : S0;        // big decoder only
6087     DECODE : S1;        // any decoder for FPU POP
6088     FPU    : S4;
6089     MEM    : S3;        // any mem
6090 %}
6091 
6092 // Float reg-mem operation
6093 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem) %{
6094     instruction_count(3);
6095     dst    : S5(write);
6096     src1   : S3(read);
6097     mem    : S3(read);
6098     D0     : S0;        // big decoder only
6099     DECODE : S1(2);     // any decoder for FPU POP
6100     FPU    : S4;
6101     MEM    : S3;        // any mem
6102 %}
6103 
6104 // Float mem-reg operation
6105 pipe_class fpu_mem_reg(memory mem, regD src) %{
6106     instruction_count(2);
6107     src    : S5(read);
6108     mem    : S3(read);
6109     DECODE : S0;        // any decoder for FPU PUSH
6110     D0     : S1;        // big decoder only
6111     FPU    : S4;
6112     MEM    : S3;        // any mem
6113 %}
6114 
6115 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2) %{
6116     instruction_count(3);
6117     src1   : S3(read);
6118     src2   : S3(read);
6119     mem    : S3(read);
6120     DECODE : S0(2);     // any decoder for FPU PUSH
6121     D0     : S1;        // big decoder only
6122     FPU    : S4;
6123     MEM    : S3;        // any mem
6124 %}
6125 
6126 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2) %{
6127     instruction_count(3);
6128     src1   : S3(read);
6129     src2   : S3(read);
6130     mem    : S4(read);
6131     DECODE : S0;        // any decoder for FPU PUSH
6132     D0     : S0(2);     // big decoder only
6133     FPU    : S4;
6134     MEM    : S3(2);     // any mem
6135 %}
6136 
6137 pipe_class fpu_mem_mem(memory dst, memory src1) %{
6138     instruction_count(2);
6139     src1   : S3(read);
6140     dst    : S4(read);
6141     D0     : S0(2);     // big decoder only
6142     MEM    : S3(2);     // any mem
6143 %}
6144 
6145 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2) %{
6146     instruction_count(3);
6147     src1   : S3(read);
6148     src2   : S3(read);
6149     dst    : S4(read);
6150     D0     : S0(3);     // big decoder only
6151     FPU    : S4;
6152     MEM    : S3(3);     // any mem
6153 %}
6154 
6155 pipe_class fpu_mem_reg_con(memory mem, regD src1) %{
6156     instruction_count(3);
6157     src1   : S4(read);
6158     mem    : S4(read);
6159     DECODE : S0;        // any decoder for FPU PUSH
6160     D0     : S0(2);     // big decoder only
6161     FPU    : S4;
6162     MEM    : S3(2);     // any mem
6163 %}
6164 
6165 // Float load constant
6166 pipe_class fpu_reg_con(regD dst) %{
6167     instruction_count(2);
6168     dst    : S5(write);
6169     D0     : S0;        // big decoder only for the load
6170     DECODE : S1;        // any decoder for FPU POP
6171     FPU    : S4;
6172     MEM    : S3;        // any mem
6173 %}
6174 
6175 // Float load constant
6176 pipe_class fpu_reg_reg_con(regD dst, regD src) %{
6177     instruction_count(3);
6178     dst    : S5(write);
6179     src    : S3(read);
6180     D0     : S0;        // big decoder only for the load
6181     DECODE : S1(2);     // any decoder for FPU POP
6182     FPU    : S4;
6183     MEM    : S3;        // any mem
6184 %}
6185 
6186 // UnConditional branch
6187 pipe_class pipe_jmp( label labl ) %{
6188     single_instruction;
6189     BR   : S3;
6190 %}
6191 
6192 // Conditional branch
6193 pipe_class pipe_jcc( cmpOp cmp, eFlagsReg cr, label labl ) %{
6194     single_instruction;
6195     cr    : S1(read);
6196     BR    : S3;
6197 %}
6198 
6199 // Allocation idiom
6200 pipe_class pipe_cmpxchg( eRegP dst, eRegP heap_ptr ) %{
6201     instruction_count(1); force_serialization;
6202     fixed_latency(6);
6203     heap_ptr : S3(read);
6204     DECODE   : S0(3);
6205     D0       : S2;
6206     MEM      : S3;
6207     ALU      : S3(2);
6208     dst      : S5(write);
6209     BR       : S5;
6210 %}
6211 
6212 // Generic big/slow expanded idiom
6213 pipe_class pipe_slow(  ) %{
6214     instruction_count(10); multiple_bundles; force_serialization;
6215     fixed_latency(100);
6216     D0  : S0(2);
6217     MEM : S3(2);
6218 %}
6219 
6220 // The real do-nothing guy
6221 pipe_class empty( ) %{
6222     instruction_count(0);
6223 %}
6224 
6225 // Define the class for the Nop node
6226 define %{
6227    MachNop = empty;
6228 %}
6229 
6230 %}
6231 
6232 //----------INSTRUCTIONS-------------------------------------------------------
6233 //
6234 // match      -- States which machine-independent subtree may be replaced
6235 //               by this instruction.
6236 // ins_cost   -- The estimated cost of this instruction is used by instruction
6237 //               selection to identify a minimum cost tree of machine
6238 //               instructions that matches a tree of machine-independent
6239 //               instructions.
6240 // format     -- A string providing the disassembly for this instruction.
6241 //               The value of an instruction's operand may be inserted
6242 //               by referring to it with a '$' prefix.
6243 // opcode     -- Three instruction opcodes may be provided.  These are referred
6244 //               to within an encode class as $primary, $secondary, and $tertiary
6245 //               respectively.  The primary opcode is commonly used to
6246 //               indicate the type of machine instruction, while secondary
6247 //               and tertiary are often used for prefix options or addressing
6248 //               modes.
6249 // ins_encode -- A list of encode classes with parameters. The encode class
6250 //               name must have been defined in an 'enc_class' specification
6251 //               in the encode section of the architecture description.
6252 
6253 //----------BSWAP-Instruction--------------------------------------------------
6254 instruct bytes_reverse_int(eRegI dst) %{
6255   match(Set dst (ReverseBytesI dst));
6256 
6257   format %{ "BSWAP  $dst" %}
6258   opcode(0x0F, 0xC8);
6259   ins_encode( OpcP, OpcSReg(dst) );
6260   ins_pipe( ialu_reg );
6261 %}
6262 
6263 instruct bytes_reverse_long(eRegL dst) %{
6264   match(Set dst (ReverseBytesL dst));
6265 
6266   format %{ "BSWAP  $dst.lo\n\t"
6267             "BSWAP  $dst.hi\n\t"
6268             "XCHG   $dst.lo $dst.hi" %}
6269 
6270   ins_cost(125);
6271   ins_encode( bswap_long_bytes(dst) );
6272   ins_pipe( ialu_reg_reg);
6273 %}
6274 
6275 instruct bytes_reverse_char(eRegI dst) %{
6276   match(Set dst (ReverseBytesC dst));
6277 
6278   format %{ "BSWAP  $dst\n\t" 
6279             "SHR    $dst,16\n\t" %}
6280   ins_encode %{
6281     __ bswapl($dst$$Register);
6282     __ shrl($dst$$Register, 16); 
6283   %}
6284   ins_pipe( ialu_reg );
6285 %}
6286 
6287 instruct bytes_reverse_short(eRegI dst) %{
6288   match(Set dst (ReverseBytesS dst));
6289 
6290   format %{ "BSWAP  $dst\n\t" 
6291             "SAR    $dst,16\n\t" %}
6292   ins_encode %{
6293     __ bswapl($dst$$Register);
6294     __ sarl($dst$$Register, 16); 
6295   %}
6296   ins_pipe( ialu_reg );
6297 %}
6298 
6299 
6300 //---------- Zeros Count Instructions ------------------------------------------
6301 
6302 instruct countLeadingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
6303   predicate(UseCountLeadingZerosInstruction);
6304   match(Set dst (CountLeadingZerosI src));
6305   effect(KILL cr);
6306 
6307   format %{ "LZCNT  $dst, $src\t# count leading zeros (int)" %}
6308   ins_encode %{
6309     __ lzcntl($dst$$Register, $src$$Register);
6310   %}
6311   ins_pipe(ialu_reg);
6312 %}
6313 
6314 instruct countLeadingZerosI_bsr(eRegI dst, eRegI src, eFlagsReg cr) %{
6315   predicate(!UseCountLeadingZerosInstruction);
6316   match(Set dst (CountLeadingZerosI src));
6317   effect(KILL cr);
6318 
6319   format %{ "BSR    $dst, $src\t# count leading zeros (int)\n\t"
6320             "JNZ    skip\n\t"
6321             "MOV    $dst, -1\n"
6322       "skip:\n\t"
6323             "NEG    $dst\n\t"
6324             "ADD    $dst, 31" %}
6325   ins_encode %{
6326     Register Rdst = $dst$$Register;
6327     Register Rsrc = $src$$Register;
6328     Label skip;
6329     __ bsrl(Rdst, Rsrc);
6330     __ jccb(Assembler::notZero, skip);
6331     __ movl(Rdst, -1);
6332     __ bind(skip);
6333     __ negl(Rdst);
6334     __ addl(Rdst, BitsPerInt - 1);
6335   %}
6336   ins_pipe(ialu_reg);
6337 %}
6338 
6339 instruct countLeadingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
6340   predicate(UseCountLeadingZerosInstruction);
6341   match(Set dst (CountLeadingZerosL src));
6342   effect(TEMP dst, KILL cr);
6343 
6344   format %{ "LZCNT  $dst, $src.hi\t# count leading zeros (long)\n\t"
6345             "JNC    done\n\t"
6346             "LZCNT  $dst, $src.lo\n\t"
6347             "ADD    $dst, 32\n"
6348       "done:" %}
6349   ins_encode %{
6350     Register Rdst = $dst$$Register;
6351     Register Rsrc = $src$$Register;
6352     Label done;
6353     __ lzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
6354     __ jccb(Assembler::carryClear, done);
6355     __ lzcntl(Rdst, Rsrc);
6356     __ addl(Rdst, BitsPerInt);
6357     __ bind(done);
6358   %}
6359   ins_pipe(ialu_reg);
6360 %}
6361 
6362 instruct countLeadingZerosL_bsr(eRegI dst, eRegL src, eFlagsReg cr) %{
6363   predicate(!UseCountLeadingZerosInstruction);
6364   match(Set dst (CountLeadingZerosL src));
6365   effect(TEMP dst, KILL cr);
6366 
6367   format %{ "BSR    $dst, $src.hi\t# count leading zeros (long)\n\t"
6368             "JZ     msw_is_zero\n\t"
6369             "ADD    $dst, 32\n\t"
6370             "JMP    not_zero\n"
6371       "msw_is_zero:\n\t"
6372             "BSR    $dst, $src.lo\n\t"
6373             "JNZ    not_zero\n\t"
6374             "MOV    $dst, -1\n"
6375       "not_zero:\n\t"
6376             "NEG    $dst\n\t"
6377             "ADD    $dst, 63\n" %}
6378  ins_encode %{
6379     Register Rdst = $dst$$Register;
6380     Register Rsrc = $src$$Register;
6381     Label msw_is_zero;
6382     Label not_zero;
6383     __ bsrl(Rdst, HIGH_FROM_LOW(Rsrc));
6384     __ jccb(Assembler::zero, msw_is_zero);
6385     __ addl(Rdst, BitsPerInt);
6386     __ jmpb(not_zero);
6387     __ bind(msw_is_zero);
6388     __ bsrl(Rdst, Rsrc);
6389     __ jccb(Assembler::notZero, not_zero);
6390     __ movl(Rdst, -1);
6391     __ bind(not_zero);
6392     __ negl(Rdst);
6393     __ addl(Rdst, BitsPerLong - 1);
6394   %}
6395   ins_pipe(ialu_reg);
6396 %}
6397 
6398 instruct countTrailingZerosI(eRegI dst, eRegI src, eFlagsReg cr) %{
6399   match(Set dst (CountTrailingZerosI src));
6400   effect(KILL cr);
6401 
6402   format %{ "BSF    $dst, $src\t# count trailing zeros (int)\n\t"
6403             "JNZ    done\n\t"
6404             "MOV    $dst, 32\n"
6405       "done:" %}
6406   ins_encode %{
6407     Register Rdst = $dst$$Register;
6408     Label done;
6409     __ bsfl(Rdst, $src$$Register);
6410     __ jccb(Assembler::notZero, done);
6411     __ movl(Rdst, BitsPerInt);
6412     __ bind(done);
6413   %}
6414   ins_pipe(ialu_reg);
6415 %}
6416 
6417 instruct countTrailingZerosL(eRegI dst, eRegL src, eFlagsReg cr) %{
6418   match(Set dst (CountTrailingZerosL src));
6419   effect(TEMP dst, KILL cr);
6420 
6421   format %{ "BSF    $dst, $src.lo\t# count trailing zeros (long)\n\t"
6422             "JNZ    done\n\t"
6423             "BSF    $dst, $src.hi\n\t"
6424             "JNZ    msw_not_zero\n\t"
6425             "MOV    $dst, 32\n"
6426       "msw_not_zero:\n\t"
6427             "ADD    $dst, 32\n"
6428       "done:" %}
6429   ins_encode %{
6430     Register Rdst = $dst$$Register;
6431     Register Rsrc = $src$$Register;
6432     Label msw_not_zero;
6433     Label done;
6434     __ bsfl(Rdst, Rsrc);
6435     __ jccb(Assembler::notZero, done);
6436     __ bsfl(Rdst, HIGH_FROM_LOW(Rsrc));
6437     __ jccb(Assembler::notZero, msw_not_zero);
6438     __ movl(Rdst, BitsPerInt);
6439     __ bind(msw_not_zero);
6440     __ addl(Rdst, BitsPerInt);
6441     __ bind(done);
6442   %}
6443   ins_pipe(ialu_reg);
6444 %}
6445 
6446 
6447 //---------- Population Count Instructions -------------------------------------
6448 
6449 instruct popCountI(eRegI dst, eRegI src) %{
6450   predicate(UsePopCountInstruction);
6451   match(Set dst (PopCountI src));
6452 
6453   format %{ "POPCNT $dst, $src" %}
6454   ins_encode %{
6455     __ popcntl($dst$$Register, $src$$Register);
6456   %}
6457   ins_pipe(ialu_reg);
6458 %}
6459 
6460 instruct popCountI_mem(eRegI dst, memory mem) %{
6461   predicate(UsePopCountInstruction);
6462   match(Set dst (PopCountI (LoadI mem)));
6463 
6464   format %{ "POPCNT $dst, $mem" %}
6465   ins_encode %{
6466     __ popcntl($dst$$Register, $mem$$Address);
6467   %}
6468   ins_pipe(ialu_reg);
6469 %}
6470 
6471 // Note: Long.bitCount(long) returns an int.
6472 instruct popCountL(eRegI dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
6473   predicate(UsePopCountInstruction);
6474   match(Set dst (PopCountL src));
6475   effect(KILL cr, TEMP tmp, TEMP dst);
6476 
6477   format %{ "POPCNT $dst, $src.lo\n\t"
6478             "POPCNT $tmp, $src.hi\n\t"
6479             "ADD    $dst, $tmp" %}
6480   ins_encode %{
6481     __ popcntl($dst$$Register, $src$$Register);
6482     __ popcntl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
6483     __ addl($dst$$Register, $tmp$$Register);
6484   %}
6485   ins_pipe(ialu_reg);
6486 %}
6487 
6488 // Note: Long.bitCount(long) returns an int.
6489 instruct popCountL_mem(eRegI dst, memory mem, eRegI tmp, eFlagsReg cr) %{
6490   predicate(UsePopCountInstruction);
6491   match(Set dst (PopCountL (LoadL mem)));
6492   effect(KILL cr, TEMP tmp, TEMP dst);
6493 
6494   format %{ "POPCNT $dst, $mem\n\t"
6495             "POPCNT $tmp, $mem+4\n\t"
6496             "ADD    $dst, $tmp" %}
6497   ins_encode %{
6498     //__ popcntl($dst$$Register, $mem$$Address$$first);
6499     //__ popcntl($tmp$$Register, $mem$$Address$$second);
6500     __ popcntl($dst$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false));
6501     __ popcntl($tmp$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false));
6502     __ addl($dst$$Register, $tmp$$Register);
6503   %}
6504   ins_pipe(ialu_reg);
6505 %}
6506 
6507 
6508 //----------Load/Store/Move Instructions---------------------------------------
6509 //----------Load Instructions--------------------------------------------------
6510 // Load Byte (8bit signed)
6511 instruct loadB(xRegI dst, memory mem) %{
6512   match(Set dst (LoadB mem));
6513 
6514   ins_cost(125);
6515   format %{ "MOVSX8 $dst,$mem\t# byte" %}
6516 
6517   ins_encode %{
6518     __ movsbl($dst$$Register, $mem$$Address);
6519   %}
6520 
6521   ins_pipe(ialu_reg_mem);
6522 %}
6523 
6524 // Load Byte (8bit signed) into Long Register
6525 instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
6526   match(Set dst (ConvI2L (LoadB mem)));
6527   effect(KILL cr);
6528 
6529   ins_cost(375);
6530   format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
6531             "MOV    $dst.hi,$dst.lo\n\t"
6532             "SAR    $dst.hi,7" %}
6533 
6534   ins_encode %{
6535     __ movsbl($dst$$Register, $mem$$Address);
6536     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6537     __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
6538   %}
6539 
6540   ins_pipe(ialu_reg_mem);
6541 %}
6542 
6543 // Load Unsigned Byte (8bit UNsigned)
6544 instruct loadUB(xRegI dst, memory mem) %{
6545   match(Set dst (LoadUB mem));
6546 
6547   ins_cost(125);
6548   format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
6549 
6550   ins_encode %{
6551     __ movzbl($dst$$Register, $mem$$Address);
6552   %}
6553 
6554   ins_pipe(ialu_reg_mem);
6555 %}
6556 
6557 // Load Unsigned Byte (8 bit UNsigned) into Long Register
6558 instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
6559   match(Set dst (ConvI2L (LoadUB mem)));
6560   effect(KILL cr);
6561 
6562   ins_cost(250);
6563   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
6564             "XOR    $dst.hi,$dst.hi" %}
6565 
6566   ins_encode %{
6567     Register Rdst = $dst$$Register;
6568     __ movzbl(Rdst, $mem$$Address);
6569     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6570   %}
6571 
6572   ins_pipe(ialu_reg_mem);
6573 %}
6574 
6575 // Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
6576 instruct loadUB2L_immI8(eRegL dst, memory mem, immI8 mask, eFlagsReg cr) %{
6577   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
6578   effect(KILL cr);
6579 
6580   format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 8-bit mask -> long\n\t"
6581             "XOR    $dst.hi,$dst.hi\n\t"
6582             "AND    $dst.lo,$mask" %}
6583   ins_encode %{
6584     Register Rdst = $dst$$Register;
6585     __ movzbl(Rdst, $mem$$Address);
6586     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6587     __ andl(Rdst, $mask$$constant);
6588   %}
6589   ins_pipe(ialu_reg_mem);
6590 %}
6591 
6592 // Load Short (16bit signed)
6593 instruct loadS(eRegI dst, memory mem) %{
6594   match(Set dst (LoadS mem));
6595 
6596   ins_cost(125);
6597   format %{ "MOVSX  $dst,$mem\t# short" %}
6598 
6599   ins_encode %{
6600     __ movswl($dst$$Register, $mem$$Address);
6601   %}
6602 
6603   ins_pipe(ialu_reg_mem);
6604 %}
6605 
6606 // Load Short (16 bit signed) to Byte (8 bit signed)
6607 instruct loadS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6608   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
6609 
6610   ins_cost(125);
6611   format %{ "MOVSX  $dst, $mem\t# short -> byte" %}
6612   ins_encode %{
6613     __ movsbl($dst$$Register, $mem$$Address);
6614   %}
6615   ins_pipe(ialu_reg_mem);
6616 %}
6617 
6618 // Load Short (16bit signed) into Long Register
6619 instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
6620   match(Set dst (ConvI2L (LoadS mem)));
6621   effect(KILL cr);
6622 
6623   ins_cost(375);
6624   format %{ "MOVSX  $dst.lo,$mem\t# short -> long\n\t"
6625             "MOV    $dst.hi,$dst.lo\n\t"
6626             "SAR    $dst.hi,15" %}
6627 
6628   ins_encode %{
6629     __ movswl($dst$$Register, $mem$$Address);
6630     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6631     __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
6632   %}
6633 
6634   ins_pipe(ialu_reg_mem);
6635 %}
6636 
6637 // Load Unsigned Short/Char (16bit unsigned)
6638 instruct loadUS(eRegI dst, memory mem) %{
6639   match(Set dst (LoadUS mem));
6640 
6641   ins_cost(125);
6642   format %{ "MOVZX  $dst,$mem\t# ushort/char -> int" %}
6643 
6644   ins_encode %{
6645     __ movzwl($dst$$Register, $mem$$Address);
6646   %}
6647 
6648   ins_pipe(ialu_reg_mem);
6649 %}
6650 
6651 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
6652 instruct loadUS2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6653   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
6654 
6655   ins_cost(125);
6656   format %{ "MOVSX  $dst, $mem\t# ushort -> byte" %}
6657   ins_encode %{
6658     __ movsbl($dst$$Register, $mem$$Address);
6659   %}
6660   ins_pipe(ialu_reg_mem);
6661 %}
6662 
6663 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
6664 instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
6665   match(Set dst (ConvI2L (LoadUS mem)));
6666   effect(KILL cr);
6667 
6668   ins_cost(250);
6669   format %{ "MOVZX  $dst.lo,$mem\t# ushort/char -> long\n\t"
6670             "XOR    $dst.hi,$dst.hi" %}
6671 
6672   ins_encode %{
6673     __ movzwl($dst$$Register, $mem$$Address);
6674     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
6675   %}
6676 
6677   ins_pipe(ialu_reg_mem);
6678 %}
6679 
6680 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
6681 instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
6682   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
6683   effect(KILL cr);
6684 
6685   format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
6686             "XOR    $dst.hi,$dst.hi" %}
6687   ins_encode %{
6688     Register Rdst = $dst$$Register;
6689     __ movzbl(Rdst, $mem$$Address);
6690     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6691   %}
6692   ins_pipe(ialu_reg_mem);
6693 %}
6694 
6695 // Load Unsigned Short/Char (16 bit UNsigned) with a 16-bit mask into Long Register
6696 instruct loadUS2L_immI16(eRegL dst, memory mem, immI16 mask, eFlagsReg cr) %{
6697   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
6698   effect(KILL cr);
6699 
6700   format %{ "MOVZX  $dst.lo, $mem\t# ushort/char & 16-bit mask -> long\n\t"
6701             "XOR    $dst.hi,$dst.hi\n\t"
6702             "AND    $dst.lo,$mask" %}
6703   ins_encode %{
6704     Register Rdst = $dst$$Register;
6705     __ movzwl(Rdst, $mem$$Address);
6706     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6707     __ andl(Rdst, $mask$$constant);
6708   %}
6709   ins_pipe(ialu_reg_mem);
6710 %}
6711 
6712 // Load Integer
6713 instruct loadI(eRegI dst, memory mem) %{
6714   match(Set dst (LoadI mem));
6715 
6716   ins_cost(125);
6717   format %{ "MOV    $dst,$mem\t# int" %}
6718 
6719   ins_encode %{
6720     __ movl($dst$$Register, $mem$$Address);
6721   %}
6722 
6723   ins_pipe(ialu_reg_mem);
6724 %}
6725 
6726 // Load Integer (32 bit signed) to Byte (8 bit signed)
6727 instruct loadI2B(eRegI dst, memory mem, immI_24 twentyfour) %{
6728   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
6729 
6730   ins_cost(125);
6731   format %{ "MOVSX  $dst, $mem\t# int -> byte" %}
6732   ins_encode %{
6733     __ movsbl($dst$$Register, $mem$$Address);
6734   %}
6735   ins_pipe(ialu_reg_mem);
6736 %}
6737 
6738 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
6739 instruct loadI2UB(eRegI dst, memory mem, immI_255 mask) %{
6740   match(Set dst (AndI (LoadI mem) mask));
6741 
6742   ins_cost(125);
6743   format %{ "MOVZX  $dst, $mem\t# int -> ubyte" %}
6744   ins_encode %{
6745     __ movzbl($dst$$Register, $mem$$Address);
6746   %}
6747   ins_pipe(ialu_reg_mem);
6748 %}
6749 
6750 // Load Integer (32 bit signed) to Short (16 bit signed)
6751 instruct loadI2S(eRegI dst, memory mem, immI_16 sixteen) %{
6752   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
6753 
6754   ins_cost(125);
6755   format %{ "MOVSX  $dst, $mem\t# int -> short" %}
6756   ins_encode %{
6757     __ movswl($dst$$Register, $mem$$Address);
6758   %}
6759   ins_pipe(ialu_reg_mem);
6760 %}
6761 
6762 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
6763 instruct loadI2US(eRegI dst, memory mem, immI_65535 mask) %{
6764   match(Set dst (AndI (LoadI mem) mask));
6765 
6766   ins_cost(125);
6767   format %{ "MOVZX  $dst, $mem\t# int -> ushort/char" %}
6768   ins_encode %{
6769     __ movzwl($dst$$Register, $mem$$Address);
6770   %}
6771   ins_pipe(ialu_reg_mem);
6772 %}
6773 
6774 // Load Integer into Long Register
6775 instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
6776   match(Set dst (ConvI2L (LoadI mem)));
6777   effect(KILL cr);
6778 
6779   ins_cost(375);
6780   format %{ "MOV    $dst.lo,$mem\t# int -> long\n\t"
6781             "MOV    $dst.hi,$dst.lo\n\t"
6782             "SAR    $dst.hi,31" %}
6783 
6784   ins_encode %{
6785     __ movl($dst$$Register, $mem$$Address);
6786     __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
6787     __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
6788   %}
6789 
6790   ins_pipe(ialu_reg_mem);
6791 %}
6792 
6793 // Load Integer with mask 0xFF into Long Register
6794 instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
6795   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6796   effect(KILL cr);
6797 
6798   format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
6799             "XOR    $dst.hi,$dst.hi" %}
6800   ins_encode %{
6801     Register Rdst = $dst$$Register;
6802     __ movzbl(Rdst, $mem$$Address);
6803     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6804   %}
6805   ins_pipe(ialu_reg_mem);
6806 %}
6807 
6808 // Load Integer with mask 0xFFFF into Long Register
6809 instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
6810   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6811   effect(KILL cr);
6812 
6813   format %{ "MOVZX  $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
6814             "XOR    $dst.hi,$dst.hi" %}
6815   ins_encode %{
6816     Register Rdst = $dst$$Register;
6817     __ movzwl(Rdst, $mem$$Address);
6818     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6819   %}
6820   ins_pipe(ialu_reg_mem);
6821 %}
6822 
6823 // Load Integer with 32-bit mask into Long Register
6824 instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
6825   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6826   effect(KILL cr);
6827 
6828   format %{ "MOV    $dst.lo,$mem\t# int & 32-bit mask -> long\n\t"
6829             "XOR    $dst.hi,$dst.hi\n\t"
6830             "AND    $dst.lo,$mask" %}
6831   ins_encode %{
6832     Register Rdst = $dst$$Register;
6833     __ movl(Rdst, $mem$$Address);
6834     __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
6835     __ andl(Rdst, $mask$$constant);
6836   %}
6837   ins_pipe(ialu_reg_mem);
6838 %}
6839 
6840 // Load Unsigned Integer into Long Register
6841 instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
6842   match(Set dst (LoadUI2L mem));
6843   effect(KILL cr);
6844 
6845   ins_cost(250);
6846   format %{ "MOV    $dst.lo,$mem\t# uint -> long\n\t"
6847             "XOR    $dst.hi,$dst.hi" %}
6848 
6849   ins_encode %{
6850     __ movl($dst$$Register, $mem$$Address);
6851     __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
6852   %}
6853 
6854   ins_pipe(ialu_reg_mem);
6855 %}
6856 
6857 // Load Long.  Cannot clobber address while loading, so restrict address
6858 // register to ESI
6859 instruct loadL(eRegL dst, load_long_memory mem) %{
6860   predicate(!((LoadLNode*)n)->require_atomic_access());
6861   match(Set dst (LoadL mem));
6862 
6863   ins_cost(250);
6864   format %{ "MOV    $dst.lo,$mem\t# long\n\t"
6865             "MOV    $dst.hi,$mem+4" %}
6866 
6867   ins_encode %{
6868     Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, false);
6869     Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, false);
6870     __ movl($dst$$Register, Amemlo);
6871     __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
6872   %}
6873 
6874   ins_pipe(ialu_reg_long_mem);
6875 %}
6876 
6877 // Volatile Load Long.  Must be atomic, so do 64-bit FILD
6878 // then store it down to the stack and reload on the int
6879 // side.
6880 instruct loadL_volatile(stackSlotL dst, memory mem) %{
6881   predicate(UseSSE<=1 && ((LoadLNode*)n)->require_atomic_access());
6882   match(Set dst (LoadL mem));
6883 
6884   ins_cost(200);
6885   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
6886             "FISTp  $dst" %}
6887   ins_encode(enc_loadL_volatile(mem,dst));
6888   ins_pipe( fpu_reg_mem );
6889 %}
6890 
6891 instruct loadLX_volatile(stackSlotL dst, memory mem, regXD tmp) %{
6892   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
6893   match(Set dst (LoadL mem));
6894   effect(TEMP tmp);
6895   ins_cost(180);
6896   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
6897             "MOVSD  $dst,$tmp" %}
6898   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
6899   ins_pipe( pipe_slow );
6900 %}
6901 
6902 instruct loadLX_reg_volatile(eRegL dst, memory mem, regXD tmp) %{
6903   predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
6904   match(Set dst (LoadL mem));
6905   effect(TEMP tmp);
6906   ins_cost(160);
6907   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
6908             "MOVD   $dst.lo,$tmp\n\t"
6909             "PSRLQ  $tmp,32\n\t"
6910             "MOVD   $dst.hi,$tmp" %}
6911   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
6912   ins_pipe( pipe_slow );
6913 %}
6914 
6915 // Load Range
6916 instruct loadRange(eRegI dst, memory mem) %{
6917   match(Set dst (LoadRange mem));
6918 
6919   ins_cost(125);
6920   format %{ "MOV    $dst,$mem" %}
6921   opcode(0x8B);
6922   ins_encode( OpcP, RegMem(dst,mem));
6923   ins_pipe( ialu_reg_mem );
6924 %}
6925 
6926 
6927 // Load Pointer
6928 instruct loadP(eRegP dst, memory mem) %{
6929   match(Set dst (LoadP mem));
6930 
6931   ins_cost(125);
6932   format %{ "MOV    $dst,$mem" %}
6933   opcode(0x8B);
6934   ins_encode( OpcP, RegMem(dst,mem));
6935   ins_pipe( ialu_reg_mem );
6936 %}
6937 
6938 // Load Klass Pointer
6939 instruct loadKlass(eRegP dst, memory mem) %{
6940   match(Set dst (LoadKlass mem));
6941 
6942   ins_cost(125);
6943   format %{ "MOV    $dst,$mem" %}
6944   opcode(0x8B);
6945   ins_encode( OpcP, RegMem(dst,mem));
6946   ins_pipe( ialu_reg_mem );
6947 %}
6948 
6949 // Load Double
6950 instruct loadD(regD dst, memory mem) %{
6951   predicate(UseSSE<=1);
6952   match(Set dst (LoadD mem));
6953 
6954   ins_cost(150);
6955   format %{ "FLD_D  ST,$mem\n\t"
6956             "FSTP   $dst" %}
6957   opcode(0xDD);               /* DD /0 */
6958   ins_encode( OpcP, RMopc_Mem(0x00,mem),
6959               Pop_Reg_D(dst) );
6960   ins_pipe( fpu_reg_mem );
6961 %}
6962 
6963 // Load Double to XMM
6964 instruct loadXD(regXD dst, memory mem) %{
6965   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
6966   match(Set dst (LoadD mem));
6967   ins_cost(145);
6968   format %{ "MOVSD  $dst,$mem" %}
6969   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
6970   ins_pipe( pipe_slow );
6971 %}
6972 
6973 instruct loadXD_partial(regXD dst, memory mem) %{
6974   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
6975   match(Set dst (LoadD mem));
6976   ins_cost(145);
6977   format %{ "MOVLPD $dst,$mem" %}
6978   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,mem));
6979   ins_pipe( pipe_slow );
6980 %}
6981 
6982 // Load to XMM register (single-precision floating point)
6983 // MOVSS instruction
6984 instruct loadX(regX dst, memory mem) %{
6985   predicate(UseSSE>=1);
6986   match(Set dst (LoadF mem));
6987   ins_cost(145);
6988   format %{ "MOVSS  $dst,$mem" %}
6989   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,mem));
6990   ins_pipe( pipe_slow );
6991 %}
6992 
6993 // Load Float
6994 instruct loadF(regF dst, memory mem) %{
6995   predicate(UseSSE==0);
6996   match(Set dst (LoadF mem));
6997 
6998   ins_cost(150);
6999   format %{ "FLD_S  ST,$mem\n\t"
7000             "FSTP   $dst" %}
7001   opcode(0xD9);               /* D9 /0 */
7002   ins_encode( OpcP, RMopc_Mem(0x00,mem),
7003               Pop_Reg_F(dst) );
7004   ins_pipe( fpu_reg_mem );
7005 %}
7006 
7007 // Load Aligned Packed Byte to XMM register
7008 instruct loadA8B(regXD dst, memory mem) %{
7009   predicate(UseSSE>=1);
7010   match(Set dst (Load8B mem));
7011   ins_cost(125);
7012   format %{ "MOVQ  $dst,$mem\t! packed8B" %}
7013   ins_encode( movq_ld(dst, mem));
7014   ins_pipe( pipe_slow );
7015 %}
7016 
7017 // Load Aligned Packed Short to XMM register
7018 instruct loadA4S(regXD dst, memory mem) %{
7019   predicate(UseSSE>=1);
7020   match(Set dst (Load4S mem));
7021   ins_cost(125);
7022   format %{ "MOVQ  $dst,$mem\t! packed4S" %}
7023   ins_encode( movq_ld(dst, mem));
7024   ins_pipe( pipe_slow );
7025 %}
7026 
7027 // Load Aligned Packed Char to XMM register
7028 instruct loadA4C(regXD dst, memory mem) %{
7029   predicate(UseSSE>=1);
7030   match(Set dst (Load4C mem));
7031   ins_cost(125);
7032   format %{ "MOVQ  $dst,$mem\t! packed4C" %}
7033   ins_encode( movq_ld(dst, mem));
7034   ins_pipe( pipe_slow );
7035 %}
7036 
7037 // Load Aligned Packed Integer to XMM register
7038 instruct load2IU(regXD dst, memory mem) %{
7039   predicate(UseSSE>=1);
7040   match(Set dst (Load2I mem));
7041   ins_cost(125);
7042   format %{ "MOVQ  $dst,$mem\t! packed2I" %}
7043   ins_encode( movq_ld(dst, mem));
7044   ins_pipe( pipe_slow );
7045 %}
7046 
7047 // Load Aligned Packed Single to XMM
7048 instruct loadA2F(regXD dst, memory mem) %{
7049   predicate(UseSSE>=1);
7050   match(Set dst (Load2F mem));
7051   ins_cost(145);
7052   format %{ "MOVQ  $dst,$mem\t! packed2F" %}
7053   ins_encode( movq_ld(dst, mem));
7054   ins_pipe( pipe_slow );
7055 %}
7056 
7057 // Load Effective Address
7058 instruct leaP8(eRegP dst, indOffset8 mem) %{
7059   match(Set dst mem);
7060 
7061   ins_cost(110);
7062   format %{ "LEA    $dst,$mem" %}
7063   opcode(0x8D);
7064   ins_encode( OpcP, RegMem(dst,mem));
7065   ins_pipe( ialu_reg_reg_fat );
7066 %}
7067 
7068 instruct leaP32(eRegP dst, indOffset32 mem) %{
7069   match(Set dst mem);
7070 
7071   ins_cost(110);
7072   format %{ "LEA    $dst,$mem" %}
7073   opcode(0x8D);
7074   ins_encode( OpcP, RegMem(dst,mem));
7075   ins_pipe( ialu_reg_reg_fat );
7076 %}
7077 
7078 instruct leaPIdxOff(eRegP dst, indIndexOffset mem) %{
7079   match(Set dst mem);
7080 
7081   ins_cost(110);
7082   format %{ "LEA    $dst,$mem" %}
7083   opcode(0x8D);
7084   ins_encode( OpcP, RegMem(dst,mem));
7085   ins_pipe( ialu_reg_reg_fat );
7086 %}
7087 
7088 instruct leaPIdxScale(eRegP dst, indIndexScale mem) %{
7089   match(Set dst mem);
7090 
7091   ins_cost(110);
7092   format %{ "LEA    $dst,$mem" %}
7093   opcode(0x8D);
7094   ins_encode( OpcP, RegMem(dst,mem));
7095   ins_pipe( ialu_reg_reg_fat );
7096 %}
7097 
7098 instruct leaPIdxScaleOff(eRegP dst, indIndexScaleOffset mem) %{
7099   match(Set dst mem);
7100 
7101   ins_cost(110);
7102   format %{ "LEA    $dst,$mem" %}
7103   opcode(0x8D);
7104   ins_encode( OpcP, RegMem(dst,mem));
7105   ins_pipe( ialu_reg_reg_fat );
7106 %}
7107 
7108 // Load Constant
7109 instruct loadConI(eRegI dst, immI src) %{
7110   match(Set dst src);
7111 
7112   format %{ "MOV    $dst,$src" %}
7113   ins_encode( LdImmI(dst, src) );
7114   ins_pipe( ialu_reg_fat );
7115 %}
7116 
7117 // Load Constant zero
7118 instruct loadConI0(eRegI dst, immI0 src, eFlagsReg cr) %{
7119   match(Set dst src);
7120   effect(KILL cr);
7121 
7122   ins_cost(50);
7123   format %{ "XOR    $dst,$dst" %}
7124   opcode(0x33);  /* + rd */
7125   ins_encode( OpcP, RegReg( dst, dst ) );
7126   ins_pipe( ialu_reg );
7127 %}
7128 
7129 instruct loadConP(eRegP dst, immP src) %{
7130   match(Set dst src);
7131 
7132   format %{ "MOV    $dst,$src" %}
7133   opcode(0xB8);  /* + rd */
7134   ins_encode( LdImmP(dst, src) );
7135   ins_pipe( ialu_reg_fat );
7136 %}
7137 
7138 instruct loadConL(eRegL dst, immL src, eFlagsReg cr) %{
7139   match(Set dst src);
7140   effect(KILL cr);
7141   ins_cost(200);
7142   format %{ "MOV    $dst.lo,$src.lo\n\t"
7143             "MOV    $dst.hi,$src.hi" %}
7144   opcode(0xB8);
7145   ins_encode( LdImmL_Lo(dst, src), LdImmL_Hi(dst, src) );
7146   ins_pipe( ialu_reg_long_fat );
7147 %}
7148 
7149 instruct loadConL0(eRegL dst, immL0 src, eFlagsReg cr) %{
7150   match(Set dst src);
7151   effect(KILL cr);
7152   ins_cost(150);
7153   format %{ "XOR    $dst.lo,$dst.lo\n\t"
7154             "XOR    $dst.hi,$dst.hi" %}
7155   opcode(0x33,0x33);
7156   ins_encode( RegReg_Lo(dst,dst), RegReg_Hi(dst, dst) );
7157   ins_pipe( ialu_reg_long );
7158 %}
7159 
7160 // The instruction usage is guarded by predicate in operand immF().
7161 instruct loadConF(regF dst, immF src) %{
7162   match(Set dst src);
7163   ins_cost(125);
7164 
7165   format %{ "FLD_S  ST,$src\n\t"
7166             "FSTP   $dst" %}
7167   opcode(0xD9, 0x00);       /* D9 /0 */
7168   ins_encode(LdImmF(src), Pop_Reg_F(dst) );
7169   ins_pipe( fpu_reg_con );
7170 %}
7171 
7172 // The instruction usage is guarded by predicate in operand immXF().
7173 instruct loadConX(regX dst, immXF con) %{
7174   match(Set dst con);
7175   ins_cost(125);
7176   format %{ "MOVSS  $dst,[$con]" %}
7177   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), LdImmX(dst, con));
7178   ins_pipe( pipe_slow );
7179 %}
7180 
7181 // The instruction usage is guarded by predicate in operand immXF0().
7182 instruct loadConX0(regX dst, immXF0 src) %{
7183   match(Set dst src);
7184   ins_cost(100);
7185   format %{ "XORPS  $dst,$dst\t# float 0.0" %}
7186   ins_encode( Opcode(0x0F), Opcode(0x57), RegReg(dst,dst));
7187   ins_pipe( pipe_slow );
7188 %}
7189 
7190 // The instruction usage is guarded by predicate in operand immD().
7191 instruct loadConD(regD dst, immD src) %{
7192   match(Set dst src);
7193   ins_cost(125);
7194 
7195   format %{ "FLD_D  ST,$src\n\t"
7196             "FSTP   $dst" %}
7197   ins_encode(LdImmD(src), Pop_Reg_D(dst) );
7198   ins_pipe( fpu_reg_con );
7199 %}
7200 
7201 // The instruction usage is guarded by predicate in operand immXD().
7202 instruct loadConXD(regXD dst, immXD con) %{
7203   match(Set dst con);
7204   ins_cost(125);
7205   format %{ "MOVSD  $dst,[$con]" %}
7206   ins_encode(load_conXD(dst, con));
7207   ins_pipe( pipe_slow );
7208 %}
7209 
7210 // The instruction usage is guarded by predicate in operand immXD0().
7211 instruct loadConXD0(regXD dst, immXD0 src) %{
7212   match(Set dst src);
7213   ins_cost(100);
7214   format %{ "XORPD  $dst,$dst\t# double 0.0" %}
7215   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x57), RegReg(dst,dst));
7216   ins_pipe( pipe_slow );
7217 %}
7218 
7219 // Load Stack Slot
7220 instruct loadSSI(eRegI dst, stackSlotI src) %{
7221   match(Set dst src);
7222   ins_cost(125);
7223 
7224   format %{ "MOV    $dst,$src" %}
7225   opcode(0x8B);
7226   ins_encode( OpcP, RegMem(dst,src));
7227   ins_pipe( ialu_reg_mem );
7228 %}
7229 
7230 instruct loadSSL(eRegL dst, stackSlotL src) %{
7231   match(Set dst src);
7232 
7233   ins_cost(200);
7234   format %{ "MOV    $dst,$src.lo\n\t"
7235             "MOV    $dst+4,$src.hi" %}
7236   opcode(0x8B, 0x8B);
7237   ins_encode( OpcP, RegMem( dst, src ), OpcS, RegMem_Hi( dst, src ) );
7238   ins_pipe( ialu_mem_long_reg );
7239 %}
7240 
7241 // Load Stack Slot
7242 instruct loadSSP(eRegP dst, stackSlotP src) %{
7243   match(Set dst src);
7244   ins_cost(125);
7245 
7246   format %{ "MOV    $dst,$src" %}
7247   opcode(0x8B);
7248   ins_encode( OpcP, RegMem(dst,src));
7249   ins_pipe( ialu_reg_mem );
7250 %}
7251 
7252 // Load Stack Slot
7253 instruct loadSSF(regF dst, stackSlotF src) %{
7254   match(Set dst src);
7255   ins_cost(125);
7256 
7257   format %{ "FLD_S  $src\n\t"
7258             "FSTP   $dst" %}
7259   opcode(0xD9);               /* D9 /0, FLD m32real */
7260   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
7261               Pop_Reg_F(dst) );
7262   ins_pipe( fpu_reg_mem );
7263 %}
7264 
7265 // Load Stack Slot
7266 instruct loadSSD(regD dst, stackSlotD src) %{
7267   match(Set dst src);
7268   ins_cost(125);
7269 
7270   format %{ "FLD_D  $src\n\t"
7271             "FSTP   $dst" %}
7272   opcode(0xDD);               /* DD /0, FLD m64real */
7273   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
7274               Pop_Reg_D(dst) );
7275   ins_pipe( fpu_reg_mem );
7276 %}
7277 
7278 // Prefetch instructions.
7279 // Must be safe to execute with invalid address (cannot fault).
7280 
7281 instruct prefetchr0( memory mem ) %{
7282   predicate(UseSSE==0 && !VM_Version::supports_3dnow());
7283   match(PrefetchRead mem);
7284   ins_cost(0);
7285   size(0);
7286   format %{ "PREFETCHR (non-SSE is empty encoding)" %}
7287   ins_encode();
7288   ins_pipe(empty);
7289 %}
7290 
7291 instruct prefetchr( memory mem ) %{
7292   predicate(UseSSE==0 && VM_Version::supports_3dnow() || ReadPrefetchInstr==3);
7293   match(PrefetchRead mem);
7294   ins_cost(100);
7295 
7296   format %{ "PREFETCHR $mem\t! Prefetch into level 1 cache for read" %}
7297   opcode(0x0F, 0x0d);     /* Opcode 0F 0d /0 */
7298   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
7299   ins_pipe(ialu_mem);
7300 %}
7301 
7302 instruct prefetchrNTA( memory mem ) %{
7303   predicate(UseSSE>=1 && ReadPrefetchInstr==0);
7304   match(PrefetchRead mem);
7305   ins_cost(100);
7306 
7307   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for read" %}
7308   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
7309   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
7310   ins_pipe(ialu_mem);
7311 %}
7312 
7313 instruct prefetchrT0( memory mem ) %{
7314   predicate(UseSSE>=1 && ReadPrefetchInstr==1);
7315   match(PrefetchRead mem);
7316   ins_cost(100);
7317 
7318   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for read" %}
7319   opcode(0x0F, 0x18);     /* Opcode 0F 18 /1 */
7320   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
7321   ins_pipe(ialu_mem);
7322 %}
7323 
7324 instruct prefetchrT2( memory mem ) %{
7325   predicate(UseSSE>=1 && ReadPrefetchInstr==2);
7326   match(PrefetchRead mem);
7327   ins_cost(100);
7328 
7329   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for read" %}
7330   opcode(0x0F, 0x18);     /* Opcode 0F 18 /3 */
7331   ins_encode(OpcP, OpcS, RMopc_Mem(0x03,mem));
7332   ins_pipe(ialu_mem);
7333 %}
7334 
7335 instruct prefetchw0( memory mem ) %{
7336   predicate(UseSSE==0 && !VM_Version::supports_3dnow());
7337   match(PrefetchWrite mem);
7338   ins_cost(0);
7339   size(0);
7340   format %{ "Prefetch (non-SSE is empty encoding)" %}
7341   ins_encode();
7342   ins_pipe(empty);
7343 %}
7344 
7345 instruct prefetchw( memory mem ) %{
7346   predicate(UseSSE==0 && VM_Version::supports_3dnow() || AllocatePrefetchInstr==3);
7347   match( PrefetchWrite mem );
7348   ins_cost(100);
7349 
7350   format %{ "PREFETCHW $mem\t! Prefetch into L1 cache and mark modified" %}
7351   opcode(0x0F, 0x0D);     /* Opcode 0F 0D /1 */
7352   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
7353   ins_pipe(ialu_mem);
7354 %}
7355 
7356 instruct prefetchwNTA( memory mem ) %{
7357   predicate(UseSSE>=1 && AllocatePrefetchInstr==0);
7358   match(PrefetchWrite mem);
7359   ins_cost(100);
7360 
7361   format %{ "PREFETCHNTA $mem\t! Prefetch into non-temporal cache for write" %}
7362   opcode(0x0F, 0x18);     /* Opcode 0F 18 /0 */
7363   ins_encode(OpcP, OpcS, RMopc_Mem(0x00,mem));
7364   ins_pipe(ialu_mem);
7365 %}
7366 
7367 instruct prefetchwT0( memory mem ) %{
7368   predicate(UseSSE>=1 && AllocatePrefetchInstr==1);
7369   match(PrefetchWrite mem);
7370   ins_cost(100);
7371 
7372   format %{ "PREFETCHT0 $mem\t! Prefetch into L1 and L2 caches for write" %}
7373   opcode(0x0F, 0x18);     /* Opcode 0F 18 /1 */
7374   ins_encode(OpcP, OpcS, RMopc_Mem(0x01,mem));
7375   ins_pipe(ialu_mem);
7376 %}
7377 
7378 instruct prefetchwT2( memory mem ) %{
7379   predicate(UseSSE>=1 && AllocatePrefetchInstr==2);
7380   match(PrefetchWrite mem);
7381   ins_cost(100);
7382 
7383   format %{ "PREFETCHT2 $mem\t! Prefetch into L2 cache for write" %}
7384   opcode(0x0F, 0x18);     /* Opcode 0F 18 /3 */
7385   ins_encode(OpcP, OpcS, RMopc_Mem(0x03,mem));
7386   ins_pipe(ialu_mem);
7387 %}
7388 
7389 //----------Store Instructions-------------------------------------------------
7390 
7391 // Store Byte
7392 instruct storeB(memory mem, xRegI src) %{
7393   match(Set mem (StoreB mem src));
7394 
7395   ins_cost(125);
7396   format %{ "MOV8   $mem,$src" %}
7397   opcode(0x88);
7398   ins_encode( OpcP, RegMem( src, mem ) );
7399   ins_pipe( ialu_mem_reg );
7400 %}
7401 
7402 // Store Char/Short
7403 instruct storeC(memory mem, eRegI src) %{
7404   match(Set mem (StoreC mem src));
7405 
7406   ins_cost(125);
7407   format %{ "MOV16  $mem,$src" %}
7408   opcode(0x89, 0x66);
7409   ins_encode( OpcS, OpcP, RegMem( src, mem ) );
7410   ins_pipe( ialu_mem_reg );
7411 %}
7412 
7413 // Store Integer
7414 instruct storeI(memory mem, eRegI src) %{
7415   match(Set mem (StoreI mem src));
7416 
7417   ins_cost(125);
7418   format %{ "MOV    $mem,$src" %}
7419   opcode(0x89);
7420   ins_encode( OpcP, RegMem( src, mem ) );
7421   ins_pipe( ialu_mem_reg );
7422 %}
7423 
7424 // Store Long
7425 instruct storeL(long_memory mem, eRegL src) %{
7426   predicate(!((StoreLNode*)n)->require_atomic_access());
7427   match(Set mem (StoreL mem src));
7428 
7429   ins_cost(200);
7430   format %{ "MOV    $mem,$src.lo\n\t"
7431             "MOV    $mem+4,$src.hi" %}
7432   opcode(0x89, 0x89);
7433   ins_encode( OpcP, RegMem( src, mem ), OpcS, RegMem_Hi( src, mem ) );
7434   ins_pipe( ialu_mem_long_reg );
7435 %}
7436 
7437 // Store Long to Integer
7438 instruct storeL2I(memory mem, eRegL src) %{
7439   match(Set mem (StoreI mem (ConvL2I src)));
7440 
7441   format %{ "MOV    $mem,$src.lo\t# long -> int" %}
7442   ins_encode %{
7443     __ movl($mem$$Address, $src$$Register);
7444   %}
7445   ins_pipe(ialu_mem_reg);
7446 %}
7447 
7448 // Volatile Store Long.  Must be atomic, so move it into
7449 // the FP TOS and then do a 64-bit FIST.  Has to probe the
7450 // target address before the store (for null-ptr checks)
7451 // so the memory operand is used twice in the encoding.
7452 instruct storeL_volatile(memory mem, stackSlotL src, eFlagsReg cr ) %{
7453   predicate(UseSSE<=1 && ((StoreLNode*)n)->require_atomic_access());
7454   match(Set mem (StoreL mem src));
7455   effect( KILL cr );
7456   ins_cost(400);
7457   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7458             "FILD   $src\n\t"
7459             "FISTp  $mem\t # 64-bit atomic volatile long store" %}
7460   opcode(0x3B);
7461   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeL_volatile(mem,src));
7462   ins_pipe( fpu_reg_mem );
7463 %}
7464 
7465 instruct storeLX_volatile(memory mem, stackSlotL src, regXD tmp, eFlagsReg cr) %{
7466   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
7467   match(Set mem (StoreL mem src));
7468   effect( TEMP tmp, KILL cr );
7469   ins_cost(380);
7470   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7471             "MOVSD  $tmp,$src\n\t"
7472             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
7473   opcode(0x3B);
7474   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_volatile(mem, src, tmp));
7475   ins_pipe( pipe_slow );
7476 %}
7477 
7478 instruct storeLX_reg_volatile(memory mem, eRegL src, regXD tmp2, regXD tmp, eFlagsReg cr) %{
7479   predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
7480   match(Set mem (StoreL mem src));
7481   effect( TEMP tmp2 , TEMP tmp, KILL cr );
7482   ins_cost(360);
7483   format %{ "CMP    $mem,EAX\t# Probe address for implicit null check\n\t"
7484             "MOVD   $tmp,$src.lo\n\t"
7485             "MOVD   $tmp2,$src.hi\n\t"
7486             "PUNPCKLDQ $tmp,$tmp2\n\t"
7487             "MOVSD  $mem,$tmp\t # 64-bit atomic volatile long store" %}
7488   opcode(0x3B);
7489   ins_encode( OpcP, RegMem( EAX, mem ), enc_storeLX_reg_volatile(mem, src, tmp, tmp2));
7490   ins_pipe( pipe_slow );
7491 %}
7492 
7493 // Store Pointer; for storing unknown oops and raw pointers
7494 instruct storeP(memory mem, anyRegP src) %{
7495   match(Set mem (StoreP mem src));
7496 
7497   ins_cost(125);
7498   format %{ "MOV    $mem,$src" %}
7499   opcode(0x89);
7500   ins_encode( OpcP, RegMem( src, mem ) );
7501   ins_pipe( ialu_mem_reg );
7502 %}
7503 
7504 // Store Integer Immediate
7505 instruct storeImmI(memory mem, immI src) %{
7506   match(Set mem (StoreI mem src));
7507 
7508   ins_cost(150);
7509   format %{ "MOV    $mem,$src" %}
7510   opcode(0xC7);               /* C7 /0 */
7511   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
7512   ins_pipe( ialu_mem_imm );
7513 %}
7514 
7515 // Store Short/Char Immediate
7516 instruct storeImmI16(memory mem, immI16 src) %{
7517   predicate(UseStoreImmI16);
7518   match(Set mem (StoreC mem src));
7519 
7520   ins_cost(150);
7521   format %{ "MOV16  $mem,$src" %}
7522   opcode(0xC7);     /* C7 /0 Same as 32 store immediate with prefix */
7523   ins_encode( SizePrefix, OpcP, RMopc_Mem(0x00,mem),  Con16( src ));
7524   ins_pipe( ialu_mem_imm );
7525 %}
7526 
7527 // Store Pointer Immediate; null pointers or constant oops that do not
7528 // need card-mark barriers.
7529 instruct storeImmP(memory mem, immP src) %{
7530   match(Set mem (StoreP mem src));
7531 
7532   ins_cost(150);
7533   format %{ "MOV    $mem,$src" %}
7534   opcode(0xC7);               /* C7 /0 */
7535   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32( src ));
7536   ins_pipe( ialu_mem_imm );
7537 %}
7538 
7539 // Store Byte Immediate
7540 instruct storeImmB(memory mem, immI8 src) %{
7541   match(Set mem (StoreB mem src));
7542 
7543   ins_cost(150);
7544   format %{ "MOV8   $mem,$src" %}
7545   opcode(0xC6);               /* C6 /0 */
7546   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
7547   ins_pipe( ialu_mem_imm );
7548 %}
7549 
7550 // Store Aligned Packed Byte XMM register to memory
7551 instruct storeA8B(memory mem, regXD src) %{
7552   predicate(UseSSE>=1);
7553   match(Set mem (Store8B mem src));
7554   ins_cost(145);
7555   format %{ "MOVQ  $mem,$src\t! packed8B" %}
7556   ins_encode( movq_st(mem, src));
7557   ins_pipe( pipe_slow );
7558 %}
7559 
7560 // Store Aligned Packed Char/Short XMM register to memory
7561 instruct storeA4C(memory mem, regXD src) %{
7562   predicate(UseSSE>=1);
7563   match(Set mem (Store4C mem src));
7564   ins_cost(145);
7565   format %{ "MOVQ  $mem,$src\t! packed4C" %}
7566   ins_encode( movq_st(mem, src));
7567   ins_pipe( pipe_slow );
7568 %}
7569 
7570 // Store Aligned Packed Integer XMM register to memory
7571 instruct storeA2I(memory mem, regXD src) %{
7572   predicate(UseSSE>=1);
7573   match(Set mem (Store2I mem src));
7574   ins_cost(145);
7575   format %{ "MOVQ  $mem,$src\t! packed2I" %}
7576   ins_encode( movq_st(mem, src));
7577   ins_pipe( pipe_slow );
7578 %}
7579 
7580 // Store CMS card-mark Immediate
7581 instruct storeImmCM(memory mem, immI8 src) %{
7582   match(Set mem (StoreCM mem src));
7583 
7584   ins_cost(150);
7585   format %{ "MOV8   $mem,$src\t! CMS card-mark imm0" %}
7586   opcode(0xC6);               /* C6 /0 */
7587   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con8or32( src ));
7588   ins_pipe( ialu_mem_imm );
7589 %}
7590 
7591 // Store Double
7592 instruct storeD( memory mem, regDPR1 src) %{
7593   predicate(UseSSE<=1);
7594   match(Set mem (StoreD mem src));
7595 
7596   ins_cost(100);
7597   format %{ "FST_D  $mem,$src" %}
7598   opcode(0xDD);       /* DD /2 */
7599   ins_encode( enc_FP_store(mem,src) );
7600   ins_pipe( fpu_mem_reg );
7601 %}
7602 
7603 // Store double does rounding on x86
7604 instruct storeD_rounded( memory mem, regDPR1 src) %{
7605   predicate(UseSSE<=1);
7606   match(Set mem (StoreD mem (RoundDouble src)));
7607 
7608   ins_cost(100);
7609   format %{ "FST_D  $mem,$src\t# round" %}
7610   opcode(0xDD);       /* DD /2 */
7611   ins_encode( enc_FP_store(mem,src) );
7612   ins_pipe( fpu_mem_reg );
7613 %}
7614 
7615 // Store XMM register to memory (double-precision floating points)
7616 // MOVSD instruction
7617 instruct storeXD(memory mem, regXD src) %{
7618   predicate(UseSSE>=2);
7619   match(Set mem (StoreD mem src));
7620   ins_cost(95);
7621   format %{ "MOVSD  $mem,$src" %}
7622   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
7623   ins_pipe( pipe_slow );
7624 %}
7625 
7626 // Store XMM register to memory (single-precision floating point)
7627 // MOVSS instruction
7628 instruct storeX(memory mem, regX src) %{
7629   predicate(UseSSE>=1);
7630   match(Set mem (StoreF mem src));
7631   ins_cost(95);
7632   format %{ "MOVSS  $mem,$src" %}
7633   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, mem));
7634   ins_pipe( pipe_slow );
7635 %}
7636 
7637 // Store Aligned Packed Single Float XMM register to memory
7638 instruct storeA2F(memory mem, regXD src) %{
7639   predicate(UseSSE>=1);
7640   match(Set mem (Store2F mem src));
7641   ins_cost(145);
7642   format %{ "MOVQ  $mem,$src\t! packed2F" %}
7643   ins_encode( movq_st(mem, src));
7644   ins_pipe( pipe_slow );
7645 %}
7646 
7647 // Store Float
7648 instruct storeF( memory mem, regFPR1 src) %{
7649   predicate(UseSSE==0);
7650   match(Set mem (StoreF mem src));
7651 
7652   ins_cost(100);
7653   format %{ "FST_S  $mem,$src" %}
7654   opcode(0xD9);       /* D9 /2 */
7655   ins_encode( enc_FP_store(mem,src) );
7656   ins_pipe( fpu_mem_reg );
7657 %}
7658 
7659 // Store Float does rounding on x86
7660 instruct storeF_rounded( memory mem, regFPR1 src) %{
7661   predicate(UseSSE==0);
7662   match(Set mem (StoreF mem (RoundFloat src)));
7663 
7664   ins_cost(100);
7665   format %{ "FST_S  $mem,$src\t# round" %}
7666   opcode(0xD9);       /* D9 /2 */
7667   ins_encode( enc_FP_store(mem,src) );
7668   ins_pipe( fpu_mem_reg );
7669 %}
7670 
7671 // Store Float does rounding on x86
7672 instruct storeF_Drounded( memory mem, regDPR1 src) %{
7673   predicate(UseSSE<=1);
7674   match(Set mem (StoreF mem (ConvD2F src)));
7675 
7676   ins_cost(100);
7677   format %{ "FST_S  $mem,$src\t# D-round" %}
7678   opcode(0xD9);       /* D9 /2 */
7679   ins_encode( enc_FP_store(mem,src) );
7680   ins_pipe( fpu_mem_reg );
7681 %}
7682 
7683 // Store immediate Float value (it is faster than store from FPU register)
7684 // The instruction usage is guarded by predicate in operand immF().
7685 instruct storeF_imm( memory mem, immF src) %{
7686   match(Set mem (StoreF mem src));
7687 
7688   ins_cost(50);
7689   format %{ "MOV    $mem,$src\t# store float" %}
7690   opcode(0xC7);               /* C7 /0 */
7691   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32F_as_bits( src ));
7692   ins_pipe( ialu_mem_imm );
7693 %}
7694 
7695 // Store immediate Float value (it is faster than store from XMM register)
7696 // The instruction usage is guarded by predicate in operand immXF().
7697 instruct storeX_imm( memory mem, immXF src) %{
7698   match(Set mem (StoreF mem src));
7699 
7700   ins_cost(50);
7701   format %{ "MOV    $mem,$src\t# store float" %}
7702   opcode(0xC7);               /* C7 /0 */
7703   ins_encode( OpcP, RMopc_Mem(0x00,mem),  Con32XF_as_bits( src ));
7704   ins_pipe( ialu_mem_imm );
7705 %}
7706 
7707 // Store Integer to stack slot
7708 instruct storeSSI(stackSlotI dst, eRegI src) %{
7709   match(Set dst src);
7710 
7711   ins_cost(100);
7712   format %{ "MOV    $dst,$src" %}
7713   opcode(0x89);
7714   ins_encode( OpcPRegSS( dst, src ) );
7715   ins_pipe( ialu_mem_reg );
7716 %}
7717 
7718 // Store Integer to stack slot
7719 instruct storeSSP(stackSlotP dst, eRegP src) %{
7720   match(Set dst src);
7721 
7722   ins_cost(100);
7723   format %{ "MOV    $dst,$src" %}
7724   opcode(0x89);
7725   ins_encode( OpcPRegSS( dst, src ) );
7726   ins_pipe( ialu_mem_reg );
7727 %}
7728 
7729 // Store Long to stack slot
7730 instruct storeSSL(stackSlotL dst, eRegL src) %{
7731   match(Set dst src);
7732 
7733   ins_cost(200);
7734   format %{ "MOV    $dst,$src.lo\n\t"
7735             "MOV    $dst+4,$src.hi" %}
7736   opcode(0x89, 0x89);
7737   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
7738   ins_pipe( ialu_mem_long_reg );
7739 %}
7740 
7741 //----------MemBar Instructions-----------------------------------------------
7742 // Memory barrier flavors
7743 
7744 instruct membar_acquire() %{
7745   match(MemBarAcquire);
7746   ins_cost(400);
7747 
7748   size(0);
7749   format %{ "MEMBAR-acquire ! (empty encoding)" %}
7750   ins_encode();
7751   ins_pipe(empty);
7752 %}
7753 
7754 instruct membar_acquire_lock() %{
7755   match(MemBarAcquire);
7756   predicate(Matcher::prior_fast_lock(n));
7757   ins_cost(0);
7758 
7759   size(0);
7760   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
7761   ins_encode( );
7762   ins_pipe(empty);
7763 %}
7764 
7765 instruct membar_release() %{
7766   match(MemBarRelease);
7767   ins_cost(400);
7768 
7769   size(0);
7770   format %{ "MEMBAR-release ! (empty encoding)" %}
7771   ins_encode( );
7772   ins_pipe(empty);
7773 %}
7774 
7775 instruct membar_release_lock() %{
7776   match(MemBarRelease);
7777   predicate(Matcher::post_fast_unlock(n));
7778   ins_cost(0);
7779 
7780   size(0);
7781   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
7782   ins_encode( );
7783   ins_pipe(empty);
7784 %}
7785 
7786 instruct membar_volatile(eFlagsReg cr) %{
7787   match(MemBarVolatile);
7788   effect(KILL cr);
7789   ins_cost(400);
7790 
7791   format %{ 
7792     $$template
7793     if (os::is_MP()) {
7794       $$emit$$"LOCK ADDL [ESP + #0], 0\t! membar_volatile"
7795     } else {
7796       $$emit$$"MEMBAR-volatile ! (empty encoding)"
7797     }
7798   %}
7799   ins_encode %{
7800     __ membar(Assembler::StoreLoad);
7801   %}
7802   ins_pipe(pipe_slow);
7803 %}
7804 
7805 instruct unnecessary_membar_volatile() %{
7806   match(MemBarVolatile);
7807   predicate(Matcher::post_store_load_barrier(n));
7808   ins_cost(0);
7809 
7810   size(0);
7811   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
7812   ins_encode( );
7813   ins_pipe(empty);
7814 %}
7815 
7816 //----------Move Instructions--------------------------------------------------
7817 instruct castX2P(eAXRegP dst, eAXRegI src) %{
7818   match(Set dst (CastX2P src));
7819   format %{ "# X2P  $dst, $src" %}
7820   ins_encode( /*empty encoding*/ );
7821   ins_cost(0);
7822   ins_pipe(empty);
7823 %}
7824 
7825 instruct castP2X(eRegI dst, eRegP src ) %{
7826   match(Set dst (CastP2X src));
7827   ins_cost(50);
7828   format %{ "MOV    $dst, $src\t# CastP2X" %}
7829   ins_encode( enc_Copy( dst, src) );
7830   ins_pipe( ialu_reg_reg );
7831 %}
7832 
7833 //----------Conditional Move---------------------------------------------------
7834 // Conditional move
7835 instruct cmovI_reg(eRegI dst, eRegI src, eFlagsReg cr, cmpOp cop ) %{
7836   predicate(VM_Version::supports_cmov() );
7837   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7838   ins_cost(200);
7839   format %{ "CMOV$cop $dst,$src" %}
7840   opcode(0x0F,0x40);
7841   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7842   ins_pipe( pipe_cmov_reg );
7843 %}
7844 
7845 instruct cmovI_regU( cmpOpU cop, eFlagsRegU cr, eRegI dst, eRegI src ) %{
7846   predicate(VM_Version::supports_cmov() );
7847   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7848   ins_cost(200);
7849   format %{ "CMOV$cop $dst,$src" %}
7850   opcode(0x0F,0x40);
7851   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7852   ins_pipe( pipe_cmov_reg );
7853 %}
7854 
7855 instruct cmovI_regUCF( cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, eRegI src ) %{
7856   predicate(VM_Version::supports_cmov() );
7857   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7858   ins_cost(200);
7859   expand %{
7860     cmovI_regU(cop, cr, dst, src);
7861   %}
7862 %}
7863 
7864 // Conditional move
7865 instruct cmovI_mem(cmpOp cop, eFlagsReg cr, eRegI dst, memory src) %{
7866   predicate(VM_Version::supports_cmov() );
7867   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7868   ins_cost(250);
7869   format %{ "CMOV$cop $dst,$src" %}
7870   opcode(0x0F,0x40);
7871   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
7872   ins_pipe( pipe_cmov_mem );
7873 %}
7874 
7875 // Conditional move
7876 instruct cmovI_memU(cmpOpU cop, eFlagsRegU cr, eRegI dst, memory src) %{
7877   predicate(VM_Version::supports_cmov() );
7878   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7879   ins_cost(250);
7880   format %{ "CMOV$cop $dst,$src" %}
7881   opcode(0x0F,0x40);
7882   ins_encode( enc_cmov(cop), RegMem( dst, src ) );
7883   ins_pipe( pipe_cmov_mem );
7884 %}
7885 
7886 instruct cmovI_memUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegI dst, memory src) %{
7887   predicate(VM_Version::supports_cmov() );
7888   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7889   ins_cost(250);
7890   expand %{
7891     cmovI_memU(cop, cr, dst, src);
7892   %}
7893 %}
7894 
7895 // Conditional move
7896 instruct cmovP_reg(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
7897   predicate(VM_Version::supports_cmov() );
7898   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7899   ins_cost(200);
7900   format %{ "CMOV$cop $dst,$src\t# ptr" %}
7901   opcode(0x0F,0x40);
7902   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7903   ins_pipe( pipe_cmov_reg );
7904 %}
7905 
7906 // Conditional move (non-P6 version)
7907 // Note:  a CMoveP is generated for  stubs and native wrappers
7908 //        regardless of whether we are on a P6, so we
7909 //        emulate a cmov here
7910 instruct cmovP_reg_nonP6(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
7911   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7912   ins_cost(300);
7913   format %{ "Jn$cop   skip\n\t"
7914           "MOV    $dst,$src\t# pointer\n"
7915       "skip:" %}
7916   opcode(0x8b);
7917   ins_encode( enc_cmov_branch(cop, 0x2), OpcP, RegReg(dst, src));
7918   ins_pipe( pipe_cmov_reg );
7919 %}
7920 
7921 // Conditional move
7922 instruct cmovP_regU(cmpOpU cop, eFlagsRegU cr, eRegP dst, eRegP src ) %{
7923   predicate(VM_Version::supports_cmov() );
7924   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7925   ins_cost(200);
7926   format %{ "CMOV$cop $dst,$src\t# ptr" %}
7927   opcode(0x0F,0x40);
7928   ins_encode( enc_cmov(cop), RegReg( dst, src ) );
7929   ins_pipe( pipe_cmov_reg );
7930 %}
7931 
7932 instruct cmovP_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegP dst, eRegP src ) %{
7933   predicate(VM_Version::supports_cmov() );
7934   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7935   ins_cost(200);
7936   expand %{
7937     cmovP_regU(cop, cr, dst, src);
7938   %}
7939 %}
7940 
7941 // DISABLED: Requires the ADLC to emit a bottom_type call that
7942 // correctly meets the two pointer arguments; one is an incoming
7943 // register but the other is a memory operand.  ALSO appears to
7944 // be buggy with implicit null checks.
7945 //
7946 //// Conditional move
7947 //instruct cmovP_mem(cmpOp cop, eFlagsReg cr, eRegP dst, memory src) %{
7948 //  predicate(VM_Version::supports_cmov() );
7949 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7950 //  ins_cost(250);
7951 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7952 //  opcode(0x0F,0x40);
7953 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
7954 //  ins_pipe( pipe_cmov_mem );
7955 //%}
7956 //
7957 //// Conditional move
7958 //instruct cmovP_memU(cmpOpU cop, eFlagsRegU cr, eRegP dst, memory src) %{
7959 //  predicate(VM_Version::supports_cmov() );
7960 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7961 //  ins_cost(250);
7962 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7963 //  opcode(0x0F,0x40);
7964 //  ins_encode( enc_cmov(cop), RegMem( dst, src ) );
7965 //  ins_pipe( pipe_cmov_mem );
7966 //%}
7967 
7968 // Conditional move
7969 instruct fcmovD_regU(cmpOp_fcmov cop, eFlagsRegU cr, regDPR1 dst, regD src) %{
7970   predicate(UseSSE<=1);
7971   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7972   ins_cost(200);
7973   format %{ "FCMOV$cop $dst,$src\t# double" %}
7974   opcode(0xDA);
7975   ins_encode( enc_cmov_d(cop,src) );
7976   ins_pipe( pipe_cmovD_reg );
7977 %}
7978 
7979 // Conditional move
7980 instruct fcmovF_regU(cmpOp_fcmov cop, eFlagsRegU cr, regFPR1 dst, regF src) %{
7981   predicate(UseSSE==0);
7982   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7983   ins_cost(200);
7984   format %{ "FCMOV$cop $dst,$src\t# float" %}
7985   opcode(0xDA);
7986   ins_encode( enc_cmov_d(cop,src) );
7987   ins_pipe( pipe_cmovD_reg );
7988 %}
7989 
7990 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
7991 instruct fcmovD_regS(cmpOp cop, eFlagsReg cr, regD dst, regD src) %{
7992   predicate(UseSSE<=1);
7993   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7994   ins_cost(200);
7995   format %{ "Jn$cop   skip\n\t"
7996             "MOV    $dst,$src\t# double\n"
7997       "skip:" %}
7998   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
7999   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_D(src), OpcP, RegOpc(dst) );
8000   ins_pipe( pipe_cmovD_reg );
8001 %}
8002 
8003 // Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
8004 instruct fcmovF_regS(cmpOp cop, eFlagsReg cr, regF dst, regF src) %{
8005   predicate(UseSSE==0);
8006   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8007   ins_cost(200);
8008   format %{ "Jn$cop    skip\n\t"
8009             "MOV    $dst,$src\t# float\n"
8010       "skip:" %}
8011   opcode (0xdd, 0x3);     /* DD D8+i or DD /3 */
8012   ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_F(src), OpcP, RegOpc(dst) );
8013   ins_pipe( pipe_cmovD_reg );
8014 %}
8015 
8016 // No CMOVE with SSE/SSE2
8017 instruct fcmovX_regS(cmpOp cop, eFlagsReg cr, regX dst, regX src) %{
8018   predicate (UseSSE>=1);
8019   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8020   ins_cost(200);
8021   format %{ "Jn$cop   skip\n\t"
8022             "MOVSS  $dst,$src\t# float\n"
8023       "skip:" %}
8024   ins_encode %{
8025     Label skip;
8026     // Invert sense of branch from sense of CMOV
8027     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8028     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
8029     __ bind(skip);
8030   %}
8031   ins_pipe( pipe_slow );
8032 %}
8033 
8034 // No CMOVE with SSE/SSE2
8035 instruct fcmovXD_regS(cmpOp cop, eFlagsReg cr, regXD dst, regXD src) %{
8036   predicate (UseSSE>=2);
8037   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8038   ins_cost(200);
8039   format %{ "Jn$cop   skip\n\t"
8040             "MOVSD  $dst,$src\t# float\n"
8041       "skip:" %}
8042   ins_encode %{
8043     Label skip;
8044     // Invert sense of branch from sense of CMOV
8045     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8046     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
8047     __ bind(skip);
8048   %}
8049   ins_pipe( pipe_slow );
8050 %}
8051 
8052 // unsigned version
8053 instruct fcmovX_regU(cmpOpU cop, eFlagsRegU cr, regX dst, regX src) %{
8054   predicate (UseSSE>=1);
8055   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8056   ins_cost(200);
8057   format %{ "Jn$cop   skip\n\t"
8058             "MOVSS  $dst,$src\t# float\n"
8059       "skip:" %}
8060   ins_encode %{
8061     Label skip;
8062     // Invert sense of branch from sense of CMOV
8063     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8064     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
8065     __ bind(skip);
8066   %}
8067   ins_pipe( pipe_slow );
8068 %}
8069 
8070 instruct fcmovX_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regX dst, regX src) %{
8071   predicate (UseSSE>=1);
8072   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
8073   ins_cost(200);
8074   expand %{
8075     fcmovX_regU(cop, cr, dst, src);
8076   %}
8077 %}
8078 
8079 // unsigned version
8080 instruct fcmovXD_regU(cmpOpU cop, eFlagsRegU cr, regXD dst, regXD src) %{
8081   predicate (UseSSE>=2);
8082   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8083   ins_cost(200);
8084   format %{ "Jn$cop   skip\n\t"
8085             "MOVSD  $dst,$src\t# float\n"
8086       "skip:" %}
8087   ins_encode %{
8088     Label skip;
8089     // Invert sense of branch from sense of CMOV
8090     __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
8091     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
8092     __ bind(skip);
8093   %}
8094   ins_pipe( pipe_slow );
8095 %}
8096 
8097 instruct fcmovXD_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regXD dst, regXD src) %{
8098   predicate (UseSSE>=2);
8099   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
8100   ins_cost(200);
8101   expand %{
8102     fcmovXD_regU(cop, cr, dst, src);
8103   %}
8104 %}
8105 
8106 instruct cmovL_reg(cmpOp cop, eFlagsReg cr, eRegL dst, eRegL src) %{
8107   predicate(VM_Version::supports_cmov() );
8108   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8109   ins_cost(200);
8110   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
8111             "CMOV$cop $dst.hi,$src.hi" %}
8112   opcode(0x0F,0x40);
8113   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
8114   ins_pipe( pipe_cmov_reg_long );
8115 %}
8116 
8117 instruct cmovL_regU(cmpOpU cop, eFlagsRegU cr, eRegL dst, eRegL src) %{
8118   predicate(VM_Version::supports_cmov() );
8119   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8120   ins_cost(200);
8121   format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
8122             "CMOV$cop $dst.hi,$src.hi" %}
8123   opcode(0x0F,0x40);
8124   ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
8125   ins_pipe( pipe_cmov_reg_long );
8126 %}
8127 
8128 instruct cmovL_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegL dst, eRegL src) %{
8129   predicate(VM_Version::supports_cmov() );
8130   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
8131   ins_cost(200);
8132   expand %{
8133     cmovL_regU(cop, cr, dst, src);
8134   %}
8135 %}
8136 
8137 //----------Arithmetic Instructions--------------------------------------------
8138 //----------Addition Instructions----------------------------------------------
8139 // Integer Addition Instructions
8140 instruct addI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8141   match(Set dst (AddI dst src));
8142   effect(KILL cr);
8143 
8144   size(2);
8145   format %{ "ADD    $dst,$src" %}
8146   opcode(0x03);
8147   ins_encode( OpcP, RegReg( dst, src) );
8148   ins_pipe( ialu_reg_reg );
8149 %}
8150 
8151 instruct addI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
8152   match(Set dst (AddI dst src));
8153   effect(KILL cr);
8154 
8155   format %{ "ADD    $dst,$src" %}
8156   opcode(0x81, 0x00); /* /0 id */
8157   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8158   ins_pipe( ialu_reg );
8159 %}
8160 
8161 instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
8162   predicate(UseIncDec);
8163   match(Set dst (AddI dst src));
8164   effect(KILL cr);
8165 
8166   size(1);
8167   format %{ "INC    $dst" %}
8168   opcode(0x40); /*  */
8169   ins_encode( Opc_plus( primary, dst ) );
8170   ins_pipe( ialu_reg );
8171 %}
8172 
8173 instruct leaI_eReg_immI(eRegI dst, eRegI src0, immI src1) %{
8174   match(Set dst (AddI src0 src1));
8175   ins_cost(110);
8176 
8177   format %{ "LEA    $dst,[$src0 + $src1]" %}
8178   opcode(0x8D); /* 0x8D /r */
8179   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
8180   ins_pipe( ialu_reg_reg );
8181 %}
8182 
8183 instruct leaP_eReg_immI(eRegP dst, eRegP src0, immI src1) %{
8184   match(Set dst (AddP src0 src1));
8185   ins_cost(110);
8186 
8187   format %{ "LEA    $dst,[$src0 + $src1]\t# ptr" %}
8188   opcode(0x8D); /* 0x8D /r */
8189   ins_encode( OpcP, RegLea( dst, src0, src1 ) );
8190   ins_pipe( ialu_reg_reg );
8191 %}
8192 
8193 instruct decI_eReg(eRegI dst, immI_M1 src, eFlagsReg cr) %{
8194   predicate(UseIncDec);
8195   match(Set dst (AddI dst src));
8196   effect(KILL cr);
8197 
8198   size(1);
8199   format %{ "DEC    $dst" %}
8200   opcode(0x48); /*  */
8201   ins_encode( Opc_plus( primary, dst ) );
8202   ins_pipe( ialu_reg );
8203 %}
8204 
8205 instruct addP_eReg(eRegP dst, eRegI src, eFlagsReg cr) %{
8206   match(Set dst (AddP dst src));
8207   effect(KILL cr);
8208 
8209   size(2);
8210   format %{ "ADD    $dst,$src" %}
8211   opcode(0x03);
8212   ins_encode( OpcP, RegReg( dst, src) );
8213   ins_pipe( ialu_reg_reg );
8214 %}
8215 
8216 instruct addP_eReg_imm(eRegP dst, immI src, eFlagsReg cr) %{
8217   match(Set dst (AddP dst src));
8218   effect(KILL cr);
8219 
8220   format %{ "ADD    $dst,$src" %}
8221   opcode(0x81,0x00); /* Opcode 81 /0 id */
8222   // ins_encode( RegImm( dst, src) );
8223   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8224   ins_pipe( ialu_reg );
8225 %}
8226 
8227 instruct addI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
8228   match(Set dst (AddI dst (LoadI src)));
8229   effect(KILL cr);
8230 
8231   ins_cost(125);
8232   format %{ "ADD    $dst,$src" %}
8233   opcode(0x03);
8234   ins_encode( OpcP, RegMem( dst, src) );
8235   ins_pipe( ialu_reg_mem );
8236 %}
8237 
8238 instruct addI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
8239   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8240   effect(KILL cr);
8241 
8242   ins_cost(150);
8243   format %{ "ADD    $dst,$src" %}
8244   opcode(0x01);  /* Opcode 01 /r */
8245   ins_encode( OpcP, RegMem( src, dst ) );
8246   ins_pipe( ialu_mem_reg );
8247 %}
8248 
8249 // Add Memory with Immediate
8250 instruct addI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
8251   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8252   effect(KILL cr);
8253 
8254   ins_cost(125);
8255   format %{ "ADD    $dst,$src" %}
8256   opcode(0x81);               /* Opcode 81 /0 id */
8257   ins_encode( OpcSE( src ), RMopc_Mem(0x00,dst), Con8or32( src ) );
8258   ins_pipe( ialu_mem_imm );
8259 %}
8260 
8261 instruct incI_mem(memory dst, immI1 src, eFlagsReg cr) %{
8262   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8263   effect(KILL cr);
8264 
8265   ins_cost(125);
8266   format %{ "INC    $dst" %}
8267   opcode(0xFF);               /* Opcode FF /0 */
8268   ins_encode( OpcP, RMopc_Mem(0x00,dst));
8269   ins_pipe( ialu_mem_imm );
8270 %}
8271 
8272 instruct decI_mem(memory dst, immI_M1 src, eFlagsReg cr) %{
8273   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
8274   effect(KILL cr);
8275 
8276   ins_cost(125);
8277   format %{ "DEC    $dst" %}
8278   opcode(0xFF);               /* Opcode FF /1 */
8279   ins_encode( OpcP, RMopc_Mem(0x01,dst));
8280   ins_pipe( ialu_mem_imm );
8281 %}
8282 
8283 
8284 instruct checkCastPP( eRegP dst ) %{
8285   match(Set dst (CheckCastPP dst));
8286 
8287   size(0);
8288   format %{ "#checkcastPP of $dst" %}
8289   ins_encode( /*empty encoding*/ );
8290   ins_pipe( empty );
8291 %}
8292 
8293 instruct castPP( eRegP dst ) %{
8294   match(Set dst (CastPP dst));
8295   format %{ "#castPP of $dst" %}
8296   ins_encode( /*empty encoding*/ );
8297   ins_pipe( empty );
8298 %}
8299 
8300 instruct castII( eRegI dst ) %{
8301   match(Set dst (CastII dst));
8302   format %{ "#castII of $dst" %}
8303   ins_encode( /*empty encoding*/ );
8304   ins_cost(0);
8305   ins_pipe( empty );
8306 %}
8307 
8308 
8309 // Load-locked - same as a regular pointer load when used with compare-swap
8310 instruct loadPLocked(eRegP dst, memory mem) %{
8311   match(Set dst (LoadPLocked mem));
8312 
8313   ins_cost(125);
8314   format %{ "MOV    $dst,$mem\t# Load ptr. locked" %}
8315   opcode(0x8B);
8316   ins_encode( OpcP, RegMem(dst,mem));
8317   ins_pipe( ialu_reg_mem );
8318 %}
8319 
8320 // LoadLong-locked - same as a volatile long load when used with compare-swap
8321 instruct loadLLocked(stackSlotL dst, load_long_memory mem) %{
8322   predicate(UseSSE<=1);
8323   match(Set dst (LoadLLocked mem));
8324 
8325   ins_cost(200);
8326   format %{ "FILD   $mem\t# Atomic volatile long load\n\t"
8327             "FISTp  $dst" %}
8328   ins_encode(enc_loadL_volatile(mem,dst));
8329   ins_pipe( fpu_reg_mem );
8330 %}
8331 
8332 instruct loadLX_Locked(stackSlotL dst, load_long_memory mem, regXD tmp) %{
8333   predicate(UseSSE>=2);
8334   match(Set dst (LoadLLocked mem));
8335   effect(TEMP tmp);
8336   ins_cost(180);
8337   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
8338             "MOVSD  $dst,$tmp" %}
8339   ins_encode(enc_loadLX_volatile(mem, dst, tmp));
8340   ins_pipe( pipe_slow );
8341 %}
8342 
8343 instruct loadLX_reg_Locked(eRegL dst, load_long_memory mem, regXD tmp) %{
8344   predicate(UseSSE>=2);
8345   match(Set dst (LoadLLocked mem));
8346   effect(TEMP tmp);
8347   ins_cost(160);
8348   format %{ "MOVSD  $tmp,$mem\t# Atomic volatile long load\n\t"
8349             "MOVD   $dst.lo,$tmp\n\t"
8350             "PSRLQ  $tmp,32\n\t"
8351             "MOVD   $dst.hi,$tmp" %}
8352   ins_encode(enc_loadLX_reg_volatile(mem, dst, tmp));
8353   ins_pipe( pipe_slow );
8354 %}
8355 
8356 // Conditional-store of the updated heap-top.
8357 // Used during allocation of the shared heap.
8358 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
8359 instruct storePConditional( memory heap_top_ptr, eAXRegP oldval, eRegP newval, eFlagsReg cr ) %{
8360   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8361   // EAX is killed if there is contention, but then it's also unused.
8362   // In the common case of no contention, EAX holds the new oop address.
8363   format %{ "CMPXCHG $heap_top_ptr,$newval\t# If EAX==$heap_top_ptr Then store $newval into $heap_top_ptr" %}
8364   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval,heap_top_ptr) );
8365   ins_pipe( pipe_cmpxchg );
8366 %}
8367 
8368 // Conditional-store of an int value.
8369 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG on Intel.
8370 instruct storeIConditional( memory mem, eAXRegI oldval, eRegI newval, eFlagsReg cr ) %{
8371   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8372   effect(KILL oldval);
8373   format %{ "CMPXCHG $mem,$newval\t# If EAX==$mem Then store $newval into $mem" %}
8374   ins_encode( lock_prefix, Opcode(0x0F), Opcode(0xB1), RegMem(newval, mem) );
8375   ins_pipe( pipe_cmpxchg );
8376 %}
8377 
8378 // Conditional-store of a long value.
8379 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG8 on Intel.
8380 instruct storeLConditional( memory mem, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
8381   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8382   effect(KILL oldval);
8383   format %{ "XCHG   EBX,ECX\t# correct order for CMPXCHG8 instruction\n\t"
8384             "CMPXCHG8 $mem,ECX:EBX\t# If EDX:EAX==$mem Then store ECX:EBX into $mem\n\t"
8385             "XCHG   EBX,ECX"
8386   %}
8387   ins_encode %{
8388     // Note: we need to swap rbx, and rcx before and after the
8389     //       cmpxchg8 instruction because the instruction uses
8390     //       rcx as the high order word of the new value to store but
8391     //       our register encoding uses rbx.
8392     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
8393     if( os::is_MP() )
8394       __ lock();
8395     __ cmpxchg8($mem$$Address);
8396     __ xchgl(as_Register(EBX_enc), as_Register(ECX_enc));
8397   %}
8398   ins_pipe( pipe_cmpxchg );
8399 %}
8400 
8401 // No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
8402 
8403 instruct compareAndSwapL( eRegI res, eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
8404   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
8405   effect(KILL cr, KILL oldval);
8406   format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8407             "MOV    $res,0\n\t"
8408             "JNE,s  fail\n\t"
8409             "MOV    $res,1\n"
8410           "fail:" %}
8411   ins_encode( enc_cmpxchg8(mem_ptr),
8412               enc_flags_ne_to_boolean(res) );
8413   ins_pipe( pipe_cmpxchg );
8414 %}
8415 
8416 instruct compareAndSwapP( eRegI res,  pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
8417   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
8418   effect(KILL cr, KILL oldval);
8419   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8420             "MOV    $res,0\n\t"
8421             "JNE,s  fail\n\t"
8422             "MOV    $res,1\n"
8423           "fail:" %}
8424   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
8425   ins_pipe( pipe_cmpxchg );
8426 %}
8427 
8428 instruct compareAndSwapI( eRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
8429   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
8430   effect(KILL cr, KILL oldval);
8431   format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
8432             "MOV    $res,0\n\t"
8433             "JNE,s  fail\n\t"
8434             "MOV    $res,1\n"
8435           "fail:" %}
8436   ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
8437   ins_pipe( pipe_cmpxchg );
8438 %}
8439 
8440 //----------Subtraction Instructions-------------------------------------------
8441 // Integer Subtraction Instructions
8442 instruct subI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8443   match(Set dst (SubI dst src));
8444   effect(KILL cr);
8445 
8446   size(2);
8447   format %{ "SUB    $dst,$src" %}
8448   opcode(0x2B);
8449   ins_encode( OpcP, RegReg( dst, src) );
8450   ins_pipe( ialu_reg_reg );
8451 %}
8452 
8453 instruct subI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
8454   match(Set dst (SubI dst src));
8455   effect(KILL cr);
8456 
8457   format %{ "SUB    $dst,$src" %}
8458   opcode(0x81,0x05);  /* Opcode 81 /5 */
8459   // ins_encode( RegImm( dst, src) );
8460   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8461   ins_pipe( ialu_reg );
8462 %}
8463 
8464 instruct subI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
8465   match(Set dst (SubI dst (LoadI src)));
8466   effect(KILL cr);
8467 
8468   ins_cost(125);
8469   format %{ "SUB    $dst,$src" %}
8470   opcode(0x2B);
8471   ins_encode( OpcP, RegMem( dst, src) );
8472   ins_pipe( ialu_reg_mem );
8473 %}
8474 
8475 instruct subI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
8476   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8477   effect(KILL cr);
8478 
8479   ins_cost(150);
8480   format %{ "SUB    $dst,$src" %}
8481   opcode(0x29);  /* Opcode 29 /r */
8482   ins_encode( OpcP, RegMem( src, dst ) );
8483   ins_pipe( ialu_mem_reg );
8484 %}
8485 
8486 // Subtract from a pointer
8487 instruct subP_eReg(eRegP dst, eRegI src, immI0 zero, eFlagsReg cr) %{
8488   match(Set dst (AddP dst (SubI zero src)));
8489   effect(KILL cr);
8490 
8491   size(2);
8492   format %{ "SUB    $dst,$src" %}
8493   opcode(0x2B);
8494   ins_encode( OpcP, RegReg( dst, src) );
8495   ins_pipe( ialu_reg_reg );
8496 %}
8497 
8498 instruct negI_eReg(eRegI dst, immI0 zero, eFlagsReg cr) %{
8499   match(Set dst (SubI zero dst));
8500   effect(KILL cr);
8501 
8502   size(2);
8503   format %{ "NEG    $dst" %}
8504   opcode(0xF7,0x03);  // Opcode F7 /3
8505   ins_encode( OpcP, RegOpc( dst ) );
8506   ins_pipe( ialu_reg );
8507 %}
8508 
8509 
8510 //----------Multiplication/Division Instructions-------------------------------
8511 // Integer Multiplication Instructions
8512 // Multiply Register
8513 instruct mulI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8514   match(Set dst (MulI dst src));
8515   effect(KILL cr);
8516 
8517   size(3);
8518   ins_cost(300);
8519   format %{ "IMUL   $dst,$src" %}
8520   opcode(0xAF, 0x0F);
8521   ins_encode( OpcS, OpcP, RegReg( dst, src) );
8522   ins_pipe( ialu_reg_reg_alu0 );
8523 %}
8524 
8525 // Multiply 32-bit Immediate
8526 instruct mulI_eReg_imm(eRegI dst, eRegI src, immI imm, eFlagsReg cr) %{
8527   match(Set dst (MulI src imm));
8528   effect(KILL cr);
8529 
8530   ins_cost(300);
8531   format %{ "IMUL   $dst,$src,$imm" %}
8532   opcode(0x69);  /* 69 /r id */
8533   ins_encode( OpcSE(imm), RegReg( dst, src ), Con8or32( imm ) );
8534   ins_pipe( ialu_reg_reg_alu0 );
8535 %}
8536 
8537 instruct loadConL_low_only(eADXRegL_low_only dst, immL32 src, eFlagsReg cr) %{
8538   match(Set dst src);
8539   effect(KILL cr);
8540 
8541   // Note that this is artificially increased to make it more expensive than loadConL
8542   ins_cost(250);
8543   format %{ "MOV    EAX,$src\t// low word only" %}
8544   opcode(0xB8);
8545   ins_encode( LdImmL_Lo(dst, src) );
8546   ins_pipe( ialu_reg_fat );
8547 %}
8548 
8549 // Multiply by 32-bit Immediate, taking the shifted high order results
8550 //  (special case for shift by 32)
8551 instruct mulI_imm_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32 cnt, eFlagsReg cr) %{
8552   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
8553   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
8554              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
8555              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
8556   effect(USE src1, KILL cr);
8557 
8558   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
8559   ins_cost(0*100 + 1*400 - 150);
8560   format %{ "IMUL   EDX:EAX,$src1" %}
8561   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
8562   ins_pipe( pipe_slow );
8563 %}
8564 
8565 // Multiply by 32-bit Immediate, taking the shifted high order results
8566 instruct mulI_imm_RShift_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr) %{
8567   match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
8568   predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
8569              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
8570              _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
8571   effect(USE src1, KILL cr);
8572 
8573   // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
8574   ins_cost(1*100 + 1*400 - 150);
8575   format %{ "IMUL   EDX:EAX,$src1\n\t"
8576             "SAR    EDX,$cnt-32" %}
8577   ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
8578   ins_pipe( pipe_slow );
8579 %}
8580 
8581 // Multiply Memory 32-bit Immediate
8582 instruct mulI_mem_imm(eRegI dst, memory src, immI imm, eFlagsReg cr) %{
8583   match(Set dst (MulI (LoadI src) imm));
8584   effect(KILL cr);
8585 
8586   ins_cost(300);
8587   format %{ "IMUL   $dst,$src,$imm" %}
8588   opcode(0x69);  /* 69 /r id */
8589   ins_encode( OpcSE(imm), RegMem( dst, src ), Con8or32( imm ) );
8590   ins_pipe( ialu_reg_mem_alu0 );
8591 %}
8592 
8593 // Multiply Memory
8594 instruct mulI(eRegI dst, memory src, eFlagsReg cr) %{
8595   match(Set dst (MulI dst (LoadI src)));
8596   effect(KILL cr);
8597 
8598   ins_cost(350);
8599   format %{ "IMUL   $dst,$src" %}
8600   opcode(0xAF, 0x0F);
8601   ins_encode( OpcS, OpcP, RegMem( dst, src) );
8602   ins_pipe( ialu_reg_mem_alu0 );
8603 %}
8604 
8605 // Multiply Register Int to Long
8606 instruct mulI2L(eADXRegL dst, eAXRegI src, nadxRegI src1, eFlagsReg flags) %{
8607   // Basic Idea: long = (long)int * (long)int
8608   match(Set dst (MulL (ConvI2L src) (ConvI2L src1)));
8609   effect(DEF dst, USE src, USE src1, KILL flags);
8610 
8611   ins_cost(300);
8612   format %{ "IMUL   $dst,$src1" %}
8613 
8614   ins_encode( long_int_multiply( dst, src1 ) );
8615   ins_pipe( ialu_reg_reg_alu0 );
8616 %}
8617 
8618 instruct mulIS_eReg(eADXRegL dst, immL_32bits mask, eFlagsReg flags, eAXRegI src, nadxRegI src1) %{
8619   // Basic Idea:  long = (int & 0xffffffffL) * (int & 0xffffffffL)
8620   match(Set dst (MulL (AndL (ConvI2L src) mask) (AndL (ConvI2L src1) mask)));
8621   effect(KILL flags);
8622 
8623   ins_cost(300);
8624   format %{ "MUL    $dst,$src1" %}
8625 
8626   ins_encode( long_uint_multiply(dst, src1) );
8627   ins_pipe( ialu_reg_reg_alu0 );
8628 %}
8629 
8630 // Multiply Register Long
8631 instruct mulL_eReg(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8632   match(Set dst (MulL dst src));
8633   effect(KILL cr, TEMP tmp);
8634   ins_cost(4*100+3*400);
8635 // Basic idea: lo(result) = lo(x_lo * y_lo)
8636 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
8637   format %{ "MOV    $tmp,$src.lo\n\t"
8638             "IMUL   $tmp,EDX\n\t"
8639             "MOV    EDX,$src.hi\n\t"
8640             "IMUL   EDX,EAX\n\t"
8641             "ADD    $tmp,EDX\n\t"
8642             "MUL    EDX:EAX,$src.lo\n\t"
8643             "ADD    EDX,$tmp" %}
8644   ins_encode( long_multiply( dst, src, tmp ) );
8645   ins_pipe( pipe_slow );
8646 %}
8647 
8648 // Multiply Register Long where the left operand's high 32 bits are zero
8649 instruct mulL_eReg_lhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8650   predicate(is_operand_hi32_zero(n->in(1)));
8651   match(Set dst (MulL dst src));
8652   effect(KILL cr, TEMP tmp);
8653   ins_cost(2*100+2*400);
8654 // Basic idea: lo(result) = lo(x_lo * y_lo)
8655 //             hi(result) = hi(x_lo * y_lo) + lo(x_lo * y_hi) where lo(x_hi * y_lo) = 0 because x_hi = 0
8656   format %{ "MOV    $tmp,$src.hi\n\t"
8657             "IMUL   $tmp,EAX\n\t"
8658             "MUL    EDX:EAX,$src.lo\n\t"
8659             "ADD    EDX,$tmp" %}
8660   ins_encode %{
8661     __ movl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
8662     __ imull($tmp$$Register, rax);
8663     __ mull($src$$Register);
8664     __ addl(rdx, $tmp$$Register);
8665   %}
8666   ins_pipe( pipe_slow );
8667 %}
8668 
8669 // Multiply Register Long where the right operand's high 32 bits are zero
8670 instruct mulL_eReg_rhi0(eADXRegL dst, eRegL src, eRegI tmp, eFlagsReg cr) %{
8671   predicate(is_operand_hi32_zero(n->in(2)));
8672   match(Set dst (MulL dst src));
8673   effect(KILL cr, TEMP tmp);
8674   ins_cost(2*100+2*400);
8675 // Basic idea: lo(result) = lo(x_lo * y_lo)
8676 //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) where lo(x_lo * y_hi) = 0 because y_hi = 0
8677   format %{ "MOV    $tmp,$src.lo\n\t"
8678             "IMUL   $tmp,EDX\n\t"
8679             "MUL    EDX:EAX,$src.lo\n\t"
8680             "ADD    EDX,$tmp" %}
8681   ins_encode %{
8682     __ movl($tmp$$Register, $src$$Register);
8683     __ imull($tmp$$Register, rdx);
8684     __ mull($src$$Register);
8685     __ addl(rdx, $tmp$$Register);
8686   %}
8687   ins_pipe( pipe_slow );
8688 %}
8689 
8690 // Multiply Register Long where the left and the right operands' high 32 bits are zero
8691 instruct mulL_eReg_hi0(eADXRegL dst, eRegL src, eFlagsReg cr) %{
8692   predicate(is_operand_hi32_zero(n->in(1)) && is_operand_hi32_zero(n->in(2)));
8693   match(Set dst (MulL dst src));
8694   effect(KILL cr);
8695   ins_cost(1*400);
8696 // Basic idea: lo(result) = lo(x_lo * y_lo)
8697 //             hi(result) = hi(x_lo * y_lo) where lo(x_hi * y_lo) = 0 and lo(x_lo * y_hi) = 0 because x_hi = 0 and y_hi = 0
8698   format %{ "MUL    EDX:EAX,$src.lo\n\t" %}
8699   ins_encode %{
8700     __ mull($src$$Register);
8701   %}
8702   ins_pipe( pipe_slow );
8703 %}
8704 
8705 // Multiply Register Long by small constant
8706 instruct mulL_eReg_con(eADXRegL dst, immL_127 src, eRegI tmp, eFlagsReg cr) %{
8707   match(Set dst (MulL dst src));
8708   effect(KILL cr, TEMP tmp);
8709   ins_cost(2*100+2*400);
8710   size(12);
8711 // Basic idea: lo(result) = lo(src * EAX)
8712 //             hi(result) = hi(src * EAX) + lo(src * EDX)
8713   format %{ "IMUL   $tmp,EDX,$src\n\t"
8714             "MOV    EDX,$src\n\t"
8715             "MUL    EDX\t# EDX*EAX -> EDX:EAX\n\t"
8716             "ADD    EDX,$tmp" %}
8717   ins_encode( long_multiply_con( dst, src, tmp ) );
8718   ins_pipe( pipe_slow );
8719 %}
8720 
8721 // Integer DIV with Register
8722 instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
8723   match(Set rax (DivI rax div));
8724   effect(KILL rdx, KILL cr);
8725   size(26);
8726   ins_cost(30*100+10*100);
8727   format %{ "CMP    EAX,0x80000000\n\t"
8728             "JNE,s  normal\n\t"
8729             "XOR    EDX,EDX\n\t"
8730             "CMP    ECX,-1\n\t"
8731             "JE,s   done\n"
8732     "normal: CDQ\n\t"
8733             "IDIV   $div\n\t"
8734     "done:"        %}
8735   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8736   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8737   ins_pipe( ialu_reg_reg_alu0 );
8738 %}
8739 
8740 // Divide Register Long
8741 instruct divL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
8742   match(Set dst (DivL src1 src2));
8743   effect( KILL cr, KILL cx, KILL bx );
8744   ins_cost(10000);
8745   format %{ "PUSH   $src1.hi\n\t"
8746             "PUSH   $src1.lo\n\t"
8747             "PUSH   $src2.hi\n\t"
8748             "PUSH   $src2.lo\n\t"
8749             "CALL   SharedRuntime::ldiv\n\t"
8750             "ADD    ESP,16" %}
8751   ins_encode( long_div(src1,src2) );
8752   ins_pipe( pipe_slow );
8753 %}
8754 
8755 // Integer DIVMOD with Register, both quotient and mod results
8756 instruct divModI_eReg_divmod(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
8757   match(DivModI rax div);
8758   effect(KILL cr);
8759   size(26);
8760   ins_cost(30*100+10*100);
8761   format %{ "CMP    EAX,0x80000000\n\t"
8762             "JNE,s  normal\n\t"
8763             "XOR    EDX,EDX\n\t"
8764             "CMP    ECX,-1\n\t"
8765             "JE,s   done\n"
8766     "normal: CDQ\n\t"
8767             "IDIV   $div\n\t"
8768     "done:"        %}
8769   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8770   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8771   ins_pipe( pipe_slow );
8772 %}
8773 
8774 // Integer MOD with Register
8775 instruct modI_eReg(eDXRegI rdx, eAXRegI rax, eCXRegI div, eFlagsReg cr) %{
8776   match(Set rdx (ModI rax div));
8777   effect(KILL rax, KILL cr);
8778 
8779   size(26);
8780   ins_cost(300);
8781   format %{ "CDQ\n\t"
8782             "IDIV   $div" %}
8783   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8784   ins_encode( cdq_enc, OpcP, RegOpc(div) );
8785   ins_pipe( ialu_reg_reg_alu0 );
8786 %}
8787 
8788 // Remainder Register Long
8789 instruct modL_eReg( eADXRegL dst, eRegL src1, eRegL src2, eFlagsReg cr, eCXRegI cx, eBXRegI bx ) %{
8790   match(Set dst (ModL src1 src2));
8791   effect( KILL cr, KILL cx, KILL bx );
8792   ins_cost(10000);
8793   format %{ "PUSH   $src1.hi\n\t"
8794             "PUSH   $src1.lo\n\t"
8795             "PUSH   $src2.hi\n\t"
8796             "PUSH   $src2.lo\n\t"
8797             "CALL   SharedRuntime::lrem\n\t"
8798             "ADD    ESP,16" %}
8799   ins_encode( long_mod(src1,src2) );
8800   ins_pipe( pipe_slow );
8801 %}
8802 
8803 // Integer Shift Instructions
8804 // Shift Left by one
8805 instruct shlI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
8806   match(Set dst (LShiftI dst shift));
8807   effect(KILL cr);
8808 
8809   size(2);
8810   format %{ "SHL    $dst,$shift" %}
8811   opcode(0xD1, 0x4);  /* D1 /4 */
8812   ins_encode( OpcP, RegOpc( dst ) );
8813   ins_pipe( ialu_reg );
8814 %}
8815 
8816 // Shift Left by 8-bit immediate
8817 instruct salI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
8818   match(Set dst (LShiftI dst shift));
8819   effect(KILL cr);
8820 
8821   size(3);
8822   format %{ "SHL    $dst,$shift" %}
8823   opcode(0xC1, 0x4);  /* C1 /4 ib */
8824   ins_encode( RegOpcImm( dst, shift) );
8825   ins_pipe( ialu_reg );
8826 %}
8827 
8828 // Shift Left by variable
8829 instruct salI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
8830   match(Set dst (LShiftI dst shift));
8831   effect(KILL cr);
8832 
8833   size(2);
8834   format %{ "SHL    $dst,$shift" %}
8835   opcode(0xD3, 0x4);  /* D3 /4 */
8836   ins_encode( OpcP, RegOpc( dst ) );
8837   ins_pipe( ialu_reg_reg );
8838 %}
8839 
8840 // Arithmetic shift right by one
8841 instruct sarI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
8842   match(Set dst (RShiftI dst shift));
8843   effect(KILL cr);
8844 
8845   size(2);
8846   format %{ "SAR    $dst,$shift" %}
8847   opcode(0xD1, 0x7);  /* D1 /7 */
8848   ins_encode( OpcP, RegOpc( dst ) );
8849   ins_pipe( ialu_reg );
8850 %}
8851 
8852 // Arithmetic shift right by one
8853 instruct sarI_mem_1(memory dst, immI1 shift, eFlagsReg cr) %{
8854   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8855   effect(KILL cr);
8856   format %{ "SAR    $dst,$shift" %}
8857   opcode(0xD1, 0x7);  /* D1 /7 */
8858   ins_encode( OpcP, RMopc_Mem(secondary,dst) );
8859   ins_pipe( ialu_mem_imm );
8860 %}
8861 
8862 // Arithmetic Shift Right by 8-bit immediate
8863 instruct sarI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
8864   match(Set dst (RShiftI dst shift));
8865   effect(KILL cr);
8866 
8867   size(3);
8868   format %{ "SAR    $dst,$shift" %}
8869   opcode(0xC1, 0x7);  /* C1 /7 ib */
8870   ins_encode( RegOpcImm( dst, shift ) );
8871   ins_pipe( ialu_mem_imm );
8872 %}
8873 
8874 // Arithmetic Shift Right by 8-bit immediate
8875 instruct sarI_mem_imm(memory dst, immI8 shift, eFlagsReg cr) %{
8876   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8877   effect(KILL cr);
8878 
8879   format %{ "SAR    $dst,$shift" %}
8880   opcode(0xC1, 0x7);  /* C1 /7 ib */
8881   ins_encode( OpcP, RMopc_Mem(secondary, dst ), Con8or32( shift ) );
8882   ins_pipe( ialu_mem_imm );
8883 %}
8884 
8885 // Arithmetic Shift Right by variable
8886 instruct sarI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
8887   match(Set dst (RShiftI dst shift));
8888   effect(KILL cr);
8889 
8890   size(2);
8891   format %{ "SAR    $dst,$shift" %}
8892   opcode(0xD3, 0x7);  /* D3 /7 */
8893   ins_encode( OpcP, RegOpc( dst ) );
8894   ins_pipe( ialu_reg_reg );
8895 %}
8896 
8897 // Logical shift right by one
8898 instruct shrI_eReg_1(eRegI dst, immI1 shift, eFlagsReg cr) %{
8899   match(Set dst (URShiftI dst shift));
8900   effect(KILL cr);
8901 
8902   size(2);
8903   format %{ "SHR    $dst,$shift" %}
8904   opcode(0xD1, 0x5);  /* D1 /5 */
8905   ins_encode( OpcP, RegOpc( dst ) );
8906   ins_pipe( ialu_reg );
8907 %}
8908 
8909 // Logical Shift Right by 8-bit immediate
8910 instruct shrI_eReg_imm(eRegI dst, immI8 shift, eFlagsReg cr) %{
8911   match(Set dst (URShiftI dst shift));
8912   effect(KILL cr);
8913 
8914   size(3);
8915   format %{ "SHR    $dst,$shift" %}
8916   opcode(0xC1, 0x5);  /* C1 /5 ib */
8917   ins_encode( RegOpcImm( dst, shift) );
8918   ins_pipe( ialu_reg );
8919 %}
8920 
8921 
8922 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8923 // This idiom is used by the compiler for the i2b bytecode.
8924 instruct i2b(eRegI dst, xRegI src, immI_24 twentyfour) %{
8925   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8926 
8927   size(3);
8928   format %{ "MOVSX  $dst,$src :8" %}
8929   ins_encode %{
8930     __ movsbl($dst$$Register, $src$$Register);
8931   %}
8932   ins_pipe(ialu_reg_reg);
8933 %}
8934 
8935 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8936 // This idiom is used by the compiler the i2s bytecode.
8937 instruct i2s(eRegI dst, xRegI src, immI_16 sixteen) %{
8938   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8939 
8940   size(3);
8941   format %{ "MOVSX  $dst,$src :16" %}
8942   ins_encode %{
8943     __ movswl($dst$$Register, $src$$Register);
8944   %}
8945   ins_pipe(ialu_reg_reg);
8946 %}
8947 
8948 
8949 // Logical Shift Right by variable
8950 instruct shrI_eReg_CL(eRegI dst, eCXRegI shift, eFlagsReg cr) %{
8951   match(Set dst (URShiftI dst shift));
8952   effect(KILL cr);
8953 
8954   size(2);
8955   format %{ "SHR    $dst,$shift" %}
8956   opcode(0xD3, 0x5);  /* D3 /5 */
8957   ins_encode( OpcP, RegOpc( dst ) );
8958   ins_pipe( ialu_reg_reg );
8959 %}
8960 
8961 
8962 //----------Logical Instructions-----------------------------------------------
8963 //----------Integer Logical Instructions---------------------------------------
8964 // And Instructions
8965 // And Register with Register
8966 instruct andI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
8967   match(Set dst (AndI dst src));
8968   effect(KILL cr);
8969 
8970   size(2);
8971   format %{ "AND    $dst,$src" %}
8972   opcode(0x23);
8973   ins_encode( OpcP, RegReg( dst, src) );
8974   ins_pipe( ialu_reg_reg );
8975 %}
8976 
8977 // And Register with Immediate
8978 instruct andI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
8979   match(Set dst (AndI dst src));
8980   effect(KILL cr);
8981 
8982   format %{ "AND    $dst,$src" %}
8983   opcode(0x81,0x04);  /* Opcode 81 /4 */
8984   // ins_encode( RegImm( dst, src) );
8985   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
8986   ins_pipe( ialu_reg );
8987 %}
8988 
8989 // And Register with Memory
8990 instruct andI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
8991   match(Set dst (AndI dst (LoadI src)));
8992   effect(KILL cr);
8993 
8994   ins_cost(125);
8995   format %{ "AND    $dst,$src" %}
8996   opcode(0x23);
8997   ins_encode( OpcP, RegMem( dst, src) );
8998   ins_pipe( ialu_reg_mem );
8999 %}
9000 
9001 // And Memory with Register
9002 instruct andI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9003   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9004   effect(KILL cr);
9005 
9006   ins_cost(150);
9007   format %{ "AND    $dst,$src" %}
9008   opcode(0x21);  /* Opcode 21 /r */
9009   ins_encode( OpcP, RegMem( src, dst ) );
9010   ins_pipe( ialu_mem_reg );
9011 %}
9012 
9013 // And Memory with Immediate
9014 instruct andI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9015   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9016   effect(KILL cr);
9017 
9018   ins_cost(125);
9019   format %{ "AND    $dst,$src" %}
9020   opcode(0x81, 0x4);  /* Opcode 81 /4 id */
9021   // ins_encode( MemImm( dst, src) );
9022   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9023   ins_pipe( ialu_mem_imm );
9024 %}
9025 
9026 // Or Instructions
9027 // Or Register with Register
9028 instruct orI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
9029   match(Set dst (OrI dst src));
9030   effect(KILL cr);
9031 
9032   size(2);
9033   format %{ "OR     $dst,$src" %}
9034   opcode(0x0B);
9035   ins_encode( OpcP, RegReg( dst, src) );
9036   ins_pipe( ialu_reg_reg );
9037 %}
9038 
9039 instruct orI_eReg_castP2X(eRegI dst, eRegP src, eFlagsReg cr) %{
9040   match(Set dst (OrI dst (CastP2X src)));
9041   effect(KILL cr);
9042 
9043   size(2);
9044   format %{ "OR     $dst,$src" %}
9045   opcode(0x0B);
9046   ins_encode( OpcP, RegReg( dst, src) );
9047   ins_pipe( ialu_reg_reg );
9048 %}
9049 
9050 
9051 // Or Register with Immediate
9052 instruct orI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
9053   match(Set dst (OrI dst src));
9054   effect(KILL cr);
9055 
9056   format %{ "OR     $dst,$src" %}
9057   opcode(0x81,0x01);  /* Opcode 81 /1 id */
9058   // ins_encode( RegImm( dst, src) );
9059   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
9060   ins_pipe( ialu_reg );
9061 %}
9062 
9063 // Or Register with Memory
9064 instruct orI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
9065   match(Set dst (OrI dst (LoadI src)));
9066   effect(KILL cr);
9067 
9068   ins_cost(125);
9069   format %{ "OR     $dst,$src" %}
9070   opcode(0x0B);
9071   ins_encode( OpcP, RegMem( dst, src) );
9072   ins_pipe( ialu_reg_mem );
9073 %}
9074 
9075 // Or Memory with Register
9076 instruct orI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9077   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9078   effect(KILL cr);
9079 
9080   ins_cost(150);
9081   format %{ "OR     $dst,$src" %}
9082   opcode(0x09);  /* Opcode 09 /r */
9083   ins_encode( OpcP, RegMem( src, dst ) );
9084   ins_pipe( ialu_mem_reg );
9085 %}
9086 
9087 // Or Memory with Immediate
9088 instruct orI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9089   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9090   effect(KILL cr);
9091 
9092   ins_cost(125);
9093   format %{ "OR     $dst,$src" %}
9094   opcode(0x81,0x1);  /* Opcode 81 /1 id */
9095   // ins_encode( MemImm( dst, src) );
9096   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9097   ins_pipe( ialu_mem_imm );
9098 %}
9099 
9100 // ROL/ROR
9101 // ROL expand
9102 instruct rolI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9103   effect(USE_DEF dst, USE shift, KILL cr);
9104 
9105   format %{ "ROL    $dst, $shift" %}
9106   opcode(0xD1, 0x0); /* Opcode D1 /0 */
9107   ins_encode( OpcP, RegOpc( dst ));
9108   ins_pipe( ialu_reg );
9109 %}
9110 
9111 instruct rolI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
9112   effect(USE_DEF dst, USE shift, KILL cr);
9113 
9114   format %{ "ROL    $dst, $shift" %}
9115   opcode(0xC1, 0x0); /*Opcode /C1  /0  */
9116   ins_encode( RegOpcImm(dst, shift) );
9117   ins_pipe(ialu_reg);
9118 %}
9119 
9120 instruct rolI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr) %{
9121   effect(USE_DEF dst, USE shift, KILL cr);
9122 
9123   format %{ "ROL    $dst, $shift" %}
9124   opcode(0xD3, 0x0);    /* Opcode D3 /0 */
9125   ins_encode(OpcP, RegOpc(dst));
9126   ins_pipe( ialu_reg_reg );
9127 %}
9128 // end of ROL expand
9129 
9130 // ROL 32bit by one once
9131 instruct rolI_eReg_i1(eRegI dst, immI1 lshift, immI_M1 rshift, eFlagsReg cr) %{
9132   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9133 
9134   expand %{
9135     rolI_eReg_imm1(dst, lshift, cr);
9136   %}
9137 %}
9138 
9139 // ROL 32bit var by imm8 once
9140 instruct rolI_eReg_i8(eRegI dst, immI8 lshift, immI8 rshift, eFlagsReg cr) %{
9141   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9142   match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9143 
9144   expand %{
9145     rolI_eReg_imm8(dst, lshift, cr);
9146   %}
9147 %}
9148 
9149 // ROL 32bit var by var once
9150 instruct rolI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
9151   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9152 
9153   expand %{
9154     rolI_eReg_CL(dst, shift, cr);
9155   %}
9156 %}
9157 
9158 // ROL 32bit var by var once
9159 instruct rolI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
9160   match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9161 
9162   expand %{
9163     rolI_eReg_CL(dst, shift, cr);
9164   %}
9165 %}
9166 
9167 // ROR expand
9168 instruct rorI_eReg_imm1(eRegI dst, immI1 shift, eFlagsReg cr) %{
9169   effect(USE_DEF dst, USE shift, KILL cr);
9170 
9171   format %{ "ROR    $dst, $shift" %}
9172   opcode(0xD1,0x1);  /* Opcode D1 /1 */
9173   ins_encode( OpcP, RegOpc( dst ) );
9174   ins_pipe( ialu_reg );
9175 %}
9176 
9177 instruct rorI_eReg_imm8(eRegI dst, immI8 shift, eFlagsReg cr) %{
9178   effect (USE_DEF dst, USE shift, KILL cr);
9179 
9180   format %{ "ROR    $dst, $shift" %}
9181   opcode(0xC1, 0x1); /* Opcode /C1 /1 ib */
9182   ins_encode( RegOpcImm(dst, shift) );
9183   ins_pipe( ialu_reg );
9184 %}
9185 
9186 instruct rorI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr)%{
9187   effect(USE_DEF dst, USE shift, KILL cr);
9188 
9189   format %{ "ROR    $dst, $shift" %}
9190   opcode(0xD3, 0x1);    /* Opcode D3 /1 */
9191   ins_encode(OpcP, RegOpc(dst));
9192   ins_pipe( ialu_reg_reg );
9193 %}
9194 // end of ROR expand
9195 
9196 // ROR right once
9197 instruct rorI_eReg_i1(eRegI dst, immI1 rshift, immI_M1 lshift, eFlagsReg cr) %{
9198   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9199 
9200   expand %{
9201     rorI_eReg_imm1(dst, rshift, cr);
9202   %}
9203 %}
9204 
9205 // ROR 32bit by immI8 once
9206 instruct rorI_eReg_i8(eRegI dst, immI8 rshift, immI8 lshift, eFlagsReg cr) %{
9207   predicate(  0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9208   match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9209 
9210   expand %{
9211     rorI_eReg_imm8(dst, rshift, cr);
9212   %}
9213 %}
9214 
9215 // ROR 32bit var by var once
9216 instruct rorI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI0 zero, eFlagsReg cr) %{
9217   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9218 
9219   expand %{
9220     rorI_eReg_CL(dst, shift, cr);
9221   %}
9222 %}
9223 
9224 // ROR 32bit var by var once
9225 instruct rorI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
9226   match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9227 
9228   expand %{
9229     rorI_eReg_CL(dst, shift, cr);
9230   %}
9231 %}
9232 
9233 // Xor Instructions
9234 // Xor Register with Register
9235 instruct xorI_eReg(eRegI dst, eRegI src, eFlagsReg cr) %{
9236   match(Set dst (XorI dst src));
9237   effect(KILL cr);
9238 
9239   size(2);
9240   format %{ "XOR    $dst,$src" %}
9241   opcode(0x33);
9242   ins_encode( OpcP, RegReg( dst, src) );
9243   ins_pipe( ialu_reg_reg );
9244 %}
9245 
9246 // Xor Register with Immediate -1
9247 instruct xorI_eReg_im1(eRegI dst, immI_M1 imm) %{
9248   match(Set dst (XorI dst imm));  
9249 
9250   size(2);
9251   format %{ "NOT    $dst" %}  
9252   ins_encode %{
9253      __ notl($dst$$Register);
9254   %}
9255   ins_pipe( ialu_reg );
9256 %}
9257 
9258 // Xor Register with Immediate
9259 instruct xorI_eReg_imm(eRegI dst, immI src, eFlagsReg cr) %{
9260   match(Set dst (XorI dst src));
9261   effect(KILL cr);
9262 
9263   format %{ "XOR    $dst,$src" %}
9264   opcode(0x81,0x06);  /* Opcode 81 /6 id */
9265   // ins_encode( RegImm( dst, src) );
9266   ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
9267   ins_pipe( ialu_reg );
9268 %}
9269 
9270 // Xor Register with Memory
9271 instruct xorI_eReg_mem(eRegI dst, memory src, eFlagsReg cr) %{
9272   match(Set dst (XorI dst (LoadI src)));
9273   effect(KILL cr);
9274 
9275   ins_cost(125);
9276   format %{ "XOR    $dst,$src" %}
9277   opcode(0x33);
9278   ins_encode( OpcP, RegMem(dst, src) );
9279   ins_pipe( ialu_reg_mem );
9280 %}
9281 
9282 // Xor Memory with Register
9283 instruct xorI_mem_eReg(memory dst, eRegI src, eFlagsReg cr) %{
9284   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9285   effect(KILL cr);
9286 
9287   ins_cost(150);
9288   format %{ "XOR    $dst,$src" %}
9289   opcode(0x31);  /* Opcode 31 /r */
9290   ins_encode( OpcP, RegMem( src, dst ) );
9291   ins_pipe( ialu_mem_reg );
9292 %}
9293 
9294 // Xor Memory with Immediate
9295 instruct xorI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
9296   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9297   effect(KILL cr);
9298 
9299   ins_cost(125);
9300   format %{ "XOR    $dst,$src" %}
9301   opcode(0x81,0x6);  /* Opcode 81 /6 id */
9302   ins_encode( OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32( src ) );
9303   ins_pipe( ialu_mem_imm );
9304 %}
9305 
9306 //----------Convert Int to Boolean---------------------------------------------
9307 
9308 instruct movI_nocopy(eRegI dst, eRegI src) %{
9309   effect( DEF dst, USE src );
9310   format %{ "MOV    $dst,$src" %}
9311   ins_encode( enc_Copy( dst, src) );
9312   ins_pipe( ialu_reg_reg );
9313 %}
9314 
9315 instruct ci2b( eRegI dst, eRegI src, eFlagsReg cr ) %{
9316   effect( USE_DEF dst, USE src, KILL cr );
9317 
9318   size(4);
9319   format %{ "NEG    $dst\n\t"
9320             "ADC    $dst,$src" %}
9321   ins_encode( neg_reg(dst),
9322               OpcRegReg(0x13,dst,src) );
9323   ins_pipe( ialu_reg_reg_long );
9324 %}
9325 
9326 instruct convI2B( eRegI dst, eRegI src, eFlagsReg cr ) %{
9327   match(Set dst (Conv2B src));
9328 
9329   expand %{
9330     movI_nocopy(dst,src);
9331     ci2b(dst,src,cr);
9332   %}
9333 %}
9334 
9335 instruct movP_nocopy(eRegI dst, eRegP src) %{
9336   effect( DEF dst, USE src );
9337   format %{ "MOV    $dst,$src" %}
9338   ins_encode( enc_Copy( dst, src) );
9339   ins_pipe( ialu_reg_reg );
9340 %}
9341 
9342 instruct cp2b( eRegI dst, eRegP src, eFlagsReg cr ) %{
9343   effect( USE_DEF dst, USE src, KILL cr );
9344   format %{ "NEG    $dst\n\t"
9345             "ADC    $dst,$src" %}
9346   ins_encode( neg_reg(dst),
9347               OpcRegReg(0x13,dst,src) );
9348   ins_pipe( ialu_reg_reg_long );
9349 %}
9350 
9351 instruct convP2B( eRegI dst, eRegP src, eFlagsReg cr ) %{
9352   match(Set dst (Conv2B src));
9353 
9354   expand %{
9355     movP_nocopy(dst,src);
9356     cp2b(dst,src,cr);
9357   %}
9358 %}
9359 
9360 instruct cmpLTMask( eCXRegI dst, ncxRegI p, ncxRegI q, eFlagsReg cr ) %{
9361   match(Set dst (CmpLTMask p q));
9362   effect( KILL cr );
9363   ins_cost(400);
9364 
9365   // SETlt can only use low byte of EAX,EBX, ECX, or EDX as destination
9366   format %{ "XOR    $dst,$dst\n\t"
9367             "CMP    $p,$q\n\t"
9368             "SETlt  $dst\n\t"
9369             "NEG    $dst" %}
9370   ins_encode( OpcRegReg(0x33,dst,dst),
9371               OpcRegReg(0x3B,p,q),
9372               setLT_reg(dst), neg_reg(dst) );
9373   ins_pipe( pipe_slow );
9374 %}
9375 
9376 instruct cmpLTMask0( eRegI dst, immI0 zero, eFlagsReg cr ) %{
9377   match(Set dst (CmpLTMask dst zero));
9378   effect( DEF dst, KILL cr );
9379   ins_cost(100);
9380 
9381   format %{ "SAR    $dst,31" %}
9382   opcode(0xC1, 0x7);  /* C1 /7 ib */
9383   ins_encode( RegOpcImm( dst, 0x1F ) );
9384   ins_pipe( ialu_reg );
9385 %}
9386 
9387 
9388 instruct cadd_cmpLTMask( ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp, eFlagsReg cr ) %{
9389   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9390   effect( KILL tmp, KILL cr );
9391   ins_cost(400);
9392   // annoyingly, $tmp has no edges so you cant ask for it in
9393   // any format or encoding
9394   format %{ "SUB    $p,$q\n\t"
9395             "SBB    ECX,ECX\n\t"
9396             "AND    ECX,$y\n\t"
9397             "ADD    $p,ECX" %}
9398   ins_encode( enc_cmpLTP(p,q,y,tmp) );
9399   ins_pipe( pipe_cmplt );
9400 %}
9401 
9402 /* If I enable this, I encourage spilling in the inner loop of compress.
9403 instruct cadd_cmpLTMask_mem( ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr ) %{
9404   match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
9405   effect( USE_KILL tmp, KILL cr );
9406   ins_cost(400);
9407 
9408   format %{ "SUB    $p,$q\n\t"
9409             "SBB    ECX,ECX\n\t"
9410             "AND    ECX,$y\n\t"
9411             "ADD    $p,ECX" %}
9412   ins_encode( enc_cmpLTP_mem(p,q,y,tmp) );
9413 %}
9414 */
9415 
9416 //----------Long Instructions------------------------------------------------
9417 // Add Long Register with Register
9418 instruct addL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9419   match(Set dst (AddL dst src));
9420   effect(KILL cr);
9421   ins_cost(200);
9422   format %{ "ADD    $dst.lo,$src.lo\n\t"
9423             "ADC    $dst.hi,$src.hi" %}
9424   opcode(0x03, 0x13);
9425   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
9426   ins_pipe( ialu_reg_reg_long );
9427 %}
9428 
9429 // Add Long Register with Immediate
9430 instruct addL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9431   match(Set dst (AddL dst src));
9432   effect(KILL cr);
9433   format %{ "ADD    $dst.lo,$src.lo\n\t"
9434             "ADC    $dst.hi,$src.hi" %}
9435   opcode(0x81,0x00,0x02);  /* Opcode 81 /0, 81 /2 */
9436   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9437   ins_pipe( ialu_reg_long );
9438 %}
9439 
9440 // Add Long Register with Memory
9441 instruct addL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9442   match(Set dst (AddL dst (LoadL mem)));
9443   effect(KILL cr);
9444   ins_cost(125);
9445   format %{ "ADD    $dst.lo,$mem\n\t"
9446             "ADC    $dst.hi,$mem+4" %}
9447   opcode(0x03, 0x13);
9448   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9449   ins_pipe( ialu_reg_long_mem );
9450 %}
9451 
9452 // Subtract Long Register with Register.
9453 instruct subL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9454   match(Set dst (SubL dst src));
9455   effect(KILL cr);
9456   ins_cost(200);
9457   format %{ "SUB    $dst.lo,$src.lo\n\t"
9458             "SBB    $dst.hi,$src.hi" %}
9459   opcode(0x2B, 0x1B);
9460   ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
9461   ins_pipe( ialu_reg_reg_long );
9462 %}
9463 
9464 // Subtract Long Register with Immediate
9465 instruct subL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9466   match(Set dst (SubL dst src));
9467   effect(KILL cr);
9468   format %{ "SUB    $dst.lo,$src.lo\n\t"
9469             "SBB    $dst.hi,$src.hi" %}
9470   opcode(0x81,0x05,0x03);  /* Opcode 81 /5, 81 /3 */
9471   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9472   ins_pipe( ialu_reg_long );
9473 %}
9474 
9475 // Subtract Long Register with Memory
9476 instruct subL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9477   match(Set dst (SubL dst (LoadL mem)));
9478   effect(KILL cr);
9479   ins_cost(125);
9480   format %{ "SUB    $dst.lo,$mem\n\t"
9481             "SBB    $dst.hi,$mem+4" %}
9482   opcode(0x2B, 0x1B);
9483   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9484   ins_pipe( ialu_reg_long_mem );
9485 %}
9486 
9487 instruct negL_eReg(eRegL dst, immL0 zero, eFlagsReg cr) %{
9488   match(Set dst (SubL zero dst));
9489   effect(KILL cr);
9490   ins_cost(300);
9491   format %{ "NEG    $dst.hi\n\tNEG    $dst.lo\n\tSBB    $dst.hi,0" %}
9492   ins_encode( neg_long(dst) );
9493   ins_pipe( ialu_reg_reg_long );
9494 %}
9495 
9496 // And Long Register with Register
9497 instruct andL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9498   match(Set dst (AndL dst src));
9499   effect(KILL cr);
9500   format %{ "AND    $dst.lo,$src.lo\n\t"
9501             "AND    $dst.hi,$src.hi" %}
9502   opcode(0x23,0x23);
9503   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9504   ins_pipe( ialu_reg_reg_long );
9505 %}
9506 
9507 // And Long Register with Immediate
9508 instruct andL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9509   match(Set dst (AndL dst src));
9510   effect(KILL cr);
9511   format %{ "AND    $dst.lo,$src.lo\n\t"
9512             "AND    $dst.hi,$src.hi" %}
9513   opcode(0x81,0x04,0x04);  /* Opcode 81 /4, 81 /4 */
9514   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9515   ins_pipe( ialu_reg_long );
9516 %}
9517 
9518 // And Long Register with Memory
9519 instruct andL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9520   match(Set dst (AndL dst (LoadL mem)));
9521   effect(KILL cr);
9522   ins_cost(125);
9523   format %{ "AND    $dst.lo,$mem\n\t"
9524             "AND    $dst.hi,$mem+4" %}
9525   opcode(0x23, 0x23);
9526   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9527   ins_pipe( ialu_reg_long_mem );
9528 %}
9529 
9530 // Or Long Register with Register
9531 instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9532   match(Set dst (OrL dst src));
9533   effect(KILL cr);
9534   format %{ "OR     $dst.lo,$src.lo\n\t"
9535             "OR     $dst.hi,$src.hi" %}
9536   opcode(0x0B,0x0B);
9537   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9538   ins_pipe( ialu_reg_reg_long );
9539 %}
9540 
9541 // Or Long Register with Immediate
9542 instruct orl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9543   match(Set dst (OrL dst src));
9544   effect(KILL cr);
9545   format %{ "OR     $dst.lo,$src.lo\n\t"
9546             "OR     $dst.hi,$src.hi" %}
9547   opcode(0x81,0x01,0x01);  /* Opcode 81 /1, 81 /1 */
9548   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9549   ins_pipe( ialu_reg_long );
9550 %}
9551 
9552 // Or Long Register with Memory
9553 instruct orl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9554   match(Set dst (OrL dst (LoadL mem)));
9555   effect(KILL cr);
9556   ins_cost(125);
9557   format %{ "OR     $dst.lo,$mem\n\t"
9558             "OR     $dst.hi,$mem+4" %}
9559   opcode(0x0B,0x0B);
9560   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9561   ins_pipe( ialu_reg_long_mem );
9562 %}
9563 
9564 // Xor Long Register with Register
9565 instruct xorl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
9566   match(Set dst (XorL dst src));
9567   effect(KILL cr);
9568   format %{ "XOR    $dst.lo,$src.lo\n\t"
9569             "XOR    $dst.hi,$src.hi" %}
9570   opcode(0x33,0x33);
9571   ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
9572   ins_pipe( ialu_reg_reg_long );
9573 %}
9574 
9575 // Xor Long Register with Immediate -1
9576 instruct xorl_eReg_im1(eRegL dst, immL_M1 imm) %{
9577   match(Set dst (XorL dst imm));  
9578   format %{ "NOT    $dst.lo\n\t"
9579             "NOT    $dst.hi" %}
9580   ins_encode %{
9581      __ notl($dst$$Register);
9582      __ notl(HIGH_FROM_LOW($dst$$Register));
9583   %}
9584   ins_pipe( ialu_reg_long );
9585 %}
9586 
9587 // Xor Long Register with Immediate
9588 instruct xorl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
9589   match(Set dst (XorL dst src));
9590   effect(KILL cr);
9591   format %{ "XOR    $dst.lo,$src.lo\n\t"
9592             "XOR    $dst.hi,$src.hi" %}
9593   opcode(0x81,0x06,0x06);  /* Opcode 81 /6, 81 /6 */
9594   ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
9595   ins_pipe( ialu_reg_long );
9596 %}
9597 
9598 // Xor Long Register with Memory
9599 instruct xorl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
9600   match(Set dst (XorL dst (LoadL mem)));
9601   effect(KILL cr);
9602   ins_cost(125);
9603   format %{ "XOR    $dst.lo,$mem\n\t"
9604             "XOR    $dst.hi,$mem+4" %}
9605   opcode(0x33,0x33);
9606   ins_encode( OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem) );
9607   ins_pipe( ialu_reg_long_mem );
9608 %}
9609 
9610 // Shift Left Long by 1
9611 instruct shlL_eReg_1(eRegL dst, immI_1 cnt, eFlagsReg cr) %{
9612   predicate(UseNewLongLShift);
9613   match(Set dst (LShiftL dst cnt));
9614   effect(KILL cr);
9615   ins_cost(100);
9616   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9617             "ADC    $dst.hi,$dst.hi" %}
9618   ins_encode %{
9619     __ addl($dst$$Register,$dst$$Register);
9620     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9621   %}
9622   ins_pipe( ialu_reg_long );
9623 %}
9624 
9625 // Shift Left Long by 2
9626 instruct shlL_eReg_2(eRegL dst, immI_2 cnt, eFlagsReg cr) %{
9627   predicate(UseNewLongLShift);
9628   match(Set dst (LShiftL dst cnt));
9629   effect(KILL cr);
9630   ins_cost(100);
9631   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9632             "ADC    $dst.hi,$dst.hi\n\t" 
9633             "ADD    $dst.lo,$dst.lo\n\t"
9634             "ADC    $dst.hi,$dst.hi" %}
9635   ins_encode %{
9636     __ addl($dst$$Register,$dst$$Register);
9637     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9638     __ addl($dst$$Register,$dst$$Register);
9639     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9640   %}
9641   ins_pipe( ialu_reg_long );
9642 %}
9643 
9644 // Shift Left Long by 3
9645 instruct shlL_eReg_3(eRegL dst, immI_3 cnt, eFlagsReg cr) %{
9646   predicate(UseNewLongLShift);
9647   match(Set dst (LShiftL dst cnt));
9648   effect(KILL cr);
9649   ins_cost(100);
9650   format %{ "ADD    $dst.lo,$dst.lo\n\t"
9651             "ADC    $dst.hi,$dst.hi\n\t" 
9652             "ADD    $dst.lo,$dst.lo\n\t"
9653             "ADC    $dst.hi,$dst.hi\n\t" 
9654             "ADD    $dst.lo,$dst.lo\n\t"
9655             "ADC    $dst.hi,$dst.hi" %}
9656   ins_encode %{
9657     __ addl($dst$$Register,$dst$$Register);
9658     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9659     __ addl($dst$$Register,$dst$$Register);
9660     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9661     __ addl($dst$$Register,$dst$$Register);
9662     __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
9663   %}
9664   ins_pipe( ialu_reg_long );
9665 %}
9666 
9667 // Shift Left Long by 1-31
9668 instruct shlL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9669   match(Set dst (LShiftL dst cnt));
9670   effect(KILL cr);
9671   ins_cost(200);
9672   format %{ "SHLD   $dst.hi,$dst.lo,$cnt\n\t"
9673             "SHL    $dst.lo,$cnt" %}
9674   opcode(0xC1, 0x4, 0xA4);  /* 0F/A4, then C1 /4 ib */
9675   ins_encode( move_long_small_shift(dst,cnt) );
9676   ins_pipe( ialu_reg_long );
9677 %}
9678 
9679 // Shift Left Long by 32-63
9680 instruct shlL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9681   match(Set dst (LShiftL dst cnt));
9682   effect(KILL cr);
9683   ins_cost(300);
9684   format %{ "MOV    $dst.hi,$dst.lo\n"
9685           "\tSHL    $dst.hi,$cnt-32\n"
9686           "\tXOR    $dst.lo,$dst.lo" %}
9687   opcode(0xC1, 0x4);  /* C1 /4 ib */
9688   ins_encode( move_long_big_shift_clr(dst,cnt) );
9689   ins_pipe( ialu_reg_long );
9690 %}
9691 
9692 // Shift Left Long by variable
9693 instruct salL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9694   match(Set dst (LShiftL dst shift));
9695   effect(KILL cr);
9696   ins_cost(500+200);
9697   size(17);
9698   format %{ "TEST   $shift,32\n\t"
9699             "JEQ,s  small\n\t"
9700             "MOV    $dst.hi,$dst.lo\n\t"
9701             "XOR    $dst.lo,$dst.lo\n"
9702     "small:\tSHLD   $dst.hi,$dst.lo,$shift\n\t"
9703             "SHL    $dst.lo,$shift" %}
9704   ins_encode( shift_left_long( dst, shift ) );
9705   ins_pipe( pipe_slow );
9706 %}
9707 
9708 // Shift Right Long by 1-31
9709 instruct shrL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9710   match(Set dst (URShiftL dst cnt));
9711   effect(KILL cr);
9712   ins_cost(200);
9713   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
9714             "SHR    $dst.hi,$cnt" %}
9715   opcode(0xC1, 0x5, 0xAC);  /* 0F/AC, then C1 /5 ib */
9716   ins_encode( move_long_small_shift(dst,cnt) );
9717   ins_pipe( ialu_reg_long );
9718 %}
9719 
9720 // Shift Right Long by 32-63
9721 instruct shrL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9722   match(Set dst (URShiftL dst cnt));
9723   effect(KILL cr);
9724   ins_cost(300);
9725   format %{ "MOV    $dst.lo,$dst.hi\n"
9726           "\tSHR    $dst.lo,$cnt-32\n"
9727           "\tXOR    $dst.hi,$dst.hi" %}
9728   opcode(0xC1, 0x5);  /* C1 /5 ib */
9729   ins_encode( move_long_big_shift_clr(dst,cnt) );
9730   ins_pipe( ialu_reg_long );
9731 %}
9732 
9733 // Shift Right Long by variable
9734 instruct shrL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9735   match(Set dst (URShiftL dst shift));
9736   effect(KILL cr);
9737   ins_cost(600);
9738   size(17);
9739   format %{ "TEST   $shift,32\n\t"
9740             "JEQ,s  small\n\t"
9741             "MOV    $dst.lo,$dst.hi\n\t"
9742             "XOR    $dst.hi,$dst.hi\n"
9743     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
9744             "SHR    $dst.hi,$shift" %}
9745   ins_encode( shift_right_long( dst, shift ) );
9746   ins_pipe( pipe_slow );
9747 %}
9748 
9749 // Shift Right Long by 1-31
9750 instruct sarL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
9751   match(Set dst (RShiftL dst cnt));
9752   effect(KILL cr);
9753   ins_cost(200);
9754   format %{ "SHRD   $dst.lo,$dst.hi,$cnt\n\t"
9755             "SAR    $dst.hi,$cnt" %}
9756   opcode(0xC1, 0x7, 0xAC);  /* 0F/AC, then C1 /7 ib */
9757   ins_encode( move_long_small_shift(dst,cnt) );
9758   ins_pipe( ialu_reg_long );
9759 %}
9760 
9761 // Shift Right Long by 32-63
9762 instruct sarL_eReg_32_63( eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
9763   match(Set dst (RShiftL dst cnt));
9764   effect(KILL cr);
9765   ins_cost(300);
9766   format %{ "MOV    $dst.lo,$dst.hi\n"
9767           "\tSAR    $dst.lo,$cnt-32\n"
9768           "\tSAR    $dst.hi,31" %}
9769   opcode(0xC1, 0x7);  /* C1 /7 ib */
9770   ins_encode( move_long_big_shift_sign(dst,cnt) );
9771   ins_pipe( ialu_reg_long );
9772 %}
9773 
9774 // Shift Right arithmetic Long by variable
9775 instruct sarL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
9776   match(Set dst (RShiftL dst shift));
9777   effect(KILL cr);
9778   ins_cost(600);
9779   size(18);
9780   format %{ "TEST   $shift,32\n\t"
9781             "JEQ,s  small\n\t"
9782             "MOV    $dst.lo,$dst.hi\n\t"
9783             "SAR    $dst.hi,31\n"
9784     "small:\tSHRD   $dst.lo,$dst.hi,$shift\n\t"
9785             "SAR    $dst.hi,$shift" %}
9786   ins_encode( shift_right_arith_long( dst, shift ) );
9787   ins_pipe( pipe_slow );
9788 %}
9789 
9790 
9791 //----------Double Instructions------------------------------------------------
9792 // Double Math
9793 
9794 // Compare & branch
9795 
9796 // P6 version of float compare, sets condition codes in EFLAGS
9797 instruct cmpD_cc_P6(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
9798   predicate(VM_Version::supports_cmov() && UseSSE <=1);
9799   match(Set cr (CmpD src1 src2));
9800   effect(KILL rax);
9801   ins_cost(150);
9802   format %{ "FLD    $src1\n\t"
9803             "FUCOMIP ST,$src2  // P6 instruction\n\t"
9804             "JNP    exit\n\t"
9805             "MOV    ah,1       // saw a NaN, set CF\n\t"
9806             "SAHF\n"
9807      "exit:\tNOP               // avoid branch to branch" %}
9808   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
9809   ins_encode( Push_Reg_D(src1),
9810               OpcP, RegOpc(src2),
9811               cmpF_P6_fixup );
9812   ins_pipe( pipe_slow );
9813 %}
9814 
9815 instruct cmpD_cc_P6CF(eFlagsRegUCF cr, regD src1, regD src2) %{
9816   predicate(VM_Version::supports_cmov() && UseSSE <=1);
9817   match(Set cr (CmpD src1 src2));
9818   ins_cost(150);
9819   format %{ "FLD    $src1\n\t"
9820             "FUCOMIP ST,$src2  // P6 instruction" %}
9821   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
9822   ins_encode( Push_Reg_D(src1),
9823               OpcP, RegOpc(src2));
9824   ins_pipe( pipe_slow );
9825 %}
9826 
9827 // Compare & branch
9828 instruct cmpD_cc(eFlagsRegU cr, regD src1, regD src2, eAXRegI rax) %{
9829   predicate(UseSSE<=1);
9830   match(Set cr (CmpD src1 src2));
9831   effect(KILL rax);
9832   ins_cost(200);
9833   format %{ "FLD    $src1\n\t"
9834             "FCOMp  $src2\n\t"
9835             "FNSTSW AX\n\t"
9836             "TEST   AX,0x400\n\t"
9837             "JZ,s   flags\n\t"
9838             "MOV    AH,1\t# unordered treat as LT\n"
9839     "flags:\tSAHF" %}
9840   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
9841   ins_encode( Push_Reg_D(src1),
9842               OpcP, RegOpc(src2),
9843               fpu_flags);
9844   ins_pipe( pipe_slow );
9845 %}
9846 
9847 // Compare vs zero into -1,0,1
9848 instruct cmpD_0(eRegI dst, regD src1, immD0 zero, eAXRegI rax, eFlagsReg cr) %{
9849   predicate(UseSSE<=1);
9850   match(Set dst (CmpD3 src1 zero));
9851   effect(KILL cr, KILL rax);
9852   ins_cost(280);
9853   format %{ "FTSTD  $dst,$src1" %}
9854   opcode(0xE4, 0xD9);
9855   ins_encode( Push_Reg_D(src1),
9856               OpcS, OpcP, PopFPU,
9857               CmpF_Result(dst));
9858   ins_pipe( pipe_slow );
9859 %}
9860 
9861 // Compare into -1,0,1
9862 instruct cmpD_reg(eRegI dst, regD src1, regD src2, eAXRegI rax, eFlagsReg cr) %{
9863   predicate(UseSSE<=1);
9864   match(Set dst (CmpD3 src1 src2));
9865   effect(KILL cr, KILL rax);
9866   ins_cost(300);
9867   format %{ "FCMPD  $dst,$src1,$src2" %}
9868   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
9869   ins_encode( Push_Reg_D(src1),
9870               OpcP, RegOpc(src2),
9871               CmpF_Result(dst));
9872   ins_pipe( pipe_slow );
9873 %}
9874 
9875 // float compare and set condition codes in EFLAGS by XMM regs
9876 instruct cmpXD_cc(eFlagsRegU cr, regXD dst, regXD src, eAXRegI rax) %{
9877   predicate(UseSSE>=2);
9878   match(Set cr (CmpD dst src));
9879   effect(KILL rax);
9880   ins_cost(125);
9881   format %{ "COMISD $dst,$src\n"
9882           "\tJNP    exit\n"
9883           "\tMOV    ah,1       // saw a NaN, set CF\n"
9884           "\tSAHF\n"
9885      "exit:\tNOP               // avoid branch to branch" %}
9886   opcode(0x66, 0x0F, 0x2F);
9887   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src), cmpF_P6_fixup);
9888   ins_pipe( pipe_slow );
9889 %}
9890 
9891 instruct cmpXD_ccCF(eFlagsRegUCF cr, regXD dst, regXD src) %{
9892   predicate(UseSSE>=2);
9893   match(Set cr (CmpD dst src));
9894   ins_cost(100);
9895   format %{ "COMISD $dst,$src" %}
9896   opcode(0x66, 0x0F, 0x2F);
9897   ins_encode(OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
9898   ins_pipe( pipe_slow );
9899 %}
9900 
9901 // float compare and set condition codes in EFLAGS by XMM regs
9902 instruct cmpXD_ccmem(eFlagsRegU cr, regXD dst, memory src, eAXRegI rax) %{
9903   predicate(UseSSE>=2);
9904   match(Set cr (CmpD dst (LoadD src)));
9905   effect(KILL rax);
9906   ins_cost(145);
9907   format %{ "COMISD $dst,$src\n"
9908           "\tJNP    exit\n"
9909           "\tMOV    ah,1       // saw a NaN, set CF\n"
9910           "\tSAHF\n"
9911      "exit:\tNOP               // avoid branch to branch" %}
9912   opcode(0x66, 0x0F, 0x2F);
9913   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src), cmpF_P6_fixup);
9914   ins_pipe( pipe_slow );
9915 %}
9916 
9917 instruct cmpXD_ccmemCF(eFlagsRegUCF cr, regXD dst, memory src) %{
9918   predicate(UseSSE>=2);
9919   match(Set cr (CmpD dst (LoadD src)));
9920   ins_cost(100);
9921   format %{ "COMISD $dst,$src" %}
9922   opcode(0x66, 0x0F, 0x2F);
9923   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(dst, src));
9924   ins_pipe( pipe_slow );
9925 %}
9926 
9927 // Compare into -1,0,1 in XMM
9928 instruct cmpXD_reg(eRegI dst, regXD src1, regXD src2, eFlagsReg cr) %{
9929   predicate(UseSSE>=2);
9930   match(Set dst (CmpD3 src1 src2));
9931   effect(KILL cr);
9932   ins_cost(255);
9933   format %{ "XOR    $dst,$dst\n"
9934           "\tCOMISD $src1,$src2\n"
9935           "\tJP,s   nan\n"
9936           "\tJEQ,s  exit\n"
9937           "\tJA,s   inc\n"
9938       "nan:\tDEC    $dst\n"
9939           "\tJMP,s  exit\n"
9940       "inc:\tINC    $dst\n"
9941       "exit:"
9942                 %}
9943   opcode(0x66, 0x0F, 0x2F);
9944   ins_encode(Xor_Reg(dst), OpcP, OpcS, Opcode(tertiary), RegReg(src1, src2),
9945              CmpX_Result(dst));
9946   ins_pipe( pipe_slow );
9947 %}
9948 
9949 // Compare into -1,0,1 in XMM and memory
9950 instruct cmpXD_regmem(eRegI dst, regXD src1, memory mem, eFlagsReg cr) %{
9951   predicate(UseSSE>=2);
9952   match(Set dst (CmpD3 src1 (LoadD mem)));
9953   effect(KILL cr);
9954   ins_cost(275);
9955   format %{ "COMISD $src1,$mem\n"
9956           "\tMOV    $dst,0\t\t# do not blow flags\n"
9957           "\tJP,s   nan\n"
9958           "\tJEQ,s  exit\n"
9959           "\tJA,s   inc\n"
9960       "nan:\tDEC    $dst\n"
9961           "\tJMP,s  exit\n"
9962       "inc:\tINC    $dst\n"
9963       "exit:"
9964                 %}
9965   opcode(0x66, 0x0F, 0x2F);
9966   ins_encode(OpcP, OpcS, Opcode(tertiary), RegMem(src1, mem),
9967              LdImmI(dst,0x0), CmpX_Result(dst));
9968   ins_pipe( pipe_slow );
9969 %}
9970 
9971 
9972 instruct subD_reg(regD dst, regD src) %{
9973   predicate (UseSSE <=1);
9974   match(Set dst (SubD dst src));
9975 
9976   format %{ "FLD    $src\n\t"
9977             "DSUBp  $dst,ST" %}
9978   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
9979   ins_cost(150);
9980   ins_encode( Push_Reg_D(src),
9981               OpcP, RegOpc(dst) );
9982   ins_pipe( fpu_reg_reg );
9983 %}
9984 
9985 instruct subD_reg_round(stackSlotD dst, regD src1, regD src2) %{
9986   predicate (UseSSE <=1);
9987   match(Set dst (RoundDouble (SubD src1 src2)));
9988   ins_cost(250);
9989 
9990   format %{ "FLD    $src2\n\t"
9991             "DSUB   ST,$src1\n\t"
9992             "FSTP_D $dst\t# D-round" %}
9993   opcode(0xD8, 0x5);
9994   ins_encode( Push_Reg_D(src2),
9995               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
9996   ins_pipe( fpu_mem_reg_reg );
9997 %}
9998 
9999 
10000 instruct subD_reg_mem(regD dst, memory src) %{
10001   predicate (UseSSE <=1);
10002   match(Set dst (SubD dst (LoadD src)));
10003   ins_cost(150);
10004 
10005   format %{ "FLD    $src\n\t"
10006             "DSUBp  $dst,ST" %}
10007   opcode(0xDE, 0x5, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
10008   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10009               OpcP, RegOpc(dst) );
10010   ins_pipe( fpu_reg_mem );
10011 %}
10012 
10013 instruct absD_reg(regDPR1 dst, regDPR1 src) %{
10014   predicate (UseSSE<=1);
10015   match(Set dst (AbsD src));
10016   ins_cost(100);
10017   format %{ "FABS" %}
10018   opcode(0xE1, 0xD9);
10019   ins_encode( OpcS, OpcP );
10020   ins_pipe( fpu_reg_reg );
10021 %}
10022 
10023 instruct absXD_reg( regXD dst ) %{
10024   predicate(UseSSE>=2);
10025   match(Set dst (AbsD dst));
10026   format %{ "ANDPD  $dst,[0x7FFFFFFFFFFFFFFF]\t# ABS D by sign masking" %}
10027   ins_encode( AbsXD_encoding(dst));
10028   ins_pipe( pipe_slow );
10029 %}
10030 
10031 instruct negD_reg(regDPR1 dst, regDPR1 src) %{
10032   predicate(UseSSE<=1);
10033   match(Set dst (NegD src));
10034   ins_cost(100);
10035   format %{ "FCHS" %}
10036   opcode(0xE0, 0xD9);
10037   ins_encode( OpcS, OpcP );
10038   ins_pipe( fpu_reg_reg );
10039 %}
10040 
10041 instruct negXD_reg( regXD dst ) %{
10042   predicate(UseSSE>=2);
10043   match(Set dst (NegD dst));
10044   format %{ "XORPD  $dst,[0x8000000000000000]\t# CHS D by sign flipping" %}
10045   ins_encode %{
10046      __ xorpd($dst$$XMMRegister,
10047               ExternalAddress((address)double_signflip_pool));
10048   %}
10049   ins_pipe( pipe_slow );
10050 %}
10051 
10052 instruct addD_reg(regD dst, regD src) %{
10053   predicate(UseSSE<=1);
10054   match(Set dst (AddD dst src));
10055   format %{ "FLD    $src\n\t"
10056             "DADD   $dst,ST" %}
10057   size(4);
10058   ins_cost(150);
10059   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
10060   ins_encode( Push_Reg_D(src),
10061               OpcP, RegOpc(dst) );
10062   ins_pipe( fpu_reg_reg );
10063 %}
10064 
10065 
10066 instruct addD_reg_round(stackSlotD dst, regD src1, regD src2) %{
10067   predicate(UseSSE<=1);
10068   match(Set dst (RoundDouble (AddD src1 src2)));
10069   ins_cost(250);
10070 
10071   format %{ "FLD    $src2\n\t"
10072             "DADD   ST,$src1\n\t"
10073             "FSTP_D $dst\t# D-round" %}
10074   opcode(0xD8, 0x0); /* D8 C0+i or D8 /0*/
10075   ins_encode( Push_Reg_D(src2),
10076               OpcP, RegOpc(src1), Pop_Mem_D(dst) );
10077   ins_pipe( fpu_mem_reg_reg );
10078 %}
10079 
10080 
10081 instruct addD_reg_mem(regD dst, memory src) %{
10082   predicate(UseSSE<=1);
10083   match(Set dst (AddD dst (LoadD src)));
10084   ins_cost(150);
10085 
10086   format %{ "FLD    $src\n\t"
10087             "DADDp  $dst,ST" %}
10088   opcode(0xDE, 0x0, 0xDD); /* DE C0+i */  /* LoadD  DD /0 */
10089   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10090               OpcP, RegOpc(dst) );
10091   ins_pipe( fpu_reg_mem );
10092 %}
10093 
10094 // add-to-memory
10095 instruct addD_mem_reg(memory dst, regD src) %{
10096   predicate(UseSSE<=1);
10097   match(Set dst (StoreD dst (RoundDouble (AddD (LoadD dst) src))));
10098   ins_cost(150);
10099 
10100   format %{ "FLD_D  $dst\n\t"
10101             "DADD   ST,$src\n\t"
10102             "FST_D  $dst" %}
10103   opcode(0xDD, 0x0);
10104   ins_encode( Opcode(0xDD), RMopc_Mem(0x00,dst),
10105               Opcode(0xD8), RegOpc(src),
10106               set_instruction_start,
10107               Opcode(0xDD), RMopc_Mem(0x03,dst) );
10108   ins_pipe( fpu_reg_mem );
10109 %}
10110 
10111 instruct addD_reg_imm1(regD dst, immD1 src) %{
10112   predicate(UseSSE<=1);
10113   match(Set dst (AddD dst src));
10114   ins_cost(125);
10115   format %{ "FLD1\n\t"
10116             "DADDp  $dst,ST" %}
10117   opcode(0xDE, 0x00);
10118   ins_encode( LdImmD(src),
10119               OpcP, RegOpc(dst) );
10120   ins_pipe( fpu_reg );
10121 %}
10122 
10123 instruct addD_reg_imm(regD dst, immD src) %{
10124   predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
10125   match(Set dst (AddD dst src));
10126   ins_cost(200);
10127   format %{ "FLD_D  [$src]\n\t"
10128             "DADDp  $dst,ST" %}
10129   opcode(0xDE, 0x00);       /* DE /0 */
10130   ins_encode( LdImmD(src),
10131               OpcP, RegOpc(dst));
10132   ins_pipe( fpu_reg_mem );
10133 %}
10134 
10135 instruct addD_reg_imm_round(stackSlotD dst, regD src, immD con) %{
10136   predicate(UseSSE<=1 && _kids[0]->_kids[1]->_leaf->getd() != 0.0 && _kids[0]->_kids[1]->_leaf->getd() != 1.0 );
10137   match(Set dst (RoundDouble (AddD src con)));
10138   ins_cost(200);
10139   format %{ "FLD_D  [$con]\n\t"
10140             "DADD   ST,$src\n\t"
10141             "FSTP_D $dst\t# D-round" %}
10142   opcode(0xD8, 0x00);       /* D8 /0 */
10143   ins_encode( LdImmD(con),
10144               OpcP, RegOpc(src), Pop_Mem_D(dst));
10145   ins_pipe( fpu_mem_reg_con );
10146 %}
10147 
10148 // Add two double precision floating point values in xmm
10149 instruct addXD_reg(regXD dst, regXD src) %{
10150   predicate(UseSSE>=2);
10151   match(Set dst (AddD dst src));
10152   format %{ "ADDSD  $dst,$src" %}
10153   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
10154   ins_pipe( pipe_slow );
10155 %}
10156 
10157 instruct addXD_imm(regXD dst, immXD con) %{
10158   predicate(UseSSE>=2);
10159   match(Set dst (AddD dst con));
10160   format %{ "ADDSD  $dst,[$con]" %}
10161   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), LdImmXD(dst, con) );
10162   ins_pipe( pipe_slow );
10163 %}
10164 
10165 instruct addXD_mem(regXD dst, memory mem) %{
10166   predicate(UseSSE>=2);
10167   match(Set dst (AddD dst (LoadD mem)));
10168   format %{ "ADDSD  $dst,$mem" %}
10169   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x58), RegMem(dst,mem));
10170   ins_pipe( pipe_slow );
10171 %}
10172 
10173 // Sub two double precision floating point values in xmm
10174 instruct subXD_reg(regXD dst, regXD src) %{
10175   predicate(UseSSE>=2);
10176   match(Set dst (SubD dst src));
10177   format %{ "SUBSD  $dst,$src" %}
10178   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
10179   ins_pipe( pipe_slow );
10180 %}
10181 
10182 instruct subXD_imm(regXD dst, immXD con) %{
10183   predicate(UseSSE>=2);
10184   match(Set dst (SubD dst con));
10185   format %{ "SUBSD  $dst,[$con]" %}
10186   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), LdImmXD(dst, con) );
10187   ins_pipe( pipe_slow );
10188 %}
10189 
10190 instruct subXD_mem(regXD dst, memory mem) %{
10191   predicate(UseSSE>=2);
10192   match(Set dst (SubD dst (LoadD mem)));
10193   format %{ "SUBSD  $dst,$mem" %}
10194   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
10195   ins_pipe( pipe_slow );
10196 %}
10197 
10198 // Mul two double precision floating point values in xmm
10199 instruct mulXD_reg(regXD dst, regXD src) %{
10200   predicate(UseSSE>=2);
10201   match(Set dst (MulD dst src));
10202   format %{ "MULSD  $dst,$src" %}
10203   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
10204   ins_pipe( pipe_slow );
10205 %}
10206 
10207 instruct mulXD_imm(regXD dst, immXD con) %{
10208   predicate(UseSSE>=2);
10209   match(Set dst (MulD dst con));
10210   format %{ "MULSD  $dst,[$con]" %}
10211   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), LdImmXD(dst, con) );
10212   ins_pipe( pipe_slow );
10213 %}
10214 
10215 instruct mulXD_mem(regXD dst, memory mem) %{
10216   predicate(UseSSE>=2);
10217   match(Set dst (MulD dst (LoadD mem)));
10218   format %{ "MULSD  $dst,$mem" %}
10219   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
10220   ins_pipe( pipe_slow );
10221 %}
10222 
10223 // Div two double precision floating point values in xmm
10224 instruct divXD_reg(regXD dst, regXD src) %{
10225   predicate(UseSSE>=2);
10226   match(Set dst (DivD dst src));
10227   format %{ "DIVSD  $dst,$src" %}
10228   opcode(0xF2, 0x0F, 0x5E);
10229   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
10230   ins_pipe( pipe_slow );
10231 %}
10232 
10233 instruct divXD_imm(regXD dst, immXD con) %{
10234   predicate(UseSSE>=2);
10235   match(Set dst (DivD dst con));
10236   format %{ "DIVSD  $dst,[$con]" %}
10237   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), LdImmXD(dst, con));
10238   ins_pipe( pipe_slow );
10239 %}
10240 
10241 instruct divXD_mem(regXD dst, memory mem) %{
10242   predicate(UseSSE>=2);
10243   match(Set dst (DivD dst (LoadD mem)));
10244   format %{ "DIVSD  $dst,$mem" %}
10245   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
10246   ins_pipe( pipe_slow );
10247 %}
10248 
10249 
10250 instruct mulD_reg(regD dst, regD src) %{
10251   predicate(UseSSE<=1);
10252   match(Set dst (MulD dst src));
10253   format %{ "FLD    $src\n\t"
10254             "DMULp  $dst,ST" %}
10255   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
10256   ins_cost(150);
10257   ins_encode( Push_Reg_D(src),
10258               OpcP, RegOpc(dst) );
10259   ins_pipe( fpu_reg_reg );
10260 %}
10261 
10262 // Strict FP instruction biases argument before multiply then
10263 // biases result to avoid double rounding of subnormals.
10264 //
10265 // scale arg1 by multiplying arg1 by 2^(-15360)
10266 // load arg2
10267 // multiply scaled arg1 by arg2
10268 // rescale product by 2^(15360)
10269 //
10270 instruct strictfp_mulD_reg(regDPR1 dst, regnotDPR1 src) %{
10271   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
10272   match(Set dst (MulD dst src));
10273   ins_cost(1);   // Select this instruction for all strict FP double multiplies
10274 
10275   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
10276             "DMULp  $dst,ST\n\t"
10277             "FLD    $src\n\t"
10278             "DMULp  $dst,ST\n\t"
10279             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
10280             "DMULp  $dst,ST\n\t" %}
10281   opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
10282   ins_encode( strictfp_bias1(dst),
10283               Push_Reg_D(src),
10284               OpcP, RegOpc(dst),
10285               strictfp_bias2(dst) );
10286   ins_pipe( fpu_reg_reg );
10287 %}
10288 
10289 instruct mulD_reg_imm(regD dst, immD src) %{
10290   predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
10291   match(Set dst (MulD dst src));
10292   ins_cost(200);
10293   format %{ "FLD_D  [$src]\n\t"
10294             "DMULp  $dst,ST" %}
10295   opcode(0xDE, 0x1); /* DE /1 */
10296   ins_encode( LdImmD(src),
10297               OpcP, RegOpc(dst) );
10298   ins_pipe( fpu_reg_mem );
10299 %}
10300 
10301 
10302 instruct mulD_reg_mem(regD dst, memory src) %{
10303   predicate( UseSSE<=1 );
10304   match(Set dst (MulD dst (LoadD src)));
10305   ins_cost(200);
10306   format %{ "FLD_D  $src\n\t"
10307             "DMULp  $dst,ST" %}
10308   opcode(0xDE, 0x1, 0xDD); /* DE C8+i or DE /1*/  /* LoadD  DD /0 */
10309   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
10310               OpcP, RegOpc(dst) );
10311   ins_pipe( fpu_reg_mem );
10312 %}
10313 
10314 //
10315 // Cisc-alternate to reg-reg multiply
10316 instruct mulD_reg_mem_cisc(regD dst, regD src, memory mem) %{
10317   predicate( UseSSE<=1 );
10318   match(Set dst (MulD src (LoadD mem)));
10319   ins_cost(250);
10320   format %{ "FLD_D  $mem\n\t"
10321             "DMUL   ST,$src\n\t"
10322             "FSTP_D $dst" %}
10323   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadD D9 /0 */
10324   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem),
10325               OpcReg_F(src),
10326               Pop_Reg_D(dst) );
10327   ins_pipe( fpu_reg_reg_mem );
10328 %}
10329 
10330 
10331 // MACRO3 -- addD a mulD
10332 // This instruction is a '2-address' instruction in that the result goes
10333 // back to src2.  This eliminates a move from the macro; possibly the
10334 // register allocator will have to add it back (and maybe not).
10335 instruct addD_mulD_reg(regD src2, regD src1, regD src0) %{
10336   predicate( UseSSE<=1 );
10337   match(Set src2 (AddD (MulD src0 src1) src2));
10338   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
10339             "DMUL   ST,$src1\n\t"
10340             "DADDp  $src2,ST" %}
10341   ins_cost(250);
10342   opcode(0xDD); /* LoadD DD /0 */
10343   ins_encode( Push_Reg_F(src0),
10344               FMul_ST_reg(src1),
10345               FAddP_reg_ST(src2) );
10346   ins_pipe( fpu_reg_reg_reg );
10347 %}
10348 
10349 
10350 // MACRO3 -- subD a mulD
10351 instruct subD_mulD_reg(regD src2, regD src1, regD src0) %{
10352   predicate( UseSSE<=1 );
10353   match(Set src2 (SubD (MulD src0 src1) src2));
10354   format %{ "FLD    $src0\t# ===MACRO3d===\n\t"
10355             "DMUL   ST,$src1\n\t"
10356             "DSUBRp $src2,ST" %}
10357   ins_cost(250);
10358   ins_encode( Push_Reg_F(src0),
10359               FMul_ST_reg(src1),
10360               Opcode(0xDE), Opc_plus(0xE0,src2));
10361   ins_pipe( fpu_reg_reg_reg );
10362 %}
10363 
10364 
10365 instruct divD_reg(regD dst, regD src) %{
10366   predicate( UseSSE<=1 );
10367   match(Set dst (DivD dst src));
10368 
10369   format %{ "FLD    $src\n\t"
10370             "FDIVp  $dst,ST" %}
10371   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10372   ins_cost(150);
10373   ins_encode( Push_Reg_D(src),
10374               OpcP, RegOpc(dst) );
10375   ins_pipe( fpu_reg_reg );
10376 %}
10377 
10378 // Strict FP instruction biases argument before division then
10379 // biases result, to avoid double rounding of subnormals.
10380 //
10381 // scale dividend by multiplying dividend by 2^(-15360)
10382 // load divisor
10383 // divide scaled dividend by divisor
10384 // rescale quotient by 2^(15360)
10385 //
10386 instruct strictfp_divD_reg(regDPR1 dst, regnotDPR1 src) %{
10387   predicate (UseSSE<=1);
10388   match(Set dst (DivD dst src));
10389   predicate( UseSSE<=1 && Compile::current()->has_method() && Compile::current()->method()->is_strict() );
10390   ins_cost(01);
10391 
10392   format %{ "FLD    StubRoutines::_fpu_subnormal_bias1\n\t"
10393             "DMULp  $dst,ST\n\t"
10394             "FLD    $src\n\t"
10395             "FDIVp  $dst,ST\n\t"
10396             "FLD    StubRoutines::_fpu_subnormal_bias2\n\t"
10397             "DMULp  $dst,ST\n\t" %}
10398   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
10399   ins_encode( strictfp_bias1(dst),
10400               Push_Reg_D(src),
10401               OpcP, RegOpc(dst),
10402               strictfp_bias2(dst) );
10403   ins_pipe( fpu_reg_reg );
10404 %}
10405 
10406 instruct divD_reg_round(stackSlotD dst, regD src1, regD src2) %{
10407   predicate( UseSSE<=1 && !(Compile::current()->has_method() && Compile::current()->method()->is_strict()) );
10408   match(Set dst (RoundDouble (DivD src1 src2)));
10409 
10410   format %{ "FLD    $src1\n\t"
10411             "FDIV   ST,$src2\n\t"
10412             "FSTP_D $dst\t# D-round" %}
10413   opcode(0xD8, 0x6); /* D8 F0+i or D8 /6 */
10414   ins_encode( Push_Reg_D(src1),
10415               OpcP, RegOpc(src2), Pop_Mem_D(dst) );
10416   ins_pipe( fpu_mem_reg_reg );
10417 %}
10418 
10419 
10420 instruct modD_reg(regD dst, regD src, eAXRegI rax, eFlagsReg cr) %{
10421   predicate(UseSSE<=1);
10422   match(Set dst (ModD dst src));
10423   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
10424 
10425   format %{ "DMOD   $dst,$src" %}
10426   ins_cost(250);
10427   ins_encode(Push_Reg_Mod_D(dst, src),
10428               emitModD(),
10429               Push_Result_Mod_D(src),
10430               Pop_Reg_D(dst));
10431   ins_pipe( pipe_slow );
10432 %}
10433 
10434 instruct modXD_reg(regXD dst, regXD src0, regXD src1, eAXRegI rax, eFlagsReg cr) %{
10435   predicate(UseSSE>=2);
10436   match(Set dst (ModD src0 src1));
10437   effect(KILL rax, KILL cr);
10438 
10439   format %{ "SUB    ESP,8\t # DMOD\n"
10440           "\tMOVSD  [ESP+0],$src1\n"
10441           "\tFLD_D  [ESP+0]\n"
10442           "\tMOVSD  [ESP+0],$src0\n"
10443           "\tFLD_D  [ESP+0]\n"
10444      "loop:\tFPREM\n"
10445           "\tFWAIT\n"
10446           "\tFNSTSW AX\n"
10447           "\tSAHF\n"
10448           "\tJP     loop\n"
10449           "\tFSTP_D [ESP+0]\n"
10450           "\tMOVSD  $dst,[ESP+0]\n"
10451           "\tADD    ESP,8\n"
10452           "\tFSTP   ST0\t # Restore FPU Stack"
10453     %}
10454   ins_cost(250);
10455   ins_encode( Push_ModD_encoding(src0, src1), emitModD(), Push_ResultXD(dst), PopFPU);
10456   ins_pipe( pipe_slow );
10457 %}
10458 
10459 instruct sinD_reg(regDPR1 dst, regDPR1 src) %{
10460   predicate (UseSSE<=1);
10461   match(Set dst (SinD src));
10462   ins_cost(1800);
10463   format %{ "DSIN   $dst" %}
10464   opcode(0xD9, 0xFE);
10465   ins_encode( OpcP, OpcS );
10466   ins_pipe( pipe_slow );
10467 %}
10468 
10469 instruct sinXD_reg(regXD dst, eFlagsReg cr) %{
10470   predicate (UseSSE>=2);
10471   match(Set dst (SinD dst));
10472   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10473   ins_cost(1800);
10474   format %{ "DSIN   $dst" %}
10475   opcode(0xD9, 0xFE);
10476   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10477   ins_pipe( pipe_slow );
10478 %}
10479 
10480 instruct cosD_reg(regDPR1 dst, regDPR1 src) %{
10481   predicate (UseSSE<=1);
10482   match(Set dst (CosD src));
10483   ins_cost(1800);
10484   format %{ "DCOS   $dst" %}
10485   opcode(0xD9, 0xFF);
10486   ins_encode( OpcP, OpcS );
10487   ins_pipe( pipe_slow );
10488 %}
10489 
10490 instruct cosXD_reg(regXD dst, eFlagsReg cr) %{
10491   predicate (UseSSE>=2);
10492   match(Set dst (CosD dst));
10493   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10494   ins_cost(1800);
10495   format %{ "DCOS   $dst" %}
10496   opcode(0xD9, 0xFF);
10497   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10498   ins_pipe( pipe_slow );
10499 %}
10500 
10501 instruct tanD_reg(regDPR1 dst, regDPR1 src) %{
10502   predicate (UseSSE<=1);
10503   match(Set dst(TanD src));
10504   format %{ "DTAN   $dst" %}
10505   ins_encode( Opcode(0xD9), Opcode(0xF2),    // fptan
10506               Opcode(0xDD), Opcode(0xD8));   // fstp st
10507   ins_pipe( pipe_slow );
10508 %}
10509 
10510 instruct tanXD_reg(regXD dst, eFlagsReg cr) %{
10511   predicate (UseSSE>=2);
10512   match(Set dst(TanD dst));
10513   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10514   format %{ "DTAN   $dst" %}
10515   ins_encode( Push_SrcXD(dst),
10516               Opcode(0xD9), Opcode(0xF2),    // fptan
10517               Opcode(0xDD), Opcode(0xD8),   // fstp st
10518               Push_ResultXD(dst) );
10519   ins_pipe( pipe_slow );
10520 %}
10521 
10522 instruct atanD_reg(regD dst, regD src) %{
10523   predicate (UseSSE<=1);
10524   match(Set dst(AtanD dst src));
10525   format %{ "DATA   $dst,$src" %}
10526   opcode(0xD9, 0xF3);
10527   ins_encode( Push_Reg_D(src),
10528               OpcP, OpcS, RegOpc(dst) );
10529   ins_pipe( pipe_slow );
10530 %}
10531 
10532 instruct atanXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
10533   predicate (UseSSE>=2);
10534   match(Set dst(AtanD dst src));
10535   effect(KILL cr); // Push_{Src|Result}XD() uses "{SUB|ADD} ESP,8"
10536   format %{ "DATA   $dst,$src" %}
10537   opcode(0xD9, 0xF3);
10538   ins_encode( Push_SrcXD(src),
10539               OpcP, OpcS, Push_ResultXD(dst) );
10540   ins_pipe( pipe_slow );
10541 %}
10542 
10543 instruct sqrtD_reg(regD dst, regD src) %{
10544   predicate (UseSSE<=1);
10545   match(Set dst (SqrtD src));
10546   format %{ "DSQRT  $dst,$src" %}
10547   opcode(0xFA, 0xD9);
10548   ins_encode( Push_Reg_D(src),
10549               OpcS, OpcP, Pop_Reg_D(dst) );
10550   ins_pipe( pipe_slow );
10551 %}
10552 
10553 instruct powD_reg(regD X, regDPR1 Y, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10554   predicate (UseSSE<=1);
10555   match(Set Y (PowD X Y));  // Raise X to the Yth power
10556   effect(KILL rax, KILL rbx, KILL rcx);
10557   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
10558             "FLD_D  $X\n\t"
10559             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
10560 
10561             "FDUP   \t\t\t# Q Q\n\t"
10562             "FRNDINT\t\t\t# int(Q) Q\n\t"
10563             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10564             "FISTP  dword [ESP]\n\t"
10565             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10566             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10567             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10568             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10569             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10570             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10571             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10572             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10573             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10574             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10575             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10576             "MOV    [ESP+0],0\n\t"
10577             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10578 
10579             "ADD    ESP,8"
10580              %}
10581   ins_encode( push_stack_temp_qword,
10582               Push_Reg_D(X),
10583               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10584               pow_exp_core_encoding,
10585               pop_stack_temp_qword);
10586   ins_pipe( pipe_slow );
10587 %}
10588 
10589 instruct powXD_reg(regXD dst, regXD src0, regXD src1, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx ) %{
10590   predicate (UseSSE>=2);
10591   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
10592   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx );
10593   format %{ "SUB    ESP,8\t\t# Fast-path POW encoding\n\t"
10594             "MOVSD  [ESP],$src1\n\t"
10595             "FLD    FPR1,$src1\n\t"
10596             "MOVSD  [ESP],$src0\n\t"
10597             "FLD    FPR1,$src0\n\t"
10598             "FYL2X  \t\t\t# Q=Y*ln2(X)\n\t"
10599 
10600             "FDUP   \t\t\t# Q Q\n\t"
10601             "FRNDINT\t\t\t# int(Q) Q\n\t"
10602             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10603             "FISTP  dword [ESP]\n\t"
10604             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10605             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10606             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10607             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10608             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10609             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10610             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10611             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10612             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10613             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10614             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10615             "MOV    [ESP+0],0\n\t"
10616             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10617 
10618             "FST_D  [ESP]\n\t"
10619             "MOVSD  $dst,[ESP]\n\t"
10620             "ADD    ESP,8"
10621              %}
10622   ins_encode( push_stack_temp_qword,
10623               push_xmm_to_fpr1(src1),
10624               push_xmm_to_fpr1(src0),
10625               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10626               pow_exp_core_encoding,
10627               Push_ResultXD(dst) );
10628   ins_pipe( pipe_slow );
10629 %}
10630 
10631 
10632 instruct expD_reg(regDPR1 dpr1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10633   predicate (UseSSE<=1);
10634   match(Set dpr1 (ExpD dpr1));
10635   effect(KILL rax, KILL rbx, KILL rcx);
10636   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding"
10637             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
10638             "FMULP  \t\t\t# Q=X*log2(e)\n\t"
10639 
10640             "FDUP   \t\t\t# Q Q\n\t"
10641             "FRNDINT\t\t\t# int(Q) Q\n\t"
10642             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10643             "FISTP  dword [ESP]\n\t"
10644             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10645             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10646             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10647             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10648             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10649             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10650             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10651             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10652             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10653             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10654             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10655             "MOV    [ESP+0],0\n\t"
10656             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10657 
10658             "ADD    ESP,8"
10659              %}
10660   ins_encode( push_stack_temp_qword,
10661               Opcode(0xD9), Opcode(0xEA),   // fldl2e
10662               Opcode(0xDE), Opcode(0xC9),   // fmulp
10663               pow_exp_core_encoding,
10664               pop_stack_temp_qword);
10665   ins_pipe( pipe_slow );
10666 %}
10667 
10668 instruct expXD_reg(regXD dst, regXD src, regDPR1 tmp1, eAXRegI rax, eBXRegI rbx, eCXRegI rcx) %{
10669   predicate (UseSSE>=2);
10670   match(Set dst (ExpD src));
10671   effect(KILL tmp1, KILL rax, KILL rbx, KILL rcx);
10672   format %{ "SUB    ESP,8\t\t# Fast-path EXP encoding\n\t"
10673             "MOVSD  [ESP],$src\n\t"
10674             "FLDL2E \t\t\t# Ld log2(e) X\n\t"
10675             "FMULP  \t\t\t# Q=X*log2(e) X\n\t"
10676 
10677             "FDUP   \t\t\t# Q Q\n\t"
10678             "FRNDINT\t\t\t# int(Q) Q\n\t"
10679             "FSUB   ST(1),ST(0)\t# int(Q) frac(Q)\n\t"
10680             "FISTP  dword [ESP]\n\t"
10681             "F2XM1  \t\t\t# 2^frac(Q)-1 int(Q)\n\t"
10682             "FLD1   \t\t\t# 1 2^frac(Q)-1 int(Q)\n\t"
10683             "FADDP  \t\t\t# 2^frac(Q) int(Q)\n\t" // could use FADD [1.000] instead
10684             "MOV    EAX,[ESP]\t# Pick up int(Q)\n\t"
10685             "MOV    ECX,0xFFFFF800\t# Overflow mask\n\t"
10686             "ADD    EAX,1023\t\t# Double exponent bias\n\t"
10687             "MOV    EBX,EAX\t\t# Preshifted biased expo\n\t"
10688             "SHL    EAX,20\t\t# Shift exponent into place\n\t"
10689             "TEST   EBX,ECX\t\t# Check for overflow\n\t"
10690             "CMOVne EAX,ECX\t\t# If overflow, stuff NaN into EAX\n\t"
10691             "MOV    [ESP+4],EAX\t# Marshal 64-bit scaling double\n\t"
10692             "MOV    [ESP+0],0\n\t"
10693             "FMUL   ST(0),[ESP+0]\t# Scale\n\t"
10694 
10695             "FST_D  [ESP]\n\t"
10696             "MOVSD  $dst,[ESP]\n\t"
10697             "ADD    ESP,8"
10698              %}
10699   ins_encode( Push_SrcXD(src),
10700               Opcode(0xD9), Opcode(0xEA),   // fldl2e
10701               Opcode(0xDE), Opcode(0xC9),   // fmulp
10702               pow_exp_core_encoding,
10703               Push_ResultXD(dst) );
10704   ins_pipe( pipe_slow );
10705 %}
10706 
10707 
10708 
10709 instruct log10D_reg(regDPR1 dst, regDPR1 src) %{
10710   predicate (UseSSE<=1);
10711   // The source Double operand on FPU stack
10712   match(Set dst (Log10D src));
10713   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10714   // fxch         ; swap ST(0) with ST(1)
10715   // fyl2x        ; compute log_10(2) * log_2(x)
10716   format %{ "FLDLG2 \t\t\t#Log10\n\t"
10717             "FXCH   \n\t"
10718             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
10719          %}
10720   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
10721               Opcode(0xD9), Opcode(0xC9),   // fxch
10722               Opcode(0xD9), Opcode(0xF1));  // fyl2x
10723 
10724   ins_pipe( pipe_slow );
10725 %}
10726 
10727 instruct log10XD_reg(regXD dst, regXD src, eFlagsReg cr) %{
10728   predicate (UseSSE>=2);
10729   effect(KILL cr);
10730   match(Set dst (Log10D src));
10731   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10732   // fyl2x        ; compute log_10(2) * log_2(x)
10733   format %{ "FLDLG2 \t\t\t#Log10\n\t"
10734             "FYL2X  \t\t\t# Q=Log10*Log_2(x)"
10735          %}
10736   ins_encode( Opcode(0xD9), Opcode(0xEC),   // fldlg2
10737               Push_SrcXD(src),
10738               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10739               Push_ResultXD(dst));
10740 
10741   ins_pipe( pipe_slow );
10742 %}
10743 
10744 instruct logD_reg(regDPR1 dst, regDPR1 src) %{
10745   predicate (UseSSE<=1);
10746   // The source Double operand on FPU stack
10747   match(Set dst (LogD src));
10748   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
10749   // fxch         ; swap ST(0) with ST(1)
10750   // fyl2x        ; compute log_e(2) * log_2(x)
10751   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
10752             "FXCH   \n\t"
10753             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
10754          %}
10755   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
10756               Opcode(0xD9), Opcode(0xC9),   // fxch
10757               Opcode(0xD9), Opcode(0xF1));  // fyl2x
10758 
10759   ins_pipe( pipe_slow );
10760 %}
10761 
10762 instruct logXD_reg(regXD dst, regXD src, eFlagsReg cr) %{
10763   predicate (UseSSE>=2);
10764   effect(KILL cr);
10765   // The source and result Double operands in XMM registers
10766   match(Set dst (LogD src));
10767   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
10768   // fyl2x        ; compute log_e(2) * log_2(x)
10769   format %{ "FLDLN2 \t\t\t#Log_e\n\t"
10770             "FYL2X  \t\t\t# Q=Log_e*Log_2(x)"
10771          %}
10772   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
10773               Push_SrcXD(src),
10774               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10775               Push_ResultXD(dst));
10776   ins_pipe( pipe_slow );
10777 %}
10778 
10779 //-------------Float Instructions-------------------------------
10780 // Float Math
10781 
10782 // Code for float compare:
10783 //     fcompp();
10784 //     fwait(); fnstsw_ax();
10785 //     sahf();
10786 //     movl(dst, unordered_result);
10787 //     jcc(Assembler::parity, exit);
10788 //     movl(dst, less_result);
10789 //     jcc(Assembler::below, exit);
10790 //     movl(dst, equal_result);
10791 //     jcc(Assembler::equal, exit);
10792 //     movl(dst, greater_result);
10793 //   exit:
10794 
10795 // P6 version of float compare, sets condition codes in EFLAGS
10796 instruct cmpF_cc_P6(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
10797   predicate(VM_Version::supports_cmov() && UseSSE == 0);
10798   match(Set cr (CmpF src1 src2));
10799   effect(KILL rax);
10800   ins_cost(150);
10801   format %{ "FLD    $src1\n\t"
10802             "FUCOMIP ST,$src2  // P6 instruction\n\t"
10803             "JNP    exit\n\t"
10804             "MOV    ah,1       // saw a NaN, set CF (treat as LT)\n\t"
10805             "SAHF\n"
10806      "exit:\tNOP               // avoid branch to branch" %}
10807   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10808   ins_encode( Push_Reg_D(src1),
10809               OpcP, RegOpc(src2),
10810               cmpF_P6_fixup );
10811   ins_pipe( pipe_slow );
10812 %}
10813 
10814 instruct cmpF_cc_P6CF(eFlagsRegUCF cr, regF src1, regF src2) %{
10815   predicate(VM_Version::supports_cmov() && UseSSE == 0);
10816   match(Set cr (CmpF src1 src2));
10817   ins_cost(100);
10818   format %{ "FLD    $src1\n\t"
10819             "FUCOMIP ST,$src2  // P6 instruction" %}
10820   opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
10821   ins_encode( Push_Reg_D(src1),
10822               OpcP, RegOpc(src2));
10823   ins_pipe( pipe_slow );
10824 %}
10825 
10826 
10827 // Compare & branch
10828 instruct cmpF_cc(eFlagsRegU cr, regF src1, regF src2, eAXRegI rax) %{
10829   predicate(UseSSE == 0);
10830   match(Set cr (CmpF src1 src2));
10831   effect(KILL rax);
10832   ins_cost(200);
10833   format %{ "FLD    $src1\n\t"
10834             "FCOMp  $src2\n\t"
10835             "FNSTSW AX\n\t"
10836             "TEST   AX,0x400\n\t"
10837             "JZ,s   flags\n\t"
10838             "MOV    AH,1\t# unordered treat as LT\n"
10839     "flags:\tSAHF" %}
10840   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10841   ins_encode( Push_Reg_D(src1),
10842               OpcP, RegOpc(src2),
10843               fpu_flags);
10844   ins_pipe( pipe_slow );
10845 %}
10846 
10847 // Compare vs zero into -1,0,1
10848 instruct cmpF_0(eRegI dst, regF src1, immF0 zero, eAXRegI rax, eFlagsReg cr) %{
10849   predicate(UseSSE == 0);
10850   match(Set dst (CmpF3 src1 zero));
10851   effect(KILL cr, KILL rax);
10852   ins_cost(280);
10853   format %{ "FTSTF  $dst,$src1" %}
10854   opcode(0xE4, 0xD9);
10855   ins_encode( Push_Reg_D(src1),
10856               OpcS, OpcP, PopFPU,
10857               CmpF_Result(dst));
10858   ins_pipe( pipe_slow );
10859 %}
10860 
10861 // Compare into -1,0,1
10862 instruct cmpF_reg(eRegI dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
10863   predicate(UseSSE == 0);
10864   match(Set dst (CmpF3 src1 src2));
10865   effect(KILL cr, KILL rax);
10866   ins_cost(300);
10867   format %{ "FCMPF  $dst,$src1,$src2" %}
10868   opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
10869   ins_encode( Push_Reg_D(src1),
10870               OpcP, RegOpc(src2),
10871               CmpF_Result(dst));
10872   ins_pipe( pipe_slow );
10873 %}
10874 
10875 // float compare and set condition codes in EFLAGS by XMM regs
10876 instruct cmpX_cc(eFlagsRegU cr, regX dst, regX src, eAXRegI rax) %{
10877   predicate(UseSSE>=1);
10878   match(Set cr (CmpF dst src));
10879   effect(KILL rax);
10880   ins_cost(145);
10881   format %{ "COMISS $dst,$src\n"
10882           "\tJNP    exit\n"
10883           "\tMOV    ah,1       // saw a NaN, set CF\n"
10884           "\tSAHF\n"
10885      "exit:\tNOP               // avoid branch to branch" %}
10886   opcode(0x0F, 0x2F);
10887   ins_encode(OpcP, OpcS, RegReg(dst, src), cmpF_P6_fixup);
10888   ins_pipe( pipe_slow );
10889 %}
10890 
10891 instruct cmpX_ccCF(eFlagsRegUCF cr, regX dst, regX src) %{
10892   predicate(UseSSE>=1);
10893   match(Set cr (CmpF dst src));
10894   ins_cost(100);
10895   format %{ "COMISS $dst,$src" %}
10896   opcode(0x0F, 0x2F);
10897   ins_encode(OpcP, OpcS, RegReg(dst, src));
10898   ins_pipe( pipe_slow );
10899 %}
10900 
10901 // float compare and set condition codes in EFLAGS by XMM regs
10902 instruct cmpX_ccmem(eFlagsRegU cr, regX dst, memory src, eAXRegI rax) %{
10903   predicate(UseSSE>=1);
10904   match(Set cr (CmpF dst (LoadF src)));
10905   effect(KILL rax);
10906   ins_cost(165);
10907   format %{ "COMISS $dst,$src\n"
10908           "\tJNP    exit\n"
10909           "\tMOV    ah,1       // saw a NaN, set CF\n"
10910           "\tSAHF\n"
10911      "exit:\tNOP               // avoid branch to branch" %}
10912   opcode(0x0F, 0x2F);
10913   ins_encode(OpcP, OpcS, RegMem(dst, src), cmpF_P6_fixup);
10914   ins_pipe( pipe_slow );
10915 %}
10916 
10917 instruct cmpX_ccmemCF(eFlagsRegUCF cr, regX dst, memory src) %{
10918   predicate(UseSSE>=1);
10919   match(Set cr (CmpF dst (LoadF src)));
10920   ins_cost(100);
10921   format %{ "COMISS $dst,$src" %}
10922   opcode(0x0F, 0x2F);
10923   ins_encode(OpcP, OpcS, RegMem(dst, src));
10924   ins_pipe( pipe_slow );
10925 %}
10926 
10927 // Compare into -1,0,1 in XMM
10928 instruct cmpX_reg(eRegI dst, regX src1, regX src2, eFlagsReg cr) %{
10929   predicate(UseSSE>=1);
10930   match(Set dst (CmpF3 src1 src2));
10931   effect(KILL cr);
10932   ins_cost(255);
10933   format %{ "XOR    $dst,$dst\n"
10934           "\tCOMISS $src1,$src2\n"
10935           "\tJP,s   nan\n"
10936           "\tJEQ,s  exit\n"
10937           "\tJA,s   inc\n"
10938       "nan:\tDEC    $dst\n"
10939           "\tJMP,s  exit\n"
10940       "inc:\tINC    $dst\n"
10941       "exit:"
10942                 %}
10943   opcode(0x0F, 0x2F);
10944   ins_encode(Xor_Reg(dst), OpcP, OpcS, RegReg(src1, src2), CmpX_Result(dst));
10945   ins_pipe( pipe_slow );
10946 %}
10947 
10948 // Compare into -1,0,1 in XMM and memory
10949 instruct cmpX_regmem(eRegI dst, regX src1, memory mem, eFlagsReg cr) %{
10950   predicate(UseSSE>=1);
10951   match(Set dst (CmpF3 src1 (LoadF mem)));
10952   effect(KILL cr);
10953   ins_cost(275);
10954   format %{ "COMISS $src1,$mem\n"
10955           "\tMOV    $dst,0\t\t# do not blow flags\n"
10956           "\tJP,s   nan\n"
10957           "\tJEQ,s  exit\n"
10958           "\tJA,s   inc\n"
10959       "nan:\tDEC    $dst\n"
10960           "\tJMP,s  exit\n"
10961       "inc:\tINC    $dst\n"
10962       "exit:"
10963                 %}
10964   opcode(0x0F, 0x2F);
10965   ins_encode(OpcP, OpcS, RegMem(src1, mem), LdImmI(dst,0x0), CmpX_Result(dst));
10966   ins_pipe( pipe_slow );
10967 %}
10968 
10969 // Spill to obtain 24-bit precision
10970 instruct subF24_reg(stackSlotF dst, regF src1, regF src2) %{
10971   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10972   match(Set dst (SubF src1 src2));
10973 
10974   format %{ "FSUB   $dst,$src1 - $src2" %}
10975   opcode(0xD8, 0x4); /* D8 E0+i or D8 /4 mod==0x3 ;; result in TOS */
10976   ins_encode( Push_Reg_F(src1),
10977               OpcReg_F(src2),
10978               Pop_Mem_F(dst) );
10979   ins_pipe( fpu_mem_reg_reg );
10980 %}
10981 //
10982 // This instruction does not round to 24-bits
10983 instruct subF_reg(regF dst, regF src) %{
10984   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
10985   match(Set dst (SubF dst src));
10986 
10987   format %{ "FSUB   $dst,$src" %}
10988   opcode(0xDE, 0x5); /* DE E8+i  or DE /5 */
10989   ins_encode( Push_Reg_F(src),
10990               OpcP, RegOpc(dst) );
10991   ins_pipe( fpu_reg_reg );
10992 %}
10993 
10994 // Spill to obtain 24-bit precision
10995 instruct addF24_reg(stackSlotF dst, regF src1, regF src2) %{
10996   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
10997   match(Set dst (AddF src1 src2));
10998 
10999   format %{ "FADD   $dst,$src1,$src2" %}
11000   opcode(0xD8, 0x0); /* D8 C0+i */
11001   ins_encode( Push_Reg_F(src2),
11002               OpcReg_F(src1),
11003               Pop_Mem_F(dst) );
11004   ins_pipe( fpu_mem_reg_reg );
11005 %}
11006 //
11007 // This instruction does not round to 24-bits
11008 instruct addF_reg(regF dst, regF src) %{
11009   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11010   match(Set dst (AddF dst src));
11011 
11012   format %{ "FLD    $src\n\t"
11013             "FADDp  $dst,ST" %}
11014   opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
11015   ins_encode( Push_Reg_F(src),
11016               OpcP, RegOpc(dst) );
11017   ins_pipe( fpu_reg_reg );
11018 %}
11019 
11020 // Add two single precision floating point values in xmm
11021 instruct addX_reg(regX dst, regX src) %{
11022   predicate(UseSSE>=1);
11023   match(Set dst (AddF dst src));
11024   format %{ "ADDSS  $dst,$src" %}
11025   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegReg(dst, src));
11026   ins_pipe( pipe_slow );
11027 %}
11028 
11029 instruct addX_imm(regX dst, immXF con) %{
11030   predicate(UseSSE>=1);
11031   match(Set dst (AddF dst con));
11032   format %{ "ADDSS  $dst,[$con]" %}
11033   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), LdImmX(dst, con) );
11034   ins_pipe( pipe_slow );
11035 %}
11036 
11037 instruct addX_mem(regX dst, memory mem) %{
11038   predicate(UseSSE>=1);
11039   match(Set dst (AddF dst (LoadF mem)));
11040   format %{ "ADDSS  $dst,$mem" %}
11041   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x58), RegMem(dst, mem));
11042   ins_pipe( pipe_slow );
11043 %}
11044 
11045 // Subtract two single precision floating point values in xmm
11046 instruct subX_reg(regX dst, regX src) %{
11047   predicate(UseSSE>=1);
11048   match(Set dst (SubF dst src));
11049   format %{ "SUBSS  $dst,$src" %}
11050   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegReg(dst, src));
11051   ins_pipe( pipe_slow );
11052 %}
11053 
11054 instruct subX_imm(regX dst, immXF con) %{
11055   predicate(UseSSE>=1);
11056   match(Set dst (SubF dst con));
11057   format %{ "SUBSS  $dst,[$con]" %}
11058   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), LdImmX(dst, con) );
11059   ins_pipe( pipe_slow );
11060 %}
11061 
11062 instruct subX_mem(regX dst, memory mem) %{
11063   predicate(UseSSE>=1);
11064   match(Set dst (SubF dst (LoadF mem)));
11065   format %{ "SUBSS  $dst,$mem" %}
11066   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5C), RegMem(dst,mem));
11067   ins_pipe( pipe_slow );
11068 %}
11069 
11070 // Multiply two single precision floating point values in xmm
11071 instruct mulX_reg(regX dst, regX src) %{
11072   predicate(UseSSE>=1);
11073   match(Set dst (MulF dst src));
11074   format %{ "MULSS  $dst,$src" %}
11075   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegReg(dst, src));
11076   ins_pipe( pipe_slow );
11077 %}
11078 
11079 instruct mulX_imm(regX dst, immXF con) %{
11080   predicate(UseSSE>=1);
11081   match(Set dst (MulF dst con));
11082   format %{ "MULSS  $dst,[$con]" %}
11083   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), LdImmX(dst, con) );
11084   ins_pipe( pipe_slow );
11085 %}
11086 
11087 instruct mulX_mem(regX dst, memory mem) %{
11088   predicate(UseSSE>=1);
11089   match(Set dst (MulF dst (LoadF mem)));
11090   format %{ "MULSS  $dst,$mem" %}
11091   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x59), RegMem(dst,mem));
11092   ins_pipe( pipe_slow );
11093 %}
11094 
11095 // Divide two single precision floating point values in xmm
11096 instruct divX_reg(regX dst, regX src) %{
11097   predicate(UseSSE>=1);
11098   match(Set dst (DivF dst src));
11099   format %{ "DIVSS  $dst,$src" %}
11100   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegReg(dst, src));
11101   ins_pipe( pipe_slow );
11102 %}
11103 
11104 instruct divX_imm(regX dst, immXF con) %{
11105   predicate(UseSSE>=1);
11106   match(Set dst (DivF dst con));
11107   format %{ "DIVSS  $dst,[$con]" %}
11108   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), LdImmX(dst, con) );
11109   ins_pipe( pipe_slow );
11110 %}
11111 
11112 instruct divX_mem(regX dst, memory mem) %{
11113   predicate(UseSSE>=1);
11114   match(Set dst (DivF dst (LoadF mem)));
11115   format %{ "DIVSS  $dst,$mem" %}
11116   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x5E), RegMem(dst,mem));
11117   ins_pipe( pipe_slow );
11118 %}
11119 
11120 // Get the square root of a single precision floating point values in xmm
11121 instruct sqrtX_reg(regX dst, regX src) %{
11122   predicate(UseSSE>=1);
11123   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
11124   format %{ "SQRTSS $dst,$src" %}
11125   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
11126   ins_pipe( pipe_slow );
11127 %}
11128 
11129 instruct sqrtX_mem(regX dst, memory mem) %{
11130   predicate(UseSSE>=1);
11131   match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF mem)))));
11132   format %{ "SQRTSS $dst,$mem" %}
11133   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
11134   ins_pipe( pipe_slow );
11135 %}
11136 
11137 // Get the square root of a double precision floating point values in xmm
11138 instruct sqrtXD_reg(regXD dst, regXD src) %{
11139   predicate(UseSSE>=2);
11140   match(Set dst (SqrtD src));
11141   format %{ "SQRTSD $dst,$src" %}
11142   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegReg(dst, src));
11143   ins_pipe( pipe_slow );
11144 %}
11145 
11146 instruct sqrtXD_mem(regXD dst, memory mem) %{
11147   predicate(UseSSE>=2);
11148   match(Set dst (SqrtD (LoadD mem)));
11149   format %{ "SQRTSD $dst,$mem" %}
11150   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x51), RegMem(dst, mem));
11151   ins_pipe( pipe_slow );
11152 %}
11153 
11154 instruct absF_reg(regFPR1 dst, regFPR1 src) %{
11155   predicate(UseSSE==0);
11156   match(Set dst (AbsF src));
11157   ins_cost(100);
11158   format %{ "FABS" %}
11159   opcode(0xE1, 0xD9);
11160   ins_encode( OpcS, OpcP );
11161   ins_pipe( fpu_reg_reg );
11162 %}
11163 
11164 instruct absX_reg(regX dst ) %{
11165   predicate(UseSSE>=1);
11166   match(Set dst (AbsF dst));
11167   format %{ "ANDPS  $dst,[0x7FFFFFFF]\t# ABS F by sign masking" %}
11168   ins_encode( AbsXF_encoding(dst));
11169   ins_pipe( pipe_slow );
11170 %}
11171 
11172 instruct negF_reg(regFPR1 dst, regFPR1 src) %{
11173   predicate(UseSSE==0);
11174   match(Set dst (NegF src));
11175   ins_cost(100);
11176   format %{ "FCHS" %}
11177   opcode(0xE0, 0xD9);
11178   ins_encode( OpcS, OpcP );
11179   ins_pipe( fpu_reg_reg );
11180 %}
11181 
11182 instruct negX_reg( regX dst ) %{
11183   predicate(UseSSE>=1);
11184   match(Set dst (NegF dst));
11185   format %{ "XORPS  $dst,[0x80000000]\t# CHS F by sign flipping" %}
11186   ins_encode( NegXF_encoding(dst));
11187   ins_pipe( pipe_slow );
11188 %}
11189 
11190 // Cisc-alternate to addF_reg
11191 // Spill to obtain 24-bit precision
11192 instruct addF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
11193   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11194   match(Set dst (AddF src1 (LoadF src2)));
11195 
11196   format %{ "FLD    $src2\n\t"
11197             "FADD   ST,$src1\n\t"
11198             "FSTP_S $dst" %}
11199   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11200   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11201               OpcReg_F(src1),
11202               Pop_Mem_F(dst) );
11203   ins_pipe( fpu_mem_reg_mem );
11204 %}
11205 //
11206 // Cisc-alternate to addF_reg
11207 // This instruction does not round to 24-bits
11208 instruct addF_reg_mem(regF dst, memory src) %{
11209   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11210   match(Set dst (AddF dst (LoadF src)));
11211 
11212   format %{ "FADD   $dst,$src" %}
11213   opcode(0xDE, 0x0, 0xD9); /* DE C0+i or DE /0*/  /* LoadF  D9 /0 */
11214   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src),
11215               OpcP, RegOpc(dst) );
11216   ins_pipe( fpu_reg_mem );
11217 %}
11218 
11219 // // Following two instructions for _222_mpegaudio
11220 // Spill to obtain 24-bit precision
11221 instruct addF24_mem_reg(stackSlotF dst, regF src2, memory src1 ) %{
11222   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11223   match(Set dst (AddF src1 src2));
11224 
11225   format %{ "FADD   $dst,$src1,$src2" %}
11226   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11227   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src1),
11228               OpcReg_F(src2),
11229               Pop_Mem_F(dst) );
11230   ins_pipe( fpu_mem_reg_mem );
11231 %}
11232 
11233 // Cisc-spill variant
11234 // Spill to obtain 24-bit precision
11235 instruct addF24_mem_cisc(stackSlotF dst, memory src1, memory src2) %{
11236   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11237   match(Set dst (AddF src1 (LoadF src2)));
11238 
11239   format %{ "FADD   $dst,$src1,$src2 cisc" %}
11240   opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */  /* LoadF  D9 /0 */
11241   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11242               set_instruction_start,
11243               OpcP, RMopc_Mem(secondary,src1),
11244               Pop_Mem_F(dst) );
11245   ins_pipe( fpu_mem_mem_mem );
11246 %}
11247 
11248 // Spill to obtain 24-bit precision
11249 instruct addF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
11250   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11251   match(Set dst (AddF src1 src2));
11252 
11253   format %{ "FADD   $dst,$src1,$src2" %}
11254   opcode(0xD8, 0x0, 0xD9); /* D8 /0 */  /* LoadF  D9 /0 */
11255   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11256               set_instruction_start,
11257               OpcP, RMopc_Mem(secondary,src1),
11258               Pop_Mem_F(dst) );
11259   ins_pipe( fpu_mem_mem_mem );
11260 %}
11261 
11262 
11263 // Spill to obtain 24-bit precision
11264 instruct addF24_reg_imm(stackSlotF dst, regF src1, immF src2) %{
11265   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11266   match(Set dst (AddF src1 src2));
11267   format %{ "FLD    $src1\n\t"
11268             "FADD   $src2\n\t"
11269             "FSTP_S $dst"  %}
11270   opcode(0xD8, 0x00);       /* D8 /0 */
11271   ins_encode( Push_Reg_F(src1),
11272               Opc_MemImm_F(src2),
11273               Pop_Mem_F(dst));
11274   ins_pipe( fpu_mem_reg_con );
11275 %}
11276 //
11277 // This instruction does not round to 24-bits
11278 instruct addF_reg_imm(regF dst, regF src1, immF src2) %{
11279   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11280   match(Set dst (AddF src1 src2));
11281   format %{ "FLD    $src1\n\t"
11282             "FADD   $src2\n\t"
11283             "FSTP_S $dst"  %}
11284   opcode(0xD8, 0x00);       /* D8 /0 */
11285   ins_encode( Push_Reg_F(src1),
11286               Opc_MemImm_F(src2),
11287               Pop_Reg_F(dst));
11288   ins_pipe( fpu_reg_reg_con );
11289 %}
11290 
11291 // Spill to obtain 24-bit precision
11292 instruct mulF24_reg(stackSlotF dst, regF src1, regF src2) %{
11293   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11294   match(Set dst (MulF src1 src2));
11295 
11296   format %{ "FLD    $src1\n\t"
11297             "FMUL   $src2\n\t"
11298             "FSTP_S $dst"  %}
11299   opcode(0xD8, 0x1); /* D8 C8+i or D8 /1 ;; result in TOS */
11300   ins_encode( Push_Reg_F(src1),
11301               OpcReg_F(src2),
11302               Pop_Mem_F(dst) );
11303   ins_pipe( fpu_mem_reg_reg );
11304 %}
11305 //
11306 // This instruction does not round to 24-bits
11307 instruct mulF_reg(regF dst, regF src1, regF src2) %{
11308   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11309   match(Set dst (MulF src1 src2));
11310 
11311   format %{ "FLD    $src1\n\t"
11312             "FMUL   $src2\n\t"
11313             "FSTP_S $dst"  %}
11314   opcode(0xD8, 0x1); /* D8 C8+i */
11315   ins_encode( Push_Reg_F(src2),
11316               OpcReg_F(src1),
11317               Pop_Reg_F(dst) );
11318   ins_pipe( fpu_reg_reg_reg );
11319 %}
11320 
11321 
11322 // Spill to obtain 24-bit precision
11323 // Cisc-alternate to reg-reg multiply
11324 instruct mulF24_reg_mem(stackSlotF dst, regF src1, memory src2) %{
11325   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11326   match(Set dst (MulF src1 (LoadF src2)));
11327 
11328   format %{ "FLD_S  $src2\n\t"
11329             "FMUL   $src1\n\t"
11330             "FSTP_S $dst"  %}
11331   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or DE /1*/  /* LoadF D9 /0 */
11332   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11333               OpcReg_F(src1),
11334               Pop_Mem_F(dst) );
11335   ins_pipe( fpu_mem_reg_mem );
11336 %}
11337 //
11338 // This instruction does not round to 24-bits
11339 // Cisc-alternate to reg-reg multiply
11340 instruct mulF_reg_mem(regF dst, regF src1, memory src2) %{
11341   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11342   match(Set dst (MulF src1 (LoadF src2)));
11343 
11344   format %{ "FMUL   $dst,$src1,$src2" %}
11345   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */  /* LoadF D9 /0 */
11346   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11347               OpcReg_F(src1),
11348               Pop_Reg_F(dst) );
11349   ins_pipe( fpu_reg_reg_mem );
11350 %}
11351 
11352 // Spill to obtain 24-bit precision
11353 instruct mulF24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
11354   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11355   match(Set dst (MulF src1 src2));
11356 
11357   format %{ "FMUL   $dst,$src1,$src2" %}
11358   opcode(0xD8, 0x1, 0xD9); /* D8 /1 */  /* LoadF D9 /0 */
11359   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,src2),
11360               set_instruction_start,
11361               OpcP, RMopc_Mem(secondary,src1),
11362               Pop_Mem_F(dst) );
11363   ins_pipe( fpu_mem_mem_mem );
11364 %}
11365 
11366 // Spill to obtain 24-bit precision
11367 instruct mulF24_reg_imm(stackSlotF dst, regF src1, immF src2) %{
11368   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11369   match(Set dst (MulF src1 src2));
11370 
11371   format %{ "FMULc $dst,$src1,$src2" %}
11372   opcode(0xD8, 0x1);  /* D8 /1*/
11373   ins_encode( Push_Reg_F(src1),
11374               Opc_MemImm_F(src2),
11375               Pop_Mem_F(dst));
11376   ins_pipe( fpu_mem_reg_con );
11377 %}
11378 //
11379 // This instruction does not round to 24-bits
11380 instruct mulF_reg_imm(regF dst, regF src1, immF src2) %{
11381   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11382   match(Set dst (MulF src1 src2));
11383 
11384   format %{ "FMULc $dst. $src1, $src2" %}
11385   opcode(0xD8, 0x1);  /* D8 /1*/
11386   ins_encode( Push_Reg_F(src1),
11387               Opc_MemImm_F(src2),
11388               Pop_Reg_F(dst));
11389   ins_pipe( fpu_reg_reg_con );
11390 %}
11391 
11392 
11393 //
11394 // MACRO1 -- subsume unshared load into mulF
11395 // This instruction does not round to 24-bits
11396 instruct mulF_reg_load1(regF dst, regF src, memory mem1 ) %{
11397   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11398   match(Set dst (MulF (LoadF mem1) src));
11399 
11400   format %{ "FLD    $mem1    ===MACRO1===\n\t"
11401             "FMUL   ST,$src\n\t"
11402             "FSTP   $dst" %}
11403   opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or D8 /1 */  /* LoadF D9 /0 */
11404   ins_encode( Opcode(tertiary), RMopc_Mem(0x00,mem1),
11405               OpcReg_F(src),
11406               Pop_Reg_F(dst) );
11407   ins_pipe( fpu_reg_reg_mem );
11408 %}
11409 //
11410 // MACRO2 -- addF a mulF which subsumed an unshared load
11411 // This instruction does not round to 24-bits
11412 instruct addF_mulF_reg_load1(regF dst, memory mem1, regF src1, regF src2) %{
11413   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11414   match(Set dst (AddF (MulF (LoadF mem1) src1) src2));
11415   ins_cost(95);
11416 
11417   format %{ "FLD    $mem1     ===MACRO2===\n\t"
11418             "FMUL   ST,$src1  subsume mulF left load\n\t"
11419             "FADD   ST,$src2\n\t"
11420             "FSTP   $dst" %}
11421   opcode(0xD9); /* LoadF D9 /0 */
11422   ins_encode( OpcP, RMopc_Mem(0x00,mem1),
11423               FMul_ST_reg(src1),
11424               FAdd_ST_reg(src2),
11425               Pop_Reg_F(dst) );
11426   ins_pipe( fpu_reg_mem_reg_reg );
11427 %}
11428 
11429 // MACRO3 -- addF a mulF
11430 // This instruction does not round to 24-bits.  It is a '2-address'
11431 // instruction in that the result goes back to src2.  This eliminates
11432 // a move from the macro; possibly the register allocator will have
11433 // to add it back (and maybe not).
11434 instruct addF_mulF_reg(regF src2, regF src1, regF src0) %{
11435   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11436   match(Set src2 (AddF (MulF src0 src1) src2));
11437 
11438   format %{ "FLD    $src0     ===MACRO3===\n\t"
11439             "FMUL   ST,$src1\n\t"
11440             "FADDP  $src2,ST" %}
11441   opcode(0xD9); /* LoadF D9 /0 */
11442   ins_encode( Push_Reg_F(src0),
11443               FMul_ST_reg(src1),
11444               FAddP_reg_ST(src2) );
11445   ins_pipe( fpu_reg_reg_reg );
11446 %}
11447 
11448 // MACRO4 -- divF subF
11449 // This instruction does not round to 24-bits
11450 instruct subF_divF_reg(regF dst, regF src1, regF src2, regF src3) %{
11451   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11452   match(Set dst (DivF (SubF src2 src1) src3));
11453 
11454   format %{ "FLD    $src2   ===MACRO4===\n\t"
11455             "FSUB   ST,$src1\n\t"
11456             "FDIV   ST,$src3\n\t"
11457             "FSTP  $dst" %}
11458   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
11459   ins_encode( Push_Reg_F(src2),
11460               subF_divF_encode(src1,src3),
11461               Pop_Reg_F(dst) );
11462   ins_pipe( fpu_reg_reg_reg_reg );
11463 %}
11464 
11465 // Spill to obtain 24-bit precision
11466 instruct divF24_reg(stackSlotF dst, regF src1, regF src2) %{
11467   predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
11468   match(Set dst (DivF src1 src2));
11469 
11470   format %{ "FDIV   $dst,$src1,$src2" %}
11471   opcode(0xD8, 0x6); /* D8 F0+i or DE /6*/
11472   ins_encode( Push_Reg_F(src1),
11473               OpcReg_F(src2),
11474               Pop_Mem_F(dst) );
11475   ins_pipe( fpu_mem_reg_reg );
11476 %}
11477 //
11478 // This instruction does not round to 24-bits
11479 instruct divF_reg(regF dst, regF src) %{
11480   predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
11481   match(Set dst (DivF dst src));
11482 
11483   format %{ "FDIV   $dst,$src" %}
11484   opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
11485   ins_encode( Push_Reg_F(src),
11486               OpcP, RegOpc(dst) );
11487   ins_pipe( fpu_reg_reg );
11488 %}
11489 
11490 
11491 // Spill to obtain 24-bit precision
11492 instruct modF24_reg(stackSlotF dst, regF src1, regF src2, eAXRegI rax, eFlagsReg cr) %{
11493   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11494   match(Set dst (ModF src1 src2));
11495   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
11496 
11497   format %{ "FMOD   $dst,$src1,$src2" %}
11498   ins_encode( Push_Reg_Mod_D(src1, src2),
11499               emitModD(),
11500               Push_Result_Mod_D(src2),
11501               Pop_Mem_F(dst));
11502   ins_pipe( pipe_slow );
11503 %}
11504 //
11505 // This instruction does not round to 24-bits
11506 instruct modF_reg(regF dst, regF src, eAXRegI rax, eFlagsReg cr) %{
11507   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11508   match(Set dst (ModF dst src));
11509   effect(KILL rax, KILL cr); // emitModD() uses EAX and EFLAGS
11510 
11511   format %{ "FMOD   $dst,$src" %}
11512   ins_encode(Push_Reg_Mod_D(dst, src),
11513               emitModD(),
11514               Push_Result_Mod_D(src),
11515               Pop_Reg_F(dst));
11516   ins_pipe( pipe_slow );
11517 %}
11518 
11519 instruct modX_reg(regX dst, regX src0, regX src1, eAXRegI rax, eFlagsReg cr) %{
11520   predicate(UseSSE>=1);
11521   match(Set dst (ModF src0 src1));
11522   effect(KILL rax, KILL cr);
11523   format %{ "SUB    ESP,4\t # FMOD\n"
11524           "\tMOVSS  [ESP+0],$src1\n"
11525           "\tFLD_S  [ESP+0]\n"
11526           "\tMOVSS  [ESP+0],$src0\n"
11527           "\tFLD_S  [ESP+0]\n"
11528      "loop:\tFPREM\n"
11529           "\tFWAIT\n"
11530           "\tFNSTSW AX\n"
11531           "\tSAHF\n"
11532           "\tJP     loop\n"
11533           "\tFSTP_S [ESP+0]\n"
11534           "\tMOVSS  $dst,[ESP+0]\n"
11535           "\tADD    ESP,4\n"
11536           "\tFSTP   ST0\t # Restore FPU Stack"
11537     %}
11538   ins_cost(250);
11539   ins_encode( Push_ModX_encoding(src0, src1), emitModD(), Push_ResultX(dst,0x4), PopFPU);
11540   ins_pipe( pipe_slow );
11541 %}
11542 
11543 
11544 //----------Arithmetic Conversion Instructions---------------------------------
11545 // The conversions operations are all Alpha sorted.  Please keep it that way!
11546 
11547 instruct roundFloat_mem_reg(stackSlotF dst, regF src) %{
11548   predicate(UseSSE==0);
11549   match(Set dst (RoundFloat src));
11550   ins_cost(125);
11551   format %{ "FST_S  $dst,$src\t# F-round" %}
11552   ins_encode( Pop_Mem_Reg_F(dst, src) );
11553   ins_pipe( fpu_mem_reg );
11554 %}
11555 
11556 instruct roundDouble_mem_reg(stackSlotD dst, regD src) %{
11557   predicate(UseSSE<=1);
11558   match(Set dst (RoundDouble src));
11559   ins_cost(125);
11560   format %{ "FST_D  $dst,$src\t# D-round" %}
11561   ins_encode( Pop_Mem_Reg_D(dst, src) );
11562   ins_pipe( fpu_mem_reg );
11563 %}
11564 
11565 // Force rounding to 24-bit precision and 6-bit exponent
11566 instruct convD2F_reg(stackSlotF dst, regD src) %{
11567   predicate(UseSSE==0);
11568   match(Set dst (ConvD2F src));
11569   format %{ "FST_S  $dst,$src\t# F-round" %}
11570   expand %{
11571     roundFloat_mem_reg(dst,src);
11572   %}
11573 %}
11574 
11575 // Force rounding to 24-bit precision and 6-bit exponent
11576 instruct convD2X_reg(regX dst, regD src, eFlagsReg cr) %{
11577   predicate(UseSSE==1);
11578   match(Set dst (ConvD2F src));
11579   effect( KILL cr );
11580   format %{ "SUB    ESP,4\n\t"
11581             "FST_S  [ESP],$src\t# F-round\n\t"
11582             "MOVSS  $dst,[ESP]\n\t"
11583             "ADD ESP,4" %}
11584   ins_encode( D2X_encoding(dst, src) );
11585   ins_pipe( pipe_slow );
11586 %}
11587 
11588 // Force rounding double precision to single precision
11589 instruct convXD2X_reg(regX dst, regXD src) %{
11590   predicate(UseSSE>=2);
11591   match(Set dst (ConvD2F src));
11592   format %{ "CVTSD2SS $dst,$src\t# F-round" %}
11593   opcode(0xF2, 0x0F, 0x5A);
11594   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11595   ins_pipe( pipe_slow );
11596 %}
11597 
11598 instruct convF2D_reg_reg(regD dst, regF src) %{
11599   predicate(UseSSE==0);
11600   match(Set dst (ConvF2D src));
11601   format %{ "FST_S  $dst,$src\t# D-round" %}
11602   ins_encode( Pop_Reg_Reg_D(dst, src));
11603   ins_pipe( fpu_reg_reg );
11604 %}
11605 
11606 instruct convF2D_reg(stackSlotD dst, regF src) %{
11607   predicate(UseSSE==1);
11608   match(Set dst (ConvF2D src));
11609   format %{ "FST_D  $dst,$src\t# D-round" %}
11610   expand %{
11611     roundDouble_mem_reg(dst,src);
11612   %}
11613 %}
11614 
11615 instruct convX2D_reg(regD dst, regX src, eFlagsReg cr) %{
11616   predicate(UseSSE==1);
11617   match(Set dst (ConvF2D src));
11618   effect( KILL cr );
11619   format %{ "SUB    ESP,4\n\t"
11620             "MOVSS  [ESP] $src\n\t"
11621             "FLD_S  [ESP]\n\t"
11622             "ADD    ESP,4\n\t"
11623             "FSTP   $dst\t# D-round" %}
11624   ins_encode( X2D_encoding(dst, src), Pop_Reg_D(dst));
11625   ins_pipe( pipe_slow );
11626 %}
11627 
11628 instruct convX2XD_reg(regXD dst, regX src) %{
11629   predicate(UseSSE>=2);
11630   match(Set dst (ConvF2D src));
11631   format %{ "CVTSS2SD $dst,$src\t# D-round" %}
11632   opcode(0xF3, 0x0F, 0x5A);
11633   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11634   ins_pipe( pipe_slow );
11635 %}
11636 
11637 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
11638 instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{
11639   predicate(UseSSE<=1);
11640   match(Set dst (ConvD2I src));
11641   effect( KILL tmp, KILL cr );
11642   format %{ "FLD    $src\t# Convert double to int \n\t"
11643             "FLDCW  trunc mode\n\t"
11644             "SUB    ESP,4\n\t"
11645             "FISTp  [ESP + #0]\n\t"
11646             "FLDCW  std/24-bit mode\n\t"
11647             "POP    EAX\n\t"
11648             "CMP    EAX,0x80000000\n\t"
11649             "JNE,s  fast\n\t"
11650             "FLD_D  $src\n\t"
11651             "CALL   d2i_wrapper\n"
11652       "fast:" %}
11653   ins_encode( Push_Reg_D(src), D2I_encoding(src) );
11654   ins_pipe( pipe_slow );
11655 %}
11656 
11657 // Convert a double to an int.  If the double is a NAN, stuff a zero in instead.
11658 instruct convXD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regXD src, eFlagsReg cr ) %{
11659   predicate(UseSSE>=2);
11660   match(Set dst (ConvD2I src));
11661   effect( KILL tmp, KILL cr );
11662   format %{ "CVTTSD2SI $dst, $src\n\t"
11663             "CMP    $dst,0x80000000\n\t"
11664             "JNE,s  fast\n\t"
11665             "SUB    ESP, 8\n\t"
11666             "MOVSD  [ESP], $src\n\t"
11667             "FLD_D  [ESP]\n\t"
11668             "ADD    ESP, 8\n\t"
11669             "CALL   d2i_wrapper\n"
11670       "fast:" %}
11671   opcode(0x1); // double-precision conversion
11672   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
11673   ins_pipe( pipe_slow );
11674 %}
11675 
11676 instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{
11677   predicate(UseSSE<=1);
11678   match(Set dst (ConvD2L src));
11679   effect( KILL cr );
11680   format %{ "FLD    $src\t# Convert double to long\n\t"
11681             "FLDCW  trunc mode\n\t"
11682             "SUB    ESP,8\n\t"
11683             "FISTp  [ESP + #0]\n\t"
11684             "FLDCW  std/24-bit mode\n\t"
11685             "POP    EAX\n\t"
11686             "POP    EDX\n\t"
11687             "CMP    EDX,0x80000000\n\t"
11688             "JNE,s  fast\n\t"
11689             "TEST   EAX,EAX\n\t"
11690             "JNE,s  fast\n\t"
11691             "FLD    $src\n\t"
11692             "CALL   d2l_wrapper\n"
11693       "fast:" %}
11694   ins_encode( Push_Reg_D(src),  D2L_encoding(src) );
11695   ins_pipe( pipe_slow );
11696 %}
11697 
11698 // XMM lacks a float/double->long conversion, so use the old FPU stack.
11699 instruct convXD2L_reg_reg( eADXRegL dst, regXD src, eFlagsReg cr ) %{
11700   predicate (UseSSE>=2);
11701   match(Set dst (ConvD2L src));
11702   effect( KILL cr );
11703   format %{ "SUB    ESP,8\t# Convert double to long\n\t"
11704             "MOVSD  [ESP],$src\n\t"
11705             "FLD_D  [ESP]\n\t"
11706             "FLDCW  trunc mode\n\t"
11707             "FISTp  [ESP + #0]\n\t"
11708             "FLDCW  std/24-bit mode\n\t"
11709             "POP    EAX\n\t"
11710             "POP    EDX\n\t"
11711             "CMP    EDX,0x80000000\n\t"
11712             "JNE,s  fast\n\t"
11713             "TEST   EAX,EAX\n\t"
11714             "JNE,s  fast\n\t"
11715             "SUB    ESP,8\n\t"
11716             "MOVSD  [ESP],$src\n\t"
11717             "FLD_D  [ESP]\n\t"
11718             "CALL   d2l_wrapper\n"
11719       "fast:" %}
11720   ins_encode( XD2L_encoding(src) );
11721   ins_pipe( pipe_slow );
11722 %}
11723 
11724 // Convert a double to an int.  Java semantics require we do complex
11725 // manglations in the corner cases.  So we set the rounding mode to
11726 // 'zero', store the darned double down as an int, and reset the
11727 // rounding mode to 'nearest'.  The hardware stores a flag value down
11728 // if we would overflow or converted a NAN; we check for this and
11729 // and go the slow path if needed.
11730 instruct convF2I_reg_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{
11731   predicate(UseSSE==0);
11732   match(Set dst (ConvF2I src));
11733   effect( KILL tmp, KILL cr );
11734   format %{ "FLD    $src\t# Convert float to int \n\t"
11735             "FLDCW  trunc mode\n\t"
11736             "SUB    ESP,4\n\t"
11737             "FISTp  [ESP + #0]\n\t"
11738             "FLDCW  std/24-bit mode\n\t"
11739             "POP    EAX\n\t"
11740             "CMP    EAX,0x80000000\n\t"
11741             "JNE,s  fast\n\t"
11742             "FLD    $src\n\t"
11743             "CALL   d2i_wrapper\n"
11744       "fast:" %}
11745   // D2I_encoding works for F2I
11746   ins_encode( Push_Reg_F(src), D2I_encoding(src) );
11747   ins_pipe( pipe_slow );
11748 %}
11749 
11750 // Convert a float in xmm to an int reg.
11751 instruct convX2I_reg(eAXRegI dst, eDXRegI tmp, regX src, eFlagsReg cr ) %{
11752   predicate(UseSSE>=1);
11753   match(Set dst (ConvF2I src));
11754   effect( KILL tmp, KILL cr );
11755   format %{ "CVTTSS2SI $dst, $src\n\t"
11756             "CMP    $dst,0x80000000\n\t"
11757             "JNE,s  fast\n\t"
11758             "SUB    ESP, 4\n\t"
11759             "MOVSS  [ESP], $src\n\t"
11760             "FLD    [ESP]\n\t"
11761             "ADD    ESP, 4\n\t"
11762             "CALL   d2i_wrapper\n"
11763       "fast:" %}
11764   opcode(0x0); // single-precision conversion
11765   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x2C), FX2I_encoding(src,dst));
11766   ins_pipe( pipe_slow );
11767 %}
11768 
11769 instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{
11770   predicate(UseSSE==0);
11771   match(Set dst (ConvF2L src));
11772   effect( KILL cr );
11773   format %{ "FLD    $src\t# Convert float to long\n\t"
11774             "FLDCW  trunc mode\n\t"
11775             "SUB    ESP,8\n\t"
11776             "FISTp  [ESP + #0]\n\t"
11777             "FLDCW  std/24-bit mode\n\t"
11778             "POP    EAX\n\t"
11779             "POP    EDX\n\t"
11780             "CMP    EDX,0x80000000\n\t"
11781             "JNE,s  fast\n\t"
11782             "TEST   EAX,EAX\n\t"
11783             "JNE,s  fast\n\t"
11784             "FLD    $src\n\t"
11785             "CALL   d2l_wrapper\n"
11786       "fast:" %}
11787   // D2L_encoding works for F2L
11788   ins_encode( Push_Reg_F(src), D2L_encoding(src) );
11789   ins_pipe( pipe_slow );
11790 %}
11791 
11792 // XMM lacks a float/double->long conversion, so use the old FPU stack.
11793 instruct convX2L_reg_reg( eADXRegL dst, regX src, eFlagsReg cr ) %{
11794   predicate (UseSSE>=1);
11795   match(Set dst (ConvF2L src));
11796   effect( KILL cr );
11797   format %{ "SUB    ESP,8\t# Convert float to long\n\t"
11798             "MOVSS  [ESP],$src\n\t"
11799             "FLD_S  [ESP]\n\t"
11800             "FLDCW  trunc mode\n\t"
11801             "FISTp  [ESP + #0]\n\t"
11802             "FLDCW  std/24-bit mode\n\t"
11803             "POP    EAX\n\t"
11804             "POP    EDX\n\t"
11805             "CMP    EDX,0x80000000\n\t"
11806             "JNE,s  fast\n\t"
11807             "TEST   EAX,EAX\n\t"
11808             "JNE,s  fast\n\t"
11809             "SUB    ESP,4\t# Convert float to long\n\t"
11810             "MOVSS  [ESP],$src\n\t"
11811             "FLD_S  [ESP]\n\t"
11812             "ADD    ESP,4\n\t"
11813             "CALL   d2l_wrapper\n"
11814       "fast:" %}
11815   ins_encode( X2L_encoding(src) );
11816   ins_pipe( pipe_slow );
11817 %}
11818 
11819 instruct convI2D_reg(regD dst, stackSlotI src) %{
11820   predicate( UseSSE<=1 );
11821   match(Set dst (ConvI2D src));
11822   format %{ "FILD   $src\n\t"
11823             "FSTP   $dst" %}
11824   opcode(0xDB, 0x0);  /* DB /0 */
11825   ins_encode(Push_Mem_I(src), Pop_Reg_D(dst));
11826   ins_pipe( fpu_reg_mem );
11827 %}
11828 
11829 instruct convI2XD_reg(regXD dst, eRegI src) %{
11830   predicate( UseSSE>=2 && !UseXmmI2D );
11831   match(Set dst (ConvI2D src));
11832   format %{ "CVTSI2SD $dst,$src" %}
11833   opcode(0xF2, 0x0F, 0x2A);
11834   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11835   ins_pipe( pipe_slow );
11836 %}
11837 
11838 instruct convI2XD_mem(regXD dst, memory mem) %{
11839   predicate( UseSSE>=2 );
11840   match(Set dst (ConvI2D (LoadI mem)));
11841   format %{ "CVTSI2SD $dst,$mem" %}
11842   opcode(0xF2, 0x0F, 0x2A);
11843   ins_encode( OpcP, OpcS, Opcode(tertiary), RegMem(dst, mem));
11844   ins_pipe( pipe_slow );
11845 %}
11846 
11847 instruct convXI2XD_reg(regXD dst, eRegI src)
11848 %{
11849   predicate( UseSSE>=2 && UseXmmI2D );
11850   match(Set dst (ConvI2D src));
11851 
11852   format %{ "MOVD  $dst,$src\n\t"
11853             "CVTDQ2PD $dst,$dst\t# i2d" %}
11854   ins_encode %{
11855     __ movdl($dst$$XMMRegister, $src$$Register);
11856     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11857   %}
11858   ins_pipe(pipe_slow); // XXX
11859 %}
11860 
11861 instruct convI2D_mem(regD dst, memory mem) %{
11862   predicate( UseSSE<=1 && !Compile::current()->select_24_bit_instr());
11863   match(Set dst (ConvI2D (LoadI mem)));
11864   format %{ "FILD   $mem\n\t"
11865             "FSTP   $dst" %}
11866   opcode(0xDB);      /* DB /0 */
11867   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11868               Pop_Reg_D(dst));
11869   ins_pipe( fpu_reg_mem );
11870 %}
11871 
11872 // Convert a byte to a float; no rounding step needed.
11873 instruct conv24I2F_reg(regF dst, stackSlotI src) %{
11874   predicate( UseSSE==0 && n->in(1)->Opcode() == Op_AndI && n->in(1)->in(2)->is_Con() && n->in(1)->in(2)->get_int() == 255 );
11875   match(Set dst (ConvI2F src));
11876   format %{ "FILD   $src\n\t"
11877             "FSTP   $dst" %}
11878 
11879   opcode(0xDB, 0x0);  /* DB /0 */
11880   ins_encode(Push_Mem_I(src), Pop_Reg_F(dst));
11881   ins_pipe( fpu_reg_mem );
11882 %}
11883 
11884 // In 24-bit mode, force exponent rounding by storing back out
11885 instruct convI2F_SSF(stackSlotF dst, stackSlotI src) %{
11886   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11887   match(Set dst (ConvI2F src));
11888   ins_cost(200);
11889   format %{ "FILD   $src\n\t"
11890             "FSTP_S $dst" %}
11891   opcode(0xDB, 0x0);  /* DB /0 */
11892   ins_encode( Push_Mem_I(src),
11893               Pop_Mem_F(dst));
11894   ins_pipe( fpu_mem_mem );
11895 %}
11896 
11897 // In 24-bit mode, force exponent rounding by storing back out
11898 instruct convI2F_SSF_mem(stackSlotF dst, memory mem) %{
11899   predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
11900   match(Set dst (ConvI2F (LoadI mem)));
11901   ins_cost(200);
11902   format %{ "FILD   $mem\n\t"
11903             "FSTP_S $dst" %}
11904   opcode(0xDB);  /* DB /0 */
11905   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11906               Pop_Mem_F(dst));
11907   ins_pipe( fpu_mem_mem );
11908 %}
11909 
11910 // This instruction does not round to 24-bits
11911 instruct convI2F_reg(regF dst, stackSlotI src) %{
11912   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11913   match(Set dst (ConvI2F src));
11914   format %{ "FILD   $src\n\t"
11915             "FSTP   $dst" %}
11916   opcode(0xDB, 0x0);  /* DB /0 */
11917   ins_encode( Push_Mem_I(src),
11918               Pop_Reg_F(dst));
11919   ins_pipe( fpu_reg_mem );
11920 %}
11921 
11922 // This instruction does not round to 24-bits
11923 instruct convI2F_mem(regF dst, memory mem) %{
11924   predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
11925   match(Set dst (ConvI2F (LoadI mem)));
11926   format %{ "FILD   $mem\n\t"
11927             "FSTP   $dst" %}
11928   opcode(0xDB);      /* DB /0 */
11929   ins_encode( OpcP, RMopc_Mem(0x00,mem),
11930               Pop_Reg_F(dst));
11931   ins_pipe( fpu_reg_mem );
11932 %}
11933 
11934 // Convert an int to a float in xmm; no rounding step needed.
11935 instruct convI2X_reg(regX dst, eRegI src) %{
11936   predicate( UseSSE==1 || UseSSE>=2 && !UseXmmI2F );
11937   match(Set dst (ConvI2F src));
11938   format %{ "CVTSI2SS $dst, $src" %}
11939 
11940   opcode(0xF3, 0x0F, 0x2A);  /* F3 0F 2A /r */
11941   ins_encode( OpcP, OpcS, Opcode(tertiary), RegReg(dst, src));
11942   ins_pipe( pipe_slow );
11943 %}
11944 
11945  instruct convXI2X_reg(regX dst, eRegI src)
11946 %{
11947   predicate( UseSSE>=2 && UseXmmI2F );
11948   match(Set dst (ConvI2F src));
11949 
11950   format %{ "MOVD  $dst,$src\n\t"
11951             "CVTDQ2PS $dst,$dst\t# i2f" %}
11952   ins_encode %{
11953     __ movdl($dst$$XMMRegister, $src$$Register);
11954     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11955   %}
11956   ins_pipe(pipe_slow); // XXX
11957 %}
11958 
11959 instruct convI2L_reg( eRegL dst, eRegI src, eFlagsReg cr) %{
11960   match(Set dst (ConvI2L src));
11961   effect(KILL cr);
11962   ins_cost(375);
11963   format %{ "MOV    $dst.lo,$src\n\t"
11964             "MOV    $dst.hi,$src\n\t"
11965             "SAR    $dst.hi,31" %}
11966   ins_encode(convert_int_long(dst,src));
11967   ins_pipe( ialu_reg_reg_long );
11968 %}
11969 
11970 // Zero-extend convert int to long
11971 instruct convI2L_reg_zex(eRegL dst, eRegI src, immL_32bits mask, eFlagsReg flags ) %{
11972   match(Set dst (AndL (ConvI2L src) mask) );
11973   effect( KILL flags );
11974   ins_cost(250);
11975   format %{ "MOV    $dst.lo,$src\n\t"
11976             "XOR    $dst.hi,$dst.hi" %}
11977   opcode(0x33); // XOR
11978   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
11979   ins_pipe( ialu_reg_reg_long );
11980 %}
11981 
11982 // Zero-extend long
11983 instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
11984   match(Set dst (AndL src mask) );
11985   effect( KILL flags );
11986   ins_cost(250);
11987   format %{ "MOV    $dst.lo,$src.lo\n\t"
11988             "XOR    $dst.hi,$dst.hi\n\t" %}
11989   opcode(0x33); // XOR
11990   ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
11991   ins_pipe( ialu_reg_reg_long );
11992 %}
11993 
11994 instruct convL2D_reg( stackSlotD dst, eRegL src, eFlagsReg cr) %{
11995   predicate (UseSSE<=1);
11996   match(Set dst (ConvL2D src));
11997   effect( KILL cr );
11998   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
11999             "PUSH   $src.lo\n\t"
12000             "FILD   ST,[ESP + #0]\n\t"
12001             "ADD    ESP,8\n\t"
12002             "FSTP_D $dst\t# D-round" %}
12003   opcode(0xDF, 0x5);  /* DF /5 */
12004   ins_encode(convert_long_double(src), Pop_Mem_D(dst));
12005   ins_pipe( pipe_slow );
12006 %}
12007 
12008 instruct convL2XD_reg( regXD dst, eRegL src, eFlagsReg cr) %{
12009   predicate (UseSSE>=2);
12010   match(Set dst (ConvL2D src));
12011   effect( KILL cr );
12012   format %{ "PUSH   $src.hi\t# Convert long to double\n\t"
12013             "PUSH   $src.lo\n\t"
12014             "FILD_D [ESP]\n\t"
12015             "FSTP_D [ESP]\n\t"
12016             "MOVSD  $dst,[ESP]\n\t"
12017             "ADD    ESP,8" %}
12018   opcode(0xDF, 0x5);  /* DF /5 */
12019   ins_encode(convert_long_double2(src), Push_ResultXD(dst));
12020   ins_pipe( pipe_slow );
12021 %}
12022 
12023 instruct convL2X_reg( regX dst, eRegL src, eFlagsReg cr) %{
12024   predicate (UseSSE>=1);
12025   match(Set dst (ConvL2F src));
12026   effect( KILL cr );
12027   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
12028             "PUSH   $src.lo\n\t"
12029             "FILD_D [ESP]\n\t"
12030             "FSTP_S [ESP]\n\t"
12031             "MOVSS  $dst,[ESP]\n\t"
12032             "ADD    ESP,8" %}
12033   opcode(0xDF, 0x5);  /* DF /5 */
12034   ins_encode(convert_long_double2(src), Push_ResultX(dst,0x8));
12035   ins_pipe( pipe_slow );
12036 %}
12037 
12038 instruct convL2F_reg( stackSlotF dst, eRegL src, eFlagsReg cr) %{
12039   match(Set dst (ConvL2F src));
12040   effect( KILL cr );
12041   format %{ "PUSH   $src.hi\t# Convert long to single float\n\t"
12042             "PUSH   $src.lo\n\t"
12043             "FILD   ST,[ESP + #0]\n\t"
12044             "ADD    ESP,8\n\t"
12045             "FSTP_S $dst\t# F-round" %}
12046   opcode(0xDF, 0x5);  /* DF /5 */
12047   ins_encode(convert_long_double(src), Pop_Mem_F(dst));
12048   ins_pipe( pipe_slow );
12049 %}
12050 
12051 instruct convL2I_reg( eRegI dst, eRegL src ) %{
12052   match(Set dst (ConvL2I src));
12053   effect( DEF dst, USE src );
12054   format %{ "MOV    $dst,$src.lo" %}
12055   ins_encode(enc_CopyL_Lo(dst,src));
12056   ins_pipe( ialu_reg_reg );
12057 %}
12058 
12059 
12060 instruct MoveF2I_stack_reg(eRegI dst, stackSlotF src) %{
12061   match(Set dst (MoveF2I src));
12062   effect( DEF dst, USE src );
12063   ins_cost(100);
12064   format %{ "MOV    $dst,$src\t# MoveF2I_stack_reg" %}
12065   opcode(0x8B);
12066   ins_encode( OpcP, RegMem(dst,src));
12067   ins_pipe( ialu_reg_mem );
12068 %}
12069 
12070 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
12071   predicate(UseSSE==0);
12072   match(Set dst (MoveF2I src));
12073   effect( DEF dst, USE src );
12074 
12075   ins_cost(125);
12076   format %{ "FST_S  $dst,$src\t# MoveF2I_reg_stack" %}
12077   ins_encode( Pop_Mem_Reg_F(dst, src) );
12078   ins_pipe( fpu_mem_reg );
12079 %}
12080 
12081 instruct MoveF2I_reg_stack_sse(stackSlotI dst, regX src) %{
12082   predicate(UseSSE>=1);
12083   match(Set dst (MoveF2I src));
12084   effect( DEF dst, USE src );
12085 
12086   ins_cost(95);
12087   format %{ "MOVSS  $dst,$src\t# MoveF2I_reg_stack_sse" %}
12088   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x11), RegMem(src, dst));
12089   ins_pipe( pipe_slow );
12090 %}
12091 
12092 instruct MoveF2I_reg_reg_sse(eRegI dst, regX src) %{
12093   predicate(UseSSE>=2);
12094   match(Set dst (MoveF2I src));
12095   effect( DEF dst, USE src );
12096   ins_cost(85);
12097   format %{ "MOVD   $dst,$src\t# MoveF2I_reg_reg_sse" %}
12098   ins_encode( MovX2I_reg(dst, src));
12099   ins_pipe( pipe_slow );
12100 %}
12101 
12102 instruct MoveI2F_reg_stack(stackSlotF dst, eRegI src) %{
12103   match(Set dst (MoveI2F src));
12104   effect( DEF dst, USE src );
12105 
12106   ins_cost(100);
12107   format %{ "MOV    $dst,$src\t# MoveI2F_reg_stack" %}
12108   opcode(0x89);
12109   ins_encode( OpcPRegSS( dst, src ) );
12110   ins_pipe( ialu_mem_reg );
12111 %}
12112 
12113 
12114 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
12115   predicate(UseSSE==0);
12116   match(Set dst (MoveI2F src));
12117   effect(DEF dst, USE src);
12118 
12119   ins_cost(125);
12120   format %{ "FLD_S  $src\n\t"
12121             "FSTP   $dst\t# MoveI2F_stack_reg" %}
12122   opcode(0xD9);               /* D9 /0, FLD m32real */
12123   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
12124               Pop_Reg_F(dst) );
12125   ins_pipe( fpu_reg_mem );
12126 %}
12127 
12128 instruct MoveI2F_stack_reg_sse(regX dst, stackSlotI src) %{
12129   predicate(UseSSE>=1);
12130   match(Set dst (MoveI2F src));
12131   effect( DEF dst, USE src );
12132 
12133   ins_cost(95);
12134   format %{ "MOVSS  $dst,$src\t# MoveI2F_stack_reg_sse" %}
12135   ins_encode( Opcode(0xF3), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
12136   ins_pipe( pipe_slow );
12137 %}
12138 
12139 instruct MoveI2F_reg_reg_sse(regX dst, eRegI src) %{
12140   predicate(UseSSE>=2);
12141   match(Set dst (MoveI2F src));
12142   effect( DEF dst, USE src );
12143 
12144   ins_cost(85);
12145   format %{ "MOVD   $dst,$src\t# MoveI2F_reg_reg_sse" %}
12146   ins_encode( MovI2X_reg(dst, src) );
12147   ins_pipe( pipe_slow );
12148 %}
12149 
12150 instruct MoveD2L_stack_reg(eRegL dst, stackSlotD src) %{
12151   match(Set dst (MoveD2L src));
12152   effect(DEF dst, USE src);
12153 
12154   ins_cost(250);
12155   format %{ "MOV    $dst.lo,$src\n\t"
12156             "MOV    $dst.hi,$src+4\t# MoveD2L_stack_reg" %}
12157   opcode(0x8B, 0x8B);
12158   ins_encode( OpcP, RegMem(dst,src), OpcS, RegMem_Hi(dst,src));
12159   ins_pipe( ialu_mem_long_reg );
12160 %}
12161 
12162 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
12163   predicate(UseSSE<=1);
12164   match(Set dst (MoveD2L src));
12165   effect(DEF dst, USE src);
12166 
12167   ins_cost(125);
12168   format %{ "FST_D  $dst,$src\t# MoveD2L_reg_stack" %}
12169   ins_encode( Pop_Mem_Reg_D(dst, src) );
12170   ins_pipe( fpu_mem_reg );
12171 %}
12172 
12173 instruct MoveD2L_reg_stack_sse(stackSlotL dst, regXD src) %{
12174   predicate(UseSSE>=2);
12175   match(Set dst (MoveD2L src));
12176   effect(DEF dst, USE src);
12177   ins_cost(95);
12178 
12179   format %{ "MOVSD  $dst,$src\t# MoveD2L_reg_stack_sse" %}
12180   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x11), RegMem(src,dst));
12181   ins_pipe( pipe_slow );
12182 %}
12183 
12184 instruct MoveD2L_reg_reg_sse(eRegL dst, regXD src, regXD tmp) %{
12185   predicate(UseSSE>=2);
12186   match(Set dst (MoveD2L src));
12187   effect(DEF dst, USE src, TEMP tmp);
12188   ins_cost(85);
12189   format %{ "MOVD   $dst.lo,$src\n\t"
12190             "PSHUFLW $tmp,$src,0x4E\n\t"
12191             "MOVD   $dst.hi,$tmp\t# MoveD2L_reg_reg_sse" %}
12192   ins_encode( MovXD2L_reg(dst, src, tmp) );
12193   ins_pipe( pipe_slow );
12194 %}
12195 
12196 instruct MoveL2D_reg_stack(stackSlotD dst, eRegL src) %{
12197   match(Set dst (MoveL2D src));
12198   effect(DEF dst, USE src);
12199 
12200   ins_cost(200);
12201   format %{ "MOV    $dst,$src.lo\n\t"
12202             "MOV    $dst+4,$src.hi\t# MoveL2D_reg_stack" %}
12203   opcode(0x89, 0x89);
12204   ins_encode( OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ) );
12205   ins_pipe( ialu_mem_long_reg );
12206 %}
12207 
12208 
12209 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
12210   predicate(UseSSE<=1);
12211   match(Set dst (MoveL2D src));
12212   effect(DEF dst, USE src);
12213   ins_cost(125);
12214 
12215   format %{ "FLD_D  $src\n\t"
12216             "FSTP   $dst\t# MoveL2D_stack_reg" %}
12217   opcode(0xDD);               /* DD /0, FLD m64real */
12218   ins_encode( OpcP, RMopc_Mem_no_oop(0x00,src),
12219               Pop_Reg_D(dst) );
12220   ins_pipe( fpu_reg_mem );
12221 %}
12222 
12223 
12224 instruct MoveL2D_stack_reg_sse(regXD dst, stackSlotL src) %{
12225   predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
12226   match(Set dst (MoveL2D src));
12227   effect(DEF dst, USE src);
12228 
12229   ins_cost(95);
12230   format %{ "MOVSD  $dst,$src\t# MoveL2D_stack_reg_sse" %}
12231   ins_encode( Opcode(0xF2), Opcode(0x0F), Opcode(0x10), RegMem(dst,src));
12232   ins_pipe( pipe_slow );
12233 %}
12234 
12235 instruct MoveL2D_stack_reg_sse_partial(regXD dst, stackSlotL src) %{
12236   predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
12237   match(Set dst (MoveL2D src));
12238   effect(DEF dst, USE src);
12239 
12240   ins_cost(95);
12241   format %{ "MOVLPD $dst,$src\t# MoveL2D_stack_reg_sse" %}
12242   ins_encode( Opcode(0x66), Opcode(0x0F), Opcode(0x12), RegMem(dst,src));
12243   ins_pipe( pipe_slow );
12244 %}
12245 
12246 instruct MoveL2D_reg_reg_sse(regXD dst, eRegL src, regXD tmp) %{
12247   predicate(UseSSE>=2);
12248   match(Set dst (MoveL2D src));
12249   effect(TEMP dst, USE src, TEMP tmp);
12250   ins_cost(85);
12251   format %{ "MOVD   $dst,$src.lo\n\t"
12252             "MOVD   $tmp,$src.hi\n\t"
12253             "PUNPCKLDQ $dst,$tmp\t# MoveL2D_reg_reg_sse" %}
12254   ins_encode( MovL2XD_reg(dst, src, tmp) );
12255   ins_pipe( pipe_slow );
12256 %}
12257 
12258 // Replicate scalar to packed byte (1 byte) values in xmm
12259 instruct Repl8B_reg(regXD dst, regXD src) %{
12260   predicate(UseSSE>=2);
12261   match(Set dst (Replicate8B src));
12262   format %{ "MOVDQA  $dst,$src\n\t"
12263             "PUNPCKLBW $dst,$dst\n\t"
12264             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
12265   ins_encode( pshufd_8x8(dst, src));
12266   ins_pipe( pipe_slow );
12267 %}
12268 
12269 // Replicate scalar to packed byte (1 byte) values in xmm
12270 instruct Repl8B_eRegI(regXD dst, eRegI src) %{
12271   predicate(UseSSE>=2);
12272   match(Set dst (Replicate8B src));
12273   format %{ "MOVD    $dst,$src\n\t"
12274             "PUNPCKLBW $dst,$dst\n\t"
12275             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
12276   ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
12277   ins_pipe( pipe_slow );
12278 %}
12279 
12280 // Replicate scalar zero to packed byte (1 byte) values in xmm
12281 instruct Repl8B_immI0(regXD dst, immI0 zero) %{
12282   predicate(UseSSE>=2);
12283   match(Set dst (Replicate8B zero));
12284   format %{ "PXOR  $dst,$dst\t! replicate8B" %}
12285   ins_encode( pxor(dst, dst));
12286   ins_pipe( fpu_reg_reg );
12287 %}
12288 
12289 // Replicate scalar to packed shore (2 byte) values in xmm
12290 instruct Repl4S_reg(regXD dst, regXD src) %{
12291   predicate(UseSSE>=2);
12292   match(Set dst (Replicate4S src));
12293   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
12294   ins_encode( pshufd_4x16(dst, src));
12295   ins_pipe( fpu_reg_reg );
12296 %}
12297 
12298 // Replicate scalar to packed shore (2 byte) values in xmm
12299 instruct Repl4S_eRegI(regXD dst, eRegI src) %{
12300   predicate(UseSSE>=2);
12301   match(Set dst (Replicate4S src));
12302   format %{ "MOVD    $dst,$src\n\t"
12303             "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
12304   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
12305   ins_pipe( fpu_reg_reg );
12306 %}
12307 
12308 // Replicate scalar zero to packed short (2 byte) values in xmm
12309 instruct Repl4S_immI0(regXD dst, immI0 zero) %{
12310   predicate(UseSSE>=2);
12311   match(Set dst (Replicate4S zero));
12312   format %{ "PXOR  $dst,$dst\t! replicate4S" %}
12313   ins_encode( pxor(dst, dst));
12314   ins_pipe( fpu_reg_reg );
12315 %}
12316 
12317 // Replicate scalar to packed char (2 byte) values in xmm
12318 instruct Repl4C_reg(regXD dst, regXD src) %{
12319   predicate(UseSSE>=2);
12320   match(Set dst (Replicate4C src));
12321   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
12322   ins_encode( pshufd_4x16(dst, src));
12323   ins_pipe( fpu_reg_reg );
12324 %}
12325 
12326 // Replicate scalar to packed char (2 byte) values in xmm
12327 instruct Repl4C_eRegI(regXD dst, eRegI src) %{
12328   predicate(UseSSE>=2);
12329   match(Set dst (Replicate4C src));
12330   format %{ "MOVD    $dst,$src\n\t"
12331             "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
12332   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
12333   ins_pipe( fpu_reg_reg );
12334 %}
12335 
12336 // Replicate scalar zero to packed char (2 byte) values in xmm
12337 instruct Repl4C_immI0(regXD dst, immI0 zero) %{
12338   predicate(UseSSE>=2);
12339   match(Set dst (Replicate4C zero));
12340   format %{ "PXOR  $dst,$dst\t! replicate4C" %}
12341   ins_encode( pxor(dst, dst));
12342   ins_pipe( fpu_reg_reg );
12343 %}
12344 
12345 // Replicate scalar to packed integer (4 byte) values in xmm
12346 instruct Repl2I_reg(regXD dst, regXD src) %{
12347   predicate(UseSSE>=2);
12348   match(Set dst (Replicate2I src));
12349   format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
12350   ins_encode( pshufd(dst, src, 0x00));
12351   ins_pipe( fpu_reg_reg );
12352 %}
12353 
12354 // Replicate scalar to packed integer (4 byte) values in xmm
12355 instruct Repl2I_eRegI(regXD dst, eRegI src) %{
12356   predicate(UseSSE>=2);
12357   match(Set dst (Replicate2I src));
12358   format %{ "MOVD   $dst,$src\n\t"
12359             "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
12360   ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
12361   ins_pipe( fpu_reg_reg );
12362 %}
12363 
12364 // Replicate scalar zero to packed integer (2 byte) values in xmm
12365 instruct Repl2I_immI0(regXD dst, immI0 zero) %{
12366   predicate(UseSSE>=2);
12367   match(Set dst (Replicate2I zero));
12368   format %{ "PXOR  $dst,$dst\t! replicate2I" %}
12369   ins_encode( pxor(dst, dst));
12370   ins_pipe( fpu_reg_reg );
12371 %}
12372 
12373 // Replicate scalar to packed single precision floating point values in xmm
12374 instruct Repl2F_reg(regXD dst, regXD src) %{
12375   predicate(UseSSE>=2);
12376   match(Set dst (Replicate2F src));
12377   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
12378   ins_encode( pshufd(dst, src, 0xe0));
12379   ins_pipe( fpu_reg_reg );
12380 %}
12381 
12382 // Replicate scalar to packed single precision floating point values in xmm
12383 instruct Repl2F_regX(regXD dst, regX src) %{
12384   predicate(UseSSE>=2);
12385   match(Set dst (Replicate2F src));
12386   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
12387   ins_encode( pshufd(dst, src, 0xe0));
12388   ins_pipe( fpu_reg_reg );
12389 %}
12390 
12391 // Replicate scalar to packed single precision floating point values in xmm
12392 instruct Repl2F_immXF0(regXD dst, immXF0 zero) %{
12393   predicate(UseSSE>=2);
12394   match(Set dst (Replicate2F zero));
12395   format %{ "PXOR  $dst,$dst\t! replicate2F" %}
12396   ins_encode( pxor(dst, dst));
12397   ins_pipe( fpu_reg_reg );
12398 %}
12399 
12400 // =======================================================================
12401 // fast clearing of an array
12402 instruct rep_stos(eCXRegI cnt, eDIRegP base, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
12403   match(Set dummy (ClearArray cnt base));
12404   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
12405   format %{ "SHL    ECX,1\t# Convert doublewords to words\n\t"
12406             "XOR    EAX,EAX\n\t"
12407             "REP STOS\t# store EAX into [EDI++] while ECX--" %}
12408   opcode(0,0x4);
12409   ins_encode( Opcode(0xD1), RegOpc(ECX),
12410               OpcRegReg(0x33,EAX,EAX),
12411               Opcode(0xF3), Opcode(0xAB) );
12412   ins_pipe( pipe_slow );
12413 %}
12414 
12415 instruct string_compare(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eBXRegI cnt2,
12416                         eAXRegI result, regXD tmp1, regXD tmp2, eFlagsReg cr) %{
12417   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
12418   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
12419 
12420   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1, $tmp2" %}
12421   ins_encode %{
12422     __ string_compare($str1$$Register, $str2$$Register,
12423                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12424                       $tmp1$$XMMRegister, $tmp2$$XMMRegister);
12425   %}
12426   ins_pipe( pipe_slow );
12427 %}
12428 
12429 // fast string equals
12430 instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
12431                        regXD tmp1, regXD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
12432   match(Set result (StrEquals (Binary str1 str2) cnt));
12433   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
12434 
12435   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
12436   ins_encode %{
12437     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
12438                           $cnt$$Register, $result$$Register, $tmp3$$Register,
12439                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
12440   %}
12441   ins_pipe( pipe_slow );
12442 %}
12443 
12444 instruct string_indexof(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
12445                         eBXRegI result, regXD tmp1, eCXRegI tmp2, eFlagsReg cr) %{
12446   predicate(UseSSE42Intrinsics);
12447   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
12448   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp2, KILL cr);
12449 
12450   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp2, $tmp1" %}
12451   ins_encode %{
12452     __ string_indexof($str1$$Register, $str2$$Register,
12453                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
12454                       $tmp1$$XMMRegister, $tmp2$$Register);
12455   %}
12456   ins_pipe( pipe_slow );
12457 %}
12458 
12459 // fast array equals
12460 instruct array_equals(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
12461                       regXD tmp1, regXD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
12462 %{
12463   match(Set result (AryEq ary1 ary2));
12464   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
12465   //ins_cost(300);
12466 
12467   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
12468   ins_encode %{
12469     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
12470                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
12471                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
12472   %}
12473   ins_pipe( pipe_slow );
12474 %}
12475 
12476 //----------Control Flow Instructions------------------------------------------
12477 // Signed compare Instructions
12478 instruct compI_eReg(eFlagsReg cr, eRegI op1, eRegI op2) %{
12479   match(Set cr (CmpI op1 op2));
12480   effect( DEF cr, USE op1, USE op2 );
12481   format %{ "CMP    $op1,$op2" %}
12482   opcode(0x3B);  /* Opcode 3B /r */
12483   ins_encode( OpcP, RegReg( op1, op2) );
12484   ins_pipe( ialu_cr_reg_reg );
12485 %}
12486 
12487 instruct compI_eReg_imm(eFlagsReg cr, eRegI op1, immI op2) %{
12488   match(Set cr (CmpI op1 op2));
12489   effect( DEF cr, USE op1 );
12490   format %{ "CMP    $op1,$op2" %}
12491   opcode(0x81,0x07);  /* Opcode 81 /7 */
12492   // ins_encode( RegImm( op1, op2) );  /* Was CmpImm */
12493   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12494   ins_pipe( ialu_cr_reg_imm );
12495 %}
12496 
12497 // Cisc-spilled version of cmpI_eReg
12498 instruct compI_eReg_mem(eFlagsReg cr, eRegI op1, memory op2) %{
12499   match(Set cr (CmpI op1 (LoadI op2)));
12500 
12501   format %{ "CMP    $op1,$op2" %}
12502   ins_cost(500);
12503   opcode(0x3B);  /* Opcode 3B /r */
12504   ins_encode( OpcP, RegMem( op1, op2) );
12505   ins_pipe( ialu_cr_reg_mem );
12506 %}
12507 
12508 instruct testI_reg( eFlagsReg cr, eRegI src, immI0 zero ) %{
12509   match(Set cr (CmpI src zero));
12510   effect( DEF cr, USE src );
12511 
12512   format %{ "TEST   $src,$src" %}
12513   opcode(0x85);
12514   ins_encode( OpcP, RegReg( src, src ) );
12515   ins_pipe( ialu_cr_reg_imm );
12516 %}
12517 
12518 instruct testI_reg_imm( eFlagsReg cr, eRegI src, immI con, immI0 zero ) %{
12519   match(Set cr (CmpI (AndI src con) zero));
12520 
12521   format %{ "TEST   $src,$con" %}
12522   opcode(0xF7,0x00);
12523   ins_encode( OpcP, RegOpc(src), Con32(con) );
12524   ins_pipe( ialu_cr_reg_imm );
12525 %}
12526 
12527 instruct testI_reg_mem( eFlagsReg cr, eRegI src, memory mem, immI0 zero ) %{
12528   match(Set cr (CmpI (AndI src mem) zero));
12529 
12530   format %{ "TEST   $src,$mem" %}
12531   opcode(0x85);
12532   ins_encode( OpcP, RegMem( src, mem ) );
12533   ins_pipe( ialu_cr_reg_mem );
12534 %}
12535 
12536 // Unsigned compare Instructions; really, same as signed except they
12537 // produce an eFlagsRegU instead of eFlagsReg.
12538 instruct compU_eReg(eFlagsRegU cr, eRegI op1, eRegI op2) %{
12539   match(Set cr (CmpU op1 op2));
12540 
12541   format %{ "CMPu   $op1,$op2" %}
12542   opcode(0x3B);  /* Opcode 3B /r */
12543   ins_encode( OpcP, RegReg( op1, op2) );
12544   ins_pipe( ialu_cr_reg_reg );
12545 %}
12546 
12547 instruct compU_eReg_imm(eFlagsRegU cr, eRegI op1, immI op2) %{
12548   match(Set cr (CmpU op1 op2));
12549 
12550   format %{ "CMPu   $op1,$op2" %}
12551   opcode(0x81,0x07);  /* Opcode 81 /7 */
12552   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12553   ins_pipe( ialu_cr_reg_imm );
12554 %}
12555 
12556 // // Cisc-spilled version of cmpU_eReg
12557 instruct compU_eReg_mem(eFlagsRegU cr, eRegI op1, memory op2) %{
12558   match(Set cr (CmpU op1 (LoadI op2)));
12559 
12560   format %{ "CMPu   $op1,$op2" %}
12561   ins_cost(500);
12562   opcode(0x3B);  /* Opcode 3B /r */
12563   ins_encode( OpcP, RegMem( op1, op2) );
12564   ins_pipe( ialu_cr_reg_mem );
12565 %}
12566 
12567 // // Cisc-spilled version of cmpU_eReg
12568 //instruct compU_mem_eReg(eFlagsRegU cr, memory op1, eRegI op2) %{
12569 //  match(Set cr (CmpU (LoadI op1) op2));
12570 //
12571 //  format %{ "CMPu   $op1,$op2" %}
12572 //  ins_cost(500);
12573 //  opcode(0x39);  /* Opcode 39 /r */
12574 //  ins_encode( OpcP, RegMem( op1, op2) );
12575 //%}
12576 
12577 instruct testU_reg( eFlagsRegU cr, eRegI src, immI0 zero ) %{
12578   match(Set cr (CmpU src zero));
12579 
12580   format %{ "TESTu  $src,$src" %}
12581   opcode(0x85);
12582   ins_encode( OpcP, RegReg( src, src ) );
12583   ins_pipe( ialu_cr_reg_imm );
12584 %}
12585 
12586 // Unsigned pointer compare Instructions
12587 instruct compP_eReg(eFlagsRegU cr, eRegP op1, eRegP op2) %{
12588   match(Set cr (CmpP op1 op2));
12589 
12590   format %{ "CMPu   $op1,$op2" %}
12591   opcode(0x3B);  /* Opcode 3B /r */
12592   ins_encode( OpcP, RegReg( op1, op2) );
12593   ins_pipe( ialu_cr_reg_reg );
12594 %}
12595 
12596 instruct compP_eReg_imm(eFlagsRegU cr, eRegP op1, immP op2) %{
12597   match(Set cr (CmpP op1 op2));
12598 
12599   format %{ "CMPu   $op1,$op2" %}
12600   opcode(0x81,0x07);  /* Opcode 81 /7 */
12601   ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
12602   ins_pipe( ialu_cr_reg_imm );
12603 %}
12604 
12605 // // Cisc-spilled version of cmpP_eReg
12606 instruct compP_eReg_mem(eFlagsRegU cr, eRegP op1, memory op2) %{
12607   match(Set cr (CmpP op1 (LoadP op2)));
12608 
12609   format %{ "CMPu   $op1,$op2" %}
12610   ins_cost(500);
12611   opcode(0x3B);  /* Opcode 3B /r */
12612   ins_encode( OpcP, RegMem( op1, op2) );
12613   ins_pipe( ialu_cr_reg_mem );
12614 %}
12615 
12616 // // Cisc-spilled version of cmpP_eReg
12617 //instruct compP_mem_eReg(eFlagsRegU cr, memory op1, eRegP op2) %{
12618 //  match(Set cr (CmpP (LoadP op1) op2));
12619 //
12620 //  format %{ "CMPu   $op1,$op2" %}
12621 //  ins_cost(500);
12622 //  opcode(0x39);  /* Opcode 39 /r */
12623 //  ins_encode( OpcP, RegMem( op1, op2) );
12624 //%}
12625 
12626 // Compare raw pointer (used in out-of-heap check).
12627 // Only works because non-oop pointers must be raw pointers
12628 // and raw pointers have no anti-dependencies.
12629 instruct compP_mem_eReg( eFlagsRegU cr, eRegP op1, memory op2 ) %{
12630   predicate( !n->in(2)->in(2)->bottom_type()->isa_oop_ptr() );
12631   match(Set cr (CmpP op1 (LoadP op2)));
12632 
12633   format %{ "CMPu   $op1,$op2" %}
12634   opcode(0x3B);  /* Opcode 3B /r */
12635   ins_encode( OpcP, RegMem( op1, op2) );
12636   ins_pipe( ialu_cr_reg_mem );
12637 %}
12638 
12639 //
12640 // This will generate a signed flags result. This should be ok
12641 // since any compare to a zero should be eq/neq.
12642 instruct testP_reg( eFlagsReg cr, eRegP src, immP0 zero ) %{
12643   match(Set cr (CmpP src zero));
12644 
12645   format %{ "TEST   $src,$src" %}
12646   opcode(0x85);
12647   ins_encode( OpcP, RegReg( src, src ) );
12648   ins_pipe( ialu_cr_reg_imm );
12649 %}
12650 
12651 // Cisc-spilled version of testP_reg
12652 // This will generate a signed flags result. This should be ok
12653 // since any compare to a zero should be eq/neq.
12654 instruct testP_Reg_mem( eFlagsReg cr, memory op, immI0 zero ) %{
12655   match(Set cr (CmpP (LoadP op) zero));
12656 
12657   format %{ "TEST   $op,0xFFFFFFFF" %}
12658   ins_cost(500);
12659   opcode(0xF7);               /* Opcode F7 /0 */
12660   ins_encode( OpcP, RMopc_Mem(0x00,op), Con_d32(0xFFFFFFFF) );
12661   ins_pipe( ialu_cr_reg_imm );
12662 %}
12663 
12664 // Yanked all unsigned pointer compare operations.
12665 // Pointer compares are done with CmpP which is already unsigned.
12666 
12667 //----------Max and Min--------------------------------------------------------
12668 // Min Instructions
12669 ////
12670 //   *** Min and Max using the conditional move are slower than the
12671 //   *** branch version on a Pentium III.
12672 // // Conditional move for min
12673 //instruct cmovI_reg_lt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
12674 //  effect( USE_DEF op2, USE op1, USE cr );
12675 //  format %{ "CMOVlt $op2,$op1\t! min" %}
12676 //  opcode(0x4C,0x0F);
12677 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
12678 //  ins_pipe( pipe_cmov_reg );
12679 //%}
12680 //
12681 //// Min Register with Register (P6 version)
12682 //instruct minI_eReg_p6( eRegI op1, eRegI op2 ) %{
12683 //  predicate(VM_Version::supports_cmov() );
12684 //  match(Set op2 (MinI op1 op2));
12685 //  ins_cost(200);
12686 //  expand %{
12687 //    eFlagsReg cr;
12688 //    compI_eReg(cr,op1,op2);
12689 //    cmovI_reg_lt(op2,op1,cr);
12690 //  %}
12691 //%}
12692 
12693 // Min Register with Register (generic version)
12694 instruct minI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
12695   match(Set dst (MinI dst src));
12696   effect(KILL flags);
12697   ins_cost(300);
12698 
12699   format %{ "MIN    $dst,$src" %}
12700   opcode(0xCC);
12701   ins_encode( min_enc(dst,src) );
12702   ins_pipe( pipe_slow );
12703 %}
12704 
12705 // Max Register with Register
12706 //   *** Min and Max using the conditional move are slower than the
12707 //   *** branch version on a Pentium III.
12708 // // Conditional move for max
12709 //instruct cmovI_reg_gt( eRegI op2, eRegI op1, eFlagsReg cr ) %{
12710 //  effect( USE_DEF op2, USE op1, USE cr );
12711 //  format %{ "CMOVgt $op2,$op1\t! max" %}
12712 //  opcode(0x4F,0x0F);
12713 //  ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
12714 //  ins_pipe( pipe_cmov_reg );
12715 //%}
12716 //
12717 // // Max Register with Register (P6 version)
12718 //instruct maxI_eReg_p6( eRegI op1, eRegI op2 ) %{
12719 //  predicate(VM_Version::supports_cmov() );
12720 //  match(Set op2 (MaxI op1 op2));
12721 //  ins_cost(200);
12722 //  expand %{
12723 //    eFlagsReg cr;
12724 //    compI_eReg(cr,op1,op2);
12725 //    cmovI_reg_gt(op2,op1,cr);
12726 //  %}
12727 //%}
12728 
12729 // Max Register with Register (generic version)
12730 instruct maxI_eReg(eRegI dst, eRegI src, eFlagsReg flags) %{
12731   match(Set dst (MaxI dst src));
12732   effect(KILL flags);
12733   ins_cost(300);
12734 
12735   format %{ "MAX    $dst,$src" %}
12736   opcode(0xCC);
12737   ins_encode( max_enc(dst,src) );
12738   ins_pipe( pipe_slow );
12739 %}
12740 
12741 // ============================================================================
12742 // Branch Instructions
12743 // Jump Table
12744 instruct jumpXtnd(eRegI switch_val) %{
12745   match(Jump switch_val);
12746   ins_cost(350);
12747 
12748   format %{  "JMP    [table_base](,$switch_val,1)\n\t" %}
12749 
12750   ins_encode %{
12751     address table_base  = __ address_table_constant(_index2label);
12752 
12753     // Jump to Address(table_base + switch_reg)
12754     InternalAddress table(table_base);
12755     Address index(noreg, $switch_val$$Register, Address::times_1);
12756     __ jump(ArrayAddress(table, index));
12757   %}
12758   ins_pc_relative(1);
12759   ins_pipe(pipe_jmp);
12760 %}
12761 
12762 // Jump Direct - Label defines a relative address from JMP+1
12763 instruct jmpDir(label labl) %{
12764   match(Goto);
12765   effect(USE labl);
12766 
12767   ins_cost(300);
12768   format %{ "JMP    $labl" %}
12769   size(5);
12770   opcode(0xE9);
12771   ins_encode( OpcP, Lbl( labl ) );
12772   ins_pipe( pipe_jmp );
12773   ins_pc_relative(1);
12774 %}
12775 
12776 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12777 instruct jmpCon(cmpOp cop, eFlagsReg cr, label labl) %{
12778   match(If cop cr);
12779   effect(USE labl);
12780 
12781   ins_cost(300);
12782   format %{ "J$cop    $labl" %}
12783   size(6);
12784   opcode(0x0F, 0x80);
12785   ins_encode( Jcc( cop, labl) );
12786   ins_pipe( pipe_jcc );
12787   ins_pc_relative(1);
12788 %}
12789 
12790 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12791 instruct jmpLoopEnd(cmpOp cop, eFlagsReg cr, label labl) %{
12792   match(CountedLoopEnd cop cr);
12793   effect(USE labl);
12794 
12795   ins_cost(300);
12796   format %{ "J$cop    $labl\t# Loop end" %}
12797   size(6);
12798   opcode(0x0F, 0x80);
12799   ins_encode( Jcc( cop, labl) );
12800   ins_pipe( pipe_jcc );
12801   ins_pc_relative(1);
12802 %}
12803 
12804 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12805 instruct jmpLoopEndU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12806   match(CountedLoopEnd cop cmp);
12807   effect(USE labl);
12808 
12809   ins_cost(300);
12810   format %{ "J$cop,u  $labl\t# Loop end" %}
12811   size(6);
12812   opcode(0x0F, 0x80);
12813   ins_encode( Jcc( cop, labl) );
12814   ins_pipe( pipe_jcc );
12815   ins_pc_relative(1);
12816 %}
12817 
12818 instruct jmpLoopEndUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12819   match(CountedLoopEnd cop cmp);
12820   effect(USE labl);
12821 
12822   ins_cost(200);
12823   format %{ "J$cop,u  $labl\t# Loop end" %}
12824   size(6);
12825   opcode(0x0F, 0x80);
12826   ins_encode( Jcc( cop, labl) );
12827   ins_pipe( pipe_jcc );
12828   ins_pc_relative(1);
12829 %}
12830 
12831 // Jump Direct Conditional - using unsigned comparison
12832 instruct jmpConU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
12833   match(If cop cmp);
12834   effect(USE labl);
12835 
12836   ins_cost(300);
12837   format %{ "J$cop,u  $labl" %}
12838   size(6);
12839   opcode(0x0F, 0x80);
12840   ins_encode(Jcc(cop, labl));
12841   ins_pipe(pipe_jcc);
12842   ins_pc_relative(1);
12843 %}
12844 
12845 instruct jmpConUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
12846   match(If cop cmp);
12847   effect(USE labl);
12848 
12849   ins_cost(200);
12850   format %{ "J$cop,u  $labl" %}
12851   size(6);
12852   opcode(0x0F, 0x80);
12853   ins_encode(Jcc(cop, labl));
12854   ins_pipe(pipe_jcc);
12855   ins_pc_relative(1);
12856 %}
12857 
12858 instruct jmpConUCF2(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
12859   match(If cop cmp);
12860   effect(USE labl);
12861 
12862   ins_cost(200);
12863   format %{ $$template
12864     if ($cop$$cmpcode == Assembler::notEqual) {
12865       $$emit$$"JP,u   $labl\n\t"
12866       $$emit$$"J$cop,u   $labl"
12867     } else {
12868       $$emit$$"JP,u   done\n\t"
12869       $$emit$$"J$cop,u   $labl\n\t"
12870       $$emit$$"done:"
12871     }
12872   %}
12873   size(12);
12874   opcode(0x0F, 0x80);
12875   ins_encode %{
12876     Label* l = $labl$$label;
12877     $$$emit8$primary;
12878     emit_cc(cbuf, $secondary, Assembler::parity);
12879     int parity_disp = -1;
12880     bool ok = false;
12881     if ($cop$$cmpcode == Assembler::notEqual) {
12882        // the two jumps 6 bytes apart so the jump distances are too
12883        parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
12884     } else if ($cop$$cmpcode == Assembler::equal) {
12885        parity_disp = 6;
12886        ok = true;
12887     } else {
12888        ShouldNotReachHere();
12889     }
12890     emit_d32(cbuf, parity_disp);
12891     $$$emit8$primary;
12892     emit_cc(cbuf, $secondary, $cop$$cmpcode);
12893     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 4)) : 0;
12894     emit_d32(cbuf, disp);
12895   %}
12896   ins_pipe(pipe_jcc);
12897   ins_pc_relative(1);
12898 %}
12899 
12900 // ============================================================================
12901 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass
12902 // array for an instance of the superklass.  Set a hidden internal cache on a
12903 // hit (cache is checked with exposed code in gen_subtype_check()).  Return
12904 // NZ for a miss or zero for a hit.  The encoding ALSO sets flags.
12905 instruct partialSubtypeCheck( eDIRegP result, eSIRegP sub, eAXRegP super, eCXRegI rcx, eFlagsReg cr ) %{
12906   match(Set result (PartialSubtypeCheck sub super));
12907   effect( KILL rcx, KILL cr );
12908 
12909   ins_cost(1100);  // slightly larger than the next version
12910   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
12911             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
12912             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
12913             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
12914             "JNE,s  miss\t\t# Missed: EDI not-zero\n\t"
12915             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache\n\t"
12916             "XOR    $result,$result\t\t Hit: EDI zero\n\t"
12917      "miss:\t" %}
12918 
12919   opcode(0x1); // Force a XOR of EDI
12920   ins_encode( enc_PartialSubtypeCheck() );
12921   ins_pipe( pipe_slow );
12922 %}
12923 
12924 instruct partialSubtypeCheck_vs_Zero( eFlagsReg cr, eSIRegP sub, eAXRegP super, eCXRegI rcx, eDIRegP result, immP0 zero ) %{
12925   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12926   effect( KILL rcx, KILL result );
12927 
12928   ins_cost(1000);
12929   format %{ "MOV    EDI,[$sub+Klass::secondary_supers]\n\t"
12930             "MOV    ECX,[EDI+arrayKlass::length]\t# length to scan\n\t"
12931             "ADD    EDI,arrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
12932             "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
12933             "JNE,s  miss\t\t# Missed: flags NZ\n\t"
12934             "MOV    [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache, flags Z\n\t"
12935      "miss:\t" %}
12936 
12937   opcode(0x0);  // No need to XOR EDI
12938   ins_encode( enc_PartialSubtypeCheck() );
12939   ins_pipe( pipe_slow );
12940 %}
12941 
12942 // ============================================================================
12943 // Branch Instructions -- short offset versions
12944 //
12945 // These instructions are used to replace jumps of a long offset (the default
12946 // match) with jumps of a shorter offset.  These instructions are all tagged
12947 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12948 // match rules in general matching.  Instead, the ADLC generates a conversion
12949 // method in the MachNode which can be used to do in-place replacement of the
12950 // long variant with the shorter variant.  The compiler will determine if a
12951 // branch can be taken by the is_short_branch_offset() predicate in the machine
12952 // specific code section of the file.
12953 
12954 // Jump Direct - Label defines a relative address from JMP+1
12955 instruct jmpDir_short(label labl) %{
12956   match(Goto);
12957   effect(USE labl);
12958 
12959   ins_cost(300);
12960   format %{ "JMP,s  $labl" %}
12961   size(2);
12962   opcode(0xEB);
12963   ins_encode( OpcP, LblShort( labl ) );
12964   ins_pipe( pipe_jmp );
12965   ins_pc_relative(1);
12966   ins_short_branch(1);
12967 %}
12968 
12969 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12970 instruct jmpCon_short(cmpOp cop, eFlagsReg cr, label labl) %{
12971   match(If cop cr);
12972   effect(USE labl);
12973 
12974   ins_cost(300);
12975   format %{ "J$cop,s  $labl" %}
12976   size(2);
12977   opcode(0x70);
12978   ins_encode( JccShort( cop, labl) );
12979   ins_pipe( pipe_jcc );
12980   ins_pc_relative(1);
12981   ins_short_branch(1);
12982 %}
12983 
12984 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12985 instruct jmpLoopEnd_short(cmpOp cop, eFlagsReg cr, label labl) %{
12986   match(CountedLoopEnd cop cr);
12987   effect(USE labl);
12988 
12989   ins_cost(300);
12990   format %{ "J$cop,s  $labl\t# Loop end" %}
12991   size(2);
12992   opcode(0x70);
12993   ins_encode( JccShort( cop, labl) );
12994   ins_pipe( pipe_jcc );
12995   ins_pc_relative(1);
12996   ins_short_branch(1);
12997 %}
12998 
12999 // Jump Direct Conditional - Label defines a relative address from Jcc+1
13000 instruct jmpLoopEndU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13001   match(CountedLoopEnd cop cmp);
13002   effect(USE labl);
13003 
13004   ins_cost(300);
13005   format %{ "J$cop,us $labl\t# Loop end" %}
13006   size(2);
13007   opcode(0x70);
13008   ins_encode( JccShort( cop, labl) );
13009   ins_pipe( pipe_jcc );
13010   ins_pc_relative(1);
13011   ins_short_branch(1);
13012 %}
13013 
13014 instruct jmpLoopEndUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13015   match(CountedLoopEnd cop cmp);
13016   effect(USE labl);
13017 
13018   ins_cost(300);
13019   format %{ "J$cop,us $labl\t# Loop end" %}
13020   size(2);
13021   opcode(0x70);
13022   ins_encode( JccShort( cop, labl) );
13023   ins_pipe( pipe_jcc );
13024   ins_pc_relative(1);
13025   ins_short_branch(1);
13026 %}
13027 
13028 // Jump Direct Conditional - using unsigned comparison
13029 instruct jmpConU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
13030   match(If cop cmp);
13031   effect(USE labl);
13032 
13033   ins_cost(300);
13034   format %{ "J$cop,us $labl" %}
13035   size(2);
13036   opcode(0x70);
13037   ins_encode( JccShort( cop, labl) );
13038   ins_pipe( pipe_jcc );
13039   ins_pc_relative(1);
13040   ins_short_branch(1);
13041 %}
13042 
13043 instruct jmpConUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
13044   match(If cop cmp);
13045   effect(USE labl);
13046 
13047   ins_cost(300);
13048   format %{ "J$cop,us $labl" %}
13049   size(2);
13050   opcode(0x70);
13051   ins_encode( JccShort( cop, labl) );
13052   ins_pipe( pipe_jcc );
13053   ins_pc_relative(1);
13054   ins_short_branch(1);
13055 %}
13056 
13057 instruct jmpConUCF2_short(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
13058   match(If cop cmp);
13059   effect(USE labl);
13060 
13061   ins_cost(300);
13062   format %{ $$template
13063     if ($cop$$cmpcode == Assembler::notEqual) {
13064       $$emit$$"JP,u,s   $labl\n\t"
13065       $$emit$$"J$cop,u,s   $labl"
13066     } else {
13067       $$emit$$"JP,u,s   done\n\t"
13068       $$emit$$"J$cop,u,s  $labl\n\t"
13069       $$emit$$"done:"
13070     }
13071   %}
13072   size(4);
13073   opcode(0x70);
13074   ins_encode %{
13075     Label* l = $labl$$label;
13076     emit_cc(cbuf, $primary, Assembler::parity);
13077     int parity_disp = -1;
13078     if ($cop$$cmpcode == Assembler::notEqual) {
13079       parity_disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
13080     } else if ($cop$$cmpcode == Assembler::equal) {
13081       parity_disp = 2;
13082     } else {
13083       ShouldNotReachHere();
13084     }
13085     emit_d8(cbuf, parity_disp);
13086     emit_cc(cbuf, $primary, $cop$$cmpcode);
13087     int disp = l ? (l->loc_pos() - (cbuf.code_size() + 1)) : 0;
13088     emit_d8(cbuf, disp);
13089     assert(-128 <= disp && disp <= 127, "Displacement too large for short jmp");
13090     assert(-128 <= parity_disp && parity_disp <= 127, "Displacement too large for short jmp");
13091   %}
13092   ins_pipe(pipe_jcc);
13093   ins_pc_relative(1);
13094   ins_short_branch(1);
13095 %}
13096 
13097 // ============================================================================
13098 // Long Compare
13099 //
13100 // Currently we hold longs in 2 registers.  Comparing such values efficiently
13101 // is tricky.  The flavor of compare used depends on whether we are testing
13102 // for LT, LE, or EQ.  For a simple LT test we can check just the sign bit.
13103 // The GE test is the negated LT test.  The LE test can be had by commuting
13104 // the operands (yielding a GE test) and then negating; negate again for the
13105 // GT test.  The EQ test is done by ORcc'ing the high and low halves, and the
13106 // NE test is negated from that.
13107 
13108 // Due to a shortcoming in the ADLC, it mixes up expressions like:
13109 // (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)).  Note the
13110 // difference between 'Y' and '0L'.  The tree-matches for the CmpI sections
13111 // are collapsed internally in the ADLC's dfa-gen code.  The match for
13112 // (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
13113 // foo match ends up with the wrong leaf.  One fix is to not match both
13114 // reg-reg and reg-zero forms of long-compare.  This is unfortunate because
13115 // both forms beat the trinary form of long-compare and both are very useful
13116 // on Intel which has so few registers.
13117 
13118 // Manifest a CmpL result in an integer register.  Very painful.
13119 // This is the test to avoid.
13120 instruct cmpL3_reg_reg(eSIRegI dst, eRegL src1, eRegL src2, eFlagsReg flags ) %{
13121   match(Set dst (CmpL3 src1 src2));
13122   effect( KILL flags );
13123   ins_cost(1000);
13124   format %{ "XOR    $dst,$dst\n\t"
13125             "CMP    $src1.hi,$src2.hi\n\t"
13126             "JLT,s  m_one\n\t"
13127             "JGT,s  p_one\n\t"
13128             "CMP    $src1.lo,$src2.lo\n\t"
13129             "JB,s   m_one\n\t"
13130             "JEQ,s  done\n"
13131     "p_one:\tINC    $dst\n\t"
13132             "JMP,s  done\n"
13133     "m_one:\tDEC    $dst\n"
13134      "done:" %}
13135   ins_encode %{
13136     Label p_one, m_one, done;
13137     __ xorptr($dst$$Register, $dst$$Register);
13138     __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register));
13139     __ jccb(Assembler::less,    m_one);
13140     __ jccb(Assembler::greater, p_one);
13141     __ cmpl($src1$$Register, $src2$$Register);
13142     __ jccb(Assembler::below,   m_one);
13143     __ jccb(Assembler::equal,   done);
13144     __ bind(p_one);
13145     __ incrementl($dst$$Register);
13146     __ jmpb(done);
13147     __ bind(m_one);
13148     __ decrementl($dst$$Register);
13149     __ bind(done);
13150   %}
13151   ins_pipe( pipe_slow );
13152 %}
13153 
13154 //======
13155 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
13156 // compares.  Can be used for LE or GT compares by reversing arguments.
13157 // NOT GOOD FOR EQ/NE tests.
13158 instruct cmpL_zero_flags_LTGE( flagsReg_long_LTGE flags, eRegL src, immL0 zero ) %{
13159   match( Set flags (CmpL src zero ));
13160   ins_cost(100);
13161   format %{ "TEST   $src.hi,$src.hi" %}
13162   opcode(0x85);
13163   ins_encode( OpcP, RegReg_Hi2( src, src ) );
13164   ins_pipe( ialu_cr_reg_reg );
13165 %}
13166 
13167 // Manifest a CmpL result in the normal flags.  Only good for LT or GE
13168 // compares.  Can be used for LE or GT compares by reversing arguments.
13169 // NOT GOOD FOR EQ/NE tests.
13170 instruct cmpL_reg_flags_LTGE( flagsReg_long_LTGE flags, eRegL src1, eRegL src2, eRegI tmp ) %{
13171   match( Set flags (CmpL src1 src2 ));
13172   effect( TEMP tmp );
13173   ins_cost(300);
13174   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
13175             "MOV    $tmp,$src1.hi\n\t"
13176             "SBB    $tmp,$src2.hi\t! Compute flags for long compare" %}
13177   ins_encode( long_cmp_flags2( src1, src2, tmp ) );
13178   ins_pipe( ialu_cr_reg_reg );
13179 %}
13180 
13181 // Long compares reg < zero/req OR reg >= zero/req.
13182 // Just a wrapper for a normal branch, plus the predicate test.
13183 instruct cmpL_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, label labl) %{
13184   match(If cmp flags);
13185   effect(USE labl);
13186   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13187   expand %{
13188     jmpCon(cmp,flags,labl);    // JLT or JGE...
13189   %}
13190 %}
13191 
13192 // Compare 2 longs and CMOVE longs.
13193 instruct cmovLL_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, eRegL src) %{
13194   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13195   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13196   ins_cost(400);
13197   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13198             "CMOV$cmp $dst.hi,$src.hi" %}
13199   opcode(0x0F,0x40);
13200   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13201   ins_pipe( pipe_cmov_reg_long );
13202 %}
13203 
13204 instruct cmovLL_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, load_long_memory src) %{
13205   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13206   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13207   ins_cost(500);
13208   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13209             "CMOV$cmp $dst.hi,$src.hi" %}
13210   opcode(0x0F,0x40);
13211   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13212   ins_pipe( pipe_cmov_reg_long );
13213 %}
13214 
13215 // Compare 2 longs and CMOVE ints.
13216 instruct cmovII_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, eRegI src) %{
13217   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13218   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13219   ins_cost(200);
13220   format %{ "CMOV$cmp $dst,$src" %}
13221   opcode(0x0F,0x40);
13222   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13223   ins_pipe( pipe_cmov_reg );
13224 %}
13225 
13226 instruct cmovII_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegI dst, memory src) %{
13227   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13228   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13229   ins_cost(250);
13230   format %{ "CMOV$cmp $dst,$src" %}
13231   opcode(0x0F,0x40);
13232   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13233   ins_pipe( pipe_cmov_mem );
13234 %}
13235 
13236 // Compare 2 longs and CMOVE ints.
13237 instruct cmovPP_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegP dst, eRegP src) %{
13238   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
13239   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13240   ins_cost(200);
13241   format %{ "CMOV$cmp $dst,$src" %}
13242   opcode(0x0F,0x40);
13243   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13244   ins_pipe( pipe_cmov_reg );
13245 %}
13246 
13247 // Compare 2 longs and CMOVE doubles
13248 instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
13249   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13250   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13251   ins_cost(200);
13252   expand %{
13253     fcmovD_regS(cmp,flags,dst,src);
13254   %}
13255 %}
13256 
13257 // Compare 2 longs and CMOVE doubles
13258 instruct cmovXDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regXD dst, regXD src) %{
13259   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13260   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13261   ins_cost(200);
13262   expand %{
13263     fcmovXD_regS(cmp,flags,dst,src);
13264   %}
13265 %}
13266 
13267 instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
13268   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13269   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13270   ins_cost(200);
13271   expand %{
13272     fcmovF_regS(cmp,flags,dst,src);
13273   %}
13274 %}
13275 
13276 instruct cmovXX_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regX dst, regX src) %{
13277   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
13278   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13279   ins_cost(200);
13280   expand %{
13281     fcmovX_regS(cmp,flags,dst,src);
13282   %}
13283 %}
13284 
13285 //======
13286 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
13287 instruct cmpL_zero_flags_EQNE( flagsReg_long_EQNE flags, eRegL src, immL0 zero, eRegI tmp ) %{
13288   match( Set flags (CmpL src zero ));
13289   effect(TEMP tmp);
13290   ins_cost(200);
13291   format %{ "MOV    $tmp,$src.lo\n\t"
13292             "OR     $tmp,$src.hi\t! Long is EQ/NE 0?" %}
13293   ins_encode( long_cmp_flags0( src, tmp ) );
13294   ins_pipe( ialu_reg_reg_long );
13295 %}
13296 
13297 // Manifest a CmpL result in the normal flags.  Only good for EQ/NE compares.
13298 instruct cmpL_reg_flags_EQNE( flagsReg_long_EQNE flags, eRegL src1, eRegL src2 ) %{
13299   match( Set flags (CmpL src1 src2 ));
13300   ins_cost(200+300);
13301   format %{ "CMP    $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
13302             "JNE,s  skip\n\t"
13303             "CMP    $src1.hi,$src2.hi\n\t"
13304      "skip:\t" %}
13305   ins_encode( long_cmp_flags1( src1, src2 ) );
13306   ins_pipe( ialu_cr_reg_reg );
13307 %}
13308 
13309 // Long compare reg == zero/reg OR reg != zero/reg
13310 // Just a wrapper for a normal branch, plus the predicate test.
13311 instruct cmpL_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, label labl) %{
13312   match(If cmp flags);
13313   effect(USE labl);
13314   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13315   expand %{
13316     jmpCon(cmp,flags,labl);    // JEQ or JNE...
13317   %}
13318 %}
13319 
13320 // Compare 2 longs and CMOVE longs.
13321 instruct cmovLL_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, eRegL src) %{
13322   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13323   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13324   ins_cost(400);
13325   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13326             "CMOV$cmp $dst.hi,$src.hi" %}
13327   opcode(0x0F,0x40);
13328   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13329   ins_pipe( pipe_cmov_reg_long );
13330 %}
13331 
13332 instruct cmovLL_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, load_long_memory src) %{
13333   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13334   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13335   ins_cost(500);
13336   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13337             "CMOV$cmp $dst.hi,$src.hi" %}
13338   opcode(0x0F,0x40);
13339   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13340   ins_pipe( pipe_cmov_reg_long );
13341 %}
13342 
13343 // Compare 2 longs and CMOVE ints.
13344 instruct cmovII_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, eRegI src) %{
13345   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13346   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13347   ins_cost(200);
13348   format %{ "CMOV$cmp $dst,$src" %}
13349   opcode(0x0F,0x40);
13350   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13351   ins_pipe( pipe_cmov_reg );
13352 %}
13353 
13354 instruct cmovII_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegI dst, memory src) %{
13355   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13356   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13357   ins_cost(250);
13358   format %{ "CMOV$cmp $dst,$src" %}
13359   opcode(0x0F,0x40);
13360   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13361   ins_pipe( pipe_cmov_mem );
13362 %}
13363 
13364 // Compare 2 longs and CMOVE ints.
13365 instruct cmovPP_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegP dst, eRegP src) %{
13366   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
13367   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13368   ins_cost(200);
13369   format %{ "CMOV$cmp $dst,$src" %}
13370   opcode(0x0F,0x40);
13371   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13372   ins_pipe( pipe_cmov_reg );
13373 %}
13374 
13375 // Compare 2 longs and CMOVE doubles
13376 instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
13377   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13378   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13379   ins_cost(200);
13380   expand %{
13381     fcmovD_regS(cmp,flags,dst,src);
13382   %}
13383 %}
13384 
13385 // Compare 2 longs and CMOVE doubles
13386 instruct cmovXDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regXD dst, regXD src) %{
13387   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13388   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13389   ins_cost(200);
13390   expand %{
13391     fcmovXD_regS(cmp,flags,dst,src);
13392   %}
13393 %}
13394 
13395 instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
13396   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13397   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13398   ins_cost(200);
13399   expand %{
13400     fcmovF_regS(cmp,flags,dst,src);
13401   %}
13402 %}
13403 
13404 instruct cmovXX_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regX dst, regX src) %{
13405   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
13406   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13407   ins_cost(200);
13408   expand %{
13409     fcmovX_regS(cmp,flags,dst,src);
13410   %}
13411 %}
13412 
13413 //======
13414 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
13415 // Same as cmpL_reg_flags_LEGT except must negate src
13416 instruct cmpL_zero_flags_LEGT( flagsReg_long_LEGT flags, eRegL src, immL0 zero, eRegI tmp ) %{
13417   match( Set flags (CmpL src zero ));
13418   effect( TEMP tmp );
13419   ins_cost(300);
13420   format %{ "XOR    $tmp,$tmp\t# Long compare for -$src < 0, use commuted test\n\t"
13421             "CMP    $tmp,$src.lo\n\t"
13422             "SBB    $tmp,$src.hi\n\t" %}
13423   ins_encode( long_cmp_flags3(src, tmp) );
13424   ins_pipe( ialu_reg_reg_long );
13425 %}
13426 
13427 // Manifest a CmpL result in the normal flags.  Only good for LE or GT compares.
13428 // Same as cmpL_reg_flags_LTGE except operands swapped.  Swapping operands
13429 // requires a commuted test to get the same result.
13430 instruct cmpL_reg_flags_LEGT( flagsReg_long_LEGT flags, eRegL src1, eRegL src2, eRegI tmp ) %{
13431   match( Set flags (CmpL src1 src2 ));
13432   effect( TEMP tmp );
13433   ins_cost(300);
13434   format %{ "CMP    $src2.lo,$src1.lo\t! Long compare, swapped operands, use with commuted test\n\t"
13435             "MOV    $tmp,$src2.hi\n\t"
13436             "SBB    $tmp,$src1.hi\t! Compute flags for long compare" %}
13437   ins_encode( long_cmp_flags2( src2, src1, tmp ) );
13438   ins_pipe( ialu_cr_reg_reg );
13439 %}
13440 
13441 // Long compares reg < zero/req OR reg >= zero/req.
13442 // Just a wrapper for a normal branch, plus the predicate test
13443 instruct cmpL_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, label labl) %{
13444   match(If cmp flags);
13445   effect(USE labl);
13446   predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
13447   ins_cost(300);
13448   expand %{
13449     jmpCon(cmp,flags,labl);    // JGT or JLE...
13450   %}
13451 %}
13452 
13453 // Compare 2 longs and CMOVE longs.
13454 instruct cmovLL_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, eRegL src) %{
13455   match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
13456   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13457   ins_cost(400);
13458   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13459             "CMOV$cmp $dst.hi,$src.hi" %}
13460   opcode(0x0F,0x40);
13461   ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
13462   ins_pipe( pipe_cmov_reg_long );
13463 %}
13464 
13465 instruct cmovLL_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, load_long_memory src) %{
13466   match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
13467   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13468   ins_cost(500);
13469   format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
13470             "CMOV$cmp $dst.hi,$src.hi+4" %}
13471   opcode(0x0F,0x40);
13472   ins_encode( enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src) );
13473   ins_pipe( pipe_cmov_reg_long );
13474 %}
13475 
13476 // Compare 2 longs and CMOVE ints.
13477 instruct cmovII_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, eRegI src) %{
13478   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13479   match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
13480   ins_cost(200);
13481   format %{ "CMOV$cmp $dst,$src" %}
13482   opcode(0x0F,0x40);
13483   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13484   ins_pipe( pipe_cmov_reg );
13485 %}
13486 
13487 instruct cmovII_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegI dst, memory src) %{
13488   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13489   match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
13490   ins_cost(250);
13491   format %{ "CMOV$cmp $dst,$src" %}
13492   opcode(0x0F,0x40);
13493   ins_encode( enc_cmov(cmp), RegMem( dst, src ) );
13494   ins_pipe( pipe_cmov_mem );
13495 %}
13496 
13497 // Compare 2 longs and CMOVE ptrs.
13498 instruct cmovPP_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegP dst, eRegP src) %{
13499   predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
13500   match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
13501   ins_cost(200);
13502   format %{ "CMOV$cmp $dst,$src" %}
13503   opcode(0x0F,0x40);
13504   ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
13505   ins_pipe( pipe_cmov_reg );
13506 %}
13507 
13508 // Compare 2 longs and CMOVE doubles
13509 instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
13510   predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13511   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13512   ins_cost(200);
13513   expand %{
13514     fcmovD_regS(cmp,flags,dst,src);
13515   %}
13516 %}
13517 
13518 // Compare 2 longs and CMOVE doubles
13519 instruct cmovXDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regXD dst, regXD src) %{
13520   predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13521   match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
13522   ins_cost(200);
13523   expand %{
13524     fcmovXD_regS(cmp,flags,dst,src);
13525   %}
13526 %}
13527 
13528 instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
13529   predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13530   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13531   ins_cost(200);
13532   expand %{
13533     fcmovF_regS(cmp,flags,dst,src);
13534   %}
13535 %}
13536 
13537 
13538 instruct cmovXX_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regX dst, regX src) %{
13539   predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
13540   match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
13541   ins_cost(200);
13542   expand %{
13543     fcmovX_regS(cmp,flags,dst,src);
13544   %}
13545 %}
13546 
13547 
13548 // ============================================================================
13549 // Procedure Call/Return Instructions
13550 // Call Java Static Instruction
13551 // Note: If this code changes, the corresponding ret_addr_offset() and
13552 //       compute_padding() functions will have to be adjusted.
13553 instruct CallStaticJavaDirect(method meth) %{
13554   match(CallStaticJava);
13555   predicate(! ((CallStaticJavaNode*)n)->is_method_handle_invoke());
13556   effect(USE meth);
13557 
13558   ins_cost(300);
13559   format %{ "CALL,static " %}
13560   opcode(0xE8); /* E8 cd */
13561   ins_encode( pre_call_FPU,
13562               Java_Static_Call( meth ),
13563               call_epilog,
13564               post_call_FPU );
13565   ins_pipe( pipe_slow );
13566   ins_pc_relative(1);
13567   ins_alignment(4);
13568 %}
13569 
13570 // Call Java Static Instruction (method handle version)
13571 // Note: If this code changes, the corresponding ret_addr_offset() and
13572 //       compute_padding() functions will have to be adjusted.
13573 instruct CallStaticJavaHandle(method meth, eBPRegP ebp) %{
13574   match(CallStaticJava);
13575   predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
13576   effect(USE meth);
13577   // EBP is saved by all callees (for interpreter stack correction).
13578   // We use it here for a similar purpose, in {preserve,restore}_SP.
13579 
13580   ins_cost(300);
13581   format %{ "CALL,static/MethodHandle " %}
13582   opcode(0xE8); /* E8 cd */
13583   ins_encode( pre_call_FPU,
13584               preserve_SP,
13585               Java_Static_Call( meth ),
13586               restore_SP,
13587               call_epilog,
13588               post_call_FPU );
13589   ins_pipe( pipe_slow );
13590   ins_pc_relative(1);
13591   ins_alignment(4);
13592 %}
13593 
13594 // Call Java Dynamic Instruction
13595 // Note: If this code changes, the corresponding ret_addr_offset() and
13596 //       compute_padding() functions will have to be adjusted.
13597 instruct CallDynamicJavaDirect(method meth) %{
13598   match(CallDynamicJava);
13599   effect(USE meth);
13600 
13601   ins_cost(300);
13602   format %{ "MOV    EAX,(oop)-1\n\t"
13603             "CALL,dynamic" %}
13604   opcode(0xE8); /* E8 cd */
13605   ins_encode( pre_call_FPU,
13606               Java_Dynamic_Call( meth ),
13607               call_epilog,
13608               post_call_FPU );
13609   ins_pipe( pipe_slow );
13610   ins_pc_relative(1);
13611   ins_alignment(4);
13612 %}
13613 
13614 // Call Runtime Instruction
13615 instruct CallRuntimeDirect(method meth) %{
13616   match(CallRuntime );
13617   effect(USE meth);
13618 
13619   ins_cost(300);
13620   format %{ "CALL,runtime " %}
13621   opcode(0xE8); /* E8 cd */
13622   // Use FFREEs to clear entries in float stack
13623   ins_encode( pre_call_FPU,
13624               FFree_Float_Stack_All,
13625               Java_To_Runtime( meth ),
13626               post_call_FPU );
13627   ins_pipe( pipe_slow );
13628   ins_pc_relative(1);
13629 %}
13630 
13631 // Call runtime without safepoint
13632 instruct CallLeafDirect(method meth) %{
13633   match(CallLeaf);
13634   effect(USE meth);
13635 
13636   ins_cost(300);
13637   format %{ "CALL_LEAF,runtime " %}
13638   opcode(0xE8); /* E8 cd */
13639   ins_encode( pre_call_FPU,
13640               FFree_Float_Stack_All,
13641               Java_To_Runtime( meth ),
13642               Verify_FPU_For_Leaf, post_call_FPU );
13643   ins_pipe( pipe_slow );
13644   ins_pc_relative(1);
13645 %}
13646 
13647 instruct CallLeafNoFPDirect(method meth) %{
13648   match(CallLeafNoFP);
13649   effect(USE meth);
13650 
13651   ins_cost(300);
13652   format %{ "CALL_LEAF_NOFP,runtime " %}
13653   opcode(0xE8); /* E8 cd */
13654   ins_encode(Java_To_Runtime(meth));
13655   ins_pipe( pipe_slow );
13656   ins_pc_relative(1);
13657 %}
13658 
13659 
13660 // Return Instruction
13661 // Remove the return address & jump to it.
13662 instruct Ret() %{
13663   match(Return);
13664   format %{ "RET" %}
13665   opcode(0xC3);
13666   ins_encode(OpcP);
13667   ins_pipe( pipe_jmp );
13668 %}
13669 
13670 // Tail Call; Jump from runtime stub to Java code.
13671 // Also known as an 'interprocedural jump'.
13672 // Target of jump will eventually return to caller.
13673 // TailJump below removes the return address.
13674 instruct TailCalljmpInd(eRegP_no_EBP jump_target, eBXRegP method_oop) %{
13675   match(TailCall jump_target method_oop );
13676   ins_cost(300);
13677   format %{ "JMP    $jump_target \t# EBX holds method oop" %}
13678   opcode(0xFF, 0x4);  /* Opcode FF /4 */
13679   ins_encode( OpcP, RegOpc(jump_target) );
13680   ins_pipe( pipe_jmp );
13681 %}
13682 
13683 
13684 // Tail Jump; remove the return address; jump to target.
13685 // TailCall above leaves the return address around.
13686 instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
13687   match( TailJump jump_target ex_oop );
13688   ins_cost(300);
13689   format %{ "POP    EDX\t# pop return address into dummy\n\t"
13690             "JMP    $jump_target " %}
13691   opcode(0xFF, 0x4);  /* Opcode FF /4 */
13692   ins_encode( enc_pop_rdx,
13693               OpcP, RegOpc(jump_target) );
13694   ins_pipe( pipe_jmp );
13695 %}
13696 
13697 // Create exception oop: created by stack-crawling runtime code.
13698 // Created exception is now available to this handler, and is setup
13699 // just prior to jumping to this handler.  No code emitted.
13700 instruct CreateException( eAXRegP ex_oop )
13701 %{
13702   match(Set ex_oop (CreateEx));
13703 
13704   size(0);
13705   // use the following format syntax
13706   format %{ "# exception oop is in EAX; no code emitted" %}
13707   ins_encode();
13708   ins_pipe( empty );
13709 %}
13710 
13711 
13712 // Rethrow exception:
13713 // The exception oop will come in the first argument position.
13714 // Then JUMP (not call) to the rethrow stub code.
13715 instruct RethrowException()
13716 %{
13717   match(Rethrow);
13718 
13719   // use the following format syntax
13720   format %{ "JMP    rethrow_stub" %}
13721   ins_encode(enc_rethrow);
13722   ins_pipe( pipe_jmp );
13723 %}
13724 
13725 // inlined locking and unlocking
13726 
13727 
13728 instruct cmpFastLock( eFlagsReg cr, eRegP object, eRegP box, eAXRegI tmp, eRegP scr) %{
13729   match( Set cr (FastLock object box) );
13730   effect( TEMP tmp, TEMP scr );
13731   ins_cost(300);
13732   format %{ "FASTLOCK $object, $box KILLS $tmp,$scr" %}
13733   ins_encode( Fast_Lock(object,box,tmp,scr) );
13734   ins_pipe( pipe_slow );
13735   ins_pc_relative(1);
13736 %}
13737 
13738 instruct cmpFastUnlock( eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
13739   match( Set cr (FastUnlock object box) );
13740   effect( TEMP tmp );
13741   ins_cost(300);
13742   format %{ "FASTUNLOCK $object, $box, $tmp" %}
13743   ins_encode( Fast_Unlock(object,box,tmp) );
13744   ins_pipe( pipe_slow );
13745   ins_pc_relative(1);
13746 %}
13747 
13748 
13749 
13750 // ============================================================================
13751 // Safepoint Instruction
13752 instruct safePoint_poll(eFlagsReg cr) %{
13753   match(SafePoint);
13754   effect(KILL cr);
13755 
13756   // TODO-FIXME: we currently poll at offset 0 of the safepoint polling page.
13757   // On SPARC that might be acceptable as we can generate the address with
13758   // just a sethi, saving an or.  By polling at offset 0 we can end up
13759   // putting additional pressure on the index-0 in the D$.  Because of
13760   // alignment (just like the situation at hand) the lower indices tend
13761   // to see more traffic.  It'd be better to change the polling address
13762   // to offset 0 of the last $line in the polling page.
13763 
13764   format %{ "TSTL   #polladdr,EAX\t! Safepoint: poll for GC" %}
13765   ins_cost(125);
13766   size(6) ;
13767   ins_encode( Safepoint_Poll() );
13768   ins_pipe( ialu_reg_mem );
13769 %}
13770 
13771 //----------PEEPHOLE RULES-----------------------------------------------------
13772 // These must follow all instruction definitions as they use the names
13773 // defined in the instructions definitions.
13774 //
13775 // peepmatch ( root_instr_name [preceding_instruction]* );
13776 //
13777 // peepconstraint %{
13778 // (instruction_number.operand_name relational_op instruction_number.operand_name
13779 //  [, ...] );
13780 // // instruction numbers are zero-based using left to right order in peepmatch
13781 //
13782 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13783 // // provide an instruction_number.operand_name for each operand that appears
13784 // // in the replacement instruction's match rule
13785 //
13786 // ---------VM FLAGS---------------------------------------------------------
13787 //
13788 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13789 //
13790 // Each peephole rule is given an identifying number starting with zero and
13791 // increasing by one in the order seen by the parser.  An individual peephole
13792 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13793 // on the command-line.
13794 //
13795 // ---------CURRENT LIMITATIONS----------------------------------------------
13796 //
13797 // Only match adjacent instructions in same basic block
13798 // Only equality constraints
13799 // Only constraints between operands, not (0.dest_reg == EAX_enc)
13800 // Only one replacement instruction
13801 //
13802 // ---------EXAMPLE----------------------------------------------------------
13803 //
13804 // // pertinent parts of existing instructions in architecture description
13805 // instruct movI(eRegI dst, eRegI src) %{
13806 //   match(Set dst (CopyI src));
13807 // %}
13808 //
13809 // instruct incI_eReg(eRegI dst, immI1 src, eFlagsReg cr) %{
13810 //   match(Set dst (AddI dst src));
13811 //   effect(KILL cr);
13812 // %}
13813 //
13814 // // Change (inc mov) to lea
13815 // peephole %{
13816 //   // increment preceeded by register-register move
13817 //   peepmatch ( incI_eReg movI );
13818 //   // require that the destination register of the increment
13819 //   // match the destination register of the move
13820 //   peepconstraint ( 0.dst == 1.dst );
13821 //   // construct a replacement instruction that sets
13822 //   // the destination to ( move's source register + one )
13823 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13824 // %}
13825 //
13826 // Implementation no longer uses movX instructions since
13827 // machine-independent system no longer uses CopyX nodes.
13828 //
13829 // peephole %{
13830 //   peepmatch ( incI_eReg movI );
13831 //   peepconstraint ( 0.dst == 1.dst );
13832 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13833 // %}
13834 //
13835 // peephole %{
13836 //   peepmatch ( decI_eReg movI );
13837 //   peepconstraint ( 0.dst == 1.dst );
13838 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13839 // %}
13840 //
13841 // peephole %{
13842 //   peepmatch ( addI_eReg_imm movI );
13843 //   peepconstraint ( 0.dst == 1.dst );
13844 //   peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
13845 // %}
13846 //
13847 // peephole %{
13848 //   peepmatch ( addP_eReg_imm movP );
13849 //   peepconstraint ( 0.dst == 1.dst );
13850 //   peepreplace ( leaP_eReg_immI( 0.dst 1.src 0.src ) );
13851 // %}
13852 
13853 // // Change load of spilled value to only a spill
13854 // instruct storeI(memory mem, eRegI src) %{
13855 //   match(Set mem (StoreI mem src));
13856 // %}
13857 //
13858 // instruct loadI(eRegI dst, memory mem) %{
13859 //   match(Set dst (LoadI mem));
13860 // %}
13861 //
13862 peephole %{
13863   peepmatch ( loadI storeI );
13864   peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
13865   peepreplace ( storeI( 1.mem 1.mem 1.src ) );
13866 %}
13867 
13868 //----------SMARTSPILL RULES---------------------------------------------------
13869 // These must follow all instruction definitions as they use the names
13870 // defined in the instructions definitions.