1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Singleton class for RAX pointer register
 262 reg_class ptr_rax_reg(RAX, RAX_H);
 263 
 264 // Singleton class for RBX pointer register
 265 reg_class ptr_rbx_reg(RBX, RBX_H);
 266 
 267 // Singleton class for RSI pointer register
 268 reg_class ptr_rsi_reg(RSI, RSI_H);
 269 
 270 // Singleton class for RDI pointer register
 271 reg_class ptr_rdi_reg(RDI, RDI_H);
 272 
 273 // Singleton class for stack pointer
 274 reg_class ptr_rsp_reg(RSP, RSP_H);
 275 
 276 // Singleton class for TLS pointer
 277 reg_class ptr_r15_reg(R15, R15_H);
 278 
 279 // Singleton class for RAX long register
 280 reg_class long_rax_reg(RAX, RAX_H);
 281 
 282 // Singleton class for RCX long register
 283 reg_class long_rcx_reg(RCX, RCX_H);
 284 
 285 // Singleton class for RDX long register
 286 reg_class long_rdx_reg(RDX, RDX_H);
 287 
 288 // Singleton class for RAX int register
 289 reg_class int_rax_reg(RAX);
 290 
 291 // Singleton class for RBX int register
 292 reg_class int_rbx_reg(RBX);
 293 
 294 // Singleton class for RCX int register
 295 reg_class int_rcx_reg(RCX);
 296 
 297 // Singleton class for RCX int register
 298 reg_class int_rdx_reg(RDX);
 299 
 300 // Singleton class for RCX int register
 301 reg_class int_rdi_reg(RDI);
 302 
 303 // Singleton class for instruction pointer
 304 // reg_class ip_reg(RIP);
 305 
 306 %}
 307 
 308 //----------SOURCE BLOCK-------------------------------------------------------
 309 // This is a block of C++ code which provides values, functions, and
 310 // definitions necessary in the rest of the architecture description
 311 source_hpp %{
 312 
 313 extern RegMask _ANY_REG_mask;
 314 extern RegMask _PTR_REG_mask;
 315 extern RegMask _PTR_REG_NO_RBP_mask;
 316 extern RegMask _PTR_NO_RAX_REG_mask;
 317 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 318 extern RegMask _LONG_REG_mask;
 319 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 320 extern RegMask _LONG_NO_RCX_REG_mask;
 321 extern RegMask _INT_REG_mask;
 322 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 323 extern RegMask _INT_NO_RCX_REG_mask;
 324 
 325 extern RegMask _STACK_OR_PTR_REG_mask;
 326 extern RegMask _STACK_OR_LONG_REG_mask;
 327 extern RegMask _STACK_OR_INT_REG_mask;
 328 
 329 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 330 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 331 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 332 
 333 %}
 334 
 335 source %{
 336 #define   RELOC_IMM64    Assembler::imm_operand
 337 #define   RELOC_DISP32   Assembler::disp32_operand
 338 
 339 #define __ _masm.
 340 
 341 RegMask _ANY_REG_mask;
 342 RegMask _PTR_REG_mask;
 343 RegMask _PTR_REG_NO_RBP_mask;
 344 RegMask _PTR_NO_RAX_REG_mask;
 345 RegMask _PTR_NO_RAX_RBX_REG_mask;
 346 RegMask _LONG_REG_mask;
 347 RegMask _LONG_NO_RAX_RDX_REG_mask;
 348 RegMask _LONG_NO_RCX_REG_mask;
 349 RegMask _INT_REG_mask;
 350 RegMask _INT_NO_RAX_RDX_REG_mask;
 351 RegMask _INT_NO_RCX_REG_mask;
 352 RegMask _STACK_OR_PTR_REG_mask;
 353 RegMask _STACK_OR_LONG_REG_mask;
 354 RegMask _STACK_OR_INT_REG_mask;
 355 
 356 static bool need_r12_heapbase() {
 357   return UseCompressedOops || UseCompressedClassPointers || UseZGC;
 358 }
 359 
 360 void reg_mask_init() {
 361   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 362   // We derive a number of subsets from it.
 363   _ANY_REG_mask = _ALL_REG_mask;
 364 
 365   if (PreserveFramePointer) {
 366     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 368   }
 369   if (need_r12_heapbase()) {
 370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 372   }
 373 
 374   _PTR_REG_mask = _ANY_REG_mask;
 375   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 379 
 380   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 381   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 382 
 383   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 384   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 386 
 387   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 388   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 390 
 391   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 392   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 394 
 395   _LONG_REG_mask = _PTR_REG_mask;
 396   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 397   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 398 
 399   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 400   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 404 
 405   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 406   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 408 
 409   _INT_REG_mask = _ALL_INT_REG_mask;
 410   if (PreserveFramePointer) {
 411     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 412   }
 413   if (need_r12_heapbase()) {
 414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 415   }
 416 
 417   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 418   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 419 
 420   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 421   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 423 
 424   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 425   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 426 }
 427 
 428 static bool generate_vzeroupper(Compile* C) {
 429   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 430 }
 431 
 432 static int clear_avx_size() {
 433   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 434 }
 435 
 436 // !!!!! Special hack to get all types of calls to specify the byte offset
 437 //       from the start of the call to the point where the return address
 438 //       will point.
 439 int MachCallStaticJavaNode::ret_addr_offset()
 440 {
 441   int offset = 5; // 5 bytes from start of call to where return address points
 442   offset += clear_avx_size();
 443   return offset;
 444 }
 445 
 446 int MachCallDynamicJavaNode::ret_addr_offset()
 447 {
 448   int offset = 15; // 15 bytes from start of call to where return address points
 449   offset += clear_avx_size();
 450   return offset;
 451 }
 452 
 453 int MachCallRuntimeNode::ret_addr_offset() {
 454   int offset = 13; // movq r10,#addr; callq (r10)
 455   offset += clear_avx_size();
 456   return offset;
 457 }
 458 
 459 // Indicate if the safepoint node needs the polling page as an input,
 460 // it does if the polling page is more than disp32 away.
 461 bool SafePointNode::needs_polling_address_input()
 462 {
 463   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 464 }
 465 
 466 //
 467 // Compute padding required for nodes which need alignment
 468 //
 469 
 470 // The address of the call instruction needs to be 4-byte aligned to
 471 // ensure that it does not span a cache line so that it can be patched.
 472 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 473 {
 474   current_offset += clear_avx_size(); // skip vzeroupper
 475   current_offset += 1; // skip call opcode byte
 476   return align_up(current_offset, alignment_required()) - current_offset;
 477 }
 478 
 479 // The address of the call instruction needs to be 4-byte aligned to
 480 // ensure that it does not span a cache line so that it can be patched.
 481 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 482 {
 483   current_offset += clear_avx_size(); // skip vzeroupper
 484   current_offset += 11; // skip movq instruction + call opcode byte
 485   return align_up(current_offset, alignment_required()) - current_offset;
 486 }
 487 
 488 // EMIT_RM()
 489 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 490   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 491   cbuf.insts()->emit_int8(c);
 492 }
 493 
 494 // EMIT_CC()
 495 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 496   unsigned char c = (unsigned char) (f1 | f2);
 497   cbuf.insts()->emit_int8(c);
 498 }
 499 
 500 // EMIT_OPCODE()
 501 void emit_opcode(CodeBuffer &cbuf, int code) {
 502   cbuf.insts()->emit_int8((unsigned char) code);
 503 }
 504 
 505 // EMIT_OPCODE() w/ relocation information
 506 void emit_opcode(CodeBuffer &cbuf,
 507                  int code, relocInfo::relocType reloc, int offset, int format)
 508 {
 509   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 510   emit_opcode(cbuf, code);
 511 }
 512 
 513 // EMIT_D8()
 514 void emit_d8(CodeBuffer &cbuf, int d8) {
 515   cbuf.insts()->emit_int8((unsigned char) d8);
 516 }
 517 
 518 // EMIT_D16()
 519 void emit_d16(CodeBuffer &cbuf, int d16) {
 520   cbuf.insts()->emit_int16(d16);
 521 }
 522 
 523 // EMIT_D32()
 524 void emit_d32(CodeBuffer &cbuf, int d32) {
 525   cbuf.insts()->emit_int32(d32);
 526 }
 527 
 528 // EMIT_D64()
 529 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 530   cbuf.insts()->emit_int64(d64);
 531 }
 532 
 533 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 534 void emit_d32_reloc(CodeBuffer& cbuf,
 535                     int d32,
 536                     relocInfo::relocType reloc,
 537                     int format)
 538 {
 539   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 540   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 541   cbuf.insts()->emit_int32(d32);
 542 }
 543 
 544 // emit 32 bit value and construct relocation entry from RelocationHolder
 545 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 546 #ifdef ASSERT
 547   if (rspec.reloc()->type() == relocInfo::oop_type &&
 548       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 549     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 550     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 551   }
 552 #endif
 553   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 554   cbuf.insts()->emit_int32(d32);
 555 }
 556 
 557 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 558   address next_ip = cbuf.insts_end() + 4;
 559   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 560                  external_word_Relocation::spec(addr),
 561                  RELOC_DISP32);
 562 }
 563 
 564 
 565 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 566 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 567   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 568   cbuf.insts()->emit_int64(d64);
 569 }
 570 
 571 // emit 64 bit value and construct relocation entry from RelocationHolder
 572 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 573 #ifdef ASSERT
 574   if (rspec.reloc()->type() == relocInfo::oop_type &&
 575       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 576     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 577     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 578            "cannot embed scavengable oops in code");
 579   }
 580 #endif
 581   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 582   cbuf.insts()->emit_int64(d64);
 583 }
 584 
 585 // Access stack slot for load or store
 586 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 587 {
 588   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 589   if (-0x80 <= disp && disp < 0x80) {
 590     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 591     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 592     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 593   } else {
 594     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 595     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 596     emit_d32(cbuf, disp);     // Displacement // R/M byte
 597   }
 598 }
 599 
 600    // rRegI ereg, memory mem) %{    // emit_reg_mem
 601 void encode_RegMem(CodeBuffer &cbuf,
 602                    int reg,
 603                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 604 {
 605   assert(disp_reloc == relocInfo::none, "cannot have disp");
 606   int regenc = reg & 7;
 607   int baseenc = base & 7;
 608   int indexenc = index & 7;
 609 
 610   // There is no index & no scale, use form without SIB byte
 611   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 612     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 613     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 614       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 615     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 616       // If 8-bit displacement, mode 0x1
 617       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 618       emit_d8(cbuf, disp);
 619     } else {
 620       // If 32-bit displacement
 621       if (base == -1) { // Special flag for absolute address
 622         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 623         if (disp_reloc != relocInfo::none) {
 624           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 625         } else {
 626           emit_d32(cbuf, disp);
 627         }
 628       } else {
 629         // Normal base + offset
 630         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 631         if (disp_reloc != relocInfo::none) {
 632           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 633         } else {
 634           emit_d32(cbuf, disp);
 635         }
 636       }
 637     }
 638   } else {
 639     // Else, encode with the SIB byte
 640     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 641     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 642       // If no displacement
 643       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 644       emit_rm(cbuf, scale, indexenc, baseenc);
 645     } else {
 646       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 647         // If 8-bit displacement, mode 0x1
 648         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 649         emit_rm(cbuf, scale, indexenc, baseenc);
 650         emit_d8(cbuf, disp);
 651       } else {
 652         // If 32-bit displacement
 653         if (base == 0x04 ) {
 654           emit_rm(cbuf, 0x2, regenc, 0x4);
 655           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 656         } else {
 657           emit_rm(cbuf, 0x2, regenc, 0x4);
 658           emit_rm(cbuf, scale, indexenc, baseenc); // *
 659         }
 660         if (disp_reloc != relocInfo::none) {
 661           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 662         } else {
 663           emit_d32(cbuf, disp);
 664         }
 665       }
 666     }
 667   }
 668 }
 669 
 670 // This could be in MacroAssembler but it's fairly C2 specific
 671 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 672   Label exit;
 673   __ jccb(Assembler::noParity, exit);
 674   __ pushf();
 675   //
 676   // comiss/ucomiss instructions set ZF,PF,CF flags and
 677   // zero OF,AF,SF for NaN values.
 678   // Fixup flags by zeroing ZF,PF so that compare of NaN
 679   // values returns 'less than' result (CF is set).
 680   // Leave the rest of flags unchanged.
 681   //
 682   //    7 6 5 4 3 2 1 0
 683   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 684   //    0 0 1 0 1 0 1 1   (0x2B)
 685   //
 686   __ andq(Address(rsp, 0), 0xffffff2b);
 687   __ popf();
 688   __ bind(exit);
 689 }
 690 
 691 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 692   Label done;
 693   __ movl(dst, -1);
 694   __ jcc(Assembler::parity, done);
 695   __ jcc(Assembler::below, done);
 696   __ setb(Assembler::notEqual, dst);
 697   __ movzbl(dst, dst);
 698   __ bind(done);
 699 }
 700 
 701 
 702 //=============================================================================
 703 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 704 
 705 int Compile::ConstantTable::calculate_table_base_offset() const {
 706   return 0;  // absolute addressing, no offset
 707 }
 708 
 709 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 710 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 711   ShouldNotReachHere();
 712 }
 713 
 714 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 715   // Empty encoding
 716 }
 717 
 718 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 719   return 0;
 720 }
 721 
 722 #ifndef PRODUCT
 723 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 724   st->print("# MachConstantBaseNode (empty encoding)");
 725 }
 726 #endif
 727 
 728 
 729 //=============================================================================
 730 #ifndef PRODUCT
 731 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 732   Compile* C = ra_->C;
 733 
 734   int framesize = C->frame_size_in_bytes();
 735   int bangsize = C->bang_size_in_bytes();
 736   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 737   // Remove wordSize for return addr which is already pushed.
 738   framesize -= wordSize;
 739 
 740   if (C->need_stack_bang(bangsize)) {
 741     framesize -= wordSize;
 742     st->print("# stack bang (%d bytes)", bangsize);
 743     st->print("\n\t");
 744     st->print("pushq   rbp\t# Save rbp");
 745     if (PreserveFramePointer) {
 746         st->print("\n\t");
 747         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 748     }
 749     if (framesize) {
 750       st->print("\n\t");
 751       st->print("subq    rsp, #%d\t# Create frame",framesize);
 752     }
 753   } else {
 754     st->print("subq    rsp, #%d\t# Create frame",framesize);
 755     st->print("\n\t");
 756     framesize -= wordSize;
 757     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 758     if (PreserveFramePointer) {
 759       st->print("\n\t");
 760       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 761       if (framesize > 0) {
 762         st->print("\n\t");
 763         st->print("addq    rbp, #%d", framesize);
 764       }
 765     }
 766   }
 767 
 768   if (VerifyStackAtCalls) {
 769     st->print("\n\t");
 770     framesize -= wordSize;
 771     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 772 #ifdef ASSERT
 773     st->print("\n\t");
 774     st->print("# stack alignment check");
 775 #endif
 776   }
 777   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 778     st->print("\n\t");
 779     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 780     st->print("\n\t");
 781     st->print("je      fast_entry\t");
 782     st->print("\n\t");
 783     st->print("call    #nmethod_entry_barrier_stub\t");
 784     st->print("\n\tfast_entry:");
 785   }
 786   st->cr();
 787 }
 788 #endif
 789 
 790 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 791   Compile* C = ra_->C;
 792   MacroAssembler _masm(&cbuf);
 793 
 794   int framesize = C->frame_size_in_bytes();
 795   int bangsize = C->bang_size_in_bytes();
 796 
 797   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
 798 
 799   C->set_frame_complete(cbuf.insts_size());
 800 
 801   if (C->has_mach_constant_base_node()) {
 802     // NOTE: We set the table base offset here because users might be
 803     // emitted before MachConstantBaseNode.
 804     Compile::ConstantTable& constant_table = C->constant_table();
 805     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 806   }
 807 }
 808 
 809 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 810 {
 811   return MachNode::size(ra_); // too many variables; just compute it
 812                               // the hard way
 813 }
 814 
 815 int MachPrologNode::reloc() const
 816 {
 817   return 0; // a large enough number
 818 }
 819 
 820 //=============================================================================
 821 #ifndef PRODUCT
 822 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 823 {
 824   Compile* C = ra_->C;
 825   if (generate_vzeroupper(C)) {
 826     st->print("vzeroupper");
 827     st->cr(); st->print("\t");
 828   }
 829 
 830   int framesize = C->frame_size_in_bytes();
 831   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 832   // Remove word for return adr already pushed
 833   // and RBP
 834   framesize -= 2*wordSize;
 835 
 836   if (framesize) {
 837     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 838     st->print("\t");
 839   }
 840 
 841   st->print_cr("popq   rbp");
 842   if (do_polling() && C->is_method_compilation()) {
 843     st->print("\t");
 844     if (SafepointMechanism::uses_thread_local_poll()) {
 845       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 846                    "testl  rax, [rscratch1]\t"
 847                    "# Safepoint: poll for GC");
 848     } else if (Assembler::is_polling_page_far()) {
 849       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 850                    "testl  rax, [rscratch1]\t"
 851                    "# Safepoint: poll for GC");
 852     } else {
 853       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 854                    "# Safepoint: poll for GC");
 855     }
 856   }
 857 }
 858 #endif
 859 
 860 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 861 {
 862   Compile* C = ra_->C;
 863   MacroAssembler _masm(&cbuf);
 864 
 865   if (generate_vzeroupper(C)) {
 866     // Clear upper bits of YMM registers when current compiled code uses
 867     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 868     __ vzeroupper();
 869   }
 870 
 871   int framesize = C->frame_size_in_bytes();
 872   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 873   // Remove word for return adr already pushed
 874   // and RBP
 875   framesize -= 2*wordSize;
 876 
 877   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 878 
 879   if (framesize) {
 880     emit_opcode(cbuf, Assembler::REX_W);
 881     if (framesize < 0x80) {
 882       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 883       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 884       emit_d8(cbuf, framesize);
 885     } else {
 886       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 887       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 888       emit_d32(cbuf, framesize);
 889     }
 890   }
 891 
 892   // popq rbp
 893   emit_opcode(cbuf, 0x58 | RBP_enc);
 894 
 895   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 896     __ reserved_stack_check();
 897   }
 898 
 899   if (do_polling() && C->is_method_compilation()) {
 900     MacroAssembler _masm(&cbuf);
 901     if (SafepointMechanism::uses_thread_local_poll()) {
 902       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
 903       __ relocate(relocInfo::poll_return_type);
 904       __ testl(rax, Address(rscratch1, 0));
 905     } else {
 906       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 907       if (Assembler::is_polling_page_far()) {
 908         __ lea(rscratch1, polling_page);
 909         __ relocate(relocInfo::poll_return_type);
 910         __ testl(rax, Address(rscratch1, 0));
 911       } else {
 912         __ testl(rax, polling_page);
 913       }
 914     }
 915   }
 916 }
 917 
 918 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 919 {
 920   return MachNode::size(ra_); // too many variables; just compute it
 921                               // the hard way
 922 }
 923 
 924 int MachEpilogNode::reloc() const
 925 {
 926   return 2; // a large enough number
 927 }
 928 
 929 const Pipeline* MachEpilogNode::pipeline() const
 930 {
 931   return MachNode::pipeline_class();
 932 }
 933 
 934 int MachEpilogNode::safepoint_offset() const
 935 {
 936   return 0;
 937 }
 938 
 939 //=============================================================================
 940 
 941 enum RC {
 942   rc_bad,
 943   rc_int,
 944   rc_float,
 945   rc_stack
 946 };
 947 
 948 static enum RC rc_class(OptoReg::Name reg)
 949 {
 950   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 951 
 952   if (OptoReg::is_stack(reg)) return rc_stack;
 953 
 954   VMReg r = OptoReg::as_VMReg(reg);
 955 
 956   if (r->is_Register()) return rc_int;
 957 
 958   assert(r->is_XMMRegister(), "must be");
 959   return rc_float;
 960 }
 961 
 962 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 963 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 964                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 965 
 966 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 967                             int stack_offset, int reg, uint ireg, outputStream* st);
 968 
 969 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 970                                       int dst_offset, uint ireg, outputStream* st) {
 971   if (cbuf) {
 972     MacroAssembler _masm(cbuf);
 973     switch (ireg) {
 974     case Op_VecS:
 975       __ movq(Address(rsp, -8), rax);
 976       __ movl(rax, Address(rsp, src_offset));
 977       __ movl(Address(rsp, dst_offset), rax);
 978       __ movq(rax, Address(rsp, -8));
 979       break;
 980     case Op_VecD:
 981       __ pushq(Address(rsp, src_offset));
 982       __ popq (Address(rsp, dst_offset));
 983       break;
 984     case Op_VecX:
 985       __ pushq(Address(rsp, src_offset));
 986       __ popq (Address(rsp, dst_offset));
 987       __ pushq(Address(rsp, src_offset+8));
 988       __ popq (Address(rsp, dst_offset+8));
 989       break;
 990     case Op_VecY:
 991       __ vmovdqu(Address(rsp, -32), xmm0);
 992       __ vmovdqu(xmm0, Address(rsp, src_offset));
 993       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 994       __ vmovdqu(xmm0, Address(rsp, -32));
 995       break;
 996     case Op_VecZ:
 997       __ evmovdquq(Address(rsp, -64), xmm0, 2);
 998       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
 999       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1000       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1001       break;
1002     default:
1003       ShouldNotReachHere();
1004     }
1005 #ifndef PRODUCT
1006   } else {
1007     switch (ireg) {
1008     case Op_VecS:
1009       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1010                 "movl    rax, [rsp + #%d]\n\t"
1011                 "movl    [rsp + #%d], rax\n\t"
1012                 "movq    rax, [rsp - #8]",
1013                 src_offset, dst_offset);
1014       break;
1015     case Op_VecD:
1016       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1017                 "popq    [rsp + #%d]",
1018                 src_offset, dst_offset);
1019       break;
1020      case Op_VecX:
1021       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1022                 "popq    [rsp + #%d]\n\t"
1023                 "pushq   [rsp + #%d]\n\t"
1024                 "popq    [rsp + #%d]",
1025                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1026       break;
1027     case Op_VecY:
1028       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1029                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1030                 "vmovdqu [rsp + #%d], xmm0\n\t"
1031                 "vmovdqu xmm0, [rsp - #32]",
1032                 src_offset, dst_offset);
1033       break;
1034     case Op_VecZ:
1035       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1036                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1037                 "vmovdqu [rsp + #%d], xmm0\n\t"
1038                 "vmovdqu xmm0, [rsp - #64]",
1039                 src_offset, dst_offset);
1040       break;
1041     default:
1042       ShouldNotReachHere();
1043     }
1044 #endif
1045   }
1046 }
1047 
1048 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1049                                        PhaseRegAlloc* ra_,
1050                                        bool do_size,
1051                                        outputStream* st) const {
1052   assert(cbuf != NULL || st  != NULL, "sanity");
1053   // Get registers to move
1054   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1055   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1056   OptoReg::Name dst_second = ra_->get_reg_second(this);
1057   OptoReg::Name dst_first = ra_->get_reg_first(this);
1058 
1059   enum RC src_second_rc = rc_class(src_second);
1060   enum RC src_first_rc = rc_class(src_first);
1061   enum RC dst_second_rc = rc_class(dst_second);
1062   enum RC dst_first_rc = rc_class(dst_first);
1063 
1064   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1065          "must move at least 1 register" );
1066 
1067   if (src_first == dst_first && src_second == dst_second) {
1068     // Self copy, no move
1069     return 0;
1070   }
1071   if (bottom_type()->isa_vect() != NULL) {
1072     uint ireg = ideal_reg();
1073     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1074     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1075     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1076       // mem -> mem
1077       int src_offset = ra_->reg2offset(src_first);
1078       int dst_offset = ra_->reg2offset(dst_first);
1079       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1080     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1081       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1082     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1083       int stack_offset = ra_->reg2offset(dst_first);
1084       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1085     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1086       int stack_offset = ra_->reg2offset(src_first);
1087       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1088     } else {
1089       ShouldNotReachHere();
1090     }
1091     return 0;
1092   }
1093   if (src_first_rc == rc_stack) {
1094     // mem ->
1095     if (dst_first_rc == rc_stack) {
1096       // mem -> mem
1097       assert(src_second != dst_first, "overlap");
1098       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1099           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1100         // 64-bit
1101         int src_offset = ra_->reg2offset(src_first);
1102         int dst_offset = ra_->reg2offset(dst_first);
1103         if (cbuf) {
1104           MacroAssembler _masm(cbuf);
1105           __ pushq(Address(rsp, src_offset));
1106           __ popq (Address(rsp, dst_offset));
1107 #ifndef PRODUCT
1108         } else {
1109           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1110                     "popq    [rsp + #%d]",
1111                      src_offset, dst_offset);
1112 #endif
1113         }
1114       } else {
1115         // 32-bit
1116         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1117         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1118         // No pushl/popl, so:
1119         int src_offset = ra_->reg2offset(src_first);
1120         int dst_offset = ra_->reg2offset(dst_first);
1121         if (cbuf) {
1122           MacroAssembler _masm(cbuf);
1123           __ movq(Address(rsp, -8), rax);
1124           __ movl(rax, Address(rsp, src_offset));
1125           __ movl(Address(rsp, dst_offset), rax);
1126           __ movq(rax, Address(rsp, -8));
1127 #ifndef PRODUCT
1128         } else {
1129           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1130                     "movl    rax, [rsp + #%d]\n\t"
1131                     "movl    [rsp + #%d], rax\n\t"
1132                     "movq    rax, [rsp - #8]",
1133                      src_offset, dst_offset);
1134 #endif
1135         }
1136       }
1137       return 0;
1138     } else if (dst_first_rc == rc_int) {
1139       // mem -> gpr
1140       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1141           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1142         // 64-bit
1143         int offset = ra_->reg2offset(src_first);
1144         if (cbuf) {
1145           MacroAssembler _masm(cbuf);
1146           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1147 #ifndef PRODUCT
1148         } else {
1149           st->print("movq    %s, [rsp + #%d]\t# spill",
1150                      Matcher::regName[dst_first],
1151                      offset);
1152 #endif
1153         }
1154       } else {
1155         // 32-bit
1156         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1157         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1158         int offset = ra_->reg2offset(src_first);
1159         if (cbuf) {
1160           MacroAssembler _masm(cbuf);
1161           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1162 #ifndef PRODUCT
1163         } else {
1164           st->print("movl    %s, [rsp + #%d]\t# spill",
1165                      Matcher::regName[dst_first],
1166                      offset);
1167 #endif
1168         }
1169       }
1170       return 0;
1171     } else if (dst_first_rc == rc_float) {
1172       // mem-> xmm
1173       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1174           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1175         // 64-bit
1176         int offset = ra_->reg2offset(src_first);
1177         if (cbuf) {
1178           MacroAssembler _masm(cbuf);
1179           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1180 #ifndef PRODUCT
1181         } else {
1182           st->print("%s  %s, [rsp + #%d]\t# spill",
1183                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1184                      Matcher::regName[dst_first],
1185                      offset);
1186 #endif
1187         }
1188       } else {
1189         // 32-bit
1190         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1191         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1192         int offset = ra_->reg2offset(src_first);
1193         if (cbuf) {
1194           MacroAssembler _masm(cbuf);
1195           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1196 #ifndef PRODUCT
1197         } else {
1198           st->print("movss   %s, [rsp + #%d]\t# spill",
1199                      Matcher::regName[dst_first],
1200                      offset);
1201 #endif
1202         }
1203       }
1204       return 0;
1205     }
1206   } else if (src_first_rc == rc_int) {
1207     // gpr ->
1208     if (dst_first_rc == rc_stack) {
1209       // gpr -> mem
1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1212         // 64-bit
1213         int offset = ra_->reg2offset(dst_first);
1214         if (cbuf) {
1215           MacroAssembler _masm(cbuf);
1216           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1217 #ifndef PRODUCT
1218         } else {
1219           st->print("movq    [rsp + #%d], %s\t# spill",
1220                      offset,
1221                      Matcher::regName[src_first]);
1222 #endif
1223         }
1224       } else {
1225         // 32-bit
1226         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1227         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1228         int offset = ra_->reg2offset(dst_first);
1229         if (cbuf) {
1230           MacroAssembler _masm(cbuf);
1231           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1232 #ifndef PRODUCT
1233         } else {
1234           st->print("movl    [rsp + #%d], %s\t# spill",
1235                      offset,
1236                      Matcher::regName[src_first]);
1237 #endif
1238         }
1239       }
1240       return 0;
1241     } else if (dst_first_rc == rc_int) {
1242       // gpr -> gpr
1243       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1244           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1245         // 64-bit
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1249                   as_Register(Matcher::_regEncode[src_first]));
1250 #ifndef PRODUCT
1251         } else {
1252           st->print("movq    %s, %s\t# spill",
1253                      Matcher::regName[dst_first],
1254                      Matcher::regName[src_first]);
1255 #endif
1256         }
1257         return 0;
1258       } else {
1259         // 32-bit
1260         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1261         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1262         if (cbuf) {
1263           MacroAssembler _masm(cbuf);
1264           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1265                   as_Register(Matcher::_regEncode[src_first]));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("movl    %s, %s\t# spill",
1269                      Matcher::regName[dst_first],
1270                      Matcher::regName[src_first]);
1271 #endif
1272         }
1273         return 0;
1274       }
1275     } else if (dst_first_rc == rc_float) {
1276       // gpr -> xmm
1277       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1278           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1279         // 64-bit
1280         if (cbuf) {
1281           MacroAssembler _masm(cbuf);
1282           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1283 #ifndef PRODUCT
1284         } else {
1285           st->print("movdq   %s, %s\t# spill",
1286                      Matcher::regName[dst_first],
1287                      Matcher::regName[src_first]);
1288 #endif
1289         }
1290       } else {
1291         // 32-bit
1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1294         if (cbuf) {
1295           MacroAssembler _masm(cbuf);
1296           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1297 #ifndef PRODUCT
1298         } else {
1299           st->print("movdl   %s, %s\t# spill",
1300                      Matcher::regName[dst_first],
1301                      Matcher::regName[src_first]);
1302 #endif
1303         }
1304       }
1305       return 0;
1306     }
1307   } else if (src_first_rc == rc_float) {
1308     // xmm ->
1309     if (dst_first_rc == rc_stack) {
1310       // xmm -> mem
1311       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1312           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1313         // 64-bit
1314         int offset = ra_->reg2offset(dst_first);
1315         if (cbuf) {
1316           MacroAssembler _masm(cbuf);
1317           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1318 #ifndef PRODUCT
1319         } else {
1320           st->print("movsd   [rsp + #%d], %s\t# spill",
1321                      offset,
1322                      Matcher::regName[src_first]);
1323 #endif
1324         }
1325       } else {
1326         // 32-bit
1327         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1328         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1329         int offset = ra_->reg2offset(dst_first);
1330         if (cbuf) {
1331           MacroAssembler _masm(cbuf);
1332           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1333 #ifndef PRODUCT
1334         } else {
1335           st->print("movss   [rsp + #%d], %s\t# spill",
1336                      offset,
1337                      Matcher::regName[src_first]);
1338 #endif
1339         }
1340       }
1341       return 0;
1342     } else if (dst_first_rc == rc_int) {
1343       // xmm -> gpr
1344       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1345           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1346         // 64-bit
1347         if (cbuf) {
1348           MacroAssembler _masm(cbuf);
1349           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1350 #ifndef PRODUCT
1351         } else {
1352           st->print("movdq   %s, %s\t# spill",
1353                      Matcher::regName[dst_first],
1354                      Matcher::regName[src_first]);
1355 #endif
1356         }
1357       } else {
1358         // 32-bit
1359         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1360         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1361         if (cbuf) {
1362           MacroAssembler _masm(cbuf);
1363           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1364 #ifndef PRODUCT
1365         } else {
1366           st->print("movdl   %s, %s\t# spill",
1367                      Matcher::regName[dst_first],
1368                      Matcher::regName[src_first]);
1369 #endif
1370         }
1371       }
1372       return 0;
1373     } else if (dst_first_rc == rc_float) {
1374       // xmm -> xmm
1375       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1376           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1377         // 64-bit
1378         if (cbuf) {
1379           MacroAssembler _masm(cbuf);
1380           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1381 #ifndef PRODUCT
1382         } else {
1383           st->print("%s  %s, %s\t# spill",
1384                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1385                      Matcher::regName[dst_first],
1386                      Matcher::regName[src_first]);
1387 #endif
1388         }
1389       } else {
1390         // 32-bit
1391         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1392         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1393         if (cbuf) {
1394           MacroAssembler _masm(cbuf);
1395           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1396 #ifndef PRODUCT
1397         } else {
1398           st->print("%s  %s, %s\t# spill",
1399                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1400                      Matcher::regName[dst_first],
1401                      Matcher::regName[src_first]);
1402 #endif
1403         }
1404       }
1405       return 0;
1406     }
1407   }
1408 
1409   assert(0," foo ");
1410   Unimplemented();
1411   return 0;
1412 }
1413 
1414 #ifndef PRODUCT
1415 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1416   implementation(NULL, ra_, false, st);
1417 }
1418 #endif
1419 
1420 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1421   implementation(&cbuf, ra_, false, NULL);
1422 }
1423 
1424 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1425   return MachNode::size(ra_);
1426 }
1427 
1428 //=============================================================================
1429 #ifndef PRODUCT
1430 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1431 {
1432   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1433   int reg = ra_->get_reg_first(this);
1434   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1435             Matcher::regName[reg], offset);
1436 }
1437 #endif
1438 
1439 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1440 {
1441   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1442   int reg = ra_->get_encode(this);
1443   if (offset >= 0x80) {
1444     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1445     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1446     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1447     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1448     emit_d32(cbuf, offset);
1449   } else {
1450     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1451     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1452     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1453     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1454     emit_d8(cbuf, offset);
1455   }
1456 }
1457 
1458 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1459 {
1460   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1461   return (offset < 0x80) ? 5 : 8; // REX
1462 }
1463 
1464 //=============================================================================
1465 #ifndef PRODUCT
1466 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1467 {
1468   if (UseCompressedClassPointers) {
1469     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1470     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1471     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1472   } else {
1473     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1474                  "# Inline cache check");
1475   }
1476   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1477   st->print_cr("\tnop\t# nops to align entry point");
1478 }
1479 #endif
1480 
1481 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1482 {
1483   MacroAssembler masm(&cbuf);
1484   uint insts_size = cbuf.insts_size();
1485   if (UseCompressedClassPointers) {
1486     masm.load_klass(rscratch1, j_rarg0);
1487     masm.cmpptr(rax, rscratch1);
1488   } else {
1489     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1490   }
1491 
1492   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1493 
1494   /* WARNING these NOPs are critical so that verified entry point is properly
1495      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1496   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1497   if (OptoBreakpoint) {
1498     // Leave space for int3
1499     nops_cnt -= 1;
1500   }
1501   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1502   if (nops_cnt > 0)
1503     masm.nop(nops_cnt);
1504 }
1505 
1506 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1507 {
1508   return MachNode::size(ra_); // too many variables; just compute it
1509                               // the hard way
1510 }
1511 
1512 
1513 //=============================================================================
1514 
1515 int Matcher::regnum_to_fpu_offset(int regnum)
1516 {
1517   return regnum - 32; // The FP registers are in the second chunk
1518 }
1519 
1520 // This is UltraSparc specific, true just means we have fast l2f conversion
1521 const bool Matcher::convL2FSupported(void) {
1522   return true;
1523 }
1524 
1525 // Is this branch offset short enough that a short branch can be used?
1526 //
1527 // NOTE: If the platform does not provide any short branch variants, then
1528 //       this method should return false for offset 0.
1529 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1530   // The passed offset is relative to address of the branch.
1531   // On 86 a branch displacement is calculated relative to address
1532   // of a next instruction.
1533   offset -= br_size;
1534 
1535   // the short version of jmpConUCF2 contains multiple branches,
1536   // making the reach slightly less
1537   if (rule == jmpConUCF2_rule)
1538     return (-126 <= offset && offset <= 125);
1539   return (-128 <= offset && offset <= 127);
1540 }
1541 
1542 const bool Matcher::isSimpleConstant64(jlong value) {
1543   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1544   //return value == (int) value;  // Cf. storeImmL and immL32.
1545 
1546   // Probably always true, even if a temp register is required.
1547   return true;
1548 }
1549 
1550 // The ecx parameter to rep stosq for the ClearArray node is in words.
1551 const bool Matcher::init_array_count_is_in_bytes = false;
1552 
1553 // No additional cost for CMOVL.
1554 const int Matcher::long_cmove_cost() { return 0; }
1555 
1556 // No CMOVF/CMOVD with SSE2
1557 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1558 
1559 // Does the CPU require late expand (see block.cpp for description of late expand)?
1560 const bool Matcher::require_postalloc_expand = false;
1561 
1562 // Do we need to mask the count passed to shift instructions or does
1563 // the cpu only look at the lower 5/6 bits anyway?
1564 const bool Matcher::need_masked_shift_count = false;
1565 
1566 bool Matcher::narrow_oop_use_complex_address() {
1567   assert(UseCompressedOops, "only for compressed oops code");
1568   return (LogMinObjAlignmentInBytes <= 3);
1569 }
1570 
1571 bool Matcher::narrow_klass_use_complex_address() {
1572   assert(UseCompressedClassPointers, "only for compressed klass code");
1573   return (LogKlassAlignmentInBytes <= 3);
1574 }
1575 
1576 bool Matcher::const_oop_prefer_decode() {
1577   // Prefer ConN+DecodeN over ConP.
1578   return true;
1579 }
1580 
1581 bool Matcher::const_klass_prefer_decode() {
1582   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1583   //       or condisider the following:
1584   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1585   //return Universe::narrow_klass_base() == NULL;
1586   return true;
1587 }
1588 
1589 // Is it better to copy float constants, or load them directly from
1590 // memory?  Intel can load a float constant from a direct address,
1591 // requiring no extra registers.  Most RISCs will have to materialize
1592 // an address into a register first, so they would do better to copy
1593 // the constant from stack.
1594 const bool Matcher::rematerialize_float_constants = true; // XXX
1595 
1596 // If CPU can load and store mis-aligned doubles directly then no
1597 // fixup is needed.  Else we split the double into 2 integer pieces
1598 // and move it piece-by-piece.  Only happens when passing doubles into
1599 // C code as the Java calling convention forces doubles to be aligned.
1600 const bool Matcher::misaligned_doubles_ok = true;
1601 
1602 // No-op on amd64
1603 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1604 
1605 // Advertise here if the CPU requires explicit rounding operations to
1606 // implement the UseStrictFP mode.
1607 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1608 
1609 // Are floats conerted to double when stored to stack during deoptimization?
1610 // On x64 it is stored without convertion so we can use normal access.
1611 bool Matcher::float_in_double() { return false; }
1612 
1613 // Do ints take an entire long register or just half?
1614 const bool Matcher::int_in_long = true;
1615 
1616 // Return whether or not this register is ever used as an argument.
1617 // This function is used on startup to build the trampoline stubs in
1618 // generateOptoStub.  Registers not mentioned will be killed by the VM
1619 // call in the trampoline, and arguments in those registers not be
1620 // available to the callee.
1621 bool Matcher::can_be_java_arg(int reg)
1622 {
1623   return
1624     reg ==  RDI_num || reg == RDI_H_num ||
1625     reg ==  RSI_num || reg == RSI_H_num ||
1626     reg ==  RDX_num || reg == RDX_H_num ||
1627     reg ==  RCX_num || reg == RCX_H_num ||
1628     reg ==   R8_num || reg ==  R8_H_num ||
1629     reg ==   R9_num || reg ==  R9_H_num ||
1630     reg ==  R12_num || reg == R12_H_num ||
1631     reg == XMM0_num || reg == XMM0b_num ||
1632     reg == XMM1_num || reg == XMM1b_num ||
1633     reg == XMM2_num || reg == XMM2b_num ||
1634     reg == XMM3_num || reg == XMM3b_num ||
1635     reg == XMM4_num || reg == XMM4b_num ||
1636     reg == XMM5_num || reg == XMM5b_num ||
1637     reg == XMM6_num || reg == XMM6b_num ||
1638     reg == XMM7_num || reg == XMM7b_num;
1639 }
1640 
1641 bool Matcher::is_spillable_arg(int reg)
1642 {
1643   return can_be_java_arg(reg);
1644 }
1645 
1646 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1647   // In 64 bit mode a code which use multiply when
1648   // devisor is constant is faster than hardware
1649   // DIV instruction (it uses MulHiL).
1650   return false;
1651 }
1652 
1653 // Register for DIVI projection of divmodI
1654 RegMask Matcher::divI_proj_mask() {
1655   return INT_RAX_REG_mask();
1656 }
1657 
1658 // Register for MODI projection of divmodI
1659 RegMask Matcher::modI_proj_mask() {
1660   return INT_RDX_REG_mask();
1661 }
1662 
1663 // Register for DIVL projection of divmodL
1664 RegMask Matcher::divL_proj_mask() {
1665   return LONG_RAX_REG_mask();
1666 }
1667 
1668 // Register for MODL projection of divmodL
1669 RegMask Matcher::modL_proj_mask() {
1670   return LONG_RDX_REG_mask();
1671 }
1672 
1673 // Register for saving SP into on method handle invokes. Not used on x86_64.
1674 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1675     return NO_REG_mask();
1676 }
1677 
1678 %}
1679 
1680 //----------ENCODING BLOCK-----------------------------------------------------
1681 // This block specifies the encoding classes used by the compiler to
1682 // output byte streams.  Encoding classes are parameterized macros
1683 // used by Machine Instruction Nodes in order to generate the bit
1684 // encoding of the instruction.  Operands specify their base encoding
1685 // interface with the interface keyword.  There are currently
1686 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1687 // COND_INTER.  REG_INTER causes an operand to generate a function
1688 // which returns its register number when queried.  CONST_INTER causes
1689 // an operand to generate a function which returns the value of the
1690 // constant when queried.  MEMORY_INTER causes an operand to generate
1691 // four functions which return the Base Register, the Index Register,
1692 // the Scale Value, and the Offset Value of the operand when queried.
1693 // COND_INTER causes an operand to generate six functions which return
1694 // the encoding code (ie - encoding bits for the instruction)
1695 // associated with each basic boolean condition for a conditional
1696 // instruction.
1697 //
1698 // Instructions specify two basic values for encoding.  Again, a
1699 // function is available to check if the constant displacement is an
1700 // oop. They use the ins_encode keyword to specify their encoding
1701 // classes (which must be a sequence of enc_class names, and their
1702 // parameters, specified in the encoding block), and they use the
1703 // opcode keyword to specify, in order, their primary, secondary, and
1704 // tertiary opcode.  Only the opcode sections which a particular
1705 // instruction needs for encoding need to be specified.
1706 encode %{
1707   // Build emit functions for each basic byte or larger field in the
1708   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1709   // from C++ code in the enc_class source block.  Emit functions will
1710   // live in the main source block for now.  In future, we can
1711   // generalize this by adding a syntax that specifies the sizes of
1712   // fields in an order, so that the adlc can build the emit functions
1713   // automagically
1714 
1715   // Emit primary opcode
1716   enc_class OpcP
1717   %{
1718     emit_opcode(cbuf, $primary);
1719   %}
1720 
1721   // Emit secondary opcode
1722   enc_class OpcS
1723   %{
1724     emit_opcode(cbuf, $secondary);
1725   %}
1726 
1727   // Emit tertiary opcode
1728   enc_class OpcT
1729   %{
1730     emit_opcode(cbuf, $tertiary);
1731   %}
1732 
1733   // Emit opcode directly
1734   enc_class Opcode(immI d8)
1735   %{
1736     emit_opcode(cbuf, $d8$$constant);
1737   %}
1738 
1739   // Emit size prefix
1740   enc_class SizePrefix
1741   %{
1742     emit_opcode(cbuf, 0x66);
1743   %}
1744 
1745   enc_class reg(rRegI reg)
1746   %{
1747     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1748   %}
1749 
1750   enc_class reg_reg(rRegI dst, rRegI src)
1751   %{
1752     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1753   %}
1754 
1755   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1756   %{
1757     emit_opcode(cbuf, $opcode$$constant);
1758     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1759   %}
1760 
1761   enc_class cdql_enc(no_rax_rdx_RegI div)
1762   %{
1763     // Full implementation of Java idiv and irem; checks for
1764     // special case as described in JVM spec., p.243 & p.271.
1765     //
1766     //         normal case                           special case
1767     //
1768     // input : rax: dividend                         min_int
1769     //         reg: divisor                          -1
1770     //
1771     // output: rax: quotient  (= rax idiv reg)       min_int
1772     //         rdx: remainder (= rax irem reg)       0
1773     //
1774     //  Code sequnce:
1775     //
1776     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1777     //    5:   75 07/08                jne    e <normal>
1778     //    7:   33 d2                   xor    %edx,%edx
1779     //  [div >= 8 -> offset + 1]
1780     //  [REX_B]
1781     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1782     //    c:   74 03/04                je     11 <done>
1783     // 000000000000000e <normal>:
1784     //    e:   99                      cltd
1785     //  [div >= 8 -> offset + 1]
1786     //  [REX_B]
1787     //    f:   f7 f9                   idiv   $div
1788     // 0000000000000011 <done>:
1789 
1790     // cmp    $0x80000000,%eax
1791     emit_opcode(cbuf, 0x3d);
1792     emit_d8(cbuf, 0x00);
1793     emit_d8(cbuf, 0x00);
1794     emit_d8(cbuf, 0x00);
1795     emit_d8(cbuf, 0x80);
1796 
1797     // jne    e <normal>
1798     emit_opcode(cbuf, 0x75);
1799     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1800 
1801     // xor    %edx,%edx
1802     emit_opcode(cbuf, 0x33);
1803     emit_d8(cbuf, 0xD2);
1804 
1805     // cmp    $0xffffffffffffffff,%ecx
1806     if ($div$$reg >= 8) {
1807       emit_opcode(cbuf, Assembler::REX_B);
1808     }
1809     emit_opcode(cbuf, 0x83);
1810     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1811     emit_d8(cbuf, 0xFF);
1812 
1813     // je     11 <done>
1814     emit_opcode(cbuf, 0x74);
1815     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1816 
1817     // <normal>
1818     // cltd
1819     emit_opcode(cbuf, 0x99);
1820 
1821     // idivl (note: must be emitted by the user of this rule)
1822     // <done>
1823   %}
1824 
1825   enc_class cdqq_enc(no_rax_rdx_RegL div)
1826   %{
1827     // Full implementation of Java ldiv and lrem; checks for
1828     // special case as described in JVM spec., p.243 & p.271.
1829     //
1830     //         normal case                           special case
1831     //
1832     // input : rax: dividend                         min_long
1833     //         reg: divisor                          -1
1834     //
1835     // output: rax: quotient  (= rax idiv reg)       min_long
1836     //         rdx: remainder (= rax irem reg)       0
1837     //
1838     //  Code sequnce:
1839     //
1840     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1841     //    7:   00 00 80
1842     //    a:   48 39 d0                cmp    %rdx,%rax
1843     //    d:   75 08                   jne    17 <normal>
1844     //    f:   33 d2                   xor    %edx,%edx
1845     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1846     //   15:   74 05                   je     1c <done>
1847     // 0000000000000017 <normal>:
1848     //   17:   48 99                   cqto
1849     //   19:   48 f7 f9                idiv   $div
1850     // 000000000000001c <done>:
1851 
1852     // mov    $0x8000000000000000,%rdx
1853     emit_opcode(cbuf, Assembler::REX_W);
1854     emit_opcode(cbuf, 0xBA);
1855     emit_d8(cbuf, 0x00);
1856     emit_d8(cbuf, 0x00);
1857     emit_d8(cbuf, 0x00);
1858     emit_d8(cbuf, 0x00);
1859     emit_d8(cbuf, 0x00);
1860     emit_d8(cbuf, 0x00);
1861     emit_d8(cbuf, 0x00);
1862     emit_d8(cbuf, 0x80);
1863 
1864     // cmp    %rdx,%rax
1865     emit_opcode(cbuf, Assembler::REX_W);
1866     emit_opcode(cbuf, 0x39);
1867     emit_d8(cbuf, 0xD0);
1868 
1869     // jne    17 <normal>
1870     emit_opcode(cbuf, 0x75);
1871     emit_d8(cbuf, 0x08);
1872 
1873     // xor    %edx,%edx
1874     emit_opcode(cbuf, 0x33);
1875     emit_d8(cbuf, 0xD2);
1876 
1877     // cmp    $0xffffffffffffffff,$div
1878     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1879     emit_opcode(cbuf, 0x83);
1880     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1881     emit_d8(cbuf, 0xFF);
1882 
1883     // je     1e <done>
1884     emit_opcode(cbuf, 0x74);
1885     emit_d8(cbuf, 0x05);
1886 
1887     // <normal>
1888     // cqto
1889     emit_opcode(cbuf, Assembler::REX_W);
1890     emit_opcode(cbuf, 0x99);
1891 
1892     // idivq (note: must be emitted by the user of this rule)
1893     // <done>
1894   %}
1895 
1896   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1897   enc_class OpcSE(immI imm)
1898   %{
1899     // Emit primary opcode and set sign-extend bit
1900     // Check for 8-bit immediate, and set sign extend bit in opcode
1901     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1902       emit_opcode(cbuf, $primary | 0x02);
1903     } else {
1904       // 32-bit immediate
1905       emit_opcode(cbuf, $primary);
1906     }
1907   %}
1908 
1909   enc_class OpcSErm(rRegI dst, immI imm)
1910   %{
1911     // OpcSEr/m
1912     int dstenc = $dst$$reg;
1913     if (dstenc >= 8) {
1914       emit_opcode(cbuf, Assembler::REX_B);
1915       dstenc -= 8;
1916     }
1917     // Emit primary opcode and set sign-extend bit
1918     // Check for 8-bit immediate, and set sign extend bit in opcode
1919     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1920       emit_opcode(cbuf, $primary | 0x02);
1921     } else {
1922       // 32-bit immediate
1923       emit_opcode(cbuf, $primary);
1924     }
1925     // Emit r/m byte with secondary opcode, after primary opcode.
1926     emit_rm(cbuf, 0x3, $secondary, dstenc);
1927   %}
1928 
1929   enc_class OpcSErm_wide(rRegL dst, immI imm)
1930   %{
1931     // OpcSEr/m
1932     int dstenc = $dst$$reg;
1933     if (dstenc < 8) {
1934       emit_opcode(cbuf, Assembler::REX_W);
1935     } else {
1936       emit_opcode(cbuf, Assembler::REX_WB);
1937       dstenc -= 8;
1938     }
1939     // Emit primary opcode and set sign-extend bit
1940     // Check for 8-bit immediate, and set sign extend bit in opcode
1941     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1942       emit_opcode(cbuf, $primary | 0x02);
1943     } else {
1944       // 32-bit immediate
1945       emit_opcode(cbuf, $primary);
1946     }
1947     // Emit r/m byte with secondary opcode, after primary opcode.
1948     emit_rm(cbuf, 0x3, $secondary, dstenc);
1949   %}
1950 
1951   enc_class Con8or32(immI imm)
1952   %{
1953     // Check for 8-bit immediate, and set sign extend bit in opcode
1954     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1955       $$$emit8$imm$$constant;
1956     } else {
1957       // 32-bit immediate
1958       $$$emit32$imm$$constant;
1959     }
1960   %}
1961 
1962   enc_class opc2_reg(rRegI dst)
1963   %{
1964     // BSWAP
1965     emit_cc(cbuf, $secondary, $dst$$reg);
1966   %}
1967 
1968   enc_class opc3_reg(rRegI dst)
1969   %{
1970     // BSWAP
1971     emit_cc(cbuf, $tertiary, $dst$$reg);
1972   %}
1973 
1974   enc_class reg_opc(rRegI div)
1975   %{
1976     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1977     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1978   %}
1979 
1980   enc_class enc_cmov(cmpOp cop)
1981   %{
1982     // CMOV
1983     $$$emit8$primary;
1984     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1985   %}
1986 
1987   enc_class enc_PartialSubtypeCheck()
1988   %{
1989     Register Rrdi = as_Register(RDI_enc); // result register
1990     Register Rrax = as_Register(RAX_enc); // super class
1991     Register Rrcx = as_Register(RCX_enc); // killed
1992     Register Rrsi = as_Register(RSI_enc); // sub class
1993     Label miss;
1994     const bool set_cond_codes = true;
1995 
1996     MacroAssembler _masm(&cbuf);
1997     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1998                                      NULL, &miss,
1999                                      /*set_cond_codes:*/ true);
2000     if ($primary) {
2001       __ xorptr(Rrdi, Rrdi);
2002     }
2003     __ bind(miss);
2004   %}
2005 
2006   enc_class clear_avx %{
2007     debug_only(int off0 = cbuf.insts_size());
2008     if (generate_vzeroupper(Compile::current())) {
2009       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2010       // Clear upper bits of YMM registers when current compiled code uses
2011       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2012       MacroAssembler _masm(&cbuf);
2013       __ vzeroupper();
2014     }
2015     debug_only(int off1 = cbuf.insts_size());
2016     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2017   %}
2018 
2019   enc_class Java_To_Runtime(method meth) %{
2020     // No relocation needed
2021     MacroAssembler _masm(&cbuf);
2022     __ mov64(r10, (int64_t) $meth$$method);
2023     __ call(r10);
2024   %}
2025 
2026   enc_class Java_To_Interpreter(method meth)
2027   %{
2028     // CALL Java_To_Interpreter
2029     // This is the instruction starting address for relocation info.
2030     cbuf.set_insts_mark();
2031     $$$emit8$primary;
2032     // CALL directly to the runtime
2033     emit_d32_reloc(cbuf,
2034                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2035                    runtime_call_Relocation::spec(),
2036                    RELOC_DISP32);
2037   %}
2038 
2039   enc_class Java_Static_Call(method meth)
2040   %{
2041     // JAVA STATIC CALL
2042     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2043     // determine who we intended to call.
2044     cbuf.set_insts_mark();
2045     $$$emit8$primary;
2046 
2047     if (!_method) {
2048       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2049                      runtime_call_Relocation::spec(),
2050                      RELOC_DISP32);
2051     } else {
2052       int method_index = resolved_method_index(cbuf);
2053       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2054                                                   : static_call_Relocation::spec(method_index);
2055       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2056                      rspec, RELOC_DISP32);
2057       // Emit stubs for static call.
2058       address mark = cbuf.insts_mark();
2059       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2060       if (stub == NULL) {
2061         ciEnv::current()->record_failure("CodeCache is full");
2062         return;
2063       }
2064 #if INCLUDE_AOT
2065       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2066 #endif
2067     }
2068   %}
2069 
2070   enc_class Java_Dynamic_Call(method meth) %{
2071     MacroAssembler _masm(&cbuf);
2072     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2073   %}
2074 
2075   enc_class Java_Compiled_Call(method meth)
2076   %{
2077     // JAVA COMPILED CALL
2078     int disp = in_bytes(Method:: from_compiled_offset());
2079 
2080     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2081     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2082 
2083     // callq *disp(%rax)
2084     cbuf.set_insts_mark();
2085     $$$emit8$primary;
2086     if (disp < 0x80) {
2087       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2088       emit_d8(cbuf, disp); // Displacement
2089     } else {
2090       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2091       emit_d32(cbuf, disp); // Displacement
2092     }
2093   %}
2094 
2095   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2096   %{
2097     // SAL, SAR, SHR
2098     int dstenc = $dst$$reg;
2099     if (dstenc >= 8) {
2100       emit_opcode(cbuf, Assembler::REX_B);
2101       dstenc -= 8;
2102     }
2103     $$$emit8$primary;
2104     emit_rm(cbuf, 0x3, $secondary, dstenc);
2105     $$$emit8$shift$$constant;
2106   %}
2107 
2108   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2109   %{
2110     // SAL, SAR, SHR
2111     int dstenc = $dst$$reg;
2112     if (dstenc < 8) {
2113       emit_opcode(cbuf, Assembler::REX_W);
2114     } else {
2115       emit_opcode(cbuf, Assembler::REX_WB);
2116       dstenc -= 8;
2117     }
2118     $$$emit8$primary;
2119     emit_rm(cbuf, 0x3, $secondary, dstenc);
2120     $$$emit8$shift$$constant;
2121   %}
2122 
2123   enc_class load_immI(rRegI dst, immI src)
2124   %{
2125     int dstenc = $dst$$reg;
2126     if (dstenc >= 8) {
2127       emit_opcode(cbuf, Assembler::REX_B);
2128       dstenc -= 8;
2129     }
2130     emit_opcode(cbuf, 0xB8 | dstenc);
2131     $$$emit32$src$$constant;
2132   %}
2133 
2134   enc_class load_immL(rRegL dst, immL src)
2135   %{
2136     int dstenc = $dst$$reg;
2137     if (dstenc < 8) {
2138       emit_opcode(cbuf, Assembler::REX_W);
2139     } else {
2140       emit_opcode(cbuf, Assembler::REX_WB);
2141       dstenc -= 8;
2142     }
2143     emit_opcode(cbuf, 0xB8 | dstenc);
2144     emit_d64(cbuf, $src$$constant);
2145   %}
2146 
2147   enc_class load_immUL32(rRegL dst, immUL32 src)
2148   %{
2149     // same as load_immI, but this time we care about zeroes in the high word
2150     int dstenc = $dst$$reg;
2151     if (dstenc >= 8) {
2152       emit_opcode(cbuf, Assembler::REX_B);
2153       dstenc -= 8;
2154     }
2155     emit_opcode(cbuf, 0xB8 | dstenc);
2156     $$$emit32$src$$constant;
2157   %}
2158 
2159   enc_class load_immL32(rRegL dst, immL32 src)
2160   %{
2161     int dstenc = $dst$$reg;
2162     if (dstenc < 8) {
2163       emit_opcode(cbuf, Assembler::REX_W);
2164     } else {
2165       emit_opcode(cbuf, Assembler::REX_WB);
2166       dstenc -= 8;
2167     }
2168     emit_opcode(cbuf, 0xC7);
2169     emit_rm(cbuf, 0x03, 0x00, dstenc);
2170     $$$emit32$src$$constant;
2171   %}
2172 
2173   enc_class load_immP31(rRegP dst, immP32 src)
2174   %{
2175     // same as load_immI, but this time we care about zeroes in the high word
2176     int dstenc = $dst$$reg;
2177     if (dstenc >= 8) {
2178       emit_opcode(cbuf, Assembler::REX_B);
2179       dstenc -= 8;
2180     }
2181     emit_opcode(cbuf, 0xB8 | dstenc);
2182     $$$emit32$src$$constant;
2183   %}
2184 
2185   enc_class load_immP(rRegP dst, immP src)
2186   %{
2187     int dstenc = $dst$$reg;
2188     if (dstenc < 8) {
2189       emit_opcode(cbuf, Assembler::REX_W);
2190     } else {
2191       emit_opcode(cbuf, Assembler::REX_WB);
2192       dstenc -= 8;
2193     }
2194     emit_opcode(cbuf, 0xB8 | dstenc);
2195     // This next line should be generated from ADLC
2196     if ($src->constant_reloc() != relocInfo::none) {
2197       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2198     } else {
2199       emit_d64(cbuf, $src$$constant);
2200     }
2201   %}
2202 
2203   enc_class Con32(immI src)
2204   %{
2205     // Output immediate
2206     $$$emit32$src$$constant;
2207   %}
2208 
2209   enc_class Con32F_as_bits(immF src)
2210   %{
2211     // Output Float immediate bits
2212     jfloat jf = $src$$constant;
2213     jint jf_as_bits = jint_cast(jf);
2214     emit_d32(cbuf, jf_as_bits);
2215   %}
2216 
2217   enc_class Con16(immI src)
2218   %{
2219     // Output immediate
2220     $$$emit16$src$$constant;
2221   %}
2222 
2223   // How is this different from Con32??? XXX
2224   enc_class Con_d32(immI src)
2225   %{
2226     emit_d32(cbuf,$src$$constant);
2227   %}
2228 
2229   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2230     // Output immediate memory reference
2231     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2232     emit_d32(cbuf, 0x00);
2233   %}
2234 
2235   enc_class lock_prefix()
2236   %{
2237     emit_opcode(cbuf, 0xF0); // lock
2238   %}
2239 
2240   enc_class REX_mem(memory mem)
2241   %{
2242     if ($mem$$base >= 8) {
2243       if ($mem$$index < 8) {
2244         emit_opcode(cbuf, Assembler::REX_B);
2245       } else {
2246         emit_opcode(cbuf, Assembler::REX_XB);
2247       }
2248     } else {
2249       if ($mem$$index >= 8) {
2250         emit_opcode(cbuf, Assembler::REX_X);
2251       }
2252     }
2253   %}
2254 
2255   enc_class REX_mem_wide(memory mem)
2256   %{
2257     if ($mem$$base >= 8) {
2258       if ($mem$$index < 8) {
2259         emit_opcode(cbuf, Assembler::REX_WB);
2260       } else {
2261         emit_opcode(cbuf, Assembler::REX_WXB);
2262       }
2263     } else {
2264       if ($mem$$index < 8) {
2265         emit_opcode(cbuf, Assembler::REX_W);
2266       } else {
2267         emit_opcode(cbuf, Assembler::REX_WX);
2268       }
2269     }
2270   %}
2271 
2272   // for byte regs
2273   enc_class REX_breg(rRegI reg)
2274   %{
2275     if ($reg$$reg >= 4) {
2276       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2277     }
2278   %}
2279 
2280   // for byte regs
2281   enc_class REX_reg_breg(rRegI dst, rRegI src)
2282   %{
2283     if ($dst$$reg < 8) {
2284       if ($src$$reg >= 4) {
2285         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2286       }
2287     } else {
2288       if ($src$$reg < 8) {
2289         emit_opcode(cbuf, Assembler::REX_R);
2290       } else {
2291         emit_opcode(cbuf, Assembler::REX_RB);
2292       }
2293     }
2294   %}
2295 
2296   // for byte regs
2297   enc_class REX_breg_mem(rRegI reg, memory mem)
2298   %{
2299     if ($reg$$reg < 8) {
2300       if ($mem$$base < 8) {
2301         if ($mem$$index >= 8) {
2302           emit_opcode(cbuf, Assembler::REX_X);
2303         } else if ($reg$$reg >= 4) {
2304           emit_opcode(cbuf, Assembler::REX);
2305         }
2306       } else {
2307         if ($mem$$index < 8) {
2308           emit_opcode(cbuf, Assembler::REX_B);
2309         } else {
2310           emit_opcode(cbuf, Assembler::REX_XB);
2311         }
2312       }
2313     } else {
2314       if ($mem$$base < 8) {
2315         if ($mem$$index < 8) {
2316           emit_opcode(cbuf, Assembler::REX_R);
2317         } else {
2318           emit_opcode(cbuf, Assembler::REX_RX);
2319         }
2320       } else {
2321         if ($mem$$index < 8) {
2322           emit_opcode(cbuf, Assembler::REX_RB);
2323         } else {
2324           emit_opcode(cbuf, Assembler::REX_RXB);
2325         }
2326       }
2327     }
2328   %}
2329 
2330   enc_class REX_reg(rRegI reg)
2331   %{
2332     if ($reg$$reg >= 8) {
2333       emit_opcode(cbuf, Assembler::REX_B);
2334     }
2335   %}
2336 
2337   enc_class REX_reg_wide(rRegI reg)
2338   %{
2339     if ($reg$$reg < 8) {
2340       emit_opcode(cbuf, Assembler::REX_W);
2341     } else {
2342       emit_opcode(cbuf, Assembler::REX_WB);
2343     }
2344   %}
2345 
2346   enc_class REX_reg_reg(rRegI dst, rRegI src)
2347   %{
2348     if ($dst$$reg < 8) {
2349       if ($src$$reg >= 8) {
2350         emit_opcode(cbuf, Assembler::REX_B);
2351       }
2352     } else {
2353       if ($src$$reg < 8) {
2354         emit_opcode(cbuf, Assembler::REX_R);
2355       } else {
2356         emit_opcode(cbuf, Assembler::REX_RB);
2357       }
2358     }
2359   %}
2360 
2361   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2362   %{
2363     if ($dst$$reg < 8) {
2364       if ($src$$reg < 8) {
2365         emit_opcode(cbuf, Assembler::REX_W);
2366       } else {
2367         emit_opcode(cbuf, Assembler::REX_WB);
2368       }
2369     } else {
2370       if ($src$$reg < 8) {
2371         emit_opcode(cbuf, Assembler::REX_WR);
2372       } else {
2373         emit_opcode(cbuf, Assembler::REX_WRB);
2374       }
2375     }
2376   %}
2377 
2378   enc_class REX_reg_mem(rRegI reg, memory mem)
2379   %{
2380     if ($reg$$reg < 8) {
2381       if ($mem$$base < 8) {
2382         if ($mem$$index >= 8) {
2383           emit_opcode(cbuf, Assembler::REX_X);
2384         }
2385       } else {
2386         if ($mem$$index < 8) {
2387           emit_opcode(cbuf, Assembler::REX_B);
2388         } else {
2389           emit_opcode(cbuf, Assembler::REX_XB);
2390         }
2391       }
2392     } else {
2393       if ($mem$$base < 8) {
2394         if ($mem$$index < 8) {
2395           emit_opcode(cbuf, Assembler::REX_R);
2396         } else {
2397           emit_opcode(cbuf, Assembler::REX_RX);
2398         }
2399       } else {
2400         if ($mem$$index < 8) {
2401           emit_opcode(cbuf, Assembler::REX_RB);
2402         } else {
2403           emit_opcode(cbuf, Assembler::REX_RXB);
2404         }
2405       }
2406     }
2407   %}
2408 
2409   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2410   %{
2411     if ($reg$$reg < 8) {
2412       if ($mem$$base < 8) {
2413         if ($mem$$index < 8) {
2414           emit_opcode(cbuf, Assembler::REX_W);
2415         } else {
2416           emit_opcode(cbuf, Assembler::REX_WX);
2417         }
2418       } else {
2419         if ($mem$$index < 8) {
2420           emit_opcode(cbuf, Assembler::REX_WB);
2421         } else {
2422           emit_opcode(cbuf, Assembler::REX_WXB);
2423         }
2424       }
2425     } else {
2426       if ($mem$$base < 8) {
2427         if ($mem$$index < 8) {
2428           emit_opcode(cbuf, Assembler::REX_WR);
2429         } else {
2430           emit_opcode(cbuf, Assembler::REX_WRX);
2431         }
2432       } else {
2433         if ($mem$$index < 8) {
2434           emit_opcode(cbuf, Assembler::REX_WRB);
2435         } else {
2436           emit_opcode(cbuf, Assembler::REX_WRXB);
2437         }
2438       }
2439     }
2440   %}
2441 
2442   enc_class reg_mem(rRegI ereg, memory mem)
2443   %{
2444     // High registers handle in encode_RegMem
2445     int reg = $ereg$$reg;
2446     int base = $mem$$base;
2447     int index = $mem$$index;
2448     int scale = $mem$$scale;
2449     int disp = $mem$$disp;
2450     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2451 
2452     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2453   %}
2454 
2455   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2456   %{
2457     int rm_byte_opcode = $rm_opcode$$constant;
2458 
2459     // High registers handle in encode_RegMem
2460     int base = $mem$$base;
2461     int index = $mem$$index;
2462     int scale = $mem$$scale;
2463     int displace = $mem$$disp;
2464 
2465     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2466                                             // working with static
2467                                             // globals
2468     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2469                   disp_reloc);
2470   %}
2471 
2472   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2473   %{
2474     int reg_encoding = $dst$$reg;
2475     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2476     int index        = 0x04;            // 0x04 indicates no index
2477     int scale        = 0x00;            // 0x00 indicates no scale
2478     int displace     = $src1$$constant; // 0x00 indicates no displacement
2479     relocInfo::relocType disp_reloc = relocInfo::none;
2480     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2481                   disp_reloc);
2482   %}
2483 
2484   enc_class neg_reg(rRegI dst)
2485   %{
2486     int dstenc = $dst$$reg;
2487     if (dstenc >= 8) {
2488       emit_opcode(cbuf, Assembler::REX_B);
2489       dstenc -= 8;
2490     }
2491     // NEG $dst
2492     emit_opcode(cbuf, 0xF7);
2493     emit_rm(cbuf, 0x3, 0x03, dstenc);
2494   %}
2495 
2496   enc_class neg_reg_wide(rRegI dst)
2497   %{
2498     int dstenc = $dst$$reg;
2499     if (dstenc < 8) {
2500       emit_opcode(cbuf, Assembler::REX_W);
2501     } else {
2502       emit_opcode(cbuf, Assembler::REX_WB);
2503       dstenc -= 8;
2504     }
2505     // NEG $dst
2506     emit_opcode(cbuf, 0xF7);
2507     emit_rm(cbuf, 0x3, 0x03, dstenc);
2508   %}
2509 
2510   enc_class setLT_reg(rRegI dst)
2511   %{
2512     int dstenc = $dst$$reg;
2513     if (dstenc >= 8) {
2514       emit_opcode(cbuf, Assembler::REX_B);
2515       dstenc -= 8;
2516     } else if (dstenc >= 4) {
2517       emit_opcode(cbuf, Assembler::REX);
2518     }
2519     // SETLT $dst
2520     emit_opcode(cbuf, 0x0F);
2521     emit_opcode(cbuf, 0x9C);
2522     emit_rm(cbuf, 0x3, 0x0, dstenc);
2523   %}
2524 
2525   enc_class setNZ_reg(rRegI dst)
2526   %{
2527     int dstenc = $dst$$reg;
2528     if (dstenc >= 8) {
2529       emit_opcode(cbuf, Assembler::REX_B);
2530       dstenc -= 8;
2531     } else if (dstenc >= 4) {
2532       emit_opcode(cbuf, Assembler::REX);
2533     }
2534     // SETNZ $dst
2535     emit_opcode(cbuf, 0x0F);
2536     emit_opcode(cbuf, 0x95);
2537     emit_rm(cbuf, 0x3, 0x0, dstenc);
2538   %}
2539 
2540 
2541   // Compare the lonogs and set -1, 0, or 1 into dst
2542   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2543   %{
2544     int src1enc = $src1$$reg;
2545     int src2enc = $src2$$reg;
2546     int dstenc = $dst$$reg;
2547 
2548     // cmpq $src1, $src2
2549     if (src1enc < 8) {
2550       if (src2enc < 8) {
2551         emit_opcode(cbuf, Assembler::REX_W);
2552       } else {
2553         emit_opcode(cbuf, Assembler::REX_WB);
2554       }
2555     } else {
2556       if (src2enc < 8) {
2557         emit_opcode(cbuf, Assembler::REX_WR);
2558       } else {
2559         emit_opcode(cbuf, Assembler::REX_WRB);
2560       }
2561     }
2562     emit_opcode(cbuf, 0x3B);
2563     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2564 
2565     // movl $dst, -1
2566     if (dstenc >= 8) {
2567       emit_opcode(cbuf, Assembler::REX_B);
2568     }
2569     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2570     emit_d32(cbuf, -1);
2571 
2572     // jl,s done
2573     emit_opcode(cbuf, 0x7C);
2574     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2575 
2576     // setne $dst
2577     if (dstenc >= 4) {
2578       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2579     }
2580     emit_opcode(cbuf, 0x0F);
2581     emit_opcode(cbuf, 0x95);
2582     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2583 
2584     // movzbl $dst, $dst
2585     if (dstenc >= 4) {
2586       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2587     }
2588     emit_opcode(cbuf, 0x0F);
2589     emit_opcode(cbuf, 0xB6);
2590     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2591   %}
2592 
2593   enc_class Push_ResultXD(regD dst) %{
2594     MacroAssembler _masm(&cbuf);
2595     __ fstp_d(Address(rsp, 0));
2596     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2597     __ addptr(rsp, 8);
2598   %}
2599 
2600   enc_class Push_SrcXD(regD src) %{
2601     MacroAssembler _masm(&cbuf);
2602     __ subptr(rsp, 8);
2603     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2604     __ fld_d(Address(rsp, 0));
2605   %}
2606 
2607 
2608   enc_class enc_rethrow()
2609   %{
2610     cbuf.set_insts_mark();
2611     emit_opcode(cbuf, 0xE9); // jmp entry
2612     emit_d32_reloc(cbuf,
2613                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2614                    runtime_call_Relocation::spec(),
2615                    RELOC_DISP32);
2616   %}
2617 
2618 %}
2619 
2620 
2621 
2622 //----------FRAME--------------------------------------------------------------
2623 // Definition of frame structure and management information.
2624 //
2625 //  S T A C K   L A Y O U T    Allocators stack-slot number
2626 //                             |   (to get allocators register number
2627 //  G  Owned by    |        |  v    add OptoReg::stack0())
2628 //  r   CALLER     |        |
2629 //  o     |        +--------+      pad to even-align allocators stack-slot
2630 //  w     V        |  pad0  |        numbers; owned by CALLER
2631 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2632 //  h     ^        |   in   |  5
2633 //        |        |  args  |  4   Holes in incoming args owned by SELF
2634 //  |     |        |        |  3
2635 //  |     |        +--------+
2636 //  V     |        | old out|      Empty on Intel, window on Sparc
2637 //        |    old |preserve|      Must be even aligned.
2638 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2639 //        |        |   in   |  3   area for Intel ret address
2640 //     Owned by    |preserve|      Empty on Sparc.
2641 //       SELF      +--------+
2642 //        |        |  pad2  |  2   pad to align old SP
2643 //        |        +--------+  1
2644 //        |        | locks  |  0
2645 //        |        +--------+----> OptoReg::stack0(), even aligned
2646 //        |        |  pad1  | 11   pad to align new SP
2647 //        |        +--------+
2648 //        |        |        | 10
2649 //        |        | spills |  9   spills
2650 //        V        |        |  8   (pad0 slot for callee)
2651 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2652 //        ^        |  out   |  7
2653 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2654 //     Owned by    +--------+
2655 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2656 //        |    new |preserve|      Must be even-aligned.
2657 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2658 //        |        |        |
2659 //
2660 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2661 //         known from SELF's arguments and the Java calling convention.
2662 //         Region 6-7 is determined per call site.
2663 // Note 2: If the calling convention leaves holes in the incoming argument
2664 //         area, those holes are owned by SELF.  Holes in the outgoing area
2665 //         are owned by the CALLEE.  Holes should not be nessecary in the
2666 //         incoming area, as the Java calling convention is completely under
2667 //         the control of the AD file.  Doubles can be sorted and packed to
2668 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2669 //         varargs C calling conventions.
2670 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2671 //         even aligned with pad0 as needed.
2672 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2673 //         region 6-11 is even aligned; it may be padded out more so that
2674 //         the region from SP to FP meets the minimum stack alignment.
2675 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2676 //         alignment.  Region 11, pad1, may be dynamically extended so that
2677 //         SP meets the minimum alignment.
2678 
2679 frame
2680 %{
2681   // What direction does stack grow in (assumed to be same for C & Java)
2682   stack_direction(TOWARDS_LOW);
2683 
2684   // These three registers define part of the calling convention
2685   // between compiled code and the interpreter.
2686   inline_cache_reg(RAX);                // Inline Cache Register
2687   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2688                                         // calling interpreter
2689 
2690   // Optional: name the operand used by cisc-spilling to access
2691   // [stack_pointer + offset]
2692   cisc_spilling_operand_name(indOffset32);
2693 
2694   // Number of stack slots consumed by locking an object
2695   sync_stack_slots(2);
2696 
2697   // Compiled code's Frame Pointer
2698   frame_pointer(RSP);
2699 
2700   // Interpreter stores its frame pointer in a register which is
2701   // stored to the stack by I2CAdaptors.
2702   // I2CAdaptors convert from interpreted java to compiled java.
2703   interpreter_frame_pointer(RBP);
2704 
2705   // Stack alignment requirement
2706   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2707 
2708   // Number of stack slots between incoming argument block and the start of
2709   // a new frame.  The PROLOG must add this many slots to the stack.  The
2710   // EPILOG must remove this many slots.  amd64 needs two slots for
2711   // return address.
2712   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2713 
2714   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2715   // for calls to C.  Supports the var-args backing area for register parms.
2716   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2717 
2718   // The after-PROLOG location of the return address.  Location of
2719   // return address specifies a type (REG or STACK) and a number
2720   // representing the register number (i.e. - use a register name) or
2721   // stack slot.
2722   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2723   // Otherwise, it is above the locks and verification slot and alignment word
2724   return_addr(STACK - 2 +
2725               align_up((Compile::current()->in_preserve_stack_slots() +
2726                         Compile::current()->fixed_slots()),
2727                        stack_alignment_in_slots()));
2728 
2729   // Body of function which returns an integer array locating
2730   // arguments either in registers or in stack slots.  Passed an array
2731   // of ideal registers called "sig" and a "length" count.  Stack-slot
2732   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2733   // arguments for a CALLEE.  Incoming stack arguments are
2734   // automatically biased by the preserve_stack_slots field above.
2735 
2736   calling_convention
2737   %{
2738     // No difference between ingoing/outgoing just pass false
2739     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2740   %}
2741 
2742   c_calling_convention
2743   %{
2744     // This is obviously always outgoing
2745     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2746   %}
2747 
2748   // Location of compiled Java return values.  Same as C for now.
2749   return_value
2750   %{
2751     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2752            "only return normal values");
2753 
2754     static const int lo[Op_RegL + 1] = {
2755       0,
2756       0,
2757       RAX_num,  // Op_RegN
2758       RAX_num,  // Op_RegI
2759       RAX_num,  // Op_RegP
2760       XMM0_num, // Op_RegF
2761       XMM0_num, // Op_RegD
2762       RAX_num   // Op_RegL
2763     };
2764     static const int hi[Op_RegL + 1] = {
2765       0,
2766       0,
2767       OptoReg::Bad, // Op_RegN
2768       OptoReg::Bad, // Op_RegI
2769       RAX_H_num,    // Op_RegP
2770       OptoReg::Bad, // Op_RegF
2771       XMM0b_num,    // Op_RegD
2772       RAX_H_num     // Op_RegL
2773     };
2774     // Excluded flags and vector registers.
2775     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2776     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2777   %}
2778 %}
2779 
2780 //----------ATTRIBUTES---------------------------------------------------------
2781 //----------Operand Attributes-------------------------------------------------
2782 op_attrib op_cost(0);        // Required cost attribute
2783 
2784 //----------Instruction Attributes---------------------------------------------
2785 ins_attrib ins_cost(100);       // Required cost attribute
2786 ins_attrib ins_size(8);         // Required size attribute (in bits)
2787 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2788                                 // a non-matching short branch variant
2789                                 // of some long branch?
2790 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2791                                 // be a power of 2) specifies the
2792                                 // alignment that some part of the
2793                                 // instruction (not necessarily the
2794                                 // start) requires.  If > 1, a
2795                                 // compute_padding() function must be
2796                                 // provided for the instruction
2797 
2798 //----------OPERANDS-----------------------------------------------------------
2799 // Operand definitions must precede instruction definitions for correct parsing
2800 // in the ADLC because operands constitute user defined types which are used in
2801 // instruction definitions.
2802 
2803 //----------Simple Operands----------------------------------------------------
2804 // Immediate Operands
2805 // Integer Immediate
2806 operand immI()
2807 %{
2808   match(ConI);
2809 
2810   op_cost(10);
2811   format %{ %}
2812   interface(CONST_INTER);
2813 %}
2814 
2815 // Constant for test vs zero
2816 operand immI0()
2817 %{
2818   predicate(n->get_int() == 0);
2819   match(ConI);
2820 
2821   op_cost(0);
2822   format %{ %}
2823   interface(CONST_INTER);
2824 %}
2825 
2826 // Constant for increment
2827 operand immI1()
2828 %{
2829   predicate(n->get_int() == 1);
2830   match(ConI);
2831 
2832   op_cost(0);
2833   format %{ %}
2834   interface(CONST_INTER);
2835 %}
2836 
2837 // Constant for decrement
2838 operand immI_M1()
2839 %{
2840   predicate(n->get_int() == -1);
2841   match(ConI);
2842 
2843   op_cost(0);
2844   format %{ %}
2845   interface(CONST_INTER);
2846 %}
2847 
2848 // Valid scale values for addressing modes
2849 operand immI2()
2850 %{
2851   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2852   match(ConI);
2853 
2854   format %{ %}
2855   interface(CONST_INTER);
2856 %}
2857 
2858 operand immI8()
2859 %{
2860   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2861   match(ConI);
2862 
2863   op_cost(5);
2864   format %{ %}
2865   interface(CONST_INTER);
2866 %}
2867 
2868 operand immU8()
2869 %{
2870   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2871   match(ConI);
2872 
2873   op_cost(5);
2874   format %{ %}
2875   interface(CONST_INTER);
2876 %}
2877 
2878 operand immI16()
2879 %{
2880   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2881   match(ConI);
2882 
2883   op_cost(10);
2884   format %{ %}
2885   interface(CONST_INTER);
2886 %}
2887 
2888 // Int Immediate non-negative
2889 operand immU31()
2890 %{
2891   predicate(n->get_int() >= 0);
2892   match(ConI);
2893 
2894   op_cost(0);
2895   format %{ %}
2896   interface(CONST_INTER);
2897 %}
2898 
2899 // Constant for long shifts
2900 operand immI_32()
2901 %{
2902   predicate( n->get_int() == 32 );
2903   match(ConI);
2904 
2905   op_cost(0);
2906   format %{ %}
2907   interface(CONST_INTER);
2908 %}
2909 
2910 // Constant for long shifts
2911 operand immI_64()
2912 %{
2913   predicate( n->get_int() == 64 );
2914   match(ConI);
2915 
2916   op_cost(0);
2917   format %{ %}
2918   interface(CONST_INTER);
2919 %}
2920 
2921 // Pointer Immediate
2922 operand immP()
2923 %{
2924   match(ConP);
2925 
2926   op_cost(10);
2927   format %{ %}
2928   interface(CONST_INTER);
2929 %}
2930 
2931 // NULL Pointer Immediate
2932 operand immP0()
2933 %{
2934   predicate(n->get_ptr() == 0);
2935   match(ConP);
2936 
2937   op_cost(5);
2938   format %{ %}
2939   interface(CONST_INTER);
2940 %}
2941 
2942 // Pointer Immediate
2943 operand immN() %{
2944   match(ConN);
2945 
2946   op_cost(10);
2947   format %{ %}
2948   interface(CONST_INTER);
2949 %}
2950 
2951 operand immNKlass() %{
2952   match(ConNKlass);
2953 
2954   op_cost(10);
2955   format %{ %}
2956   interface(CONST_INTER);
2957 %}
2958 
2959 // NULL Pointer Immediate
2960 operand immN0() %{
2961   predicate(n->get_narrowcon() == 0);
2962   match(ConN);
2963 
2964   op_cost(5);
2965   format %{ %}
2966   interface(CONST_INTER);
2967 %}
2968 
2969 operand immP31()
2970 %{
2971   predicate(n->as_Type()->type()->reloc() == relocInfo::none
2972             && (n->get_ptr() >> 31) == 0);
2973   match(ConP);
2974 
2975   op_cost(5);
2976   format %{ %}
2977   interface(CONST_INTER);
2978 %}
2979 
2980 
2981 // Long Immediate
2982 operand immL()
2983 %{
2984   match(ConL);
2985 
2986   op_cost(20);
2987   format %{ %}
2988   interface(CONST_INTER);
2989 %}
2990 
2991 // Long Immediate 8-bit
2992 operand immL8()
2993 %{
2994   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2995   match(ConL);
2996 
2997   op_cost(5);
2998   format %{ %}
2999   interface(CONST_INTER);
3000 %}
3001 
3002 // Long Immediate 32-bit unsigned
3003 operand immUL32()
3004 %{
3005   predicate(n->get_long() == (unsigned int) (n->get_long()));
3006   match(ConL);
3007 
3008   op_cost(10);
3009   format %{ %}
3010   interface(CONST_INTER);
3011 %}
3012 
3013 // Long Immediate 32-bit signed
3014 operand immL32()
3015 %{
3016   predicate(n->get_long() == (int) (n->get_long()));
3017   match(ConL);
3018 
3019   op_cost(15);
3020   format %{ %}
3021   interface(CONST_INTER);
3022 %}
3023 
3024 // Long Immediate zero
3025 operand immL0()
3026 %{
3027   predicate(n->get_long() == 0L);
3028   match(ConL);
3029 
3030   op_cost(10);
3031   format %{ %}
3032   interface(CONST_INTER);
3033 %}
3034 
3035 // Constant for increment
3036 operand immL1()
3037 %{
3038   predicate(n->get_long() == 1);
3039   match(ConL);
3040 
3041   format %{ %}
3042   interface(CONST_INTER);
3043 %}
3044 
3045 // Constant for decrement
3046 operand immL_M1()
3047 %{
3048   predicate(n->get_long() == -1);
3049   match(ConL);
3050 
3051   format %{ %}
3052   interface(CONST_INTER);
3053 %}
3054 
3055 // Long Immediate: the value 10
3056 operand immL10()
3057 %{
3058   predicate(n->get_long() == 10);
3059   match(ConL);
3060 
3061   format %{ %}
3062   interface(CONST_INTER);
3063 %}
3064 
3065 // Long immediate from 0 to 127.
3066 // Used for a shorter form of long mul by 10.
3067 operand immL_127()
3068 %{
3069   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3070   match(ConL);
3071 
3072   op_cost(10);
3073   format %{ %}
3074   interface(CONST_INTER);
3075 %}
3076 
3077 // Long Immediate: low 32-bit mask
3078 operand immL_32bits()
3079 %{
3080   predicate(n->get_long() == 0xFFFFFFFFL);
3081   match(ConL);
3082   op_cost(20);
3083 
3084   format %{ %}
3085   interface(CONST_INTER);
3086 %}
3087 
3088 // Float Immediate zero
3089 operand immF0()
3090 %{
3091   predicate(jint_cast(n->getf()) == 0);
3092   match(ConF);
3093 
3094   op_cost(5);
3095   format %{ %}
3096   interface(CONST_INTER);
3097 %}
3098 
3099 // Float Immediate
3100 operand immF()
3101 %{
3102   match(ConF);
3103 
3104   op_cost(15);
3105   format %{ %}
3106   interface(CONST_INTER);
3107 %}
3108 
3109 // Double Immediate zero
3110 operand immD0()
3111 %{
3112   predicate(jlong_cast(n->getd()) == 0);
3113   match(ConD);
3114 
3115   op_cost(5);
3116   format %{ %}
3117   interface(CONST_INTER);
3118 %}
3119 
3120 // Double Immediate
3121 operand immD()
3122 %{
3123   match(ConD);
3124 
3125   op_cost(15);
3126   format %{ %}
3127   interface(CONST_INTER);
3128 %}
3129 
3130 // Immediates for special shifts (sign extend)
3131 
3132 // Constants for increment
3133 operand immI_16()
3134 %{
3135   predicate(n->get_int() == 16);
3136   match(ConI);
3137 
3138   format %{ %}
3139   interface(CONST_INTER);
3140 %}
3141 
3142 operand immI_24()
3143 %{
3144   predicate(n->get_int() == 24);
3145   match(ConI);
3146 
3147   format %{ %}
3148   interface(CONST_INTER);
3149 %}
3150 
3151 // Constant for byte-wide masking
3152 operand immI_255()
3153 %{
3154   predicate(n->get_int() == 255);
3155   match(ConI);
3156 
3157   format %{ %}
3158   interface(CONST_INTER);
3159 %}
3160 
3161 // Constant for short-wide masking
3162 operand immI_65535()
3163 %{
3164   predicate(n->get_int() == 65535);
3165   match(ConI);
3166 
3167   format %{ %}
3168   interface(CONST_INTER);
3169 %}
3170 
3171 // Constant for byte-wide masking
3172 operand immL_255()
3173 %{
3174   predicate(n->get_long() == 255);
3175   match(ConL);
3176 
3177   format %{ %}
3178   interface(CONST_INTER);
3179 %}
3180 
3181 // Constant for short-wide masking
3182 operand immL_65535()
3183 %{
3184   predicate(n->get_long() == 65535);
3185   match(ConL);
3186 
3187   format %{ %}
3188   interface(CONST_INTER);
3189 %}
3190 
3191 // Register Operands
3192 // Integer Register
3193 operand rRegI()
3194 %{
3195   constraint(ALLOC_IN_RC(int_reg));
3196   match(RegI);
3197 
3198   match(rax_RegI);
3199   match(rbx_RegI);
3200   match(rcx_RegI);
3201   match(rdx_RegI);
3202   match(rdi_RegI);
3203 
3204   format %{ %}
3205   interface(REG_INTER);
3206 %}
3207 
3208 // Special Registers
3209 operand rax_RegI()
3210 %{
3211   constraint(ALLOC_IN_RC(int_rax_reg));
3212   match(RegI);
3213   match(rRegI);
3214 
3215   format %{ "RAX" %}
3216   interface(REG_INTER);
3217 %}
3218 
3219 // Special Registers
3220 operand rbx_RegI()
3221 %{
3222   constraint(ALLOC_IN_RC(int_rbx_reg));
3223   match(RegI);
3224   match(rRegI);
3225 
3226   format %{ "RBX" %}
3227   interface(REG_INTER);
3228 %}
3229 
3230 operand rcx_RegI()
3231 %{
3232   constraint(ALLOC_IN_RC(int_rcx_reg));
3233   match(RegI);
3234   match(rRegI);
3235 
3236   format %{ "RCX" %}
3237   interface(REG_INTER);
3238 %}
3239 
3240 operand rdx_RegI()
3241 %{
3242   constraint(ALLOC_IN_RC(int_rdx_reg));
3243   match(RegI);
3244   match(rRegI);
3245 
3246   format %{ "RDX" %}
3247   interface(REG_INTER);
3248 %}
3249 
3250 operand rdi_RegI()
3251 %{
3252   constraint(ALLOC_IN_RC(int_rdi_reg));
3253   match(RegI);
3254   match(rRegI);
3255 
3256   format %{ "RDI" %}
3257   interface(REG_INTER);
3258 %}
3259 
3260 operand no_rcx_RegI()
3261 %{
3262   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3263   match(RegI);
3264   match(rax_RegI);
3265   match(rbx_RegI);
3266   match(rdx_RegI);
3267   match(rdi_RegI);
3268 
3269   format %{ %}
3270   interface(REG_INTER);
3271 %}
3272 
3273 operand no_rax_rdx_RegI()
3274 %{
3275   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3276   match(RegI);
3277   match(rbx_RegI);
3278   match(rcx_RegI);
3279   match(rdi_RegI);
3280 
3281   format %{ %}
3282   interface(REG_INTER);
3283 %}
3284 
3285 // Pointer Register
3286 operand any_RegP()
3287 %{
3288   constraint(ALLOC_IN_RC(any_reg));
3289   match(RegP);
3290   match(rax_RegP);
3291   match(rbx_RegP);
3292   match(rdi_RegP);
3293   match(rsi_RegP);
3294   match(rbp_RegP);
3295   match(r15_RegP);
3296   match(rRegP);
3297 
3298   format %{ %}
3299   interface(REG_INTER);
3300 %}
3301 
3302 operand rRegP()
3303 %{
3304   constraint(ALLOC_IN_RC(ptr_reg));
3305   match(RegP);
3306   match(rax_RegP);
3307   match(rbx_RegP);
3308   match(rdi_RegP);
3309   match(rsi_RegP);
3310   match(rbp_RegP);  // See Q&A below about
3311   match(r15_RegP);  // r15_RegP and rbp_RegP.
3312 
3313   format %{ %}
3314   interface(REG_INTER);
3315 %}
3316 
3317 operand rRegN() %{
3318   constraint(ALLOC_IN_RC(int_reg));
3319   match(RegN);
3320 
3321   format %{ %}
3322   interface(REG_INTER);
3323 %}
3324 
3325 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3326 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3327 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3328 // The output of an instruction is controlled by the allocator, which respects
3329 // register class masks, not match rules.  Unless an instruction mentions
3330 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3331 // by the allocator as an input.
3332 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3333 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3334 // result, RBP is not included in the output of the instruction either.
3335 
3336 operand no_rax_RegP()
3337 %{
3338   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3339   match(RegP);
3340   match(rbx_RegP);
3341   match(rsi_RegP);
3342   match(rdi_RegP);
3343 
3344   format %{ %}
3345   interface(REG_INTER);
3346 %}
3347 
3348 // This operand is not allowed to use RBP even if
3349 // RBP is not used to hold the frame pointer.
3350 operand no_rbp_RegP()
3351 %{
3352   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3353   match(RegP);
3354   match(rbx_RegP);
3355   match(rsi_RegP);
3356   match(rdi_RegP);
3357 
3358   format %{ %}
3359   interface(REG_INTER);
3360 %}
3361 
3362 operand no_rax_rbx_RegP()
3363 %{
3364   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3365   match(RegP);
3366   match(rsi_RegP);
3367   match(rdi_RegP);
3368 
3369   format %{ %}
3370   interface(REG_INTER);
3371 %}
3372 
3373 // Special Registers
3374 // Return a pointer value
3375 operand rax_RegP()
3376 %{
3377   constraint(ALLOC_IN_RC(ptr_rax_reg));
3378   match(RegP);
3379   match(rRegP);
3380 
3381   format %{ %}
3382   interface(REG_INTER);
3383 %}
3384 
3385 // Special Registers
3386 // Return a compressed pointer value
3387 operand rax_RegN()
3388 %{
3389   constraint(ALLOC_IN_RC(int_rax_reg));
3390   match(RegN);
3391   match(rRegN);
3392 
3393   format %{ %}
3394   interface(REG_INTER);
3395 %}
3396 
3397 // Used in AtomicAdd
3398 operand rbx_RegP()
3399 %{
3400   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3401   match(RegP);
3402   match(rRegP);
3403 
3404   format %{ %}
3405   interface(REG_INTER);
3406 %}
3407 
3408 operand rsi_RegP()
3409 %{
3410   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3411   match(RegP);
3412   match(rRegP);
3413 
3414   format %{ %}
3415   interface(REG_INTER);
3416 %}
3417 
3418 // Used in rep stosq
3419 operand rdi_RegP()
3420 %{
3421   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3422   match(RegP);
3423   match(rRegP);
3424 
3425   format %{ %}
3426   interface(REG_INTER);
3427 %}
3428 
3429 operand r15_RegP()
3430 %{
3431   constraint(ALLOC_IN_RC(ptr_r15_reg));
3432   match(RegP);
3433   match(rRegP);
3434 
3435   format %{ %}
3436   interface(REG_INTER);
3437 %}
3438 
3439 operand rRegL()
3440 %{
3441   constraint(ALLOC_IN_RC(long_reg));
3442   match(RegL);
3443   match(rax_RegL);
3444   match(rdx_RegL);
3445 
3446   format %{ %}
3447   interface(REG_INTER);
3448 %}
3449 
3450 // Special Registers
3451 operand no_rax_rdx_RegL()
3452 %{
3453   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3454   match(RegL);
3455   match(rRegL);
3456 
3457   format %{ %}
3458   interface(REG_INTER);
3459 %}
3460 
3461 operand no_rax_RegL()
3462 %{
3463   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3464   match(RegL);
3465   match(rRegL);
3466   match(rdx_RegL);
3467 
3468   format %{ %}
3469   interface(REG_INTER);
3470 %}
3471 
3472 operand no_rcx_RegL()
3473 %{
3474   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3475   match(RegL);
3476   match(rRegL);
3477 
3478   format %{ %}
3479   interface(REG_INTER);
3480 %}
3481 
3482 operand rax_RegL()
3483 %{
3484   constraint(ALLOC_IN_RC(long_rax_reg));
3485   match(RegL);
3486   match(rRegL);
3487 
3488   format %{ "RAX" %}
3489   interface(REG_INTER);
3490 %}
3491 
3492 operand rcx_RegL()
3493 %{
3494   constraint(ALLOC_IN_RC(long_rcx_reg));
3495   match(RegL);
3496   match(rRegL);
3497 
3498   format %{ %}
3499   interface(REG_INTER);
3500 %}
3501 
3502 operand rdx_RegL()
3503 %{
3504   constraint(ALLOC_IN_RC(long_rdx_reg));
3505   match(RegL);
3506   match(rRegL);
3507 
3508   format %{ %}
3509   interface(REG_INTER);
3510 %}
3511 
3512 // Flags register, used as output of compare instructions
3513 operand rFlagsReg()
3514 %{
3515   constraint(ALLOC_IN_RC(int_flags));
3516   match(RegFlags);
3517 
3518   format %{ "RFLAGS" %}
3519   interface(REG_INTER);
3520 %}
3521 
3522 // Flags register, used as output of FLOATING POINT compare instructions
3523 operand rFlagsRegU()
3524 %{
3525   constraint(ALLOC_IN_RC(int_flags));
3526   match(RegFlags);
3527 
3528   format %{ "RFLAGS_U" %}
3529   interface(REG_INTER);
3530 %}
3531 
3532 operand rFlagsRegUCF() %{
3533   constraint(ALLOC_IN_RC(int_flags));
3534   match(RegFlags);
3535   predicate(false);
3536 
3537   format %{ "RFLAGS_U_CF" %}
3538   interface(REG_INTER);
3539 %}
3540 
3541 // Float register operands
3542 operand regF() %{
3543    constraint(ALLOC_IN_RC(float_reg));
3544    match(RegF);
3545 
3546    format %{ %}
3547    interface(REG_INTER);
3548 %}
3549 
3550 // Float register operands
3551 operand vlRegF() %{
3552    constraint(ALLOC_IN_RC(float_reg_vl));
3553    match(RegF);
3554 
3555    format %{ %}
3556    interface(REG_INTER);
3557 %}
3558 
3559 // Double register operands
3560 operand regD() %{
3561    constraint(ALLOC_IN_RC(double_reg));
3562    match(RegD);
3563 
3564    format %{ %}
3565    interface(REG_INTER);
3566 %}
3567 
3568 // Double register operands
3569 operand vlRegD() %{
3570    constraint(ALLOC_IN_RC(double_reg_vl));
3571    match(RegD);
3572 
3573    format %{ %}
3574    interface(REG_INTER);
3575 %}
3576 
3577 // Vectors
3578 operand vecS() %{
3579   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3580   match(VecS);
3581 
3582   format %{ %}
3583   interface(REG_INTER);
3584 %}
3585 
3586 // Vectors
3587 operand legVecS() %{
3588   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3589   match(VecS);
3590 
3591   format %{ %}
3592   interface(REG_INTER);
3593 %}
3594 
3595 operand vecD() %{
3596   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3597   match(VecD);
3598 
3599   format %{ %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 operand legVecD() %{
3604   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3605   match(VecD);
3606 
3607   format %{ %}
3608   interface(REG_INTER);
3609 %}
3610 
3611 operand vecX() %{
3612   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3613   match(VecX);
3614 
3615   format %{ %}
3616   interface(REG_INTER);
3617 %}
3618 
3619 operand legVecX() %{
3620   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3621   match(VecX);
3622 
3623   format %{ %}
3624   interface(REG_INTER);
3625 %}
3626 
3627 operand vecY() %{
3628   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3629   match(VecY);
3630 
3631   format %{ %}
3632   interface(REG_INTER);
3633 %}
3634 
3635 operand legVecY() %{
3636   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3637   match(VecY);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 //----------Memory Operands----------------------------------------------------
3644 // Direct Memory Operand
3645 // operand direct(immP addr)
3646 // %{
3647 //   match(addr);
3648 
3649 //   format %{ "[$addr]" %}
3650 //   interface(MEMORY_INTER) %{
3651 //     base(0xFFFFFFFF);
3652 //     index(0x4);
3653 //     scale(0x0);
3654 //     disp($addr);
3655 //   %}
3656 // %}
3657 
3658 // Indirect Memory Operand
3659 operand indirect(any_RegP reg)
3660 %{
3661   constraint(ALLOC_IN_RC(ptr_reg));
3662   match(reg);
3663 
3664   format %{ "[$reg]" %}
3665   interface(MEMORY_INTER) %{
3666     base($reg);
3667     index(0x4);
3668     scale(0x0);
3669     disp(0x0);
3670   %}
3671 %}
3672 
3673 // Indirect Memory Plus Short Offset Operand
3674 operand indOffset8(any_RegP reg, immL8 off)
3675 %{
3676   constraint(ALLOC_IN_RC(ptr_reg));
3677   match(AddP reg off);
3678 
3679   format %{ "[$reg + $off (8-bit)]" %}
3680   interface(MEMORY_INTER) %{
3681     base($reg);
3682     index(0x4);
3683     scale(0x0);
3684     disp($off);
3685   %}
3686 %}
3687 
3688 // Indirect Memory Plus Long Offset Operand
3689 operand indOffset32(any_RegP reg, immL32 off)
3690 %{
3691   constraint(ALLOC_IN_RC(ptr_reg));
3692   match(AddP reg off);
3693 
3694   format %{ "[$reg + $off (32-bit)]" %}
3695   interface(MEMORY_INTER) %{
3696     base($reg);
3697     index(0x4);
3698     scale(0x0);
3699     disp($off);
3700   %}
3701 %}
3702 
3703 // Indirect Memory Plus Index Register Plus Offset Operand
3704 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3705 %{
3706   constraint(ALLOC_IN_RC(ptr_reg));
3707   match(AddP (AddP reg lreg) off);
3708 
3709   op_cost(10);
3710   format %{"[$reg + $off + $lreg]" %}
3711   interface(MEMORY_INTER) %{
3712     base($reg);
3713     index($lreg);
3714     scale(0x0);
3715     disp($off);
3716   %}
3717 %}
3718 
3719 // Indirect Memory Plus Index Register Plus Offset Operand
3720 operand indIndex(any_RegP reg, rRegL lreg)
3721 %{
3722   constraint(ALLOC_IN_RC(ptr_reg));
3723   match(AddP reg lreg);
3724 
3725   op_cost(10);
3726   format %{"[$reg + $lreg]" %}
3727   interface(MEMORY_INTER) %{
3728     base($reg);
3729     index($lreg);
3730     scale(0x0);
3731     disp(0x0);
3732   %}
3733 %}
3734 
3735 // Indirect Memory Times Scale Plus Index Register
3736 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3737 %{
3738   constraint(ALLOC_IN_RC(ptr_reg));
3739   match(AddP reg (LShiftL lreg scale));
3740 
3741   op_cost(10);
3742   format %{"[$reg + $lreg << $scale]" %}
3743   interface(MEMORY_INTER) %{
3744     base($reg);
3745     index($lreg);
3746     scale($scale);
3747     disp(0x0);
3748   %}
3749 %}
3750 
3751 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3752 %{
3753   constraint(ALLOC_IN_RC(ptr_reg));
3754   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3755   match(AddP reg (LShiftL (ConvI2L idx) scale));
3756 
3757   op_cost(10);
3758   format %{"[$reg + pos $idx << $scale]" %}
3759   interface(MEMORY_INTER) %{
3760     base($reg);
3761     index($idx);
3762     scale($scale);
3763     disp(0x0);
3764   %}
3765 %}
3766 
3767 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3768 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3769 %{
3770   constraint(ALLOC_IN_RC(ptr_reg));
3771   match(AddP (AddP reg (LShiftL lreg scale)) off);
3772 
3773   op_cost(10);
3774   format %{"[$reg + $off + $lreg << $scale]" %}
3775   interface(MEMORY_INTER) %{
3776     base($reg);
3777     index($lreg);
3778     scale($scale);
3779     disp($off);
3780   %}
3781 %}
3782 
3783 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3784 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3785 %{
3786   constraint(ALLOC_IN_RC(ptr_reg));
3787   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3788   match(AddP (AddP reg (ConvI2L idx)) off);
3789 
3790   op_cost(10);
3791   format %{"[$reg + $off + $idx]" %}
3792   interface(MEMORY_INTER) %{
3793     base($reg);
3794     index($idx);
3795     scale(0x0);
3796     disp($off);
3797   %}
3798 %}
3799 
3800 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3801 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3802 %{
3803   constraint(ALLOC_IN_RC(ptr_reg));
3804   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3805   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3806 
3807   op_cost(10);
3808   format %{"[$reg + $off + $idx << $scale]" %}
3809   interface(MEMORY_INTER) %{
3810     base($reg);
3811     index($idx);
3812     scale($scale);
3813     disp($off);
3814   %}
3815 %}
3816 
3817 // Indirect Narrow Oop Plus Offset Operand
3818 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3819 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3820 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3821   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3822   constraint(ALLOC_IN_RC(ptr_reg));
3823   match(AddP (DecodeN reg) off);
3824 
3825   op_cost(10);
3826   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3827   interface(MEMORY_INTER) %{
3828     base(0xc); // R12
3829     index($reg);
3830     scale(0x3);
3831     disp($off);
3832   %}
3833 %}
3834 
3835 // Indirect Memory Operand
3836 operand indirectNarrow(rRegN reg)
3837 %{
3838   predicate(Universe::narrow_oop_shift() == 0);
3839   constraint(ALLOC_IN_RC(ptr_reg));
3840   match(DecodeN reg);
3841 
3842   format %{ "[$reg]" %}
3843   interface(MEMORY_INTER) %{
3844     base($reg);
3845     index(0x4);
3846     scale(0x0);
3847     disp(0x0);
3848   %}
3849 %}
3850 
3851 // Indirect Memory Plus Short Offset Operand
3852 operand indOffset8Narrow(rRegN reg, immL8 off)
3853 %{
3854   predicate(Universe::narrow_oop_shift() == 0);
3855   constraint(ALLOC_IN_RC(ptr_reg));
3856   match(AddP (DecodeN reg) off);
3857 
3858   format %{ "[$reg + $off (8-bit)]" %}
3859   interface(MEMORY_INTER) %{
3860     base($reg);
3861     index(0x4);
3862     scale(0x0);
3863     disp($off);
3864   %}
3865 %}
3866 
3867 // Indirect Memory Plus Long Offset Operand
3868 operand indOffset32Narrow(rRegN reg, immL32 off)
3869 %{
3870   predicate(Universe::narrow_oop_shift() == 0);
3871   constraint(ALLOC_IN_RC(ptr_reg));
3872   match(AddP (DecodeN reg) off);
3873 
3874   format %{ "[$reg + $off (32-bit)]" %}
3875   interface(MEMORY_INTER) %{
3876     base($reg);
3877     index(0x4);
3878     scale(0x0);
3879     disp($off);
3880   %}
3881 %}
3882 
3883 // Indirect Memory Plus Index Register Plus Offset Operand
3884 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3885 %{
3886   predicate(Universe::narrow_oop_shift() == 0);
3887   constraint(ALLOC_IN_RC(ptr_reg));
3888   match(AddP (AddP (DecodeN reg) lreg) off);
3889 
3890   op_cost(10);
3891   format %{"[$reg + $off + $lreg]" %}
3892   interface(MEMORY_INTER) %{
3893     base($reg);
3894     index($lreg);
3895     scale(0x0);
3896     disp($off);
3897   %}
3898 %}
3899 
3900 // Indirect Memory Plus Index Register Plus Offset Operand
3901 operand indIndexNarrow(rRegN reg, rRegL lreg)
3902 %{
3903   predicate(Universe::narrow_oop_shift() == 0);
3904   constraint(ALLOC_IN_RC(ptr_reg));
3905   match(AddP (DecodeN reg) lreg);
3906 
3907   op_cost(10);
3908   format %{"[$reg + $lreg]" %}
3909   interface(MEMORY_INTER) %{
3910     base($reg);
3911     index($lreg);
3912     scale(0x0);
3913     disp(0x0);
3914   %}
3915 %}
3916 
3917 // Indirect Memory Times Scale Plus Index Register
3918 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3919 %{
3920   predicate(Universe::narrow_oop_shift() == 0);
3921   constraint(ALLOC_IN_RC(ptr_reg));
3922   match(AddP (DecodeN reg) (LShiftL lreg scale));
3923 
3924   op_cost(10);
3925   format %{"[$reg + $lreg << $scale]" %}
3926   interface(MEMORY_INTER) %{
3927     base($reg);
3928     index($lreg);
3929     scale($scale);
3930     disp(0x0);
3931   %}
3932 %}
3933 
3934 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3935 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3936 %{
3937   predicate(Universe::narrow_oop_shift() == 0);
3938   constraint(ALLOC_IN_RC(ptr_reg));
3939   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3940 
3941   op_cost(10);
3942   format %{"[$reg + $off + $lreg << $scale]" %}
3943   interface(MEMORY_INTER) %{
3944     base($reg);
3945     index($lreg);
3946     scale($scale);
3947     disp($off);
3948   %}
3949 %}
3950 
3951 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3952 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3953 %{
3954   constraint(ALLOC_IN_RC(ptr_reg));
3955   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3956   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3957 
3958   op_cost(10);
3959   format %{"[$reg + $off + $idx]" %}
3960   interface(MEMORY_INTER) %{
3961     base($reg);
3962     index($idx);
3963     scale(0x0);
3964     disp($off);
3965   %}
3966 %}
3967 
3968 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3969 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3970 %{
3971   constraint(ALLOC_IN_RC(ptr_reg));
3972   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3973   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3974 
3975   op_cost(10);
3976   format %{"[$reg + $off + $idx << $scale]" %}
3977   interface(MEMORY_INTER) %{
3978     base($reg);
3979     index($idx);
3980     scale($scale);
3981     disp($off);
3982   %}
3983 %}
3984 
3985 //----------Special Memory Operands--------------------------------------------
3986 // Stack Slot Operand - This operand is used for loading and storing temporary
3987 //                      values on the stack where a match requires a value to
3988 //                      flow through memory.
3989 operand stackSlotP(sRegP reg)
3990 %{
3991   constraint(ALLOC_IN_RC(stack_slots));
3992   // No match rule because this operand is only generated in matching
3993 
3994   format %{ "[$reg]" %}
3995   interface(MEMORY_INTER) %{
3996     base(0x4);   // RSP
3997     index(0x4);  // No Index
3998     scale(0x0);  // No Scale
3999     disp($reg);  // Stack Offset
4000   %}
4001 %}
4002 
4003 operand stackSlotI(sRegI reg)
4004 %{
4005   constraint(ALLOC_IN_RC(stack_slots));
4006   // No match rule because this operand is only generated in matching
4007 
4008   format %{ "[$reg]" %}
4009   interface(MEMORY_INTER) %{
4010     base(0x4);   // RSP
4011     index(0x4);  // No Index
4012     scale(0x0);  // No Scale
4013     disp($reg);  // Stack Offset
4014   %}
4015 %}
4016 
4017 operand stackSlotF(sRegF reg)
4018 %{
4019   constraint(ALLOC_IN_RC(stack_slots));
4020   // No match rule because this operand is only generated in matching
4021 
4022   format %{ "[$reg]" %}
4023   interface(MEMORY_INTER) %{
4024     base(0x4);   // RSP
4025     index(0x4);  // No Index
4026     scale(0x0);  // No Scale
4027     disp($reg);  // Stack Offset
4028   %}
4029 %}
4030 
4031 operand stackSlotD(sRegD reg)
4032 %{
4033   constraint(ALLOC_IN_RC(stack_slots));
4034   // No match rule because this operand is only generated in matching
4035 
4036   format %{ "[$reg]" %}
4037   interface(MEMORY_INTER) %{
4038     base(0x4);   // RSP
4039     index(0x4);  // No Index
4040     scale(0x0);  // No Scale
4041     disp($reg);  // Stack Offset
4042   %}
4043 %}
4044 operand stackSlotL(sRegL reg)
4045 %{
4046   constraint(ALLOC_IN_RC(stack_slots));
4047   // No match rule because this operand is only generated in matching
4048 
4049   format %{ "[$reg]" %}
4050   interface(MEMORY_INTER) %{
4051     base(0x4);   // RSP
4052     index(0x4);  // No Index
4053     scale(0x0);  // No Scale
4054     disp($reg);  // Stack Offset
4055   %}
4056 %}
4057 
4058 //----------Conditional Branch Operands----------------------------------------
4059 // Comparison Op  - This is the operation of the comparison, and is limited to
4060 //                  the following set of codes:
4061 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4062 //
4063 // Other attributes of the comparison, such as unsignedness, are specified
4064 // by the comparison instruction that sets a condition code flags register.
4065 // That result is represented by a flags operand whose subtype is appropriate
4066 // to the unsignedness (etc.) of the comparison.
4067 //
4068 // Later, the instruction which matches both the Comparison Op (a Bool) and
4069 // the flags (produced by the Cmp) specifies the coding of the comparison op
4070 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4071 
4072 // Comparision Code
4073 operand cmpOp()
4074 %{
4075   match(Bool);
4076 
4077   format %{ "" %}
4078   interface(COND_INTER) %{
4079     equal(0x4, "e");
4080     not_equal(0x5, "ne");
4081     less(0xC, "l");
4082     greater_equal(0xD, "ge");
4083     less_equal(0xE, "le");
4084     greater(0xF, "g");
4085     overflow(0x0, "o");
4086     no_overflow(0x1, "no");
4087   %}
4088 %}
4089 
4090 // Comparison Code, unsigned compare.  Used by FP also, with
4091 // C2 (unordered) turned into GT or LT already.  The other bits
4092 // C0 and C3 are turned into Carry & Zero flags.
4093 operand cmpOpU()
4094 %{
4095   match(Bool);
4096 
4097   format %{ "" %}
4098   interface(COND_INTER) %{
4099     equal(0x4, "e");
4100     not_equal(0x5, "ne");
4101     less(0x2, "b");
4102     greater_equal(0x3, "nb");
4103     less_equal(0x6, "be");
4104     greater(0x7, "nbe");
4105     overflow(0x0, "o");
4106     no_overflow(0x1, "no");
4107   %}
4108 %}
4109 
4110 
4111 // Floating comparisons that don't require any fixup for the unordered case
4112 operand cmpOpUCF() %{
4113   match(Bool);
4114   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4115             n->as_Bool()->_test._test == BoolTest::ge ||
4116             n->as_Bool()->_test._test == BoolTest::le ||
4117             n->as_Bool()->_test._test == BoolTest::gt);
4118   format %{ "" %}
4119   interface(COND_INTER) %{
4120     equal(0x4, "e");
4121     not_equal(0x5, "ne");
4122     less(0x2, "b");
4123     greater_equal(0x3, "nb");
4124     less_equal(0x6, "be");
4125     greater(0x7, "nbe");
4126     overflow(0x0, "o");
4127     no_overflow(0x1, "no");
4128   %}
4129 %}
4130 
4131 
4132 // Floating comparisons that can be fixed up with extra conditional jumps
4133 operand cmpOpUCF2() %{
4134   match(Bool);
4135   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4136             n->as_Bool()->_test._test == BoolTest::eq);
4137   format %{ "" %}
4138   interface(COND_INTER) %{
4139     equal(0x4, "e");
4140     not_equal(0x5, "ne");
4141     less(0x2, "b");
4142     greater_equal(0x3, "nb");
4143     less_equal(0x6, "be");
4144     greater(0x7, "nbe");
4145     overflow(0x0, "o");
4146     no_overflow(0x1, "no");
4147   %}
4148 %}
4149 
4150 // Operands for bound floating pointer register arguments
4151 operand rxmm0() %{
4152   constraint(ALLOC_IN_RC(xmm0_reg));
4153   match(VecX);
4154   format%{%}
4155   interface(REG_INTER);
4156 %}
4157 operand rxmm1() %{
4158   constraint(ALLOC_IN_RC(xmm1_reg));
4159   match(VecX);
4160   format%{%}
4161   interface(REG_INTER);
4162 %}
4163 operand rxmm2() %{
4164   constraint(ALLOC_IN_RC(xmm2_reg));
4165   match(VecX);
4166   format%{%}
4167   interface(REG_INTER);
4168 %}
4169 operand rxmm3() %{
4170   constraint(ALLOC_IN_RC(xmm3_reg));
4171   match(VecX);
4172   format%{%}
4173   interface(REG_INTER);
4174 %}
4175 operand rxmm4() %{
4176   constraint(ALLOC_IN_RC(xmm4_reg));
4177   match(VecX);
4178   format%{%}
4179   interface(REG_INTER);
4180 %}
4181 operand rxmm5() %{
4182   constraint(ALLOC_IN_RC(xmm5_reg));
4183   match(VecX);
4184   format%{%}
4185   interface(REG_INTER);
4186 %}
4187 operand rxmm6() %{
4188   constraint(ALLOC_IN_RC(xmm6_reg));
4189   match(VecX);
4190   format%{%}
4191   interface(REG_INTER);
4192 %}
4193 operand rxmm7() %{
4194   constraint(ALLOC_IN_RC(xmm7_reg));
4195   match(VecX);
4196   format%{%}
4197   interface(REG_INTER);
4198 %}
4199 operand rxmm8() %{
4200   constraint(ALLOC_IN_RC(xmm8_reg));
4201   match(VecX);
4202   format%{%}
4203   interface(REG_INTER);
4204 %}
4205 operand rxmm9() %{
4206   constraint(ALLOC_IN_RC(xmm9_reg));
4207   match(VecX);
4208   format%{%}
4209   interface(REG_INTER);
4210 %}
4211 operand rxmm10() %{
4212   constraint(ALLOC_IN_RC(xmm10_reg));
4213   match(VecX);
4214   format%{%}
4215   interface(REG_INTER);
4216 %}
4217 operand rxmm11() %{
4218   constraint(ALLOC_IN_RC(xmm11_reg));
4219   match(VecX);
4220   format%{%}
4221   interface(REG_INTER);
4222 %}
4223 operand rxmm12() %{
4224   constraint(ALLOC_IN_RC(xmm12_reg));
4225   match(VecX);
4226   format%{%}
4227   interface(REG_INTER);
4228 %}
4229 operand rxmm13() %{
4230   constraint(ALLOC_IN_RC(xmm13_reg));
4231   match(VecX);
4232   format%{%}
4233   interface(REG_INTER);
4234 %}
4235 operand rxmm14() %{
4236   constraint(ALLOC_IN_RC(xmm14_reg));
4237   match(VecX);
4238   format%{%}
4239   interface(REG_INTER);
4240 %}
4241 operand rxmm15() %{
4242   constraint(ALLOC_IN_RC(xmm15_reg));
4243   match(VecX);
4244   format%{%}
4245   interface(REG_INTER);
4246 %}
4247 operand rxmm16() %{
4248   constraint(ALLOC_IN_RC(xmm16_reg));
4249   match(VecX);
4250   format%{%}
4251   interface(REG_INTER);
4252 %}
4253 operand rxmm17() %{
4254   constraint(ALLOC_IN_RC(xmm17_reg));
4255   match(VecX);
4256   format%{%}
4257   interface(REG_INTER);
4258 %}
4259 operand rxmm18() %{
4260   constraint(ALLOC_IN_RC(xmm18_reg));
4261   match(VecX);
4262   format%{%}
4263   interface(REG_INTER);
4264 %}
4265 operand rxmm19() %{
4266   constraint(ALLOC_IN_RC(xmm19_reg));
4267   match(VecX);
4268   format%{%}
4269   interface(REG_INTER);
4270 %}
4271 operand rxmm20() %{
4272   constraint(ALLOC_IN_RC(xmm20_reg));
4273   match(VecX);
4274   format%{%}
4275   interface(REG_INTER);
4276 %}
4277 operand rxmm21() %{
4278   constraint(ALLOC_IN_RC(xmm21_reg));
4279   match(VecX);
4280   format%{%}
4281   interface(REG_INTER);
4282 %}
4283 operand rxmm22() %{
4284   constraint(ALLOC_IN_RC(xmm22_reg));
4285   match(VecX);
4286   format%{%}
4287   interface(REG_INTER);
4288 %}
4289 operand rxmm23() %{
4290   constraint(ALLOC_IN_RC(xmm23_reg));
4291   match(VecX);
4292   format%{%}
4293   interface(REG_INTER);
4294 %}
4295 operand rxmm24() %{
4296   constraint(ALLOC_IN_RC(xmm24_reg));
4297   match(VecX);
4298   format%{%}
4299   interface(REG_INTER);
4300 %}
4301 operand rxmm25() %{
4302   constraint(ALLOC_IN_RC(xmm25_reg));
4303   match(VecX);
4304   format%{%}
4305   interface(REG_INTER);
4306 %}
4307 operand rxmm26() %{
4308   constraint(ALLOC_IN_RC(xmm26_reg));
4309   match(VecX);
4310   format%{%}
4311   interface(REG_INTER);
4312 %}
4313 operand rxmm27() %{
4314   constraint(ALLOC_IN_RC(xmm27_reg));
4315   match(VecX);
4316   format%{%}
4317   interface(REG_INTER);
4318 %}
4319 operand rxmm28() %{
4320   constraint(ALLOC_IN_RC(xmm28_reg));
4321   match(VecX);
4322   format%{%}
4323   interface(REG_INTER);
4324 %}
4325 operand rxmm29() %{
4326   constraint(ALLOC_IN_RC(xmm29_reg));
4327   match(VecX);
4328   format%{%}
4329   interface(REG_INTER);
4330 %}
4331 operand rxmm30() %{
4332   constraint(ALLOC_IN_RC(xmm30_reg));
4333   match(VecX);
4334   format%{%}
4335   interface(REG_INTER);
4336 %}
4337 operand rxmm31() %{
4338   constraint(ALLOC_IN_RC(xmm31_reg));
4339   match(VecX);
4340   format%{%}
4341   interface(REG_INTER);
4342 %}
4343 
4344 //----------OPERAND CLASSES----------------------------------------------------
4345 // Operand Classes are groups of operands that are used as to simplify
4346 // instruction definitions by not requiring the AD writer to specify separate
4347 // instructions for every form of operand when the instruction accepts
4348 // multiple operand types with the same basic encoding and format.  The classic
4349 // case of this is memory operands.
4350 
4351 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4352                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4353                indCompressedOopOffset,
4354                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4355                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4356                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4357 
4358 //----------PIPELINE-----------------------------------------------------------
4359 // Rules which define the behavior of the target architectures pipeline.
4360 pipeline %{
4361 
4362 //----------ATTRIBUTES---------------------------------------------------------
4363 attributes %{
4364   variable_size_instructions;        // Fixed size instructions
4365   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4366   instruction_unit_size = 1;         // An instruction is 1 bytes long
4367   instruction_fetch_unit_size = 16;  // The processor fetches one line
4368   instruction_fetch_units = 1;       // of 16 bytes
4369 
4370   // List of nop instructions
4371   nops( MachNop );
4372 %}
4373 
4374 //----------RESOURCES----------------------------------------------------------
4375 // Resources are the functional units available to the machine
4376 
4377 // Generic P2/P3 pipeline
4378 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4379 // 3 instructions decoded per cycle.
4380 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4381 // 3 ALU op, only ALU0 handles mul instructions.
4382 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4383            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4384            BR, FPU,
4385            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4386 
4387 //----------PIPELINE DESCRIPTION-----------------------------------------------
4388 // Pipeline Description specifies the stages in the machine's pipeline
4389 
4390 // Generic P2/P3 pipeline
4391 pipe_desc(S0, S1, S2, S3, S4, S5);
4392 
4393 //----------PIPELINE CLASSES---------------------------------------------------
4394 // Pipeline Classes describe the stages in which input and output are
4395 // referenced by the hardware pipeline.
4396 
4397 // Naming convention: ialu or fpu
4398 // Then: _reg
4399 // Then: _reg if there is a 2nd register
4400 // Then: _long if it's a pair of instructions implementing a long
4401 // Then: _fat if it requires the big decoder
4402 //   Or: _mem if it requires the big decoder and a memory unit.
4403 
4404 // Integer ALU reg operation
4405 pipe_class ialu_reg(rRegI dst)
4406 %{
4407     single_instruction;
4408     dst    : S4(write);
4409     dst    : S3(read);
4410     DECODE : S0;        // any decoder
4411     ALU    : S3;        // any alu
4412 %}
4413 
4414 // Long ALU reg operation
4415 pipe_class ialu_reg_long(rRegL dst)
4416 %{
4417     instruction_count(2);
4418     dst    : S4(write);
4419     dst    : S3(read);
4420     DECODE : S0(2);     // any 2 decoders
4421     ALU    : S3(2);     // both alus
4422 %}
4423 
4424 // Integer ALU reg operation using big decoder
4425 pipe_class ialu_reg_fat(rRegI dst)
4426 %{
4427     single_instruction;
4428     dst    : S4(write);
4429     dst    : S3(read);
4430     D0     : S0;        // big decoder only
4431     ALU    : S3;        // any alu
4432 %}
4433 
4434 // Long ALU reg operation using big decoder
4435 pipe_class ialu_reg_long_fat(rRegL dst)
4436 %{
4437     instruction_count(2);
4438     dst    : S4(write);
4439     dst    : S3(read);
4440     D0     : S0(2);     // big decoder only; twice
4441     ALU    : S3(2);     // any 2 alus
4442 %}
4443 
4444 // Integer ALU reg-reg operation
4445 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4446 %{
4447     single_instruction;
4448     dst    : S4(write);
4449     src    : S3(read);
4450     DECODE : S0;        // any decoder
4451     ALU    : S3;        // any alu
4452 %}
4453 
4454 // Long ALU reg-reg operation
4455 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4456 %{
4457     instruction_count(2);
4458     dst    : S4(write);
4459     src    : S3(read);
4460     DECODE : S0(2);     // any 2 decoders
4461     ALU    : S3(2);     // both alus
4462 %}
4463 
4464 // Integer ALU reg-reg operation
4465 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4466 %{
4467     single_instruction;
4468     dst    : S4(write);
4469     src    : S3(read);
4470     D0     : S0;        // big decoder only
4471     ALU    : S3;        // any alu
4472 %}
4473 
4474 // Long ALU reg-reg operation
4475 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4476 %{
4477     instruction_count(2);
4478     dst    : S4(write);
4479     src    : S3(read);
4480     D0     : S0(2);     // big decoder only; twice
4481     ALU    : S3(2);     // both alus
4482 %}
4483 
4484 // Integer ALU reg-mem operation
4485 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4486 %{
4487     single_instruction;
4488     dst    : S5(write);
4489     mem    : S3(read);
4490     D0     : S0;        // big decoder only
4491     ALU    : S4;        // any alu
4492     MEM    : S3;        // any mem
4493 %}
4494 
4495 // Integer mem operation (prefetch)
4496 pipe_class ialu_mem(memory mem)
4497 %{
4498     single_instruction;
4499     mem    : S3(read);
4500     D0     : S0;        // big decoder only
4501     MEM    : S3;        // any mem
4502 %}
4503 
4504 // Integer Store to Memory
4505 pipe_class ialu_mem_reg(memory mem, rRegI src)
4506 %{
4507     single_instruction;
4508     mem    : S3(read);
4509     src    : S5(read);
4510     D0     : S0;        // big decoder only
4511     ALU    : S4;        // any alu
4512     MEM    : S3;
4513 %}
4514 
4515 // // Long Store to Memory
4516 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4517 // %{
4518 //     instruction_count(2);
4519 //     mem    : S3(read);
4520 //     src    : S5(read);
4521 //     D0     : S0(2);          // big decoder only; twice
4522 //     ALU    : S4(2);     // any 2 alus
4523 //     MEM    : S3(2);  // Both mems
4524 // %}
4525 
4526 // Integer Store to Memory
4527 pipe_class ialu_mem_imm(memory mem)
4528 %{
4529     single_instruction;
4530     mem    : S3(read);
4531     D0     : S0;        // big decoder only
4532     ALU    : S4;        // any alu
4533     MEM    : S3;
4534 %}
4535 
4536 // Integer ALU0 reg-reg operation
4537 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4538 %{
4539     single_instruction;
4540     dst    : S4(write);
4541     src    : S3(read);
4542     D0     : S0;        // Big decoder only
4543     ALU0   : S3;        // only alu0
4544 %}
4545 
4546 // Integer ALU0 reg-mem operation
4547 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4548 %{
4549     single_instruction;
4550     dst    : S5(write);
4551     mem    : S3(read);
4552     D0     : S0;        // big decoder only
4553     ALU0   : S4;        // ALU0 only
4554     MEM    : S3;        // any mem
4555 %}
4556 
4557 // Integer ALU reg-reg operation
4558 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4559 %{
4560     single_instruction;
4561     cr     : S4(write);
4562     src1   : S3(read);
4563     src2   : S3(read);
4564     DECODE : S0;        // any decoder
4565     ALU    : S3;        // any alu
4566 %}
4567 
4568 // Integer ALU reg-imm operation
4569 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4570 %{
4571     single_instruction;
4572     cr     : S4(write);
4573     src1   : S3(read);
4574     DECODE : S0;        // any decoder
4575     ALU    : S3;        // any alu
4576 %}
4577 
4578 // Integer ALU reg-mem operation
4579 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4580 %{
4581     single_instruction;
4582     cr     : S4(write);
4583     src1   : S3(read);
4584     src2   : S3(read);
4585     D0     : S0;        // big decoder only
4586     ALU    : S4;        // any alu
4587     MEM    : S3;
4588 %}
4589 
4590 // Conditional move reg-reg
4591 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4592 %{
4593     instruction_count(4);
4594     y      : S4(read);
4595     q      : S3(read);
4596     p      : S3(read);
4597     DECODE : S0(4);     // any decoder
4598 %}
4599 
4600 // Conditional move reg-reg
4601 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4602 %{
4603     single_instruction;
4604     dst    : S4(write);
4605     src    : S3(read);
4606     cr     : S3(read);
4607     DECODE : S0;        // any decoder
4608 %}
4609 
4610 // Conditional move reg-mem
4611 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4612 %{
4613     single_instruction;
4614     dst    : S4(write);
4615     src    : S3(read);
4616     cr     : S3(read);
4617     DECODE : S0;        // any decoder
4618     MEM    : S3;
4619 %}
4620 
4621 // Conditional move reg-reg long
4622 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4623 %{
4624     single_instruction;
4625     dst    : S4(write);
4626     src    : S3(read);
4627     cr     : S3(read);
4628     DECODE : S0(2);     // any 2 decoders
4629 %}
4630 
4631 // XXX
4632 // // Conditional move double reg-reg
4633 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4634 // %{
4635 //     single_instruction;
4636 //     dst    : S4(write);
4637 //     src    : S3(read);
4638 //     cr     : S3(read);
4639 //     DECODE : S0;     // any decoder
4640 // %}
4641 
4642 // Float reg-reg operation
4643 pipe_class fpu_reg(regD dst)
4644 %{
4645     instruction_count(2);
4646     dst    : S3(read);
4647     DECODE : S0(2);     // any 2 decoders
4648     FPU    : S3;
4649 %}
4650 
4651 // Float reg-reg operation
4652 pipe_class fpu_reg_reg(regD dst, regD src)
4653 %{
4654     instruction_count(2);
4655     dst    : S4(write);
4656     src    : S3(read);
4657     DECODE : S0(2);     // any 2 decoders
4658     FPU    : S3;
4659 %}
4660 
4661 // Float reg-reg operation
4662 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4663 %{
4664     instruction_count(3);
4665     dst    : S4(write);
4666     src1   : S3(read);
4667     src2   : S3(read);
4668     DECODE : S0(3);     // any 3 decoders
4669     FPU    : S3(2);
4670 %}
4671 
4672 // Float reg-reg operation
4673 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4674 %{
4675     instruction_count(4);
4676     dst    : S4(write);
4677     src1   : S3(read);
4678     src2   : S3(read);
4679     src3   : S3(read);
4680     DECODE : S0(4);     // any 3 decoders
4681     FPU    : S3(2);
4682 %}
4683 
4684 // Float reg-reg operation
4685 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4686 %{
4687     instruction_count(4);
4688     dst    : S4(write);
4689     src1   : S3(read);
4690     src2   : S3(read);
4691     src3   : S3(read);
4692     DECODE : S1(3);     // any 3 decoders
4693     D0     : S0;        // Big decoder only
4694     FPU    : S3(2);
4695     MEM    : S3;
4696 %}
4697 
4698 // Float reg-mem operation
4699 pipe_class fpu_reg_mem(regD dst, memory mem)
4700 %{
4701     instruction_count(2);
4702     dst    : S5(write);
4703     mem    : S3(read);
4704     D0     : S0;        // big decoder only
4705     DECODE : S1;        // any decoder for FPU POP
4706     FPU    : S4;
4707     MEM    : S3;        // any mem
4708 %}
4709 
4710 // Float reg-mem operation
4711 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4712 %{
4713     instruction_count(3);
4714     dst    : S5(write);
4715     src1   : S3(read);
4716     mem    : S3(read);
4717     D0     : S0;        // big decoder only
4718     DECODE : S1(2);     // any decoder for FPU POP
4719     FPU    : S4;
4720     MEM    : S3;        // any mem
4721 %}
4722 
4723 // Float mem-reg operation
4724 pipe_class fpu_mem_reg(memory mem, regD src)
4725 %{
4726     instruction_count(2);
4727     src    : S5(read);
4728     mem    : S3(read);
4729     DECODE : S0;        // any decoder for FPU PUSH
4730     D0     : S1;        // big decoder only
4731     FPU    : S4;
4732     MEM    : S3;        // any mem
4733 %}
4734 
4735 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4736 %{
4737     instruction_count(3);
4738     src1   : S3(read);
4739     src2   : S3(read);
4740     mem    : S3(read);
4741     DECODE : S0(2);     // any decoder for FPU PUSH
4742     D0     : S1;        // big decoder only
4743     FPU    : S4;
4744     MEM    : S3;        // any mem
4745 %}
4746 
4747 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4748 %{
4749     instruction_count(3);
4750     src1   : S3(read);
4751     src2   : S3(read);
4752     mem    : S4(read);
4753     DECODE : S0;        // any decoder for FPU PUSH
4754     D0     : S0(2);     // big decoder only
4755     FPU    : S4;
4756     MEM    : S3(2);     // any mem
4757 %}
4758 
4759 pipe_class fpu_mem_mem(memory dst, memory src1)
4760 %{
4761     instruction_count(2);
4762     src1   : S3(read);
4763     dst    : S4(read);
4764     D0     : S0(2);     // big decoder only
4765     MEM    : S3(2);     // any mem
4766 %}
4767 
4768 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4769 %{
4770     instruction_count(3);
4771     src1   : S3(read);
4772     src2   : S3(read);
4773     dst    : S4(read);
4774     D0     : S0(3);     // big decoder only
4775     FPU    : S4;
4776     MEM    : S3(3);     // any mem
4777 %}
4778 
4779 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4780 %{
4781     instruction_count(3);
4782     src1   : S4(read);
4783     mem    : S4(read);
4784     DECODE : S0;        // any decoder for FPU PUSH
4785     D0     : S0(2);     // big decoder only
4786     FPU    : S4;
4787     MEM    : S3(2);     // any mem
4788 %}
4789 
4790 // Float load constant
4791 pipe_class fpu_reg_con(regD dst)
4792 %{
4793     instruction_count(2);
4794     dst    : S5(write);
4795     D0     : S0;        // big decoder only for the load
4796     DECODE : S1;        // any decoder for FPU POP
4797     FPU    : S4;
4798     MEM    : S3;        // any mem
4799 %}
4800 
4801 // Float load constant
4802 pipe_class fpu_reg_reg_con(regD dst, regD src)
4803 %{
4804     instruction_count(3);
4805     dst    : S5(write);
4806     src    : S3(read);
4807     D0     : S0;        // big decoder only for the load
4808     DECODE : S1(2);     // any decoder for FPU POP
4809     FPU    : S4;
4810     MEM    : S3;        // any mem
4811 %}
4812 
4813 // UnConditional branch
4814 pipe_class pipe_jmp(label labl)
4815 %{
4816     single_instruction;
4817     BR   : S3;
4818 %}
4819 
4820 // Conditional branch
4821 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4822 %{
4823     single_instruction;
4824     cr    : S1(read);
4825     BR    : S3;
4826 %}
4827 
4828 // Allocation idiom
4829 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4830 %{
4831     instruction_count(1); force_serialization;
4832     fixed_latency(6);
4833     heap_ptr : S3(read);
4834     DECODE   : S0(3);
4835     D0       : S2;
4836     MEM      : S3;
4837     ALU      : S3(2);
4838     dst      : S5(write);
4839     BR       : S5;
4840 %}
4841 
4842 // Generic big/slow expanded idiom
4843 pipe_class pipe_slow()
4844 %{
4845     instruction_count(10); multiple_bundles; force_serialization;
4846     fixed_latency(100);
4847     D0  : S0(2);
4848     MEM : S3(2);
4849 %}
4850 
4851 // The real do-nothing guy
4852 pipe_class empty()
4853 %{
4854     instruction_count(0);
4855 %}
4856 
4857 // Define the class for the Nop node
4858 define
4859 %{
4860    MachNop = empty;
4861 %}
4862 
4863 %}
4864 
4865 //----------INSTRUCTIONS-------------------------------------------------------
4866 //
4867 // match      -- States which machine-independent subtree may be replaced
4868 //               by this instruction.
4869 // ins_cost   -- The estimated cost of this instruction is used by instruction
4870 //               selection to identify a minimum cost tree of machine
4871 //               instructions that matches a tree of machine-independent
4872 //               instructions.
4873 // format     -- A string providing the disassembly for this instruction.
4874 //               The value of an instruction's operand may be inserted
4875 //               by referring to it with a '$' prefix.
4876 // opcode     -- Three instruction opcodes may be provided.  These are referred
4877 //               to within an encode class as $primary, $secondary, and $tertiary
4878 //               rrspectively.  The primary opcode is commonly used to
4879 //               indicate the type of machine instruction, while secondary
4880 //               and tertiary are often used for prefix options or addressing
4881 //               modes.
4882 // ins_encode -- A list of encode classes with parameters. The encode class
4883 //               name must have been defined in an 'enc_class' specification
4884 //               in the encode section of the architecture description.
4885 
4886 
4887 //----------Load/Store/Move Instructions---------------------------------------
4888 //----------Load Instructions--------------------------------------------------
4889 
4890 // Load Byte (8 bit signed)
4891 instruct loadB(rRegI dst, memory mem)
4892 %{
4893   match(Set dst (LoadB mem));
4894 
4895   ins_cost(125);
4896   format %{ "movsbl  $dst, $mem\t# byte" %}
4897 
4898   ins_encode %{
4899     __ movsbl($dst$$Register, $mem$$Address);
4900   %}
4901 
4902   ins_pipe(ialu_reg_mem);
4903 %}
4904 
4905 // Load Byte (8 bit signed) into Long Register
4906 instruct loadB2L(rRegL dst, memory mem)
4907 %{
4908   match(Set dst (ConvI2L (LoadB mem)));
4909 
4910   ins_cost(125);
4911   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4912 
4913   ins_encode %{
4914     __ movsbq($dst$$Register, $mem$$Address);
4915   %}
4916 
4917   ins_pipe(ialu_reg_mem);
4918 %}
4919 
4920 // Load Unsigned Byte (8 bit UNsigned)
4921 instruct loadUB(rRegI dst, memory mem)
4922 %{
4923   match(Set dst (LoadUB mem));
4924 
4925   ins_cost(125);
4926   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4927 
4928   ins_encode %{
4929     __ movzbl($dst$$Register, $mem$$Address);
4930   %}
4931 
4932   ins_pipe(ialu_reg_mem);
4933 %}
4934 
4935 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4936 instruct loadUB2L(rRegL dst, memory mem)
4937 %{
4938   match(Set dst (ConvI2L (LoadUB mem)));
4939 
4940   ins_cost(125);
4941   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4942 
4943   ins_encode %{
4944     __ movzbq($dst$$Register, $mem$$Address);
4945   %}
4946 
4947   ins_pipe(ialu_reg_mem);
4948 %}
4949 
4950 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4951 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4952   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4953   effect(KILL cr);
4954 
4955   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4956             "andl    $dst, right_n_bits($mask, 8)" %}
4957   ins_encode %{
4958     Register Rdst = $dst$$Register;
4959     __ movzbq(Rdst, $mem$$Address);
4960     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4961   %}
4962   ins_pipe(ialu_reg_mem);
4963 %}
4964 
4965 // Load Short (16 bit signed)
4966 instruct loadS(rRegI dst, memory mem)
4967 %{
4968   match(Set dst (LoadS mem));
4969 
4970   ins_cost(125);
4971   format %{ "movswl $dst, $mem\t# short" %}
4972 
4973   ins_encode %{
4974     __ movswl($dst$$Register, $mem$$Address);
4975   %}
4976 
4977   ins_pipe(ialu_reg_mem);
4978 %}
4979 
4980 // Load Short (16 bit signed) to Byte (8 bit signed)
4981 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4982   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4983 
4984   ins_cost(125);
4985   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4986   ins_encode %{
4987     __ movsbl($dst$$Register, $mem$$Address);
4988   %}
4989   ins_pipe(ialu_reg_mem);
4990 %}
4991 
4992 // Load Short (16 bit signed) into Long Register
4993 instruct loadS2L(rRegL dst, memory mem)
4994 %{
4995   match(Set dst (ConvI2L (LoadS mem)));
4996 
4997   ins_cost(125);
4998   format %{ "movswq $dst, $mem\t# short -> long" %}
4999 
5000   ins_encode %{
5001     __ movswq($dst$$Register, $mem$$Address);
5002   %}
5003 
5004   ins_pipe(ialu_reg_mem);
5005 %}
5006 
5007 // Load Unsigned Short/Char (16 bit UNsigned)
5008 instruct loadUS(rRegI dst, memory mem)
5009 %{
5010   match(Set dst (LoadUS mem));
5011 
5012   ins_cost(125);
5013   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5014 
5015   ins_encode %{
5016     __ movzwl($dst$$Register, $mem$$Address);
5017   %}
5018 
5019   ins_pipe(ialu_reg_mem);
5020 %}
5021 
5022 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5023 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5024   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5025 
5026   ins_cost(125);
5027   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5028   ins_encode %{
5029     __ movsbl($dst$$Register, $mem$$Address);
5030   %}
5031   ins_pipe(ialu_reg_mem);
5032 %}
5033 
5034 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5035 instruct loadUS2L(rRegL dst, memory mem)
5036 %{
5037   match(Set dst (ConvI2L (LoadUS mem)));
5038 
5039   ins_cost(125);
5040   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5041 
5042   ins_encode %{
5043     __ movzwq($dst$$Register, $mem$$Address);
5044   %}
5045 
5046   ins_pipe(ialu_reg_mem);
5047 %}
5048 
5049 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5050 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5051   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5052 
5053   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5054   ins_encode %{
5055     __ movzbq($dst$$Register, $mem$$Address);
5056   %}
5057   ins_pipe(ialu_reg_mem);
5058 %}
5059 
5060 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5061 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5062   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5063   effect(KILL cr);
5064 
5065   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5066             "andl    $dst, right_n_bits($mask, 16)" %}
5067   ins_encode %{
5068     Register Rdst = $dst$$Register;
5069     __ movzwq(Rdst, $mem$$Address);
5070     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5071   %}
5072   ins_pipe(ialu_reg_mem);
5073 %}
5074 
5075 // Load Integer
5076 instruct loadI(rRegI dst, memory mem)
5077 %{
5078   match(Set dst (LoadI mem));
5079 
5080   ins_cost(125);
5081   format %{ "movl    $dst, $mem\t# int" %}
5082 
5083   ins_encode %{
5084     __ movl($dst$$Register, $mem$$Address);
5085   %}
5086 
5087   ins_pipe(ialu_reg_mem);
5088 %}
5089 
5090 // Load Integer (32 bit signed) to Byte (8 bit signed)
5091 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5092   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5093 
5094   ins_cost(125);
5095   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5096   ins_encode %{
5097     __ movsbl($dst$$Register, $mem$$Address);
5098   %}
5099   ins_pipe(ialu_reg_mem);
5100 %}
5101 
5102 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5103 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5104   match(Set dst (AndI (LoadI mem) mask));
5105 
5106   ins_cost(125);
5107   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5108   ins_encode %{
5109     __ movzbl($dst$$Register, $mem$$Address);
5110   %}
5111   ins_pipe(ialu_reg_mem);
5112 %}
5113 
5114 // Load Integer (32 bit signed) to Short (16 bit signed)
5115 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5116   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5117 
5118   ins_cost(125);
5119   format %{ "movswl  $dst, $mem\t# int -> short" %}
5120   ins_encode %{
5121     __ movswl($dst$$Register, $mem$$Address);
5122   %}
5123   ins_pipe(ialu_reg_mem);
5124 %}
5125 
5126 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5127 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5128   match(Set dst (AndI (LoadI mem) mask));
5129 
5130   ins_cost(125);
5131   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5132   ins_encode %{
5133     __ movzwl($dst$$Register, $mem$$Address);
5134   %}
5135   ins_pipe(ialu_reg_mem);
5136 %}
5137 
5138 // Load Integer into Long Register
5139 instruct loadI2L(rRegL dst, memory mem)
5140 %{
5141   match(Set dst (ConvI2L (LoadI mem)));
5142 
5143   ins_cost(125);
5144   format %{ "movslq  $dst, $mem\t# int -> long" %}
5145 
5146   ins_encode %{
5147     __ movslq($dst$$Register, $mem$$Address);
5148   %}
5149 
5150   ins_pipe(ialu_reg_mem);
5151 %}
5152 
5153 // Load Integer with mask 0xFF into Long Register
5154 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5155   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5156 
5157   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5158   ins_encode %{
5159     __ movzbq($dst$$Register, $mem$$Address);
5160   %}
5161   ins_pipe(ialu_reg_mem);
5162 %}
5163 
5164 // Load Integer with mask 0xFFFF into Long Register
5165 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5166   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5167 
5168   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5169   ins_encode %{
5170     __ movzwq($dst$$Register, $mem$$Address);
5171   %}
5172   ins_pipe(ialu_reg_mem);
5173 %}
5174 
5175 // Load Integer with a 31-bit mask into Long Register
5176 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5177   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5178   effect(KILL cr);
5179 
5180   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5181             "andl    $dst, $mask" %}
5182   ins_encode %{
5183     Register Rdst = $dst$$Register;
5184     __ movl(Rdst, $mem$$Address);
5185     __ andl(Rdst, $mask$$constant);
5186   %}
5187   ins_pipe(ialu_reg_mem);
5188 %}
5189 
5190 // Load Unsigned Integer into Long Register
5191 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5192 %{
5193   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5194 
5195   ins_cost(125);
5196   format %{ "movl    $dst, $mem\t# uint -> long" %}
5197 
5198   ins_encode %{
5199     __ movl($dst$$Register, $mem$$Address);
5200   %}
5201 
5202   ins_pipe(ialu_reg_mem);
5203 %}
5204 
5205 // Load Long
5206 instruct loadL(rRegL dst, memory mem)
5207 %{
5208   match(Set dst (LoadL mem));
5209 
5210   ins_cost(125);
5211   format %{ "movq    $dst, $mem\t# long" %}
5212 
5213   ins_encode %{
5214     __ movq($dst$$Register, $mem$$Address);
5215   %}
5216 
5217   ins_pipe(ialu_reg_mem); // XXX
5218 %}
5219 
5220 // Load Range
5221 instruct loadRange(rRegI dst, memory mem)
5222 %{
5223   match(Set dst (LoadRange mem));
5224 
5225   ins_cost(125); // XXX
5226   format %{ "movl    $dst, $mem\t# range" %}
5227   opcode(0x8B);
5228   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5229   ins_pipe(ialu_reg_mem);
5230 %}
5231 
5232 // Load Pointer
5233 instruct loadP(rRegP dst, memory mem)
5234 %{
5235   match(Set dst (LoadP mem));
5236 
5237   ins_cost(125); // XXX
5238   format %{ "movq    $dst, $mem\t# ptr" %}
5239   opcode(0x8B);
5240   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5241   ins_pipe(ialu_reg_mem); // XXX
5242 %}
5243 
5244 // Load Compressed Pointer
5245 instruct loadN(rRegN dst, memory mem)
5246 %{
5247    match(Set dst (LoadN mem));
5248 
5249    ins_cost(125); // XXX
5250    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5251    ins_encode %{
5252      __ movl($dst$$Register, $mem$$Address);
5253    %}
5254    ins_pipe(ialu_reg_mem); // XXX
5255 %}
5256 
5257 
5258 // Load Klass Pointer
5259 instruct loadKlass(rRegP dst, memory mem)
5260 %{
5261   match(Set dst (LoadKlass mem));
5262 
5263   ins_cost(125); // XXX
5264   format %{ "movq    $dst, $mem\t# class" %}
5265   opcode(0x8B);
5266   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5267   ins_pipe(ialu_reg_mem); // XXX
5268 %}
5269 
5270 // Load narrow Klass Pointer
5271 instruct loadNKlass(rRegN dst, memory mem)
5272 %{
5273   match(Set dst (LoadNKlass mem));
5274 
5275   ins_cost(125); // XXX
5276   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5277   ins_encode %{
5278     __ movl($dst$$Register, $mem$$Address);
5279   %}
5280   ins_pipe(ialu_reg_mem); // XXX
5281 %}
5282 
5283 // Load Float
5284 instruct loadF(regF dst, memory mem)
5285 %{
5286   match(Set dst (LoadF mem));
5287 
5288   ins_cost(145); // XXX
5289   format %{ "movss   $dst, $mem\t# float" %}
5290   ins_encode %{
5291     __ movflt($dst$$XMMRegister, $mem$$Address);
5292   %}
5293   ins_pipe(pipe_slow); // XXX
5294 %}
5295 
5296 // Load Float
5297 instruct MoveF2VL(vlRegF dst, regF src) %{
5298   match(Set dst src);
5299   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5300   ins_encode %{
5301     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5302   %}
5303   ins_pipe( fpu_reg_reg );
5304 %}
5305 
5306 // Load Float
5307 instruct MoveVL2F(regF dst, vlRegF src) %{
5308   match(Set dst src);
5309   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5310   ins_encode %{
5311     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5312   %}
5313   ins_pipe( fpu_reg_reg );
5314 %}
5315 
5316 // Load Double
5317 instruct loadD_partial(regD dst, memory mem)
5318 %{
5319   predicate(!UseXmmLoadAndClearUpper);
5320   match(Set dst (LoadD mem));
5321 
5322   ins_cost(145); // XXX
5323   format %{ "movlpd  $dst, $mem\t# double" %}
5324   ins_encode %{
5325     __ movdbl($dst$$XMMRegister, $mem$$Address);
5326   %}
5327   ins_pipe(pipe_slow); // XXX
5328 %}
5329 
5330 instruct loadD(regD dst, memory mem)
5331 %{
5332   predicate(UseXmmLoadAndClearUpper);
5333   match(Set dst (LoadD mem));
5334 
5335   ins_cost(145); // XXX
5336   format %{ "movsd   $dst, $mem\t# double" %}
5337   ins_encode %{
5338     __ movdbl($dst$$XMMRegister, $mem$$Address);
5339   %}
5340   ins_pipe(pipe_slow); // XXX
5341 %}
5342 
5343 // Load Double
5344 instruct MoveD2VL(vlRegD dst, regD src) %{
5345   match(Set dst src);
5346   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5347   ins_encode %{
5348     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5349   %}
5350   ins_pipe( fpu_reg_reg );
5351 %}
5352 
5353 // Load Double
5354 instruct MoveVL2D(regD dst, vlRegD src) %{
5355   match(Set dst src);
5356   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5357   ins_encode %{
5358     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5359   %}
5360   ins_pipe( fpu_reg_reg );
5361 %}
5362 
5363 // Load Effective Address
5364 instruct leaP8(rRegP dst, indOffset8 mem)
5365 %{
5366   match(Set dst mem);
5367 
5368   ins_cost(110); // XXX
5369   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5370   opcode(0x8D);
5371   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5372   ins_pipe(ialu_reg_reg_fat);
5373 %}
5374 
5375 instruct leaP32(rRegP dst, indOffset32 mem)
5376 %{
5377   match(Set dst mem);
5378 
5379   ins_cost(110);
5380   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5381   opcode(0x8D);
5382   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5383   ins_pipe(ialu_reg_reg_fat);
5384 %}
5385 
5386 // instruct leaPIdx(rRegP dst, indIndex mem)
5387 // %{
5388 //   match(Set dst mem);
5389 
5390 //   ins_cost(110);
5391 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5392 //   opcode(0x8D);
5393 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5394 //   ins_pipe(ialu_reg_reg_fat);
5395 // %}
5396 
5397 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5398 %{
5399   match(Set dst mem);
5400 
5401   ins_cost(110);
5402   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5403   opcode(0x8D);
5404   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5405   ins_pipe(ialu_reg_reg_fat);
5406 %}
5407 
5408 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5409 %{
5410   match(Set dst mem);
5411 
5412   ins_cost(110);
5413   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5414   opcode(0x8D);
5415   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5416   ins_pipe(ialu_reg_reg_fat);
5417 %}
5418 
5419 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5420 %{
5421   match(Set dst mem);
5422 
5423   ins_cost(110);
5424   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5425   opcode(0x8D);
5426   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5427   ins_pipe(ialu_reg_reg_fat);
5428 %}
5429 
5430 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5431 %{
5432   match(Set dst mem);
5433 
5434   ins_cost(110);
5435   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5436   opcode(0x8D);
5437   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5438   ins_pipe(ialu_reg_reg_fat);
5439 %}
5440 
5441 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5442 %{
5443   match(Set dst mem);
5444 
5445   ins_cost(110);
5446   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5447   opcode(0x8D);
5448   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5449   ins_pipe(ialu_reg_reg_fat);
5450 %}
5451 
5452 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5453 %{
5454   match(Set dst mem);
5455 
5456   ins_cost(110);
5457   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5458   opcode(0x8D);
5459   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5460   ins_pipe(ialu_reg_reg_fat);
5461 %}
5462 
5463 // Load Effective Address which uses Narrow (32-bits) oop
5464 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5465 %{
5466   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5467   match(Set dst mem);
5468 
5469   ins_cost(110);
5470   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5471   opcode(0x8D);
5472   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5473   ins_pipe(ialu_reg_reg_fat);
5474 %}
5475 
5476 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5477 %{
5478   predicate(Universe::narrow_oop_shift() == 0);
5479   match(Set dst mem);
5480 
5481   ins_cost(110); // XXX
5482   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5483   opcode(0x8D);
5484   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5485   ins_pipe(ialu_reg_reg_fat);
5486 %}
5487 
5488 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5489 %{
5490   predicate(Universe::narrow_oop_shift() == 0);
5491   match(Set dst mem);
5492 
5493   ins_cost(110);
5494   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5495   opcode(0x8D);
5496   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5497   ins_pipe(ialu_reg_reg_fat);
5498 %}
5499 
5500 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5501 %{
5502   predicate(Universe::narrow_oop_shift() == 0);
5503   match(Set dst mem);
5504 
5505   ins_cost(110);
5506   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5507   opcode(0x8D);
5508   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5509   ins_pipe(ialu_reg_reg_fat);
5510 %}
5511 
5512 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5513 %{
5514   predicate(Universe::narrow_oop_shift() == 0);
5515   match(Set dst mem);
5516 
5517   ins_cost(110);
5518   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5519   opcode(0x8D);
5520   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5521   ins_pipe(ialu_reg_reg_fat);
5522 %}
5523 
5524 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5525 %{
5526   predicate(Universe::narrow_oop_shift() == 0);
5527   match(Set dst mem);
5528 
5529   ins_cost(110);
5530   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5531   opcode(0x8D);
5532   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5533   ins_pipe(ialu_reg_reg_fat);
5534 %}
5535 
5536 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5537 %{
5538   predicate(Universe::narrow_oop_shift() == 0);
5539   match(Set dst mem);
5540 
5541   ins_cost(110);
5542   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5543   opcode(0x8D);
5544   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5545   ins_pipe(ialu_reg_reg_fat);
5546 %}
5547 
5548 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5549 %{
5550   predicate(Universe::narrow_oop_shift() == 0);
5551   match(Set dst mem);
5552 
5553   ins_cost(110);
5554   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5555   opcode(0x8D);
5556   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5557   ins_pipe(ialu_reg_reg_fat);
5558 %}
5559 
5560 instruct loadConI(rRegI dst, immI src)
5561 %{
5562   match(Set dst src);
5563 
5564   format %{ "movl    $dst, $src\t# int" %}
5565   ins_encode(load_immI(dst, src));
5566   ins_pipe(ialu_reg_fat); // XXX
5567 %}
5568 
5569 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5570 %{
5571   match(Set dst src);
5572   effect(KILL cr);
5573 
5574   ins_cost(50);
5575   format %{ "xorl    $dst, $dst\t# int" %}
5576   opcode(0x33); /* + rd */
5577   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5578   ins_pipe(ialu_reg);
5579 %}
5580 
5581 instruct loadConL(rRegL dst, immL src)
5582 %{
5583   match(Set dst src);
5584 
5585   ins_cost(150);
5586   format %{ "movq    $dst, $src\t# long" %}
5587   ins_encode(load_immL(dst, src));
5588   ins_pipe(ialu_reg);
5589 %}
5590 
5591 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5592 %{
5593   match(Set dst src);
5594   effect(KILL cr);
5595 
5596   ins_cost(50);
5597   format %{ "xorl    $dst, $dst\t# long" %}
5598   opcode(0x33); /* + rd */
5599   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5600   ins_pipe(ialu_reg); // XXX
5601 %}
5602 
5603 instruct loadConUL32(rRegL dst, immUL32 src)
5604 %{
5605   match(Set dst src);
5606 
5607   ins_cost(60);
5608   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5609   ins_encode(load_immUL32(dst, src));
5610   ins_pipe(ialu_reg);
5611 %}
5612 
5613 instruct loadConL32(rRegL dst, immL32 src)
5614 %{
5615   match(Set dst src);
5616 
5617   ins_cost(70);
5618   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5619   ins_encode(load_immL32(dst, src));
5620   ins_pipe(ialu_reg);
5621 %}
5622 
5623 instruct loadConP(rRegP dst, immP con) %{
5624   match(Set dst con);
5625 
5626   format %{ "movq    $dst, $con\t# ptr" %}
5627   ins_encode(load_immP(dst, con));
5628   ins_pipe(ialu_reg_fat); // XXX
5629 %}
5630 
5631 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5632 %{
5633   match(Set dst src);
5634   effect(KILL cr);
5635 
5636   ins_cost(50);
5637   format %{ "xorl    $dst, $dst\t# ptr" %}
5638   opcode(0x33); /* + rd */
5639   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5640   ins_pipe(ialu_reg);
5641 %}
5642 
5643 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5644 %{
5645   match(Set dst src);
5646   effect(KILL cr);
5647 
5648   ins_cost(60);
5649   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5650   ins_encode(load_immP31(dst, src));
5651   ins_pipe(ialu_reg);
5652 %}
5653 
5654 instruct loadConF(regF dst, immF con) %{
5655   match(Set dst con);
5656   ins_cost(125);
5657   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5658   ins_encode %{
5659     __ movflt($dst$$XMMRegister, $constantaddress($con));
5660   %}
5661   ins_pipe(pipe_slow);
5662 %}
5663 
5664 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5665   match(Set dst src);
5666   effect(KILL cr);
5667   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5668   ins_encode %{
5669     __ xorq($dst$$Register, $dst$$Register);
5670   %}
5671   ins_pipe(ialu_reg);
5672 %}
5673 
5674 instruct loadConN(rRegN dst, immN src) %{
5675   match(Set dst src);
5676 
5677   ins_cost(125);
5678   format %{ "movl    $dst, $src\t# compressed ptr" %}
5679   ins_encode %{
5680     address con = (address)$src$$constant;
5681     if (con == NULL) {
5682       ShouldNotReachHere();
5683     } else {
5684       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5685     }
5686   %}
5687   ins_pipe(ialu_reg_fat); // XXX
5688 %}
5689 
5690 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5691   match(Set dst src);
5692 
5693   ins_cost(125);
5694   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5695   ins_encode %{
5696     address con = (address)$src$$constant;
5697     if (con == NULL) {
5698       ShouldNotReachHere();
5699     } else {
5700       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5701     }
5702   %}
5703   ins_pipe(ialu_reg_fat); // XXX
5704 %}
5705 
5706 instruct loadConF0(regF dst, immF0 src)
5707 %{
5708   match(Set dst src);
5709   ins_cost(100);
5710 
5711   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5712   ins_encode %{
5713     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5714   %}
5715   ins_pipe(pipe_slow);
5716 %}
5717 
5718 // Use the same format since predicate() can not be used here.
5719 instruct loadConD(regD dst, immD con) %{
5720   match(Set dst con);
5721   ins_cost(125);
5722   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5723   ins_encode %{
5724     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5725   %}
5726   ins_pipe(pipe_slow);
5727 %}
5728 
5729 instruct loadConD0(regD dst, immD0 src)
5730 %{
5731   match(Set dst src);
5732   ins_cost(100);
5733 
5734   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5735   ins_encode %{
5736     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5737   %}
5738   ins_pipe(pipe_slow);
5739 %}
5740 
5741 instruct loadSSI(rRegI dst, stackSlotI src)
5742 %{
5743   match(Set dst src);
5744 
5745   ins_cost(125);
5746   format %{ "movl    $dst, $src\t# int stk" %}
5747   opcode(0x8B);
5748   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5749   ins_pipe(ialu_reg_mem);
5750 %}
5751 
5752 instruct loadSSL(rRegL dst, stackSlotL src)
5753 %{
5754   match(Set dst src);
5755 
5756   ins_cost(125);
5757   format %{ "movq    $dst, $src\t# long stk" %}
5758   opcode(0x8B);
5759   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5760   ins_pipe(ialu_reg_mem);
5761 %}
5762 
5763 instruct loadSSP(rRegP dst, stackSlotP src)
5764 %{
5765   match(Set dst src);
5766 
5767   ins_cost(125);
5768   format %{ "movq    $dst, $src\t# ptr stk" %}
5769   opcode(0x8B);
5770   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5771   ins_pipe(ialu_reg_mem);
5772 %}
5773 
5774 instruct loadSSF(regF dst, stackSlotF src)
5775 %{
5776   match(Set dst src);
5777 
5778   ins_cost(125);
5779   format %{ "movss   $dst, $src\t# float stk" %}
5780   ins_encode %{
5781     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5782   %}
5783   ins_pipe(pipe_slow); // XXX
5784 %}
5785 
5786 // Use the same format since predicate() can not be used here.
5787 instruct loadSSD(regD dst, stackSlotD src)
5788 %{
5789   match(Set dst src);
5790 
5791   ins_cost(125);
5792   format %{ "movsd   $dst, $src\t# double stk" %}
5793   ins_encode  %{
5794     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5795   %}
5796   ins_pipe(pipe_slow); // XXX
5797 %}
5798 
5799 // Prefetch instructions for allocation.
5800 // Must be safe to execute with invalid address (cannot fault).
5801 
5802 instruct prefetchAlloc( memory mem ) %{
5803   predicate(AllocatePrefetchInstr==3);
5804   match(PrefetchAllocation mem);
5805   ins_cost(125);
5806 
5807   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5808   ins_encode %{
5809     __ prefetchw($mem$$Address);
5810   %}
5811   ins_pipe(ialu_mem);
5812 %}
5813 
5814 instruct prefetchAllocNTA( memory mem ) %{
5815   predicate(AllocatePrefetchInstr==0);
5816   match(PrefetchAllocation mem);
5817   ins_cost(125);
5818 
5819   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5820   ins_encode %{
5821     __ prefetchnta($mem$$Address);
5822   %}
5823   ins_pipe(ialu_mem);
5824 %}
5825 
5826 instruct prefetchAllocT0( memory mem ) %{
5827   predicate(AllocatePrefetchInstr==1);
5828   match(PrefetchAllocation mem);
5829   ins_cost(125);
5830 
5831   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5832   ins_encode %{
5833     __ prefetcht0($mem$$Address);
5834   %}
5835   ins_pipe(ialu_mem);
5836 %}
5837 
5838 instruct prefetchAllocT2( memory mem ) %{
5839   predicate(AllocatePrefetchInstr==2);
5840   match(PrefetchAllocation mem);
5841   ins_cost(125);
5842 
5843   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5844   ins_encode %{
5845     __ prefetcht2($mem$$Address);
5846   %}
5847   ins_pipe(ialu_mem);
5848 %}
5849 
5850 //----------Store Instructions-------------------------------------------------
5851 
5852 // Store Byte
5853 instruct storeB(memory mem, rRegI src)
5854 %{
5855   match(Set mem (StoreB mem src));
5856 
5857   ins_cost(125); // XXX
5858   format %{ "movb    $mem, $src\t# byte" %}
5859   opcode(0x88);
5860   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5861   ins_pipe(ialu_mem_reg);
5862 %}
5863 
5864 // Store Char/Short
5865 instruct storeC(memory mem, rRegI src)
5866 %{
5867   match(Set mem (StoreC mem src));
5868 
5869   ins_cost(125); // XXX
5870   format %{ "movw    $mem, $src\t# char/short" %}
5871   opcode(0x89);
5872   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5873   ins_pipe(ialu_mem_reg);
5874 %}
5875 
5876 // Store Integer
5877 instruct storeI(memory mem, rRegI src)
5878 %{
5879   match(Set mem (StoreI mem src));
5880 
5881   ins_cost(125); // XXX
5882   format %{ "movl    $mem, $src\t# int" %}
5883   opcode(0x89);
5884   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5885   ins_pipe(ialu_mem_reg);
5886 %}
5887 
5888 // Store Long
5889 instruct storeL(memory mem, rRegL src)
5890 %{
5891   match(Set mem (StoreL mem src));
5892 
5893   ins_cost(125); // XXX
5894   format %{ "movq    $mem, $src\t# long" %}
5895   opcode(0x89);
5896   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5897   ins_pipe(ialu_mem_reg); // XXX
5898 %}
5899 
5900 // Store Pointer
5901 instruct storeP(memory mem, any_RegP src)
5902 %{
5903   match(Set mem (StoreP mem src));
5904 
5905   ins_cost(125); // XXX
5906   format %{ "movq    $mem, $src\t# ptr" %}
5907   opcode(0x89);
5908   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5909   ins_pipe(ialu_mem_reg);
5910 %}
5911 
5912 instruct storeImmP0(memory mem, immP0 zero)
5913 %{
5914   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5915   match(Set mem (StoreP mem zero));
5916 
5917   ins_cost(125); // XXX
5918   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5919   ins_encode %{
5920     __ movq($mem$$Address, r12);
5921   %}
5922   ins_pipe(ialu_mem_reg);
5923 %}
5924 
5925 // Store NULL Pointer, mark word, or other simple pointer constant.
5926 instruct storeImmP(memory mem, immP31 src)
5927 %{
5928   match(Set mem (StoreP mem src));
5929 
5930   ins_cost(150); // XXX
5931   format %{ "movq    $mem, $src\t# ptr" %}
5932   opcode(0xC7); /* C7 /0 */
5933   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5934   ins_pipe(ialu_mem_imm);
5935 %}
5936 
5937 // Store Compressed Pointer
5938 instruct storeN(memory mem, rRegN src)
5939 %{
5940   match(Set mem (StoreN mem src));
5941 
5942   ins_cost(125); // XXX
5943   format %{ "movl    $mem, $src\t# compressed ptr" %}
5944   ins_encode %{
5945     __ movl($mem$$Address, $src$$Register);
5946   %}
5947   ins_pipe(ialu_mem_reg);
5948 %}
5949 
5950 instruct storeNKlass(memory mem, rRegN src)
5951 %{
5952   match(Set mem (StoreNKlass mem src));
5953 
5954   ins_cost(125); // XXX
5955   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5956   ins_encode %{
5957     __ movl($mem$$Address, $src$$Register);
5958   %}
5959   ins_pipe(ialu_mem_reg);
5960 %}
5961 
5962 instruct storeImmN0(memory mem, immN0 zero)
5963 %{
5964   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5965   match(Set mem (StoreN mem zero));
5966 
5967   ins_cost(125); // XXX
5968   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5969   ins_encode %{
5970     __ movl($mem$$Address, r12);
5971   %}
5972   ins_pipe(ialu_mem_reg);
5973 %}
5974 
5975 instruct storeImmN(memory mem, immN src)
5976 %{
5977   match(Set mem (StoreN mem src));
5978 
5979   ins_cost(150); // XXX
5980   format %{ "movl    $mem, $src\t# compressed ptr" %}
5981   ins_encode %{
5982     address con = (address)$src$$constant;
5983     if (con == NULL) {
5984       __ movl($mem$$Address, (int32_t)0);
5985     } else {
5986       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5987     }
5988   %}
5989   ins_pipe(ialu_mem_imm);
5990 %}
5991 
5992 instruct storeImmNKlass(memory mem, immNKlass src)
5993 %{
5994   match(Set mem (StoreNKlass mem src));
5995 
5996   ins_cost(150); // XXX
5997   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5998   ins_encode %{
5999     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6000   %}
6001   ins_pipe(ialu_mem_imm);
6002 %}
6003 
6004 // Store Integer Immediate
6005 instruct storeImmI0(memory mem, immI0 zero)
6006 %{
6007   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6008   match(Set mem (StoreI mem zero));
6009 
6010   ins_cost(125); // XXX
6011   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6012   ins_encode %{
6013     __ movl($mem$$Address, r12);
6014   %}
6015   ins_pipe(ialu_mem_reg);
6016 %}
6017 
6018 instruct storeImmI(memory mem, immI src)
6019 %{
6020   match(Set mem (StoreI mem src));
6021 
6022   ins_cost(150);
6023   format %{ "movl    $mem, $src\t# int" %}
6024   opcode(0xC7); /* C7 /0 */
6025   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6026   ins_pipe(ialu_mem_imm);
6027 %}
6028 
6029 // Store Long Immediate
6030 instruct storeImmL0(memory mem, immL0 zero)
6031 %{
6032   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6033   match(Set mem (StoreL mem zero));
6034 
6035   ins_cost(125); // XXX
6036   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6037   ins_encode %{
6038     __ movq($mem$$Address, r12);
6039   %}
6040   ins_pipe(ialu_mem_reg);
6041 %}
6042 
6043 instruct storeImmL(memory mem, immL32 src)
6044 %{
6045   match(Set mem (StoreL mem src));
6046 
6047   ins_cost(150);
6048   format %{ "movq    $mem, $src\t# long" %}
6049   opcode(0xC7); /* C7 /0 */
6050   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6051   ins_pipe(ialu_mem_imm);
6052 %}
6053 
6054 // Store Short/Char Immediate
6055 instruct storeImmC0(memory mem, immI0 zero)
6056 %{
6057   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6058   match(Set mem (StoreC mem zero));
6059 
6060   ins_cost(125); // XXX
6061   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6062   ins_encode %{
6063     __ movw($mem$$Address, r12);
6064   %}
6065   ins_pipe(ialu_mem_reg);
6066 %}
6067 
6068 instruct storeImmI16(memory mem, immI16 src)
6069 %{
6070   predicate(UseStoreImmI16);
6071   match(Set mem (StoreC mem src));
6072 
6073   ins_cost(150);
6074   format %{ "movw    $mem, $src\t# short/char" %}
6075   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6076   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6077   ins_pipe(ialu_mem_imm);
6078 %}
6079 
6080 // Store Byte Immediate
6081 instruct storeImmB0(memory mem, immI0 zero)
6082 %{
6083   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6084   match(Set mem (StoreB mem zero));
6085 
6086   ins_cost(125); // XXX
6087   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6088   ins_encode %{
6089     __ movb($mem$$Address, r12);
6090   %}
6091   ins_pipe(ialu_mem_reg);
6092 %}
6093 
6094 instruct storeImmB(memory mem, immI8 src)
6095 %{
6096   match(Set mem (StoreB mem src));
6097 
6098   ins_cost(150); // XXX
6099   format %{ "movb    $mem, $src\t# byte" %}
6100   opcode(0xC6); /* C6 /0 */
6101   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6102   ins_pipe(ialu_mem_imm);
6103 %}
6104 
6105 // Store CMS card-mark Immediate
6106 instruct storeImmCM0_reg(memory mem, immI0 zero)
6107 %{
6108   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6109   match(Set mem (StoreCM mem zero));
6110 
6111   ins_cost(125); // XXX
6112   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6113   ins_encode %{
6114     __ movb($mem$$Address, r12);
6115   %}
6116   ins_pipe(ialu_mem_reg);
6117 %}
6118 
6119 instruct storeImmCM0(memory mem, immI0 src)
6120 %{
6121   match(Set mem (StoreCM mem src));
6122 
6123   ins_cost(150); // XXX
6124   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6125   opcode(0xC6); /* C6 /0 */
6126   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6127   ins_pipe(ialu_mem_imm);
6128 %}
6129 
6130 // Store Float
6131 instruct storeF(memory mem, regF src)
6132 %{
6133   match(Set mem (StoreF mem src));
6134 
6135   ins_cost(95); // XXX
6136   format %{ "movss   $mem, $src\t# float" %}
6137   ins_encode %{
6138     __ movflt($mem$$Address, $src$$XMMRegister);
6139   %}
6140   ins_pipe(pipe_slow); // XXX
6141 %}
6142 
6143 // Store immediate Float value (it is faster than store from XMM register)
6144 instruct storeF0(memory mem, immF0 zero)
6145 %{
6146   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6147   match(Set mem (StoreF mem zero));
6148 
6149   ins_cost(25); // XXX
6150   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6151   ins_encode %{
6152     __ movl($mem$$Address, r12);
6153   %}
6154   ins_pipe(ialu_mem_reg);
6155 %}
6156 
6157 instruct storeF_imm(memory mem, immF src)
6158 %{
6159   match(Set mem (StoreF mem src));
6160 
6161   ins_cost(50);
6162   format %{ "movl    $mem, $src\t# float" %}
6163   opcode(0xC7); /* C7 /0 */
6164   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6165   ins_pipe(ialu_mem_imm);
6166 %}
6167 
6168 // Store Double
6169 instruct storeD(memory mem, regD src)
6170 %{
6171   match(Set mem (StoreD mem src));
6172 
6173   ins_cost(95); // XXX
6174   format %{ "movsd   $mem, $src\t# double" %}
6175   ins_encode %{
6176     __ movdbl($mem$$Address, $src$$XMMRegister);
6177   %}
6178   ins_pipe(pipe_slow); // XXX
6179 %}
6180 
6181 // Store immediate double 0.0 (it is faster than store from XMM register)
6182 instruct storeD0_imm(memory mem, immD0 src)
6183 %{
6184   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6185   match(Set mem (StoreD mem src));
6186 
6187   ins_cost(50);
6188   format %{ "movq    $mem, $src\t# double 0." %}
6189   opcode(0xC7); /* C7 /0 */
6190   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6191   ins_pipe(ialu_mem_imm);
6192 %}
6193 
6194 instruct storeD0(memory mem, immD0 zero)
6195 %{
6196   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6197   match(Set mem (StoreD mem zero));
6198 
6199   ins_cost(25); // XXX
6200   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6201   ins_encode %{
6202     __ movq($mem$$Address, r12);
6203   %}
6204   ins_pipe(ialu_mem_reg);
6205 %}
6206 
6207 instruct storeSSI(stackSlotI dst, rRegI src)
6208 %{
6209   match(Set dst src);
6210 
6211   ins_cost(100);
6212   format %{ "movl    $dst, $src\t# int stk" %}
6213   opcode(0x89);
6214   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6215   ins_pipe( ialu_mem_reg );
6216 %}
6217 
6218 instruct storeSSL(stackSlotL dst, rRegL src)
6219 %{
6220   match(Set dst src);
6221 
6222   ins_cost(100);
6223   format %{ "movq    $dst, $src\t# long stk" %}
6224   opcode(0x89);
6225   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6226   ins_pipe(ialu_mem_reg);
6227 %}
6228 
6229 instruct storeSSP(stackSlotP dst, rRegP src)
6230 %{
6231   match(Set dst src);
6232 
6233   ins_cost(100);
6234   format %{ "movq    $dst, $src\t# ptr stk" %}
6235   opcode(0x89);
6236   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6237   ins_pipe(ialu_mem_reg);
6238 %}
6239 
6240 instruct storeSSF(stackSlotF dst, regF src)
6241 %{
6242   match(Set dst src);
6243 
6244   ins_cost(95); // XXX
6245   format %{ "movss   $dst, $src\t# float stk" %}
6246   ins_encode %{
6247     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6248   %}
6249   ins_pipe(pipe_slow); // XXX
6250 %}
6251 
6252 instruct storeSSD(stackSlotD dst, regD src)
6253 %{
6254   match(Set dst src);
6255 
6256   ins_cost(95); // XXX
6257   format %{ "movsd   $dst, $src\t# double stk" %}
6258   ins_encode %{
6259     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6260   %}
6261   ins_pipe(pipe_slow); // XXX
6262 %}
6263 
6264 //----------BSWAP Instructions-------------------------------------------------
6265 instruct bytes_reverse_int(rRegI dst) %{
6266   match(Set dst (ReverseBytesI dst));
6267 
6268   format %{ "bswapl  $dst" %}
6269   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6270   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6271   ins_pipe( ialu_reg );
6272 %}
6273 
6274 instruct bytes_reverse_long(rRegL dst) %{
6275   match(Set dst (ReverseBytesL dst));
6276 
6277   format %{ "bswapq  $dst" %}
6278   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6279   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6280   ins_pipe( ialu_reg);
6281 %}
6282 
6283 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6284   match(Set dst (ReverseBytesUS dst));
6285   effect(KILL cr);
6286 
6287   format %{ "bswapl  $dst\n\t"
6288             "shrl    $dst,16\n\t" %}
6289   ins_encode %{
6290     __ bswapl($dst$$Register);
6291     __ shrl($dst$$Register, 16);
6292   %}
6293   ins_pipe( ialu_reg );
6294 %}
6295 
6296 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6297   match(Set dst (ReverseBytesS dst));
6298   effect(KILL cr);
6299 
6300   format %{ "bswapl  $dst\n\t"
6301             "sar     $dst,16\n\t" %}
6302   ins_encode %{
6303     __ bswapl($dst$$Register);
6304     __ sarl($dst$$Register, 16);
6305   %}
6306   ins_pipe( ialu_reg );
6307 %}
6308 
6309 //---------- Zeros Count Instructions ------------------------------------------
6310 
6311 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6312   predicate(UseCountLeadingZerosInstruction);
6313   match(Set dst (CountLeadingZerosI src));
6314   effect(KILL cr);
6315 
6316   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6317   ins_encode %{
6318     __ lzcntl($dst$$Register, $src$$Register);
6319   %}
6320   ins_pipe(ialu_reg);
6321 %}
6322 
6323 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6324   predicate(!UseCountLeadingZerosInstruction);
6325   match(Set dst (CountLeadingZerosI src));
6326   effect(KILL cr);
6327 
6328   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6329             "jnz     skip\n\t"
6330             "movl    $dst, -1\n"
6331       "skip:\n\t"
6332             "negl    $dst\n\t"
6333             "addl    $dst, 31" %}
6334   ins_encode %{
6335     Register Rdst = $dst$$Register;
6336     Register Rsrc = $src$$Register;
6337     Label skip;
6338     __ bsrl(Rdst, Rsrc);
6339     __ jccb(Assembler::notZero, skip);
6340     __ movl(Rdst, -1);
6341     __ bind(skip);
6342     __ negl(Rdst);
6343     __ addl(Rdst, BitsPerInt - 1);
6344   %}
6345   ins_pipe(ialu_reg);
6346 %}
6347 
6348 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6349   predicate(UseCountLeadingZerosInstruction);
6350   match(Set dst (CountLeadingZerosL src));
6351   effect(KILL cr);
6352 
6353   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6354   ins_encode %{
6355     __ lzcntq($dst$$Register, $src$$Register);
6356   %}
6357   ins_pipe(ialu_reg);
6358 %}
6359 
6360 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6361   predicate(!UseCountLeadingZerosInstruction);
6362   match(Set dst (CountLeadingZerosL src));
6363   effect(KILL cr);
6364 
6365   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6366             "jnz     skip\n\t"
6367             "movl    $dst, -1\n"
6368       "skip:\n\t"
6369             "negl    $dst\n\t"
6370             "addl    $dst, 63" %}
6371   ins_encode %{
6372     Register Rdst = $dst$$Register;
6373     Register Rsrc = $src$$Register;
6374     Label skip;
6375     __ bsrq(Rdst, Rsrc);
6376     __ jccb(Assembler::notZero, skip);
6377     __ movl(Rdst, -1);
6378     __ bind(skip);
6379     __ negl(Rdst);
6380     __ addl(Rdst, BitsPerLong - 1);
6381   %}
6382   ins_pipe(ialu_reg);
6383 %}
6384 
6385 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6386   predicate(UseCountTrailingZerosInstruction);
6387   match(Set dst (CountTrailingZerosI src));
6388   effect(KILL cr);
6389 
6390   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6391   ins_encode %{
6392     __ tzcntl($dst$$Register, $src$$Register);
6393   %}
6394   ins_pipe(ialu_reg);
6395 %}
6396 
6397 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6398   predicate(!UseCountTrailingZerosInstruction);
6399   match(Set dst (CountTrailingZerosI src));
6400   effect(KILL cr);
6401 
6402   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6403             "jnz     done\n\t"
6404             "movl    $dst, 32\n"
6405       "done:" %}
6406   ins_encode %{
6407     Register Rdst = $dst$$Register;
6408     Label done;
6409     __ bsfl(Rdst, $src$$Register);
6410     __ jccb(Assembler::notZero, done);
6411     __ movl(Rdst, BitsPerInt);
6412     __ bind(done);
6413   %}
6414   ins_pipe(ialu_reg);
6415 %}
6416 
6417 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6418   predicate(UseCountTrailingZerosInstruction);
6419   match(Set dst (CountTrailingZerosL src));
6420   effect(KILL cr);
6421 
6422   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6423   ins_encode %{
6424     __ tzcntq($dst$$Register, $src$$Register);
6425   %}
6426   ins_pipe(ialu_reg);
6427 %}
6428 
6429 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6430   predicate(!UseCountTrailingZerosInstruction);
6431   match(Set dst (CountTrailingZerosL src));
6432   effect(KILL cr);
6433 
6434   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6435             "jnz     done\n\t"
6436             "movl    $dst, 64\n"
6437       "done:" %}
6438   ins_encode %{
6439     Register Rdst = $dst$$Register;
6440     Label done;
6441     __ bsfq(Rdst, $src$$Register);
6442     __ jccb(Assembler::notZero, done);
6443     __ movl(Rdst, BitsPerLong);
6444     __ bind(done);
6445   %}
6446   ins_pipe(ialu_reg);
6447 %}
6448 
6449 
6450 //---------- Population Count Instructions -------------------------------------
6451 
6452 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6453   predicate(UsePopCountInstruction);
6454   match(Set dst (PopCountI src));
6455   effect(KILL cr);
6456 
6457   format %{ "popcnt  $dst, $src" %}
6458   ins_encode %{
6459     __ popcntl($dst$$Register, $src$$Register);
6460   %}
6461   ins_pipe(ialu_reg);
6462 %}
6463 
6464 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6465   predicate(UsePopCountInstruction);
6466   match(Set dst (PopCountI (LoadI mem)));
6467   effect(KILL cr);
6468 
6469   format %{ "popcnt  $dst, $mem" %}
6470   ins_encode %{
6471     __ popcntl($dst$$Register, $mem$$Address);
6472   %}
6473   ins_pipe(ialu_reg);
6474 %}
6475 
6476 // Note: Long.bitCount(long) returns an int.
6477 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6478   predicate(UsePopCountInstruction);
6479   match(Set dst (PopCountL src));
6480   effect(KILL cr);
6481 
6482   format %{ "popcnt  $dst, $src" %}
6483   ins_encode %{
6484     __ popcntq($dst$$Register, $src$$Register);
6485   %}
6486   ins_pipe(ialu_reg);
6487 %}
6488 
6489 // Note: Long.bitCount(long) returns an int.
6490 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6491   predicate(UsePopCountInstruction);
6492   match(Set dst (PopCountL (LoadL mem)));
6493   effect(KILL cr);
6494 
6495   format %{ "popcnt  $dst, $mem" %}
6496   ins_encode %{
6497     __ popcntq($dst$$Register, $mem$$Address);
6498   %}
6499   ins_pipe(ialu_reg);
6500 %}
6501 
6502 
6503 //----------MemBar Instructions-----------------------------------------------
6504 // Memory barrier flavors
6505 
6506 instruct membar_acquire()
6507 %{
6508   match(MemBarAcquire);
6509   match(LoadFence);
6510   ins_cost(0);
6511 
6512   size(0);
6513   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6514   ins_encode();
6515   ins_pipe(empty);
6516 %}
6517 
6518 instruct membar_acquire_lock()
6519 %{
6520   match(MemBarAcquireLock);
6521   ins_cost(0);
6522 
6523   size(0);
6524   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6525   ins_encode();
6526   ins_pipe(empty);
6527 %}
6528 
6529 instruct membar_release()
6530 %{
6531   match(MemBarRelease);
6532   match(StoreFence);
6533   ins_cost(0);
6534 
6535   size(0);
6536   format %{ "MEMBAR-release ! (empty encoding)" %}
6537   ins_encode();
6538   ins_pipe(empty);
6539 %}
6540 
6541 instruct membar_release_lock()
6542 %{
6543   match(MemBarReleaseLock);
6544   ins_cost(0);
6545 
6546   size(0);
6547   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6548   ins_encode();
6549   ins_pipe(empty);
6550 %}
6551 
6552 instruct membar_volatile(rFlagsReg cr) %{
6553   match(MemBarVolatile);
6554   effect(KILL cr);
6555   ins_cost(400);
6556 
6557   format %{
6558     $$template
6559     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6560   %}
6561   ins_encode %{
6562     __ membar(Assembler::StoreLoad);
6563   %}
6564   ins_pipe(pipe_slow);
6565 %}
6566 
6567 instruct unnecessary_membar_volatile()
6568 %{
6569   match(MemBarVolatile);
6570   predicate(Matcher::post_store_load_barrier(n));
6571   ins_cost(0);
6572 
6573   size(0);
6574   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6575   ins_encode();
6576   ins_pipe(empty);
6577 %}
6578 
6579 instruct membar_storestore() %{
6580   match(MemBarStoreStore);
6581   ins_cost(0);
6582 
6583   size(0);
6584   format %{ "MEMBAR-storestore (empty encoding)" %}
6585   ins_encode( );
6586   ins_pipe(empty);
6587 %}
6588 
6589 //----------Move Instructions--------------------------------------------------
6590 
6591 instruct castX2P(rRegP dst, rRegL src)
6592 %{
6593   match(Set dst (CastX2P src));
6594 
6595   format %{ "movq    $dst, $src\t# long->ptr" %}
6596   ins_encode %{
6597     if ($dst$$reg != $src$$reg) {
6598       __ movptr($dst$$Register, $src$$Register);
6599     }
6600   %}
6601   ins_pipe(ialu_reg_reg); // XXX
6602 %}
6603 
6604 instruct castP2X(rRegL dst, rRegP src)
6605 %{
6606   match(Set dst (CastP2X src));
6607 
6608   format %{ "movq    $dst, $src\t# ptr -> long" %}
6609   ins_encode %{
6610     if ($dst$$reg != $src$$reg) {
6611       __ movptr($dst$$Register, $src$$Register);
6612     }
6613   %}
6614   ins_pipe(ialu_reg_reg); // XXX
6615 %}
6616 
6617 // Convert oop into int for vectors alignment masking
6618 instruct convP2I(rRegI dst, rRegP src)
6619 %{
6620   match(Set dst (ConvL2I (CastP2X src)));
6621 
6622   format %{ "movl    $dst, $src\t# ptr -> int" %}
6623   ins_encode %{
6624     __ movl($dst$$Register, $src$$Register);
6625   %}
6626   ins_pipe(ialu_reg_reg); // XXX
6627 %}
6628 
6629 // Convert compressed oop into int for vectors alignment masking
6630 // in case of 32bit oops (heap < 4Gb).
6631 instruct convN2I(rRegI dst, rRegN src)
6632 %{
6633   predicate(Universe::narrow_oop_shift() == 0);
6634   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6635 
6636   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6637   ins_encode %{
6638     __ movl($dst$$Register, $src$$Register);
6639   %}
6640   ins_pipe(ialu_reg_reg); // XXX
6641 %}
6642 
6643 // Convert oop pointer into compressed form
6644 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6645   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6646   match(Set dst (EncodeP src));
6647   effect(KILL cr);
6648   format %{ "encode_heap_oop $dst,$src" %}
6649   ins_encode %{
6650     Register s = $src$$Register;
6651     Register d = $dst$$Register;
6652     if (s != d) {
6653       __ movq(d, s);
6654     }
6655     __ encode_heap_oop(d);
6656   %}
6657   ins_pipe(ialu_reg_long);
6658 %}
6659 
6660 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6661   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6662   match(Set dst (EncodeP src));
6663   effect(KILL cr);
6664   format %{ "encode_heap_oop_not_null $dst,$src" %}
6665   ins_encode %{
6666     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6667   %}
6668   ins_pipe(ialu_reg_long);
6669 %}
6670 
6671 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6672   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6673             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6674   match(Set dst (DecodeN src));
6675   effect(KILL cr);
6676   format %{ "decode_heap_oop $dst,$src" %}
6677   ins_encode %{
6678     Register s = $src$$Register;
6679     Register d = $dst$$Register;
6680     if (s != d) {
6681       __ movq(d, s);
6682     }
6683     __ decode_heap_oop(d);
6684   %}
6685   ins_pipe(ialu_reg_long);
6686 %}
6687 
6688 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6689   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6690             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6691   match(Set dst (DecodeN src));
6692   effect(KILL cr);
6693   format %{ "decode_heap_oop_not_null $dst,$src" %}
6694   ins_encode %{
6695     Register s = $src$$Register;
6696     Register d = $dst$$Register;
6697     if (s != d) {
6698       __ decode_heap_oop_not_null(d, s);
6699     } else {
6700       __ decode_heap_oop_not_null(d);
6701     }
6702   %}
6703   ins_pipe(ialu_reg_long);
6704 %}
6705 
6706 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6707   match(Set dst (EncodePKlass src));
6708   effect(KILL cr);
6709   format %{ "encode_klass_not_null $dst,$src" %}
6710   ins_encode %{
6711     __ encode_klass_not_null($dst$$Register, $src$$Register);
6712   %}
6713   ins_pipe(ialu_reg_long);
6714 %}
6715 
6716 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6717   match(Set dst (DecodeNKlass src));
6718   effect(KILL cr);
6719   format %{ "decode_klass_not_null $dst,$src" %}
6720   ins_encode %{
6721     Register s = $src$$Register;
6722     Register d = $dst$$Register;
6723     if (s != d) {
6724       __ decode_klass_not_null(d, s);
6725     } else {
6726       __ decode_klass_not_null(d);
6727     }
6728   %}
6729   ins_pipe(ialu_reg_long);
6730 %}
6731 
6732 
6733 //----------Conditional Move---------------------------------------------------
6734 // Jump
6735 // dummy instruction for generating temp registers
6736 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6737   match(Jump (LShiftL switch_val shift));
6738   ins_cost(350);
6739   predicate(false);
6740   effect(TEMP dest);
6741 
6742   format %{ "leaq    $dest, [$constantaddress]\n\t"
6743             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6744   ins_encode %{
6745     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6746     // to do that and the compiler is using that register as one it can allocate.
6747     // So we build it all by hand.
6748     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6749     // ArrayAddress dispatch(table, index);
6750     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6751     __ lea($dest$$Register, $constantaddress);
6752     __ jmp(dispatch);
6753   %}
6754   ins_pipe(pipe_jmp);
6755 %}
6756 
6757 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6758   match(Jump (AddL (LShiftL switch_val shift) offset));
6759   ins_cost(350);
6760   effect(TEMP dest);
6761 
6762   format %{ "leaq    $dest, [$constantaddress]\n\t"
6763             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6764   ins_encode %{
6765     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6766     // to do that and the compiler is using that register as one it can allocate.
6767     // So we build it all by hand.
6768     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6769     // ArrayAddress dispatch(table, index);
6770     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6771     __ lea($dest$$Register, $constantaddress);
6772     __ jmp(dispatch);
6773   %}
6774   ins_pipe(pipe_jmp);
6775 %}
6776 
6777 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6778   match(Jump switch_val);
6779   ins_cost(350);
6780   effect(TEMP dest);
6781 
6782   format %{ "leaq    $dest, [$constantaddress]\n\t"
6783             "jmp     [$dest + $switch_val]\n\t" %}
6784   ins_encode %{
6785     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6786     // to do that and the compiler is using that register as one it can allocate.
6787     // So we build it all by hand.
6788     // Address index(noreg, switch_reg, Address::times_1);
6789     // ArrayAddress dispatch(table, index);
6790     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6791     __ lea($dest$$Register, $constantaddress);
6792     __ jmp(dispatch);
6793   %}
6794   ins_pipe(pipe_jmp);
6795 %}
6796 
6797 // Conditional move
6798 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6799 %{
6800   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6801 
6802   ins_cost(200); // XXX
6803   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6804   opcode(0x0F, 0x40);
6805   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6806   ins_pipe(pipe_cmov_reg);
6807 %}
6808 
6809 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6810   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6811 
6812   ins_cost(200); // XXX
6813   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6814   opcode(0x0F, 0x40);
6815   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6816   ins_pipe(pipe_cmov_reg);
6817 %}
6818 
6819 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6820   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6821   ins_cost(200);
6822   expand %{
6823     cmovI_regU(cop, cr, dst, src);
6824   %}
6825 %}
6826 
6827 // Conditional move
6828 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6829   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6830 
6831   ins_cost(250); // XXX
6832   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6833   opcode(0x0F, 0x40);
6834   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6835   ins_pipe(pipe_cmov_mem);
6836 %}
6837 
6838 // Conditional move
6839 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6840 %{
6841   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6842 
6843   ins_cost(250); // XXX
6844   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6845   opcode(0x0F, 0x40);
6846   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6847   ins_pipe(pipe_cmov_mem);
6848 %}
6849 
6850 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6851   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6852   ins_cost(250);
6853   expand %{
6854     cmovI_memU(cop, cr, dst, src);
6855   %}
6856 %}
6857 
6858 // Conditional move
6859 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6860 %{
6861   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6862 
6863   ins_cost(200); // XXX
6864   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6865   opcode(0x0F, 0x40);
6866   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6867   ins_pipe(pipe_cmov_reg);
6868 %}
6869 
6870 // Conditional move
6871 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6872 %{
6873   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6874 
6875   ins_cost(200); // XXX
6876   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6877   opcode(0x0F, 0x40);
6878   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6879   ins_pipe(pipe_cmov_reg);
6880 %}
6881 
6882 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6883   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6884   ins_cost(200);
6885   expand %{
6886     cmovN_regU(cop, cr, dst, src);
6887   %}
6888 %}
6889 
6890 // Conditional move
6891 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6892 %{
6893   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6894 
6895   ins_cost(200); // XXX
6896   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6897   opcode(0x0F, 0x40);
6898   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6899   ins_pipe(pipe_cmov_reg);  // XXX
6900 %}
6901 
6902 // Conditional move
6903 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6904 %{
6905   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6906 
6907   ins_cost(200); // XXX
6908   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6909   opcode(0x0F, 0x40);
6910   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6911   ins_pipe(pipe_cmov_reg); // XXX
6912 %}
6913 
6914 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6915   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6916   ins_cost(200);
6917   expand %{
6918     cmovP_regU(cop, cr, dst, src);
6919   %}
6920 %}
6921 
6922 // DISABLED: Requires the ADLC to emit a bottom_type call that
6923 // correctly meets the two pointer arguments; one is an incoming
6924 // register but the other is a memory operand.  ALSO appears to
6925 // be buggy with implicit null checks.
6926 //
6927 //// Conditional move
6928 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6929 //%{
6930 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6931 //  ins_cost(250);
6932 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6933 //  opcode(0x0F,0x40);
6934 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6935 //  ins_pipe( pipe_cmov_mem );
6936 //%}
6937 //
6938 //// Conditional move
6939 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6940 //%{
6941 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6942 //  ins_cost(250);
6943 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6944 //  opcode(0x0F,0x40);
6945 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6946 //  ins_pipe( pipe_cmov_mem );
6947 //%}
6948 
6949 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6950 %{
6951   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6952 
6953   ins_cost(200); // XXX
6954   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6955   opcode(0x0F, 0x40);
6956   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6957   ins_pipe(pipe_cmov_reg);  // XXX
6958 %}
6959 
6960 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6961 %{
6962   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6963 
6964   ins_cost(200); // XXX
6965   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6966   opcode(0x0F, 0x40);
6967   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6968   ins_pipe(pipe_cmov_mem);  // XXX
6969 %}
6970 
6971 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6972 %{
6973   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6974 
6975   ins_cost(200); // XXX
6976   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6977   opcode(0x0F, 0x40);
6978   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6979   ins_pipe(pipe_cmov_reg); // XXX
6980 %}
6981 
6982 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6983   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6984   ins_cost(200);
6985   expand %{
6986     cmovL_regU(cop, cr, dst, src);
6987   %}
6988 %}
6989 
6990 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6991 %{
6992   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6993 
6994   ins_cost(200); // XXX
6995   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6996   opcode(0x0F, 0x40);
6997   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6998   ins_pipe(pipe_cmov_mem); // XXX
6999 %}
7000 
7001 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7002   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7003   ins_cost(200);
7004   expand %{
7005     cmovL_memU(cop, cr, dst, src);
7006   %}
7007 %}
7008 
7009 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7010 %{
7011   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7012 
7013   ins_cost(200); // XXX
7014   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7015             "movss     $dst, $src\n"
7016     "skip:" %}
7017   ins_encode %{
7018     Label Lskip;
7019     // Invert sense of branch from sense of CMOV
7020     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7021     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7022     __ bind(Lskip);
7023   %}
7024   ins_pipe(pipe_slow);
7025 %}
7026 
7027 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7028 // %{
7029 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7030 
7031 //   ins_cost(200); // XXX
7032 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7033 //             "movss     $dst, $src\n"
7034 //     "skip:" %}
7035 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7036 //   ins_pipe(pipe_slow);
7037 // %}
7038 
7039 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7040 %{
7041   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7042 
7043   ins_cost(200); // XXX
7044   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7045             "movss     $dst, $src\n"
7046     "skip:" %}
7047   ins_encode %{
7048     Label Lskip;
7049     // Invert sense of branch from sense of CMOV
7050     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7051     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7052     __ bind(Lskip);
7053   %}
7054   ins_pipe(pipe_slow);
7055 %}
7056 
7057 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7058   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7059   ins_cost(200);
7060   expand %{
7061     cmovF_regU(cop, cr, dst, src);
7062   %}
7063 %}
7064 
7065 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7066 %{
7067   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7068 
7069   ins_cost(200); // XXX
7070   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7071             "movsd     $dst, $src\n"
7072     "skip:" %}
7073   ins_encode %{
7074     Label Lskip;
7075     // Invert sense of branch from sense of CMOV
7076     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7077     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7078     __ bind(Lskip);
7079   %}
7080   ins_pipe(pipe_slow);
7081 %}
7082 
7083 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7084 %{
7085   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7086 
7087   ins_cost(200); // XXX
7088   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7089             "movsd     $dst, $src\n"
7090     "skip:" %}
7091   ins_encode %{
7092     Label Lskip;
7093     // Invert sense of branch from sense of CMOV
7094     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7095     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7096     __ bind(Lskip);
7097   %}
7098   ins_pipe(pipe_slow);
7099 %}
7100 
7101 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7102   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7103   ins_cost(200);
7104   expand %{
7105     cmovD_regU(cop, cr, dst, src);
7106   %}
7107 %}
7108 
7109 //----------Arithmetic Instructions--------------------------------------------
7110 //----------Addition Instructions----------------------------------------------
7111 
7112 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7113 %{
7114   match(Set dst (AddI dst src));
7115   effect(KILL cr);
7116 
7117   format %{ "addl    $dst, $src\t# int" %}
7118   opcode(0x03);
7119   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7120   ins_pipe(ialu_reg_reg);
7121 %}
7122 
7123 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7124 %{
7125   match(Set dst (AddI dst src));
7126   effect(KILL cr);
7127 
7128   format %{ "addl    $dst, $src\t# int" %}
7129   opcode(0x81, 0x00); /* /0 id */
7130   ins_encode(OpcSErm(dst, src), Con8or32(src));
7131   ins_pipe( ialu_reg );
7132 %}
7133 
7134 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7135 %{
7136   match(Set dst (AddI dst (LoadI src)));
7137   effect(KILL cr);
7138 
7139   ins_cost(125); // XXX
7140   format %{ "addl    $dst, $src\t# int" %}
7141   opcode(0x03);
7142   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7143   ins_pipe(ialu_reg_mem);
7144 %}
7145 
7146 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7147 %{
7148   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7149   effect(KILL cr);
7150 
7151   ins_cost(150); // XXX
7152   format %{ "addl    $dst, $src\t# int" %}
7153   opcode(0x01); /* Opcode 01 /r */
7154   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7155   ins_pipe(ialu_mem_reg);
7156 %}
7157 
7158 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7159 %{
7160   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7161   effect(KILL cr);
7162 
7163   ins_cost(125); // XXX
7164   format %{ "addl    $dst, $src\t# int" %}
7165   opcode(0x81); /* Opcode 81 /0 id */
7166   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7167   ins_pipe(ialu_mem_imm);
7168 %}
7169 
7170 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7171 %{
7172   predicate(UseIncDec);
7173   match(Set dst (AddI dst src));
7174   effect(KILL cr);
7175 
7176   format %{ "incl    $dst\t# int" %}
7177   opcode(0xFF, 0x00); // FF /0
7178   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7179   ins_pipe(ialu_reg);
7180 %}
7181 
7182 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7183 %{
7184   predicate(UseIncDec);
7185   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7186   effect(KILL cr);
7187 
7188   ins_cost(125); // XXX
7189   format %{ "incl    $dst\t# int" %}
7190   opcode(0xFF); /* Opcode FF /0 */
7191   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7192   ins_pipe(ialu_mem_imm);
7193 %}
7194 
7195 // XXX why does that use AddI
7196 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7197 %{
7198   predicate(UseIncDec);
7199   match(Set dst (AddI dst src));
7200   effect(KILL cr);
7201 
7202   format %{ "decl    $dst\t# int" %}
7203   opcode(0xFF, 0x01); // FF /1
7204   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7205   ins_pipe(ialu_reg);
7206 %}
7207 
7208 // XXX why does that use AddI
7209 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7210 %{
7211   predicate(UseIncDec);
7212   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7213   effect(KILL cr);
7214 
7215   ins_cost(125); // XXX
7216   format %{ "decl    $dst\t# int" %}
7217   opcode(0xFF); /* Opcode FF /1 */
7218   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7219   ins_pipe(ialu_mem_imm);
7220 %}
7221 
7222 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7223 %{
7224   match(Set dst (AddI src0 src1));
7225 
7226   ins_cost(110);
7227   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7228   opcode(0x8D); /* 0x8D /r */
7229   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7230   ins_pipe(ialu_reg_reg);
7231 %}
7232 
7233 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7234 %{
7235   match(Set dst (AddL dst src));
7236   effect(KILL cr);
7237 
7238   format %{ "addq    $dst, $src\t# long" %}
7239   opcode(0x03);
7240   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7241   ins_pipe(ialu_reg_reg);
7242 %}
7243 
7244 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7245 %{
7246   match(Set dst (AddL dst src));
7247   effect(KILL cr);
7248 
7249   format %{ "addq    $dst, $src\t# long" %}
7250   opcode(0x81, 0x00); /* /0 id */
7251   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7252   ins_pipe( ialu_reg );
7253 %}
7254 
7255 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7256 %{
7257   match(Set dst (AddL dst (LoadL src)));
7258   effect(KILL cr);
7259 
7260   ins_cost(125); // XXX
7261   format %{ "addq    $dst, $src\t# long" %}
7262   opcode(0x03);
7263   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7264   ins_pipe(ialu_reg_mem);
7265 %}
7266 
7267 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7268 %{
7269   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7270   effect(KILL cr);
7271 
7272   ins_cost(150); // XXX
7273   format %{ "addq    $dst, $src\t# long" %}
7274   opcode(0x01); /* Opcode 01 /r */
7275   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7276   ins_pipe(ialu_mem_reg);
7277 %}
7278 
7279 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7280 %{
7281   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7282   effect(KILL cr);
7283 
7284   ins_cost(125); // XXX
7285   format %{ "addq    $dst, $src\t# long" %}
7286   opcode(0x81); /* Opcode 81 /0 id */
7287   ins_encode(REX_mem_wide(dst),
7288              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7289   ins_pipe(ialu_mem_imm);
7290 %}
7291 
7292 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7293 %{
7294   predicate(UseIncDec);
7295   match(Set dst (AddL dst src));
7296   effect(KILL cr);
7297 
7298   format %{ "incq    $dst\t# long" %}
7299   opcode(0xFF, 0x00); // FF /0
7300   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7301   ins_pipe(ialu_reg);
7302 %}
7303 
7304 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7305 %{
7306   predicate(UseIncDec);
7307   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7308   effect(KILL cr);
7309 
7310   ins_cost(125); // XXX
7311   format %{ "incq    $dst\t# long" %}
7312   opcode(0xFF); /* Opcode FF /0 */
7313   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7314   ins_pipe(ialu_mem_imm);
7315 %}
7316 
7317 // XXX why does that use AddL
7318 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7319 %{
7320   predicate(UseIncDec);
7321   match(Set dst (AddL dst src));
7322   effect(KILL cr);
7323 
7324   format %{ "decq    $dst\t# long" %}
7325   opcode(0xFF, 0x01); // FF /1
7326   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7327   ins_pipe(ialu_reg);
7328 %}
7329 
7330 // XXX why does that use AddL
7331 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7332 %{
7333   predicate(UseIncDec);
7334   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7335   effect(KILL cr);
7336 
7337   ins_cost(125); // XXX
7338   format %{ "decq    $dst\t# long" %}
7339   opcode(0xFF); /* Opcode FF /1 */
7340   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7341   ins_pipe(ialu_mem_imm);
7342 %}
7343 
7344 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7345 %{
7346   match(Set dst (AddL src0 src1));
7347 
7348   ins_cost(110);
7349   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7350   opcode(0x8D); /* 0x8D /r */
7351   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7352   ins_pipe(ialu_reg_reg);
7353 %}
7354 
7355 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7356 %{
7357   match(Set dst (AddP dst src));
7358   effect(KILL cr);
7359 
7360   format %{ "addq    $dst, $src\t# ptr" %}
7361   opcode(0x03);
7362   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7363   ins_pipe(ialu_reg_reg);
7364 %}
7365 
7366 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7367 %{
7368   match(Set dst (AddP dst src));
7369   effect(KILL cr);
7370 
7371   format %{ "addq    $dst, $src\t# ptr" %}
7372   opcode(0x81, 0x00); /* /0 id */
7373   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7374   ins_pipe( ialu_reg );
7375 %}
7376 
7377 // XXX addP mem ops ????
7378 
7379 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7380 %{
7381   match(Set dst (AddP src0 src1));
7382 
7383   ins_cost(110);
7384   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7385   opcode(0x8D); /* 0x8D /r */
7386   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7387   ins_pipe(ialu_reg_reg);
7388 %}
7389 
7390 instruct checkCastPP(rRegP dst)
7391 %{
7392   match(Set dst (CheckCastPP dst));
7393 
7394   size(0);
7395   format %{ "# checkcastPP of $dst" %}
7396   ins_encode(/* empty encoding */);
7397   ins_pipe(empty);
7398 %}
7399 
7400 instruct castPP(rRegP dst)
7401 %{
7402   match(Set dst (CastPP dst));
7403 
7404   size(0);
7405   format %{ "# castPP of $dst" %}
7406   ins_encode(/* empty encoding */);
7407   ins_pipe(empty);
7408 %}
7409 
7410 instruct castII(rRegI dst)
7411 %{
7412   match(Set dst (CastII dst));
7413 
7414   size(0);
7415   format %{ "# castII of $dst" %}
7416   ins_encode(/* empty encoding */);
7417   ins_cost(0);
7418   ins_pipe(empty);
7419 %}
7420 
7421 // LoadP-locked same as a regular LoadP when used with compare-swap
7422 instruct loadPLocked(rRegP dst, memory mem)
7423 %{
7424   match(Set dst (LoadPLocked mem));
7425 
7426   ins_cost(125); // XXX
7427   format %{ "movq    $dst, $mem\t# ptr locked" %}
7428   opcode(0x8B);
7429   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7430   ins_pipe(ialu_reg_mem); // XXX
7431 %}
7432 
7433 // Conditional-store of the updated heap-top.
7434 // Used during allocation of the shared heap.
7435 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7436 
7437 instruct storePConditional(memory heap_top_ptr,
7438                            rax_RegP oldval, rRegP newval,
7439                            rFlagsReg cr)
7440 %{
7441   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7442 
7443   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7444             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7445   opcode(0x0F, 0xB1);
7446   ins_encode(lock_prefix,
7447              REX_reg_mem_wide(newval, heap_top_ptr),
7448              OpcP, OpcS,
7449              reg_mem(newval, heap_top_ptr));
7450   ins_pipe(pipe_cmpxchg);
7451 %}
7452 
7453 // Conditional-store of an int value.
7454 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7455 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7456 %{
7457   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7458   effect(KILL oldval);
7459 
7460   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7461   opcode(0x0F, 0xB1);
7462   ins_encode(lock_prefix,
7463              REX_reg_mem(newval, mem),
7464              OpcP, OpcS,
7465              reg_mem(newval, mem));
7466   ins_pipe(pipe_cmpxchg);
7467 %}
7468 
7469 // Conditional-store of a long value.
7470 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7471 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7472 %{
7473   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7474   effect(KILL oldval);
7475 
7476   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7477   opcode(0x0F, 0xB1);
7478   ins_encode(lock_prefix,
7479              REX_reg_mem_wide(newval, mem),
7480              OpcP, OpcS,
7481              reg_mem(newval, mem));
7482   ins_pipe(pipe_cmpxchg);
7483 %}
7484 
7485 
7486 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7487 instruct compareAndSwapP(rRegI res,
7488                          memory mem_ptr,
7489                          rax_RegP oldval, rRegP newval,
7490                          rFlagsReg cr)
7491 %{
7492   predicate(VM_Version::supports_cx8());
7493   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7494   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7495   effect(KILL cr, KILL oldval);
7496 
7497   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7498             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7499             "sete    $res\n\t"
7500             "movzbl  $res, $res" %}
7501   opcode(0x0F, 0xB1);
7502   ins_encode(lock_prefix,
7503              REX_reg_mem_wide(newval, mem_ptr),
7504              OpcP, OpcS,
7505              reg_mem(newval, mem_ptr),
7506              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7507              REX_reg_breg(res, res), // movzbl
7508              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7509   ins_pipe( pipe_cmpxchg );
7510 %}
7511 
7512 instruct compareAndSwapL(rRegI res,
7513                          memory mem_ptr,
7514                          rax_RegL oldval, rRegL newval,
7515                          rFlagsReg cr)
7516 %{
7517   predicate(VM_Version::supports_cx8());
7518   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7519   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7520   effect(KILL cr, KILL oldval);
7521 
7522   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7523             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7524             "sete    $res\n\t"
7525             "movzbl  $res, $res" %}
7526   opcode(0x0F, 0xB1);
7527   ins_encode(lock_prefix,
7528              REX_reg_mem_wide(newval, mem_ptr),
7529              OpcP, OpcS,
7530              reg_mem(newval, mem_ptr),
7531              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7532              REX_reg_breg(res, res), // movzbl
7533              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7534   ins_pipe( pipe_cmpxchg );
7535 %}
7536 
7537 instruct compareAndSwapI(rRegI res,
7538                          memory mem_ptr,
7539                          rax_RegI oldval, rRegI newval,
7540                          rFlagsReg cr)
7541 %{
7542   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7543   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7544   effect(KILL cr, KILL oldval);
7545 
7546   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7547             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7548             "sete    $res\n\t"
7549             "movzbl  $res, $res" %}
7550   opcode(0x0F, 0xB1);
7551   ins_encode(lock_prefix,
7552              REX_reg_mem(newval, mem_ptr),
7553              OpcP, OpcS,
7554              reg_mem(newval, mem_ptr),
7555              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7556              REX_reg_breg(res, res), // movzbl
7557              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7558   ins_pipe( pipe_cmpxchg );
7559 %}
7560 
7561 instruct compareAndSwapB(rRegI res,
7562                          memory mem_ptr,
7563                          rax_RegI oldval, rRegI newval,
7564                          rFlagsReg cr)
7565 %{
7566   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7567   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7568   effect(KILL cr, KILL oldval);
7569 
7570   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7571             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7572             "sete    $res\n\t"
7573             "movzbl  $res, $res" %}
7574   opcode(0x0F, 0xB0);
7575   ins_encode(lock_prefix,
7576              REX_breg_mem(newval, mem_ptr),
7577              OpcP, OpcS,
7578              reg_mem(newval, mem_ptr),
7579              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7580              REX_reg_breg(res, res), // movzbl
7581              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7582   ins_pipe( pipe_cmpxchg );
7583 %}
7584 
7585 instruct compareAndSwapS(rRegI res,
7586                          memory mem_ptr,
7587                          rax_RegI oldval, rRegI newval,
7588                          rFlagsReg cr)
7589 %{
7590   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7591   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7592   effect(KILL cr, KILL oldval);
7593 
7594   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7595             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7596             "sete    $res\n\t"
7597             "movzbl  $res, $res" %}
7598   opcode(0x0F, 0xB1);
7599   ins_encode(lock_prefix,
7600              SizePrefix,
7601              REX_reg_mem(newval, mem_ptr),
7602              OpcP, OpcS,
7603              reg_mem(newval, mem_ptr),
7604              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7605              REX_reg_breg(res, res), // movzbl
7606              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7607   ins_pipe( pipe_cmpxchg );
7608 %}
7609 
7610 instruct compareAndSwapN(rRegI res,
7611                           memory mem_ptr,
7612                           rax_RegN oldval, rRegN newval,
7613                           rFlagsReg cr) %{
7614   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7615   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7616   effect(KILL cr, KILL oldval);
7617 
7618   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7619             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7620             "sete    $res\n\t"
7621             "movzbl  $res, $res" %}
7622   opcode(0x0F, 0xB1);
7623   ins_encode(lock_prefix,
7624              REX_reg_mem(newval, mem_ptr),
7625              OpcP, OpcS,
7626              reg_mem(newval, mem_ptr),
7627              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7628              REX_reg_breg(res, res), // movzbl
7629              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7630   ins_pipe( pipe_cmpxchg );
7631 %}
7632 
7633 instruct compareAndExchangeB(
7634                          memory mem_ptr,
7635                          rax_RegI oldval, rRegI newval,
7636                          rFlagsReg cr)
7637 %{
7638   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7639   effect(KILL cr);
7640 
7641   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7642             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7643   opcode(0x0F, 0xB0);
7644   ins_encode(lock_prefix,
7645              REX_breg_mem(newval, mem_ptr),
7646              OpcP, OpcS,
7647              reg_mem(newval, mem_ptr) // lock cmpxchg
7648              );
7649   ins_pipe( pipe_cmpxchg );
7650 %}
7651 
7652 instruct compareAndExchangeS(
7653                          memory mem_ptr,
7654                          rax_RegI oldval, rRegI newval,
7655                          rFlagsReg cr)
7656 %{
7657   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7658   effect(KILL cr);
7659 
7660   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7661             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7662   opcode(0x0F, 0xB1);
7663   ins_encode(lock_prefix,
7664              SizePrefix,
7665              REX_reg_mem(newval, mem_ptr),
7666              OpcP, OpcS,
7667              reg_mem(newval, mem_ptr) // lock cmpxchg
7668              );
7669   ins_pipe( pipe_cmpxchg );
7670 %}
7671 
7672 instruct compareAndExchangeI(
7673                          memory mem_ptr,
7674                          rax_RegI oldval, rRegI newval,
7675                          rFlagsReg cr)
7676 %{
7677   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7678   effect(KILL cr);
7679 
7680   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7681             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7682   opcode(0x0F, 0xB1);
7683   ins_encode(lock_prefix,
7684              REX_reg_mem(newval, mem_ptr),
7685              OpcP, OpcS,
7686              reg_mem(newval, mem_ptr) // lock cmpxchg
7687              );
7688   ins_pipe( pipe_cmpxchg );
7689 %}
7690 
7691 instruct compareAndExchangeL(
7692                          memory mem_ptr,
7693                          rax_RegL oldval, rRegL newval,
7694                          rFlagsReg cr)
7695 %{
7696   predicate(VM_Version::supports_cx8());
7697   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7698   effect(KILL cr);
7699 
7700   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7701             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7702   opcode(0x0F, 0xB1);
7703   ins_encode(lock_prefix,
7704              REX_reg_mem_wide(newval, mem_ptr),
7705              OpcP, OpcS,
7706              reg_mem(newval, mem_ptr)  // lock cmpxchg
7707             );
7708   ins_pipe( pipe_cmpxchg );
7709 %}
7710 
7711 instruct compareAndExchangeN(
7712                           memory mem_ptr,
7713                           rax_RegN oldval, rRegN newval,
7714                           rFlagsReg cr) %{
7715   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7716   effect(KILL cr);
7717 
7718   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7719             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7720   opcode(0x0F, 0xB1);
7721   ins_encode(lock_prefix,
7722              REX_reg_mem(newval, mem_ptr),
7723              OpcP, OpcS,
7724              reg_mem(newval, mem_ptr)  // lock cmpxchg
7725           );
7726   ins_pipe( pipe_cmpxchg );
7727 %}
7728 
7729 instruct compareAndExchangeP(
7730                          memory mem_ptr,
7731                          rax_RegP oldval, rRegP newval,
7732                          rFlagsReg cr)
7733 %{
7734   predicate(VM_Version::supports_cx8());
7735   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7736   effect(KILL cr);
7737 
7738   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7739             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7740   opcode(0x0F, 0xB1);
7741   ins_encode(lock_prefix,
7742              REX_reg_mem_wide(newval, mem_ptr),
7743              OpcP, OpcS,
7744              reg_mem(newval, mem_ptr)  // lock cmpxchg
7745           );
7746   ins_pipe( pipe_cmpxchg );
7747 %}
7748 
7749 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7750   predicate(n->as_LoadStore()->result_not_used());
7751   match(Set dummy (GetAndAddB mem add));
7752   effect(KILL cr);
7753   format %{ "ADDB  [$mem],$add" %}
7754   ins_encode %{
7755     __ lock();
7756     __ addb($mem$$Address, $add$$constant);
7757   %}
7758   ins_pipe( pipe_cmpxchg );
7759 %}
7760 
7761 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7762   match(Set newval (GetAndAddB mem newval));
7763   effect(KILL cr);
7764   format %{ "XADDB  [$mem],$newval" %}
7765   ins_encode %{
7766     __ lock();
7767     __ xaddb($mem$$Address, $newval$$Register);
7768   %}
7769   ins_pipe( pipe_cmpxchg );
7770 %}
7771 
7772 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7773   predicate(n->as_LoadStore()->result_not_used());
7774   match(Set dummy (GetAndAddS mem add));
7775   effect(KILL cr);
7776   format %{ "ADDW  [$mem],$add" %}
7777   ins_encode %{
7778     __ lock();
7779     __ addw($mem$$Address, $add$$constant);
7780   %}
7781   ins_pipe( pipe_cmpxchg );
7782 %}
7783 
7784 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7785   match(Set newval (GetAndAddS mem newval));
7786   effect(KILL cr);
7787   format %{ "XADDW  [$mem],$newval" %}
7788   ins_encode %{
7789     __ lock();
7790     __ xaddw($mem$$Address, $newval$$Register);
7791   %}
7792   ins_pipe( pipe_cmpxchg );
7793 %}
7794 
7795 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7796   predicate(n->as_LoadStore()->result_not_used());
7797   match(Set dummy (GetAndAddI mem add));
7798   effect(KILL cr);
7799   format %{ "ADDL  [$mem],$add" %}
7800   ins_encode %{
7801     __ lock();
7802     __ addl($mem$$Address, $add$$constant);
7803   %}
7804   ins_pipe( pipe_cmpxchg );
7805 %}
7806 
7807 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7808   match(Set newval (GetAndAddI mem newval));
7809   effect(KILL cr);
7810   format %{ "XADDL  [$mem],$newval" %}
7811   ins_encode %{
7812     __ lock();
7813     __ xaddl($mem$$Address, $newval$$Register);
7814   %}
7815   ins_pipe( pipe_cmpxchg );
7816 %}
7817 
7818 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7819   predicate(n->as_LoadStore()->result_not_used());
7820   match(Set dummy (GetAndAddL mem add));
7821   effect(KILL cr);
7822   format %{ "ADDQ  [$mem],$add" %}
7823   ins_encode %{
7824     __ lock();
7825     __ addq($mem$$Address, $add$$constant);
7826   %}
7827   ins_pipe( pipe_cmpxchg );
7828 %}
7829 
7830 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7831   match(Set newval (GetAndAddL mem newval));
7832   effect(KILL cr);
7833   format %{ "XADDQ  [$mem],$newval" %}
7834   ins_encode %{
7835     __ lock();
7836     __ xaddq($mem$$Address, $newval$$Register);
7837   %}
7838   ins_pipe( pipe_cmpxchg );
7839 %}
7840 
7841 instruct xchgB( memory mem, rRegI newval) %{
7842   match(Set newval (GetAndSetB mem newval));
7843   format %{ "XCHGB  $newval,[$mem]" %}
7844   ins_encode %{
7845     __ xchgb($newval$$Register, $mem$$Address);
7846   %}
7847   ins_pipe( pipe_cmpxchg );
7848 %}
7849 
7850 instruct xchgS( memory mem, rRegI newval) %{
7851   match(Set newval (GetAndSetS mem newval));
7852   format %{ "XCHGW  $newval,[$mem]" %}
7853   ins_encode %{
7854     __ xchgw($newval$$Register, $mem$$Address);
7855   %}
7856   ins_pipe( pipe_cmpxchg );
7857 %}
7858 
7859 instruct xchgI( memory mem, rRegI newval) %{
7860   match(Set newval (GetAndSetI mem newval));
7861   format %{ "XCHGL  $newval,[$mem]" %}
7862   ins_encode %{
7863     __ xchgl($newval$$Register, $mem$$Address);
7864   %}
7865   ins_pipe( pipe_cmpxchg );
7866 %}
7867 
7868 instruct xchgL( memory mem, rRegL newval) %{
7869   match(Set newval (GetAndSetL mem newval));
7870   format %{ "XCHGL  $newval,[$mem]" %}
7871   ins_encode %{
7872     __ xchgq($newval$$Register, $mem$$Address);
7873   %}
7874   ins_pipe( pipe_cmpxchg );
7875 %}
7876 
7877 instruct xchgP( memory mem, rRegP newval) %{
7878   match(Set newval (GetAndSetP mem newval));
7879   format %{ "XCHGQ  $newval,[$mem]" %}
7880   ins_encode %{
7881     __ xchgq($newval$$Register, $mem$$Address);
7882   %}
7883   ins_pipe( pipe_cmpxchg );
7884 %}
7885 
7886 instruct xchgN( memory mem, rRegN newval) %{
7887   match(Set newval (GetAndSetN mem newval));
7888   format %{ "XCHGL  $newval,$mem]" %}
7889   ins_encode %{
7890     __ xchgl($newval$$Register, $mem$$Address);
7891   %}
7892   ins_pipe( pipe_cmpxchg );
7893 %}
7894 
7895 //----------Subtraction Instructions-------------------------------------------
7896 
7897 // Integer Subtraction Instructions
7898 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7899 %{
7900   match(Set dst (SubI dst src));
7901   effect(KILL cr);
7902 
7903   format %{ "subl    $dst, $src\t# int" %}
7904   opcode(0x2B);
7905   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7906   ins_pipe(ialu_reg_reg);
7907 %}
7908 
7909 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7910 %{
7911   match(Set dst (SubI dst src));
7912   effect(KILL cr);
7913 
7914   format %{ "subl    $dst, $src\t# int" %}
7915   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7916   ins_encode(OpcSErm(dst, src), Con8or32(src));
7917   ins_pipe(ialu_reg);
7918 %}
7919 
7920 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7921 %{
7922   match(Set dst (SubI dst (LoadI src)));
7923   effect(KILL cr);
7924 
7925   ins_cost(125);
7926   format %{ "subl    $dst, $src\t# int" %}
7927   opcode(0x2B);
7928   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7929   ins_pipe(ialu_reg_mem);
7930 %}
7931 
7932 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7933 %{
7934   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7935   effect(KILL cr);
7936 
7937   ins_cost(150);
7938   format %{ "subl    $dst, $src\t# int" %}
7939   opcode(0x29); /* Opcode 29 /r */
7940   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7941   ins_pipe(ialu_mem_reg);
7942 %}
7943 
7944 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7945 %{
7946   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7947   effect(KILL cr);
7948 
7949   ins_cost(125); // XXX
7950   format %{ "subl    $dst, $src\t# int" %}
7951   opcode(0x81); /* Opcode 81 /5 id */
7952   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7953   ins_pipe(ialu_mem_imm);
7954 %}
7955 
7956 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7957 %{
7958   match(Set dst (SubL dst src));
7959   effect(KILL cr);
7960 
7961   format %{ "subq    $dst, $src\t# long" %}
7962   opcode(0x2B);
7963   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7964   ins_pipe(ialu_reg_reg);
7965 %}
7966 
7967 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7968 %{
7969   match(Set dst (SubL dst src));
7970   effect(KILL cr);
7971 
7972   format %{ "subq    $dst, $src\t# long" %}
7973   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7974   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7975   ins_pipe(ialu_reg);
7976 %}
7977 
7978 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7979 %{
7980   match(Set dst (SubL dst (LoadL src)));
7981   effect(KILL cr);
7982 
7983   ins_cost(125);
7984   format %{ "subq    $dst, $src\t# long" %}
7985   opcode(0x2B);
7986   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7987   ins_pipe(ialu_reg_mem);
7988 %}
7989 
7990 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7991 %{
7992   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7993   effect(KILL cr);
7994 
7995   ins_cost(150);
7996   format %{ "subq    $dst, $src\t# long" %}
7997   opcode(0x29); /* Opcode 29 /r */
7998   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7999   ins_pipe(ialu_mem_reg);
8000 %}
8001 
8002 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8003 %{
8004   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8005   effect(KILL cr);
8006 
8007   ins_cost(125); // XXX
8008   format %{ "subq    $dst, $src\t# long" %}
8009   opcode(0x81); /* Opcode 81 /5 id */
8010   ins_encode(REX_mem_wide(dst),
8011              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8012   ins_pipe(ialu_mem_imm);
8013 %}
8014 
8015 // Subtract from a pointer
8016 // XXX hmpf???
8017 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8018 %{
8019   match(Set dst (AddP dst (SubI zero src)));
8020   effect(KILL cr);
8021 
8022   format %{ "subq    $dst, $src\t# ptr - int" %}
8023   opcode(0x2B);
8024   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8025   ins_pipe(ialu_reg_reg);
8026 %}
8027 
8028 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8029 %{
8030   match(Set dst (SubI zero dst));
8031   effect(KILL cr);
8032 
8033   format %{ "negl    $dst\t# int" %}
8034   opcode(0xF7, 0x03);  // Opcode F7 /3
8035   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8036   ins_pipe(ialu_reg);
8037 %}
8038 
8039 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8040 %{
8041   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8042   effect(KILL cr);
8043 
8044   format %{ "negl    $dst\t# int" %}
8045   opcode(0xF7, 0x03);  // Opcode F7 /3
8046   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8047   ins_pipe(ialu_reg);
8048 %}
8049 
8050 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8051 %{
8052   match(Set dst (SubL zero dst));
8053   effect(KILL cr);
8054 
8055   format %{ "negq    $dst\t# long" %}
8056   opcode(0xF7, 0x03);  // Opcode F7 /3
8057   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8058   ins_pipe(ialu_reg);
8059 %}
8060 
8061 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8062 %{
8063   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8064   effect(KILL cr);
8065 
8066   format %{ "negq    $dst\t# long" %}
8067   opcode(0xF7, 0x03);  // Opcode F7 /3
8068   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8069   ins_pipe(ialu_reg);
8070 %}
8071 
8072 //----------Multiplication/Division Instructions-------------------------------
8073 // Integer Multiplication Instructions
8074 // Multiply Register
8075 
8076 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8077 %{
8078   match(Set dst (MulI dst src));
8079   effect(KILL cr);
8080 
8081   ins_cost(300);
8082   format %{ "imull   $dst, $src\t# int" %}
8083   opcode(0x0F, 0xAF);
8084   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8085   ins_pipe(ialu_reg_reg_alu0);
8086 %}
8087 
8088 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8089 %{
8090   match(Set dst (MulI src imm));
8091   effect(KILL cr);
8092 
8093   ins_cost(300);
8094   format %{ "imull   $dst, $src, $imm\t# int" %}
8095   opcode(0x69); /* 69 /r id */
8096   ins_encode(REX_reg_reg(dst, src),
8097              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8098   ins_pipe(ialu_reg_reg_alu0);
8099 %}
8100 
8101 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8102 %{
8103   match(Set dst (MulI dst (LoadI src)));
8104   effect(KILL cr);
8105 
8106   ins_cost(350);
8107   format %{ "imull   $dst, $src\t# int" %}
8108   opcode(0x0F, 0xAF);
8109   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8110   ins_pipe(ialu_reg_mem_alu0);
8111 %}
8112 
8113 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8114 %{
8115   match(Set dst (MulI (LoadI src) imm));
8116   effect(KILL cr);
8117 
8118   ins_cost(300);
8119   format %{ "imull   $dst, $src, $imm\t# int" %}
8120   opcode(0x69); /* 69 /r id */
8121   ins_encode(REX_reg_mem(dst, src),
8122              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8123   ins_pipe(ialu_reg_mem_alu0);
8124 %}
8125 
8126 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8127 %{
8128   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8129   effect(KILL cr, KILL src2);
8130 
8131   expand %{ mulI_rReg(dst, src1, cr);
8132            mulI_rReg(src2, src3, cr);
8133            addI_rReg(dst, src2, cr); %}
8134 %}
8135 
8136 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8137 %{
8138   match(Set dst (MulL dst src));
8139   effect(KILL cr);
8140 
8141   ins_cost(300);
8142   format %{ "imulq   $dst, $src\t# long" %}
8143   opcode(0x0F, 0xAF);
8144   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8145   ins_pipe(ialu_reg_reg_alu0);
8146 %}
8147 
8148 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8149 %{
8150   match(Set dst (MulL src imm));
8151   effect(KILL cr);
8152 
8153   ins_cost(300);
8154   format %{ "imulq   $dst, $src, $imm\t# long" %}
8155   opcode(0x69); /* 69 /r id */
8156   ins_encode(REX_reg_reg_wide(dst, src),
8157              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8158   ins_pipe(ialu_reg_reg_alu0);
8159 %}
8160 
8161 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8162 %{
8163   match(Set dst (MulL dst (LoadL src)));
8164   effect(KILL cr);
8165 
8166   ins_cost(350);
8167   format %{ "imulq   $dst, $src\t# long" %}
8168   opcode(0x0F, 0xAF);
8169   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8170   ins_pipe(ialu_reg_mem_alu0);
8171 %}
8172 
8173 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8174 %{
8175   match(Set dst (MulL (LoadL src) imm));
8176   effect(KILL cr);
8177 
8178   ins_cost(300);
8179   format %{ "imulq   $dst, $src, $imm\t# long" %}
8180   opcode(0x69); /* 69 /r id */
8181   ins_encode(REX_reg_mem_wide(dst, src),
8182              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8183   ins_pipe(ialu_reg_mem_alu0);
8184 %}
8185 
8186 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8187 %{
8188   match(Set dst (MulHiL src rax));
8189   effect(USE_KILL rax, KILL cr);
8190 
8191   ins_cost(300);
8192   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8193   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8194   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8195   ins_pipe(ialu_reg_reg_alu0);
8196 %}
8197 
8198 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8199                    rFlagsReg cr)
8200 %{
8201   match(Set rax (DivI rax div));
8202   effect(KILL rdx, KILL cr);
8203 
8204   ins_cost(30*100+10*100); // XXX
8205   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8206             "jne,s   normal\n\t"
8207             "xorl    rdx, rdx\n\t"
8208             "cmpl    $div, -1\n\t"
8209             "je,s    done\n"
8210     "normal: cdql\n\t"
8211             "idivl   $div\n"
8212     "done:"        %}
8213   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8214   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8215   ins_pipe(ialu_reg_reg_alu0);
8216 %}
8217 
8218 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8219                    rFlagsReg cr)
8220 %{
8221   match(Set rax (DivL rax div));
8222   effect(KILL rdx, KILL cr);
8223 
8224   ins_cost(30*100+10*100); // XXX
8225   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8226             "cmpq    rax, rdx\n\t"
8227             "jne,s   normal\n\t"
8228             "xorl    rdx, rdx\n\t"
8229             "cmpq    $div, -1\n\t"
8230             "je,s    done\n"
8231     "normal: cdqq\n\t"
8232             "idivq   $div\n"
8233     "done:"        %}
8234   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8235   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8236   ins_pipe(ialu_reg_reg_alu0);
8237 %}
8238 
8239 // Integer DIVMOD with Register, both quotient and mod results
8240 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8241                              rFlagsReg cr)
8242 %{
8243   match(DivModI rax div);
8244   effect(KILL cr);
8245 
8246   ins_cost(30*100+10*100); // XXX
8247   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8248             "jne,s   normal\n\t"
8249             "xorl    rdx, rdx\n\t"
8250             "cmpl    $div, -1\n\t"
8251             "je,s    done\n"
8252     "normal: cdql\n\t"
8253             "idivl   $div\n"
8254     "done:"        %}
8255   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8256   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8257   ins_pipe(pipe_slow);
8258 %}
8259 
8260 // Long DIVMOD with Register, both quotient and mod results
8261 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8262                              rFlagsReg cr)
8263 %{
8264   match(DivModL rax div);
8265   effect(KILL cr);
8266 
8267   ins_cost(30*100+10*100); // XXX
8268   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8269             "cmpq    rax, rdx\n\t"
8270             "jne,s   normal\n\t"
8271             "xorl    rdx, rdx\n\t"
8272             "cmpq    $div, -1\n\t"
8273             "je,s    done\n"
8274     "normal: cdqq\n\t"
8275             "idivq   $div\n"
8276     "done:"        %}
8277   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8278   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8279   ins_pipe(pipe_slow);
8280 %}
8281 
8282 //----------- DivL-By-Constant-Expansions--------------------------------------
8283 // DivI cases are handled by the compiler
8284 
8285 // Magic constant, reciprocal of 10
8286 instruct loadConL_0x6666666666666667(rRegL dst)
8287 %{
8288   effect(DEF dst);
8289 
8290   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8291   ins_encode(load_immL(dst, 0x6666666666666667));
8292   ins_pipe(ialu_reg);
8293 %}
8294 
8295 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8296 %{
8297   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8298 
8299   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8300   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8301   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8302   ins_pipe(ialu_reg_reg_alu0);
8303 %}
8304 
8305 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8306 %{
8307   effect(USE_DEF dst, KILL cr);
8308 
8309   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8310   opcode(0xC1, 0x7); /* C1 /7 ib */
8311   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8312   ins_pipe(ialu_reg);
8313 %}
8314 
8315 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8316 %{
8317   effect(USE_DEF dst, KILL cr);
8318 
8319   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8320   opcode(0xC1, 0x7); /* C1 /7 ib */
8321   ins_encode(reg_opc_imm_wide(dst, 0x2));
8322   ins_pipe(ialu_reg);
8323 %}
8324 
8325 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8326 %{
8327   match(Set dst (DivL src div));
8328 
8329   ins_cost((5+8)*100);
8330   expand %{
8331     rax_RegL rax;                     // Killed temp
8332     rFlagsReg cr;                     // Killed
8333     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8334     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8335     sarL_rReg_63(src, cr);            // sarq  src, 63
8336     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8337     subL_rReg(dst, src, cr);          // subl  rdx, src
8338   %}
8339 %}
8340 
8341 //-----------------------------------------------------------------------------
8342 
8343 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8344                    rFlagsReg cr)
8345 %{
8346   match(Set rdx (ModI rax div));
8347   effect(KILL rax, KILL cr);
8348 
8349   ins_cost(300); // XXX
8350   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8351             "jne,s   normal\n\t"
8352             "xorl    rdx, rdx\n\t"
8353             "cmpl    $div, -1\n\t"
8354             "je,s    done\n"
8355     "normal: cdql\n\t"
8356             "idivl   $div\n"
8357     "done:"        %}
8358   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8359   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8360   ins_pipe(ialu_reg_reg_alu0);
8361 %}
8362 
8363 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8364                    rFlagsReg cr)
8365 %{
8366   match(Set rdx (ModL rax div));
8367   effect(KILL rax, KILL cr);
8368 
8369   ins_cost(300); // XXX
8370   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8371             "cmpq    rax, rdx\n\t"
8372             "jne,s   normal\n\t"
8373             "xorl    rdx, rdx\n\t"
8374             "cmpq    $div, -1\n\t"
8375             "je,s    done\n"
8376     "normal: cdqq\n\t"
8377             "idivq   $div\n"
8378     "done:"        %}
8379   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8380   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8381   ins_pipe(ialu_reg_reg_alu0);
8382 %}
8383 
8384 // Integer Shift Instructions
8385 // Shift Left by one
8386 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8387 %{
8388   match(Set dst (LShiftI dst shift));
8389   effect(KILL cr);
8390 
8391   format %{ "sall    $dst, $shift" %}
8392   opcode(0xD1, 0x4); /* D1 /4 */
8393   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8394   ins_pipe(ialu_reg);
8395 %}
8396 
8397 // Shift Left by one
8398 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8399 %{
8400   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8401   effect(KILL cr);
8402 
8403   format %{ "sall    $dst, $shift\t" %}
8404   opcode(0xD1, 0x4); /* D1 /4 */
8405   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8406   ins_pipe(ialu_mem_imm);
8407 %}
8408 
8409 // Shift Left by 8-bit immediate
8410 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8411 %{
8412   match(Set dst (LShiftI dst shift));
8413   effect(KILL cr);
8414 
8415   format %{ "sall    $dst, $shift" %}
8416   opcode(0xC1, 0x4); /* C1 /4 ib */
8417   ins_encode(reg_opc_imm(dst, shift));
8418   ins_pipe(ialu_reg);
8419 %}
8420 
8421 // Shift Left by 8-bit immediate
8422 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8423 %{
8424   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8425   effect(KILL cr);
8426 
8427   format %{ "sall    $dst, $shift" %}
8428   opcode(0xC1, 0x4); /* C1 /4 ib */
8429   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8430   ins_pipe(ialu_mem_imm);
8431 %}
8432 
8433 // Shift Left by variable
8434 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8435 %{
8436   match(Set dst (LShiftI dst shift));
8437   effect(KILL cr);
8438 
8439   format %{ "sall    $dst, $shift" %}
8440   opcode(0xD3, 0x4); /* D3 /4 */
8441   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8442   ins_pipe(ialu_reg_reg);
8443 %}
8444 
8445 // Shift Left by variable
8446 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8447 %{
8448   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8449   effect(KILL cr);
8450 
8451   format %{ "sall    $dst, $shift" %}
8452   opcode(0xD3, 0x4); /* D3 /4 */
8453   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8454   ins_pipe(ialu_mem_reg);
8455 %}
8456 
8457 // Arithmetic shift right by one
8458 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8459 %{
8460   match(Set dst (RShiftI dst shift));
8461   effect(KILL cr);
8462 
8463   format %{ "sarl    $dst, $shift" %}
8464   opcode(0xD1, 0x7); /* D1 /7 */
8465   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8466   ins_pipe(ialu_reg);
8467 %}
8468 
8469 // Arithmetic shift right by one
8470 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8471 %{
8472   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8473   effect(KILL cr);
8474 
8475   format %{ "sarl    $dst, $shift" %}
8476   opcode(0xD1, 0x7); /* D1 /7 */
8477   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8478   ins_pipe(ialu_mem_imm);
8479 %}
8480 
8481 // Arithmetic Shift Right by 8-bit immediate
8482 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8483 %{
8484   match(Set dst (RShiftI dst shift));
8485   effect(KILL cr);
8486 
8487   format %{ "sarl    $dst, $shift" %}
8488   opcode(0xC1, 0x7); /* C1 /7 ib */
8489   ins_encode(reg_opc_imm(dst, shift));
8490   ins_pipe(ialu_mem_imm);
8491 %}
8492 
8493 // Arithmetic Shift Right by 8-bit immediate
8494 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8495 %{
8496   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8497   effect(KILL cr);
8498 
8499   format %{ "sarl    $dst, $shift" %}
8500   opcode(0xC1, 0x7); /* C1 /7 ib */
8501   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8502   ins_pipe(ialu_mem_imm);
8503 %}
8504 
8505 // Arithmetic Shift Right by variable
8506 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8507 %{
8508   match(Set dst (RShiftI dst shift));
8509   effect(KILL cr);
8510 
8511   format %{ "sarl    $dst, $shift" %}
8512   opcode(0xD3, 0x7); /* D3 /7 */
8513   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8514   ins_pipe(ialu_reg_reg);
8515 %}
8516 
8517 // Arithmetic Shift Right by variable
8518 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8519 %{
8520   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8521   effect(KILL cr);
8522 
8523   format %{ "sarl    $dst, $shift" %}
8524   opcode(0xD3, 0x7); /* D3 /7 */
8525   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8526   ins_pipe(ialu_mem_reg);
8527 %}
8528 
8529 // Logical shift right by one
8530 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8531 %{
8532   match(Set dst (URShiftI dst shift));
8533   effect(KILL cr);
8534 
8535   format %{ "shrl    $dst, $shift" %}
8536   opcode(0xD1, 0x5); /* D1 /5 */
8537   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8538   ins_pipe(ialu_reg);
8539 %}
8540 
8541 // Logical shift right by one
8542 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8543 %{
8544   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8545   effect(KILL cr);
8546 
8547   format %{ "shrl    $dst, $shift" %}
8548   opcode(0xD1, 0x5); /* D1 /5 */
8549   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8550   ins_pipe(ialu_mem_imm);
8551 %}
8552 
8553 // Logical Shift Right by 8-bit immediate
8554 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8555 %{
8556   match(Set dst (URShiftI dst shift));
8557   effect(KILL cr);
8558 
8559   format %{ "shrl    $dst, $shift" %}
8560   opcode(0xC1, 0x5); /* C1 /5 ib */
8561   ins_encode(reg_opc_imm(dst, shift));
8562   ins_pipe(ialu_reg);
8563 %}
8564 
8565 // Logical Shift Right by 8-bit immediate
8566 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8567 %{
8568   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8569   effect(KILL cr);
8570 
8571   format %{ "shrl    $dst, $shift" %}
8572   opcode(0xC1, 0x5); /* C1 /5 ib */
8573   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8574   ins_pipe(ialu_mem_imm);
8575 %}
8576 
8577 // Logical Shift Right by variable
8578 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8579 %{
8580   match(Set dst (URShiftI dst shift));
8581   effect(KILL cr);
8582 
8583   format %{ "shrl    $dst, $shift" %}
8584   opcode(0xD3, 0x5); /* D3 /5 */
8585   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8586   ins_pipe(ialu_reg_reg);
8587 %}
8588 
8589 // Logical Shift Right by variable
8590 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8591 %{
8592   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8593   effect(KILL cr);
8594 
8595   format %{ "shrl    $dst, $shift" %}
8596   opcode(0xD3, 0x5); /* D3 /5 */
8597   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8598   ins_pipe(ialu_mem_reg);
8599 %}
8600 
8601 // Long Shift Instructions
8602 // Shift Left by one
8603 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8604 %{
8605   match(Set dst (LShiftL dst shift));
8606   effect(KILL cr);
8607 
8608   format %{ "salq    $dst, $shift" %}
8609   opcode(0xD1, 0x4); /* D1 /4 */
8610   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8611   ins_pipe(ialu_reg);
8612 %}
8613 
8614 // Shift Left by one
8615 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8616 %{
8617   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8618   effect(KILL cr);
8619 
8620   format %{ "salq    $dst, $shift" %}
8621   opcode(0xD1, 0x4); /* D1 /4 */
8622   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8623   ins_pipe(ialu_mem_imm);
8624 %}
8625 
8626 // Shift Left by 8-bit immediate
8627 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8628 %{
8629   match(Set dst (LShiftL dst shift));
8630   effect(KILL cr);
8631 
8632   format %{ "salq    $dst, $shift" %}
8633   opcode(0xC1, 0x4); /* C1 /4 ib */
8634   ins_encode(reg_opc_imm_wide(dst, shift));
8635   ins_pipe(ialu_reg);
8636 %}
8637 
8638 // Shift Left by 8-bit immediate
8639 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8640 %{
8641   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8642   effect(KILL cr);
8643 
8644   format %{ "salq    $dst, $shift" %}
8645   opcode(0xC1, 0x4); /* C1 /4 ib */
8646   ins_encode(REX_mem_wide(dst), OpcP,
8647              RM_opc_mem(secondary, dst), Con8or32(shift));
8648   ins_pipe(ialu_mem_imm);
8649 %}
8650 
8651 // Shift Left by variable
8652 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8653 %{
8654   match(Set dst (LShiftL dst shift));
8655   effect(KILL cr);
8656 
8657   format %{ "salq    $dst, $shift" %}
8658   opcode(0xD3, 0x4); /* D3 /4 */
8659   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8660   ins_pipe(ialu_reg_reg);
8661 %}
8662 
8663 // Shift Left by variable
8664 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8665 %{
8666   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8667   effect(KILL cr);
8668 
8669   format %{ "salq    $dst, $shift" %}
8670   opcode(0xD3, 0x4); /* D3 /4 */
8671   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8672   ins_pipe(ialu_mem_reg);
8673 %}
8674 
8675 // Arithmetic shift right by one
8676 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8677 %{
8678   match(Set dst (RShiftL dst shift));
8679   effect(KILL cr);
8680 
8681   format %{ "sarq    $dst, $shift" %}
8682   opcode(0xD1, 0x7); /* D1 /7 */
8683   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8684   ins_pipe(ialu_reg);
8685 %}
8686 
8687 // Arithmetic shift right by one
8688 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8689 %{
8690   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8691   effect(KILL cr);
8692 
8693   format %{ "sarq    $dst, $shift" %}
8694   opcode(0xD1, 0x7); /* D1 /7 */
8695   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8696   ins_pipe(ialu_mem_imm);
8697 %}
8698 
8699 // Arithmetic Shift Right by 8-bit immediate
8700 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8701 %{
8702   match(Set dst (RShiftL dst shift));
8703   effect(KILL cr);
8704 
8705   format %{ "sarq    $dst, $shift" %}
8706   opcode(0xC1, 0x7); /* C1 /7 ib */
8707   ins_encode(reg_opc_imm_wide(dst, shift));
8708   ins_pipe(ialu_mem_imm);
8709 %}
8710 
8711 // Arithmetic Shift Right by 8-bit immediate
8712 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8713 %{
8714   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8715   effect(KILL cr);
8716 
8717   format %{ "sarq    $dst, $shift" %}
8718   opcode(0xC1, 0x7); /* C1 /7 ib */
8719   ins_encode(REX_mem_wide(dst), OpcP,
8720              RM_opc_mem(secondary, dst), Con8or32(shift));
8721   ins_pipe(ialu_mem_imm);
8722 %}
8723 
8724 // Arithmetic Shift Right by variable
8725 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8726 %{
8727   match(Set dst (RShiftL dst shift));
8728   effect(KILL cr);
8729 
8730   format %{ "sarq    $dst, $shift" %}
8731   opcode(0xD3, 0x7); /* D3 /7 */
8732   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8733   ins_pipe(ialu_reg_reg);
8734 %}
8735 
8736 // Arithmetic Shift Right by variable
8737 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8738 %{
8739   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8740   effect(KILL cr);
8741 
8742   format %{ "sarq    $dst, $shift" %}
8743   opcode(0xD3, 0x7); /* D3 /7 */
8744   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8745   ins_pipe(ialu_mem_reg);
8746 %}
8747 
8748 // Logical shift right by one
8749 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8750 %{
8751   match(Set dst (URShiftL dst shift));
8752   effect(KILL cr);
8753 
8754   format %{ "shrq    $dst, $shift" %}
8755   opcode(0xD1, 0x5); /* D1 /5 */
8756   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8757   ins_pipe(ialu_reg);
8758 %}
8759 
8760 // Logical shift right by one
8761 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8762 %{
8763   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8764   effect(KILL cr);
8765 
8766   format %{ "shrq    $dst, $shift" %}
8767   opcode(0xD1, 0x5); /* D1 /5 */
8768   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8769   ins_pipe(ialu_mem_imm);
8770 %}
8771 
8772 // Logical Shift Right by 8-bit immediate
8773 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8774 %{
8775   match(Set dst (URShiftL dst shift));
8776   effect(KILL cr);
8777 
8778   format %{ "shrq    $dst, $shift" %}
8779   opcode(0xC1, 0x5); /* C1 /5 ib */
8780   ins_encode(reg_opc_imm_wide(dst, shift));
8781   ins_pipe(ialu_reg);
8782 %}
8783 
8784 
8785 // Logical Shift Right by 8-bit immediate
8786 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8787 %{
8788   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8789   effect(KILL cr);
8790 
8791   format %{ "shrq    $dst, $shift" %}
8792   opcode(0xC1, 0x5); /* C1 /5 ib */
8793   ins_encode(REX_mem_wide(dst), OpcP,
8794              RM_opc_mem(secondary, dst), Con8or32(shift));
8795   ins_pipe(ialu_mem_imm);
8796 %}
8797 
8798 // Logical Shift Right by variable
8799 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8800 %{
8801   match(Set dst (URShiftL dst shift));
8802   effect(KILL cr);
8803 
8804   format %{ "shrq    $dst, $shift" %}
8805   opcode(0xD3, 0x5); /* D3 /5 */
8806   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8807   ins_pipe(ialu_reg_reg);
8808 %}
8809 
8810 // Logical Shift Right by variable
8811 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8812 %{
8813   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8814   effect(KILL cr);
8815 
8816   format %{ "shrq    $dst, $shift" %}
8817   opcode(0xD3, 0x5); /* D3 /5 */
8818   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8819   ins_pipe(ialu_mem_reg);
8820 %}
8821 
8822 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8823 // This idiom is used by the compiler for the i2b bytecode.
8824 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8825 %{
8826   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8827 
8828   format %{ "movsbl  $dst, $src\t# i2b" %}
8829   opcode(0x0F, 0xBE);
8830   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8831   ins_pipe(ialu_reg_reg);
8832 %}
8833 
8834 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8835 // This idiom is used by the compiler the i2s bytecode.
8836 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8837 %{
8838   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8839 
8840   format %{ "movswl  $dst, $src\t# i2s" %}
8841   opcode(0x0F, 0xBF);
8842   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8843   ins_pipe(ialu_reg_reg);
8844 %}
8845 
8846 // ROL/ROR instructions
8847 
8848 // ROL expand
8849 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8850   effect(KILL cr, USE_DEF dst);
8851 
8852   format %{ "roll    $dst" %}
8853   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8854   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8855   ins_pipe(ialu_reg);
8856 %}
8857 
8858 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8859   effect(USE_DEF dst, USE shift, KILL cr);
8860 
8861   format %{ "roll    $dst, $shift" %}
8862   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8863   ins_encode( reg_opc_imm(dst, shift) );
8864   ins_pipe(ialu_reg);
8865 %}
8866 
8867 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8868 %{
8869   effect(USE_DEF dst, USE shift, KILL cr);
8870 
8871   format %{ "roll    $dst, $shift" %}
8872   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8873   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8874   ins_pipe(ialu_reg_reg);
8875 %}
8876 // end of ROL expand
8877 
8878 // Rotate Left by one
8879 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8880 %{
8881   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8882 
8883   expand %{
8884     rolI_rReg_imm1(dst, cr);
8885   %}
8886 %}
8887 
8888 // Rotate Left by 8-bit immediate
8889 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8890 %{
8891   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8892   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8893 
8894   expand %{
8895     rolI_rReg_imm8(dst, lshift, cr);
8896   %}
8897 %}
8898 
8899 // Rotate Left by variable
8900 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8901 %{
8902   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8903 
8904   expand %{
8905     rolI_rReg_CL(dst, shift, cr);
8906   %}
8907 %}
8908 
8909 // Rotate Left by variable
8910 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8911 %{
8912   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8913 
8914   expand %{
8915     rolI_rReg_CL(dst, shift, cr);
8916   %}
8917 %}
8918 
8919 // ROR expand
8920 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8921 %{
8922   effect(USE_DEF dst, KILL cr);
8923 
8924   format %{ "rorl    $dst" %}
8925   opcode(0xD1, 0x1); /* D1 /1 */
8926   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8927   ins_pipe(ialu_reg);
8928 %}
8929 
8930 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8931 %{
8932   effect(USE_DEF dst, USE shift, KILL cr);
8933 
8934   format %{ "rorl    $dst, $shift" %}
8935   opcode(0xC1, 0x1); /* C1 /1 ib */
8936   ins_encode(reg_opc_imm(dst, shift));
8937   ins_pipe(ialu_reg);
8938 %}
8939 
8940 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8941 %{
8942   effect(USE_DEF dst, USE shift, KILL cr);
8943 
8944   format %{ "rorl    $dst, $shift" %}
8945   opcode(0xD3, 0x1); /* D3 /1 */
8946   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8947   ins_pipe(ialu_reg_reg);
8948 %}
8949 // end of ROR expand
8950 
8951 // Rotate Right by one
8952 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8953 %{
8954   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8955 
8956   expand %{
8957     rorI_rReg_imm1(dst, cr);
8958   %}
8959 %}
8960 
8961 // Rotate Right by 8-bit immediate
8962 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8963 %{
8964   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8965   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8966 
8967   expand %{
8968     rorI_rReg_imm8(dst, rshift, cr);
8969   %}
8970 %}
8971 
8972 // Rotate Right by variable
8973 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8974 %{
8975   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8976 
8977   expand %{
8978     rorI_rReg_CL(dst, shift, cr);
8979   %}
8980 %}
8981 
8982 // Rotate Right by variable
8983 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8984 %{
8985   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8986 
8987   expand %{
8988     rorI_rReg_CL(dst, shift, cr);
8989   %}
8990 %}
8991 
8992 // for long rotate
8993 // ROL expand
8994 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8995   effect(USE_DEF dst, KILL cr);
8996 
8997   format %{ "rolq    $dst" %}
8998   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8999   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9000   ins_pipe(ialu_reg);
9001 %}
9002 
9003 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9004   effect(USE_DEF dst, USE shift, KILL cr);
9005 
9006   format %{ "rolq    $dst, $shift" %}
9007   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9008   ins_encode( reg_opc_imm_wide(dst, shift) );
9009   ins_pipe(ialu_reg);
9010 %}
9011 
9012 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9013 %{
9014   effect(USE_DEF dst, USE shift, KILL cr);
9015 
9016   format %{ "rolq    $dst, $shift" %}
9017   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9018   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9019   ins_pipe(ialu_reg_reg);
9020 %}
9021 // end of ROL expand
9022 
9023 // Rotate Left by one
9024 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9025 %{
9026   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9027 
9028   expand %{
9029     rolL_rReg_imm1(dst, cr);
9030   %}
9031 %}
9032 
9033 // Rotate Left by 8-bit immediate
9034 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9035 %{
9036   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9037   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9038 
9039   expand %{
9040     rolL_rReg_imm8(dst, lshift, cr);
9041   %}
9042 %}
9043 
9044 // Rotate Left by variable
9045 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9046 %{
9047   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9048 
9049   expand %{
9050     rolL_rReg_CL(dst, shift, cr);
9051   %}
9052 %}
9053 
9054 // Rotate Left by variable
9055 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9056 %{
9057   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9058 
9059   expand %{
9060     rolL_rReg_CL(dst, shift, cr);
9061   %}
9062 %}
9063 
9064 // ROR expand
9065 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9066 %{
9067   effect(USE_DEF dst, KILL cr);
9068 
9069   format %{ "rorq    $dst" %}
9070   opcode(0xD1, 0x1); /* D1 /1 */
9071   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9072   ins_pipe(ialu_reg);
9073 %}
9074 
9075 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9076 %{
9077   effect(USE_DEF dst, USE shift, KILL cr);
9078 
9079   format %{ "rorq    $dst, $shift" %}
9080   opcode(0xC1, 0x1); /* C1 /1 ib */
9081   ins_encode(reg_opc_imm_wide(dst, shift));
9082   ins_pipe(ialu_reg);
9083 %}
9084 
9085 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9086 %{
9087   effect(USE_DEF dst, USE shift, KILL cr);
9088 
9089   format %{ "rorq    $dst, $shift" %}
9090   opcode(0xD3, 0x1); /* D3 /1 */
9091   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9092   ins_pipe(ialu_reg_reg);
9093 %}
9094 // end of ROR expand
9095 
9096 // Rotate Right by one
9097 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9098 %{
9099   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9100 
9101   expand %{
9102     rorL_rReg_imm1(dst, cr);
9103   %}
9104 %}
9105 
9106 // Rotate Right by 8-bit immediate
9107 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9108 %{
9109   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9110   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9111 
9112   expand %{
9113     rorL_rReg_imm8(dst, rshift, cr);
9114   %}
9115 %}
9116 
9117 // Rotate Right by variable
9118 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9119 %{
9120   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9121 
9122   expand %{
9123     rorL_rReg_CL(dst, shift, cr);
9124   %}
9125 %}
9126 
9127 // Rotate Right by variable
9128 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9129 %{
9130   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9131 
9132   expand %{
9133     rorL_rReg_CL(dst, shift, cr);
9134   %}
9135 %}
9136 
9137 // Logical Instructions
9138 
9139 // Integer Logical Instructions
9140 
9141 // And Instructions
9142 // And Register with Register
9143 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9144 %{
9145   match(Set dst (AndI dst src));
9146   effect(KILL cr);
9147 
9148   format %{ "andl    $dst, $src\t# int" %}
9149   opcode(0x23);
9150   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9151   ins_pipe(ialu_reg_reg);
9152 %}
9153 
9154 // And Register with Immediate 255
9155 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9156 %{
9157   match(Set dst (AndI dst src));
9158 
9159   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9160   opcode(0x0F, 0xB6);
9161   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9162   ins_pipe(ialu_reg);
9163 %}
9164 
9165 // And Register with Immediate 255 and promote to long
9166 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9167 %{
9168   match(Set dst (ConvI2L (AndI src mask)));
9169 
9170   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9171   opcode(0x0F, 0xB6);
9172   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9173   ins_pipe(ialu_reg);
9174 %}
9175 
9176 // And Register with Immediate 65535
9177 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9178 %{
9179   match(Set dst (AndI dst src));
9180 
9181   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9182   opcode(0x0F, 0xB7);
9183   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9184   ins_pipe(ialu_reg);
9185 %}
9186 
9187 // And Register with Immediate 65535 and promote to long
9188 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9189 %{
9190   match(Set dst (ConvI2L (AndI src mask)));
9191 
9192   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9193   opcode(0x0F, 0xB7);
9194   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9195   ins_pipe(ialu_reg);
9196 %}
9197 
9198 // And Register with Immediate
9199 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9200 %{
9201   match(Set dst (AndI dst src));
9202   effect(KILL cr);
9203 
9204   format %{ "andl    $dst, $src\t# int" %}
9205   opcode(0x81, 0x04); /* Opcode 81 /4 */
9206   ins_encode(OpcSErm(dst, src), Con8or32(src));
9207   ins_pipe(ialu_reg);
9208 %}
9209 
9210 // And Register with Memory
9211 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9212 %{
9213   match(Set dst (AndI dst (LoadI src)));
9214   effect(KILL cr);
9215 
9216   ins_cost(125);
9217   format %{ "andl    $dst, $src\t# int" %}
9218   opcode(0x23);
9219   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9220   ins_pipe(ialu_reg_mem);
9221 %}
9222 
9223 // And Memory with Register
9224 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9225 %{
9226   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9227   effect(KILL cr);
9228 
9229   ins_cost(150);
9230   format %{ "andb    $dst, $src\t# byte" %}
9231   opcode(0x20);
9232   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9233   ins_pipe(ialu_mem_reg);
9234 %}
9235 
9236 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9237 %{
9238   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9239   effect(KILL cr);
9240 
9241   ins_cost(150);
9242   format %{ "andl    $dst, $src\t# int" %}
9243   opcode(0x21); /* Opcode 21 /r */
9244   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9245   ins_pipe(ialu_mem_reg);
9246 %}
9247 
9248 // And Memory with Immediate
9249 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9250 %{
9251   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9252   effect(KILL cr);
9253 
9254   ins_cost(125);
9255   format %{ "andl    $dst, $src\t# int" %}
9256   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9257   ins_encode(REX_mem(dst), OpcSE(src),
9258              RM_opc_mem(secondary, dst), Con8or32(src));
9259   ins_pipe(ialu_mem_imm);
9260 %}
9261 
9262 // BMI1 instructions
9263 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9264   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9265   predicate(UseBMI1Instructions);
9266   effect(KILL cr);
9267 
9268   ins_cost(125);
9269   format %{ "andnl  $dst, $src1, $src2" %}
9270 
9271   ins_encode %{
9272     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9273   %}
9274   ins_pipe(ialu_reg_mem);
9275 %}
9276 
9277 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9278   match(Set dst (AndI (XorI src1 minus_1) src2));
9279   predicate(UseBMI1Instructions);
9280   effect(KILL cr);
9281 
9282   format %{ "andnl  $dst, $src1, $src2" %}
9283 
9284   ins_encode %{
9285     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9286   %}
9287   ins_pipe(ialu_reg);
9288 %}
9289 
9290 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9291   match(Set dst (AndI (SubI imm_zero src) src));
9292   predicate(UseBMI1Instructions);
9293   effect(KILL cr);
9294 
9295   format %{ "blsil  $dst, $src" %}
9296 
9297   ins_encode %{
9298     __ blsil($dst$$Register, $src$$Register);
9299   %}
9300   ins_pipe(ialu_reg);
9301 %}
9302 
9303 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9304   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9305   predicate(UseBMI1Instructions);
9306   effect(KILL cr);
9307 
9308   ins_cost(125);
9309   format %{ "blsil  $dst, $src" %}
9310 
9311   ins_encode %{
9312     __ blsil($dst$$Register, $src$$Address);
9313   %}
9314   ins_pipe(ialu_reg_mem);
9315 %}
9316 
9317 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9318 %{
9319   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9320   predicate(UseBMI1Instructions);
9321   effect(KILL cr);
9322 
9323   ins_cost(125);
9324   format %{ "blsmskl $dst, $src" %}
9325 
9326   ins_encode %{
9327     __ blsmskl($dst$$Register, $src$$Address);
9328   %}
9329   ins_pipe(ialu_reg_mem);
9330 %}
9331 
9332 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9333 %{
9334   match(Set dst (XorI (AddI src minus_1) src));
9335   predicate(UseBMI1Instructions);
9336   effect(KILL cr);
9337 
9338   format %{ "blsmskl $dst, $src" %}
9339 
9340   ins_encode %{
9341     __ blsmskl($dst$$Register, $src$$Register);
9342   %}
9343 
9344   ins_pipe(ialu_reg);
9345 %}
9346 
9347 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9348 %{
9349   match(Set dst (AndI (AddI src minus_1) src) );
9350   predicate(UseBMI1Instructions);
9351   effect(KILL cr);
9352 
9353   format %{ "blsrl  $dst, $src" %}
9354 
9355   ins_encode %{
9356     __ blsrl($dst$$Register, $src$$Register);
9357   %}
9358 
9359   ins_pipe(ialu_reg_mem);
9360 %}
9361 
9362 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9363 %{
9364   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9365   predicate(UseBMI1Instructions);
9366   effect(KILL cr);
9367 
9368   ins_cost(125);
9369   format %{ "blsrl  $dst, $src" %}
9370 
9371   ins_encode %{
9372     __ blsrl($dst$$Register, $src$$Address);
9373   %}
9374 
9375   ins_pipe(ialu_reg);
9376 %}
9377 
9378 // Or Instructions
9379 // Or Register with Register
9380 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9381 %{
9382   match(Set dst (OrI dst src));
9383   effect(KILL cr);
9384 
9385   format %{ "orl     $dst, $src\t# int" %}
9386   opcode(0x0B);
9387   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9388   ins_pipe(ialu_reg_reg);
9389 %}
9390 
9391 // Or Register with Immediate
9392 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9393 %{
9394   match(Set dst (OrI dst src));
9395   effect(KILL cr);
9396 
9397   format %{ "orl     $dst, $src\t# int" %}
9398   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9399   ins_encode(OpcSErm(dst, src), Con8or32(src));
9400   ins_pipe(ialu_reg);
9401 %}
9402 
9403 // Or Register with Memory
9404 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9405 %{
9406   match(Set dst (OrI dst (LoadI src)));
9407   effect(KILL cr);
9408 
9409   ins_cost(125);
9410   format %{ "orl     $dst, $src\t# int" %}
9411   opcode(0x0B);
9412   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9413   ins_pipe(ialu_reg_mem);
9414 %}
9415 
9416 // Or Memory with Register
9417 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9418 %{
9419   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9420   effect(KILL cr);
9421 
9422   ins_cost(150);
9423   format %{ "orb    $dst, $src\t# byte" %}
9424   opcode(0x08);
9425   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9426   ins_pipe(ialu_mem_reg);
9427 %}
9428 
9429 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9430 %{
9431   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9432   effect(KILL cr);
9433 
9434   ins_cost(150);
9435   format %{ "orl     $dst, $src\t# int" %}
9436   opcode(0x09); /* Opcode 09 /r */
9437   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9438   ins_pipe(ialu_mem_reg);
9439 %}
9440 
9441 // Or Memory with Immediate
9442 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9443 %{
9444   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9445   effect(KILL cr);
9446 
9447   ins_cost(125);
9448   format %{ "orl     $dst, $src\t# int" %}
9449   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9450   ins_encode(REX_mem(dst), OpcSE(src),
9451              RM_opc_mem(secondary, dst), Con8or32(src));
9452   ins_pipe(ialu_mem_imm);
9453 %}
9454 
9455 // Xor Instructions
9456 // Xor Register with Register
9457 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9458 %{
9459   match(Set dst (XorI dst src));
9460   effect(KILL cr);
9461 
9462   format %{ "xorl    $dst, $src\t# int" %}
9463   opcode(0x33);
9464   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9465   ins_pipe(ialu_reg_reg);
9466 %}
9467 
9468 // Xor Register with Immediate -1
9469 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9470   match(Set dst (XorI dst imm));
9471 
9472   format %{ "not    $dst" %}
9473   ins_encode %{
9474      __ notl($dst$$Register);
9475   %}
9476   ins_pipe(ialu_reg);
9477 %}
9478 
9479 // Xor Register with Immediate
9480 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9481 %{
9482   match(Set dst (XorI dst src));
9483   effect(KILL cr);
9484 
9485   format %{ "xorl    $dst, $src\t# int" %}
9486   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9487   ins_encode(OpcSErm(dst, src), Con8or32(src));
9488   ins_pipe(ialu_reg);
9489 %}
9490 
9491 // Xor Register with Memory
9492 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9493 %{
9494   match(Set dst (XorI dst (LoadI src)));
9495   effect(KILL cr);
9496 
9497   ins_cost(125);
9498   format %{ "xorl    $dst, $src\t# int" %}
9499   opcode(0x33);
9500   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9501   ins_pipe(ialu_reg_mem);
9502 %}
9503 
9504 // Xor Memory with Register
9505 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9506 %{
9507   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9508   effect(KILL cr);
9509 
9510   ins_cost(150);
9511   format %{ "xorb    $dst, $src\t# byte" %}
9512   opcode(0x30);
9513   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9514   ins_pipe(ialu_mem_reg);
9515 %}
9516 
9517 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9518 %{
9519   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9520   effect(KILL cr);
9521 
9522   ins_cost(150);
9523   format %{ "xorl    $dst, $src\t# int" %}
9524   opcode(0x31); /* Opcode 31 /r */
9525   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9526   ins_pipe(ialu_mem_reg);
9527 %}
9528 
9529 // Xor Memory with Immediate
9530 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9531 %{
9532   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9533   effect(KILL cr);
9534 
9535   ins_cost(125);
9536   format %{ "xorl    $dst, $src\t# int" %}
9537   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9538   ins_encode(REX_mem(dst), OpcSE(src),
9539              RM_opc_mem(secondary, dst), Con8or32(src));
9540   ins_pipe(ialu_mem_imm);
9541 %}
9542 
9543 
9544 // Long Logical Instructions
9545 
9546 // And Instructions
9547 // And Register with Register
9548 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9549 %{
9550   match(Set dst (AndL dst src));
9551   effect(KILL cr);
9552 
9553   format %{ "andq    $dst, $src\t# long" %}
9554   opcode(0x23);
9555   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9556   ins_pipe(ialu_reg_reg);
9557 %}
9558 
9559 // And Register with Immediate 255
9560 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9561 %{
9562   match(Set dst (AndL dst src));
9563 
9564   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9565   opcode(0x0F, 0xB6);
9566   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9567   ins_pipe(ialu_reg);
9568 %}
9569 
9570 // And Register with Immediate 65535
9571 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9572 %{
9573   match(Set dst (AndL dst src));
9574 
9575   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9576   opcode(0x0F, 0xB7);
9577   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9578   ins_pipe(ialu_reg);
9579 %}
9580 
9581 // And Register with Immediate
9582 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9583 %{
9584   match(Set dst (AndL dst src));
9585   effect(KILL cr);
9586 
9587   format %{ "andq    $dst, $src\t# long" %}
9588   opcode(0x81, 0x04); /* Opcode 81 /4 */
9589   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9590   ins_pipe(ialu_reg);
9591 %}
9592 
9593 // And Register with Memory
9594 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9595 %{
9596   match(Set dst (AndL dst (LoadL src)));
9597   effect(KILL cr);
9598 
9599   ins_cost(125);
9600   format %{ "andq    $dst, $src\t# long" %}
9601   opcode(0x23);
9602   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9603   ins_pipe(ialu_reg_mem);
9604 %}
9605 
9606 // And Memory with Register
9607 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9608 %{
9609   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9610   effect(KILL cr);
9611 
9612   ins_cost(150);
9613   format %{ "andq    $dst, $src\t# long" %}
9614   opcode(0x21); /* Opcode 21 /r */
9615   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9616   ins_pipe(ialu_mem_reg);
9617 %}
9618 
9619 // And Memory with Immediate
9620 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9621 %{
9622   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9623   effect(KILL cr);
9624 
9625   ins_cost(125);
9626   format %{ "andq    $dst, $src\t# long" %}
9627   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9628   ins_encode(REX_mem_wide(dst), OpcSE(src),
9629              RM_opc_mem(secondary, dst), Con8or32(src));
9630   ins_pipe(ialu_mem_imm);
9631 %}
9632 
9633 // BMI1 instructions
9634 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9635   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9636   predicate(UseBMI1Instructions);
9637   effect(KILL cr);
9638 
9639   ins_cost(125);
9640   format %{ "andnq  $dst, $src1, $src2" %}
9641 
9642   ins_encode %{
9643     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9644   %}
9645   ins_pipe(ialu_reg_mem);
9646 %}
9647 
9648 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9649   match(Set dst (AndL (XorL src1 minus_1) src2));
9650   predicate(UseBMI1Instructions);
9651   effect(KILL cr);
9652 
9653   format %{ "andnq  $dst, $src1, $src2" %}
9654 
9655   ins_encode %{
9656   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9657   %}
9658   ins_pipe(ialu_reg_mem);
9659 %}
9660 
9661 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9662   match(Set dst (AndL (SubL imm_zero src) src));
9663   predicate(UseBMI1Instructions);
9664   effect(KILL cr);
9665 
9666   format %{ "blsiq  $dst, $src" %}
9667 
9668   ins_encode %{
9669     __ blsiq($dst$$Register, $src$$Register);
9670   %}
9671   ins_pipe(ialu_reg);
9672 %}
9673 
9674 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9675   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9676   predicate(UseBMI1Instructions);
9677   effect(KILL cr);
9678 
9679   ins_cost(125);
9680   format %{ "blsiq  $dst, $src" %}
9681 
9682   ins_encode %{
9683     __ blsiq($dst$$Register, $src$$Address);
9684   %}
9685   ins_pipe(ialu_reg_mem);
9686 %}
9687 
9688 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9689 %{
9690   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9691   predicate(UseBMI1Instructions);
9692   effect(KILL cr);
9693 
9694   ins_cost(125);
9695   format %{ "blsmskq $dst, $src" %}
9696 
9697   ins_encode %{
9698     __ blsmskq($dst$$Register, $src$$Address);
9699   %}
9700   ins_pipe(ialu_reg_mem);
9701 %}
9702 
9703 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9704 %{
9705   match(Set dst (XorL (AddL src minus_1) src));
9706   predicate(UseBMI1Instructions);
9707   effect(KILL cr);
9708 
9709   format %{ "blsmskq $dst, $src" %}
9710 
9711   ins_encode %{
9712     __ blsmskq($dst$$Register, $src$$Register);
9713   %}
9714 
9715   ins_pipe(ialu_reg);
9716 %}
9717 
9718 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9719 %{
9720   match(Set dst (AndL (AddL src minus_1) src) );
9721   predicate(UseBMI1Instructions);
9722   effect(KILL cr);
9723 
9724   format %{ "blsrq  $dst, $src" %}
9725 
9726   ins_encode %{
9727     __ blsrq($dst$$Register, $src$$Register);
9728   %}
9729 
9730   ins_pipe(ialu_reg);
9731 %}
9732 
9733 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9734 %{
9735   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9736   predicate(UseBMI1Instructions);
9737   effect(KILL cr);
9738 
9739   ins_cost(125);
9740   format %{ "blsrq  $dst, $src" %}
9741 
9742   ins_encode %{
9743     __ blsrq($dst$$Register, $src$$Address);
9744   %}
9745 
9746   ins_pipe(ialu_reg);
9747 %}
9748 
9749 // Or Instructions
9750 // Or Register with Register
9751 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9752 %{
9753   match(Set dst (OrL dst src));
9754   effect(KILL cr);
9755 
9756   format %{ "orq     $dst, $src\t# long" %}
9757   opcode(0x0B);
9758   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9759   ins_pipe(ialu_reg_reg);
9760 %}
9761 
9762 // Use any_RegP to match R15 (TLS register) without spilling.
9763 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9764   match(Set dst (OrL dst (CastP2X src)));
9765   effect(KILL cr);
9766 
9767   format %{ "orq     $dst, $src\t# long" %}
9768   opcode(0x0B);
9769   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9770   ins_pipe(ialu_reg_reg);
9771 %}
9772 
9773 
9774 // Or Register with Immediate
9775 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9776 %{
9777   match(Set dst (OrL dst src));
9778   effect(KILL cr);
9779 
9780   format %{ "orq     $dst, $src\t# long" %}
9781   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9782   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9783   ins_pipe(ialu_reg);
9784 %}
9785 
9786 // Or Register with Memory
9787 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9788 %{
9789   match(Set dst (OrL dst (LoadL src)));
9790   effect(KILL cr);
9791 
9792   ins_cost(125);
9793   format %{ "orq     $dst, $src\t# long" %}
9794   opcode(0x0B);
9795   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9796   ins_pipe(ialu_reg_mem);
9797 %}
9798 
9799 // Or Memory with Register
9800 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9801 %{
9802   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9803   effect(KILL cr);
9804 
9805   ins_cost(150);
9806   format %{ "orq     $dst, $src\t# long" %}
9807   opcode(0x09); /* Opcode 09 /r */
9808   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9809   ins_pipe(ialu_mem_reg);
9810 %}
9811 
9812 // Or Memory with Immediate
9813 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9814 %{
9815   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9816   effect(KILL cr);
9817 
9818   ins_cost(125);
9819   format %{ "orq     $dst, $src\t# long" %}
9820   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9821   ins_encode(REX_mem_wide(dst), OpcSE(src),
9822              RM_opc_mem(secondary, dst), Con8or32(src));
9823   ins_pipe(ialu_mem_imm);
9824 %}
9825 
9826 // Xor Instructions
9827 // Xor Register with Register
9828 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9829 %{
9830   match(Set dst (XorL dst src));
9831   effect(KILL cr);
9832 
9833   format %{ "xorq    $dst, $src\t# long" %}
9834   opcode(0x33);
9835   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9836   ins_pipe(ialu_reg_reg);
9837 %}
9838 
9839 // Xor Register with Immediate -1
9840 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9841   match(Set dst (XorL dst imm));
9842 
9843   format %{ "notq   $dst" %}
9844   ins_encode %{
9845      __ notq($dst$$Register);
9846   %}
9847   ins_pipe(ialu_reg);
9848 %}
9849 
9850 // Xor Register with Immediate
9851 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9852 %{
9853   match(Set dst (XorL dst src));
9854   effect(KILL cr);
9855 
9856   format %{ "xorq    $dst, $src\t# long" %}
9857   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9858   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9859   ins_pipe(ialu_reg);
9860 %}
9861 
9862 // Xor Register with Memory
9863 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9864 %{
9865   match(Set dst (XorL dst (LoadL src)));
9866   effect(KILL cr);
9867 
9868   ins_cost(125);
9869   format %{ "xorq    $dst, $src\t# long" %}
9870   opcode(0x33);
9871   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9872   ins_pipe(ialu_reg_mem);
9873 %}
9874 
9875 // Xor Memory with Register
9876 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9877 %{
9878   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9879   effect(KILL cr);
9880 
9881   ins_cost(150);
9882   format %{ "xorq    $dst, $src\t# long" %}
9883   opcode(0x31); /* Opcode 31 /r */
9884   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9885   ins_pipe(ialu_mem_reg);
9886 %}
9887 
9888 // Xor Memory with Immediate
9889 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9890 %{
9891   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9892   effect(KILL cr);
9893 
9894   ins_cost(125);
9895   format %{ "xorq    $dst, $src\t# long" %}
9896   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9897   ins_encode(REX_mem_wide(dst), OpcSE(src),
9898              RM_opc_mem(secondary, dst), Con8or32(src));
9899   ins_pipe(ialu_mem_imm);
9900 %}
9901 
9902 // Convert Int to Boolean
9903 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9904 %{
9905   match(Set dst (Conv2B src));
9906   effect(KILL cr);
9907 
9908   format %{ "testl   $src, $src\t# ci2b\n\t"
9909             "setnz   $dst\n\t"
9910             "movzbl  $dst, $dst" %}
9911   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9912              setNZ_reg(dst),
9913              REX_reg_breg(dst, dst), // movzbl
9914              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9915   ins_pipe(pipe_slow); // XXX
9916 %}
9917 
9918 // Convert Pointer to Boolean
9919 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9920 %{
9921   match(Set dst (Conv2B src));
9922   effect(KILL cr);
9923 
9924   format %{ "testq   $src, $src\t# cp2b\n\t"
9925             "setnz   $dst\n\t"
9926             "movzbl  $dst, $dst" %}
9927   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9928              setNZ_reg(dst),
9929              REX_reg_breg(dst, dst), // movzbl
9930              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9931   ins_pipe(pipe_slow); // XXX
9932 %}
9933 
9934 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9935 %{
9936   match(Set dst (CmpLTMask p q));
9937   effect(KILL cr);
9938 
9939   ins_cost(400);
9940   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9941             "setlt   $dst\n\t"
9942             "movzbl  $dst, $dst\n\t"
9943             "negl    $dst" %}
9944   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9945              setLT_reg(dst),
9946              REX_reg_breg(dst, dst), // movzbl
9947              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9948              neg_reg(dst));
9949   ins_pipe(pipe_slow);
9950 %}
9951 
9952 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9953 %{
9954   match(Set dst (CmpLTMask dst zero));
9955   effect(KILL cr);
9956 
9957   ins_cost(100);
9958   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9959   ins_encode %{
9960   __ sarl($dst$$Register, 31);
9961   %}
9962   ins_pipe(ialu_reg);
9963 %}
9964 
9965 /* Better to save a register than avoid a branch */
9966 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9967 %{
9968   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9969   effect(KILL cr);
9970   ins_cost(300);
9971   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9972             "jge    done\n\t"
9973             "addl   $p,$y\n"
9974             "done:  " %}
9975   ins_encode %{
9976     Register Rp = $p$$Register;
9977     Register Rq = $q$$Register;
9978     Register Ry = $y$$Register;
9979     Label done;
9980     __ subl(Rp, Rq);
9981     __ jccb(Assembler::greaterEqual, done);
9982     __ addl(Rp, Ry);
9983     __ bind(done);
9984   %}
9985   ins_pipe(pipe_cmplt);
9986 %}
9987 
9988 /* Better to save a register than avoid a branch */
9989 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9990 %{
9991   match(Set y (AndI (CmpLTMask p q) y));
9992   effect(KILL cr);
9993 
9994   ins_cost(300);
9995 
9996   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9997             "jlt      done\n\t"
9998             "xorl     $y, $y\n"
9999             "done:  " %}
10000   ins_encode %{
10001     Register Rp = $p$$Register;
10002     Register Rq = $q$$Register;
10003     Register Ry = $y$$Register;
10004     Label done;
10005     __ cmpl(Rp, Rq);
10006     __ jccb(Assembler::less, done);
10007     __ xorl(Ry, Ry);
10008     __ bind(done);
10009   %}
10010   ins_pipe(pipe_cmplt);
10011 %}
10012 
10013 
10014 //---------- FP Instructions------------------------------------------------
10015 
10016 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10017 %{
10018   match(Set cr (CmpF src1 src2));
10019 
10020   ins_cost(145);
10021   format %{ "ucomiss $src1, $src2\n\t"
10022             "jnp,s   exit\n\t"
10023             "pushfq\t# saw NaN, set CF\n\t"
10024             "andq    [rsp], #0xffffff2b\n\t"
10025             "popfq\n"
10026     "exit:" %}
10027   ins_encode %{
10028     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10029     emit_cmpfp_fixup(_masm);
10030   %}
10031   ins_pipe(pipe_slow);
10032 %}
10033 
10034 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10035   match(Set cr (CmpF src1 src2));
10036 
10037   ins_cost(100);
10038   format %{ "ucomiss $src1, $src2" %}
10039   ins_encode %{
10040     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10041   %}
10042   ins_pipe(pipe_slow);
10043 %}
10044 
10045 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10046 %{
10047   match(Set cr (CmpF src1 (LoadF src2)));
10048 
10049   ins_cost(145);
10050   format %{ "ucomiss $src1, $src2\n\t"
10051             "jnp,s   exit\n\t"
10052             "pushfq\t# saw NaN, set CF\n\t"
10053             "andq    [rsp], #0xffffff2b\n\t"
10054             "popfq\n"
10055     "exit:" %}
10056   ins_encode %{
10057     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10058     emit_cmpfp_fixup(_masm);
10059   %}
10060   ins_pipe(pipe_slow);
10061 %}
10062 
10063 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10064   match(Set cr (CmpF src1 (LoadF src2)));
10065 
10066   ins_cost(100);
10067   format %{ "ucomiss $src1, $src2" %}
10068   ins_encode %{
10069     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10070   %}
10071   ins_pipe(pipe_slow);
10072 %}
10073 
10074 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10075   match(Set cr (CmpF src con));
10076 
10077   ins_cost(145);
10078   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10079             "jnp,s   exit\n\t"
10080             "pushfq\t# saw NaN, set CF\n\t"
10081             "andq    [rsp], #0xffffff2b\n\t"
10082             "popfq\n"
10083     "exit:" %}
10084   ins_encode %{
10085     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10086     emit_cmpfp_fixup(_masm);
10087   %}
10088   ins_pipe(pipe_slow);
10089 %}
10090 
10091 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10092   match(Set cr (CmpF src con));
10093   ins_cost(100);
10094   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10095   ins_encode %{
10096     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10097   %}
10098   ins_pipe(pipe_slow);
10099 %}
10100 
10101 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10102 %{
10103   match(Set cr (CmpD src1 src2));
10104 
10105   ins_cost(145);
10106   format %{ "ucomisd $src1, $src2\n\t"
10107             "jnp,s   exit\n\t"
10108             "pushfq\t# saw NaN, set CF\n\t"
10109             "andq    [rsp], #0xffffff2b\n\t"
10110             "popfq\n"
10111     "exit:" %}
10112   ins_encode %{
10113     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10114     emit_cmpfp_fixup(_masm);
10115   %}
10116   ins_pipe(pipe_slow);
10117 %}
10118 
10119 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10120   match(Set cr (CmpD src1 src2));
10121 
10122   ins_cost(100);
10123   format %{ "ucomisd $src1, $src2 test" %}
10124   ins_encode %{
10125     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10126   %}
10127   ins_pipe(pipe_slow);
10128 %}
10129 
10130 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10131 %{
10132   match(Set cr (CmpD src1 (LoadD src2)));
10133 
10134   ins_cost(145);
10135   format %{ "ucomisd $src1, $src2\n\t"
10136             "jnp,s   exit\n\t"
10137             "pushfq\t# saw NaN, set CF\n\t"
10138             "andq    [rsp], #0xffffff2b\n\t"
10139             "popfq\n"
10140     "exit:" %}
10141   ins_encode %{
10142     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10143     emit_cmpfp_fixup(_masm);
10144   %}
10145   ins_pipe(pipe_slow);
10146 %}
10147 
10148 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10149   match(Set cr (CmpD src1 (LoadD src2)));
10150 
10151   ins_cost(100);
10152   format %{ "ucomisd $src1, $src2" %}
10153   ins_encode %{
10154     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10155   %}
10156   ins_pipe(pipe_slow);
10157 %}
10158 
10159 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10160   match(Set cr (CmpD src con));
10161 
10162   ins_cost(145);
10163   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10164             "jnp,s   exit\n\t"
10165             "pushfq\t# saw NaN, set CF\n\t"
10166             "andq    [rsp], #0xffffff2b\n\t"
10167             "popfq\n"
10168     "exit:" %}
10169   ins_encode %{
10170     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10171     emit_cmpfp_fixup(_masm);
10172   %}
10173   ins_pipe(pipe_slow);
10174 %}
10175 
10176 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10177   match(Set cr (CmpD src con));
10178   ins_cost(100);
10179   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10180   ins_encode %{
10181     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10182   %}
10183   ins_pipe(pipe_slow);
10184 %}
10185 
10186 // Compare into -1,0,1
10187 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10188 %{
10189   match(Set dst (CmpF3 src1 src2));
10190   effect(KILL cr);
10191 
10192   ins_cost(275);
10193   format %{ "ucomiss $src1, $src2\n\t"
10194             "movl    $dst, #-1\n\t"
10195             "jp,s    done\n\t"
10196             "jb,s    done\n\t"
10197             "setne   $dst\n\t"
10198             "movzbl  $dst, $dst\n"
10199     "done:" %}
10200   ins_encode %{
10201     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10202     emit_cmpfp3(_masm, $dst$$Register);
10203   %}
10204   ins_pipe(pipe_slow);
10205 %}
10206 
10207 // Compare into -1,0,1
10208 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10209 %{
10210   match(Set dst (CmpF3 src1 (LoadF src2)));
10211   effect(KILL cr);
10212 
10213   ins_cost(275);
10214   format %{ "ucomiss $src1, $src2\n\t"
10215             "movl    $dst, #-1\n\t"
10216             "jp,s    done\n\t"
10217             "jb,s    done\n\t"
10218             "setne   $dst\n\t"
10219             "movzbl  $dst, $dst\n"
10220     "done:" %}
10221   ins_encode %{
10222     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10223     emit_cmpfp3(_masm, $dst$$Register);
10224   %}
10225   ins_pipe(pipe_slow);
10226 %}
10227 
10228 // Compare into -1,0,1
10229 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10230   match(Set dst (CmpF3 src con));
10231   effect(KILL cr);
10232 
10233   ins_cost(275);
10234   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10235             "movl    $dst, #-1\n\t"
10236             "jp,s    done\n\t"
10237             "jb,s    done\n\t"
10238             "setne   $dst\n\t"
10239             "movzbl  $dst, $dst\n"
10240     "done:" %}
10241   ins_encode %{
10242     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10243     emit_cmpfp3(_masm, $dst$$Register);
10244   %}
10245   ins_pipe(pipe_slow);
10246 %}
10247 
10248 // Compare into -1,0,1
10249 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10250 %{
10251   match(Set dst (CmpD3 src1 src2));
10252   effect(KILL cr);
10253 
10254   ins_cost(275);
10255   format %{ "ucomisd $src1, $src2\n\t"
10256             "movl    $dst, #-1\n\t"
10257             "jp,s    done\n\t"
10258             "jb,s    done\n\t"
10259             "setne   $dst\n\t"
10260             "movzbl  $dst, $dst\n"
10261     "done:" %}
10262   ins_encode %{
10263     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10264     emit_cmpfp3(_masm, $dst$$Register);
10265   %}
10266   ins_pipe(pipe_slow);
10267 %}
10268 
10269 // Compare into -1,0,1
10270 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10271 %{
10272   match(Set dst (CmpD3 src1 (LoadD src2)));
10273   effect(KILL cr);
10274 
10275   ins_cost(275);
10276   format %{ "ucomisd $src1, $src2\n\t"
10277             "movl    $dst, #-1\n\t"
10278             "jp,s    done\n\t"
10279             "jb,s    done\n\t"
10280             "setne   $dst\n\t"
10281             "movzbl  $dst, $dst\n"
10282     "done:" %}
10283   ins_encode %{
10284     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10285     emit_cmpfp3(_masm, $dst$$Register);
10286   %}
10287   ins_pipe(pipe_slow);
10288 %}
10289 
10290 // Compare into -1,0,1
10291 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10292   match(Set dst (CmpD3 src con));
10293   effect(KILL cr);
10294 
10295   ins_cost(275);
10296   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10297             "movl    $dst, #-1\n\t"
10298             "jp,s    done\n\t"
10299             "jb,s    done\n\t"
10300             "setne   $dst\n\t"
10301             "movzbl  $dst, $dst\n"
10302     "done:" %}
10303   ins_encode %{
10304     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10305     emit_cmpfp3(_masm, $dst$$Register);
10306   %}
10307   ins_pipe(pipe_slow);
10308 %}
10309 
10310 //----------Arithmetic Conversion Instructions---------------------------------
10311 
10312 instruct roundFloat_nop(regF dst)
10313 %{
10314   match(Set dst (RoundFloat dst));
10315 
10316   ins_cost(0);
10317   ins_encode();
10318   ins_pipe(empty);
10319 %}
10320 
10321 instruct roundDouble_nop(regD dst)
10322 %{
10323   match(Set dst (RoundDouble dst));
10324 
10325   ins_cost(0);
10326   ins_encode();
10327   ins_pipe(empty);
10328 %}
10329 
10330 instruct convF2D_reg_reg(regD dst, regF src)
10331 %{
10332   match(Set dst (ConvF2D src));
10333 
10334   format %{ "cvtss2sd $dst, $src" %}
10335   ins_encode %{
10336     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10337   %}
10338   ins_pipe(pipe_slow); // XXX
10339 %}
10340 
10341 instruct convF2D_reg_mem(regD dst, memory src)
10342 %{
10343   match(Set dst (ConvF2D (LoadF src)));
10344 
10345   format %{ "cvtss2sd $dst, $src" %}
10346   ins_encode %{
10347     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10348   %}
10349   ins_pipe(pipe_slow); // XXX
10350 %}
10351 
10352 instruct convD2F_reg_reg(regF dst, regD src)
10353 %{
10354   match(Set dst (ConvD2F src));
10355 
10356   format %{ "cvtsd2ss $dst, $src" %}
10357   ins_encode %{
10358     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10359   %}
10360   ins_pipe(pipe_slow); // XXX
10361 %}
10362 
10363 instruct convD2F_reg_mem(regF dst, memory src)
10364 %{
10365   match(Set dst (ConvD2F (LoadD src)));
10366 
10367   format %{ "cvtsd2ss $dst, $src" %}
10368   ins_encode %{
10369     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10370   %}
10371   ins_pipe(pipe_slow); // XXX
10372 %}
10373 
10374 // XXX do mem variants
10375 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10376 %{
10377   match(Set dst (ConvF2I src));
10378   effect(KILL cr);
10379 
10380   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10381             "cmpl    $dst, #0x80000000\n\t"
10382             "jne,s   done\n\t"
10383             "subq    rsp, #8\n\t"
10384             "movss   [rsp], $src\n\t"
10385             "call    f2i_fixup\n\t"
10386             "popq    $dst\n"
10387     "done:   "%}
10388   ins_encode %{
10389     Label done;
10390     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10391     __ cmpl($dst$$Register, 0x80000000);
10392     __ jccb(Assembler::notEqual, done);
10393     __ subptr(rsp, 8);
10394     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10395     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10396     __ pop($dst$$Register);
10397     __ bind(done);
10398   %}
10399   ins_pipe(pipe_slow);
10400 %}
10401 
10402 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10403 %{
10404   match(Set dst (ConvF2L src));
10405   effect(KILL cr);
10406 
10407   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10408             "cmpq    $dst, [0x8000000000000000]\n\t"
10409             "jne,s   done\n\t"
10410             "subq    rsp, #8\n\t"
10411             "movss   [rsp], $src\n\t"
10412             "call    f2l_fixup\n\t"
10413             "popq    $dst\n"
10414     "done:   "%}
10415   ins_encode %{
10416     Label done;
10417     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10418     __ cmp64($dst$$Register,
10419              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10420     __ jccb(Assembler::notEqual, done);
10421     __ subptr(rsp, 8);
10422     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10423     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10424     __ pop($dst$$Register);
10425     __ bind(done);
10426   %}
10427   ins_pipe(pipe_slow);
10428 %}
10429 
10430 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10431 %{
10432   match(Set dst (ConvD2I src));
10433   effect(KILL cr);
10434 
10435   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10436             "cmpl    $dst, #0x80000000\n\t"
10437             "jne,s   done\n\t"
10438             "subq    rsp, #8\n\t"
10439             "movsd   [rsp], $src\n\t"
10440             "call    d2i_fixup\n\t"
10441             "popq    $dst\n"
10442     "done:   "%}
10443   ins_encode %{
10444     Label done;
10445     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10446     __ cmpl($dst$$Register, 0x80000000);
10447     __ jccb(Assembler::notEqual, done);
10448     __ subptr(rsp, 8);
10449     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10450     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10451     __ pop($dst$$Register);
10452     __ bind(done);
10453   %}
10454   ins_pipe(pipe_slow);
10455 %}
10456 
10457 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10458 %{
10459   match(Set dst (ConvD2L src));
10460   effect(KILL cr);
10461 
10462   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10463             "cmpq    $dst, [0x8000000000000000]\n\t"
10464             "jne,s   done\n\t"
10465             "subq    rsp, #8\n\t"
10466             "movsd   [rsp], $src\n\t"
10467             "call    d2l_fixup\n\t"
10468             "popq    $dst\n"
10469     "done:   "%}
10470   ins_encode %{
10471     Label done;
10472     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10473     __ cmp64($dst$$Register,
10474              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10475     __ jccb(Assembler::notEqual, done);
10476     __ subptr(rsp, 8);
10477     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10478     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10479     __ pop($dst$$Register);
10480     __ bind(done);
10481   %}
10482   ins_pipe(pipe_slow);
10483 %}
10484 
10485 instruct convI2F_reg_reg(regF dst, rRegI src)
10486 %{
10487   predicate(!UseXmmI2F);
10488   match(Set dst (ConvI2F src));
10489 
10490   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10491   ins_encode %{
10492     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10493   %}
10494   ins_pipe(pipe_slow); // XXX
10495 %}
10496 
10497 instruct convI2F_reg_mem(regF dst, memory src)
10498 %{
10499   match(Set dst (ConvI2F (LoadI src)));
10500 
10501   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10502   ins_encode %{
10503     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10504   %}
10505   ins_pipe(pipe_slow); // XXX
10506 %}
10507 
10508 instruct convI2D_reg_reg(regD dst, rRegI src)
10509 %{
10510   predicate(!UseXmmI2D);
10511   match(Set dst (ConvI2D src));
10512 
10513   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10514   ins_encode %{
10515     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10516   %}
10517   ins_pipe(pipe_slow); // XXX
10518 %}
10519 
10520 instruct convI2D_reg_mem(regD dst, memory src)
10521 %{
10522   match(Set dst (ConvI2D (LoadI src)));
10523 
10524   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10525   ins_encode %{
10526     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10527   %}
10528   ins_pipe(pipe_slow); // XXX
10529 %}
10530 
10531 instruct convXI2F_reg(regF dst, rRegI src)
10532 %{
10533   predicate(UseXmmI2F);
10534   match(Set dst (ConvI2F src));
10535 
10536   format %{ "movdl $dst, $src\n\t"
10537             "cvtdq2psl $dst, $dst\t# i2f" %}
10538   ins_encode %{
10539     __ movdl($dst$$XMMRegister, $src$$Register);
10540     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10541   %}
10542   ins_pipe(pipe_slow); // XXX
10543 %}
10544 
10545 instruct convXI2D_reg(regD dst, rRegI src)
10546 %{
10547   predicate(UseXmmI2D);
10548   match(Set dst (ConvI2D src));
10549 
10550   format %{ "movdl $dst, $src\n\t"
10551             "cvtdq2pdl $dst, $dst\t# i2d" %}
10552   ins_encode %{
10553     __ movdl($dst$$XMMRegister, $src$$Register);
10554     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10555   %}
10556   ins_pipe(pipe_slow); // XXX
10557 %}
10558 
10559 instruct convL2F_reg_reg(regF dst, rRegL src)
10560 %{
10561   match(Set dst (ConvL2F src));
10562 
10563   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10564   ins_encode %{
10565     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10566   %}
10567   ins_pipe(pipe_slow); // XXX
10568 %}
10569 
10570 instruct convL2F_reg_mem(regF dst, memory src)
10571 %{
10572   match(Set dst (ConvL2F (LoadL src)));
10573 
10574   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10575   ins_encode %{
10576     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10577   %}
10578   ins_pipe(pipe_slow); // XXX
10579 %}
10580 
10581 instruct convL2D_reg_reg(regD dst, rRegL src)
10582 %{
10583   match(Set dst (ConvL2D src));
10584 
10585   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10586   ins_encode %{
10587     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10588   %}
10589   ins_pipe(pipe_slow); // XXX
10590 %}
10591 
10592 instruct convL2D_reg_mem(regD dst, memory src)
10593 %{
10594   match(Set dst (ConvL2D (LoadL src)));
10595 
10596   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10597   ins_encode %{
10598     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10599   %}
10600   ins_pipe(pipe_slow); // XXX
10601 %}
10602 
10603 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10604 %{
10605   match(Set dst (ConvI2L src));
10606 
10607   ins_cost(125);
10608   format %{ "movslq  $dst, $src\t# i2l" %}
10609   ins_encode %{
10610     __ movslq($dst$$Register, $src$$Register);
10611   %}
10612   ins_pipe(ialu_reg_reg);
10613 %}
10614 
10615 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10616 // %{
10617 //   match(Set dst (ConvI2L src));
10618 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10619 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10620 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10621 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10622 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10623 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10624 
10625 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10626 //   ins_encode(enc_copy(dst, src));
10627 // //   opcode(0x63); // needs REX.W
10628 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10629 //   ins_pipe(ialu_reg_reg);
10630 // %}
10631 
10632 // Zero-extend convert int to long
10633 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10634 %{
10635   match(Set dst (AndL (ConvI2L src) mask));
10636 
10637   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10638   ins_encode %{
10639     if ($dst$$reg != $src$$reg) {
10640       __ movl($dst$$Register, $src$$Register);
10641     }
10642   %}
10643   ins_pipe(ialu_reg_reg);
10644 %}
10645 
10646 // Zero-extend convert int to long
10647 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10648 %{
10649   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10650 
10651   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10652   ins_encode %{
10653     __ movl($dst$$Register, $src$$Address);
10654   %}
10655   ins_pipe(ialu_reg_mem);
10656 %}
10657 
10658 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10659 %{
10660   match(Set dst (AndL src mask));
10661 
10662   format %{ "movl    $dst, $src\t# zero-extend long" %}
10663   ins_encode %{
10664     __ movl($dst$$Register, $src$$Register);
10665   %}
10666   ins_pipe(ialu_reg_reg);
10667 %}
10668 
10669 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10670 %{
10671   match(Set dst (ConvL2I src));
10672 
10673   format %{ "movl    $dst, $src\t# l2i" %}
10674   ins_encode %{
10675     __ movl($dst$$Register, $src$$Register);
10676   %}
10677   ins_pipe(ialu_reg_reg);
10678 %}
10679 
10680 
10681 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10682   match(Set dst (MoveF2I src));
10683   effect(DEF dst, USE src);
10684 
10685   ins_cost(125);
10686   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10687   ins_encode %{
10688     __ movl($dst$$Register, Address(rsp, $src$$disp));
10689   %}
10690   ins_pipe(ialu_reg_mem);
10691 %}
10692 
10693 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10694   match(Set dst (MoveI2F src));
10695   effect(DEF dst, USE src);
10696 
10697   ins_cost(125);
10698   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10699   ins_encode %{
10700     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10701   %}
10702   ins_pipe(pipe_slow);
10703 %}
10704 
10705 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10706   match(Set dst (MoveD2L src));
10707   effect(DEF dst, USE src);
10708 
10709   ins_cost(125);
10710   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10711   ins_encode %{
10712     __ movq($dst$$Register, Address(rsp, $src$$disp));
10713   %}
10714   ins_pipe(ialu_reg_mem);
10715 %}
10716 
10717 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10718   predicate(!UseXmmLoadAndClearUpper);
10719   match(Set dst (MoveL2D src));
10720   effect(DEF dst, USE src);
10721 
10722   ins_cost(125);
10723   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10724   ins_encode %{
10725     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10726   %}
10727   ins_pipe(pipe_slow);
10728 %}
10729 
10730 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10731   predicate(UseXmmLoadAndClearUpper);
10732   match(Set dst (MoveL2D src));
10733   effect(DEF dst, USE src);
10734 
10735   ins_cost(125);
10736   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10737   ins_encode %{
10738     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10739   %}
10740   ins_pipe(pipe_slow);
10741 %}
10742 
10743 
10744 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10745   match(Set dst (MoveF2I src));
10746   effect(DEF dst, USE src);
10747 
10748   ins_cost(95); // XXX
10749   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10750   ins_encode %{
10751     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10752   %}
10753   ins_pipe(pipe_slow);
10754 %}
10755 
10756 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10757   match(Set dst (MoveI2F src));
10758   effect(DEF dst, USE src);
10759 
10760   ins_cost(100);
10761   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10762   ins_encode %{
10763     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10764   %}
10765   ins_pipe( ialu_mem_reg );
10766 %}
10767 
10768 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10769   match(Set dst (MoveD2L src));
10770   effect(DEF dst, USE src);
10771 
10772   ins_cost(95); // XXX
10773   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10774   ins_encode %{
10775     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10776   %}
10777   ins_pipe(pipe_slow);
10778 %}
10779 
10780 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10781   match(Set dst (MoveL2D src));
10782   effect(DEF dst, USE src);
10783 
10784   ins_cost(100);
10785   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10786   ins_encode %{
10787     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10788   %}
10789   ins_pipe(ialu_mem_reg);
10790 %}
10791 
10792 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10793   match(Set dst (MoveF2I src));
10794   effect(DEF dst, USE src);
10795   ins_cost(85);
10796   format %{ "movd    $dst,$src\t# MoveF2I" %}
10797   ins_encode %{
10798     __ movdl($dst$$Register, $src$$XMMRegister);
10799   %}
10800   ins_pipe( pipe_slow );
10801 %}
10802 
10803 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10804   match(Set dst (MoveD2L src));
10805   effect(DEF dst, USE src);
10806   ins_cost(85);
10807   format %{ "movd    $dst,$src\t# MoveD2L" %}
10808   ins_encode %{
10809     __ movdq($dst$$Register, $src$$XMMRegister);
10810   %}
10811   ins_pipe( pipe_slow );
10812 %}
10813 
10814 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10815   match(Set dst (MoveI2F src));
10816   effect(DEF dst, USE src);
10817   ins_cost(100);
10818   format %{ "movd    $dst,$src\t# MoveI2F" %}
10819   ins_encode %{
10820     __ movdl($dst$$XMMRegister, $src$$Register);
10821   %}
10822   ins_pipe( pipe_slow );
10823 %}
10824 
10825 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10826   match(Set dst (MoveL2D src));
10827   effect(DEF dst, USE src);
10828   ins_cost(100);
10829   format %{ "movd    $dst,$src\t# MoveL2D" %}
10830   ins_encode %{
10831      __ movdq($dst$$XMMRegister, $src$$Register);
10832   %}
10833   ins_pipe( pipe_slow );
10834 %}
10835 
10836 
10837 // =======================================================================
10838 // fast clearing of an array
10839 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10840                   Universe dummy, rFlagsReg cr)
10841 %{
10842   predicate(!((ClearArrayNode*)n)->is_large());
10843   match(Set dummy (ClearArray cnt base));
10844   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10845 
10846   format %{ $$template
10847     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10848     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10849     $$emit$$"jg      LARGE\n\t"
10850     $$emit$$"dec     rcx\n\t"
10851     $$emit$$"js      DONE\t# Zero length\n\t"
10852     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10853     $$emit$$"dec     rcx\n\t"
10854     $$emit$$"jge     LOOP\n\t"
10855     $$emit$$"jmp     DONE\n\t"
10856     $$emit$$"# LARGE:\n\t"
10857     if (UseFastStosb) {
10858        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10859        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10860     } else if (UseXMMForObjInit) {
10861        $$emit$$"mov     rdi,rax\n\t"
10862        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10863        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10864        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10865        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10866        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10867        $$emit$$"add     0x40,rax\n\t"
10868        $$emit$$"# L_zero_64_bytes:\n\t"
10869        $$emit$$"sub     0x8,rcx\n\t"
10870        $$emit$$"jge     L_loop\n\t"
10871        $$emit$$"add     0x4,rcx\n\t"
10872        $$emit$$"jl      L_tail\n\t"
10873        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10874        $$emit$$"add     0x20,rax\n\t"
10875        $$emit$$"sub     0x4,rcx\n\t"
10876        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10877        $$emit$$"add     0x4,rcx\n\t"
10878        $$emit$$"jle     L_end\n\t"
10879        $$emit$$"dec     rcx\n\t"
10880        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10881        $$emit$$"vmovq   xmm0,(rax)\n\t"
10882        $$emit$$"add     0x8,rax\n\t"
10883        $$emit$$"dec     rcx\n\t"
10884        $$emit$$"jge     L_sloop\n\t"
10885        $$emit$$"# L_end:\n\t"
10886     } else {
10887        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10888     }
10889     $$emit$$"# DONE"
10890   %}
10891   ins_encode %{
10892     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10893                  $tmp$$XMMRegister, false);
10894   %}
10895   ins_pipe(pipe_slow);
10896 %}
10897 
10898 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10899                         Universe dummy, rFlagsReg cr)
10900 %{
10901   predicate(((ClearArrayNode*)n)->is_large());
10902   match(Set dummy (ClearArray cnt base));
10903   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10904 
10905   format %{ $$template
10906     if (UseFastStosb) {
10907        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10908        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10909        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10910     } else if (UseXMMForObjInit) {
10911        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10912        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10913        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10914        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10915        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10916        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10917        $$emit$$"add     0x40,rax\n\t"
10918        $$emit$$"# L_zero_64_bytes:\n\t"
10919        $$emit$$"sub     0x8,rcx\n\t"
10920        $$emit$$"jge     L_loop\n\t"
10921        $$emit$$"add     0x4,rcx\n\t"
10922        $$emit$$"jl      L_tail\n\t"
10923        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10924        $$emit$$"add     0x20,rax\n\t"
10925        $$emit$$"sub     0x4,rcx\n\t"
10926        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10927        $$emit$$"add     0x4,rcx\n\t"
10928        $$emit$$"jle     L_end\n\t"
10929        $$emit$$"dec     rcx\n\t"
10930        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10931        $$emit$$"vmovq   xmm0,(rax)\n\t"
10932        $$emit$$"add     0x8,rax\n\t"
10933        $$emit$$"dec     rcx\n\t"
10934        $$emit$$"jge     L_sloop\n\t"
10935        $$emit$$"# L_end:\n\t"
10936     } else {
10937        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10938        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10939     }
10940   %}
10941   ins_encode %{
10942     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10943                  $tmp$$XMMRegister, true);
10944   %}
10945   ins_pipe(pipe_slow);
10946 %}
10947 
10948 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10949                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
10950 %{
10951   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10952   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10953   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10954 
10955   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10956   ins_encode %{
10957     __ string_compare($str1$$Register, $str2$$Register,
10958                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10959                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10960   %}
10961   ins_pipe( pipe_slow );
10962 %}
10963 
10964 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10965                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
10966 %{
10967   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10968   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10969   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10970 
10971   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10972   ins_encode %{
10973     __ string_compare($str1$$Register, $str2$$Register,
10974                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10975                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10976   %}
10977   ins_pipe( pipe_slow );
10978 %}
10979 
10980 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10981                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
10982 %{
10983   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10984   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10985   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10986 
10987   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10988   ins_encode %{
10989     __ string_compare($str1$$Register, $str2$$Register,
10990                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10991                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10992   %}
10993   ins_pipe( pipe_slow );
10994 %}
10995 
10996 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10997                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
10998 %{
10999   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11000   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11001   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11002 
11003   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11004   ins_encode %{
11005     __ string_compare($str2$$Register, $str1$$Register,
11006                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11007                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11008   %}
11009   ins_pipe( pipe_slow );
11010 %}
11011 
11012 // fast search of substring with known size.
11013 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11014                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11015 %{
11016   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11017   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11018   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11019 
11020   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11021   ins_encode %{
11022     int icnt2 = (int)$int_cnt2$$constant;
11023     if (icnt2 >= 16) {
11024       // IndexOf for constant substrings with size >= 16 elements
11025       // which don't need to be loaded through stack.
11026       __ string_indexofC8($str1$$Register, $str2$$Register,
11027                           $cnt1$$Register, $cnt2$$Register,
11028                           icnt2, $result$$Register,
11029                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11030     } else {
11031       // Small strings are loaded through stack if they cross page boundary.
11032       __ string_indexof($str1$$Register, $str2$$Register,
11033                         $cnt1$$Register, $cnt2$$Register,
11034                         icnt2, $result$$Register,
11035                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11036     }
11037   %}
11038   ins_pipe( pipe_slow );
11039 %}
11040 
11041 // fast search of substring with known size.
11042 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11043                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11044 %{
11045   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11046   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11047   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11048 
11049   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11050   ins_encode %{
11051     int icnt2 = (int)$int_cnt2$$constant;
11052     if (icnt2 >= 8) {
11053       // IndexOf for constant substrings with size >= 8 elements
11054       // which don't need to be loaded through stack.
11055       __ string_indexofC8($str1$$Register, $str2$$Register,
11056                           $cnt1$$Register, $cnt2$$Register,
11057                           icnt2, $result$$Register,
11058                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11059     } else {
11060       // Small strings are loaded through stack if they cross page boundary.
11061       __ string_indexof($str1$$Register, $str2$$Register,
11062                         $cnt1$$Register, $cnt2$$Register,
11063                         icnt2, $result$$Register,
11064                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11065     }
11066   %}
11067   ins_pipe( pipe_slow );
11068 %}
11069 
11070 // fast search of substring with known size.
11071 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11072                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11073 %{
11074   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11075   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11076   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11077 
11078   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11079   ins_encode %{
11080     int icnt2 = (int)$int_cnt2$$constant;
11081     if (icnt2 >= 8) {
11082       // IndexOf for constant substrings with size >= 8 elements
11083       // which don't need to be loaded through stack.
11084       __ string_indexofC8($str1$$Register, $str2$$Register,
11085                           $cnt1$$Register, $cnt2$$Register,
11086                           icnt2, $result$$Register,
11087                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11088     } else {
11089       // Small strings are loaded through stack if they cross page boundary.
11090       __ string_indexof($str1$$Register, $str2$$Register,
11091                         $cnt1$$Register, $cnt2$$Register,
11092                         icnt2, $result$$Register,
11093                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11094     }
11095   %}
11096   ins_pipe( pipe_slow );
11097 %}
11098 
11099 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11100                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11101 %{
11102   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11103   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11104   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11105 
11106   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11107   ins_encode %{
11108     __ string_indexof($str1$$Register, $str2$$Register,
11109                       $cnt1$$Register, $cnt2$$Register,
11110                       (-1), $result$$Register,
11111                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11112   %}
11113   ins_pipe( pipe_slow );
11114 %}
11115 
11116 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11117                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11118 %{
11119   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11120   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11121   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11122 
11123   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11124   ins_encode %{
11125     __ string_indexof($str1$$Register, $str2$$Register,
11126                       $cnt1$$Register, $cnt2$$Register,
11127                       (-1), $result$$Register,
11128                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11129   %}
11130   ins_pipe( pipe_slow );
11131 %}
11132 
11133 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11134                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11135 %{
11136   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11137   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11138   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11139 
11140   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11141   ins_encode %{
11142     __ string_indexof($str1$$Register, $str2$$Register,
11143                       $cnt1$$Register, $cnt2$$Register,
11144                       (-1), $result$$Register,
11145                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11146   %}
11147   ins_pipe( pipe_slow );
11148 %}
11149 
11150 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11151                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11152 %{
11153   predicate(UseSSE42Intrinsics);
11154   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11155   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11156   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11157   ins_encode %{
11158     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11159                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11160   %}
11161   ins_pipe( pipe_slow );
11162 %}
11163 
11164 // fast string equals
11165 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11166                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11167 %{
11168   match(Set result (StrEquals (Binary str1 str2) cnt));
11169   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11170 
11171   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11172   ins_encode %{
11173     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11174                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11175                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11176   %}
11177   ins_pipe( pipe_slow );
11178 %}
11179 
11180 // fast array equals
11181 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11182                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11183 %{
11184   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11185   match(Set result (AryEq ary1 ary2));
11186   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11187 
11188   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11189   ins_encode %{
11190     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11191                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11192                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11193   %}
11194   ins_pipe( pipe_slow );
11195 %}
11196 
11197 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11198                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11199 %{
11200   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11201   match(Set result (AryEq ary1 ary2));
11202   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11203 
11204   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11205   ins_encode %{
11206     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11207                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11208                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11209   %}
11210   ins_pipe( pipe_slow );
11211 %}
11212 
11213 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11214                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11215 %{
11216   match(Set result (HasNegatives ary1 len));
11217   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11218 
11219   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11220   ins_encode %{
11221     __ has_negatives($ary1$$Register, $len$$Register,
11222                      $result$$Register, $tmp3$$Register,
11223                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11224   %}
11225   ins_pipe( pipe_slow );
11226 %}
11227 
11228 // fast char[] to byte[] compression
11229 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11230                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11231   match(Set result (StrCompressedCopy src (Binary dst len)));
11232   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11233 
11234   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11235   ins_encode %{
11236     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11237                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11238                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11239   %}
11240   ins_pipe( pipe_slow );
11241 %}
11242 
11243 // fast byte[] to char[] inflation
11244 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11245                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11246   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11247   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11248 
11249   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11250   ins_encode %{
11251     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11252                           $tmp1$$XMMRegister, $tmp2$$Register);
11253   %}
11254   ins_pipe( pipe_slow );
11255 %}
11256 
11257 // encode char[] to byte[] in ISO_8859_1
11258 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11259                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11260                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11261   match(Set result (EncodeISOArray src (Binary dst len)));
11262   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11263 
11264   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11265   ins_encode %{
11266     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11267                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11268                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11269   %}
11270   ins_pipe( pipe_slow );
11271 %}
11272 
11273 //----------Overflow Math Instructions-----------------------------------------
11274 
11275 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11276 %{
11277   match(Set cr (OverflowAddI op1 op2));
11278   effect(DEF cr, USE_KILL op1, USE op2);
11279 
11280   format %{ "addl    $op1, $op2\t# overflow check int" %}
11281 
11282   ins_encode %{
11283     __ addl($op1$$Register, $op2$$Register);
11284   %}
11285   ins_pipe(ialu_reg_reg);
11286 %}
11287 
11288 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11289 %{
11290   match(Set cr (OverflowAddI op1 op2));
11291   effect(DEF cr, USE_KILL op1, USE op2);
11292 
11293   format %{ "addl    $op1, $op2\t# overflow check int" %}
11294 
11295   ins_encode %{
11296     __ addl($op1$$Register, $op2$$constant);
11297   %}
11298   ins_pipe(ialu_reg_reg);
11299 %}
11300 
11301 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11302 %{
11303   match(Set cr (OverflowAddL op1 op2));
11304   effect(DEF cr, USE_KILL op1, USE op2);
11305 
11306   format %{ "addq    $op1, $op2\t# overflow check long" %}
11307   ins_encode %{
11308     __ addq($op1$$Register, $op2$$Register);
11309   %}
11310   ins_pipe(ialu_reg_reg);
11311 %}
11312 
11313 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11314 %{
11315   match(Set cr (OverflowAddL op1 op2));
11316   effect(DEF cr, USE_KILL op1, USE op2);
11317 
11318   format %{ "addq    $op1, $op2\t# overflow check long" %}
11319   ins_encode %{
11320     __ addq($op1$$Register, $op2$$constant);
11321   %}
11322   ins_pipe(ialu_reg_reg);
11323 %}
11324 
11325 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11326 %{
11327   match(Set cr (OverflowSubI op1 op2));
11328 
11329   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11330   ins_encode %{
11331     __ cmpl($op1$$Register, $op2$$Register);
11332   %}
11333   ins_pipe(ialu_reg_reg);
11334 %}
11335 
11336 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11337 %{
11338   match(Set cr (OverflowSubI op1 op2));
11339 
11340   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11341   ins_encode %{
11342     __ cmpl($op1$$Register, $op2$$constant);
11343   %}
11344   ins_pipe(ialu_reg_reg);
11345 %}
11346 
11347 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11348 %{
11349   match(Set cr (OverflowSubL op1 op2));
11350 
11351   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11352   ins_encode %{
11353     __ cmpq($op1$$Register, $op2$$Register);
11354   %}
11355   ins_pipe(ialu_reg_reg);
11356 %}
11357 
11358 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11359 %{
11360   match(Set cr (OverflowSubL op1 op2));
11361 
11362   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11363   ins_encode %{
11364     __ cmpq($op1$$Register, $op2$$constant);
11365   %}
11366   ins_pipe(ialu_reg_reg);
11367 %}
11368 
11369 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11370 %{
11371   match(Set cr (OverflowSubI zero op2));
11372   effect(DEF cr, USE_KILL op2);
11373 
11374   format %{ "negl    $op2\t# overflow check int" %}
11375   ins_encode %{
11376     __ negl($op2$$Register);
11377   %}
11378   ins_pipe(ialu_reg_reg);
11379 %}
11380 
11381 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11382 %{
11383   match(Set cr (OverflowSubL zero op2));
11384   effect(DEF cr, USE_KILL op2);
11385 
11386   format %{ "negq    $op2\t# overflow check long" %}
11387   ins_encode %{
11388     __ negq($op2$$Register);
11389   %}
11390   ins_pipe(ialu_reg_reg);
11391 %}
11392 
11393 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11394 %{
11395   match(Set cr (OverflowMulI op1 op2));
11396   effect(DEF cr, USE_KILL op1, USE op2);
11397 
11398   format %{ "imull    $op1, $op2\t# overflow check int" %}
11399   ins_encode %{
11400     __ imull($op1$$Register, $op2$$Register);
11401   %}
11402   ins_pipe(ialu_reg_reg_alu0);
11403 %}
11404 
11405 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11406 %{
11407   match(Set cr (OverflowMulI op1 op2));
11408   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11409 
11410   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11411   ins_encode %{
11412     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11413   %}
11414   ins_pipe(ialu_reg_reg_alu0);
11415 %}
11416 
11417 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11418 %{
11419   match(Set cr (OverflowMulL op1 op2));
11420   effect(DEF cr, USE_KILL op1, USE op2);
11421 
11422   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11423   ins_encode %{
11424     __ imulq($op1$$Register, $op2$$Register);
11425   %}
11426   ins_pipe(ialu_reg_reg_alu0);
11427 %}
11428 
11429 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11430 %{
11431   match(Set cr (OverflowMulL op1 op2));
11432   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11433 
11434   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11435   ins_encode %{
11436     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11437   %}
11438   ins_pipe(ialu_reg_reg_alu0);
11439 %}
11440 
11441 
11442 //----------Control Flow Instructions------------------------------------------
11443 // Signed compare Instructions
11444 
11445 // XXX more variants!!
11446 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11447 %{
11448   match(Set cr (CmpI op1 op2));
11449   effect(DEF cr, USE op1, USE op2);
11450 
11451   format %{ "cmpl    $op1, $op2" %}
11452   opcode(0x3B);  /* Opcode 3B /r */
11453   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11454   ins_pipe(ialu_cr_reg_reg);
11455 %}
11456 
11457 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11458 %{
11459   match(Set cr (CmpI op1 op2));
11460 
11461   format %{ "cmpl    $op1, $op2" %}
11462   opcode(0x81, 0x07); /* Opcode 81 /7 */
11463   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11464   ins_pipe(ialu_cr_reg_imm);
11465 %}
11466 
11467 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11468 %{
11469   match(Set cr (CmpI op1 (LoadI op2)));
11470 
11471   ins_cost(500); // XXX
11472   format %{ "cmpl    $op1, $op2" %}
11473   opcode(0x3B); /* Opcode 3B /r */
11474   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11475   ins_pipe(ialu_cr_reg_mem);
11476 %}
11477 
11478 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11479 %{
11480   match(Set cr (CmpI src zero));
11481 
11482   format %{ "testl   $src, $src" %}
11483   opcode(0x85);
11484   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11485   ins_pipe(ialu_cr_reg_imm);
11486 %}
11487 
11488 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11489 %{
11490   match(Set cr (CmpI (AndI src con) zero));
11491 
11492   format %{ "testl   $src, $con" %}
11493   opcode(0xF7, 0x00);
11494   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11495   ins_pipe(ialu_cr_reg_imm);
11496 %}
11497 
11498 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11499 %{
11500   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11501 
11502   format %{ "testl   $src, $mem" %}
11503   opcode(0x85);
11504   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11505   ins_pipe(ialu_cr_reg_mem);
11506 %}
11507 
11508 // Unsigned compare Instructions; really, same as signed except they
11509 // produce an rFlagsRegU instead of rFlagsReg.
11510 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11511 %{
11512   match(Set cr (CmpU op1 op2));
11513 
11514   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11515   opcode(0x3B); /* Opcode 3B /r */
11516   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11517   ins_pipe(ialu_cr_reg_reg);
11518 %}
11519 
11520 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11521 %{
11522   match(Set cr (CmpU op1 op2));
11523 
11524   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11525   opcode(0x81,0x07); /* Opcode 81 /7 */
11526   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11527   ins_pipe(ialu_cr_reg_imm);
11528 %}
11529 
11530 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11531 %{
11532   match(Set cr (CmpU op1 (LoadI op2)));
11533 
11534   ins_cost(500); // XXX
11535   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11536   opcode(0x3B); /* Opcode 3B /r */
11537   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11538   ins_pipe(ialu_cr_reg_mem);
11539 %}
11540 
11541 // // // Cisc-spilled version of cmpU_rReg
11542 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11543 // //%{
11544 // //  match(Set cr (CmpU (LoadI op1) op2));
11545 // //
11546 // //  format %{ "CMPu   $op1,$op2" %}
11547 // //  ins_cost(500);
11548 // //  opcode(0x39);  /* Opcode 39 /r */
11549 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11550 // //%}
11551 
11552 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11553 %{
11554   match(Set cr (CmpU src zero));
11555 
11556   format %{ "testl  $src, $src\t# unsigned" %}
11557   opcode(0x85);
11558   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11559   ins_pipe(ialu_cr_reg_imm);
11560 %}
11561 
11562 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11563 %{
11564   match(Set cr (CmpP op1 op2));
11565 
11566   format %{ "cmpq    $op1, $op2\t# ptr" %}
11567   opcode(0x3B); /* Opcode 3B /r */
11568   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11569   ins_pipe(ialu_cr_reg_reg);
11570 %}
11571 
11572 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11573 %{
11574   match(Set cr (CmpP op1 (LoadP op2)));
11575 
11576   ins_cost(500); // XXX
11577   format %{ "cmpq    $op1, $op2\t# ptr" %}
11578   opcode(0x3B); /* Opcode 3B /r */
11579   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11580   ins_pipe(ialu_cr_reg_mem);
11581 %}
11582 
11583 // // // Cisc-spilled version of cmpP_rReg
11584 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11585 // //%{
11586 // //  match(Set cr (CmpP (LoadP op1) op2));
11587 // //
11588 // //  format %{ "CMPu   $op1,$op2" %}
11589 // //  ins_cost(500);
11590 // //  opcode(0x39);  /* Opcode 39 /r */
11591 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11592 // //%}
11593 
11594 // XXX this is generalized by compP_rReg_mem???
11595 // Compare raw pointer (used in out-of-heap check).
11596 // Only works because non-oop pointers must be raw pointers
11597 // and raw pointers have no anti-dependencies.
11598 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11599 %{
11600   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11601   match(Set cr (CmpP op1 (LoadP op2)));
11602 
11603   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11604   opcode(0x3B); /* Opcode 3B /r */
11605   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11606   ins_pipe(ialu_cr_reg_mem);
11607 %}
11608 
11609 // This will generate a signed flags result. This should be OK since
11610 // any compare to a zero should be eq/neq.
11611 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11612 %{
11613   match(Set cr (CmpP src zero));
11614 
11615   format %{ "testq   $src, $src\t# ptr" %}
11616   opcode(0x85);
11617   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11618   ins_pipe(ialu_cr_reg_imm);
11619 %}
11620 
11621 // This will generate a signed flags result. This should be OK since
11622 // any compare to a zero should be eq/neq.
11623 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11624 %{
11625   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11626   match(Set cr (CmpP (LoadP op) zero));
11627 
11628   ins_cost(500); // XXX
11629   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11630   opcode(0xF7); /* Opcode F7 /0 */
11631   ins_encode(REX_mem_wide(op),
11632              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11633   ins_pipe(ialu_cr_reg_imm);
11634 %}
11635 
11636 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11637 %{
11638   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11639   match(Set cr (CmpP (LoadP mem) zero));
11640 
11641   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11642   ins_encode %{
11643     __ cmpq(r12, $mem$$Address);
11644   %}
11645   ins_pipe(ialu_cr_reg_mem);
11646 %}
11647 
11648 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11649 %{
11650   match(Set cr (CmpN op1 op2));
11651 
11652   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11653   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11654   ins_pipe(ialu_cr_reg_reg);
11655 %}
11656 
11657 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11658 %{
11659   match(Set cr (CmpN src (LoadN mem)));
11660 
11661   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11662   ins_encode %{
11663     __ cmpl($src$$Register, $mem$$Address);
11664   %}
11665   ins_pipe(ialu_cr_reg_mem);
11666 %}
11667 
11668 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11669   match(Set cr (CmpN op1 op2));
11670 
11671   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11672   ins_encode %{
11673     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11674   %}
11675   ins_pipe(ialu_cr_reg_imm);
11676 %}
11677 
11678 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11679 %{
11680   match(Set cr (CmpN src (LoadN mem)));
11681 
11682   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11683   ins_encode %{
11684     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11685   %}
11686   ins_pipe(ialu_cr_reg_mem);
11687 %}
11688 
11689 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11690   match(Set cr (CmpN op1 op2));
11691 
11692   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11693   ins_encode %{
11694     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11695   %}
11696   ins_pipe(ialu_cr_reg_imm);
11697 %}
11698 
11699 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11700 %{
11701   match(Set cr (CmpN src (LoadNKlass mem)));
11702 
11703   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11704   ins_encode %{
11705     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11706   %}
11707   ins_pipe(ialu_cr_reg_mem);
11708 %}
11709 
11710 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11711   match(Set cr (CmpN src zero));
11712 
11713   format %{ "testl   $src, $src\t# compressed ptr" %}
11714   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11715   ins_pipe(ialu_cr_reg_imm);
11716 %}
11717 
11718 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11719 %{
11720   predicate(Universe::narrow_oop_base() != NULL);
11721   match(Set cr (CmpN (LoadN mem) zero));
11722 
11723   ins_cost(500); // XXX
11724   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11725   ins_encode %{
11726     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11727   %}
11728   ins_pipe(ialu_cr_reg_mem);
11729 %}
11730 
11731 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11732 %{
11733   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11734   match(Set cr (CmpN (LoadN mem) zero));
11735 
11736   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11737   ins_encode %{
11738     __ cmpl(r12, $mem$$Address);
11739   %}
11740   ins_pipe(ialu_cr_reg_mem);
11741 %}
11742 
11743 // Yanked all unsigned pointer compare operations.
11744 // Pointer compares are done with CmpP which is already unsigned.
11745 
11746 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11747 %{
11748   match(Set cr (CmpL op1 op2));
11749 
11750   format %{ "cmpq    $op1, $op2" %}
11751   opcode(0x3B);  /* Opcode 3B /r */
11752   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11753   ins_pipe(ialu_cr_reg_reg);
11754 %}
11755 
11756 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11757 %{
11758   match(Set cr (CmpL op1 op2));
11759 
11760   format %{ "cmpq    $op1, $op2" %}
11761   opcode(0x81, 0x07); /* Opcode 81 /7 */
11762   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11763   ins_pipe(ialu_cr_reg_imm);
11764 %}
11765 
11766 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11767 %{
11768   match(Set cr (CmpL op1 (LoadL op2)));
11769 
11770   format %{ "cmpq    $op1, $op2" %}
11771   opcode(0x3B); /* Opcode 3B /r */
11772   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11773   ins_pipe(ialu_cr_reg_mem);
11774 %}
11775 
11776 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11777 %{
11778   match(Set cr (CmpL src zero));
11779 
11780   format %{ "testq   $src, $src" %}
11781   opcode(0x85);
11782   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11783   ins_pipe(ialu_cr_reg_imm);
11784 %}
11785 
11786 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11787 %{
11788   match(Set cr (CmpL (AndL src con) zero));
11789 
11790   format %{ "testq   $src, $con\t# long" %}
11791   opcode(0xF7, 0x00);
11792   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11793   ins_pipe(ialu_cr_reg_imm);
11794 %}
11795 
11796 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11797 %{
11798   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11799 
11800   format %{ "testq   $src, $mem" %}
11801   opcode(0x85);
11802   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11803   ins_pipe(ialu_cr_reg_mem);
11804 %}
11805 
11806 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11807 %{
11808   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11809 
11810   format %{ "testq   $src, $mem" %}
11811   opcode(0x85);
11812   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11813   ins_pipe(ialu_cr_reg_mem);
11814 %}
11815 
11816 // Manifest a CmpL result in an integer register.  Very painful.
11817 // This is the test to avoid.
11818 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11819 %{
11820   match(Set dst (CmpL3 src1 src2));
11821   effect(KILL flags);
11822 
11823   ins_cost(275); // XXX
11824   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11825             "movl    $dst, -1\n\t"
11826             "jl,s    done\n\t"
11827             "setne   $dst\n\t"
11828             "movzbl  $dst, $dst\n\t"
11829     "done:" %}
11830   ins_encode(cmpl3_flag(src1, src2, dst));
11831   ins_pipe(pipe_slow);
11832 %}
11833 
11834 // Unsigned long compare Instructions; really, same as signed long except they
11835 // produce an rFlagsRegU instead of rFlagsReg.
11836 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11837 %{
11838   match(Set cr (CmpUL op1 op2));
11839 
11840   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11841   opcode(0x3B);  /* Opcode 3B /r */
11842   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11843   ins_pipe(ialu_cr_reg_reg);
11844 %}
11845 
11846 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11847 %{
11848   match(Set cr (CmpUL op1 op2));
11849 
11850   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11851   opcode(0x81, 0x07); /* Opcode 81 /7 */
11852   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11853   ins_pipe(ialu_cr_reg_imm);
11854 %}
11855 
11856 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11857 %{
11858   match(Set cr (CmpUL op1 (LoadL op2)));
11859 
11860   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11861   opcode(0x3B); /* Opcode 3B /r */
11862   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11863   ins_pipe(ialu_cr_reg_mem);
11864 %}
11865 
11866 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11867 %{
11868   match(Set cr (CmpUL src zero));
11869 
11870   format %{ "testq   $src, $src\t# unsigned" %}
11871   opcode(0x85);
11872   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11873   ins_pipe(ialu_cr_reg_imm);
11874 %}
11875 
11876 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11877 %{
11878   match(Set cr (CmpI (LoadB mem) imm));
11879 
11880   ins_cost(125);
11881   format %{ "cmpb    $mem, $imm" %}
11882   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11883   ins_pipe(ialu_cr_reg_mem);
11884 %}
11885 
11886 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
11887 %{
11888   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
11889 
11890   ins_cost(125);
11891   format %{ "testb   $mem, $imm\t# ubyte" %}
11892   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11893   ins_pipe(ialu_cr_reg_mem);
11894 %}
11895 
11896 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
11897 %{
11898   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11899 
11900   ins_cost(125);
11901   format %{ "testb   $mem, $imm\t# byte" %}
11902   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11903   ins_pipe(ialu_cr_reg_mem);
11904 %}
11905 
11906 //----------Max and Min--------------------------------------------------------
11907 // Min Instructions
11908 
11909 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11910 %{
11911   effect(USE_DEF dst, USE src, USE cr);
11912 
11913   format %{ "cmovlgt $dst, $src\t# min" %}
11914   opcode(0x0F, 0x4F);
11915   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11916   ins_pipe(pipe_cmov_reg);
11917 %}
11918 
11919 
11920 instruct minI_rReg(rRegI dst, rRegI src)
11921 %{
11922   match(Set dst (MinI dst src));
11923 
11924   ins_cost(200);
11925   expand %{
11926     rFlagsReg cr;
11927     compI_rReg(cr, dst, src);
11928     cmovI_reg_g(dst, src, cr);
11929   %}
11930 %}
11931 
11932 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11933 %{
11934   effect(USE_DEF dst, USE src, USE cr);
11935 
11936   format %{ "cmovllt $dst, $src\t# max" %}
11937   opcode(0x0F, 0x4C);
11938   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11939   ins_pipe(pipe_cmov_reg);
11940 %}
11941 
11942 
11943 instruct maxI_rReg(rRegI dst, rRegI src)
11944 %{
11945   match(Set dst (MaxI dst src));
11946 
11947   ins_cost(200);
11948   expand %{
11949     rFlagsReg cr;
11950     compI_rReg(cr, dst, src);
11951     cmovI_reg_l(dst, src, cr);
11952   %}
11953 %}
11954 
11955 // ============================================================================
11956 // Branch Instructions
11957 
11958 // Jump Direct - Label defines a relative address from JMP+1
11959 instruct jmpDir(label labl)
11960 %{
11961   match(Goto);
11962   effect(USE labl);
11963 
11964   ins_cost(300);
11965   format %{ "jmp     $labl" %}
11966   size(5);
11967   ins_encode %{
11968     Label* L = $labl$$label;
11969     __ jmp(*L, false); // Always long jump
11970   %}
11971   ins_pipe(pipe_jmp);
11972 %}
11973 
11974 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11975 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11976 %{
11977   match(If cop cr);
11978   effect(USE labl);
11979 
11980   ins_cost(300);
11981   format %{ "j$cop     $labl" %}
11982   size(6);
11983   ins_encode %{
11984     Label* L = $labl$$label;
11985     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11986   %}
11987   ins_pipe(pipe_jcc);
11988 %}
11989 
11990 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11991 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11992 %{
11993   predicate(!n->has_vector_mask_set());
11994   match(CountedLoopEnd cop cr);
11995   effect(USE labl);
11996 
11997   ins_cost(300);
11998   format %{ "j$cop     $labl\t# loop end" %}
11999   size(6);
12000   ins_encode %{
12001     Label* L = $labl$$label;
12002     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12003   %}
12004   ins_pipe(pipe_jcc);
12005 %}
12006 
12007 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12008 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12009   predicate(!n->has_vector_mask_set());
12010   match(CountedLoopEnd cop cmp);
12011   effect(USE labl);
12012 
12013   ins_cost(300);
12014   format %{ "j$cop,u   $labl\t# loop end" %}
12015   size(6);
12016   ins_encode %{
12017     Label* L = $labl$$label;
12018     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12019   %}
12020   ins_pipe(pipe_jcc);
12021 %}
12022 
12023 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12024   predicate(!n->has_vector_mask_set());
12025   match(CountedLoopEnd cop cmp);
12026   effect(USE labl);
12027 
12028   ins_cost(200);
12029   format %{ "j$cop,u   $labl\t# loop end" %}
12030   size(6);
12031   ins_encode %{
12032     Label* L = $labl$$label;
12033     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12034   %}
12035   ins_pipe(pipe_jcc);
12036 %}
12037 
12038 // mask version
12039 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12040 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12041 %{
12042   predicate(n->has_vector_mask_set());
12043   match(CountedLoopEnd cop cr);
12044   effect(USE labl);
12045 
12046   ins_cost(400);
12047   format %{ "j$cop     $labl\t# loop end\n\t"
12048             "restorevectmask \t# vector mask restore for loops" %}
12049   size(10);
12050   ins_encode %{
12051     Label* L = $labl$$label;
12052     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12053     __ restorevectmask();
12054   %}
12055   ins_pipe(pipe_jcc);
12056 %}
12057 
12058 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12059 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12060   predicate(n->has_vector_mask_set());
12061   match(CountedLoopEnd cop cmp);
12062   effect(USE labl);
12063 
12064   ins_cost(400);
12065   format %{ "j$cop,u   $labl\t# loop end\n\t"
12066             "restorevectmask \t# vector mask restore for loops" %}
12067   size(10);
12068   ins_encode %{
12069     Label* L = $labl$$label;
12070     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12071     __ restorevectmask();
12072   %}
12073   ins_pipe(pipe_jcc);
12074 %}
12075 
12076 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12077   predicate(n->has_vector_mask_set());
12078   match(CountedLoopEnd cop cmp);
12079   effect(USE labl);
12080 
12081   ins_cost(300);
12082   format %{ "j$cop,u   $labl\t# loop end\n\t"
12083             "restorevectmask \t# vector mask restore for loops" %}
12084   size(10);
12085   ins_encode %{
12086     Label* L = $labl$$label;
12087     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12088     __ restorevectmask();
12089   %}
12090   ins_pipe(pipe_jcc);
12091 %}
12092 
12093 // Jump Direct Conditional - using unsigned comparison
12094 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12095   match(If cop cmp);
12096   effect(USE labl);
12097 
12098   ins_cost(300);
12099   format %{ "j$cop,u  $labl" %}
12100   size(6);
12101   ins_encode %{
12102     Label* L = $labl$$label;
12103     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12104   %}
12105   ins_pipe(pipe_jcc);
12106 %}
12107 
12108 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12109   match(If cop cmp);
12110   effect(USE labl);
12111 
12112   ins_cost(200);
12113   format %{ "j$cop,u  $labl" %}
12114   size(6);
12115   ins_encode %{
12116     Label* L = $labl$$label;
12117     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12118   %}
12119   ins_pipe(pipe_jcc);
12120 %}
12121 
12122 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12123   match(If cop cmp);
12124   effect(USE labl);
12125 
12126   ins_cost(200);
12127   format %{ $$template
12128     if ($cop$$cmpcode == Assembler::notEqual) {
12129       $$emit$$"jp,u   $labl\n\t"
12130       $$emit$$"j$cop,u   $labl"
12131     } else {
12132       $$emit$$"jp,u   done\n\t"
12133       $$emit$$"j$cop,u   $labl\n\t"
12134       $$emit$$"done:"
12135     }
12136   %}
12137   ins_encode %{
12138     Label* l = $labl$$label;
12139     if ($cop$$cmpcode == Assembler::notEqual) {
12140       __ jcc(Assembler::parity, *l, false);
12141       __ jcc(Assembler::notEqual, *l, false);
12142     } else if ($cop$$cmpcode == Assembler::equal) {
12143       Label done;
12144       __ jccb(Assembler::parity, done);
12145       __ jcc(Assembler::equal, *l, false);
12146       __ bind(done);
12147     } else {
12148        ShouldNotReachHere();
12149     }
12150   %}
12151   ins_pipe(pipe_jcc);
12152 %}
12153 
12154 // ============================================================================
12155 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12156 // superklass array for an instance of the superklass.  Set a hidden
12157 // internal cache on a hit (cache is checked with exposed code in
12158 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12159 // encoding ALSO sets flags.
12160 
12161 instruct partialSubtypeCheck(rdi_RegP result,
12162                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12163                              rFlagsReg cr)
12164 %{
12165   match(Set result (PartialSubtypeCheck sub super));
12166   effect(KILL rcx, KILL cr);
12167 
12168   ins_cost(1100);  // slightly larger than the next version
12169   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12170             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12171             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12172             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12173             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12174             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12175             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12176     "miss:\t" %}
12177 
12178   opcode(0x1); // Force a XOR of RDI
12179   ins_encode(enc_PartialSubtypeCheck());
12180   ins_pipe(pipe_slow);
12181 %}
12182 
12183 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12184                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12185                                      immP0 zero,
12186                                      rdi_RegP result)
12187 %{
12188   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12189   effect(KILL rcx, KILL result);
12190 
12191   ins_cost(1000);
12192   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12193             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12194             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12195             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12196             "jne,s   miss\t\t# Missed: flags nz\n\t"
12197             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12198     "miss:\t" %}
12199 
12200   opcode(0x0); // No need to XOR RDI
12201   ins_encode(enc_PartialSubtypeCheck());
12202   ins_pipe(pipe_slow);
12203 %}
12204 
12205 // ============================================================================
12206 // Branch Instructions -- short offset versions
12207 //
12208 // These instructions are used to replace jumps of a long offset (the default
12209 // match) with jumps of a shorter offset.  These instructions are all tagged
12210 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12211 // match rules in general matching.  Instead, the ADLC generates a conversion
12212 // method in the MachNode which can be used to do in-place replacement of the
12213 // long variant with the shorter variant.  The compiler will determine if a
12214 // branch can be taken by the is_short_branch_offset() predicate in the machine
12215 // specific code section of the file.
12216 
12217 // Jump Direct - Label defines a relative address from JMP+1
12218 instruct jmpDir_short(label labl) %{
12219   match(Goto);
12220   effect(USE labl);
12221 
12222   ins_cost(300);
12223   format %{ "jmp,s   $labl" %}
12224   size(2);
12225   ins_encode %{
12226     Label* L = $labl$$label;
12227     __ jmpb(*L);
12228   %}
12229   ins_pipe(pipe_jmp);
12230   ins_short_branch(1);
12231 %}
12232 
12233 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12234 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12235   match(If cop cr);
12236   effect(USE labl);
12237 
12238   ins_cost(300);
12239   format %{ "j$cop,s   $labl" %}
12240   size(2);
12241   ins_encode %{
12242     Label* L = $labl$$label;
12243     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12244   %}
12245   ins_pipe(pipe_jcc);
12246   ins_short_branch(1);
12247 %}
12248 
12249 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12250 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12251   match(CountedLoopEnd cop cr);
12252   effect(USE labl);
12253 
12254   ins_cost(300);
12255   format %{ "j$cop,s   $labl\t# loop end" %}
12256   size(2);
12257   ins_encode %{
12258     Label* L = $labl$$label;
12259     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12260   %}
12261   ins_pipe(pipe_jcc);
12262   ins_short_branch(1);
12263 %}
12264 
12265 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12266 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12267   match(CountedLoopEnd cop cmp);
12268   effect(USE labl);
12269 
12270   ins_cost(300);
12271   format %{ "j$cop,us  $labl\t# loop end" %}
12272   size(2);
12273   ins_encode %{
12274     Label* L = $labl$$label;
12275     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12276   %}
12277   ins_pipe(pipe_jcc);
12278   ins_short_branch(1);
12279 %}
12280 
12281 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12282   match(CountedLoopEnd cop cmp);
12283   effect(USE labl);
12284 
12285   ins_cost(300);
12286   format %{ "j$cop,us  $labl\t# loop end" %}
12287   size(2);
12288   ins_encode %{
12289     Label* L = $labl$$label;
12290     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12291   %}
12292   ins_pipe(pipe_jcc);
12293   ins_short_branch(1);
12294 %}
12295 
12296 // Jump Direct Conditional - using unsigned comparison
12297 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12298   match(If cop cmp);
12299   effect(USE labl);
12300 
12301   ins_cost(300);
12302   format %{ "j$cop,us  $labl" %}
12303   size(2);
12304   ins_encode %{
12305     Label* L = $labl$$label;
12306     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12307   %}
12308   ins_pipe(pipe_jcc);
12309   ins_short_branch(1);
12310 %}
12311 
12312 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12313   match(If cop cmp);
12314   effect(USE labl);
12315 
12316   ins_cost(300);
12317   format %{ "j$cop,us  $labl" %}
12318   size(2);
12319   ins_encode %{
12320     Label* L = $labl$$label;
12321     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12322   %}
12323   ins_pipe(pipe_jcc);
12324   ins_short_branch(1);
12325 %}
12326 
12327 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12328   match(If cop cmp);
12329   effect(USE labl);
12330 
12331   ins_cost(300);
12332   format %{ $$template
12333     if ($cop$$cmpcode == Assembler::notEqual) {
12334       $$emit$$"jp,u,s   $labl\n\t"
12335       $$emit$$"j$cop,u,s   $labl"
12336     } else {
12337       $$emit$$"jp,u,s   done\n\t"
12338       $$emit$$"j$cop,u,s  $labl\n\t"
12339       $$emit$$"done:"
12340     }
12341   %}
12342   size(4);
12343   ins_encode %{
12344     Label* l = $labl$$label;
12345     if ($cop$$cmpcode == Assembler::notEqual) {
12346       __ jccb(Assembler::parity, *l);
12347       __ jccb(Assembler::notEqual, *l);
12348     } else if ($cop$$cmpcode == Assembler::equal) {
12349       Label done;
12350       __ jccb(Assembler::parity, done);
12351       __ jccb(Assembler::equal, *l);
12352       __ bind(done);
12353     } else {
12354        ShouldNotReachHere();
12355     }
12356   %}
12357   ins_pipe(pipe_jcc);
12358   ins_short_branch(1);
12359 %}
12360 
12361 // ============================================================================
12362 // inlined locking and unlocking
12363 
12364 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12365   predicate(Compile::current()->use_rtm());
12366   match(Set cr (FastLock object box));
12367   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12368   ins_cost(300);
12369   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12370   ins_encode %{
12371     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12372                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12373                  _counters, _rtm_counters, _stack_rtm_counters,
12374                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12375                  true, ra_->C->profile_rtm());
12376   %}
12377   ins_pipe(pipe_slow);
12378 %}
12379 
12380 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12381   predicate(!Compile::current()->use_rtm());
12382   match(Set cr (FastLock object box));
12383   effect(TEMP tmp, TEMP scr, USE_KILL box);
12384   ins_cost(300);
12385   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12386   ins_encode %{
12387     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12388                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12389   %}
12390   ins_pipe(pipe_slow);
12391 %}
12392 
12393 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12394   match(Set cr (FastUnlock object box));
12395   effect(TEMP tmp, USE_KILL box);
12396   ins_cost(300);
12397   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12398   ins_encode %{
12399     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12400   %}
12401   ins_pipe(pipe_slow);
12402 %}
12403 
12404 
12405 // ============================================================================
12406 // Safepoint Instructions
12407 instruct safePoint_poll(rFlagsReg cr)
12408 %{
12409   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12410   match(SafePoint);
12411   effect(KILL cr);
12412 
12413   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12414             "# Safepoint: poll for GC" %}
12415   ins_cost(125);
12416   ins_encode %{
12417     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12418     __ testl(rax, addr);
12419   %}
12420   ins_pipe(ialu_reg_mem);
12421 %}
12422 
12423 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12424 %{
12425   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12426   match(SafePoint poll);
12427   effect(KILL cr, USE poll);
12428 
12429   format %{ "testl  rax, [$poll]\t"
12430             "# Safepoint: poll for GC" %}
12431   ins_cost(125);
12432   ins_encode %{
12433     __ relocate(relocInfo::poll_type);
12434     __ testl(rax, Address($poll$$Register, 0));
12435   %}
12436   ins_pipe(ialu_reg_mem);
12437 %}
12438 
12439 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12440 %{
12441   predicate(SafepointMechanism::uses_thread_local_poll());
12442   match(SafePoint poll);
12443   effect(KILL cr, USE poll);
12444 
12445   format %{ "testl  rax, [$poll]\t"
12446             "# Safepoint: poll for GC" %}
12447   ins_cost(125);
12448   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12449   ins_encode %{
12450     __ relocate(relocInfo::poll_type);
12451     address pre_pc = __ pc();
12452     __ testl(rax, Address($poll$$Register, 0));
12453     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12454   %}
12455   ins_pipe(ialu_reg_mem);
12456 %}
12457 
12458 // ============================================================================
12459 // Procedure Call/Return Instructions
12460 // Call Java Static Instruction
12461 // Note: If this code changes, the corresponding ret_addr_offset() and
12462 //       compute_padding() functions will have to be adjusted.
12463 instruct CallStaticJavaDirect(method meth) %{
12464   match(CallStaticJava);
12465   effect(USE meth);
12466 
12467   ins_cost(300);
12468   format %{ "call,static " %}
12469   opcode(0xE8); /* E8 cd */
12470   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12471   ins_pipe(pipe_slow);
12472   ins_alignment(4);
12473 %}
12474 
12475 // Call Java Dynamic Instruction
12476 // Note: If this code changes, the corresponding ret_addr_offset() and
12477 //       compute_padding() functions will have to be adjusted.
12478 instruct CallDynamicJavaDirect(method meth)
12479 %{
12480   match(CallDynamicJava);
12481   effect(USE meth);
12482 
12483   ins_cost(300);
12484   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12485             "call,dynamic " %}
12486   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12487   ins_pipe(pipe_slow);
12488   ins_alignment(4);
12489 %}
12490 
12491 // Call Runtime Instruction
12492 instruct CallRuntimeDirect(method meth)
12493 %{
12494   match(CallRuntime);
12495   effect(USE meth);
12496 
12497   ins_cost(300);
12498   format %{ "call,runtime " %}
12499   ins_encode(clear_avx, Java_To_Runtime(meth));
12500   ins_pipe(pipe_slow);
12501 %}
12502 
12503 // Call runtime without safepoint
12504 instruct CallLeafDirect(method meth)
12505 %{
12506   match(CallLeaf);
12507   effect(USE meth);
12508 
12509   ins_cost(300);
12510   format %{ "call_leaf,runtime " %}
12511   ins_encode(clear_avx, Java_To_Runtime(meth));
12512   ins_pipe(pipe_slow);
12513 %}
12514 
12515 // Call runtime without safepoint
12516 instruct CallLeafNoFPDirect(method meth)
12517 %{
12518   match(CallLeafNoFP);
12519   effect(USE meth);
12520 
12521   ins_cost(300);
12522   format %{ "call_leaf_nofp,runtime " %}
12523   ins_encode(clear_avx, Java_To_Runtime(meth));
12524   ins_pipe(pipe_slow);
12525 %}
12526 
12527 // Return Instruction
12528 // Remove the return address & jump to it.
12529 // Notice: We always emit a nop after a ret to make sure there is room
12530 // for safepoint patching
12531 instruct Ret()
12532 %{
12533   match(Return);
12534 
12535   format %{ "ret" %}
12536   opcode(0xC3);
12537   ins_encode(OpcP);
12538   ins_pipe(pipe_jmp);
12539 %}
12540 
12541 // Tail Call; Jump from runtime stub to Java code.
12542 // Also known as an 'interprocedural jump'.
12543 // Target of jump will eventually return to caller.
12544 // TailJump below removes the return address.
12545 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12546 %{
12547   match(TailCall jump_target method_oop);
12548 
12549   ins_cost(300);
12550   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12551   opcode(0xFF, 0x4); /* Opcode FF /4 */
12552   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12553   ins_pipe(pipe_jmp);
12554 %}
12555 
12556 // Tail Jump; remove the return address; jump to target.
12557 // TailCall above leaves the return address around.
12558 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12559 %{
12560   match(TailJump jump_target ex_oop);
12561 
12562   ins_cost(300);
12563   format %{ "popq    rdx\t# pop return address\n\t"
12564             "jmp     $jump_target" %}
12565   opcode(0xFF, 0x4); /* Opcode FF /4 */
12566   ins_encode(Opcode(0x5a), // popq rdx
12567              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12568   ins_pipe(pipe_jmp);
12569 %}
12570 
12571 // Create exception oop: created by stack-crawling runtime code.
12572 // Created exception is now available to this handler, and is setup
12573 // just prior to jumping to this handler.  No code emitted.
12574 instruct CreateException(rax_RegP ex_oop)
12575 %{
12576   match(Set ex_oop (CreateEx));
12577 
12578   size(0);
12579   // use the following format syntax
12580   format %{ "# exception oop is in rax; no code emitted" %}
12581   ins_encode();
12582   ins_pipe(empty);
12583 %}
12584 
12585 // Rethrow exception:
12586 // The exception oop will come in the first argument position.
12587 // Then JUMP (not call) to the rethrow stub code.
12588 instruct RethrowException()
12589 %{
12590   match(Rethrow);
12591 
12592   // use the following format syntax
12593   format %{ "jmp     rethrow_stub" %}
12594   ins_encode(enc_rethrow);
12595   ins_pipe(pipe_jmp);
12596 %}
12597 
12598 // ============================================================================
12599 // This name is KNOWN by the ADLC and cannot be changed.
12600 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12601 // for this guy.
12602 instruct tlsLoadP(r15_RegP dst) %{
12603   match(Set dst (ThreadLocal));
12604   effect(DEF dst);
12605 
12606   size(0);
12607   format %{ "# TLS is in R15" %}
12608   ins_encode( /*empty encoding*/ );
12609   ins_pipe(ialu_reg_reg);
12610 %}
12611 
12612 
12613 //----------PEEPHOLE RULES-----------------------------------------------------
12614 // These must follow all instruction definitions as they use the names
12615 // defined in the instructions definitions.
12616 //
12617 // peepmatch ( root_instr_name [preceding_instruction]* );
12618 //
12619 // peepconstraint %{
12620 // (instruction_number.operand_name relational_op instruction_number.operand_name
12621 //  [, ...] );
12622 // // instruction numbers are zero-based using left to right order in peepmatch
12623 //
12624 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12625 // // provide an instruction_number.operand_name for each operand that appears
12626 // // in the replacement instruction's match rule
12627 //
12628 // ---------VM FLAGS---------------------------------------------------------
12629 //
12630 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12631 //
12632 // Each peephole rule is given an identifying number starting with zero and
12633 // increasing by one in the order seen by the parser.  An individual peephole
12634 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12635 // on the command-line.
12636 //
12637 // ---------CURRENT LIMITATIONS----------------------------------------------
12638 //
12639 // Only match adjacent instructions in same basic block
12640 // Only equality constraints
12641 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12642 // Only one replacement instruction
12643 //
12644 // ---------EXAMPLE----------------------------------------------------------
12645 //
12646 // // pertinent parts of existing instructions in architecture description
12647 // instruct movI(rRegI dst, rRegI src)
12648 // %{
12649 //   match(Set dst (CopyI src));
12650 // %}
12651 //
12652 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12653 // %{
12654 //   match(Set dst (AddI dst src));
12655 //   effect(KILL cr);
12656 // %}
12657 //
12658 // // Change (inc mov) to lea
12659 // peephole %{
12660 //   // increment preceeded by register-register move
12661 //   peepmatch ( incI_rReg movI );
12662 //   // require that the destination register of the increment
12663 //   // match the destination register of the move
12664 //   peepconstraint ( 0.dst == 1.dst );
12665 //   // construct a replacement instruction that sets
12666 //   // the destination to ( move's source register + one )
12667 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12668 // %}
12669 //
12670 
12671 // Implementation no longer uses movX instructions since
12672 // machine-independent system no longer uses CopyX nodes.
12673 //
12674 // peephole
12675 // %{
12676 //   peepmatch (incI_rReg movI);
12677 //   peepconstraint (0.dst == 1.dst);
12678 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12679 // %}
12680 
12681 // peephole
12682 // %{
12683 //   peepmatch (decI_rReg movI);
12684 //   peepconstraint (0.dst == 1.dst);
12685 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12686 // %}
12687 
12688 // peephole
12689 // %{
12690 //   peepmatch (addI_rReg_imm movI);
12691 //   peepconstraint (0.dst == 1.dst);
12692 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12693 // %}
12694 
12695 // peephole
12696 // %{
12697 //   peepmatch (incL_rReg movL);
12698 //   peepconstraint (0.dst == 1.dst);
12699 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12700 // %}
12701 
12702 // peephole
12703 // %{
12704 //   peepmatch (decL_rReg movL);
12705 //   peepconstraint (0.dst == 1.dst);
12706 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12707 // %}
12708 
12709 // peephole
12710 // %{
12711 //   peepmatch (addL_rReg_imm movL);
12712 //   peepconstraint (0.dst == 1.dst);
12713 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12714 // %}
12715 
12716 // peephole
12717 // %{
12718 //   peepmatch (addP_rReg_imm movP);
12719 //   peepconstraint (0.dst == 1.dst);
12720 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12721 // %}
12722 
12723 // // Change load of spilled value to only a spill
12724 // instruct storeI(memory mem, rRegI src)
12725 // %{
12726 //   match(Set mem (StoreI mem src));
12727 // %}
12728 //
12729 // instruct loadI(rRegI dst, memory mem)
12730 // %{
12731 //   match(Set dst (LoadI mem));
12732 // %}
12733 //
12734 
12735 peephole
12736 %{
12737   peepmatch (loadI storeI);
12738   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12739   peepreplace (storeI(1.mem 1.mem 1.src));
12740 %}
12741 
12742 peephole
12743 %{
12744   peepmatch (loadL storeL);
12745   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12746   peepreplace (storeL(1.mem 1.mem 1.src));
12747 %}
12748 
12749 //----------SMARTSPILL RULES---------------------------------------------------
12750 // These must follow all instruction definitions as they use the names
12751 // defined in the instructions definitions.
--- EOF ---