1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Singleton class for RAX pointer register
 262 reg_class ptr_rax_reg(RAX, RAX_H);
 263 
 264 // Singleton class for RBX pointer register
 265 reg_class ptr_rbx_reg(RBX, RBX_H);
 266 
 267 // Singleton class for RSI pointer register
 268 reg_class ptr_rsi_reg(RSI, RSI_H);
 269 
 270 // Singleton class for RDI pointer register
 271 reg_class ptr_rdi_reg(RDI, RDI_H);
 272 
 273 // Singleton class for stack pointer
 274 reg_class ptr_rsp_reg(RSP, RSP_H);
 275 
 276 // Singleton class for TLS pointer
 277 reg_class ptr_r15_reg(R15, R15_H);
 278 
 279 // Singleton class for RAX long register
 280 reg_class long_rax_reg(RAX, RAX_H);
 281 
 282 // Singleton class for RCX long register
 283 reg_class long_rcx_reg(RCX, RCX_H);
 284 
 285 // Singleton class for RDX long register
 286 reg_class long_rdx_reg(RDX, RDX_H);
 287 
 288 // Singleton class for RAX int register
 289 reg_class int_rax_reg(RAX);
 290 
 291 // Singleton class for RBX int register
 292 reg_class int_rbx_reg(RBX);
 293 
 294 // Singleton class for RCX int register
 295 reg_class int_rcx_reg(RCX);
 296 
 297 // Singleton class for RCX int register
 298 reg_class int_rdx_reg(RDX);
 299 
 300 // Singleton class for RCX int register
 301 reg_class int_rdi_reg(RDI);
 302 
 303 // Singleton class for instruction pointer
 304 // reg_class ip_reg(RIP);
 305 
 306 %}
 307 
 308 //----------SOURCE BLOCK-------------------------------------------------------
 309 // This is a block of C++ code which provides values, functions, and
 310 // definitions necessary in the rest of the architecture description
 311 source_hpp %{
 312 
 313 extern RegMask _ANY_REG_mask;
 314 extern RegMask _PTR_REG_mask;
 315 extern RegMask _PTR_REG_NO_RBP_mask;
 316 extern RegMask _PTR_NO_RAX_REG_mask;
 317 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 318 extern RegMask _LONG_REG_mask;
 319 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 320 extern RegMask _LONG_NO_RCX_REG_mask;
 321 extern RegMask _INT_REG_mask;
 322 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 323 extern RegMask _INT_NO_RCX_REG_mask;
 324 
 325 extern RegMask _STACK_OR_PTR_REG_mask;
 326 extern RegMask _STACK_OR_LONG_REG_mask;
 327 extern RegMask _STACK_OR_INT_REG_mask;
 328 
 329 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 330 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 331 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 332 
 333 %}
 334 
 335 source %{
 336 #define   RELOC_IMM64    Assembler::imm_operand
 337 #define   RELOC_DISP32   Assembler::disp32_operand
 338 
 339 #define __ _masm.
 340 
 341 RegMask _ANY_REG_mask;
 342 RegMask _PTR_REG_mask;
 343 RegMask _PTR_REG_NO_RBP_mask;
 344 RegMask _PTR_NO_RAX_REG_mask;
 345 RegMask _PTR_NO_RAX_RBX_REG_mask;
 346 RegMask _LONG_REG_mask;
 347 RegMask _LONG_NO_RAX_RDX_REG_mask;
 348 RegMask _LONG_NO_RCX_REG_mask;
 349 RegMask _INT_REG_mask;
 350 RegMask _INT_NO_RAX_RDX_REG_mask;
 351 RegMask _INT_NO_RCX_REG_mask;
 352 RegMask _STACK_OR_PTR_REG_mask;
 353 RegMask _STACK_OR_LONG_REG_mask;
 354 RegMask _STACK_OR_INT_REG_mask;
 355 
 356 static bool need_r12_heapbase() {
 357   return UseCompressedOops || UseCompressedClassPointers;
 358 }
 359 
 360 void reg_mask_init() {
 361   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 362   // We derive a number of subsets from it.
 363   _ANY_REG_mask = _ALL_REG_mask;
 364 
 365   if (PreserveFramePointer) {
 366     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 368   }
 369   if (need_r12_heapbase()) {
 370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 372   }
 373 
 374   _PTR_REG_mask = _ANY_REG_mask;
 375   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 379 
 380   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 381   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 382 
 383   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 384   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 386 
 387   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 388   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 390 
 391   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 392   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 394 
 395   _LONG_REG_mask = _PTR_REG_mask;
 396   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 397   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 398 
 399   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 400   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 404 
 405   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 406   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 408 
 409   _INT_REG_mask = _ALL_INT_REG_mask;
 410   if (PreserveFramePointer) {
 411     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 412   }
 413   if (need_r12_heapbase()) {
 414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 415   }
 416 
 417   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 418   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 419 
 420   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 421   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 423 
 424   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 425   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 426 }
 427 
 428 static bool generate_vzeroupper(Compile* C) {
 429   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 430 }
 431 
 432 static int clear_avx_size() {
 433   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 434 }
 435 
 436 // !!!!! Special hack to get all types of calls to specify the byte offset
 437 //       from the start of the call to the point where the return address
 438 //       will point.
 439 int MachCallStaticJavaNode::ret_addr_offset()
 440 {
 441   int offset = 5; // 5 bytes from start of call to where return address points
 442   offset += clear_avx_size();
 443   return offset;
 444 }
 445 
 446 int MachCallDynamicJavaNode::ret_addr_offset()
 447 {
 448   int offset = 15; // 15 bytes from start of call to where return address points
 449   offset += clear_avx_size();
 450   return offset;
 451 }
 452 
 453 int MachCallRuntimeNode::ret_addr_offset() {
 454   int offset = 13; // movq r10,#addr; callq (r10)
 455   offset += clear_avx_size();
 456   return offset;
 457 }
 458 
 459 // Indicate if the safepoint node needs the polling page as an input,
 460 // it does if the polling page is more than disp32 away.
 461 bool SafePointNode::needs_polling_address_input()
 462 {
 463   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 464 }
 465 
 466 //
 467 // Compute padding required for nodes which need alignment
 468 //
 469 
 470 // The address of the call instruction needs to be 4-byte aligned to
 471 // ensure that it does not span a cache line so that it can be patched.
 472 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 473 {
 474   current_offset += clear_avx_size(); // skip vzeroupper
 475   current_offset += 1; // skip call opcode byte
 476   return align_up(current_offset, alignment_required()) - current_offset;
 477 }
 478 
 479 // The address of the call instruction needs to be 4-byte aligned to
 480 // ensure that it does not span a cache line so that it can be patched.
 481 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 482 {
 483   current_offset += clear_avx_size(); // skip vzeroupper
 484   current_offset += 11; // skip movq instruction + call opcode byte
 485   return align_up(current_offset, alignment_required()) - current_offset;
 486 }
 487 
 488 // EMIT_RM()
 489 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 490   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 491   cbuf.insts()->emit_int8(c);
 492 }
 493 
 494 // EMIT_CC()
 495 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 496   unsigned char c = (unsigned char) (f1 | f2);
 497   cbuf.insts()->emit_int8(c);
 498 }
 499 
 500 // EMIT_OPCODE()
 501 void emit_opcode(CodeBuffer &cbuf, int code) {
 502   cbuf.insts()->emit_int8((unsigned char) code);
 503 }
 504 
 505 // EMIT_OPCODE() w/ relocation information
 506 void emit_opcode(CodeBuffer &cbuf,
 507                  int code, relocInfo::relocType reloc, int offset, int format)
 508 {
 509   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 510   emit_opcode(cbuf, code);
 511 }
 512 
 513 // EMIT_D8()
 514 void emit_d8(CodeBuffer &cbuf, int d8) {
 515   cbuf.insts()->emit_int8((unsigned char) d8);
 516 }
 517 
 518 // EMIT_D16()
 519 void emit_d16(CodeBuffer &cbuf, int d16) {
 520   cbuf.insts()->emit_int16(d16);
 521 }
 522 
 523 // EMIT_D32()
 524 void emit_d32(CodeBuffer &cbuf, int d32) {
 525   cbuf.insts()->emit_int32(d32);
 526 }
 527 
 528 // EMIT_D64()
 529 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 530   cbuf.insts()->emit_int64(d64);
 531 }
 532 
 533 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 534 void emit_d32_reloc(CodeBuffer& cbuf,
 535                     int d32,
 536                     relocInfo::relocType reloc,
 537                     int format)
 538 {
 539   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 540   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 541   cbuf.insts()->emit_int32(d32);
 542 }
 543 
 544 // emit 32 bit value and construct relocation entry from RelocationHolder
 545 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 546 #ifdef ASSERT
 547   if (rspec.reloc()->type() == relocInfo::oop_type &&
 548       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 549     assert(Universe::heap()->is_in((address)(intptr_t)d32), "should be real oop");
 550     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)), "cannot embed broken oops in code");
 551   }
 552 #endif
 553   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 554   cbuf.insts()->emit_int32(d32);
 555 }
 556 
 557 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 558   address next_ip = cbuf.insts_end() + 4;
 559   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 560                  external_word_Relocation::spec(addr),
 561                  RELOC_DISP32);
 562 }
 563 
 564 
 565 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 566 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 567   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 568   cbuf.insts()->emit_int64(d64);
 569 }
 570 
 571 // emit 64 bit value and construct relocation entry from RelocationHolder
 572 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 573 #ifdef ASSERT
 574   if (rspec.reloc()->type() == relocInfo::oop_type &&
 575       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 576     assert(Universe::heap()->is_in((address)d64), "should be real oop");
 577     assert(oopDesc::is_oop(cast_to_oop(d64)), "cannot embed broken oops in code");
 578   }
 579 #endif
 580   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 581   cbuf.insts()->emit_int64(d64);
 582 }
 583 
 584 // Access stack slot for load or store
 585 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 586 {
 587   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 588   if (-0x80 <= disp && disp < 0x80) {
 589     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 590     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 591     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 592   } else {
 593     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 594     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 595     emit_d32(cbuf, disp);     // Displacement // R/M byte
 596   }
 597 }
 598 
 599    // rRegI ereg, memory mem) %{    // emit_reg_mem
 600 void encode_RegMem(CodeBuffer &cbuf,
 601                    int reg,
 602                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 603 {
 604   assert(disp_reloc == relocInfo::none, "cannot have disp");
 605   int regenc = reg & 7;
 606   int baseenc = base & 7;
 607   int indexenc = index & 7;
 608 
 609   // There is no index & no scale, use form without SIB byte
 610   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 611     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 612     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 613       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 614     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 615       // If 8-bit displacement, mode 0x1
 616       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 617       emit_d8(cbuf, disp);
 618     } else {
 619       // If 32-bit displacement
 620       if (base == -1) { // Special flag for absolute address
 621         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 622         if (disp_reloc != relocInfo::none) {
 623           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 624         } else {
 625           emit_d32(cbuf, disp);
 626         }
 627       } else {
 628         // Normal base + offset
 629         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 630         if (disp_reloc != relocInfo::none) {
 631           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 632         } else {
 633           emit_d32(cbuf, disp);
 634         }
 635       }
 636     }
 637   } else {
 638     // Else, encode with the SIB byte
 639     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 640     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 641       // If no displacement
 642       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 643       emit_rm(cbuf, scale, indexenc, baseenc);
 644     } else {
 645       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 646         // If 8-bit displacement, mode 0x1
 647         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 648         emit_rm(cbuf, scale, indexenc, baseenc);
 649         emit_d8(cbuf, disp);
 650       } else {
 651         // If 32-bit displacement
 652         if (base == 0x04 ) {
 653           emit_rm(cbuf, 0x2, regenc, 0x4);
 654           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 655         } else {
 656           emit_rm(cbuf, 0x2, regenc, 0x4);
 657           emit_rm(cbuf, scale, indexenc, baseenc); // *
 658         }
 659         if (disp_reloc != relocInfo::none) {
 660           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 661         } else {
 662           emit_d32(cbuf, disp);
 663         }
 664       }
 665     }
 666   }
 667 }
 668 
 669 // This could be in MacroAssembler but it's fairly C2 specific
 670 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 671   Label exit;
 672   __ jccb(Assembler::noParity, exit);
 673   __ pushf();
 674   //
 675   // comiss/ucomiss instructions set ZF,PF,CF flags and
 676   // zero OF,AF,SF for NaN values.
 677   // Fixup flags by zeroing ZF,PF so that compare of NaN
 678   // values returns 'less than' result (CF is set).
 679   // Leave the rest of flags unchanged.
 680   //
 681   //    7 6 5 4 3 2 1 0
 682   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 683   //    0 0 1 0 1 0 1 1   (0x2B)
 684   //
 685   __ andq(Address(rsp, 0), 0xffffff2b);
 686   __ popf();
 687   __ bind(exit);
 688 }
 689 
 690 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 691   Label done;
 692   __ movl(dst, -1);
 693   __ jcc(Assembler::parity, done);
 694   __ jcc(Assembler::below, done);
 695   __ setb(Assembler::notEqual, dst);
 696   __ movzbl(dst, dst);
 697   __ bind(done);
 698 }
 699 
 700 // Math.min()    # Math.max()
 701 // --------------------------
 702 // ucomis[s/d]   #
 703 // ja   -> b     # a
 704 // jp   -> NaN   # NaN
 705 // jb   -> a     # b
 706 // je            #
 707 // |-jz -> a | b # a & b
 708 // |    -> a     #
 709 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
 710                      XMMRegister a, XMMRegister b,
 711                      XMMRegister xmmt, Register rt,
 712                      bool min, bool single) {
 713 
 714   Label nan, zero, below, above, done;
 715 
 716   if (single)
 717     __ ucomiss(a, b);
 718   else
 719     __ ucomisd(a, b);
 720 
 721   if (dst->encoding() != (min ? b : a)->encoding())
 722     __ jccb(Assembler::above, above); // CF=0 & ZF=0
 723   else
 724     __ jccb(Assembler::above, done);
 725 
 726   __ jccb(Assembler::parity, nan);  // PF=1
 727   __ jccb(Assembler::below, below); // CF=1
 728 
 729   // equal
 730   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
 731   if (single) {
 732     __ ucomiss(a, xmmt);
 733     __ jccb(Assembler::equal, zero);
 734 
 735     __ movflt(dst, a);
 736     __ jmp(done);
 737   }
 738   else {
 739     __ ucomisd(a, xmmt);
 740     __ jccb(Assembler::equal, zero);
 741 
 742     __ movdbl(dst, a);
 743     __ jmp(done);
 744   }
 745 
 746   __ bind(zero);
 747   if (min)
 748     __ vpor(dst, a, b, Assembler::AVX_128bit);
 749   else
 750     __ vpand(dst, a, b, Assembler::AVX_128bit);
 751 
 752   __ jmp(done);
 753 
 754   __ bind(above);
 755   if (single)
 756     __ movflt(dst, min ? b : a);
 757   else
 758     __ movdbl(dst, min ? b : a);
 759 
 760   __ jmp(done);
 761 
 762   __ bind(nan);
 763   if (single) {
 764     __ movl(rt, 0x7fc00000); // Float.NaN
 765     __ movdl(dst, rt);
 766   }
 767   else {
 768     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
 769     __ movdq(dst, rt);
 770   }
 771   __ jmp(done);
 772 
 773   __ bind(below);
 774   if (single)
 775     __ movflt(dst, min ? a : b);
 776   else
 777     __ movdbl(dst, min ? a : b);
 778 
 779   __ bind(done);
 780 }
 781 
 782 //=============================================================================
 783 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 784 
 785 int Compile::ConstantTable::calculate_table_base_offset() const {
 786   return 0;  // absolute addressing, no offset
 787 }
 788 
 789 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 790 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 791   ShouldNotReachHere();
 792 }
 793 
 794 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 795   // Empty encoding
 796 }
 797 
 798 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 799   return 0;
 800 }
 801 
 802 #ifndef PRODUCT
 803 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 804   st->print("# MachConstantBaseNode (empty encoding)");
 805 }
 806 #endif
 807 
 808 
 809 //=============================================================================
 810 #ifndef PRODUCT
 811 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 812   Compile* C = ra_->C;
 813 
 814   int framesize = C->frame_size_in_bytes();
 815   int bangsize = C->bang_size_in_bytes();
 816   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 817   // Remove wordSize for return addr which is already pushed.
 818   framesize -= wordSize;
 819 
 820   if (C->need_stack_bang(bangsize)) {
 821     framesize -= wordSize;
 822     st->print("# stack bang (%d bytes)", bangsize);
 823     st->print("\n\t");
 824     st->print("pushq   rbp\t# Save rbp");
 825     if (PreserveFramePointer) {
 826         st->print("\n\t");
 827         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 828     }
 829     if (framesize) {
 830       st->print("\n\t");
 831       st->print("subq    rsp, #%d\t# Create frame",framesize);
 832     }
 833   } else {
 834     st->print("subq    rsp, #%d\t# Create frame",framesize);
 835     st->print("\n\t");
 836     framesize -= wordSize;
 837     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 838     if (PreserveFramePointer) {
 839       st->print("\n\t");
 840       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 841       if (framesize > 0) {
 842         st->print("\n\t");
 843         st->print("addq    rbp, #%d", framesize);
 844       }
 845     }
 846   }
 847 
 848   if (VerifyStackAtCalls) {
 849     st->print("\n\t");
 850     framesize -= wordSize;
 851     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 852 #ifdef ASSERT
 853     st->print("\n\t");
 854     st->print("# stack alignment check");
 855 #endif
 856   }
 857   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 858     st->print("\n\t");
 859     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 860     st->print("\n\t");
 861     st->print("je      fast_entry\t");
 862     st->print("\n\t");
 863     st->print("call    #nmethod_entry_barrier_stub\t");
 864     st->print("\n\tfast_entry:");
 865   }
 866   st->cr();
 867 }
 868 #endif
 869 
 870 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 871   Compile* C = ra_->C;
 872   MacroAssembler _masm(&cbuf);
 873 
 874   if (C->clinit_barrier_on_entry()) {
 875     assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 876     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
 877 
 878     Label L_skip_barrier;
 879     Register klass = rscratch1;
 880 
 881     __ mov_metadata(klass, C->method()->holder()->constant_encoding());
 882     __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
 883 
 884     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 885 
 886     __ bind(L_skip_barrier);
 887   }
 888 
 889   __ verified_entry(C);
 890   __ bind(*_verified_entry);
 891 
 892   C->set_frame_complete(cbuf.insts_size());
 893 
 894   if (C->has_mach_constant_base_node()) {
 895     // NOTE: We set the table base offset here because users might be
 896     // emitted before MachConstantBaseNode.
 897     Compile::ConstantTable& constant_table = C->constant_table();
 898     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 899   }
 900 }
 901 
 902 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 903 {
 904   return MachNode::size(ra_); // too many variables; just compute it
 905                               // the hard way
 906 }
 907 
 908 int MachPrologNode::reloc() const
 909 {
 910   return 0; // a large enough number
 911 }
 912 
 913 //=============================================================================
 914 #ifndef PRODUCT
 915 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 916 {
 917   Compile* C = ra_->C;
 918   if (generate_vzeroupper(C)) {
 919     st->print("vzeroupper");
 920     st->cr(); st->print("\t");
 921   }
 922 
 923   int framesize = C->frame_size_in_bytes();
 924   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 925   // Remove word for return adr already pushed
 926   // and RBP
 927   framesize -= 2*wordSize;
 928 
 929   if (framesize) {
 930     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 931     st->print("\t");
 932   }
 933 
 934   st->print_cr("popq    rbp");
 935   if (do_polling() && C->is_method_compilation()) {
 936     st->print("\t");
 937     if (SafepointMechanism::uses_thread_local_poll()) {
 938       st->print_cr("movq    rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 939                    "testl   rax, [rscratch1]\t"
 940                    "# Safepoint: poll for GC");
 941     } else if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq    rscratch1, #polling_page_address\n\t"
 943                    "testl   rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl   rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   MacroAssembler _masm(&cbuf);
 957 
 958   if (generate_vzeroupper(C)) {
 959     // Clear upper bits of YMM registers when current compiled code uses
 960     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 961     __ vzeroupper();
 962   }
 963 
 964   __ restore_stack(C);
 965 
 966 
 967   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 968     __ reserved_stack_check();
 969   }
 970 
 971   if (do_polling() && C->is_method_compilation()) {
 972     MacroAssembler _masm(&cbuf);
 973     if (SafepointMechanism::uses_thread_local_poll()) {
 974       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
 975       __ relocate(relocInfo::poll_return_type);
 976       __ testl(rax, Address(rscratch1, 0));
 977     } else {
 978       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 979       if (Assembler::is_polling_page_far()) {
 980         __ lea(rscratch1, polling_page);
 981         __ relocate(relocInfo::poll_return_type);
 982         __ testl(rax, Address(rscratch1, 0));
 983       } else {
 984         __ testl(rax, polling_page);
 985       }
 986     }
 987   }
 988 }
 989 
 990 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 991 {
 992   return MachNode::size(ra_); // too many variables; just compute it
 993                               // the hard way
 994 }
 995 
 996 int MachEpilogNode::reloc() const
 997 {
 998   return 2; // a large enough number
 999 }
1000 
1001 const Pipeline* MachEpilogNode::pipeline() const
1002 {
1003   return MachNode::pipeline_class();
1004 }
1005 
1006 int MachEpilogNode::safepoint_offset() const
1007 {
1008   return 0;
1009 }
1010 
1011 //=============================================================================
1012 
1013 enum RC {
1014   rc_bad,
1015   rc_int,
1016   rc_float,
1017   rc_stack
1018 };
1019 
1020 static enum RC rc_class(OptoReg::Name reg)
1021 {
1022   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1023 
1024   if (OptoReg::is_stack(reg)) return rc_stack;
1025 
1026   VMReg r = OptoReg::as_VMReg(reg);
1027 
1028   if (r->is_Register()) return rc_int;
1029 
1030   assert(r->is_XMMRegister(), "must be");
1031   return rc_float;
1032 }
1033 
1034 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1035 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1036                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1037 
1038 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1039                             int stack_offset, int reg, uint ireg, outputStream* st);
1040 
1041 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1042                                       int dst_offset, uint ireg, outputStream* st) {
1043   if (cbuf) {
1044     MacroAssembler _masm(cbuf);
1045     switch (ireg) {
1046     case Op_VecS:
1047       __ movq(Address(rsp, -8), rax);
1048       __ movl(rax, Address(rsp, src_offset));
1049       __ movl(Address(rsp, dst_offset), rax);
1050       __ movq(rax, Address(rsp, -8));
1051       break;
1052     case Op_VecD:
1053       __ pushq(Address(rsp, src_offset));
1054       __ popq (Address(rsp, dst_offset));
1055       break;
1056     case Op_VecX:
1057       __ pushq(Address(rsp, src_offset));
1058       __ popq (Address(rsp, dst_offset));
1059       __ pushq(Address(rsp, src_offset+8));
1060       __ popq (Address(rsp, dst_offset+8));
1061       break;
1062     case Op_VecY:
1063       __ vmovdqu(Address(rsp, -32), xmm0);
1064       __ vmovdqu(xmm0, Address(rsp, src_offset));
1065       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1066       __ vmovdqu(xmm0, Address(rsp, -32));
1067       break;
1068     case Op_VecZ:
1069       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1070       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1071       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1072       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1073       break;
1074     default:
1075       ShouldNotReachHere();
1076     }
1077 #ifndef PRODUCT
1078   } else {
1079     switch (ireg) {
1080     case Op_VecS:
1081       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1082                 "movl    rax, [rsp + #%d]\n\t"
1083                 "movl    [rsp + #%d], rax\n\t"
1084                 "movq    rax, [rsp - #8]",
1085                 src_offset, dst_offset);
1086       break;
1087     case Op_VecD:
1088       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1089                 "popq    [rsp + #%d]",
1090                 src_offset, dst_offset);
1091       break;
1092      case Op_VecX:
1093       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1094                 "popq    [rsp + #%d]\n\t"
1095                 "pushq   [rsp + #%d]\n\t"
1096                 "popq    [rsp + #%d]",
1097                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1098       break;
1099     case Op_VecY:
1100       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1101                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1102                 "vmovdqu [rsp + #%d], xmm0\n\t"
1103                 "vmovdqu xmm0, [rsp - #32]",
1104                 src_offset, dst_offset);
1105       break;
1106     case Op_VecZ:
1107       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1108                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1109                 "vmovdqu [rsp + #%d], xmm0\n\t"
1110                 "vmovdqu xmm0, [rsp - #64]",
1111                 src_offset, dst_offset);
1112       break;
1113     default:
1114       ShouldNotReachHere();
1115     }
1116 #endif
1117   }
1118 }
1119 
1120 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1121                                        PhaseRegAlloc* ra_,
1122                                        bool do_size,
1123                                        outputStream* st) const {
1124   assert(cbuf != NULL || st  != NULL, "sanity");
1125   // Get registers to move
1126   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1127   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1128   OptoReg::Name dst_second = ra_->get_reg_second(this);
1129   OptoReg::Name dst_first = ra_->get_reg_first(this);
1130 
1131   enum RC src_second_rc = rc_class(src_second);
1132   enum RC src_first_rc = rc_class(src_first);
1133   enum RC dst_second_rc = rc_class(dst_second);
1134   enum RC dst_first_rc = rc_class(dst_first);
1135 
1136   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1137          "must move at least 1 register" );
1138 
1139   if (src_first == dst_first && src_second == dst_second) {
1140     // Self copy, no move
1141     return 0;
1142   }
1143   if (bottom_type()->isa_vect() != NULL) {
1144     uint ireg = ideal_reg();
1145     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1146     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1147     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1148       // mem -> mem
1149       int src_offset = ra_->reg2offset(src_first);
1150       int dst_offset = ra_->reg2offset(dst_first);
1151       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1152     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1153       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1154     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1155       int stack_offset = ra_->reg2offset(dst_first);
1156       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1157     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1158       int stack_offset = ra_->reg2offset(src_first);
1159       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1160     } else {
1161       ShouldNotReachHere();
1162     }
1163     return 0;
1164   }
1165   if (src_first_rc == rc_stack) {
1166     // mem ->
1167     if (dst_first_rc == rc_stack) {
1168       // mem -> mem
1169       assert(src_second != dst_first, "overlap");
1170       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1171           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1172         // 64-bit
1173         int src_offset = ra_->reg2offset(src_first);
1174         int dst_offset = ra_->reg2offset(dst_first);
1175         if (cbuf) {
1176           MacroAssembler _masm(cbuf);
1177           __ pushq(Address(rsp, src_offset));
1178           __ popq (Address(rsp, dst_offset));
1179 #ifndef PRODUCT
1180         } else {
1181           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1182                     "popq    [rsp + #%d]",
1183                      src_offset, dst_offset);
1184 #endif
1185         }
1186       } else {
1187         // 32-bit
1188         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1189         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1190         // No pushl/popl, so:
1191         int src_offset = ra_->reg2offset(src_first);
1192         int dst_offset = ra_->reg2offset(dst_first);
1193         if (cbuf) {
1194           MacroAssembler _masm(cbuf);
1195           __ movq(Address(rsp, -8), rax);
1196           __ movl(rax, Address(rsp, src_offset));
1197           __ movl(Address(rsp, dst_offset), rax);
1198           __ movq(rax, Address(rsp, -8));
1199 #ifndef PRODUCT
1200         } else {
1201           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1202                     "movl    rax, [rsp + #%d]\n\t"
1203                     "movl    [rsp + #%d], rax\n\t"
1204                     "movq    rax, [rsp - #8]",
1205                      src_offset, dst_offset);
1206 #endif
1207         }
1208       }
1209       return 0;
1210     } else if (dst_first_rc == rc_int) {
1211       // mem -> gpr
1212       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1213           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1214         // 64-bit
1215         int offset = ra_->reg2offset(src_first);
1216         if (cbuf) {
1217           MacroAssembler _masm(cbuf);
1218           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1219 #ifndef PRODUCT
1220         } else {
1221           st->print("movq    %s, [rsp + #%d]\t# spill",
1222                      Matcher::regName[dst_first],
1223                      offset);
1224 #endif
1225         }
1226       } else {
1227         // 32-bit
1228         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1229         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1230         int offset = ra_->reg2offset(src_first);
1231         if (cbuf) {
1232           MacroAssembler _masm(cbuf);
1233           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1234 #ifndef PRODUCT
1235         } else {
1236           st->print("movl    %s, [rsp + #%d]\t# spill",
1237                      Matcher::regName[dst_first],
1238                      offset);
1239 #endif
1240         }
1241       }
1242       return 0;
1243     } else if (dst_first_rc == rc_float) {
1244       // mem-> xmm
1245       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1246           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1247         // 64-bit
1248         int offset = ra_->reg2offset(src_first);
1249         if (cbuf) {
1250           MacroAssembler _masm(cbuf);
1251           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1252 #ifndef PRODUCT
1253         } else {
1254           st->print("%s  %s, [rsp + #%d]\t# spill",
1255                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1256                      Matcher::regName[dst_first],
1257                      offset);
1258 #endif
1259         }
1260       } else {
1261         // 32-bit
1262         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1263         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1264         int offset = ra_->reg2offset(src_first);
1265         if (cbuf) {
1266           MacroAssembler _masm(cbuf);
1267           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1268 #ifndef PRODUCT
1269         } else {
1270           st->print("movss   %s, [rsp + #%d]\t# spill",
1271                      Matcher::regName[dst_first],
1272                      offset);
1273 #endif
1274         }
1275       }
1276       return 0;
1277     }
1278   } else if (src_first_rc == rc_int) {
1279     // gpr ->
1280     if (dst_first_rc == rc_stack) {
1281       // gpr -> mem
1282       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1283           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1284         // 64-bit
1285         int offset = ra_->reg2offset(dst_first);
1286         if (cbuf) {
1287           MacroAssembler _masm(cbuf);
1288           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1289 #ifndef PRODUCT
1290         } else {
1291           st->print("movq    [rsp + #%d], %s\t# spill",
1292                      offset,
1293                      Matcher::regName[src_first]);
1294 #endif
1295         }
1296       } else {
1297         // 32-bit
1298         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1299         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1300         int offset = ra_->reg2offset(dst_first);
1301         if (cbuf) {
1302           MacroAssembler _masm(cbuf);
1303           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1304 #ifndef PRODUCT
1305         } else {
1306           st->print("movl    [rsp + #%d], %s\t# spill",
1307                      offset,
1308                      Matcher::regName[src_first]);
1309 #endif
1310         }
1311       }
1312       return 0;
1313     } else if (dst_first_rc == rc_int) {
1314       // gpr -> gpr
1315       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1316           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1317         // 64-bit
1318         if (cbuf) {
1319           MacroAssembler _masm(cbuf);
1320           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1321                   as_Register(Matcher::_regEncode[src_first]));
1322 #ifndef PRODUCT
1323         } else {
1324           st->print("movq    %s, %s\t# spill",
1325                      Matcher::regName[dst_first],
1326                      Matcher::regName[src_first]);
1327 #endif
1328         }
1329         return 0;
1330       } else {
1331         // 32-bit
1332         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1333         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1334         if (cbuf) {
1335           MacroAssembler _masm(cbuf);
1336           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1337                   as_Register(Matcher::_regEncode[src_first]));
1338 #ifndef PRODUCT
1339         } else {
1340           st->print("movl    %s, %s\t# spill",
1341                      Matcher::regName[dst_first],
1342                      Matcher::regName[src_first]);
1343 #endif
1344         }
1345         return 0;
1346       }
1347     } else if (dst_first_rc == rc_float) {
1348       // gpr -> xmm
1349       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1350           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1351         // 64-bit
1352         if (cbuf) {
1353           MacroAssembler _masm(cbuf);
1354           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1355 #ifndef PRODUCT
1356         } else {
1357           st->print("movdq   %s, %s\t# spill",
1358                      Matcher::regName[dst_first],
1359                      Matcher::regName[src_first]);
1360 #endif
1361         }
1362       } else {
1363         // 32-bit
1364         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1365         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1366         if (cbuf) {
1367           MacroAssembler _masm(cbuf);
1368           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1369 #ifndef PRODUCT
1370         } else {
1371           st->print("movdl   %s, %s\t# spill",
1372                      Matcher::regName[dst_first],
1373                      Matcher::regName[src_first]);
1374 #endif
1375         }
1376       }
1377       return 0;
1378     }
1379   } else if (src_first_rc == rc_float) {
1380     // xmm ->
1381     if (dst_first_rc == rc_stack) {
1382       // xmm -> mem
1383       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1384           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1385         // 64-bit
1386         int offset = ra_->reg2offset(dst_first);
1387         if (cbuf) {
1388           MacroAssembler _masm(cbuf);
1389           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1390 #ifndef PRODUCT
1391         } else {
1392           st->print("movsd   [rsp + #%d], %s\t# spill",
1393                      offset,
1394                      Matcher::regName[src_first]);
1395 #endif
1396         }
1397       } else {
1398         // 32-bit
1399         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1400         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1401         int offset = ra_->reg2offset(dst_first);
1402         if (cbuf) {
1403           MacroAssembler _masm(cbuf);
1404           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1405 #ifndef PRODUCT
1406         } else {
1407           st->print("movss   [rsp + #%d], %s\t# spill",
1408                      offset,
1409                      Matcher::regName[src_first]);
1410 #endif
1411         }
1412       }
1413       return 0;
1414     } else if (dst_first_rc == rc_int) {
1415       // xmm -> gpr
1416       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1417           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1418         // 64-bit
1419         if (cbuf) {
1420           MacroAssembler _masm(cbuf);
1421           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1422 #ifndef PRODUCT
1423         } else {
1424           st->print("movdq   %s, %s\t# spill",
1425                      Matcher::regName[dst_first],
1426                      Matcher::regName[src_first]);
1427 #endif
1428         }
1429       } else {
1430         // 32-bit
1431         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1432         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1433         if (cbuf) {
1434           MacroAssembler _masm(cbuf);
1435           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1436 #ifndef PRODUCT
1437         } else {
1438           st->print("movdl   %s, %s\t# spill",
1439                      Matcher::regName[dst_first],
1440                      Matcher::regName[src_first]);
1441 #endif
1442         }
1443       }
1444       return 0;
1445     } else if (dst_first_rc == rc_float) {
1446       // xmm -> xmm
1447       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1448           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1449         // 64-bit
1450         if (cbuf) {
1451           MacroAssembler _masm(cbuf);
1452           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1453 #ifndef PRODUCT
1454         } else {
1455           st->print("%s  %s, %s\t# spill",
1456                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1457                      Matcher::regName[dst_first],
1458                      Matcher::regName[src_first]);
1459 #endif
1460         }
1461       } else {
1462         // 32-bit
1463         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1464         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1465         if (cbuf) {
1466           MacroAssembler _masm(cbuf);
1467           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1468 #ifndef PRODUCT
1469         } else {
1470           st->print("%s  %s, %s\t# spill",
1471                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1472                      Matcher::regName[dst_first],
1473                      Matcher::regName[src_first]);
1474 #endif
1475         }
1476       }
1477       return 0;
1478     }
1479   }
1480 
1481   assert(0," foo ");
1482   Unimplemented();
1483   return 0;
1484 }
1485 
1486 #ifndef PRODUCT
1487 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1488   implementation(NULL, ra_, false, st);
1489 }
1490 #endif
1491 
1492 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1493   implementation(&cbuf, ra_, false, NULL);
1494 }
1495 
1496 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1497   return MachNode::size(ra_);
1498 }
1499 
1500 //=============================================================================
1501 #ifndef PRODUCT
1502 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1503 {
1504   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1505   int reg = ra_->get_reg_first(this);
1506   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1507             Matcher::regName[reg], offset);
1508 }
1509 #endif
1510 
1511 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1512 {
1513   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1514   int reg = ra_->get_encode(this);
1515   if (offset >= 0x80) {
1516     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1517     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1518     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1519     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1520     emit_d32(cbuf, offset);
1521   } else {
1522     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1523     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1524     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1525     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1526     emit_d8(cbuf, offset);
1527   }
1528 }
1529 
1530 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1531 {
1532   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1533   return (offset < 0x80) ? 5 : 8; // REX
1534 }
1535 
1536 //=============================================================================
1537 #ifndef PRODUCT
1538 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1539 {
1540   st->print_cr("MachVEPNode");
1541 }
1542 #endif
1543 
1544 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1545 {
1546   MacroAssembler masm(&cbuf);
1547   if (!_verified) {  
1548     uint insts_size = cbuf.insts_size();
1549     if (UseCompressedClassPointers) {
1550       masm.load_klass(rscratch1, j_rarg0);
1551       masm.cmpptr(rax, rscratch1);
1552     } else {
1553       masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1554     }
1555     masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1556   } else {
1557     // Unpack value type args passed as oop and then jump to
1558     // the verified entry point (skipping the unverified entry).
1559     masm.unpack_value_args(ra_->C, _receiver_only);
1560     masm.jmp(*_verified_entry);
1561   }
1562 }
1563 
1564 uint MachVEPNode::size(PhaseRegAlloc* ra_) const
1565 {
1566   return MachNode::size(ra_); // too many variables; just compute it the hard way
1567 }
1568 
1569 //=============================================================================
1570 #ifndef PRODUCT
1571 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1572 {
1573   if (UseCompressedClassPointers) {
1574     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1575     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1576     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1577   } else {
1578     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1579                  "# Inline cache check");
1580   }
1581   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1582   st->print_cr("\tnop\t# nops to align entry point");
1583 }
1584 #endif
1585 
1586 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1587 {
1588   MacroAssembler masm(&cbuf);
1589   uint insts_size = cbuf.insts_size();
1590   if (UseCompressedClassPointers) {
1591     masm.load_klass(rscratch1, j_rarg0);
1592     masm.cmpptr(rax, rscratch1);
1593   } else {
1594     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1595   }
1596 
1597   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1598 
1599   /* WARNING these NOPs are critical so that verified entry point is properly
1600      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1601   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1602   if (OptoBreakpoint) {
1603     // Leave space for int3
1604     nops_cnt -= 1;
1605   }
1606   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1607   if (nops_cnt > 0)
1608     masm.nop(nops_cnt);
1609 }
1610 
1611 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1612 {
1613   return MachNode::size(ra_); // too many variables; just compute it
1614                               // the hard way
1615 }
1616 
1617 
1618 //=============================================================================
1619 
1620 int Matcher::regnum_to_fpu_offset(int regnum)
1621 {
1622   return regnum - 32; // The FP registers are in the second chunk
1623 }
1624 
1625 // This is UltraSparc specific, true just means we have fast l2f conversion
1626 const bool Matcher::convL2FSupported(void) {
1627   return true;
1628 }
1629 
1630 // Is this branch offset short enough that a short branch can be used?
1631 //
1632 // NOTE: If the platform does not provide any short branch variants, then
1633 //       this method should return false for offset 0.
1634 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1635   // The passed offset is relative to address of the branch.
1636   // On 86 a branch displacement is calculated relative to address
1637   // of a next instruction.
1638   offset -= br_size;
1639 
1640   // the short version of jmpConUCF2 contains multiple branches,
1641   // making the reach slightly less
1642   if (rule == jmpConUCF2_rule)
1643     return (-126 <= offset && offset <= 125);
1644   return (-128 <= offset && offset <= 127);
1645 }
1646 
1647 const bool Matcher::isSimpleConstant64(jlong value) {
1648   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1649   //return value == (int) value;  // Cf. storeImmL and immL32.
1650 
1651   // Probably always true, even if a temp register is required.
1652   return true;
1653 }
1654 
1655 // The ecx parameter to rep stosq for the ClearArray node is in words.
1656 const bool Matcher::init_array_count_is_in_bytes = false;
1657 
1658 // No additional cost for CMOVL.
1659 const int Matcher::long_cmove_cost() { return 0; }
1660 
1661 // No CMOVF/CMOVD with SSE2
1662 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1663 
1664 // Does the CPU require late expand (see block.cpp for description of late expand)?
1665 const bool Matcher::require_postalloc_expand = false;
1666 
1667 // Do we need to mask the count passed to shift instructions or does
1668 // the cpu only look at the lower 5/6 bits anyway?
1669 const bool Matcher::need_masked_shift_count = false;
1670 
1671 bool Matcher::narrow_oop_use_complex_address() {
1672   assert(UseCompressedOops, "only for compressed oops code");
1673   return (LogMinObjAlignmentInBytes <= 3);
1674 }
1675 
1676 bool Matcher::narrow_klass_use_complex_address() {
1677   assert(UseCompressedClassPointers, "only for compressed klass code");
1678   return (LogKlassAlignmentInBytes <= 3);
1679 }
1680 
1681 bool Matcher::const_oop_prefer_decode() {
1682   // Prefer ConN+DecodeN over ConP.
1683   return true;
1684 }
1685 
1686 bool Matcher::const_klass_prefer_decode() {
1687   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1688   //       or condisider the following:
1689   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1690   //return CompressedKlassPointers::base() == NULL;
1691   return true;
1692 }
1693 
1694 // Is it better to copy float constants, or load them directly from
1695 // memory?  Intel can load a float constant from a direct address,
1696 // requiring no extra registers.  Most RISCs will have to materialize
1697 // an address into a register first, so they would do better to copy
1698 // the constant from stack.
1699 const bool Matcher::rematerialize_float_constants = true; // XXX
1700 
1701 // If CPU can load and store mis-aligned doubles directly then no
1702 // fixup is needed.  Else we split the double into 2 integer pieces
1703 // and move it piece-by-piece.  Only happens when passing doubles into
1704 // C code as the Java calling convention forces doubles to be aligned.
1705 const bool Matcher::misaligned_doubles_ok = true;
1706 
1707 // No-op on amd64
1708 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1709 
1710 // Advertise here if the CPU requires explicit rounding operations to
1711 // implement the UseStrictFP mode.
1712 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1713 
1714 // Are floats conerted to double when stored to stack during deoptimization?
1715 // On x64 it is stored without convertion so we can use normal access.
1716 bool Matcher::float_in_double() { return false; }
1717 
1718 // Do ints take an entire long register or just half?
1719 const bool Matcher::int_in_long = true;
1720 
1721 // Return whether or not this register is ever used as an argument.
1722 // This function is used on startup to build the trampoline stubs in
1723 // generateOptoStub.  Registers not mentioned will be killed by the VM
1724 // call in the trampoline, and arguments in those registers not be
1725 // available to the callee.
1726 bool Matcher::can_be_java_arg(int reg)
1727 {
1728   return
1729     reg ==  RDI_num || reg == RDI_H_num ||
1730     reg ==  RSI_num || reg == RSI_H_num ||
1731     reg ==  RDX_num || reg == RDX_H_num ||
1732     reg ==  RCX_num || reg == RCX_H_num ||
1733     reg ==   R8_num || reg ==  R8_H_num ||
1734     reg ==   R9_num || reg ==  R9_H_num ||
1735     reg ==  R12_num || reg == R12_H_num ||
1736     reg == XMM0_num || reg == XMM0b_num ||
1737     reg == XMM1_num || reg == XMM1b_num ||
1738     reg == XMM2_num || reg == XMM2b_num ||
1739     reg == XMM3_num || reg == XMM3b_num ||
1740     reg == XMM4_num || reg == XMM4b_num ||
1741     reg == XMM5_num || reg == XMM5b_num ||
1742     reg == XMM6_num || reg == XMM6b_num ||
1743     reg == XMM7_num || reg == XMM7b_num;
1744 }
1745 
1746 bool Matcher::is_spillable_arg(int reg)
1747 {
1748   return can_be_java_arg(reg);
1749 }
1750 
1751 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1752   // In 64 bit mode a code which use multiply when
1753   // devisor is constant is faster than hardware
1754   // DIV instruction (it uses MulHiL).
1755   return false;
1756 }
1757 
1758 // Register for DIVI projection of divmodI
1759 RegMask Matcher::divI_proj_mask() {
1760   return INT_RAX_REG_mask();
1761 }
1762 
1763 // Register for MODI projection of divmodI
1764 RegMask Matcher::modI_proj_mask() {
1765   return INT_RDX_REG_mask();
1766 }
1767 
1768 // Register for DIVL projection of divmodL
1769 RegMask Matcher::divL_proj_mask() {
1770   return LONG_RAX_REG_mask();
1771 }
1772 
1773 // Register for MODL projection of divmodL
1774 RegMask Matcher::modL_proj_mask() {
1775   return LONG_RDX_REG_mask();
1776 }
1777 
1778 // Register for saving SP into on method handle invokes. Not used on x86_64.
1779 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1780     return NO_REG_mask();
1781 }
1782 
1783 %}
1784 
1785 //----------ENCODING BLOCK-----------------------------------------------------
1786 // This block specifies the encoding classes used by the compiler to
1787 // output byte streams.  Encoding classes are parameterized macros
1788 // used by Machine Instruction Nodes in order to generate the bit
1789 // encoding of the instruction.  Operands specify their base encoding
1790 // interface with the interface keyword.  There are currently
1791 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1792 // COND_INTER.  REG_INTER causes an operand to generate a function
1793 // which returns its register number when queried.  CONST_INTER causes
1794 // an operand to generate a function which returns the value of the
1795 // constant when queried.  MEMORY_INTER causes an operand to generate
1796 // four functions which return the Base Register, the Index Register,
1797 // the Scale Value, and the Offset Value of the operand when queried.
1798 // COND_INTER causes an operand to generate six functions which return
1799 // the encoding code (ie - encoding bits for the instruction)
1800 // associated with each basic boolean condition for a conditional
1801 // instruction.
1802 //
1803 // Instructions specify two basic values for encoding.  Again, a
1804 // function is available to check if the constant displacement is an
1805 // oop. They use the ins_encode keyword to specify their encoding
1806 // classes (which must be a sequence of enc_class names, and their
1807 // parameters, specified in the encoding block), and they use the
1808 // opcode keyword to specify, in order, their primary, secondary, and
1809 // tertiary opcode.  Only the opcode sections which a particular
1810 // instruction needs for encoding need to be specified.
1811 encode %{
1812   // Build emit functions for each basic byte or larger field in the
1813   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1814   // from C++ code in the enc_class source block.  Emit functions will
1815   // live in the main source block for now.  In future, we can
1816   // generalize this by adding a syntax that specifies the sizes of
1817   // fields in an order, so that the adlc can build the emit functions
1818   // automagically
1819 
1820   // Emit primary opcode
1821   enc_class OpcP
1822   %{
1823     emit_opcode(cbuf, $primary);
1824   %}
1825 
1826   // Emit secondary opcode
1827   enc_class OpcS
1828   %{
1829     emit_opcode(cbuf, $secondary);
1830   %}
1831 
1832   // Emit tertiary opcode
1833   enc_class OpcT
1834   %{
1835     emit_opcode(cbuf, $tertiary);
1836   %}
1837 
1838   // Emit opcode directly
1839   enc_class Opcode(immI d8)
1840   %{
1841     emit_opcode(cbuf, $d8$$constant);
1842   %}
1843 
1844   // Emit size prefix
1845   enc_class SizePrefix
1846   %{
1847     emit_opcode(cbuf, 0x66);
1848   %}
1849 
1850   enc_class reg(rRegI reg)
1851   %{
1852     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1853   %}
1854 
1855   enc_class reg_reg(rRegI dst, rRegI src)
1856   %{
1857     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1858   %}
1859 
1860   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1861   %{
1862     emit_opcode(cbuf, $opcode$$constant);
1863     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1864   %}
1865 
1866   enc_class cdql_enc(no_rax_rdx_RegI div)
1867   %{
1868     // Full implementation of Java idiv and irem; checks for
1869     // special case as described in JVM spec., p.243 & p.271.
1870     //
1871     //         normal case                           special case
1872     //
1873     // input : rax: dividend                         min_int
1874     //         reg: divisor                          -1
1875     //
1876     // output: rax: quotient  (= rax idiv reg)       min_int
1877     //         rdx: remainder (= rax irem reg)       0
1878     //
1879     //  Code sequnce:
1880     //
1881     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1882     //    5:   75 07/08                jne    e <normal>
1883     //    7:   33 d2                   xor    %edx,%edx
1884     //  [div >= 8 -> offset + 1]
1885     //  [REX_B]
1886     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1887     //    c:   74 03/04                je     11 <done>
1888     // 000000000000000e <normal>:
1889     //    e:   99                      cltd
1890     //  [div >= 8 -> offset + 1]
1891     //  [REX_B]
1892     //    f:   f7 f9                   idiv   $div
1893     // 0000000000000011 <done>:
1894 
1895     // cmp    $0x80000000,%eax
1896     emit_opcode(cbuf, 0x3d);
1897     emit_d8(cbuf, 0x00);
1898     emit_d8(cbuf, 0x00);
1899     emit_d8(cbuf, 0x00);
1900     emit_d8(cbuf, 0x80);
1901 
1902     // jne    e <normal>
1903     emit_opcode(cbuf, 0x75);
1904     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1905 
1906     // xor    %edx,%edx
1907     emit_opcode(cbuf, 0x33);
1908     emit_d8(cbuf, 0xD2);
1909 
1910     // cmp    $0xffffffffffffffff,%ecx
1911     if ($div$$reg >= 8) {
1912       emit_opcode(cbuf, Assembler::REX_B);
1913     }
1914     emit_opcode(cbuf, 0x83);
1915     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1916     emit_d8(cbuf, 0xFF);
1917 
1918     // je     11 <done>
1919     emit_opcode(cbuf, 0x74);
1920     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1921 
1922     // <normal>
1923     // cltd
1924     emit_opcode(cbuf, 0x99);
1925 
1926     // idivl (note: must be emitted by the user of this rule)
1927     // <done>
1928   %}
1929 
1930   enc_class cdqq_enc(no_rax_rdx_RegL div)
1931   %{
1932     // Full implementation of Java ldiv and lrem; checks for
1933     // special case as described in JVM spec., p.243 & p.271.
1934     //
1935     //         normal case                           special case
1936     //
1937     // input : rax: dividend                         min_long
1938     //         reg: divisor                          -1
1939     //
1940     // output: rax: quotient  (= rax idiv reg)       min_long
1941     //         rdx: remainder (= rax irem reg)       0
1942     //
1943     //  Code sequnce:
1944     //
1945     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1946     //    7:   00 00 80
1947     //    a:   48 39 d0                cmp    %rdx,%rax
1948     //    d:   75 08                   jne    17 <normal>
1949     //    f:   33 d2                   xor    %edx,%edx
1950     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1951     //   15:   74 05                   je     1c <done>
1952     // 0000000000000017 <normal>:
1953     //   17:   48 99                   cqto
1954     //   19:   48 f7 f9                idiv   $div
1955     // 000000000000001c <done>:
1956 
1957     // mov    $0x8000000000000000,%rdx
1958     emit_opcode(cbuf, Assembler::REX_W);
1959     emit_opcode(cbuf, 0xBA);
1960     emit_d8(cbuf, 0x00);
1961     emit_d8(cbuf, 0x00);
1962     emit_d8(cbuf, 0x00);
1963     emit_d8(cbuf, 0x00);
1964     emit_d8(cbuf, 0x00);
1965     emit_d8(cbuf, 0x00);
1966     emit_d8(cbuf, 0x00);
1967     emit_d8(cbuf, 0x80);
1968 
1969     // cmp    %rdx,%rax
1970     emit_opcode(cbuf, Assembler::REX_W);
1971     emit_opcode(cbuf, 0x39);
1972     emit_d8(cbuf, 0xD0);
1973 
1974     // jne    17 <normal>
1975     emit_opcode(cbuf, 0x75);
1976     emit_d8(cbuf, 0x08);
1977 
1978     // xor    %edx,%edx
1979     emit_opcode(cbuf, 0x33);
1980     emit_d8(cbuf, 0xD2);
1981 
1982     // cmp    $0xffffffffffffffff,$div
1983     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1984     emit_opcode(cbuf, 0x83);
1985     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1986     emit_d8(cbuf, 0xFF);
1987 
1988     // je     1e <done>
1989     emit_opcode(cbuf, 0x74);
1990     emit_d8(cbuf, 0x05);
1991 
1992     // <normal>
1993     // cqto
1994     emit_opcode(cbuf, Assembler::REX_W);
1995     emit_opcode(cbuf, 0x99);
1996 
1997     // idivq (note: must be emitted by the user of this rule)
1998     // <done>
1999   %}
2000 
2001   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2002   enc_class OpcSE(immI imm)
2003   %{
2004     // Emit primary opcode and set sign-extend bit
2005     // Check for 8-bit immediate, and set sign extend bit in opcode
2006     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2007       emit_opcode(cbuf, $primary | 0x02);
2008     } else {
2009       // 32-bit immediate
2010       emit_opcode(cbuf, $primary);
2011     }
2012   %}
2013 
2014   enc_class OpcSErm(rRegI dst, immI imm)
2015   %{
2016     // OpcSEr/m
2017     int dstenc = $dst$$reg;
2018     if (dstenc >= 8) {
2019       emit_opcode(cbuf, Assembler::REX_B);
2020       dstenc -= 8;
2021     }
2022     // Emit primary opcode and set sign-extend bit
2023     // Check for 8-bit immediate, and set sign extend bit in opcode
2024     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2025       emit_opcode(cbuf, $primary | 0x02);
2026     } else {
2027       // 32-bit immediate
2028       emit_opcode(cbuf, $primary);
2029     }
2030     // Emit r/m byte with secondary opcode, after primary opcode.
2031     emit_rm(cbuf, 0x3, $secondary, dstenc);
2032   %}
2033 
2034   enc_class OpcSErm_wide(rRegL dst, immI imm)
2035   %{
2036     // OpcSEr/m
2037     int dstenc = $dst$$reg;
2038     if (dstenc < 8) {
2039       emit_opcode(cbuf, Assembler::REX_W);
2040     } else {
2041       emit_opcode(cbuf, Assembler::REX_WB);
2042       dstenc -= 8;
2043     }
2044     // Emit primary opcode and set sign-extend bit
2045     // Check for 8-bit immediate, and set sign extend bit in opcode
2046     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2047       emit_opcode(cbuf, $primary | 0x02);
2048     } else {
2049       // 32-bit immediate
2050       emit_opcode(cbuf, $primary);
2051     }
2052     // Emit r/m byte with secondary opcode, after primary opcode.
2053     emit_rm(cbuf, 0x3, $secondary, dstenc);
2054   %}
2055 
2056   enc_class Con8or32(immI imm)
2057   %{
2058     // Check for 8-bit immediate, and set sign extend bit in opcode
2059     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2060       $$$emit8$imm$$constant;
2061     } else {
2062       // 32-bit immediate
2063       $$$emit32$imm$$constant;
2064     }
2065   %}
2066 
2067   enc_class opc2_reg(rRegI dst)
2068   %{
2069     // BSWAP
2070     emit_cc(cbuf, $secondary, $dst$$reg);
2071   %}
2072 
2073   enc_class opc3_reg(rRegI dst)
2074   %{
2075     // BSWAP
2076     emit_cc(cbuf, $tertiary, $dst$$reg);
2077   %}
2078 
2079   enc_class reg_opc(rRegI div)
2080   %{
2081     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2082     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2083   %}
2084 
2085   enc_class enc_cmov(cmpOp cop)
2086   %{
2087     // CMOV
2088     $$$emit8$primary;
2089     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2090   %}
2091 
2092   enc_class enc_PartialSubtypeCheck()
2093   %{
2094     Register Rrdi = as_Register(RDI_enc); // result register
2095     Register Rrax = as_Register(RAX_enc); // super class
2096     Register Rrcx = as_Register(RCX_enc); // killed
2097     Register Rrsi = as_Register(RSI_enc); // sub class
2098     Label miss;
2099     const bool set_cond_codes = true;
2100 
2101     MacroAssembler _masm(&cbuf);
2102     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2103                                      NULL, &miss,
2104                                      /*set_cond_codes:*/ true);
2105     if ($primary) {
2106       __ xorptr(Rrdi, Rrdi);
2107     }
2108     __ bind(miss);
2109   %}
2110 
2111   enc_class clear_avx %{
2112     debug_only(int off0 = cbuf.insts_size());
2113     if (generate_vzeroupper(Compile::current())) {
2114       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2115       // Clear upper bits of YMM registers when current compiled code uses
2116       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2117       MacroAssembler _masm(&cbuf);
2118       __ vzeroupper();
2119     }
2120     debug_only(int off1 = cbuf.insts_size());
2121     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2122   %}
2123 
2124   enc_class Java_To_Runtime(method meth) %{
2125     // No relocation needed
2126     MacroAssembler _masm(&cbuf);
2127     __ mov64(r10, (int64_t) $meth$$method);
2128     __ call(r10);
2129   %}
2130 
2131   enc_class Java_To_Interpreter(method meth)
2132   %{
2133     // CALL Java_To_Interpreter
2134     // This is the instruction starting address for relocation info.
2135     cbuf.set_insts_mark();
2136     $$$emit8$primary;
2137     // CALL directly to the runtime
2138     emit_d32_reloc(cbuf,
2139                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2140                    runtime_call_Relocation::spec(),
2141                    RELOC_DISP32);
2142   %}
2143 
2144   enc_class Java_Static_Call(method meth)
2145   %{
2146     // JAVA STATIC CALL
2147     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2148     // determine who we intended to call.
2149     cbuf.set_insts_mark();
2150     $$$emit8$primary;
2151 
2152     if (!_method) {
2153       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2154                      runtime_call_Relocation::spec(),
2155                      RELOC_DISP32);
2156     } else {
2157       int method_index = resolved_method_index(cbuf);
2158       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2159                                                   : static_call_Relocation::spec(method_index);
2160       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2161                      rspec, RELOC_DISP32);
2162       // Emit stubs for static call.
2163       address mark = cbuf.insts_mark();
2164       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2165       if (stub == NULL) {
2166         ciEnv::current()->record_failure("CodeCache is full");
2167         return;
2168       }
2169 #if INCLUDE_AOT
2170       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2171 #endif
2172     }
2173   %}
2174 
2175   enc_class Java_Dynamic_Call(method meth) %{
2176     MacroAssembler _masm(&cbuf);
2177     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2178   %}
2179 
2180   enc_class Java_Compiled_Call(method meth)
2181   %{
2182     // JAVA COMPILED CALL
2183     int disp = in_bytes(Method:: from_compiled_offset());
2184 
2185     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2186     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2187 
2188     // callq *disp(%rax)
2189     cbuf.set_insts_mark();
2190     $$$emit8$primary;
2191     if (disp < 0x80) {
2192       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2193       emit_d8(cbuf, disp); // Displacement
2194     } else {
2195       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2196       emit_d32(cbuf, disp); // Displacement
2197     }
2198   %}
2199 
2200   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2201   %{
2202     // SAL, SAR, SHR
2203     int dstenc = $dst$$reg;
2204     if (dstenc >= 8) {
2205       emit_opcode(cbuf, Assembler::REX_B);
2206       dstenc -= 8;
2207     }
2208     $$$emit8$primary;
2209     emit_rm(cbuf, 0x3, $secondary, dstenc);
2210     $$$emit8$shift$$constant;
2211   %}
2212 
2213   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2214   %{
2215     // SAL, SAR, SHR
2216     int dstenc = $dst$$reg;
2217     if (dstenc < 8) {
2218       emit_opcode(cbuf, Assembler::REX_W);
2219     } else {
2220       emit_opcode(cbuf, Assembler::REX_WB);
2221       dstenc -= 8;
2222     }
2223     $$$emit8$primary;
2224     emit_rm(cbuf, 0x3, $secondary, dstenc);
2225     $$$emit8$shift$$constant;
2226   %}
2227 
2228   enc_class load_immI(rRegI dst, immI src)
2229   %{
2230     int dstenc = $dst$$reg;
2231     if (dstenc >= 8) {
2232       emit_opcode(cbuf, Assembler::REX_B);
2233       dstenc -= 8;
2234     }
2235     emit_opcode(cbuf, 0xB8 | dstenc);
2236     $$$emit32$src$$constant;
2237   %}
2238 
2239   enc_class load_immL(rRegL dst, immL src)
2240   %{
2241     int dstenc = $dst$$reg;
2242     if (dstenc < 8) {
2243       emit_opcode(cbuf, Assembler::REX_W);
2244     } else {
2245       emit_opcode(cbuf, Assembler::REX_WB);
2246       dstenc -= 8;
2247     }
2248     emit_opcode(cbuf, 0xB8 | dstenc);
2249     emit_d64(cbuf, $src$$constant);
2250   %}
2251 
2252   enc_class load_immUL32(rRegL dst, immUL32 src)
2253   %{
2254     // same as load_immI, but this time we care about zeroes in the high word
2255     int dstenc = $dst$$reg;
2256     if (dstenc >= 8) {
2257       emit_opcode(cbuf, Assembler::REX_B);
2258       dstenc -= 8;
2259     }
2260     emit_opcode(cbuf, 0xB8 | dstenc);
2261     $$$emit32$src$$constant;
2262   %}
2263 
2264   enc_class load_immL32(rRegL dst, immL32 src)
2265   %{
2266     int dstenc = $dst$$reg;
2267     if (dstenc < 8) {
2268       emit_opcode(cbuf, Assembler::REX_W);
2269     } else {
2270       emit_opcode(cbuf, Assembler::REX_WB);
2271       dstenc -= 8;
2272     }
2273     emit_opcode(cbuf, 0xC7);
2274     emit_rm(cbuf, 0x03, 0x00, dstenc);
2275     $$$emit32$src$$constant;
2276   %}
2277 
2278   enc_class load_immP31(rRegP dst, immP32 src)
2279   %{
2280     // same as load_immI, but this time we care about zeroes in the high word
2281     int dstenc = $dst$$reg;
2282     if (dstenc >= 8) {
2283       emit_opcode(cbuf, Assembler::REX_B);
2284       dstenc -= 8;
2285     }
2286     emit_opcode(cbuf, 0xB8 | dstenc);
2287     $$$emit32$src$$constant;
2288   %}
2289 
2290   enc_class load_immP(rRegP dst, immP src)
2291   %{
2292     int dstenc = $dst$$reg;
2293     if (dstenc < 8) {
2294       emit_opcode(cbuf, Assembler::REX_W);
2295     } else {
2296       emit_opcode(cbuf, Assembler::REX_WB);
2297       dstenc -= 8;
2298     }
2299     emit_opcode(cbuf, 0xB8 | dstenc);
2300     // This next line should be generated from ADLC
2301     if ($src->constant_reloc() != relocInfo::none) {
2302       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2303     } else {
2304       emit_d64(cbuf, $src$$constant);
2305     }
2306   %}
2307 
2308   enc_class Con32(immI src)
2309   %{
2310     // Output immediate
2311     $$$emit32$src$$constant;
2312   %}
2313 
2314   enc_class Con32F_as_bits(immF src)
2315   %{
2316     // Output Float immediate bits
2317     jfloat jf = $src$$constant;
2318     jint jf_as_bits = jint_cast(jf);
2319     emit_d32(cbuf, jf_as_bits);
2320   %}
2321 
2322   enc_class Con16(immI src)
2323   %{
2324     // Output immediate
2325     $$$emit16$src$$constant;
2326   %}
2327 
2328   // How is this different from Con32??? XXX
2329   enc_class Con_d32(immI src)
2330   %{
2331     emit_d32(cbuf,$src$$constant);
2332   %}
2333 
2334   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2335     // Output immediate memory reference
2336     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2337     emit_d32(cbuf, 0x00);
2338   %}
2339 
2340   enc_class lock_prefix()
2341   %{
2342     emit_opcode(cbuf, 0xF0); // lock
2343   %}
2344 
2345   enc_class REX_mem(memory mem)
2346   %{
2347     if ($mem$$base >= 8) {
2348       if ($mem$$index < 8) {
2349         emit_opcode(cbuf, Assembler::REX_B);
2350       } else {
2351         emit_opcode(cbuf, Assembler::REX_XB);
2352       }
2353     } else {
2354       if ($mem$$index >= 8) {
2355         emit_opcode(cbuf, Assembler::REX_X);
2356       }
2357     }
2358   %}
2359 
2360   enc_class REX_mem_wide(memory mem)
2361   %{
2362     if ($mem$$base >= 8) {
2363       if ($mem$$index < 8) {
2364         emit_opcode(cbuf, Assembler::REX_WB);
2365       } else {
2366         emit_opcode(cbuf, Assembler::REX_WXB);
2367       }
2368     } else {
2369       if ($mem$$index < 8) {
2370         emit_opcode(cbuf, Assembler::REX_W);
2371       } else {
2372         emit_opcode(cbuf, Assembler::REX_WX);
2373       }
2374     }
2375   %}
2376 
2377   // for byte regs
2378   enc_class REX_breg(rRegI reg)
2379   %{
2380     if ($reg$$reg >= 4) {
2381       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2382     }
2383   %}
2384 
2385   // for byte regs
2386   enc_class REX_reg_breg(rRegI dst, rRegI src)
2387   %{
2388     if ($dst$$reg < 8) {
2389       if ($src$$reg >= 4) {
2390         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2391       }
2392     } else {
2393       if ($src$$reg < 8) {
2394         emit_opcode(cbuf, Assembler::REX_R);
2395       } else {
2396         emit_opcode(cbuf, Assembler::REX_RB);
2397       }
2398     }
2399   %}
2400 
2401   // for byte regs
2402   enc_class REX_breg_mem(rRegI reg, memory mem)
2403   %{
2404     if ($reg$$reg < 8) {
2405       if ($mem$$base < 8) {
2406         if ($mem$$index >= 8) {
2407           emit_opcode(cbuf, Assembler::REX_X);
2408         } else if ($reg$$reg >= 4) {
2409           emit_opcode(cbuf, Assembler::REX);
2410         }
2411       } else {
2412         if ($mem$$index < 8) {
2413           emit_opcode(cbuf, Assembler::REX_B);
2414         } else {
2415           emit_opcode(cbuf, Assembler::REX_XB);
2416         }
2417       }
2418     } else {
2419       if ($mem$$base < 8) {
2420         if ($mem$$index < 8) {
2421           emit_opcode(cbuf, Assembler::REX_R);
2422         } else {
2423           emit_opcode(cbuf, Assembler::REX_RX);
2424         }
2425       } else {
2426         if ($mem$$index < 8) {
2427           emit_opcode(cbuf, Assembler::REX_RB);
2428         } else {
2429           emit_opcode(cbuf, Assembler::REX_RXB);
2430         }
2431       }
2432     }
2433   %}
2434 
2435   enc_class REX_reg(rRegI reg)
2436   %{
2437     if ($reg$$reg >= 8) {
2438       emit_opcode(cbuf, Assembler::REX_B);
2439     }
2440   %}
2441 
2442   enc_class REX_reg_wide(rRegI reg)
2443   %{
2444     if ($reg$$reg < 8) {
2445       emit_opcode(cbuf, Assembler::REX_W);
2446     } else {
2447       emit_opcode(cbuf, Assembler::REX_WB);
2448     }
2449   %}
2450 
2451   enc_class REX_reg_reg(rRegI dst, rRegI src)
2452   %{
2453     if ($dst$$reg < 8) {
2454       if ($src$$reg >= 8) {
2455         emit_opcode(cbuf, Assembler::REX_B);
2456       }
2457     } else {
2458       if ($src$$reg < 8) {
2459         emit_opcode(cbuf, Assembler::REX_R);
2460       } else {
2461         emit_opcode(cbuf, Assembler::REX_RB);
2462       }
2463     }
2464   %}
2465 
2466   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2467   %{
2468     if ($dst$$reg < 8) {
2469       if ($src$$reg < 8) {
2470         emit_opcode(cbuf, Assembler::REX_W);
2471       } else {
2472         emit_opcode(cbuf, Assembler::REX_WB);
2473       }
2474     } else {
2475       if ($src$$reg < 8) {
2476         emit_opcode(cbuf, Assembler::REX_WR);
2477       } else {
2478         emit_opcode(cbuf, Assembler::REX_WRB);
2479       }
2480     }
2481   %}
2482 
2483   enc_class REX_reg_mem(rRegI reg, memory mem)
2484   %{
2485     if ($reg$$reg < 8) {
2486       if ($mem$$base < 8) {
2487         if ($mem$$index >= 8) {
2488           emit_opcode(cbuf, Assembler::REX_X);
2489         }
2490       } else {
2491         if ($mem$$index < 8) {
2492           emit_opcode(cbuf, Assembler::REX_B);
2493         } else {
2494           emit_opcode(cbuf, Assembler::REX_XB);
2495         }
2496       }
2497     } else {
2498       if ($mem$$base < 8) {
2499         if ($mem$$index < 8) {
2500           emit_opcode(cbuf, Assembler::REX_R);
2501         } else {
2502           emit_opcode(cbuf, Assembler::REX_RX);
2503         }
2504       } else {
2505         if ($mem$$index < 8) {
2506           emit_opcode(cbuf, Assembler::REX_RB);
2507         } else {
2508           emit_opcode(cbuf, Assembler::REX_RXB);
2509         }
2510       }
2511     }
2512   %}
2513 
2514   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2515   %{
2516     if ($reg$$reg < 8) {
2517       if ($mem$$base < 8) {
2518         if ($mem$$index < 8) {
2519           emit_opcode(cbuf, Assembler::REX_W);
2520         } else {
2521           emit_opcode(cbuf, Assembler::REX_WX);
2522         }
2523       } else {
2524         if ($mem$$index < 8) {
2525           emit_opcode(cbuf, Assembler::REX_WB);
2526         } else {
2527           emit_opcode(cbuf, Assembler::REX_WXB);
2528         }
2529       }
2530     } else {
2531       if ($mem$$base < 8) {
2532         if ($mem$$index < 8) {
2533           emit_opcode(cbuf, Assembler::REX_WR);
2534         } else {
2535           emit_opcode(cbuf, Assembler::REX_WRX);
2536         }
2537       } else {
2538         if ($mem$$index < 8) {
2539           emit_opcode(cbuf, Assembler::REX_WRB);
2540         } else {
2541           emit_opcode(cbuf, Assembler::REX_WRXB);
2542         }
2543       }
2544     }
2545   %}
2546 
2547   enc_class reg_mem(rRegI ereg, memory mem)
2548   %{
2549     // High registers handle in encode_RegMem
2550     int reg = $ereg$$reg;
2551     int base = $mem$$base;
2552     int index = $mem$$index;
2553     int scale = $mem$$scale;
2554     int disp = $mem$$disp;
2555     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2556 
2557     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2558   %}
2559 
2560   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2561   %{
2562     int rm_byte_opcode = $rm_opcode$$constant;
2563 
2564     // High registers handle in encode_RegMem
2565     int base = $mem$$base;
2566     int index = $mem$$index;
2567     int scale = $mem$$scale;
2568     int displace = $mem$$disp;
2569 
2570     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2571                                             // working with static
2572                                             // globals
2573     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2574                   disp_reloc);
2575   %}
2576 
2577   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2578   %{
2579     int reg_encoding = $dst$$reg;
2580     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2581     int index        = 0x04;            // 0x04 indicates no index
2582     int scale        = 0x00;            // 0x00 indicates no scale
2583     int displace     = $src1$$constant; // 0x00 indicates no displacement
2584     relocInfo::relocType disp_reloc = relocInfo::none;
2585     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2586                   disp_reloc);
2587   %}
2588 
2589   enc_class neg_reg(rRegI dst)
2590   %{
2591     int dstenc = $dst$$reg;
2592     if (dstenc >= 8) {
2593       emit_opcode(cbuf, Assembler::REX_B);
2594       dstenc -= 8;
2595     }
2596     // NEG $dst
2597     emit_opcode(cbuf, 0xF7);
2598     emit_rm(cbuf, 0x3, 0x03, dstenc);
2599   %}
2600 
2601   enc_class neg_reg_wide(rRegI dst)
2602   %{
2603     int dstenc = $dst$$reg;
2604     if (dstenc < 8) {
2605       emit_opcode(cbuf, Assembler::REX_W);
2606     } else {
2607       emit_opcode(cbuf, Assembler::REX_WB);
2608       dstenc -= 8;
2609     }
2610     // NEG $dst
2611     emit_opcode(cbuf, 0xF7);
2612     emit_rm(cbuf, 0x3, 0x03, dstenc);
2613   %}
2614 
2615   enc_class setLT_reg(rRegI dst)
2616   %{
2617     int dstenc = $dst$$reg;
2618     if (dstenc >= 8) {
2619       emit_opcode(cbuf, Assembler::REX_B);
2620       dstenc -= 8;
2621     } else if (dstenc >= 4) {
2622       emit_opcode(cbuf, Assembler::REX);
2623     }
2624     // SETLT $dst
2625     emit_opcode(cbuf, 0x0F);
2626     emit_opcode(cbuf, 0x9C);
2627     emit_rm(cbuf, 0x3, 0x0, dstenc);
2628   %}
2629 
2630   enc_class setNZ_reg(rRegI dst)
2631   %{
2632     int dstenc = $dst$$reg;
2633     if (dstenc >= 8) {
2634       emit_opcode(cbuf, Assembler::REX_B);
2635       dstenc -= 8;
2636     } else if (dstenc >= 4) {
2637       emit_opcode(cbuf, Assembler::REX);
2638     }
2639     // SETNZ $dst
2640     emit_opcode(cbuf, 0x0F);
2641     emit_opcode(cbuf, 0x95);
2642     emit_rm(cbuf, 0x3, 0x0, dstenc);
2643   %}
2644 
2645 
2646   // Compare the lonogs and set -1, 0, or 1 into dst
2647   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2648   %{
2649     int src1enc = $src1$$reg;
2650     int src2enc = $src2$$reg;
2651     int dstenc = $dst$$reg;
2652 
2653     // cmpq $src1, $src2
2654     if (src1enc < 8) {
2655       if (src2enc < 8) {
2656         emit_opcode(cbuf, Assembler::REX_W);
2657       } else {
2658         emit_opcode(cbuf, Assembler::REX_WB);
2659       }
2660     } else {
2661       if (src2enc < 8) {
2662         emit_opcode(cbuf, Assembler::REX_WR);
2663       } else {
2664         emit_opcode(cbuf, Assembler::REX_WRB);
2665       }
2666     }
2667     emit_opcode(cbuf, 0x3B);
2668     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2669 
2670     // movl $dst, -1
2671     if (dstenc >= 8) {
2672       emit_opcode(cbuf, Assembler::REX_B);
2673     }
2674     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2675     emit_d32(cbuf, -1);
2676 
2677     // jl,s done
2678     emit_opcode(cbuf, 0x7C);
2679     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2680 
2681     // setne $dst
2682     if (dstenc >= 4) {
2683       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2684     }
2685     emit_opcode(cbuf, 0x0F);
2686     emit_opcode(cbuf, 0x95);
2687     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2688 
2689     // movzbl $dst, $dst
2690     if (dstenc >= 4) {
2691       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2692     }
2693     emit_opcode(cbuf, 0x0F);
2694     emit_opcode(cbuf, 0xB6);
2695     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2696   %}
2697 
2698   enc_class Push_ResultXD(regD dst) %{
2699     MacroAssembler _masm(&cbuf);
2700     __ fstp_d(Address(rsp, 0));
2701     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2702     __ addptr(rsp, 8);
2703   %}
2704 
2705   enc_class Push_SrcXD(regD src) %{
2706     MacroAssembler _masm(&cbuf);
2707     __ subptr(rsp, 8);
2708     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2709     __ fld_d(Address(rsp, 0));
2710   %}
2711 
2712 
2713   enc_class enc_rethrow()
2714   %{
2715     cbuf.set_insts_mark();
2716     emit_opcode(cbuf, 0xE9); // jmp entry
2717     emit_d32_reloc(cbuf,
2718                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2719                    runtime_call_Relocation::spec(),
2720                    RELOC_DISP32);
2721   %}
2722 
2723 %}
2724 
2725 
2726 
2727 //----------FRAME--------------------------------------------------------------
2728 // Definition of frame structure and management information.
2729 //
2730 //  S T A C K   L A Y O U T    Allocators stack-slot number
2731 //                             |   (to get allocators register number
2732 //  G  Owned by    |        |  v    add OptoReg::stack0())
2733 //  r   CALLER     |        |
2734 //  o     |        +--------+      pad to even-align allocators stack-slot
2735 //  w     V        |  pad0  |        numbers; owned by CALLER
2736 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2737 //  h     ^        |   in   |  5
2738 //        |        |  args  |  4   Holes in incoming args owned by SELF
2739 //  |     |        |        |  3
2740 //  |     |        +--------+
2741 //  V     |        | old out|      Empty on Intel, window on Sparc
2742 //        |    old |preserve|      Must be even aligned.
2743 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2744 //        |        |   in   |  3   area for Intel ret address
2745 //     Owned by    |preserve|      Empty on Sparc.
2746 //       SELF      +--------+
2747 //        |        |  pad2  |  2   pad to align old SP
2748 //        |        +--------+  1
2749 //        |        | locks  |  0
2750 //        |        +--------+----> OptoReg::stack0(), even aligned
2751 //        |        |  pad1  | 11   pad to align new SP
2752 //        |        +--------+
2753 //        |        |        | 10
2754 //        |        | spills |  9   spills
2755 //        V        |        |  8   (pad0 slot for callee)
2756 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2757 //        ^        |  out   |  7
2758 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2759 //     Owned by    +--------+
2760 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2761 //        |    new |preserve|      Must be even-aligned.
2762 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2763 //        |        |        |
2764 //
2765 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2766 //         known from SELF's arguments and the Java calling convention.
2767 //         Region 6-7 is determined per call site.
2768 // Note 2: If the calling convention leaves holes in the incoming argument
2769 //         area, those holes are owned by SELF.  Holes in the outgoing area
2770 //         are owned by the CALLEE.  Holes should not be nessecary in the
2771 //         incoming area, as the Java calling convention is completely under
2772 //         the control of the AD file.  Doubles can be sorted and packed to
2773 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2774 //         varargs C calling conventions.
2775 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2776 //         even aligned with pad0 as needed.
2777 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2778 //         region 6-11 is even aligned; it may be padded out more so that
2779 //         the region from SP to FP meets the minimum stack alignment.
2780 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2781 //         alignment.  Region 11, pad1, may be dynamically extended so that
2782 //         SP meets the minimum alignment.
2783 
2784 frame
2785 %{
2786   // What direction does stack grow in (assumed to be same for C & Java)
2787   stack_direction(TOWARDS_LOW);
2788 
2789   // These three registers define part of the calling convention
2790   // between compiled code and the interpreter.
2791   inline_cache_reg(RAX);                // Inline Cache Register
2792   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2793                                         // calling interpreter
2794 
2795   // Optional: name the operand used by cisc-spilling to access
2796   // [stack_pointer + offset]
2797   cisc_spilling_operand_name(indOffset32);
2798 
2799   // Number of stack slots consumed by locking an object
2800   sync_stack_slots(2);
2801 
2802   // Compiled code's Frame Pointer
2803   frame_pointer(RSP);
2804 
2805   // Interpreter stores its frame pointer in a register which is
2806   // stored to the stack by I2CAdaptors.
2807   // I2CAdaptors convert from interpreted java to compiled java.
2808   interpreter_frame_pointer(RBP);
2809 
2810   // Stack alignment requirement
2811   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2812 
2813   // Number of stack slots between incoming argument block and the start of
2814   // a new frame.  The PROLOG must add this many slots to the stack.  The
2815   // EPILOG must remove this many slots.  amd64 needs two slots for
2816   // return address.
2817   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2818 
2819   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2820   // for calls to C.  Supports the var-args backing area for register parms.
2821   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2822 
2823   // The after-PROLOG location of the return address.  Location of
2824   // return address specifies a type (REG or STACK) and a number
2825   // representing the register number (i.e. - use a register name) or
2826   // stack slot.
2827   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2828   // Otherwise, it is above the locks and verification slot and alignment word
2829   return_addr(STACK - 2 +
2830               align_up((Compile::current()->in_preserve_stack_slots() +
2831                         Compile::current()->fixed_slots()),
2832                        stack_alignment_in_slots()));
2833 
2834   // Body of function which returns an integer array locating
2835   // arguments either in registers or in stack slots.  Passed an array
2836   // of ideal registers called "sig" and a "length" count.  Stack-slot
2837   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2838   // arguments for a CALLEE.  Incoming stack arguments are
2839   // automatically biased by the preserve_stack_slots field above.
2840 
2841   calling_convention
2842   %{
2843     // No difference between ingoing/outgoing just pass false
2844     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2845   %}
2846 
2847   c_calling_convention
2848   %{
2849     // This is obviously always outgoing
2850     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2851   %}
2852 
2853   // Location of compiled Java return values.  Same as C for now.
2854   return_value
2855   %{
2856     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2857            "only return normal values");
2858 
2859     static const int lo[Op_RegL + 1] = {
2860       0,
2861       0,
2862       RAX_num,  // Op_RegN
2863       RAX_num,  // Op_RegI
2864       RAX_num,  // Op_RegP
2865       XMM0_num, // Op_RegF
2866       XMM0_num, // Op_RegD
2867       RAX_num   // Op_RegL
2868     };
2869     static const int hi[Op_RegL + 1] = {
2870       0,
2871       0,
2872       OptoReg::Bad, // Op_RegN
2873       OptoReg::Bad, // Op_RegI
2874       RAX_H_num,    // Op_RegP
2875       OptoReg::Bad, // Op_RegF
2876       XMM0b_num,    // Op_RegD
2877       RAX_H_num     // Op_RegL
2878     };
2879     // Excluded flags and vector registers.
2880     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2881     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2882   %}
2883 %}
2884 
2885 //----------ATTRIBUTES---------------------------------------------------------
2886 //----------Operand Attributes-------------------------------------------------
2887 op_attrib op_cost(0);        // Required cost attribute
2888 
2889 //----------Instruction Attributes---------------------------------------------
2890 ins_attrib ins_cost(100);       // Required cost attribute
2891 ins_attrib ins_size(8);         // Required size attribute (in bits)
2892 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2893                                 // a non-matching short branch variant
2894                                 // of some long branch?
2895 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2896                                 // be a power of 2) specifies the
2897                                 // alignment that some part of the
2898                                 // instruction (not necessarily the
2899                                 // start) requires.  If > 1, a
2900                                 // compute_padding() function must be
2901                                 // provided for the instruction
2902 
2903 //----------OPERANDS-----------------------------------------------------------
2904 // Operand definitions must precede instruction definitions for correct parsing
2905 // in the ADLC because operands constitute user defined types which are used in
2906 // instruction definitions.
2907 
2908 //----------Simple Operands----------------------------------------------------
2909 // Immediate Operands
2910 // Integer Immediate
2911 operand immI()
2912 %{
2913   match(ConI);
2914 
2915   op_cost(10);
2916   format %{ %}
2917   interface(CONST_INTER);
2918 %}
2919 
2920 // Constant for test vs zero
2921 operand immI0()
2922 %{
2923   predicate(n->get_int() == 0);
2924   match(ConI);
2925 
2926   op_cost(0);
2927   format %{ %}
2928   interface(CONST_INTER);
2929 %}
2930 
2931 // Constant for increment
2932 operand immI1()
2933 %{
2934   predicate(n->get_int() == 1);
2935   match(ConI);
2936 
2937   op_cost(0);
2938   format %{ %}
2939   interface(CONST_INTER);
2940 %}
2941 
2942 // Constant for decrement
2943 operand immI_M1()
2944 %{
2945   predicate(n->get_int() == -1);
2946   match(ConI);
2947 
2948   op_cost(0);
2949   format %{ %}
2950   interface(CONST_INTER);
2951 %}
2952 
2953 // Valid scale values for addressing modes
2954 operand immI2()
2955 %{
2956   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2957   match(ConI);
2958 
2959   format %{ %}
2960   interface(CONST_INTER);
2961 %}
2962 
2963 operand immI8()
2964 %{
2965   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2966   match(ConI);
2967 
2968   op_cost(5);
2969   format %{ %}
2970   interface(CONST_INTER);
2971 %}
2972 
2973 operand immU8()
2974 %{
2975   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2976   match(ConI);
2977 
2978   op_cost(5);
2979   format %{ %}
2980   interface(CONST_INTER);
2981 %}
2982 
2983 operand immI16()
2984 %{
2985   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2986   match(ConI);
2987 
2988   op_cost(10);
2989   format %{ %}
2990   interface(CONST_INTER);
2991 %}
2992 
2993 // Int Immediate non-negative
2994 operand immU31()
2995 %{
2996   predicate(n->get_int() >= 0);
2997   match(ConI);
2998 
2999   op_cost(0);
3000   format %{ %}
3001   interface(CONST_INTER);
3002 %}
3003 
3004 // Constant for long shifts
3005 operand immI_32()
3006 %{
3007   predicate( n->get_int() == 32 );
3008   match(ConI);
3009 
3010   op_cost(0);
3011   format %{ %}
3012   interface(CONST_INTER);
3013 %}
3014 
3015 // Constant for long shifts
3016 operand immI_64()
3017 %{
3018   predicate( n->get_int() == 64 );
3019   match(ConI);
3020 
3021   op_cost(0);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // Pointer Immediate
3027 operand immP()
3028 %{
3029   match(ConP);
3030 
3031   op_cost(10);
3032   format %{ %}
3033   interface(CONST_INTER);
3034 %}
3035 
3036 // NULL Pointer Immediate
3037 operand immP0()
3038 %{
3039   predicate(n->get_ptr() == 0);
3040   match(ConP);
3041 
3042   op_cost(5);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 // Pointer Immediate
3048 operand immN() %{
3049   match(ConN);
3050 
3051   op_cost(10);
3052   format %{ %}
3053   interface(CONST_INTER);
3054 %}
3055 
3056 operand immNKlass() %{
3057   match(ConNKlass);
3058 
3059   op_cost(10);
3060   format %{ %}
3061   interface(CONST_INTER);
3062 %}
3063 
3064 // NULL Pointer Immediate
3065 operand immN0() %{
3066   predicate(n->get_narrowcon() == 0);
3067   match(ConN);
3068 
3069   op_cost(5);
3070   format %{ %}
3071   interface(CONST_INTER);
3072 %}
3073 
3074 operand immP31()
3075 %{
3076   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3077             && (n->get_ptr() >> 31) == 0);
3078   match(ConP);
3079 
3080   op_cost(5);
3081   format %{ %}
3082   interface(CONST_INTER);
3083 %}
3084 
3085 
3086 // Long Immediate
3087 operand immL()
3088 %{
3089   match(ConL);
3090 
3091   op_cost(20);
3092   format %{ %}
3093   interface(CONST_INTER);
3094 %}
3095 
3096 // Long Immediate 8-bit
3097 operand immL8()
3098 %{
3099   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3100   match(ConL);
3101 
3102   op_cost(5);
3103   format %{ %}
3104   interface(CONST_INTER);
3105 %}
3106 
3107 // Long Immediate 32-bit unsigned
3108 operand immUL32()
3109 %{
3110   predicate(n->get_long() == (unsigned int) (n->get_long()));
3111   match(ConL);
3112 
3113   op_cost(10);
3114   format %{ %}
3115   interface(CONST_INTER);
3116 %}
3117 
3118 // Long Immediate 32-bit signed
3119 operand immL32()
3120 %{
3121   predicate(n->get_long() == (int) (n->get_long()));
3122   match(ConL);
3123 
3124   op_cost(15);
3125   format %{ %}
3126   interface(CONST_INTER);
3127 %}
3128 
3129 // Long Immediate zero
3130 operand immL0()
3131 %{
3132   predicate(n->get_long() == 0L);
3133   match(ConL);
3134 
3135   op_cost(10);
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // Constant for increment
3141 operand immL1()
3142 %{
3143   predicate(n->get_long() == 1);
3144   match(ConL);
3145 
3146   format %{ %}
3147   interface(CONST_INTER);
3148 %}
3149 
3150 // Constant for decrement
3151 operand immL_M1()
3152 %{
3153   predicate(n->get_long() == -1);
3154   match(ConL);
3155 
3156   format %{ %}
3157   interface(CONST_INTER);
3158 %}
3159 
3160 // Long Immediate: the value 10
3161 operand immL10()
3162 %{
3163   predicate(n->get_long() == 10);
3164   match(ConL);
3165 
3166   format %{ %}
3167   interface(CONST_INTER);
3168 %}
3169 
3170 // Long immediate from 0 to 127.
3171 // Used for a shorter form of long mul by 10.
3172 operand immL_127()
3173 %{
3174   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3175   match(ConL);
3176 
3177   op_cost(10);
3178   format %{ %}
3179   interface(CONST_INTER);
3180 %}
3181 
3182 // Long Immediate: low 32-bit mask
3183 operand immL_32bits()
3184 %{
3185   predicate(n->get_long() == 0xFFFFFFFFL);
3186   match(ConL);
3187   op_cost(20);
3188 
3189   format %{ %}
3190   interface(CONST_INTER);
3191 %}
3192 
3193 // Float Immediate zero
3194 operand immF0()
3195 %{
3196   predicate(jint_cast(n->getf()) == 0);
3197   match(ConF);
3198 
3199   op_cost(5);
3200   format %{ %}
3201   interface(CONST_INTER);
3202 %}
3203 
3204 // Float Immediate
3205 operand immF()
3206 %{
3207   match(ConF);
3208 
3209   op_cost(15);
3210   format %{ %}
3211   interface(CONST_INTER);
3212 %}
3213 
3214 // Double Immediate zero
3215 operand immD0()
3216 %{
3217   predicate(jlong_cast(n->getd()) == 0);
3218   match(ConD);
3219 
3220   op_cost(5);
3221   format %{ %}
3222   interface(CONST_INTER);
3223 %}
3224 
3225 // Double Immediate
3226 operand immD()
3227 %{
3228   match(ConD);
3229 
3230   op_cost(15);
3231   format %{ %}
3232   interface(CONST_INTER);
3233 %}
3234 
3235 // Immediates for special shifts (sign extend)
3236 
3237 // Constants for increment
3238 operand immI_16()
3239 %{
3240   predicate(n->get_int() == 16);
3241   match(ConI);
3242 
3243   format %{ %}
3244   interface(CONST_INTER);
3245 %}
3246 
3247 operand immI_24()
3248 %{
3249   predicate(n->get_int() == 24);
3250   match(ConI);
3251 
3252   format %{ %}
3253   interface(CONST_INTER);
3254 %}
3255 
3256 // Constant for byte-wide masking
3257 operand immI_255()
3258 %{
3259   predicate(n->get_int() == 255);
3260   match(ConI);
3261 
3262   format %{ %}
3263   interface(CONST_INTER);
3264 %}
3265 
3266 // Constant for short-wide masking
3267 operand immI_65535()
3268 %{
3269   predicate(n->get_int() == 65535);
3270   match(ConI);
3271 
3272   format %{ %}
3273   interface(CONST_INTER);
3274 %}
3275 
3276 // Constant for byte-wide masking
3277 operand immL_255()
3278 %{
3279   predicate(n->get_long() == 255);
3280   match(ConL);
3281 
3282   format %{ %}
3283   interface(CONST_INTER);
3284 %}
3285 
3286 // Constant for short-wide masking
3287 operand immL_65535()
3288 %{
3289   predicate(n->get_long() == 65535);
3290   match(ConL);
3291 
3292   format %{ %}
3293   interface(CONST_INTER);
3294 %}
3295 
3296 // Register Operands
3297 // Integer Register
3298 operand rRegI()
3299 %{
3300   constraint(ALLOC_IN_RC(int_reg));
3301   match(RegI);
3302 
3303   match(rax_RegI);
3304   match(rbx_RegI);
3305   match(rcx_RegI);
3306   match(rdx_RegI);
3307   match(rdi_RegI);
3308 
3309   format %{ %}
3310   interface(REG_INTER);
3311 %}
3312 
3313 // Special Registers
3314 operand rax_RegI()
3315 %{
3316   constraint(ALLOC_IN_RC(int_rax_reg));
3317   match(RegI);
3318   match(rRegI);
3319 
3320   format %{ "RAX" %}
3321   interface(REG_INTER);
3322 %}
3323 
3324 // Special Registers
3325 operand rbx_RegI()
3326 %{
3327   constraint(ALLOC_IN_RC(int_rbx_reg));
3328   match(RegI);
3329   match(rRegI);
3330 
3331   format %{ "RBX" %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 operand rcx_RegI()
3336 %{
3337   constraint(ALLOC_IN_RC(int_rcx_reg));
3338   match(RegI);
3339   match(rRegI);
3340 
3341   format %{ "RCX" %}
3342   interface(REG_INTER);
3343 %}
3344 
3345 operand rdx_RegI()
3346 %{
3347   constraint(ALLOC_IN_RC(int_rdx_reg));
3348   match(RegI);
3349   match(rRegI);
3350 
3351   format %{ "RDX" %}
3352   interface(REG_INTER);
3353 %}
3354 
3355 operand rdi_RegI()
3356 %{
3357   constraint(ALLOC_IN_RC(int_rdi_reg));
3358   match(RegI);
3359   match(rRegI);
3360 
3361   format %{ "RDI" %}
3362   interface(REG_INTER);
3363 %}
3364 
3365 operand no_rcx_RegI()
3366 %{
3367   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3368   match(RegI);
3369   match(rax_RegI);
3370   match(rbx_RegI);
3371   match(rdx_RegI);
3372   match(rdi_RegI);
3373 
3374   format %{ %}
3375   interface(REG_INTER);
3376 %}
3377 
3378 operand no_rax_rdx_RegI()
3379 %{
3380   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3381   match(RegI);
3382   match(rbx_RegI);
3383   match(rcx_RegI);
3384   match(rdi_RegI);
3385 
3386   format %{ %}
3387   interface(REG_INTER);
3388 %}
3389 
3390 // Pointer Register
3391 operand any_RegP()
3392 %{
3393   constraint(ALLOC_IN_RC(any_reg));
3394   match(RegP);
3395   match(rax_RegP);
3396   match(rbx_RegP);
3397   match(rdi_RegP);
3398   match(rsi_RegP);
3399   match(rbp_RegP);
3400   match(r15_RegP);
3401   match(rRegP);
3402 
3403   format %{ %}
3404   interface(REG_INTER);
3405 %}
3406 
3407 operand rRegP()
3408 %{
3409   constraint(ALLOC_IN_RC(ptr_reg));
3410   match(RegP);
3411   match(rax_RegP);
3412   match(rbx_RegP);
3413   match(rdi_RegP);
3414   match(rsi_RegP);
3415   match(rbp_RegP);  // See Q&A below about
3416   match(r15_RegP);  // r15_RegP and rbp_RegP.
3417 
3418   format %{ %}
3419   interface(REG_INTER);
3420 %}
3421 
3422 operand rRegN() %{
3423   constraint(ALLOC_IN_RC(int_reg));
3424   match(RegN);
3425 
3426   format %{ %}
3427   interface(REG_INTER);
3428 %}
3429 
3430 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3431 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3432 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3433 // The output of an instruction is controlled by the allocator, which respects
3434 // register class masks, not match rules.  Unless an instruction mentions
3435 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3436 // by the allocator as an input.
3437 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3438 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3439 // result, RBP is not included in the output of the instruction either.
3440 
3441 operand no_rax_RegP()
3442 %{
3443   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3444   match(RegP);
3445   match(rbx_RegP);
3446   match(rsi_RegP);
3447   match(rdi_RegP);
3448 
3449   format %{ %}
3450   interface(REG_INTER);
3451 %}
3452 
3453 // This operand is not allowed to use RBP even if
3454 // RBP is not used to hold the frame pointer.
3455 operand no_rbp_RegP()
3456 %{
3457   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3458   match(RegP);
3459   match(rbx_RegP);
3460   match(rsi_RegP);
3461   match(rdi_RegP);
3462 
3463   format %{ %}
3464   interface(REG_INTER);
3465 %}
3466 
3467 operand no_rax_rbx_RegP()
3468 %{
3469   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3470   match(RegP);
3471   match(rsi_RegP);
3472   match(rdi_RegP);
3473 
3474   format %{ %}
3475   interface(REG_INTER);
3476 %}
3477 
3478 // Special Registers
3479 // Return a pointer value
3480 operand rax_RegP()
3481 %{
3482   constraint(ALLOC_IN_RC(ptr_rax_reg));
3483   match(RegP);
3484   match(rRegP);
3485 
3486   format %{ %}
3487   interface(REG_INTER);
3488 %}
3489 
3490 // Special Registers
3491 // Return a compressed pointer value
3492 operand rax_RegN()
3493 %{
3494   constraint(ALLOC_IN_RC(int_rax_reg));
3495   match(RegN);
3496   match(rRegN);
3497 
3498   format %{ %}
3499   interface(REG_INTER);
3500 %}
3501 
3502 // Used in AtomicAdd
3503 operand rbx_RegP()
3504 %{
3505   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3506   match(RegP);
3507   match(rRegP);
3508 
3509   format %{ %}
3510   interface(REG_INTER);
3511 %}
3512 
3513 operand rsi_RegP()
3514 %{
3515   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3516   match(RegP);
3517   match(rRegP);
3518 
3519   format %{ %}
3520   interface(REG_INTER);
3521 %}
3522 
3523 // Used in rep stosq
3524 operand rdi_RegP()
3525 %{
3526   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3527   match(RegP);
3528   match(rRegP);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 operand r15_RegP()
3535 %{
3536   constraint(ALLOC_IN_RC(ptr_r15_reg));
3537   match(RegP);
3538   match(rRegP);
3539 
3540   format %{ %}
3541   interface(REG_INTER);
3542 %}
3543 
3544 operand rRegL()
3545 %{
3546   constraint(ALLOC_IN_RC(long_reg));
3547   match(RegL);
3548   match(rax_RegL);
3549   match(rdx_RegL);
3550 
3551   format %{ %}
3552   interface(REG_INTER);
3553 %}
3554 
3555 // Special Registers
3556 operand no_rax_rdx_RegL()
3557 %{
3558   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3559   match(RegL);
3560   match(rRegL);
3561 
3562   format %{ %}
3563   interface(REG_INTER);
3564 %}
3565 
3566 operand no_rax_RegL()
3567 %{
3568   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3569   match(RegL);
3570   match(rRegL);
3571   match(rdx_RegL);
3572 
3573   format %{ %}
3574   interface(REG_INTER);
3575 %}
3576 
3577 operand no_rcx_RegL()
3578 %{
3579   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3580   match(RegL);
3581   match(rRegL);
3582 
3583   format %{ %}
3584   interface(REG_INTER);
3585 %}
3586 
3587 operand rax_RegL()
3588 %{
3589   constraint(ALLOC_IN_RC(long_rax_reg));
3590   match(RegL);
3591   match(rRegL);
3592 
3593   format %{ "RAX" %}
3594   interface(REG_INTER);
3595 %}
3596 
3597 operand rcx_RegL()
3598 %{
3599   constraint(ALLOC_IN_RC(long_rcx_reg));
3600   match(RegL);
3601   match(rRegL);
3602 
3603   format %{ %}
3604   interface(REG_INTER);
3605 %}
3606 
3607 operand rdx_RegL()
3608 %{
3609   constraint(ALLOC_IN_RC(long_rdx_reg));
3610   match(RegL);
3611   match(rRegL);
3612 
3613   format %{ %}
3614   interface(REG_INTER);
3615 %}
3616 
3617 // Flags register, used as output of compare instructions
3618 operand rFlagsReg()
3619 %{
3620   constraint(ALLOC_IN_RC(int_flags));
3621   match(RegFlags);
3622 
3623   format %{ "RFLAGS" %}
3624   interface(REG_INTER);
3625 %}
3626 
3627 // Flags register, used as output of FLOATING POINT compare instructions
3628 operand rFlagsRegU()
3629 %{
3630   constraint(ALLOC_IN_RC(int_flags));
3631   match(RegFlags);
3632 
3633   format %{ "RFLAGS_U" %}
3634   interface(REG_INTER);
3635 %}
3636 
3637 operand rFlagsRegUCF() %{
3638   constraint(ALLOC_IN_RC(int_flags));
3639   match(RegFlags);
3640   predicate(false);
3641 
3642   format %{ "RFLAGS_U_CF" %}
3643   interface(REG_INTER);
3644 %}
3645 
3646 // Float register operands
3647 operand regF() %{
3648    constraint(ALLOC_IN_RC(float_reg));
3649    match(RegF);
3650 
3651    format %{ %}
3652    interface(REG_INTER);
3653 %}
3654 
3655 // Float register operands
3656 operand legRegF() %{
3657    constraint(ALLOC_IN_RC(float_reg_legacy));
3658    match(RegF);
3659 
3660    format %{ %}
3661    interface(REG_INTER);
3662 %}
3663 
3664 // Float register operands
3665 operand vlRegF() %{
3666    constraint(ALLOC_IN_RC(float_reg_vl));
3667    match(RegF);
3668 
3669    format %{ %}
3670    interface(REG_INTER);
3671 %}
3672 
3673 // Double register operands
3674 operand regD() %{
3675    constraint(ALLOC_IN_RC(double_reg));
3676    match(RegD);
3677 
3678    format %{ %}
3679    interface(REG_INTER);
3680 %}
3681 
3682 // Double register operands
3683 operand legRegD() %{
3684    constraint(ALLOC_IN_RC(double_reg_legacy));
3685    match(RegD);
3686 
3687    format %{ %}
3688    interface(REG_INTER);
3689 %}
3690 
3691 // Double register operands
3692 operand vlRegD() %{
3693    constraint(ALLOC_IN_RC(double_reg_vl));
3694    match(RegD);
3695 
3696    format %{ %}
3697    interface(REG_INTER);
3698 %}
3699 
3700 // Vectors
3701 operand vecS() %{
3702   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3703   match(VecS);
3704 
3705   format %{ %}
3706   interface(REG_INTER);
3707 %}
3708 
3709 // Vectors
3710 operand legVecS() %{
3711   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3712   match(VecS);
3713 
3714   format %{ %}
3715   interface(REG_INTER);
3716 %}
3717 
3718 operand vecD() %{
3719   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3720   match(VecD);
3721 
3722   format %{ %}
3723   interface(REG_INTER);
3724 %}
3725 
3726 operand legVecD() %{
3727   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3728   match(VecD);
3729 
3730   format %{ %}
3731   interface(REG_INTER);
3732 %}
3733 
3734 operand vecX() %{
3735   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3736   match(VecX);
3737 
3738   format %{ %}
3739   interface(REG_INTER);
3740 %}
3741 
3742 operand legVecX() %{
3743   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3744   match(VecX);
3745 
3746   format %{ %}
3747   interface(REG_INTER);
3748 %}
3749 
3750 operand vecY() %{
3751   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3752   match(VecY);
3753 
3754   format %{ %}
3755   interface(REG_INTER);
3756 %}
3757 
3758 operand legVecY() %{
3759   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3760   match(VecY);
3761 
3762   format %{ %}
3763   interface(REG_INTER);
3764 %}
3765 
3766 //----------Memory Operands----------------------------------------------------
3767 // Direct Memory Operand
3768 // operand direct(immP addr)
3769 // %{
3770 //   match(addr);
3771 
3772 //   format %{ "[$addr]" %}
3773 //   interface(MEMORY_INTER) %{
3774 //     base(0xFFFFFFFF);
3775 //     index(0x4);
3776 //     scale(0x0);
3777 //     disp($addr);
3778 //   %}
3779 // %}
3780 
3781 // Indirect Memory Operand
3782 operand indirect(any_RegP reg)
3783 %{
3784   constraint(ALLOC_IN_RC(ptr_reg));
3785   match(reg);
3786 
3787   format %{ "[$reg]" %}
3788   interface(MEMORY_INTER) %{
3789     base($reg);
3790     index(0x4);
3791     scale(0x0);
3792     disp(0x0);
3793   %}
3794 %}
3795 
3796 // Indirect Memory Plus Short Offset Operand
3797 operand indOffset8(any_RegP reg, immL8 off)
3798 %{
3799   constraint(ALLOC_IN_RC(ptr_reg));
3800   match(AddP reg off);
3801 
3802   format %{ "[$reg + $off (8-bit)]" %}
3803   interface(MEMORY_INTER) %{
3804     base($reg);
3805     index(0x4);
3806     scale(0x0);
3807     disp($off);
3808   %}
3809 %}
3810 
3811 // Indirect Memory Plus Long Offset Operand
3812 operand indOffset32(any_RegP reg, immL32 off)
3813 %{
3814   constraint(ALLOC_IN_RC(ptr_reg));
3815   match(AddP reg off);
3816 
3817   format %{ "[$reg + $off (32-bit)]" %}
3818   interface(MEMORY_INTER) %{
3819     base($reg);
3820     index(0x4);
3821     scale(0x0);
3822     disp($off);
3823   %}
3824 %}
3825 
3826 // Indirect Memory Plus Index Register Plus Offset Operand
3827 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3828 %{
3829   constraint(ALLOC_IN_RC(ptr_reg));
3830   match(AddP (AddP reg lreg) off);
3831 
3832   op_cost(10);
3833   format %{"[$reg + $off + $lreg]" %}
3834   interface(MEMORY_INTER) %{
3835     base($reg);
3836     index($lreg);
3837     scale(0x0);
3838     disp($off);
3839   %}
3840 %}
3841 
3842 // Indirect Memory Plus Index Register Plus Offset Operand
3843 operand indIndex(any_RegP reg, rRegL lreg)
3844 %{
3845   constraint(ALLOC_IN_RC(ptr_reg));
3846   match(AddP reg lreg);
3847 
3848   op_cost(10);
3849   format %{"[$reg + $lreg]" %}
3850   interface(MEMORY_INTER) %{
3851     base($reg);
3852     index($lreg);
3853     scale(0x0);
3854     disp(0x0);
3855   %}
3856 %}
3857 
3858 // Indirect Memory Times Scale Plus Index Register
3859 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3860 %{
3861   constraint(ALLOC_IN_RC(ptr_reg));
3862   match(AddP reg (LShiftL lreg scale));
3863 
3864   op_cost(10);
3865   format %{"[$reg + $lreg << $scale]" %}
3866   interface(MEMORY_INTER) %{
3867     base($reg);
3868     index($lreg);
3869     scale($scale);
3870     disp(0x0);
3871   %}
3872 %}
3873 
3874 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3875 %{
3876   constraint(ALLOC_IN_RC(ptr_reg));
3877   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3878   match(AddP reg (LShiftL (ConvI2L idx) scale));
3879 
3880   op_cost(10);
3881   format %{"[$reg + pos $idx << $scale]" %}
3882   interface(MEMORY_INTER) %{
3883     base($reg);
3884     index($idx);
3885     scale($scale);
3886     disp(0x0);
3887   %}
3888 %}
3889 
3890 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3891 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3892 %{
3893   constraint(ALLOC_IN_RC(ptr_reg));
3894   match(AddP (AddP reg (LShiftL lreg scale)) off);
3895 
3896   op_cost(10);
3897   format %{"[$reg + $off + $lreg << $scale]" %}
3898   interface(MEMORY_INTER) %{
3899     base($reg);
3900     index($lreg);
3901     scale($scale);
3902     disp($off);
3903   %}
3904 %}
3905 
3906 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3907 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3908 %{
3909   constraint(ALLOC_IN_RC(ptr_reg));
3910   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3911   match(AddP (AddP reg (ConvI2L idx)) off);
3912 
3913   op_cost(10);
3914   format %{"[$reg + $off + $idx]" %}
3915   interface(MEMORY_INTER) %{
3916     base($reg);
3917     index($idx);
3918     scale(0x0);
3919     disp($off);
3920   %}
3921 %}
3922 
3923 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3924 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3925 %{
3926   constraint(ALLOC_IN_RC(ptr_reg));
3927   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3928   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3929 
3930   op_cost(10);
3931   format %{"[$reg + $off + $idx << $scale]" %}
3932   interface(MEMORY_INTER) %{
3933     base($reg);
3934     index($idx);
3935     scale($scale);
3936     disp($off);
3937   %}
3938 %}
3939 
















3940 // Indirect Narrow Oop Plus Offset Operand
3941 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3942 // we can't free r12 even with CompressedOops::base() == NULL.
3943 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3944   predicate(UseCompressedOops && (CompressedOops::shift() == Address::times_8));
3945   constraint(ALLOC_IN_RC(ptr_reg));
3946   match(AddP (DecodeN reg) off);
3947 
3948   op_cost(10);
3949   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3950   interface(MEMORY_INTER) %{
3951     base(0xc); // R12
3952     index($reg);
3953     scale(0x3);
3954     disp($off);
3955   %}
3956 %}
3957 
3958 // Indirect Memory Operand
3959 operand indirectNarrow(rRegN reg)
3960 %{
3961   predicate(CompressedOops::shift() == 0);
3962   constraint(ALLOC_IN_RC(ptr_reg));
3963   match(DecodeN reg);
3964 
3965   format %{ "[$reg]" %}
3966   interface(MEMORY_INTER) %{
3967     base($reg);
3968     index(0x4);
3969     scale(0x0);
3970     disp(0x0);
3971   %}
3972 %}
3973 
3974 // Indirect Memory Plus Short Offset Operand
3975 operand indOffset8Narrow(rRegN reg, immL8 off)
3976 %{
3977   predicate(CompressedOops::shift() == 0);
3978   constraint(ALLOC_IN_RC(ptr_reg));
3979   match(AddP (DecodeN reg) off);
3980 
3981   format %{ "[$reg + $off (8-bit)]" %}
3982   interface(MEMORY_INTER) %{
3983     base($reg);
3984     index(0x4);
3985     scale(0x0);
3986     disp($off);
3987   %}
3988 %}
3989 
3990 // Indirect Memory Plus Long Offset Operand
3991 operand indOffset32Narrow(rRegN reg, immL32 off)
3992 %{
3993   predicate(CompressedOops::shift() == 0);
3994   constraint(ALLOC_IN_RC(ptr_reg));
3995   match(AddP (DecodeN reg) off);
3996 
3997   format %{ "[$reg + $off (32-bit)]" %}
3998   interface(MEMORY_INTER) %{
3999     base($reg);
4000     index(0x4);
4001     scale(0x0);
4002     disp($off);
4003   %}
4004 %}
4005 
4006 // Indirect Memory Plus Index Register Plus Offset Operand
4007 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4008 %{
4009   predicate(CompressedOops::shift() == 0);
4010   constraint(ALLOC_IN_RC(ptr_reg));
4011   match(AddP (AddP (DecodeN reg) lreg) off);
4012 
4013   op_cost(10);
4014   format %{"[$reg + $off + $lreg]" %}
4015   interface(MEMORY_INTER) %{
4016     base($reg);
4017     index($lreg);
4018     scale(0x0);
4019     disp($off);
4020   %}
4021 %}
4022 
4023 // Indirect Memory Plus Index Register Plus Offset Operand
4024 operand indIndexNarrow(rRegN reg, rRegL lreg)
4025 %{
4026   predicate(CompressedOops::shift() == 0);
4027   constraint(ALLOC_IN_RC(ptr_reg));
4028   match(AddP (DecodeN reg) lreg);
4029 
4030   op_cost(10);
4031   format %{"[$reg + $lreg]" %}
4032   interface(MEMORY_INTER) %{
4033     base($reg);
4034     index($lreg);
4035     scale(0x0);
4036     disp(0x0);
4037   %}
4038 %}
4039 
4040 // Indirect Memory Times Scale Plus Index Register
4041 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4042 %{
4043   predicate(CompressedOops::shift() == 0);
4044   constraint(ALLOC_IN_RC(ptr_reg));
4045   match(AddP (DecodeN reg) (LShiftL lreg scale));
4046 
4047   op_cost(10);
4048   format %{"[$reg + $lreg << $scale]" %}
4049   interface(MEMORY_INTER) %{
4050     base($reg);
4051     index($lreg);
4052     scale($scale);
4053     disp(0x0);
4054   %}
4055 %}
4056 
4057 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4058 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4059 %{
4060   predicate(CompressedOops::shift() == 0);
4061   constraint(ALLOC_IN_RC(ptr_reg));
4062   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4063 
4064   op_cost(10);
4065   format %{"[$reg + $off + $lreg << $scale]" %}
4066   interface(MEMORY_INTER) %{
4067     base($reg);
4068     index($lreg);
4069     scale($scale);
4070     disp($off);
4071   %}
4072 %}
4073 
4074 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4075 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4076 %{
4077   constraint(ALLOC_IN_RC(ptr_reg));
4078   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4079   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4080 
4081   op_cost(10);
4082   format %{"[$reg + $off + $idx]" %}
4083   interface(MEMORY_INTER) %{
4084     base($reg);
4085     index($idx);
4086     scale(0x0);
4087     disp($off);
4088   %}
4089 %}
4090 
4091 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4092 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4093 %{
4094   constraint(ALLOC_IN_RC(ptr_reg));
4095   predicate(CompressedOops::shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4096   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4097 
4098   op_cost(10);
4099   format %{"[$reg + $off + $idx << $scale]" %}
4100   interface(MEMORY_INTER) %{
4101     base($reg);
4102     index($idx);
4103     scale($scale);
4104     disp($off);
4105   %}
4106 %}
4107 
4108 //----------Special Memory Operands--------------------------------------------
4109 // Stack Slot Operand - This operand is used for loading and storing temporary
4110 //                      values on the stack where a match requires a value to
4111 //                      flow through memory.
4112 operand stackSlotP(sRegP reg)
4113 %{
4114   constraint(ALLOC_IN_RC(stack_slots));
4115   // No match rule because this operand is only generated in matching
4116 
4117   format %{ "[$reg]" %}
4118   interface(MEMORY_INTER) %{
4119     base(0x4);   // RSP
4120     index(0x4);  // No Index
4121     scale(0x0);  // No Scale
4122     disp($reg);  // Stack Offset
4123   %}
4124 %}
4125 
4126 operand stackSlotI(sRegI reg)
4127 %{
4128   constraint(ALLOC_IN_RC(stack_slots));
4129   // No match rule because this operand is only generated in matching
4130 
4131   format %{ "[$reg]" %}
4132   interface(MEMORY_INTER) %{
4133     base(0x4);   // RSP
4134     index(0x4);  // No Index
4135     scale(0x0);  // No Scale
4136     disp($reg);  // Stack Offset
4137   %}
4138 %}
4139 
4140 operand stackSlotF(sRegF reg)
4141 %{
4142   constraint(ALLOC_IN_RC(stack_slots));
4143   // No match rule because this operand is only generated in matching
4144 
4145   format %{ "[$reg]" %}
4146   interface(MEMORY_INTER) %{
4147     base(0x4);   // RSP
4148     index(0x4);  // No Index
4149     scale(0x0);  // No Scale
4150     disp($reg);  // Stack Offset
4151   %}
4152 %}
4153 
4154 operand stackSlotD(sRegD reg)
4155 %{
4156   constraint(ALLOC_IN_RC(stack_slots));
4157   // No match rule because this operand is only generated in matching
4158 
4159   format %{ "[$reg]" %}
4160   interface(MEMORY_INTER) %{
4161     base(0x4);   // RSP
4162     index(0x4);  // No Index
4163     scale(0x0);  // No Scale
4164     disp($reg);  // Stack Offset
4165   %}
4166 %}
4167 operand stackSlotL(sRegL reg)
4168 %{
4169   constraint(ALLOC_IN_RC(stack_slots));
4170   // No match rule because this operand is only generated in matching
4171 
4172   format %{ "[$reg]" %}
4173   interface(MEMORY_INTER) %{
4174     base(0x4);   // RSP
4175     index(0x4);  // No Index
4176     scale(0x0);  // No Scale
4177     disp($reg);  // Stack Offset
4178   %}
4179 %}
4180 
4181 //----------Conditional Branch Operands----------------------------------------
4182 // Comparison Op  - This is the operation of the comparison, and is limited to
4183 //                  the following set of codes:
4184 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4185 //
4186 // Other attributes of the comparison, such as unsignedness, are specified
4187 // by the comparison instruction that sets a condition code flags register.
4188 // That result is represented by a flags operand whose subtype is appropriate
4189 // to the unsignedness (etc.) of the comparison.
4190 //
4191 // Later, the instruction which matches both the Comparison Op (a Bool) and
4192 // the flags (produced by the Cmp) specifies the coding of the comparison op
4193 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4194 
4195 // Comparision Code
4196 operand cmpOp()
4197 %{
4198   match(Bool);
4199 
4200   format %{ "" %}
4201   interface(COND_INTER) %{
4202     equal(0x4, "e");
4203     not_equal(0x5, "ne");
4204     less(0xC, "l");
4205     greater_equal(0xD, "ge");
4206     less_equal(0xE, "le");
4207     greater(0xF, "g");
4208     overflow(0x0, "o");
4209     no_overflow(0x1, "no");
4210   %}
4211 %}
4212 
4213 // Comparison Code, unsigned compare.  Used by FP also, with
4214 // C2 (unordered) turned into GT or LT already.  The other bits
4215 // C0 and C3 are turned into Carry & Zero flags.
4216 operand cmpOpU()
4217 %{
4218   match(Bool);
4219 
4220   format %{ "" %}
4221   interface(COND_INTER) %{
4222     equal(0x4, "e");
4223     not_equal(0x5, "ne");
4224     less(0x2, "b");
4225     greater_equal(0x3, "nb");
4226     less_equal(0x6, "be");
4227     greater(0x7, "nbe");
4228     overflow(0x0, "o");
4229     no_overflow(0x1, "no");
4230   %}
4231 %}
4232 
4233 
4234 // Floating comparisons that don't require any fixup for the unordered case
4235 operand cmpOpUCF() %{
4236   match(Bool);
4237   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4238             n->as_Bool()->_test._test == BoolTest::ge ||
4239             n->as_Bool()->_test._test == BoolTest::le ||
4240             n->as_Bool()->_test._test == BoolTest::gt);
4241   format %{ "" %}
4242   interface(COND_INTER) %{
4243     equal(0x4, "e");
4244     not_equal(0x5, "ne");
4245     less(0x2, "b");
4246     greater_equal(0x3, "nb");
4247     less_equal(0x6, "be");
4248     greater(0x7, "nbe");
4249     overflow(0x0, "o");
4250     no_overflow(0x1, "no");
4251   %}
4252 %}
4253 
4254 
4255 // Floating comparisons that can be fixed up with extra conditional jumps
4256 operand cmpOpUCF2() %{
4257   match(Bool);
4258   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4259             n->as_Bool()->_test._test == BoolTest::eq);
4260   format %{ "" %}
4261   interface(COND_INTER) %{
4262     equal(0x4, "e");
4263     not_equal(0x5, "ne");
4264     less(0x2, "b");
4265     greater_equal(0x3, "nb");
4266     less_equal(0x6, "be");
4267     greater(0x7, "nbe");
4268     overflow(0x0, "o");
4269     no_overflow(0x1, "no");
4270   %}
4271 %}
4272 
4273 // Operands for bound floating pointer register arguments
4274 operand rxmm0() %{
4275   constraint(ALLOC_IN_RC(xmm0_reg));
4276   match(VecX);
4277   format%{%}
4278   interface(REG_INTER);
4279 %}
4280 operand rxmm1() %{
4281   constraint(ALLOC_IN_RC(xmm1_reg));
4282   match(VecX);
4283   format%{%}
4284   interface(REG_INTER);
4285 %}
4286 operand rxmm2() %{
4287   constraint(ALLOC_IN_RC(xmm2_reg));
4288   match(VecX);
4289   format%{%}
4290   interface(REG_INTER);
4291 %}
4292 operand rxmm3() %{
4293   constraint(ALLOC_IN_RC(xmm3_reg));
4294   match(VecX);
4295   format%{%}
4296   interface(REG_INTER);
4297 %}
4298 operand rxmm4() %{
4299   constraint(ALLOC_IN_RC(xmm4_reg));
4300   match(VecX);
4301   format%{%}
4302   interface(REG_INTER);
4303 %}
4304 operand rxmm5() %{
4305   constraint(ALLOC_IN_RC(xmm5_reg));
4306   match(VecX);
4307   format%{%}
4308   interface(REG_INTER);
4309 %}
4310 operand rxmm6() %{
4311   constraint(ALLOC_IN_RC(xmm6_reg));
4312   match(VecX);
4313   format%{%}
4314   interface(REG_INTER);
4315 %}
4316 operand rxmm7() %{
4317   constraint(ALLOC_IN_RC(xmm7_reg));
4318   match(VecX);
4319   format%{%}
4320   interface(REG_INTER);
4321 %}
4322 operand rxmm8() %{
4323   constraint(ALLOC_IN_RC(xmm8_reg));
4324   match(VecX);
4325   format%{%}
4326   interface(REG_INTER);
4327 %}
4328 operand rxmm9() %{
4329   constraint(ALLOC_IN_RC(xmm9_reg));
4330   match(VecX);
4331   format%{%}
4332   interface(REG_INTER);
4333 %}
4334 operand rxmm10() %{
4335   constraint(ALLOC_IN_RC(xmm10_reg));
4336   match(VecX);
4337   format%{%}
4338   interface(REG_INTER);
4339 %}
4340 operand rxmm11() %{
4341   constraint(ALLOC_IN_RC(xmm11_reg));
4342   match(VecX);
4343   format%{%}
4344   interface(REG_INTER);
4345 %}
4346 operand rxmm12() %{
4347   constraint(ALLOC_IN_RC(xmm12_reg));
4348   match(VecX);
4349   format%{%}
4350   interface(REG_INTER);
4351 %}
4352 operand rxmm13() %{
4353   constraint(ALLOC_IN_RC(xmm13_reg));
4354   match(VecX);
4355   format%{%}
4356   interface(REG_INTER);
4357 %}
4358 operand rxmm14() %{
4359   constraint(ALLOC_IN_RC(xmm14_reg));
4360   match(VecX);
4361   format%{%}
4362   interface(REG_INTER);
4363 %}
4364 operand rxmm15() %{
4365   constraint(ALLOC_IN_RC(xmm15_reg));
4366   match(VecX);
4367   format%{%}
4368   interface(REG_INTER);
4369 %}
4370 operand rxmm16() %{
4371   constraint(ALLOC_IN_RC(xmm16_reg));
4372   match(VecX);
4373   format%{%}
4374   interface(REG_INTER);
4375 %}
4376 operand rxmm17() %{
4377   constraint(ALLOC_IN_RC(xmm17_reg));
4378   match(VecX);
4379   format%{%}
4380   interface(REG_INTER);
4381 %}
4382 operand rxmm18() %{
4383   constraint(ALLOC_IN_RC(xmm18_reg));
4384   match(VecX);
4385   format%{%}
4386   interface(REG_INTER);
4387 %}
4388 operand rxmm19() %{
4389   constraint(ALLOC_IN_RC(xmm19_reg));
4390   match(VecX);
4391   format%{%}
4392   interface(REG_INTER);
4393 %}
4394 operand rxmm20() %{
4395   constraint(ALLOC_IN_RC(xmm20_reg));
4396   match(VecX);
4397   format%{%}
4398   interface(REG_INTER);
4399 %}
4400 operand rxmm21() %{
4401   constraint(ALLOC_IN_RC(xmm21_reg));
4402   match(VecX);
4403   format%{%}
4404   interface(REG_INTER);
4405 %}
4406 operand rxmm22() %{
4407   constraint(ALLOC_IN_RC(xmm22_reg));
4408   match(VecX);
4409   format%{%}
4410   interface(REG_INTER);
4411 %}
4412 operand rxmm23() %{
4413   constraint(ALLOC_IN_RC(xmm23_reg));
4414   match(VecX);
4415   format%{%}
4416   interface(REG_INTER);
4417 %}
4418 operand rxmm24() %{
4419   constraint(ALLOC_IN_RC(xmm24_reg));
4420   match(VecX);
4421   format%{%}
4422   interface(REG_INTER);
4423 %}
4424 operand rxmm25() %{
4425   constraint(ALLOC_IN_RC(xmm25_reg));
4426   match(VecX);
4427   format%{%}
4428   interface(REG_INTER);
4429 %}
4430 operand rxmm26() %{
4431   constraint(ALLOC_IN_RC(xmm26_reg));
4432   match(VecX);
4433   format%{%}
4434   interface(REG_INTER);
4435 %}
4436 operand rxmm27() %{
4437   constraint(ALLOC_IN_RC(xmm27_reg));
4438   match(VecX);
4439   format%{%}
4440   interface(REG_INTER);
4441 %}
4442 operand rxmm28() %{
4443   constraint(ALLOC_IN_RC(xmm28_reg));
4444   match(VecX);
4445   format%{%}
4446   interface(REG_INTER);
4447 %}
4448 operand rxmm29() %{
4449   constraint(ALLOC_IN_RC(xmm29_reg));
4450   match(VecX);
4451   format%{%}
4452   interface(REG_INTER);
4453 %}
4454 operand rxmm30() %{
4455   constraint(ALLOC_IN_RC(xmm30_reg));
4456   match(VecX);
4457   format%{%}
4458   interface(REG_INTER);
4459 %}
4460 operand rxmm31() %{
4461   constraint(ALLOC_IN_RC(xmm31_reg));
4462   match(VecX);
4463   format%{%}
4464   interface(REG_INTER);
4465 %}
4466 
4467 //----------OPERAND CLASSES----------------------------------------------------
4468 // Operand Classes are groups of operands that are used as to simplify
4469 // instruction definitions by not requiring the AD writer to specify separate
4470 // instructions for every form of operand when the instruction accepts
4471 // multiple operand types with the same basic encoding and format.  The classic
4472 // case of this is memory operands.
4473 
4474 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4475                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4476                indCompressedOopOffset,
4477                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4478                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4479                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4480 
4481 //----------PIPELINE-----------------------------------------------------------
4482 // Rules which define the behavior of the target architectures pipeline.
4483 pipeline %{
4484 
4485 //----------ATTRIBUTES---------------------------------------------------------
4486 attributes %{
4487   variable_size_instructions;        // Fixed size instructions
4488   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4489   instruction_unit_size = 1;         // An instruction is 1 bytes long
4490   instruction_fetch_unit_size = 16;  // The processor fetches one line
4491   instruction_fetch_units = 1;       // of 16 bytes
4492 
4493   // List of nop instructions
4494   nops( MachNop );
4495 %}
4496 
4497 //----------RESOURCES----------------------------------------------------------
4498 // Resources are the functional units available to the machine
4499 
4500 // Generic P2/P3 pipeline
4501 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4502 // 3 instructions decoded per cycle.
4503 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4504 // 3 ALU op, only ALU0 handles mul instructions.
4505 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4506            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4507            BR, FPU,
4508            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4509 
4510 //----------PIPELINE DESCRIPTION-----------------------------------------------
4511 // Pipeline Description specifies the stages in the machine's pipeline
4512 
4513 // Generic P2/P3 pipeline
4514 pipe_desc(S0, S1, S2, S3, S4, S5);
4515 
4516 //----------PIPELINE CLASSES---------------------------------------------------
4517 // Pipeline Classes describe the stages in which input and output are
4518 // referenced by the hardware pipeline.
4519 
4520 // Naming convention: ialu or fpu
4521 // Then: _reg
4522 // Then: _reg if there is a 2nd register
4523 // Then: _long if it's a pair of instructions implementing a long
4524 // Then: _fat if it requires the big decoder
4525 //   Or: _mem if it requires the big decoder and a memory unit.
4526 
4527 // Integer ALU reg operation
4528 pipe_class ialu_reg(rRegI dst)
4529 %{
4530     single_instruction;
4531     dst    : S4(write);
4532     dst    : S3(read);
4533     DECODE : S0;        // any decoder
4534     ALU    : S3;        // any alu
4535 %}
4536 
4537 // Long ALU reg operation
4538 pipe_class ialu_reg_long(rRegL dst)
4539 %{
4540     instruction_count(2);
4541     dst    : S4(write);
4542     dst    : S3(read);
4543     DECODE : S0(2);     // any 2 decoders
4544     ALU    : S3(2);     // both alus
4545 %}
4546 
4547 // Integer ALU reg operation using big decoder
4548 pipe_class ialu_reg_fat(rRegI dst)
4549 %{
4550     single_instruction;
4551     dst    : S4(write);
4552     dst    : S3(read);
4553     D0     : S0;        // big decoder only
4554     ALU    : S3;        // any alu
4555 %}
4556 
4557 // Long ALU reg operation using big decoder
4558 pipe_class ialu_reg_long_fat(rRegL dst)
4559 %{
4560     instruction_count(2);
4561     dst    : S4(write);
4562     dst    : S3(read);
4563     D0     : S0(2);     // big decoder only; twice
4564     ALU    : S3(2);     // any 2 alus
4565 %}
4566 
4567 // Integer ALU reg-reg operation
4568 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4569 %{
4570     single_instruction;
4571     dst    : S4(write);
4572     src    : S3(read);
4573     DECODE : S0;        // any decoder
4574     ALU    : S3;        // any alu
4575 %}
4576 
4577 // Long ALU reg-reg operation
4578 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4579 %{
4580     instruction_count(2);
4581     dst    : S4(write);
4582     src    : S3(read);
4583     DECODE : S0(2);     // any 2 decoders
4584     ALU    : S3(2);     // both alus
4585 %}
4586 
4587 // Integer ALU reg-reg operation
4588 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4589 %{
4590     single_instruction;
4591     dst    : S4(write);
4592     src    : S3(read);
4593     D0     : S0;        // big decoder only
4594     ALU    : S3;        // any alu
4595 %}
4596 
4597 // Long ALU reg-reg operation
4598 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4599 %{
4600     instruction_count(2);
4601     dst    : S4(write);
4602     src    : S3(read);
4603     D0     : S0(2);     // big decoder only; twice
4604     ALU    : S3(2);     // both alus
4605 %}
4606 
4607 // Integer ALU reg-mem operation
4608 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4609 %{
4610     single_instruction;
4611     dst    : S5(write);
4612     mem    : S3(read);
4613     D0     : S0;        // big decoder only
4614     ALU    : S4;        // any alu
4615     MEM    : S3;        // any mem
4616 %}
4617 
4618 // Integer mem operation (prefetch)
4619 pipe_class ialu_mem(memory mem)
4620 %{
4621     single_instruction;
4622     mem    : S3(read);
4623     D0     : S0;        // big decoder only
4624     MEM    : S3;        // any mem
4625 %}
4626 
4627 // Integer Store to Memory
4628 pipe_class ialu_mem_reg(memory mem, rRegI src)
4629 %{
4630     single_instruction;
4631     mem    : S3(read);
4632     src    : S5(read);
4633     D0     : S0;        // big decoder only
4634     ALU    : S4;        // any alu
4635     MEM    : S3;
4636 %}
4637 
4638 // // Long Store to Memory
4639 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4640 // %{
4641 //     instruction_count(2);
4642 //     mem    : S3(read);
4643 //     src    : S5(read);
4644 //     D0     : S0(2);          // big decoder only; twice
4645 //     ALU    : S4(2);     // any 2 alus
4646 //     MEM    : S3(2);  // Both mems
4647 // %}
4648 
4649 // Integer Store to Memory
4650 pipe_class ialu_mem_imm(memory mem)
4651 %{
4652     single_instruction;
4653     mem    : S3(read);
4654     D0     : S0;        // big decoder only
4655     ALU    : S4;        // any alu
4656     MEM    : S3;
4657 %}
4658 
4659 // Integer ALU0 reg-reg operation
4660 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4661 %{
4662     single_instruction;
4663     dst    : S4(write);
4664     src    : S3(read);
4665     D0     : S0;        // Big decoder only
4666     ALU0   : S3;        // only alu0
4667 %}
4668 
4669 // Integer ALU0 reg-mem operation
4670 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4671 %{
4672     single_instruction;
4673     dst    : S5(write);
4674     mem    : S3(read);
4675     D0     : S0;        // big decoder only
4676     ALU0   : S4;        // ALU0 only
4677     MEM    : S3;        // any mem
4678 %}
4679 
4680 // Integer ALU reg-reg operation
4681 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4682 %{
4683     single_instruction;
4684     cr     : S4(write);
4685     src1   : S3(read);
4686     src2   : S3(read);
4687     DECODE : S0;        // any decoder
4688     ALU    : S3;        // any alu
4689 %}
4690 
4691 // Integer ALU reg-imm operation
4692 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4693 %{
4694     single_instruction;
4695     cr     : S4(write);
4696     src1   : S3(read);
4697     DECODE : S0;        // any decoder
4698     ALU    : S3;        // any alu
4699 %}
4700 
4701 // Integer ALU reg-mem operation
4702 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4703 %{
4704     single_instruction;
4705     cr     : S4(write);
4706     src1   : S3(read);
4707     src2   : S3(read);
4708     D0     : S0;        // big decoder only
4709     ALU    : S4;        // any alu
4710     MEM    : S3;
4711 %}
4712 
4713 // Conditional move reg-reg
4714 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4715 %{
4716     instruction_count(4);
4717     y      : S4(read);
4718     q      : S3(read);
4719     p      : S3(read);
4720     DECODE : S0(4);     // any decoder
4721 %}
4722 
4723 // Conditional move reg-reg
4724 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4725 %{
4726     single_instruction;
4727     dst    : S4(write);
4728     src    : S3(read);
4729     cr     : S3(read);
4730     DECODE : S0;        // any decoder
4731 %}
4732 
4733 // Conditional move reg-mem
4734 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4735 %{
4736     single_instruction;
4737     dst    : S4(write);
4738     src    : S3(read);
4739     cr     : S3(read);
4740     DECODE : S0;        // any decoder
4741     MEM    : S3;
4742 %}
4743 
4744 // Conditional move reg-reg long
4745 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4746 %{
4747     single_instruction;
4748     dst    : S4(write);
4749     src    : S3(read);
4750     cr     : S3(read);
4751     DECODE : S0(2);     // any 2 decoders
4752 %}
4753 
4754 // XXX
4755 // // Conditional move double reg-reg
4756 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4757 // %{
4758 //     single_instruction;
4759 //     dst    : S4(write);
4760 //     src    : S3(read);
4761 //     cr     : S3(read);
4762 //     DECODE : S0;     // any decoder
4763 // %}
4764 
4765 // Float reg-reg operation
4766 pipe_class fpu_reg(regD dst)
4767 %{
4768     instruction_count(2);
4769     dst    : S3(read);
4770     DECODE : S0(2);     // any 2 decoders
4771     FPU    : S3;
4772 %}
4773 
4774 // Float reg-reg operation
4775 pipe_class fpu_reg_reg(regD dst, regD src)
4776 %{
4777     instruction_count(2);
4778     dst    : S4(write);
4779     src    : S3(read);
4780     DECODE : S0(2);     // any 2 decoders
4781     FPU    : S3;
4782 %}
4783 
4784 // Float reg-reg operation
4785 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4786 %{
4787     instruction_count(3);
4788     dst    : S4(write);
4789     src1   : S3(read);
4790     src2   : S3(read);
4791     DECODE : S0(3);     // any 3 decoders
4792     FPU    : S3(2);
4793 %}
4794 
4795 // Float reg-reg operation
4796 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4797 %{
4798     instruction_count(4);
4799     dst    : S4(write);
4800     src1   : S3(read);
4801     src2   : S3(read);
4802     src3   : S3(read);
4803     DECODE : S0(4);     // any 3 decoders
4804     FPU    : S3(2);
4805 %}
4806 
4807 // Float reg-reg operation
4808 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4809 %{
4810     instruction_count(4);
4811     dst    : S4(write);
4812     src1   : S3(read);
4813     src2   : S3(read);
4814     src3   : S3(read);
4815     DECODE : S1(3);     // any 3 decoders
4816     D0     : S0;        // Big decoder only
4817     FPU    : S3(2);
4818     MEM    : S3;
4819 %}
4820 
4821 // Float reg-mem operation
4822 pipe_class fpu_reg_mem(regD dst, memory mem)
4823 %{
4824     instruction_count(2);
4825     dst    : S5(write);
4826     mem    : S3(read);
4827     D0     : S0;        // big decoder only
4828     DECODE : S1;        // any decoder for FPU POP
4829     FPU    : S4;
4830     MEM    : S3;        // any mem
4831 %}
4832 
4833 // Float reg-mem operation
4834 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4835 %{
4836     instruction_count(3);
4837     dst    : S5(write);
4838     src1   : S3(read);
4839     mem    : S3(read);
4840     D0     : S0;        // big decoder only
4841     DECODE : S1(2);     // any decoder for FPU POP
4842     FPU    : S4;
4843     MEM    : S3;        // any mem
4844 %}
4845 
4846 // Float mem-reg operation
4847 pipe_class fpu_mem_reg(memory mem, regD src)
4848 %{
4849     instruction_count(2);
4850     src    : S5(read);
4851     mem    : S3(read);
4852     DECODE : S0;        // any decoder for FPU PUSH
4853     D0     : S1;        // big decoder only
4854     FPU    : S4;
4855     MEM    : S3;        // any mem
4856 %}
4857 
4858 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4859 %{
4860     instruction_count(3);
4861     src1   : S3(read);
4862     src2   : S3(read);
4863     mem    : S3(read);
4864     DECODE : S0(2);     // any decoder for FPU PUSH
4865     D0     : S1;        // big decoder only
4866     FPU    : S4;
4867     MEM    : S3;        // any mem
4868 %}
4869 
4870 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4871 %{
4872     instruction_count(3);
4873     src1   : S3(read);
4874     src2   : S3(read);
4875     mem    : S4(read);
4876     DECODE : S0;        // any decoder for FPU PUSH
4877     D0     : S0(2);     // big decoder only
4878     FPU    : S4;
4879     MEM    : S3(2);     // any mem
4880 %}
4881 
4882 pipe_class fpu_mem_mem(memory dst, memory src1)
4883 %{
4884     instruction_count(2);
4885     src1   : S3(read);
4886     dst    : S4(read);
4887     D0     : S0(2);     // big decoder only
4888     MEM    : S3(2);     // any mem
4889 %}
4890 
4891 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4892 %{
4893     instruction_count(3);
4894     src1   : S3(read);
4895     src2   : S3(read);
4896     dst    : S4(read);
4897     D0     : S0(3);     // big decoder only
4898     FPU    : S4;
4899     MEM    : S3(3);     // any mem
4900 %}
4901 
4902 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4903 %{
4904     instruction_count(3);
4905     src1   : S4(read);
4906     mem    : S4(read);
4907     DECODE : S0;        // any decoder for FPU PUSH
4908     D0     : S0(2);     // big decoder only
4909     FPU    : S4;
4910     MEM    : S3(2);     // any mem
4911 %}
4912 
4913 // Float load constant
4914 pipe_class fpu_reg_con(regD dst)
4915 %{
4916     instruction_count(2);
4917     dst    : S5(write);
4918     D0     : S0;        // big decoder only for the load
4919     DECODE : S1;        // any decoder for FPU POP
4920     FPU    : S4;
4921     MEM    : S3;        // any mem
4922 %}
4923 
4924 // Float load constant
4925 pipe_class fpu_reg_reg_con(regD dst, regD src)
4926 %{
4927     instruction_count(3);
4928     dst    : S5(write);
4929     src    : S3(read);
4930     D0     : S0;        // big decoder only for the load
4931     DECODE : S1(2);     // any decoder for FPU POP
4932     FPU    : S4;
4933     MEM    : S3;        // any mem
4934 %}
4935 
4936 // UnConditional branch
4937 pipe_class pipe_jmp(label labl)
4938 %{
4939     single_instruction;
4940     BR   : S3;
4941 %}
4942 
4943 // Conditional branch
4944 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4945 %{
4946     single_instruction;
4947     cr    : S1(read);
4948     BR    : S3;
4949 %}
4950 
4951 // Allocation idiom
4952 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4953 %{
4954     instruction_count(1); force_serialization;
4955     fixed_latency(6);
4956     heap_ptr : S3(read);
4957     DECODE   : S0(3);
4958     D0       : S2;
4959     MEM      : S3;
4960     ALU      : S3(2);
4961     dst      : S5(write);
4962     BR       : S5;
4963 %}
4964 
4965 // Generic big/slow expanded idiom
4966 pipe_class pipe_slow()
4967 %{
4968     instruction_count(10); multiple_bundles; force_serialization;
4969     fixed_latency(100);
4970     D0  : S0(2);
4971     MEM : S3(2);
4972 %}
4973 
4974 // The real do-nothing guy
4975 pipe_class empty()
4976 %{
4977     instruction_count(0);
4978 %}
4979 
4980 // Define the class for the Nop node
4981 define
4982 %{
4983    MachNop = empty;
4984 %}
4985 
4986 %}
4987 
4988 //----------INSTRUCTIONS-------------------------------------------------------
4989 //
4990 // match      -- States which machine-independent subtree may be replaced
4991 //               by this instruction.
4992 // ins_cost   -- The estimated cost of this instruction is used by instruction
4993 //               selection to identify a minimum cost tree of machine
4994 //               instructions that matches a tree of machine-independent
4995 //               instructions.
4996 // format     -- A string providing the disassembly for this instruction.
4997 //               The value of an instruction's operand may be inserted
4998 //               by referring to it with a '$' prefix.
4999 // opcode     -- Three instruction opcodes may be provided.  These are referred
5000 //               to within an encode class as $primary, $secondary, and $tertiary
5001 //               rrspectively.  The primary opcode is commonly used to
5002 //               indicate the type of machine instruction, while secondary
5003 //               and tertiary are often used for prefix options or addressing
5004 //               modes.
5005 // ins_encode -- A list of encode classes with parameters. The encode class
5006 //               name must have been defined in an 'enc_class' specification
5007 //               in the encode section of the architecture description.
5008 
5009 
5010 //----------Load/Store/Move Instructions---------------------------------------
5011 //----------Load Instructions--------------------------------------------------
5012 
5013 // Load Byte (8 bit signed)
5014 instruct loadB(rRegI dst, memory mem)
5015 %{
5016   match(Set dst (LoadB mem));
5017 
5018   ins_cost(125);
5019   format %{ "movsbl  $dst, $mem\t# byte" %}
5020 
5021   ins_encode %{
5022     __ movsbl($dst$$Register, $mem$$Address);
5023   %}
5024 
5025   ins_pipe(ialu_reg_mem);
5026 %}
5027 
5028 // Load Byte (8 bit signed) into Long Register
5029 instruct loadB2L(rRegL dst, memory mem)
5030 %{
5031   match(Set dst (ConvI2L (LoadB mem)));
5032 
5033   ins_cost(125);
5034   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5035 
5036   ins_encode %{
5037     __ movsbq($dst$$Register, $mem$$Address);
5038   %}
5039 
5040   ins_pipe(ialu_reg_mem);
5041 %}
5042 
5043 // Load Unsigned Byte (8 bit UNsigned)
5044 instruct loadUB(rRegI dst, memory mem)
5045 %{
5046   match(Set dst (LoadUB mem));
5047 
5048   ins_cost(125);
5049   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5050 
5051   ins_encode %{
5052     __ movzbl($dst$$Register, $mem$$Address);
5053   %}
5054 
5055   ins_pipe(ialu_reg_mem);
5056 %}
5057 
5058 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5059 instruct loadUB2L(rRegL dst, memory mem)
5060 %{
5061   match(Set dst (ConvI2L (LoadUB mem)));
5062 
5063   ins_cost(125);
5064   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5065 
5066   ins_encode %{
5067     __ movzbq($dst$$Register, $mem$$Address);
5068   %}
5069 
5070   ins_pipe(ialu_reg_mem);
5071 %}
5072 
5073 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5074 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5075   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5076   effect(KILL cr);
5077 
5078   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5079             "andl    $dst, right_n_bits($mask, 8)" %}
5080   ins_encode %{
5081     Register Rdst = $dst$$Register;
5082     __ movzbq(Rdst, $mem$$Address);
5083     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5084   %}
5085   ins_pipe(ialu_reg_mem);
5086 %}
5087 
5088 // Load Short (16 bit signed)
5089 instruct loadS(rRegI dst, memory mem)
5090 %{
5091   match(Set dst (LoadS mem));
5092 
5093   ins_cost(125);
5094   format %{ "movswl $dst, $mem\t# short" %}
5095 
5096   ins_encode %{
5097     __ movswl($dst$$Register, $mem$$Address);
5098   %}
5099 
5100   ins_pipe(ialu_reg_mem);
5101 %}
5102 
5103 // Load Short (16 bit signed) to Byte (8 bit signed)
5104 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5105   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5106 
5107   ins_cost(125);
5108   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5109   ins_encode %{
5110     __ movsbl($dst$$Register, $mem$$Address);
5111   %}
5112   ins_pipe(ialu_reg_mem);
5113 %}
5114 
5115 // Load Short (16 bit signed) into Long Register
5116 instruct loadS2L(rRegL dst, memory mem)
5117 %{
5118   match(Set dst (ConvI2L (LoadS mem)));
5119 
5120   ins_cost(125);
5121   format %{ "movswq $dst, $mem\t# short -> long" %}
5122 
5123   ins_encode %{
5124     __ movswq($dst$$Register, $mem$$Address);
5125   %}
5126 
5127   ins_pipe(ialu_reg_mem);
5128 %}
5129 
5130 // Load Unsigned Short/Char (16 bit UNsigned)
5131 instruct loadUS(rRegI dst, memory mem)
5132 %{
5133   match(Set dst (LoadUS mem));
5134 
5135   ins_cost(125);
5136   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5137 
5138   ins_encode %{
5139     __ movzwl($dst$$Register, $mem$$Address);
5140   %}
5141 
5142   ins_pipe(ialu_reg_mem);
5143 %}
5144 
5145 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5146 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5147   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5148 
5149   ins_cost(125);
5150   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5151   ins_encode %{
5152     __ movsbl($dst$$Register, $mem$$Address);
5153   %}
5154   ins_pipe(ialu_reg_mem);
5155 %}
5156 
5157 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5158 instruct loadUS2L(rRegL dst, memory mem)
5159 %{
5160   match(Set dst (ConvI2L (LoadUS mem)));
5161 
5162   ins_cost(125);
5163   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5164 
5165   ins_encode %{
5166     __ movzwq($dst$$Register, $mem$$Address);
5167   %}
5168 
5169   ins_pipe(ialu_reg_mem);
5170 %}
5171 
5172 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5173 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5174   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5175 
5176   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5177   ins_encode %{
5178     __ movzbq($dst$$Register, $mem$$Address);
5179   %}
5180   ins_pipe(ialu_reg_mem);
5181 %}
5182 
5183 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5184 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5185   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5186   effect(KILL cr);
5187 
5188   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5189             "andl    $dst, right_n_bits($mask, 16)" %}
5190   ins_encode %{
5191     Register Rdst = $dst$$Register;
5192     __ movzwq(Rdst, $mem$$Address);
5193     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5194   %}
5195   ins_pipe(ialu_reg_mem);
5196 %}
5197 
5198 // Load Integer
5199 instruct loadI(rRegI dst, memory mem)
5200 %{
5201   match(Set dst (LoadI mem));
5202 
5203   ins_cost(125);
5204   format %{ "movl    $dst, $mem\t# int" %}
5205 
5206   ins_encode %{
5207     __ movl($dst$$Register, $mem$$Address);
5208   %}
5209 
5210   ins_pipe(ialu_reg_mem);
5211 %}
5212 
5213 // Load Integer (32 bit signed) to Byte (8 bit signed)
5214 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5215   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5216 
5217   ins_cost(125);
5218   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5219   ins_encode %{
5220     __ movsbl($dst$$Register, $mem$$Address);
5221   %}
5222   ins_pipe(ialu_reg_mem);
5223 %}
5224 
5225 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5226 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5227   match(Set dst (AndI (LoadI mem) mask));
5228 
5229   ins_cost(125);
5230   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5231   ins_encode %{
5232     __ movzbl($dst$$Register, $mem$$Address);
5233   %}
5234   ins_pipe(ialu_reg_mem);
5235 %}
5236 
5237 // Load Integer (32 bit signed) to Short (16 bit signed)
5238 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5239   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5240 
5241   ins_cost(125);
5242   format %{ "movswl  $dst, $mem\t# int -> short" %}
5243   ins_encode %{
5244     __ movswl($dst$$Register, $mem$$Address);
5245   %}
5246   ins_pipe(ialu_reg_mem);
5247 %}
5248 
5249 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5250 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5251   match(Set dst (AndI (LoadI mem) mask));
5252 
5253   ins_cost(125);
5254   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5255   ins_encode %{
5256     __ movzwl($dst$$Register, $mem$$Address);
5257   %}
5258   ins_pipe(ialu_reg_mem);
5259 %}
5260 
5261 // Load Integer into Long Register
5262 instruct loadI2L(rRegL dst, memory mem)
5263 %{
5264   match(Set dst (ConvI2L (LoadI mem)));
5265 
5266   ins_cost(125);
5267   format %{ "movslq  $dst, $mem\t# int -> long" %}
5268 
5269   ins_encode %{
5270     __ movslq($dst$$Register, $mem$$Address);
5271   %}
5272 
5273   ins_pipe(ialu_reg_mem);
5274 %}
5275 
5276 // Load Integer with mask 0xFF into Long Register
5277 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5278   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5279 
5280   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5281   ins_encode %{
5282     __ movzbq($dst$$Register, $mem$$Address);
5283   %}
5284   ins_pipe(ialu_reg_mem);
5285 %}
5286 
5287 // Load Integer with mask 0xFFFF into Long Register
5288 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5289   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5290 
5291   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5292   ins_encode %{
5293     __ movzwq($dst$$Register, $mem$$Address);
5294   %}
5295   ins_pipe(ialu_reg_mem);
5296 %}
5297 
5298 // Load Integer with a 31-bit mask into Long Register
5299 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5300   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5301   effect(KILL cr);
5302 
5303   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5304             "andl    $dst, $mask" %}
5305   ins_encode %{
5306     Register Rdst = $dst$$Register;
5307     __ movl(Rdst, $mem$$Address);
5308     __ andl(Rdst, $mask$$constant);
5309   %}
5310   ins_pipe(ialu_reg_mem);
5311 %}
5312 
5313 // Load Unsigned Integer into Long Register
5314 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5315 %{
5316   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5317 
5318   ins_cost(125);
5319   format %{ "movl    $dst, $mem\t# uint -> long" %}
5320 
5321   ins_encode %{
5322     __ movl($dst$$Register, $mem$$Address);
5323   %}
5324 
5325   ins_pipe(ialu_reg_mem);
5326 %}
5327 
5328 // Load Long
5329 instruct loadL(rRegL dst, memory mem)
5330 %{
5331   match(Set dst (LoadL mem));
5332 
5333   ins_cost(125);
5334   format %{ "movq    $dst, $mem\t# long" %}
5335 
5336   ins_encode %{
5337     __ movq($dst$$Register, $mem$$Address);
5338   %}
5339 
5340   ins_pipe(ialu_reg_mem); // XXX
5341 %}
5342 
5343 // Load Range
5344 instruct loadRange(rRegI dst, memory mem)
5345 %{
5346   match(Set dst (LoadRange mem));
5347 
5348   ins_cost(125); // XXX
5349   format %{ "movl    $dst, $mem\t# range" %}
5350   opcode(0x8B);
5351   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5352   ins_pipe(ialu_reg_mem);
5353 %}
5354 
5355 // Load Pointer
5356 instruct loadP(rRegP dst, memory mem)
5357 %{
5358   match(Set dst (LoadP mem));
5359 
5360   ins_cost(125); // XXX
5361   format %{ "movq    $dst, $mem\t# ptr" %}
5362   opcode(0x8B);
5363   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5364   ins_pipe(ialu_reg_mem); // XXX
5365 %}
5366 
5367 // Load Compressed Pointer
5368 instruct loadN(rRegN dst, memory mem)
5369 %{
5370    match(Set dst (LoadN mem));
5371 
5372    ins_cost(125); // XXX
5373    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5374    ins_encode %{
5375      __ movl($dst$$Register, $mem$$Address);
5376    %}
5377    ins_pipe(ialu_reg_mem); // XXX
5378 %}
5379 
5380 
5381 // Load Klass Pointer
5382 instruct loadKlass(rRegP dst, memory mem)
5383 %{
5384   match(Set dst (LoadKlass mem));
5385 
5386   ins_cost(125); // XXX
5387   format %{ "movq    $dst, $mem\t# class" %}
5388   opcode(0x8B);
5389   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5390   ins_pipe(ialu_reg_mem); // XXX
5391 %}
5392 
5393 // Load narrow Klass Pointer
5394 instruct loadNKlass(rRegN dst, memory mem)
5395 %{
5396   match(Set dst (LoadNKlass mem));
5397 
5398   ins_cost(125); // XXX
5399   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5400   ins_encode %{
5401     __ movl($dst$$Register, $mem$$Address);
5402   %}
5403   ins_pipe(ialu_reg_mem); // XXX
5404 %}
5405 
5406 // Load Float
5407 instruct loadF(regF dst, memory mem)
5408 %{
5409   match(Set dst (LoadF mem));
5410 
5411   ins_cost(145); // XXX
5412   format %{ "movss   $dst, $mem\t# float" %}
5413   ins_encode %{
5414     __ movflt($dst$$XMMRegister, $mem$$Address);
5415   %}
5416   ins_pipe(pipe_slow); // XXX
5417 %}
5418 
5419 // Load Float
5420 instruct MoveF2VL(vlRegF dst, regF src) %{
5421   match(Set dst src);
5422   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5423   ins_encode %{
5424     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5425   %}
5426   ins_pipe( fpu_reg_reg );
5427 %}
5428 
5429 // Load Float
5430 instruct MoveF2LEG(legRegF dst, regF src) %{
5431   match(Set dst src);
5432   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5433   ins_encode %{
5434     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5435   %}
5436   ins_pipe( fpu_reg_reg );
5437 %}
5438 
5439 // Load Float
5440 instruct MoveVL2F(regF dst, vlRegF src) %{
5441   match(Set dst src);
5442   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5443   ins_encode %{
5444     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5445   %}
5446   ins_pipe( fpu_reg_reg );
5447 %}
5448 
5449 // Load Float
5450 instruct MoveLEG2F(regF dst, legRegF src) %{
5451   match(Set dst src);
5452   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5453   ins_encode %{
5454     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5455   %}
5456   ins_pipe( fpu_reg_reg );
5457 %}
5458 
5459 // Load Double
5460 instruct loadD_partial(regD dst, memory mem)
5461 %{
5462   predicate(!UseXmmLoadAndClearUpper);
5463   match(Set dst (LoadD mem));
5464 
5465   ins_cost(145); // XXX
5466   format %{ "movlpd  $dst, $mem\t# double" %}
5467   ins_encode %{
5468     __ movdbl($dst$$XMMRegister, $mem$$Address);
5469   %}
5470   ins_pipe(pipe_slow); // XXX
5471 %}
5472 
5473 instruct loadD(regD dst, memory mem)
5474 %{
5475   predicate(UseXmmLoadAndClearUpper);
5476   match(Set dst (LoadD mem));
5477 
5478   ins_cost(145); // XXX
5479   format %{ "movsd   $dst, $mem\t# double" %}
5480   ins_encode %{
5481     __ movdbl($dst$$XMMRegister, $mem$$Address);
5482   %}
5483   ins_pipe(pipe_slow); // XXX
5484 %}
5485 
5486 // Load Double
5487 instruct MoveD2VL(vlRegD dst, regD src) %{
5488   match(Set dst src);
5489   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5490   ins_encode %{
5491     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5492   %}
5493   ins_pipe( fpu_reg_reg );
5494 %}
5495 
5496 // Load Double
5497 instruct MoveD2LEG(legRegD dst, regD src) %{
5498   match(Set dst src);
5499   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5500   ins_encode %{
5501     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5502   %}
5503   ins_pipe( fpu_reg_reg );
5504 %}
5505 
5506 // Load Double
5507 instruct MoveVL2D(regD dst, vlRegD src) %{
5508   match(Set dst src);
5509   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5510   ins_encode %{
5511     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5512   %}
5513   ins_pipe( fpu_reg_reg );
5514 %}
5515 
5516 // Load Double
5517 instruct MoveLEG2D(regD dst, legRegD src) %{
5518   match(Set dst src);
5519   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5520   ins_encode %{
5521     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5522   %}
5523   ins_pipe( fpu_reg_reg );
5524 %}
5525 
5526 // Following pseudo code describes the algorithm for max[FD]:
5527 // Min algorithm is on similar lines
5528 //  btmp = (b < +0.0) ? a : b
5529 //  atmp = (b < +0.0) ? b : a
5530 //  Tmp  = Max_Float(atmp , btmp)
5531 //  Res  = (atmp == NaN) ? atmp : Tmp
5532 
5533 // max = java.lang.Math.max(float a, float b)
5534 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5535   predicate(UseAVX > 0 && !n->is_reduction());
5536   match(Set dst (MaxF a b));
5537   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5538   format %{
5539      "blendvps         $btmp,$b,$a,$b           \n\t"
5540      "blendvps         $atmp,$a,$b,$b           \n\t"
5541      "vmaxss           $tmp,$atmp,$btmp         \n\t"
5542      "cmpps.unordered  $btmp,$atmp,$atmp        \n\t"
5543      "blendvps         $dst,$tmp,$atmp,$btmp    \n\t"
5544   %}
5545   ins_encode %{
5546     int vector_len = Assembler::AVX_128bit;
5547     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5548     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5549     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5550     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5551     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5552  %}
5553   ins_pipe( pipe_slow );
5554 %}
5555 
5556 instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5557   predicate(UseAVX > 0 && n->is_reduction());
5558   match(Set dst (MaxF a b));
5559   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5560 
5561   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
5562   ins_encode %{
5563     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5564                     false /*min*/, true /*single*/);
5565   %}
5566   ins_pipe( pipe_slow );
5567 %}
5568 
5569 // max = java.lang.Math.max(double a, double b)
5570 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5571   predicate(UseAVX > 0 && !n->is_reduction());
5572   match(Set dst (MaxD a b));
5573   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
5574   format %{
5575      "blendvpd         $btmp,$b,$a,$b            \n\t"
5576      "blendvpd         $atmp,$a,$b,$b            \n\t"
5577      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
5578      "cmppd.unordered  $btmp,$atmp,$atmp         \n\t"
5579      "blendvpd         $dst,$tmp,$atmp,$btmp     \n\t"
5580   %}
5581   ins_encode %{
5582     int vector_len = Assembler::AVX_128bit;
5583     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5584     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5585     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5586     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5587     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5588   %}
5589   ins_pipe( pipe_slow );
5590 %}
5591 
5592 instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5593   predicate(UseAVX > 0 && n->is_reduction());
5594   match(Set dst (MaxD a b));
5595   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5596 
5597   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
5598   ins_encode %{
5599     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5600                     false /*min*/, false /*single*/);
5601   %}
5602   ins_pipe( pipe_slow );
5603 %}
5604 
5605 // min = java.lang.Math.min(float a, float b)
5606 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5607   predicate(UseAVX > 0 && !n->is_reduction());
5608   match(Set dst (MinF a b));
5609   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5610   format %{
5611      "blendvps         $atmp,$a,$b,$a             \n\t"
5612      "blendvps         $btmp,$b,$a,$a             \n\t"
5613      "vminss           $tmp,$atmp,$btmp           \n\t"
5614      "cmpps.unordered  $btmp,$atmp,$atmp          \n\t"
5615      "blendvps         $dst,$tmp,$atmp,$btmp      \n\t"
5616   %}
5617   ins_encode %{
5618     int vector_len = Assembler::AVX_128bit;
5619     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5620     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5621     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5622     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5623     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5624   %}
5625   ins_pipe( pipe_slow );
5626 %}
5627 
5628 instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
5629   predicate(UseAVX > 0 && n->is_reduction());
5630   match(Set dst (MinF a b));
5631   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5632 
5633   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
5634   ins_encode %{
5635     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5636                     true /*min*/, true /*single*/);
5637   %}
5638   ins_pipe( pipe_slow );
5639 %}
5640 
5641 // min = java.lang.Math.min(double a, double b)
5642 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5643   predicate(UseAVX > 0 && !n->is_reduction());
5644   match(Set dst (MinD a b));
5645   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5646   format %{
5647      "blendvpd         $atmp,$a,$b,$a           \n\t"
5648      "blendvpd         $btmp,$b,$a,$a           \n\t"
5649      "vminsd           $tmp,$atmp,$btmp         \n\t"
5650      "cmppd.unordered  $btmp,$atmp,$atmp        \n\t"
5651      "blendvpd         $dst,$tmp,$atmp,$btmp    \n\t"
5652   %}
5653   ins_encode %{
5654     int vector_len = Assembler::AVX_128bit;
5655     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5656     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5657     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5658     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5659     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5660   %}
5661   ins_pipe( pipe_slow );
5662 %}
5663 
5664 instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
5665   predicate(UseAVX > 0 && n->is_reduction());
5666   match(Set dst (MinD a b));
5667   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5668 
5669   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
5670   ins_encode %{
5671     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5672                     true /*min*/, false /*single*/);
5673   %}
5674   ins_pipe( pipe_slow );
5675 %}
5676 
5677 // Load Effective Address
5678 instruct leaP8(rRegP dst, indOffset8 mem)
5679 %{
5680   match(Set dst mem);
5681 
5682   ins_cost(110); // XXX
5683   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5684   opcode(0x8D);
5685   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5686   ins_pipe(ialu_reg_reg_fat);
5687 %}
5688 
5689 instruct leaP32(rRegP dst, indOffset32 mem)
5690 %{
5691   match(Set dst mem);
5692 
5693   ins_cost(110);
5694   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5695   opcode(0x8D);
5696   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5697   ins_pipe(ialu_reg_reg_fat);
5698 %}
5699 
5700 // instruct leaPIdx(rRegP dst, indIndex mem)
5701 // %{
5702 //   match(Set dst mem);
5703 
5704 //   ins_cost(110);
5705 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5706 //   opcode(0x8D);
5707 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5708 //   ins_pipe(ialu_reg_reg_fat);
5709 // %}
5710 
5711 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5712 %{
5713   match(Set dst mem);
5714 
5715   ins_cost(110);
5716   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5717   opcode(0x8D);
5718   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5719   ins_pipe(ialu_reg_reg_fat);
5720 %}
5721 
5722 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5723 %{
5724   match(Set dst mem);
5725 
5726   ins_cost(110);
5727   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5728   opcode(0x8D);
5729   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5730   ins_pipe(ialu_reg_reg_fat);
5731 %}
5732 
5733 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5734 %{
5735   match(Set dst mem);
5736 
5737   ins_cost(110);
5738   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5739   opcode(0x8D);
5740   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5741   ins_pipe(ialu_reg_reg_fat);
5742 %}
5743 
5744 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5745 %{
5746   match(Set dst mem);
5747 
5748   ins_cost(110);
5749   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5750   opcode(0x8D);
5751   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5752   ins_pipe(ialu_reg_reg_fat);
5753 %}
5754 
5755 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5756 %{
5757   match(Set dst mem);
5758 
5759   ins_cost(110);
5760   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5761   opcode(0x8D);
5762   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5763   ins_pipe(ialu_reg_reg_fat);
5764 %}
5765 
5766 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5767 %{
5768   match(Set dst mem);
5769 
5770   ins_cost(110);
5771   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5772   opcode(0x8D);
5773   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5774   ins_pipe(ialu_reg_reg_fat);
5775 %}
5776 
5777 // Load Effective Address which uses Narrow (32-bits) oop
5778 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5779 %{
5780   predicate(UseCompressedOops && (CompressedOops::shift() != 0));
5781   match(Set dst mem);
5782 
5783   ins_cost(110);
5784   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5785   opcode(0x8D);
5786   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5787   ins_pipe(ialu_reg_reg_fat);
5788 %}
5789 
5790 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5791 %{
5792   predicate(CompressedOops::shift() == 0);
5793   match(Set dst mem);
5794 
5795   ins_cost(110); // XXX
5796   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5797   opcode(0x8D);
5798   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5799   ins_pipe(ialu_reg_reg_fat);
5800 %}
5801 
5802 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5803 %{
5804   predicate(CompressedOops::shift() == 0);
5805   match(Set dst mem);
5806 
5807   ins_cost(110);
5808   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5809   opcode(0x8D);
5810   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5811   ins_pipe(ialu_reg_reg_fat);
5812 %}
5813 
5814 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5815 %{
5816   predicate(CompressedOops::shift() == 0);
5817   match(Set dst mem);
5818 
5819   ins_cost(110);
5820   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5821   opcode(0x8D);
5822   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5823   ins_pipe(ialu_reg_reg_fat);
5824 %}
5825 
5826 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5827 %{
5828   predicate(CompressedOops::shift() == 0);
5829   match(Set dst mem);
5830 
5831   ins_cost(110);
5832   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5833   opcode(0x8D);
5834   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5835   ins_pipe(ialu_reg_reg_fat);
5836 %}
5837 
5838 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5839 %{
5840   predicate(CompressedOops::shift() == 0);
5841   match(Set dst mem);
5842 
5843   ins_cost(110);
5844   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5845   opcode(0x8D);
5846   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5847   ins_pipe(ialu_reg_reg_fat);
5848 %}
5849 
5850 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5851 %{
5852   predicate(CompressedOops::shift() == 0);
5853   match(Set dst mem);
5854 
5855   ins_cost(110);
5856   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5857   opcode(0x8D);
5858   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5859   ins_pipe(ialu_reg_reg_fat);
5860 %}
5861 
5862 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5863 %{
5864   predicate(CompressedOops::shift() == 0);
5865   match(Set dst mem);
5866 
5867   ins_cost(110);
5868   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5869   opcode(0x8D);
5870   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5871   ins_pipe(ialu_reg_reg_fat);
5872 %}
5873 
5874 instruct loadConI(rRegI dst, immI src)
5875 %{
5876   match(Set dst src);
5877 
5878   format %{ "movl    $dst, $src\t# int" %}
5879   ins_encode(load_immI(dst, src));
5880   ins_pipe(ialu_reg_fat); // XXX
5881 %}
5882 
5883 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5884 %{
5885   match(Set dst src);
5886   effect(KILL cr);
5887 
5888   ins_cost(50);
5889   format %{ "xorl    $dst, $dst\t# int" %}
5890   opcode(0x33); /* + rd */
5891   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5892   ins_pipe(ialu_reg);
5893 %}
5894 
5895 instruct loadConL(rRegL dst, immL src)
5896 %{
5897   match(Set dst src);
5898 
5899   ins_cost(150);
5900   format %{ "movq    $dst, $src\t# long" %}
5901   ins_encode(load_immL(dst, src));
5902   ins_pipe(ialu_reg);
5903 %}
5904 
5905 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5906 %{
5907   match(Set dst src);
5908   effect(KILL cr);
5909 
5910   ins_cost(50);
5911   format %{ "xorl    $dst, $dst\t# long" %}
5912   opcode(0x33); /* + rd */
5913   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5914   ins_pipe(ialu_reg); // XXX
5915 %}
5916 
5917 instruct loadConUL32(rRegL dst, immUL32 src)
5918 %{
5919   match(Set dst src);
5920 
5921   ins_cost(60);
5922   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5923   ins_encode(load_immUL32(dst, src));
5924   ins_pipe(ialu_reg);
5925 %}
5926 
5927 instruct loadConL32(rRegL dst, immL32 src)
5928 %{
5929   match(Set dst src);
5930 
5931   ins_cost(70);
5932   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5933   ins_encode(load_immL32(dst, src));
5934   ins_pipe(ialu_reg);
5935 %}
5936 
5937 instruct loadConP(rRegP dst, immP con) %{
5938   match(Set dst con);
5939 
5940   format %{ "movq    $dst, $con\t# ptr" %}
5941   ins_encode(load_immP(dst, con));
5942   ins_pipe(ialu_reg_fat); // XXX
5943 %}
5944 
5945 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5946 %{
5947   match(Set dst src);
5948   effect(KILL cr);
5949 
5950   ins_cost(50);
5951   format %{ "xorl    $dst, $dst\t# ptr" %}
5952   opcode(0x33); /* + rd */
5953   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5954   ins_pipe(ialu_reg);
5955 %}
5956 
5957 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5958 %{
5959   match(Set dst src);
5960   effect(KILL cr);
5961 
5962   ins_cost(60);
5963   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5964   ins_encode(load_immP31(dst, src));
5965   ins_pipe(ialu_reg);
5966 %}
5967 
5968 instruct loadConF(regF dst, immF con) %{
5969   match(Set dst con);
5970   ins_cost(125);
5971   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5972   ins_encode %{
5973     __ movflt($dst$$XMMRegister, $constantaddress($con));
5974   %}
5975   ins_pipe(pipe_slow);
5976 %}
5977 
5978 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5979   match(Set dst src);
5980   effect(KILL cr);
5981   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5982   ins_encode %{
5983     __ xorq($dst$$Register, $dst$$Register);
5984   %}
5985   ins_pipe(ialu_reg);
5986 %}
5987 
5988 instruct loadConN(rRegN dst, immN src) %{
5989   match(Set dst src);
5990 
5991   ins_cost(125);
5992   format %{ "movl    $dst, $src\t# compressed ptr" %}
5993   ins_encode %{
5994     address con = (address)$src$$constant;
5995     if (con == NULL) {
5996       ShouldNotReachHere();
5997     } else {
5998       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5999     }
6000   %}
6001   ins_pipe(ialu_reg_fat); // XXX
6002 %}
6003 
6004 instruct loadConNKlass(rRegN dst, immNKlass src) %{
6005   match(Set dst src);
6006 
6007   ins_cost(125);
6008   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
6009   ins_encode %{
6010     address con = (address)$src$$constant;
6011     if (con == NULL) {
6012       ShouldNotReachHere();
6013     } else {
6014       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
6015     }
6016   %}
6017   ins_pipe(ialu_reg_fat); // XXX
6018 %}
6019 
6020 instruct loadConF0(regF dst, immF0 src)
6021 %{
6022   match(Set dst src);
6023   ins_cost(100);
6024 
6025   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6026   ins_encode %{
6027     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6028   %}
6029   ins_pipe(pipe_slow);
6030 %}
6031 
6032 // Use the same format since predicate() can not be used here.
6033 instruct loadConD(regD dst, immD con) %{
6034   match(Set dst con);
6035   ins_cost(125);
6036   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6037   ins_encode %{
6038     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6039   %}
6040   ins_pipe(pipe_slow);
6041 %}
6042 
6043 instruct loadConD0(regD dst, immD0 src)
6044 %{
6045   match(Set dst src);
6046   ins_cost(100);
6047 
6048   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6049   ins_encode %{
6050     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6051   %}
6052   ins_pipe(pipe_slow);
6053 %}
6054 
6055 instruct loadSSI(rRegI dst, stackSlotI src)
6056 %{
6057   match(Set dst src);
6058 
6059   ins_cost(125);
6060   format %{ "movl    $dst, $src\t# int stk" %}
6061   opcode(0x8B);
6062   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6063   ins_pipe(ialu_reg_mem);
6064 %}
6065 
6066 instruct loadSSL(rRegL dst, stackSlotL src)
6067 %{
6068   match(Set dst src);
6069 
6070   ins_cost(125);
6071   format %{ "movq    $dst, $src\t# long stk" %}
6072   opcode(0x8B);
6073   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6074   ins_pipe(ialu_reg_mem);
6075 %}
6076 
6077 instruct loadSSP(rRegP dst, stackSlotP src)
6078 %{
6079   match(Set dst src);
6080 
6081   ins_cost(125);
6082   format %{ "movq    $dst, $src\t# ptr stk" %}
6083   opcode(0x8B);
6084   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6085   ins_pipe(ialu_reg_mem);
6086 %}
6087 
6088 instruct loadSSF(regF dst, stackSlotF src)
6089 %{
6090   match(Set dst src);
6091 
6092   ins_cost(125);
6093   format %{ "movss   $dst, $src\t# float stk" %}
6094   ins_encode %{
6095     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
6096   %}
6097   ins_pipe(pipe_slow); // XXX
6098 %}
6099 
6100 // Use the same format since predicate() can not be used here.
6101 instruct loadSSD(regD dst, stackSlotD src)
6102 %{
6103   match(Set dst src);
6104 
6105   ins_cost(125);
6106   format %{ "movsd   $dst, $src\t# double stk" %}
6107   ins_encode  %{
6108     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6109   %}
6110   ins_pipe(pipe_slow); // XXX
6111 %}
6112 
6113 // Prefetch instructions for allocation.
6114 // Must be safe to execute with invalid address (cannot fault).
6115 
6116 instruct prefetchAlloc( memory mem ) %{
6117   predicate(AllocatePrefetchInstr==3);
6118   match(PrefetchAllocation mem);
6119   ins_cost(125);
6120 
6121   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6122   ins_encode %{
6123     __ prefetchw($mem$$Address);
6124   %}
6125   ins_pipe(ialu_mem);
6126 %}
6127 
6128 instruct prefetchAllocNTA( memory mem ) %{
6129   predicate(AllocatePrefetchInstr==0);
6130   match(PrefetchAllocation mem);
6131   ins_cost(125);
6132 
6133   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6134   ins_encode %{
6135     __ prefetchnta($mem$$Address);
6136   %}
6137   ins_pipe(ialu_mem);
6138 %}
6139 
6140 instruct prefetchAllocT0( memory mem ) %{
6141   predicate(AllocatePrefetchInstr==1);
6142   match(PrefetchAllocation mem);
6143   ins_cost(125);
6144 
6145   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6146   ins_encode %{
6147     __ prefetcht0($mem$$Address);
6148   %}
6149   ins_pipe(ialu_mem);
6150 %}
6151 
6152 instruct prefetchAllocT2( memory mem ) %{
6153   predicate(AllocatePrefetchInstr==2);
6154   match(PrefetchAllocation mem);
6155   ins_cost(125);
6156 
6157   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6158   ins_encode %{
6159     __ prefetcht2($mem$$Address);
6160   %}
6161   ins_pipe(ialu_mem);
6162 %}
6163 
6164 //----------Store Instructions-------------------------------------------------
6165 
6166 // Store Byte
6167 instruct storeB(memory mem, rRegI src)
6168 %{
6169   match(Set mem (StoreB mem src));
6170 
6171   ins_cost(125); // XXX
6172   format %{ "movb    $mem, $src\t# byte" %}
6173   opcode(0x88);
6174   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6175   ins_pipe(ialu_mem_reg);
6176 %}
6177 
6178 // Store Char/Short
6179 instruct storeC(memory mem, rRegI src)
6180 %{
6181   match(Set mem (StoreC mem src));
6182 
6183   ins_cost(125); // XXX
6184   format %{ "movw    $mem, $src\t# char/short" %}
6185   opcode(0x89);
6186   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6187   ins_pipe(ialu_mem_reg);
6188 %}
6189 
6190 // Store Integer
6191 instruct storeI(memory mem, rRegI src)
6192 %{
6193   match(Set mem (StoreI mem src));
6194 
6195   ins_cost(125); // XXX
6196   format %{ "movl    $mem, $src\t# int" %}
6197   opcode(0x89);
6198   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6199   ins_pipe(ialu_mem_reg);
6200 %}
6201 
6202 // Store Long
6203 instruct storeL(memory mem, rRegL src)
6204 %{
6205   match(Set mem (StoreL mem src));
6206 
6207   ins_cost(125); // XXX
6208   format %{ "movq    $mem, $src\t# long" %}
6209   opcode(0x89);
6210   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6211   ins_pipe(ialu_mem_reg); // XXX
6212 %}
6213 
6214 // Store Pointer
6215 instruct storeP(memory mem, any_RegP src)
6216 %{
6217   match(Set mem (StoreP mem src));
6218 
6219   ins_cost(125); // XXX
6220   format %{ "movq    $mem, $src\t# ptr" %}
6221   opcode(0x89);
6222   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6223   ins_pipe(ialu_mem_reg);
6224 %}
6225 
6226 instruct storeImmP0(memory mem, immP0 zero)
6227 %{
6228   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6229   match(Set mem (StoreP mem zero));
6230 
6231   ins_cost(125); // XXX
6232   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6233   ins_encode %{
6234     __ movq($mem$$Address, r12);
6235   %}
6236   ins_pipe(ialu_mem_reg);
6237 %}
6238 
6239 // Store NULL Pointer, mark word, or other simple pointer constant.
6240 instruct storeImmP(memory mem, immP31 src)
6241 %{
6242   match(Set mem (StoreP mem src));
6243 
6244   ins_cost(150); // XXX
6245   format %{ "movq    $mem, $src\t# ptr" %}
6246   opcode(0xC7); /* C7 /0 */
6247   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6248   ins_pipe(ialu_mem_imm);
6249 %}
6250 
6251 // Store Compressed Pointer
6252 instruct storeN(memory mem, rRegN src)
6253 %{
6254   match(Set mem (StoreN mem src));
6255 
6256   ins_cost(125); // XXX
6257   format %{ "movl    $mem, $src\t# compressed ptr" %}
6258   ins_encode %{
6259     __ movl($mem$$Address, $src$$Register);
6260   %}
6261   ins_pipe(ialu_mem_reg);
6262 %}
6263 
6264 instruct storeNKlass(memory mem, rRegN src)
6265 %{
6266   match(Set mem (StoreNKlass mem src));
6267 
6268   ins_cost(125); // XXX
6269   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6270   ins_encode %{
6271     __ movl($mem$$Address, $src$$Register);
6272   %}
6273   ins_pipe(ialu_mem_reg);
6274 %}
6275 
6276 instruct storeImmN0(memory mem, immN0 zero)
6277 %{
6278   predicate(CompressedOops::base() == NULL && CompressedKlassPointers::base() == NULL);
6279   match(Set mem (StoreN mem zero));
6280 
6281   ins_cost(125); // XXX
6282   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6283   ins_encode %{
6284     __ movl($mem$$Address, r12);
6285   %}
6286   ins_pipe(ialu_mem_reg);
6287 %}
6288 
6289 instruct storeImmN(memory mem, immN src)
6290 %{
6291   match(Set mem (StoreN mem src));
6292 
6293   ins_cost(150); // XXX
6294   format %{ "movl    $mem, $src\t# compressed ptr" %}
6295   ins_encode %{
6296     address con = (address)$src$$constant;
6297     if (con == NULL) {
6298       __ movl($mem$$Address, (int32_t)0);
6299     } else {
6300       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6301     }
6302   %}
6303   ins_pipe(ialu_mem_imm);
6304 %}
6305 
6306 instruct storeImmNKlass(memory mem, immNKlass src)
6307 %{
6308   match(Set mem (StoreNKlass mem src));
6309 
6310   ins_cost(150); // XXX
6311   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6312   ins_encode %{
6313     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6314   %}
6315   ins_pipe(ialu_mem_imm);
6316 %}
6317 
6318 // Store Integer Immediate
6319 instruct storeImmI0(memory mem, immI0 zero)
6320 %{
6321   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6322   match(Set mem (StoreI mem zero));
6323 
6324   ins_cost(125); // XXX
6325   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6326   ins_encode %{
6327     __ movl($mem$$Address, r12);
6328   %}
6329   ins_pipe(ialu_mem_reg);
6330 %}
6331 
6332 instruct storeImmI(memory mem, immI src)
6333 %{
6334   match(Set mem (StoreI mem src));
6335 
6336   ins_cost(150);
6337   format %{ "movl    $mem, $src\t# int" %}
6338   opcode(0xC7); /* C7 /0 */
6339   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6340   ins_pipe(ialu_mem_imm);
6341 %}
6342 
6343 // Store Long Immediate
6344 instruct storeImmL0(memory mem, immL0 zero)
6345 %{
6346   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6347   match(Set mem (StoreL mem zero));
6348 
6349   ins_cost(125); // XXX
6350   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6351   ins_encode %{
6352     __ movq($mem$$Address, r12);
6353   %}
6354   ins_pipe(ialu_mem_reg);
6355 %}
6356 
6357 instruct storeImmL(memory mem, immL32 src)
6358 %{
6359   match(Set mem (StoreL mem src));
6360 
6361   ins_cost(150);
6362   format %{ "movq    $mem, $src\t# long" %}
6363   opcode(0xC7); /* C7 /0 */
6364   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6365   ins_pipe(ialu_mem_imm);
6366 %}
6367 
6368 // Store Short/Char Immediate
6369 instruct storeImmC0(memory mem, immI0 zero)
6370 %{
6371   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6372   match(Set mem (StoreC mem zero));
6373 
6374   ins_cost(125); // XXX
6375   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6376   ins_encode %{
6377     __ movw($mem$$Address, r12);
6378   %}
6379   ins_pipe(ialu_mem_reg);
6380 %}
6381 
6382 instruct storeImmI16(memory mem, immI16 src)
6383 %{
6384   predicate(UseStoreImmI16);
6385   match(Set mem (StoreC mem src));
6386 
6387   ins_cost(150);
6388   format %{ "movw    $mem, $src\t# short/char" %}
6389   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6390   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6391   ins_pipe(ialu_mem_imm);
6392 %}
6393 
6394 // Store Byte Immediate
6395 instruct storeImmB0(memory mem, immI0 zero)
6396 %{
6397   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6398   match(Set mem (StoreB mem zero));
6399 
6400   ins_cost(125); // XXX
6401   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6402   ins_encode %{
6403     __ movb($mem$$Address, r12);
6404   %}
6405   ins_pipe(ialu_mem_reg);
6406 %}
6407 
6408 instruct storeImmB(memory mem, immI8 src)
6409 %{
6410   match(Set mem (StoreB mem src));
6411 
6412   ins_cost(150); // XXX
6413   format %{ "movb    $mem, $src\t# byte" %}
6414   opcode(0xC6); /* C6 /0 */
6415   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6416   ins_pipe(ialu_mem_imm);
6417 %}
6418 
6419 // Store CMS card-mark Immediate
6420 instruct storeImmCM0_reg(memory mem, immI0 zero)
6421 %{
6422   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6423   match(Set mem (StoreCM mem zero));
6424 
6425   ins_cost(125); // XXX
6426   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6427   ins_encode %{
6428     __ movb($mem$$Address, r12);
6429   %}
6430   ins_pipe(ialu_mem_reg);
6431 %}
6432 
6433 instruct storeImmCM0(memory mem, immI0 src)
6434 %{
6435   match(Set mem (StoreCM mem src));
6436 
6437   ins_cost(150); // XXX
6438   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6439   opcode(0xC6); /* C6 /0 */
6440   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6441   ins_pipe(ialu_mem_imm);
6442 %}
6443 
6444 // Store Float
6445 instruct storeF(memory mem, regF src)
6446 %{
6447   match(Set mem (StoreF mem src));
6448 
6449   ins_cost(95); // XXX
6450   format %{ "movss   $mem, $src\t# float" %}
6451   ins_encode %{
6452     __ movflt($mem$$Address, $src$$XMMRegister);
6453   %}
6454   ins_pipe(pipe_slow); // XXX
6455 %}
6456 
6457 // Store immediate Float value (it is faster than store from XMM register)
6458 instruct storeF0(memory mem, immF0 zero)
6459 %{
6460   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6461   match(Set mem (StoreF mem zero));
6462 
6463   ins_cost(25); // XXX
6464   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6465   ins_encode %{
6466     __ movl($mem$$Address, r12);
6467   %}
6468   ins_pipe(ialu_mem_reg);
6469 %}
6470 
6471 instruct storeF_imm(memory mem, immF src)
6472 %{
6473   match(Set mem (StoreF mem src));
6474 
6475   ins_cost(50);
6476   format %{ "movl    $mem, $src\t# float" %}
6477   opcode(0xC7); /* C7 /0 */
6478   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6479   ins_pipe(ialu_mem_imm);
6480 %}
6481 
6482 // Store Double
6483 instruct storeD(memory mem, regD src)
6484 %{
6485   match(Set mem (StoreD mem src));
6486 
6487   ins_cost(95); // XXX
6488   format %{ "movsd   $mem, $src\t# double" %}
6489   ins_encode %{
6490     __ movdbl($mem$$Address, $src$$XMMRegister);
6491   %}
6492   ins_pipe(pipe_slow); // XXX
6493 %}
6494 
6495 // Store immediate double 0.0 (it is faster than store from XMM register)
6496 instruct storeD0_imm(memory mem, immD0 src)
6497 %{
6498   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
6499   match(Set mem (StoreD mem src));
6500 
6501   ins_cost(50);
6502   format %{ "movq    $mem, $src\t# double 0." %}
6503   opcode(0xC7); /* C7 /0 */
6504   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6505   ins_pipe(ialu_mem_imm);
6506 %}
6507 
6508 instruct storeD0(memory mem, immD0 zero)
6509 %{
6510   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
6511   match(Set mem (StoreD mem zero));
6512 
6513   ins_cost(25); // XXX
6514   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6515   ins_encode %{
6516     __ movq($mem$$Address, r12);
6517   %}
6518   ins_pipe(ialu_mem_reg);
6519 %}
6520 
6521 instruct storeSSI(stackSlotI dst, rRegI src)
6522 %{
6523   match(Set dst src);
6524 
6525   ins_cost(100);
6526   format %{ "movl    $dst, $src\t# int stk" %}
6527   opcode(0x89);
6528   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6529   ins_pipe( ialu_mem_reg );
6530 %}
6531 
6532 instruct storeSSL(stackSlotL dst, rRegL src)
6533 %{
6534   match(Set dst src);
6535 
6536   ins_cost(100);
6537   format %{ "movq    $dst, $src\t# long stk" %}
6538   opcode(0x89);
6539   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6540   ins_pipe(ialu_mem_reg);
6541 %}
6542 
6543 instruct storeSSP(stackSlotP dst, rRegP src)
6544 %{
6545   match(Set dst src);
6546 
6547   ins_cost(100);
6548   format %{ "movq    $dst, $src\t# ptr stk" %}
6549   opcode(0x89);
6550   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6551   ins_pipe(ialu_mem_reg);
6552 %}
6553 
6554 instruct storeSSF(stackSlotF dst, regF src)
6555 %{
6556   match(Set dst src);
6557 
6558   ins_cost(95); // XXX
6559   format %{ "movss   $dst, $src\t# float stk" %}
6560   ins_encode %{
6561     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6562   %}
6563   ins_pipe(pipe_slow); // XXX
6564 %}
6565 
6566 instruct storeSSD(stackSlotD dst, regD src)
6567 %{
6568   match(Set dst src);
6569 
6570   ins_cost(95); // XXX
6571   format %{ "movsd   $dst, $src\t# double stk" %}
6572   ins_encode %{
6573     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6574   %}
6575   ins_pipe(pipe_slow); // XXX
6576 %}
6577 
6578 instruct cacheWB(indirect addr)
6579 %{
6580   predicate(VM_Version::supports_data_cache_line_flush());
6581   match(CacheWB addr);
6582 
6583   ins_cost(100);
6584   format %{"cache wb $addr" %}
6585   ins_encode %{
6586     assert($addr->index_position() < 0, "should be");
6587     assert($addr$$disp == 0, "should be");
6588     __ cache_wb(Address($addr$$base$$Register, 0));
6589   %}
6590   ins_pipe(pipe_slow); // XXX
6591 %}
6592 
6593 instruct cacheWBPreSync()
6594 %{
6595   predicate(VM_Version::supports_data_cache_line_flush());
6596   match(CacheWBPreSync);
6597 
6598   ins_cost(100);
6599   format %{"cache wb presync" %}
6600   ins_encode %{
6601     __ cache_wbsync(true);
6602   %}
6603   ins_pipe(pipe_slow); // XXX
6604 %}
6605 
6606 instruct cacheWBPostSync()
6607 %{
6608   predicate(VM_Version::supports_data_cache_line_flush());
6609   match(CacheWBPostSync);
6610 
6611   ins_cost(100);
6612   format %{"cache wb postsync" %}
6613   ins_encode %{
6614     __ cache_wbsync(false);
6615   %}
6616   ins_pipe(pipe_slow); // XXX
6617 %}
6618 
6619 //----------BSWAP Instructions-------------------------------------------------
6620 instruct bytes_reverse_int(rRegI dst) %{
6621   match(Set dst (ReverseBytesI dst));
6622 
6623   format %{ "bswapl  $dst" %}
6624   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6625   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6626   ins_pipe( ialu_reg );
6627 %}
6628 
6629 instruct bytes_reverse_long(rRegL dst) %{
6630   match(Set dst (ReverseBytesL dst));
6631 
6632   format %{ "bswapq  $dst" %}
6633   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6634   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6635   ins_pipe( ialu_reg);
6636 %}
6637 
6638 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6639   match(Set dst (ReverseBytesUS dst));
6640   effect(KILL cr);
6641 
6642   format %{ "bswapl  $dst\n\t"
6643             "shrl    $dst,16\n\t" %}
6644   ins_encode %{
6645     __ bswapl($dst$$Register);
6646     __ shrl($dst$$Register, 16);
6647   %}
6648   ins_pipe( ialu_reg );
6649 %}
6650 
6651 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6652   match(Set dst (ReverseBytesS dst));
6653   effect(KILL cr);
6654 
6655   format %{ "bswapl  $dst\n\t"
6656             "sar     $dst,16\n\t" %}
6657   ins_encode %{
6658     __ bswapl($dst$$Register);
6659     __ sarl($dst$$Register, 16);
6660   %}
6661   ins_pipe( ialu_reg );
6662 %}
6663 
6664 //---------- Zeros Count Instructions ------------------------------------------
6665 
6666 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6667   predicate(UseCountLeadingZerosInstruction);
6668   match(Set dst (CountLeadingZerosI src));
6669   effect(KILL cr);
6670 
6671   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6672   ins_encode %{
6673     __ lzcntl($dst$$Register, $src$$Register);
6674   %}
6675   ins_pipe(ialu_reg);
6676 %}
6677 
6678 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6679   predicate(!UseCountLeadingZerosInstruction);
6680   match(Set dst (CountLeadingZerosI src));
6681   effect(KILL cr);
6682 
6683   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6684             "jnz     skip\n\t"
6685             "movl    $dst, -1\n"
6686       "skip:\n\t"
6687             "negl    $dst\n\t"
6688             "addl    $dst, 31" %}
6689   ins_encode %{
6690     Register Rdst = $dst$$Register;
6691     Register Rsrc = $src$$Register;
6692     Label skip;
6693     __ bsrl(Rdst, Rsrc);
6694     __ jccb(Assembler::notZero, skip);
6695     __ movl(Rdst, -1);
6696     __ bind(skip);
6697     __ negl(Rdst);
6698     __ addl(Rdst, BitsPerInt - 1);
6699   %}
6700   ins_pipe(ialu_reg);
6701 %}
6702 
6703 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6704   predicate(UseCountLeadingZerosInstruction);
6705   match(Set dst (CountLeadingZerosL src));
6706   effect(KILL cr);
6707 
6708   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6709   ins_encode %{
6710     __ lzcntq($dst$$Register, $src$$Register);
6711   %}
6712   ins_pipe(ialu_reg);
6713 %}
6714 
6715 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6716   predicate(!UseCountLeadingZerosInstruction);
6717   match(Set dst (CountLeadingZerosL src));
6718   effect(KILL cr);
6719 
6720   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6721             "jnz     skip\n\t"
6722             "movl    $dst, -1\n"
6723       "skip:\n\t"
6724             "negl    $dst\n\t"
6725             "addl    $dst, 63" %}
6726   ins_encode %{
6727     Register Rdst = $dst$$Register;
6728     Register Rsrc = $src$$Register;
6729     Label skip;
6730     __ bsrq(Rdst, Rsrc);
6731     __ jccb(Assembler::notZero, skip);
6732     __ movl(Rdst, -1);
6733     __ bind(skip);
6734     __ negl(Rdst);
6735     __ addl(Rdst, BitsPerLong - 1);
6736   %}
6737   ins_pipe(ialu_reg);
6738 %}
6739 
6740 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6741   predicate(UseCountTrailingZerosInstruction);
6742   match(Set dst (CountTrailingZerosI src));
6743   effect(KILL cr);
6744 
6745   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6746   ins_encode %{
6747     __ tzcntl($dst$$Register, $src$$Register);
6748   %}
6749   ins_pipe(ialu_reg);
6750 %}
6751 
6752 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6753   predicate(!UseCountTrailingZerosInstruction);
6754   match(Set dst (CountTrailingZerosI src));
6755   effect(KILL cr);
6756 
6757   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6758             "jnz     done\n\t"
6759             "movl    $dst, 32\n"
6760       "done:" %}
6761   ins_encode %{
6762     Register Rdst = $dst$$Register;
6763     Label done;
6764     __ bsfl(Rdst, $src$$Register);
6765     __ jccb(Assembler::notZero, done);
6766     __ movl(Rdst, BitsPerInt);
6767     __ bind(done);
6768   %}
6769   ins_pipe(ialu_reg);
6770 %}
6771 
6772 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6773   predicate(UseCountTrailingZerosInstruction);
6774   match(Set dst (CountTrailingZerosL src));
6775   effect(KILL cr);
6776 
6777   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6778   ins_encode %{
6779     __ tzcntq($dst$$Register, $src$$Register);
6780   %}
6781   ins_pipe(ialu_reg);
6782 %}
6783 
6784 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6785   predicate(!UseCountTrailingZerosInstruction);
6786   match(Set dst (CountTrailingZerosL src));
6787   effect(KILL cr);
6788 
6789   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6790             "jnz     done\n\t"
6791             "movl    $dst, 64\n"
6792       "done:" %}
6793   ins_encode %{
6794     Register Rdst = $dst$$Register;
6795     Label done;
6796     __ bsfq(Rdst, $src$$Register);
6797     __ jccb(Assembler::notZero, done);
6798     __ movl(Rdst, BitsPerLong);
6799     __ bind(done);
6800   %}
6801   ins_pipe(ialu_reg);
6802 %}
6803 
6804 
6805 //---------- Population Count Instructions -------------------------------------
6806 
6807 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6808   predicate(UsePopCountInstruction);
6809   match(Set dst (PopCountI src));
6810   effect(KILL cr);
6811 
6812   format %{ "popcnt  $dst, $src" %}
6813   ins_encode %{
6814     __ popcntl($dst$$Register, $src$$Register);
6815   %}
6816   ins_pipe(ialu_reg);
6817 %}
6818 
6819 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6820   predicate(UsePopCountInstruction);
6821   match(Set dst (PopCountI (LoadI mem)));
6822   effect(KILL cr);
6823 
6824   format %{ "popcnt  $dst, $mem" %}
6825   ins_encode %{
6826     __ popcntl($dst$$Register, $mem$$Address);
6827   %}
6828   ins_pipe(ialu_reg);
6829 %}
6830 
6831 // Note: Long.bitCount(long) returns an int.
6832 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6833   predicate(UsePopCountInstruction);
6834   match(Set dst (PopCountL src));
6835   effect(KILL cr);
6836 
6837   format %{ "popcnt  $dst, $src" %}
6838   ins_encode %{
6839     __ popcntq($dst$$Register, $src$$Register);
6840   %}
6841   ins_pipe(ialu_reg);
6842 %}
6843 
6844 // Note: Long.bitCount(long) returns an int.
6845 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6846   predicate(UsePopCountInstruction);
6847   match(Set dst (PopCountL (LoadL mem)));
6848   effect(KILL cr);
6849 
6850   format %{ "popcnt  $dst, $mem" %}
6851   ins_encode %{
6852     __ popcntq($dst$$Register, $mem$$Address);
6853   %}
6854   ins_pipe(ialu_reg);
6855 %}
6856 
6857 
6858 //----------MemBar Instructions-----------------------------------------------
6859 // Memory barrier flavors
6860 
6861 instruct membar_acquire()
6862 %{
6863   match(MemBarAcquire);
6864   match(LoadFence);
6865   ins_cost(0);
6866 
6867   size(0);
6868   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6869   ins_encode();
6870   ins_pipe(empty);
6871 %}
6872 
6873 instruct membar_acquire_lock()
6874 %{
6875   match(MemBarAcquireLock);
6876   ins_cost(0);
6877 
6878   size(0);
6879   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6880   ins_encode();
6881   ins_pipe(empty);
6882 %}
6883 
6884 instruct membar_release()
6885 %{
6886   match(MemBarRelease);
6887   match(StoreFence);
6888   ins_cost(0);
6889 
6890   size(0);
6891   format %{ "MEMBAR-release ! (empty encoding)" %}
6892   ins_encode();
6893   ins_pipe(empty);
6894 %}
6895 
6896 instruct membar_release_lock()
6897 %{
6898   match(MemBarReleaseLock);
6899   ins_cost(0);
6900 
6901   size(0);
6902   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6903   ins_encode();
6904   ins_pipe(empty);
6905 %}
6906 
6907 instruct membar_volatile(rFlagsReg cr) %{
6908   match(MemBarVolatile);
6909   effect(KILL cr);
6910   ins_cost(400);
6911 
6912   format %{
6913     $$template
6914     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6915   %}
6916   ins_encode %{
6917     __ membar(Assembler::StoreLoad);
6918   %}
6919   ins_pipe(pipe_slow);
6920 %}
6921 
6922 instruct unnecessary_membar_volatile()
6923 %{
6924   match(MemBarVolatile);
6925   predicate(Matcher::post_store_load_barrier(n));
6926   ins_cost(0);
6927 
6928   size(0);
6929   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6930   ins_encode();
6931   ins_pipe(empty);
6932 %}
6933 
6934 instruct membar_storestore() %{
6935   match(MemBarStoreStore);
6936   ins_cost(0);
6937 
6938   size(0);
6939   format %{ "MEMBAR-storestore (empty encoding)" %}
6940   ins_encode( );
6941   ins_pipe(empty);
6942 %}
6943 
6944 //----------Move Instructions--------------------------------------------------
6945 
6946 instruct castX2P(rRegP dst, rRegL src)
6947 %{
6948   match(Set dst (CastX2P src));
6949 
6950   format %{ "movq    $dst, $src\t# long->ptr" %}
6951   ins_encode %{
6952     if ($dst$$reg != $src$$reg) {
6953       __ movptr($dst$$Register, $src$$Register);
6954     }
6955   %}
6956   ins_pipe(ialu_reg_reg); // XXX
6957 %}
6958 
6959 instruct castN2X(rRegL dst, rRegN src)
6960 %{
6961   match(Set dst (CastP2X src));
6962 
6963   format %{ "movq    $dst, $src\t# ptr -> long" %}
6964   ins_encode %{
6965     if ($dst$$reg != $src$$reg) {
6966       __ movptr($dst$$Register, $src$$Register);
6967     }
6968   %}
6969   ins_pipe(ialu_reg_reg); // XXX
6970 %}
6971 
6972 instruct castP2X(rRegL dst, rRegP src)
6973 %{
6974   match(Set dst (CastP2X src));
6975 
6976   format %{ "movq    $dst, $src\t# ptr -> long" %}
6977   ins_encode %{
6978     if ($dst$$reg != $src$$reg) {
6979       __ movptr($dst$$Register, $src$$Register);
6980     }
6981   %}
6982   ins_pipe(ialu_reg_reg); // XXX
6983 %}
6984 
6985 instruct castN2I(rRegI dst, rRegN src)
6986 %{
6987   match(Set dst (CastN2I src));
6988 
6989   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6990   ins_encode %{
6991     if ($dst$$reg != $src$$reg) {
6992       __ movl($dst$$Register, $src$$Register);
6993     }
6994   %}
6995   ins_pipe(ialu_reg_reg); // XXX
6996 %}
6997 
6998 instruct castI2N(rRegN dst, rRegI src)
6999 %{
7000   match(Set dst (CastI2N src));
7001 
7002   format %{ "movl    $dst, $src\t# int -> compressed ptr" %}
7003   ins_encode %{
7004     if ($dst$$reg != $src$$reg) {
7005       __ movl($dst$$Register, $src$$Register);
7006     }
7007   %}
7008   ins_pipe(ialu_reg_reg); // XXX
7009 %}
7010 
7011 
7012 // Convert oop into int for vectors alignment masking
7013 instruct convP2I(rRegI dst, rRegP src)
7014 %{
7015   match(Set dst (ConvL2I (CastP2X src)));
7016 
7017   format %{ "movl    $dst, $src\t# ptr -> int" %}
7018   ins_encode %{
7019     __ movl($dst$$Register, $src$$Register);
7020   %}
7021   ins_pipe(ialu_reg_reg); // XXX
7022 %}
7023 
7024 // Convert compressed oop into int for vectors alignment masking
7025 // in case of 32bit oops (heap < 4Gb).
7026 instruct convN2I(rRegI dst, rRegN src)
7027 %{
7028   predicate(CompressedOops::shift() == 0);
7029   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7030 
7031   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
7032   ins_encode %{
7033     __ movl($dst$$Register, $src$$Register);
7034   %}
7035   ins_pipe(ialu_reg_reg); // XXX
7036 %}
7037 
7038 // Convert oop pointer into compressed form
7039 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
7040   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7041   match(Set dst (EncodeP src));
7042   effect(KILL cr);
7043   format %{ "encode_heap_oop $dst,$src" %}
7044   ins_encode %{
7045     Register s = $src$$Register;
7046     Register d = $dst$$Register;
7047     if (s != d) {
7048       __ movq(d, s);
7049     }
7050     __ encode_heap_oop(d);
7051   %}
7052   ins_pipe(ialu_reg_long);
7053 %}
7054 
7055 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7056   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7057   match(Set dst (EncodeP src));
7058   effect(KILL cr);
7059   format %{ "encode_heap_oop_not_null $dst,$src" %}
7060   ins_encode %{
7061     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7062   %}
7063   ins_pipe(ialu_reg_long);
7064 %}
7065 
7066 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
7067   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7068             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7069   match(Set dst (DecodeN src));
7070   effect(KILL cr);
7071   format %{ "decode_heap_oop $dst,$src" %}
7072   ins_encode %{
7073     Register s = $src$$Register;
7074     Register d = $dst$$Register;
7075     if (s != d) {
7076       __ movq(d, s);
7077     }
7078     __ decode_heap_oop(d);
7079   %}
7080   ins_pipe(ialu_reg_long);
7081 %}
7082 
7083 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7084   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7085             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7086   match(Set dst (DecodeN src));
7087   effect(KILL cr);
7088   format %{ "decode_heap_oop_not_null $dst,$src" %}
7089   ins_encode %{
7090     Register s = $src$$Register;
7091     Register d = $dst$$Register;
7092     if (s != d) {
7093       __ decode_heap_oop_not_null(d, s);
7094     } else {
7095       __ decode_heap_oop_not_null(d);
7096     }
7097   %}
7098   ins_pipe(ialu_reg_long);
7099 %}
7100 
7101 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7102   match(Set dst (EncodePKlass src));
7103   effect(KILL cr);
7104   format %{ "encode_klass_not_null $dst,$src" %}
7105   ins_encode %{
7106     __ encode_klass_not_null($dst$$Register, $src$$Register);
7107   %}
7108   ins_pipe(ialu_reg_long);
7109 %}
7110 
7111 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7112   match(Set dst (DecodeNKlass src));
7113   effect(KILL cr);
7114   format %{ "decode_klass_not_null $dst,$src" %}
7115   ins_encode %{
7116     Register s = $src$$Register;
7117     Register d = $dst$$Register;
7118     if (s != d) {
7119       __ decode_klass_not_null(d, s);
7120     } else {
7121       __ decode_klass_not_null(d);
7122     }
7123   %}
7124   ins_pipe(ialu_reg_long);
7125 %}
7126 
7127 
7128 //----------Conditional Move---------------------------------------------------
7129 // Jump
7130 // dummy instruction for generating temp registers
7131 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7132   match(Jump (LShiftL switch_val shift));
7133   ins_cost(350);
7134   predicate(false);
7135   effect(TEMP dest);
7136 
7137   format %{ "leaq    $dest, [$constantaddress]\n\t"
7138             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7139   ins_encode %{
7140     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7141     // to do that and the compiler is using that register as one it can allocate.
7142     // So we build it all by hand.
7143     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7144     // ArrayAddress dispatch(table, index);
7145     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7146     __ lea($dest$$Register, $constantaddress);
7147     __ jmp(dispatch);
7148   %}
7149   ins_pipe(pipe_jmp);
7150 %}
7151 
7152 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7153   match(Jump (AddL (LShiftL switch_val shift) offset));
7154   ins_cost(350);
7155   effect(TEMP dest);
7156 
7157   format %{ "leaq    $dest, [$constantaddress]\n\t"
7158             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7159   ins_encode %{
7160     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7161     // to do that and the compiler is using that register as one it can allocate.
7162     // So we build it all by hand.
7163     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7164     // ArrayAddress dispatch(table, index);
7165     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7166     __ lea($dest$$Register, $constantaddress);
7167     __ jmp(dispatch);
7168   %}
7169   ins_pipe(pipe_jmp);
7170 %}
7171 
7172 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7173   match(Jump switch_val);
7174   ins_cost(350);
7175   effect(TEMP dest);
7176 
7177   format %{ "leaq    $dest, [$constantaddress]\n\t"
7178             "jmp     [$dest + $switch_val]\n\t" %}
7179   ins_encode %{
7180     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7181     // to do that and the compiler is using that register as one it can allocate.
7182     // So we build it all by hand.
7183     // Address index(noreg, switch_reg, Address::times_1);
7184     // ArrayAddress dispatch(table, index);
7185     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7186     __ lea($dest$$Register, $constantaddress);
7187     __ jmp(dispatch);
7188   %}
7189   ins_pipe(pipe_jmp);
7190 %}
7191 
7192 // Conditional move
7193 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7194 %{
7195   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7196 
7197   ins_cost(200); // XXX
7198   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7199   opcode(0x0F, 0x40);
7200   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7201   ins_pipe(pipe_cmov_reg);
7202 %}
7203 
7204 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7205   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7206 
7207   ins_cost(200); // XXX
7208   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7209   opcode(0x0F, 0x40);
7210   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7211   ins_pipe(pipe_cmov_reg);
7212 %}
7213 
7214 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7215   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7216   ins_cost(200);
7217   expand %{
7218     cmovI_regU(cop, cr, dst, src);
7219   %}
7220 %}
7221 
7222 // Conditional move
7223 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7224   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7225 
7226   ins_cost(250); // XXX
7227   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7228   opcode(0x0F, 0x40);
7229   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7230   ins_pipe(pipe_cmov_mem);
7231 %}
7232 
7233 // Conditional move
7234 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7235 %{
7236   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7237 
7238   ins_cost(250); // XXX
7239   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7240   opcode(0x0F, 0x40);
7241   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7242   ins_pipe(pipe_cmov_mem);
7243 %}
7244 
7245 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7246   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7247   ins_cost(250);
7248   expand %{
7249     cmovI_memU(cop, cr, dst, src);
7250   %}
7251 %}
7252 
7253 // Conditional move
7254 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7255 %{
7256   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7257 
7258   ins_cost(200); // XXX
7259   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7260   opcode(0x0F, 0x40);
7261   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7262   ins_pipe(pipe_cmov_reg);
7263 %}
7264 
7265 // Conditional move
7266 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7267 %{
7268   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7269 
7270   ins_cost(200); // XXX
7271   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7272   opcode(0x0F, 0x40);
7273   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7274   ins_pipe(pipe_cmov_reg);
7275 %}
7276 
7277 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7278   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7279   ins_cost(200);
7280   expand %{
7281     cmovN_regU(cop, cr, dst, src);
7282   %}
7283 %}
7284 
7285 // Conditional move
7286 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7287 %{
7288   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7289 
7290   ins_cost(200); // XXX
7291   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7292   opcode(0x0F, 0x40);
7293   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7294   ins_pipe(pipe_cmov_reg);  // XXX
7295 %}
7296 
7297 // Conditional move
7298 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7299 %{
7300   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7301 
7302   ins_cost(200); // XXX
7303   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7304   opcode(0x0F, 0x40);
7305   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7306   ins_pipe(pipe_cmov_reg); // XXX
7307 %}
7308 
7309 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7310   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7311   ins_cost(200);
7312   expand %{
7313     cmovP_regU(cop, cr, dst, src);
7314   %}
7315 %}
7316 
7317 // DISABLED: Requires the ADLC to emit a bottom_type call that
7318 // correctly meets the two pointer arguments; one is an incoming
7319 // register but the other is a memory operand.  ALSO appears to
7320 // be buggy with implicit null checks.
7321 //
7322 //// Conditional move
7323 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7324 //%{
7325 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7326 //  ins_cost(250);
7327 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7328 //  opcode(0x0F,0x40);
7329 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7330 //  ins_pipe( pipe_cmov_mem );
7331 //%}
7332 //
7333 //// Conditional move
7334 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7335 //%{
7336 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7337 //  ins_cost(250);
7338 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7339 //  opcode(0x0F,0x40);
7340 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7341 //  ins_pipe( pipe_cmov_mem );
7342 //%}
7343 
7344 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7345 %{
7346   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7347 
7348   ins_cost(200); // XXX
7349   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7350   opcode(0x0F, 0x40);
7351   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7352   ins_pipe(pipe_cmov_reg);  // XXX
7353 %}
7354 
7355 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7356 %{
7357   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7358 
7359   ins_cost(200); // XXX
7360   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7361   opcode(0x0F, 0x40);
7362   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7363   ins_pipe(pipe_cmov_mem);  // XXX
7364 %}
7365 
7366 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7367 %{
7368   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7369 
7370   ins_cost(200); // XXX
7371   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7372   opcode(0x0F, 0x40);
7373   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7374   ins_pipe(pipe_cmov_reg); // XXX
7375 %}
7376 
7377 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7378   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7379   ins_cost(200);
7380   expand %{
7381     cmovL_regU(cop, cr, dst, src);
7382   %}
7383 %}
7384 
7385 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7386 %{
7387   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7388 
7389   ins_cost(200); // XXX
7390   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7391   opcode(0x0F, 0x40);
7392   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7393   ins_pipe(pipe_cmov_mem); // XXX
7394 %}
7395 
7396 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7397   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7398   ins_cost(200);
7399   expand %{
7400     cmovL_memU(cop, cr, dst, src);
7401   %}
7402 %}
7403 
7404 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7405 %{
7406   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7407 
7408   ins_cost(200); // XXX
7409   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7410             "movss     $dst, $src\n"
7411     "skip:" %}
7412   ins_encode %{
7413     Label Lskip;
7414     // Invert sense of branch from sense of CMOV
7415     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7416     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7417     __ bind(Lskip);
7418   %}
7419   ins_pipe(pipe_slow);
7420 %}
7421 
7422 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7423 // %{
7424 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7425 
7426 //   ins_cost(200); // XXX
7427 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7428 //             "movss     $dst, $src\n"
7429 //     "skip:" %}
7430 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7431 //   ins_pipe(pipe_slow);
7432 // %}
7433 
7434 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7435 %{
7436   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7437 
7438   ins_cost(200); // XXX
7439   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7440             "movss     $dst, $src\n"
7441     "skip:" %}
7442   ins_encode %{
7443     Label Lskip;
7444     // Invert sense of branch from sense of CMOV
7445     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7446     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7447     __ bind(Lskip);
7448   %}
7449   ins_pipe(pipe_slow);
7450 %}
7451 
7452 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7453   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7454   ins_cost(200);
7455   expand %{
7456     cmovF_regU(cop, cr, dst, src);
7457   %}
7458 %}
7459 
7460 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7461 %{
7462   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7463 
7464   ins_cost(200); // XXX
7465   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7466             "movsd     $dst, $src\n"
7467     "skip:" %}
7468   ins_encode %{
7469     Label Lskip;
7470     // Invert sense of branch from sense of CMOV
7471     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7472     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7473     __ bind(Lskip);
7474   %}
7475   ins_pipe(pipe_slow);
7476 %}
7477 
7478 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7479 %{
7480   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7481 
7482   ins_cost(200); // XXX
7483   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7484             "movsd     $dst, $src\n"
7485     "skip:" %}
7486   ins_encode %{
7487     Label Lskip;
7488     // Invert sense of branch from sense of CMOV
7489     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7490     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7491     __ bind(Lskip);
7492   %}
7493   ins_pipe(pipe_slow);
7494 %}
7495 
7496 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7497   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7498   ins_cost(200);
7499   expand %{
7500     cmovD_regU(cop, cr, dst, src);
7501   %}
7502 %}
7503 
7504 //----------Arithmetic Instructions--------------------------------------------
7505 //----------Addition Instructions----------------------------------------------
7506 
7507 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7508 %{
7509   match(Set dst (AddI dst src));
7510   effect(KILL cr);
7511 
7512   format %{ "addl    $dst, $src\t# int" %}
7513   opcode(0x03);
7514   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7515   ins_pipe(ialu_reg_reg);
7516 %}
7517 
7518 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7519 %{
7520   match(Set dst (AddI dst src));
7521   effect(KILL cr);
7522 
7523   format %{ "addl    $dst, $src\t# int" %}
7524   opcode(0x81, 0x00); /* /0 id */
7525   ins_encode(OpcSErm(dst, src), Con8or32(src));
7526   ins_pipe( ialu_reg );
7527 %}
7528 
7529 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7530 %{
7531   match(Set dst (AddI dst (LoadI src)));
7532   effect(KILL cr);
7533 
7534   ins_cost(125); // XXX
7535   format %{ "addl    $dst, $src\t# int" %}
7536   opcode(0x03);
7537   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7538   ins_pipe(ialu_reg_mem);
7539 %}
7540 
7541 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7542 %{
7543   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7544   effect(KILL cr);
7545 
7546   ins_cost(150); // XXX
7547   format %{ "addl    $dst, $src\t# int" %}
7548   opcode(0x01); /* Opcode 01 /r */
7549   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7550   ins_pipe(ialu_mem_reg);
7551 %}
7552 
7553 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7554 %{
7555   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7556   effect(KILL cr);
7557 
7558   ins_cost(125); // XXX
7559   format %{ "addl    $dst, $src\t# int" %}
7560   opcode(0x81); /* Opcode 81 /0 id */
7561   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7562   ins_pipe(ialu_mem_imm);
7563 %}
7564 
7565 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7566 %{
7567   predicate(UseIncDec);
7568   match(Set dst (AddI dst src));
7569   effect(KILL cr);
7570 
7571   format %{ "incl    $dst\t# int" %}
7572   opcode(0xFF, 0x00); // FF /0
7573   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7574   ins_pipe(ialu_reg);
7575 %}
7576 
7577 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7578 %{
7579   predicate(UseIncDec);
7580   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7581   effect(KILL cr);
7582 
7583   ins_cost(125); // XXX
7584   format %{ "incl    $dst\t# int" %}
7585   opcode(0xFF); /* Opcode FF /0 */
7586   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7587   ins_pipe(ialu_mem_imm);
7588 %}
7589 
7590 // XXX why does that use AddI
7591 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7592 %{
7593   predicate(UseIncDec);
7594   match(Set dst (AddI dst src));
7595   effect(KILL cr);
7596 
7597   format %{ "decl    $dst\t# int" %}
7598   opcode(0xFF, 0x01); // FF /1
7599   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7600   ins_pipe(ialu_reg);
7601 %}
7602 
7603 // XXX why does that use AddI
7604 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7605 %{
7606   predicate(UseIncDec);
7607   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7608   effect(KILL cr);
7609 
7610   ins_cost(125); // XXX
7611   format %{ "decl    $dst\t# int" %}
7612   opcode(0xFF); /* Opcode FF /1 */
7613   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7614   ins_pipe(ialu_mem_imm);
7615 %}
7616 
7617 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7618 %{
7619   match(Set dst (AddI src0 src1));
7620 
7621   ins_cost(110);
7622   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7623   opcode(0x8D); /* 0x8D /r */
7624   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7625   ins_pipe(ialu_reg_reg);
7626 %}
7627 
7628 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7629 %{
7630   match(Set dst (AddL dst src));
7631   effect(KILL cr);
7632 
7633   format %{ "addq    $dst, $src\t# long" %}
7634   opcode(0x03);
7635   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7636   ins_pipe(ialu_reg_reg);
7637 %}
7638 
7639 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7640 %{
7641   match(Set dst (AddL dst src));
7642   effect(KILL cr);
7643 
7644   format %{ "addq    $dst, $src\t# long" %}
7645   opcode(0x81, 0x00); /* /0 id */
7646   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7647   ins_pipe( ialu_reg );
7648 %}
7649 
7650 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7651 %{
7652   match(Set dst (AddL dst (LoadL src)));
7653   effect(KILL cr);
7654 
7655   ins_cost(125); // XXX
7656   format %{ "addq    $dst, $src\t# long" %}
7657   opcode(0x03);
7658   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7659   ins_pipe(ialu_reg_mem);
7660 %}
7661 
7662 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7663 %{
7664   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7665   effect(KILL cr);
7666 
7667   ins_cost(150); // XXX
7668   format %{ "addq    $dst, $src\t# long" %}
7669   opcode(0x01); /* Opcode 01 /r */
7670   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7671   ins_pipe(ialu_mem_reg);
7672 %}
7673 
7674 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7675 %{
7676   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7677   effect(KILL cr);
7678 
7679   ins_cost(125); // XXX
7680   format %{ "addq    $dst, $src\t# long" %}
7681   opcode(0x81); /* Opcode 81 /0 id */
7682   ins_encode(REX_mem_wide(dst),
7683              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7684   ins_pipe(ialu_mem_imm);
7685 %}
7686 
7687 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7688 %{
7689   predicate(UseIncDec);
7690   match(Set dst (AddL dst src));
7691   effect(KILL cr);
7692 
7693   format %{ "incq    $dst\t# long" %}
7694   opcode(0xFF, 0x00); // FF /0
7695   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7696   ins_pipe(ialu_reg);
7697 %}
7698 
7699 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7700 %{
7701   predicate(UseIncDec);
7702   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7703   effect(KILL cr);
7704 
7705   ins_cost(125); // XXX
7706   format %{ "incq    $dst\t# long" %}
7707   opcode(0xFF); /* Opcode FF /0 */
7708   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7709   ins_pipe(ialu_mem_imm);
7710 %}
7711 
7712 // XXX why does that use AddL
7713 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7714 %{
7715   predicate(UseIncDec);
7716   match(Set dst (AddL dst src));
7717   effect(KILL cr);
7718 
7719   format %{ "decq    $dst\t# long" %}
7720   opcode(0xFF, 0x01); // FF /1
7721   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7722   ins_pipe(ialu_reg);
7723 %}
7724 
7725 // XXX why does that use AddL
7726 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7727 %{
7728   predicate(UseIncDec);
7729   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7730   effect(KILL cr);
7731 
7732   ins_cost(125); // XXX
7733   format %{ "decq    $dst\t# long" %}
7734   opcode(0xFF); /* Opcode FF /1 */
7735   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7736   ins_pipe(ialu_mem_imm);
7737 %}
7738 
7739 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7740 %{
7741   match(Set dst (AddL src0 src1));
7742 
7743   ins_cost(110);
7744   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7745   opcode(0x8D); /* 0x8D /r */
7746   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7747   ins_pipe(ialu_reg_reg);
7748 %}
7749 
7750 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7751 %{
7752   match(Set dst (AddP dst src));
7753   effect(KILL cr);
7754 
7755   format %{ "addq    $dst, $src\t# ptr" %}
7756   opcode(0x03);
7757   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7758   ins_pipe(ialu_reg_reg);
7759 %}
7760 
7761 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7762 %{
7763   match(Set dst (AddP dst src));
7764   effect(KILL cr);
7765 
7766   format %{ "addq    $dst, $src\t# ptr" %}
7767   opcode(0x81, 0x00); /* /0 id */
7768   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7769   ins_pipe( ialu_reg );
7770 %}
7771 
7772 // XXX addP mem ops ????
7773 
7774 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7775 %{
7776   match(Set dst (AddP src0 src1));
7777 
7778   ins_cost(110);
7779   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7780   opcode(0x8D); /* 0x8D /r */
7781   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7782   ins_pipe(ialu_reg_reg);
7783 %}
7784 
7785 instruct checkCastPP(rRegP dst)
7786 %{
7787   match(Set dst (CheckCastPP dst));
7788 
7789   size(0);
7790   format %{ "# checkcastPP of $dst" %}
7791   ins_encode(/* empty encoding */);
7792   ins_pipe(empty);
7793 %}
7794 
7795 instruct castPP(rRegP dst)
7796 %{
7797   match(Set dst (CastPP dst));
7798 
7799   size(0);
7800   format %{ "# castPP of $dst" %}
7801   ins_encode(/* empty encoding */);
7802   ins_pipe(empty);
7803 %}
7804 
7805 instruct castII(rRegI dst)
7806 %{
7807   match(Set dst (CastII dst));
7808 
7809   size(0);
7810   format %{ "# castII of $dst" %}
7811   ins_encode(/* empty encoding */);
7812   ins_cost(0);
7813   ins_pipe(empty);
7814 %}
7815 
7816 instruct castLL(rRegL dst)
7817 %{
7818   match(Set dst (CastLL dst));
7819 
7820   size(0);
7821   format %{ "# castLL of $dst" %}
7822   ins_encode(/* empty encoding */);
7823   ins_cost(0);
7824   ins_pipe(empty);
7825 %}
7826 
7827 // LoadP-locked same as a regular LoadP when used with compare-swap
7828 instruct loadPLocked(rRegP dst, memory mem)
7829 %{
7830   match(Set dst (LoadPLocked mem));
7831 
7832   ins_cost(125); // XXX
7833   format %{ "movq    $dst, $mem\t# ptr locked" %}
7834   opcode(0x8B);
7835   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7836   ins_pipe(ialu_reg_mem); // XXX
7837 %}
7838 
7839 // Conditional-store of the updated heap-top.
7840 // Used during allocation of the shared heap.
7841 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7842 
7843 instruct storePConditional(memory heap_top_ptr,
7844                            rax_RegP oldval, rRegP newval,
7845                            rFlagsReg cr)
7846 %{
7847   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7848 
7849   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7850             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7851   opcode(0x0F, 0xB1);
7852   ins_encode(lock_prefix,
7853              REX_reg_mem_wide(newval, heap_top_ptr),
7854              OpcP, OpcS,
7855              reg_mem(newval, heap_top_ptr));
7856   ins_pipe(pipe_cmpxchg);
7857 %}
7858 
7859 // Conditional-store of an int value.
7860 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7861 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7862 %{
7863   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7864   effect(KILL oldval);
7865 
7866   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7867   opcode(0x0F, 0xB1);
7868   ins_encode(lock_prefix,
7869              REX_reg_mem(newval, mem),
7870              OpcP, OpcS,
7871              reg_mem(newval, mem));
7872   ins_pipe(pipe_cmpxchg);
7873 %}
7874 
7875 // Conditional-store of a long value.
7876 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7877 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7878 %{
7879   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7880   effect(KILL oldval);
7881 
7882   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7883   opcode(0x0F, 0xB1);
7884   ins_encode(lock_prefix,
7885              REX_reg_mem_wide(newval, mem),
7886              OpcP, OpcS,
7887              reg_mem(newval, mem));
7888   ins_pipe(pipe_cmpxchg);
7889 %}
7890 
7891 
7892 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7893 instruct compareAndSwapP(rRegI res,
7894                          memory mem_ptr,
7895                          rax_RegP oldval, rRegP newval,
7896                          rFlagsReg cr)
7897 %{
7898   predicate(VM_Version::supports_cx8());
7899   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7900   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7901   effect(KILL cr, KILL oldval);
7902 
7903   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7904             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7905             "sete    $res\n\t"
7906             "movzbl  $res, $res" %}
7907   opcode(0x0F, 0xB1);
7908   ins_encode(lock_prefix,
7909              REX_reg_mem_wide(newval, mem_ptr),
7910              OpcP, OpcS,
7911              reg_mem(newval, mem_ptr),
7912              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7913              REX_reg_breg(res, res), // movzbl
7914              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7915   ins_pipe( pipe_cmpxchg );
7916 %}
7917 
7918 instruct compareAndSwapL(rRegI res,
7919                          memory mem_ptr,
7920                          rax_RegL oldval, rRegL newval,
7921                          rFlagsReg cr)
7922 %{
7923   predicate(VM_Version::supports_cx8());
7924   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7925   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7926   effect(KILL cr, KILL oldval);
7927 
7928   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7929             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7930             "sete    $res\n\t"
7931             "movzbl  $res, $res" %}
7932   opcode(0x0F, 0xB1);
7933   ins_encode(lock_prefix,
7934              REX_reg_mem_wide(newval, mem_ptr),
7935              OpcP, OpcS,
7936              reg_mem(newval, mem_ptr),
7937              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7938              REX_reg_breg(res, res), // movzbl
7939              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7940   ins_pipe( pipe_cmpxchg );
7941 %}
7942 
7943 instruct compareAndSwapI(rRegI res,
7944                          memory mem_ptr,
7945                          rax_RegI oldval, rRegI newval,
7946                          rFlagsReg cr)
7947 %{
7948   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7949   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7950   effect(KILL cr, KILL oldval);
7951 
7952   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7953             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7954             "sete    $res\n\t"
7955             "movzbl  $res, $res" %}
7956   opcode(0x0F, 0xB1);
7957   ins_encode(lock_prefix,
7958              REX_reg_mem(newval, mem_ptr),
7959              OpcP, OpcS,
7960              reg_mem(newval, mem_ptr),
7961              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7962              REX_reg_breg(res, res), // movzbl
7963              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7964   ins_pipe( pipe_cmpxchg );
7965 %}
7966 
7967 instruct compareAndSwapB(rRegI res,
7968                          memory mem_ptr,
7969                          rax_RegI oldval, rRegI newval,
7970                          rFlagsReg cr)
7971 %{
7972   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7973   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7974   effect(KILL cr, KILL oldval);
7975 
7976   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7977             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7978             "sete    $res\n\t"
7979             "movzbl  $res, $res" %}
7980   opcode(0x0F, 0xB0);
7981   ins_encode(lock_prefix,
7982              REX_breg_mem(newval, mem_ptr),
7983              OpcP, OpcS,
7984              reg_mem(newval, mem_ptr),
7985              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7986              REX_reg_breg(res, res), // movzbl
7987              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7988   ins_pipe( pipe_cmpxchg );
7989 %}
7990 
7991 instruct compareAndSwapS(rRegI res,
7992                          memory mem_ptr,
7993                          rax_RegI oldval, rRegI newval,
7994                          rFlagsReg cr)
7995 %{
7996   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7997   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7998   effect(KILL cr, KILL oldval);
7999 
8000   format %{ "cmpxchgw $mem_ptr,$newval\t# "
8001             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8002             "sete    $res\n\t"
8003             "movzbl  $res, $res" %}
8004   opcode(0x0F, 0xB1);
8005   ins_encode(lock_prefix,
8006              SizePrefix,
8007              REX_reg_mem(newval, mem_ptr),
8008              OpcP, OpcS,
8009              reg_mem(newval, mem_ptr),
8010              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8011              REX_reg_breg(res, res), // movzbl
8012              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8013   ins_pipe( pipe_cmpxchg );
8014 %}
8015 
8016 instruct compareAndSwapN(rRegI res,
8017                           memory mem_ptr,
8018                           rax_RegN oldval, rRegN newval,
8019                           rFlagsReg cr) %{
8020   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
8021   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
8022   effect(KILL cr, KILL oldval);
8023 
8024   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8025             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8026             "sete    $res\n\t"
8027             "movzbl  $res, $res" %}
8028   opcode(0x0F, 0xB1);
8029   ins_encode(lock_prefix,
8030              REX_reg_mem(newval, mem_ptr),
8031              OpcP, OpcS,
8032              reg_mem(newval, mem_ptr),
8033              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8034              REX_reg_breg(res, res), // movzbl
8035              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8036   ins_pipe( pipe_cmpxchg );
8037 %}
8038 
8039 instruct compareAndExchangeB(
8040                          memory mem_ptr,
8041                          rax_RegI oldval, rRegI newval,
8042                          rFlagsReg cr)
8043 %{
8044   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
8045   effect(KILL cr);
8046 
8047   format %{ "cmpxchgb $mem_ptr,$newval\t# "
8048             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8049   opcode(0x0F, 0xB0);
8050   ins_encode(lock_prefix,
8051              REX_breg_mem(newval, mem_ptr),
8052              OpcP, OpcS,
8053              reg_mem(newval, mem_ptr) // lock cmpxchg
8054              );
8055   ins_pipe( pipe_cmpxchg );
8056 %}
8057 
8058 instruct compareAndExchangeS(
8059                          memory mem_ptr,
8060                          rax_RegI oldval, rRegI newval,
8061                          rFlagsReg cr)
8062 %{
8063   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
8064   effect(KILL cr);
8065 
8066   format %{ "cmpxchgw $mem_ptr,$newval\t# "
8067             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8068   opcode(0x0F, 0xB1);
8069   ins_encode(lock_prefix,
8070              SizePrefix,
8071              REX_reg_mem(newval, mem_ptr),
8072              OpcP, OpcS,
8073              reg_mem(newval, mem_ptr) // lock cmpxchg
8074              );
8075   ins_pipe( pipe_cmpxchg );
8076 %}
8077 
8078 instruct compareAndExchangeI(
8079                          memory mem_ptr,
8080                          rax_RegI oldval, rRegI newval,
8081                          rFlagsReg cr)
8082 %{
8083   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
8084   effect(KILL cr);
8085 
8086   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8087             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8088   opcode(0x0F, 0xB1);
8089   ins_encode(lock_prefix,
8090              REX_reg_mem(newval, mem_ptr),
8091              OpcP, OpcS,
8092              reg_mem(newval, mem_ptr) // lock cmpxchg
8093              );
8094   ins_pipe( pipe_cmpxchg );
8095 %}
8096 
8097 instruct compareAndExchangeL(
8098                          memory mem_ptr,
8099                          rax_RegL oldval, rRegL newval,
8100                          rFlagsReg cr)
8101 %{
8102   predicate(VM_Version::supports_cx8());
8103   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
8104   effect(KILL cr);
8105 
8106   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8107             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8108   opcode(0x0F, 0xB1);
8109   ins_encode(lock_prefix,
8110              REX_reg_mem_wide(newval, mem_ptr),
8111              OpcP, OpcS,
8112              reg_mem(newval, mem_ptr)  // lock cmpxchg
8113             );
8114   ins_pipe( pipe_cmpxchg );
8115 %}
8116 
8117 instruct compareAndExchangeN(
8118                           memory mem_ptr,
8119                           rax_RegN oldval, rRegN newval,
8120                           rFlagsReg cr) %{
8121   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
8122   effect(KILL cr);
8123 
8124   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8125             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8126   opcode(0x0F, 0xB1);
8127   ins_encode(lock_prefix,
8128              REX_reg_mem(newval, mem_ptr),
8129              OpcP, OpcS,
8130              reg_mem(newval, mem_ptr)  // lock cmpxchg
8131           );
8132   ins_pipe( pipe_cmpxchg );
8133 %}
8134 
8135 instruct compareAndExchangeP(
8136                          memory mem_ptr,
8137                          rax_RegP oldval, rRegP newval,
8138                          rFlagsReg cr)
8139 %{
8140   predicate(VM_Version::supports_cx8());
8141   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
8142   effect(KILL cr);
8143 
8144   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8145             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8146   opcode(0x0F, 0xB1);
8147   ins_encode(lock_prefix,
8148              REX_reg_mem_wide(newval, mem_ptr),
8149              OpcP, OpcS,
8150              reg_mem(newval, mem_ptr)  // lock cmpxchg
8151           );
8152   ins_pipe( pipe_cmpxchg );
8153 %}
8154 
8155 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8156   predicate(n->as_LoadStore()->result_not_used());
8157   match(Set dummy (GetAndAddB mem add));
8158   effect(KILL cr);
8159   format %{ "ADDB  [$mem],$add" %}
8160   ins_encode %{
8161     __ lock();
8162     __ addb($mem$$Address, $add$$constant);
8163   %}
8164   ins_pipe( pipe_cmpxchg );
8165 %}
8166 
8167 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
8168   match(Set newval (GetAndAddB mem newval));
8169   effect(KILL cr);
8170   format %{ "XADDB  [$mem],$newval" %}
8171   ins_encode %{
8172     __ lock();
8173     __ xaddb($mem$$Address, $newval$$Register);
8174   %}
8175   ins_pipe( pipe_cmpxchg );
8176 %}
8177 
8178 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8179   predicate(n->as_LoadStore()->result_not_used());
8180   match(Set dummy (GetAndAddS mem add));
8181   effect(KILL cr);
8182   format %{ "ADDW  [$mem],$add" %}
8183   ins_encode %{
8184     __ lock();
8185     __ addw($mem$$Address, $add$$constant);
8186   %}
8187   ins_pipe( pipe_cmpxchg );
8188 %}
8189 
8190 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
8191   match(Set newval (GetAndAddS mem newval));
8192   effect(KILL cr);
8193   format %{ "XADDW  [$mem],$newval" %}
8194   ins_encode %{
8195     __ lock();
8196     __ xaddw($mem$$Address, $newval$$Register);
8197   %}
8198   ins_pipe( pipe_cmpxchg );
8199 %}
8200 
8201 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8202   predicate(n->as_LoadStore()->result_not_used());
8203   match(Set dummy (GetAndAddI mem add));
8204   effect(KILL cr);
8205   format %{ "ADDL  [$mem],$add" %}
8206   ins_encode %{
8207     __ lock();
8208     __ addl($mem$$Address, $add$$constant);
8209   %}
8210   ins_pipe( pipe_cmpxchg );
8211 %}
8212 
8213 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
8214   match(Set newval (GetAndAddI mem newval));
8215   effect(KILL cr);
8216   format %{ "XADDL  [$mem],$newval" %}
8217   ins_encode %{
8218     __ lock();
8219     __ xaddl($mem$$Address, $newval$$Register);
8220   %}
8221   ins_pipe( pipe_cmpxchg );
8222 %}
8223 
8224 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
8225   predicate(n->as_LoadStore()->result_not_used());
8226   match(Set dummy (GetAndAddL mem add));
8227   effect(KILL cr);
8228   format %{ "ADDQ  [$mem],$add" %}
8229   ins_encode %{
8230     __ lock();
8231     __ addq($mem$$Address, $add$$constant);
8232   %}
8233   ins_pipe( pipe_cmpxchg );
8234 %}
8235 
8236 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8237   match(Set newval (GetAndAddL mem newval));
8238   effect(KILL cr);
8239   format %{ "XADDQ  [$mem],$newval" %}
8240   ins_encode %{
8241     __ lock();
8242     __ xaddq($mem$$Address, $newval$$Register);
8243   %}
8244   ins_pipe( pipe_cmpxchg );
8245 %}
8246 
8247 instruct xchgB( memory mem, rRegI newval) %{
8248   match(Set newval (GetAndSetB mem newval));
8249   format %{ "XCHGB  $newval,[$mem]" %}
8250   ins_encode %{
8251     __ xchgb($newval$$Register, $mem$$Address);
8252   %}
8253   ins_pipe( pipe_cmpxchg );
8254 %}
8255 
8256 instruct xchgS( memory mem, rRegI newval) %{
8257   match(Set newval (GetAndSetS mem newval));
8258   format %{ "XCHGW  $newval,[$mem]" %}
8259   ins_encode %{
8260     __ xchgw($newval$$Register, $mem$$Address);
8261   %}
8262   ins_pipe( pipe_cmpxchg );
8263 %}
8264 
8265 instruct xchgI( memory mem, rRegI newval) %{
8266   match(Set newval (GetAndSetI mem newval));
8267   format %{ "XCHGL  $newval,[$mem]" %}
8268   ins_encode %{
8269     __ xchgl($newval$$Register, $mem$$Address);
8270   %}
8271   ins_pipe( pipe_cmpxchg );
8272 %}
8273 
8274 instruct xchgL( memory mem, rRegL newval) %{
8275   match(Set newval (GetAndSetL mem newval));
8276   format %{ "XCHGL  $newval,[$mem]" %}
8277   ins_encode %{
8278     __ xchgq($newval$$Register, $mem$$Address);
8279   %}
8280   ins_pipe( pipe_cmpxchg );
8281 %}
8282 
8283 instruct xchgP( memory mem, rRegP newval) %{
8284   match(Set newval (GetAndSetP mem newval));
8285   format %{ "XCHGQ  $newval,[$mem]" %}
8286   ins_encode %{
8287     __ xchgq($newval$$Register, $mem$$Address);
8288   %}
8289   ins_pipe( pipe_cmpxchg );
8290 %}
8291 
8292 instruct xchgN( memory mem, rRegN newval) %{
8293   match(Set newval (GetAndSetN mem newval));
8294   format %{ "XCHGL  $newval,$mem]" %}
8295   ins_encode %{
8296     __ xchgl($newval$$Register, $mem$$Address);
8297   %}
8298   ins_pipe( pipe_cmpxchg );
8299 %}
8300 
8301 //----------Abs Instructions-------------------------------------------
8302 
8303 // Integer Absolute Instructions
8304 instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, rFlagsReg cr)
8305 %{
8306   match(Set dst (AbsI src));
8307   effect(TEMP dst, TEMP tmp, KILL cr);
8308   format %{ "movl $tmp, $src\n\t"
8309             "sarl $tmp, 31\n\t"
8310             "movl $dst, $src\n\t"
8311             "xorl $dst, $tmp\n\t"
8312             "subl $dst, $tmp\n"
8313           %}
8314   ins_encode %{
8315     __ movl($tmp$$Register, $src$$Register);
8316     __ sarl($tmp$$Register, 31);
8317     __ movl($dst$$Register, $src$$Register);
8318     __ xorl($dst$$Register, $tmp$$Register);
8319     __ subl($dst$$Register, $tmp$$Register);
8320   %}
8321 
8322   ins_pipe(ialu_reg_reg);
8323 %}
8324 
8325 // Long Absolute Instructions
8326 instruct absL_rReg(rRegL dst, rRegL src, rRegL tmp, rFlagsReg cr)
8327 %{
8328   match(Set dst (AbsL src));
8329   effect(TEMP dst, TEMP tmp, KILL cr);
8330   format %{ "movq $tmp, $src\n\t"
8331             "sarq $tmp, 63\n\t"
8332             "movq $dst, $src\n\t"
8333             "xorq $dst, $tmp\n\t"
8334             "subq $dst, $tmp\n"
8335           %}
8336   ins_encode %{
8337     __ movq($tmp$$Register, $src$$Register);
8338     __ sarq($tmp$$Register, 63);
8339     __ movq($dst$$Register, $src$$Register);
8340     __ xorq($dst$$Register, $tmp$$Register);
8341     __ subq($dst$$Register, $tmp$$Register);
8342   %}
8343 
8344   ins_pipe(ialu_reg_reg);
8345 %}
8346 
8347 //----------Subtraction Instructions-------------------------------------------
8348 
8349 // Integer Subtraction Instructions
8350 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8351 %{
8352   match(Set dst (SubI dst src));
8353   effect(KILL cr);
8354 
8355   format %{ "subl    $dst, $src\t# int" %}
8356   opcode(0x2B);
8357   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8358   ins_pipe(ialu_reg_reg);
8359 %}
8360 
8361 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8362 %{
8363   match(Set dst (SubI dst src));
8364   effect(KILL cr);
8365 
8366   format %{ "subl    $dst, $src\t# int" %}
8367   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8368   ins_encode(OpcSErm(dst, src), Con8or32(src));
8369   ins_pipe(ialu_reg);
8370 %}
8371 
8372 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8373 %{
8374   match(Set dst (SubI dst (LoadI src)));
8375   effect(KILL cr);
8376 
8377   ins_cost(125);
8378   format %{ "subl    $dst, $src\t# int" %}
8379   opcode(0x2B);
8380   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8381   ins_pipe(ialu_reg_mem);
8382 %}
8383 
8384 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8385 %{
8386   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8387   effect(KILL cr);
8388 
8389   ins_cost(150);
8390   format %{ "subl    $dst, $src\t# int" %}
8391   opcode(0x29); /* Opcode 29 /r */
8392   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8393   ins_pipe(ialu_mem_reg);
8394 %}
8395 
8396 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8397 %{
8398   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8399   effect(KILL cr);
8400 
8401   ins_cost(125); // XXX
8402   format %{ "subl    $dst, $src\t# int" %}
8403   opcode(0x81); /* Opcode 81 /5 id */
8404   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8405   ins_pipe(ialu_mem_imm);
8406 %}
8407 
8408 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8409 %{
8410   match(Set dst (SubL dst src));
8411   effect(KILL cr);
8412 
8413   format %{ "subq    $dst, $src\t# long" %}
8414   opcode(0x2B);
8415   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8416   ins_pipe(ialu_reg_reg);
8417 %}
8418 
8419 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8420 %{
8421   match(Set dst (SubL dst src));
8422   effect(KILL cr);
8423 
8424   format %{ "subq    $dst, $src\t# long" %}
8425   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8426   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8427   ins_pipe(ialu_reg);
8428 %}
8429 
8430 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8431 %{
8432   match(Set dst (SubL dst (LoadL src)));
8433   effect(KILL cr);
8434 
8435   ins_cost(125);
8436   format %{ "subq    $dst, $src\t# long" %}
8437   opcode(0x2B);
8438   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8439   ins_pipe(ialu_reg_mem);
8440 %}
8441 
8442 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8443 %{
8444   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8445   effect(KILL cr);
8446 
8447   ins_cost(150);
8448   format %{ "subq    $dst, $src\t# long" %}
8449   opcode(0x29); /* Opcode 29 /r */
8450   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8451   ins_pipe(ialu_mem_reg);
8452 %}
8453 
8454 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8455 %{
8456   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8457   effect(KILL cr);
8458 
8459   ins_cost(125); // XXX
8460   format %{ "subq    $dst, $src\t# long" %}
8461   opcode(0x81); /* Opcode 81 /5 id */
8462   ins_encode(REX_mem_wide(dst),
8463              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8464   ins_pipe(ialu_mem_imm);
8465 %}
8466 
8467 // Subtract from a pointer
8468 // XXX hmpf???
8469 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8470 %{
8471   match(Set dst (AddP dst (SubI zero src)));
8472   effect(KILL cr);
8473 
8474   format %{ "subq    $dst, $src\t# ptr - int" %}
8475   opcode(0x2B);
8476   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8477   ins_pipe(ialu_reg_reg);
8478 %}
8479 
8480 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8481 %{
8482   match(Set dst (SubI zero dst));
8483   effect(KILL cr);
8484 
8485   format %{ "negl    $dst\t# int" %}
8486   opcode(0xF7, 0x03);  // Opcode F7 /3
8487   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8488   ins_pipe(ialu_reg);
8489 %}
8490 
8491 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8492 %{
8493   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8494   effect(KILL cr);
8495 
8496   format %{ "negl    $dst\t# int" %}
8497   opcode(0xF7, 0x03);  // Opcode F7 /3
8498   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8499   ins_pipe(ialu_reg);
8500 %}
8501 
8502 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8503 %{
8504   match(Set dst (SubL zero dst));
8505   effect(KILL cr);
8506 
8507   format %{ "negq    $dst\t# long" %}
8508   opcode(0xF7, 0x03);  // Opcode F7 /3
8509   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8510   ins_pipe(ialu_reg);
8511 %}
8512 
8513 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8514 %{
8515   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8516   effect(KILL cr);
8517 
8518   format %{ "negq    $dst\t# long" %}
8519   opcode(0xF7, 0x03);  // Opcode F7 /3
8520   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8521   ins_pipe(ialu_reg);
8522 %}
8523 
8524 //----------Multiplication/Division Instructions-------------------------------
8525 // Integer Multiplication Instructions
8526 // Multiply Register
8527 
8528 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8529 %{
8530   match(Set dst (MulI dst src));
8531   effect(KILL cr);
8532 
8533   ins_cost(300);
8534   format %{ "imull   $dst, $src\t# int" %}
8535   opcode(0x0F, 0xAF);
8536   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8537   ins_pipe(ialu_reg_reg_alu0);
8538 %}
8539 
8540 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8541 %{
8542   match(Set dst (MulI src imm));
8543   effect(KILL cr);
8544 
8545   ins_cost(300);
8546   format %{ "imull   $dst, $src, $imm\t# int" %}
8547   opcode(0x69); /* 69 /r id */
8548   ins_encode(REX_reg_reg(dst, src),
8549              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8550   ins_pipe(ialu_reg_reg_alu0);
8551 %}
8552 
8553 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8554 %{
8555   match(Set dst (MulI dst (LoadI src)));
8556   effect(KILL cr);
8557 
8558   ins_cost(350);
8559   format %{ "imull   $dst, $src\t# int" %}
8560   opcode(0x0F, 0xAF);
8561   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8562   ins_pipe(ialu_reg_mem_alu0);
8563 %}
8564 
8565 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8566 %{
8567   match(Set dst (MulI (LoadI src) imm));
8568   effect(KILL cr);
8569 
8570   ins_cost(300);
8571   format %{ "imull   $dst, $src, $imm\t# int" %}
8572   opcode(0x69); /* 69 /r id */
8573   ins_encode(REX_reg_mem(dst, src),
8574              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8575   ins_pipe(ialu_reg_mem_alu0);
8576 %}
8577 
8578 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8579 %{
8580   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8581   effect(KILL cr, KILL src2);
8582 
8583   expand %{ mulI_rReg(dst, src1, cr);
8584            mulI_rReg(src2, src3, cr);
8585            addI_rReg(dst, src2, cr); %}
8586 %}
8587 
8588 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8589 %{
8590   match(Set dst (MulL dst src));
8591   effect(KILL cr);
8592 
8593   ins_cost(300);
8594   format %{ "imulq   $dst, $src\t# long" %}
8595   opcode(0x0F, 0xAF);
8596   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8597   ins_pipe(ialu_reg_reg_alu0);
8598 %}
8599 
8600 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8601 %{
8602   match(Set dst (MulL src imm));
8603   effect(KILL cr);
8604 
8605   ins_cost(300);
8606   format %{ "imulq   $dst, $src, $imm\t# long" %}
8607   opcode(0x69); /* 69 /r id */
8608   ins_encode(REX_reg_reg_wide(dst, src),
8609              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8610   ins_pipe(ialu_reg_reg_alu0);
8611 %}
8612 
8613 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8614 %{
8615   match(Set dst (MulL dst (LoadL src)));
8616   effect(KILL cr);
8617 
8618   ins_cost(350);
8619   format %{ "imulq   $dst, $src\t# long" %}
8620   opcode(0x0F, 0xAF);
8621   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8622   ins_pipe(ialu_reg_mem_alu0);
8623 %}
8624 
8625 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8626 %{
8627   match(Set dst (MulL (LoadL src) imm));
8628   effect(KILL cr);
8629 
8630   ins_cost(300);
8631   format %{ "imulq   $dst, $src, $imm\t# long" %}
8632   opcode(0x69); /* 69 /r id */
8633   ins_encode(REX_reg_mem_wide(dst, src),
8634              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8635   ins_pipe(ialu_reg_mem_alu0);
8636 %}
8637 
8638 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8639 %{
8640   match(Set dst (MulHiL src rax));
8641   effect(USE_KILL rax, KILL cr);
8642 
8643   ins_cost(300);
8644   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8645   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8646   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8647   ins_pipe(ialu_reg_reg_alu0);
8648 %}
8649 
8650 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8651                    rFlagsReg cr)
8652 %{
8653   match(Set rax (DivI rax div));
8654   effect(KILL rdx, KILL cr);
8655 
8656   ins_cost(30*100+10*100); // XXX
8657   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8658             "jne,s   normal\n\t"
8659             "xorl    rdx, rdx\n\t"
8660             "cmpl    $div, -1\n\t"
8661             "je,s    done\n"
8662     "normal: cdql\n\t"
8663             "idivl   $div\n"
8664     "done:"        %}
8665   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8666   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8667   ins_pipe(ialu_reg_reg_alu0);
8668 %}
8669 
8670 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8671                    rFlagsReg cr)
8672 %{
8673   match(Set rax (DivL rax div));
8674   effect(KILL rdx, KILL cr);
8675 
8676   ins_cost(30*100+10*100); // XXX
8677   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8678             "cmpq    rax, rdx\n\t"
8679             "jne,s   normal\n\t"
8680             "xorl    rdx, rdx\n\t"
8681             "cmpq    $div, -1\n\t"
8682             "je,s    done\n"
8683     "normal: cdqq\n\t"
8684             "idivq   $div\n"
8685     "done:"        %}
8686   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8687   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8688   ins_pipe(ialu_reg_reg_alu0);
8689 %}
8690 
8691 // Integer DIVMOD with Register, both quotient and mod results
8692 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8693                              rFlagsReg cr)
8694 %{
8695   match(DivModI rax div);
8696   effect(KILL cr);
8697 
8698   ins_cost(30*100+10*100); // XXX
8699   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8700             "jne,s   normal\n\t"
8701             "xorl    rdx, rdx\n\t"
8702             "cmpl    $div, -1\n\t"
8703             "je,s    done\n"
8704     "normal: cdql\n\t"
8705             "idivl   $div\n"
8706     "done:"        %}
8707   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8708   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8709   ins_pipe(pipe_slow);
8710 %}
8711 
8712 // Long DIVMOD with Register, both quotient and mod results
8713 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8714                              rFlagsReg cr)
8715 %{
8716   match(DivModL rax div);
8717   effect(KILL cr);
8718 
8719   ins_cost(30*100+10*100); // XXX
8720   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8721             "cmpq    rax, rdx\n\t"
8722             "jne,s   normal\n\t"
8723             "xorl    rdx, rdx\n\t"
8724             "cmpq    $div, -1\n\t"
8725             "je,s    done\n"
8726     "normal: cdqq\n\t"
8727             "idivq   $div\n"
8728     "done:"        %}
8729   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8730   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8731   ins_pipe(pipe_slow);
8732 %}
8733 
8734 //----------- DivL-By-Constant-Expansions--------------------------------------
8735 // DivI cases are handled by the compiler
8736 
8737 // Magic constant, reciprocal of 10
8738 instruct loadConL_0x6666666666666667(rRegL dst)
8739 %{
8740   effect(DEF dst);
8741 
8742   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8743   ins_encode(load_immL(dst, 0x6666666666666667));
8744   ins_pipe(ialu_reg);
8745 %}
8746 
8747 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8748 %{
8749   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8750 
8751   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8752   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8753   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8754   ins_pipe(ialu_reg_reg_alu0);
8755 %}
8756 
8757 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8758 %{
8759   effect(USE_DEF dst, KILL cr);
8760 
8761   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8762   opcode(0xC1, 0x7); /* C1 /7 ib */
8763   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8764   ins_pipe(ialu_reg);
8765 %}
8766 
8767 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8768 %{
8769   effect(USE_DEF dst, KILL cr);
8770 
8771   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8772   opcode(0xC1, 0x7); /* C1 /7 ib */
8773   ins_encode(reg_opc_imm_wide(dst, 0x2));
8774   ins_pipe(ialu_reg);
8775 %}
8776 
8777 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8778 %{
8779   match(Set dst (DivL src div));
8780 
8781   ins_cost((5+8)*100);
8782   expand %{
8783     rax_RegL rax;                     // Killed temp
8784     rFlagsReg cr;                     // Killed
8785     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8786     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8787     sarL_rReg_63(src, cr);            // sarq  src, 63
8788     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8789     subL_rReg(dst, src, cr);          // subl  rdx, src
8790   %}
8791 %}
8792 
8793 //-----------------------------------------------------------------------------
8794 
8795 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8796                    rFlagsReg cr)
8797 %{
8798   match(Set rdx (ModI rax div));
8799   effect(KILL rax, KILL cr);
8800 
8801   ins_cost(300); // XXX
8802   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8803             "jne,s   normal\n\t"
8804             "xorl    rdx, rdx\n\t"
8805             "cmpl    $div, -1\n\t"
8806             "je,s    done\n"
8807     "normal: cdql\n\t"
8808             "idivl   $div\n"
8809     "done:"        %}
8810   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8811   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8812   ins_pipe(ialu_reg_reg_alu0);
8813 %}
8814 
8815 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8816                    rFlagsReg cr)
8817 %{
8818   match(Set rdx (ModL rax div));
8819   effect(KILL rax, KILL cr);
8820 
8821   ins_cost(300); // XXX
8822   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8823             "cmpq    rax, rdx\n\t"
8824             "jne,s   normal\n\t"
8825             "xorl    rdx, rdx\n\t"
8826             "cmpq    $div, -1\n\t"
8827             "je,s    done\n"
8828     "normal: cdqq\n\t"
8829             "idivq   $div\n"
8830     "done:"        %}
8831   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8832   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8833   ins_pipe(ialu_reg_reg_alu0);
8834 %}
8835 
8836 // Integer Shift Instructions
8837 // Shift Left by one
8838 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8839 %{
8840   match(Set dst (LShiftI dst shift));
8841   effect(KILL cr);
8842 
8843   format %{ "sall    $dst, $shift" %}
8844   opcode(0xD1, 0x4); /* D1 /4 */
8845   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8846   ins_pipe(ialu_reg);
8847 %}
8848 
8849 // Shift Left by one
8850 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8851 %{
8852   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8853   effect(KILL cr);
8854 
8855   format %{ "sall    $dst, $shift\t" %}
8856   opcode(0xD1, 0x4); /* D1 /4 */
8857   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8858   ins_pipe(ialu_mem_imm);
8859 %}
8860 
8861 // Shift Left by 8-bit immediate
8862 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8863 %{
8864   match(Set dst (LShiftI dst shift));
8865   effect(KILL cr);
8866 
8867   format %{ "sall    $dst, $shift" %}
8868   opcode(0xC1, 0x4); /* C1 /4 ib */
8869   ins_encode(reg_opc_imm(dst, shift));
8870   ins_pipe(ialu_reg);
8871 %}
8872 
8873 // Shift Left by 8-bit immediate
8874 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8875 %{
8876   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8877   effect(KILL cr);
8878 
8879   format %{ "sall    $dst, $shift" %}
8880   opcode(0xC1, 0x4); /* C1 /4 ib */
8881   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8882   ins_pipe(ialu_mem_imm);
8883 %}
8884 
8885 // Shift Left by variable
8886 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8887 %{
8888   match(Set dst (LShiftI dst shift));
8889   effect(KILL cr);
8890 
8891   format %{ "sall    $dst, $shift" %}
8892   opcode(0xD3, 0x4); /* D3 /4 */
8893   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8894   ins_pipe(ialu_reg_reg);
8895 %}
8896 
8897 // Shift Left by variable
8898 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8899 %{
8900   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8901   effect(KILL cr);
8902 
8903   format %{ "sall    $dst, $shift" %}
8904   opcode(0xD3, 0x4); /* D3 /4 */
8905   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8906   ins_pipe(ialu_mem_reg);
8907 %}
8908 
8909 // Arithmetic shift right by one
8910 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8911 %{
8912   match(Set dst (RShiftI dst shift));
8913   effect(KILL cr);
8914 
8915   format %{ "sarl    $dst, $shift" %}
8916   opcode(0xD1, 0x7); /* D1 /7 */
8917   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8918   ins_pipe(ialu_reg);
8919 %}
8920 
8921 // Arithmetic shift right by one
8922 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8923 %{
8924   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8925   effect(KILL cr);
8926 
8927   format %{ "sarl    $dst, $shift" %}
8928   opcode(0xD1, 0x7); /* D1 /7 */
8929   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8930   ins_pipe(ialu_mem_imm);
8931 %}
8932 
8933 // Arithmetic Shift Right by 8-bit immediate
8934 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8935 %{
8936   match(Set dst (RShiftI dst shift));
8937   effect(KILL cr);
8938 
8939   format %{ "sarl    $dst, $shift" %}
8940   opcode(0xC1, 0x7); /* C1 /7 ib */
8941   ins_encode(reg_opc_imm(dst, shift));
8942   ins_pipe(ialu_mem_imm);
8943 %}
8944 
8945 // Arithmetic Shift Right by 8-bit immediate
8946 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8947 %{
8948   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8949   effect(KILL cr);
8950 
8951   format %{ "sarl    $dst, $shift" %}
8952   opcode(0xC1, 0x7); /* C1 /7 ib */
8953   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8954   ins_pipe(ialu_mem_imm);
8955 %}
8956 
8957 // Arithmetic Shift Right by variable
8958 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8959 %{
8960   match(Set dst (RShiftI dst shift));
8961   effect(KILL cr);
8962 
8963   format %{ "sarl    $dst, $shift" %}
8964   opcode(0xD3, 0x7); /* D3 /7 */
8965   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8966   ins_pipe(ialu_reg_reg);
8967 %}
8968 
8969 // Arithmetic Shift Right by variable
8970 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8971 %{
8972   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8973   effect(KILL cr);
8974 
8975   format %{ "sarl    $dst, $shift" %}
8976   opcode(0xD3, 0x7); /* D3 /7 */
8977   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8978   ins_pipe(ialu_mem_reg);
8979 %}
8980 
8981 // Logical shift right by one
8982 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8983 %{
8984   match(Set dst (URShiftI dst shift));
8985   effect(KILL cr);
8986 
8987   format %{ "shrl    $dst, $shift" %}
8988   opcode(0xD1, 0x5); /* D1 /5 */
8989   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8990   ins_pipe(ialu_reg);
8991 %}
8992 
8993 // Logical shift right by one
8994 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8995 %{
8996   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8997   effect(KILL cr);
8998 
8999   format %{ "shrl    $dst, $shift" %}
9000   opcode(0xD1, 0x5); /* D1 /5 */
9001   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
9002   ins_pipe(ialu_mem_imm);
9003 %}
9004 
9005 // Logical Shift Right by 8-bit immediate
9006 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
9007 %{
9008   match(Set dst (URShiftI dst shift));
9009   effect(KILL cr);
9010 
9011   format %{ "shrl    $dst, $shift" %}
9012   opcode(0xC1, 0x5); /* C1 /5 ib */
9013   ins_encode(reg_opc_imm(dst, shift));
9014   ins_pipe(ialu_reg);
9015 %}
9016 
9017 // Logical Shift Right by 8-bit immediate
9018 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9019 %{
9020   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
9021   effect(KILL cr);
9022 
9023   format %{ "shrl    $dst, $shift" %}
9024   opcode(0xC1, 0x5); /* C1 /5 ib */
9025   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
9026   ins_pipe(ialu_mem_imm);
9027 %}
9028 
9029 // Logical Shift Right by variable
9030 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
9031 %{
9032   match(Set dst (URShiftI dst shift));
9033   effect(KILL cr);
9034 
9035   format %{ "shrl    $dst, $shift" %}
9036   opcode(0xD3, 0x5); /* D3 /5 */
9037   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9038   ins_pipe(ialu_reg_reg);
9039 %}
9040 
9041 // Logical Shift Right by variable
9042 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9043 %{
9044   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
9045   effect(KILL cr);
9046 
9047   format %{ "shrl    $dst, $shift" %}
9048   opcode(0xD3, 0x5); /* D3 /5 */
9049   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
9050   ins_pipe(ialu_mem_reg);
9051 %}
9052 
9053 // Long Shift Instructions
9054 // Shift Left by one
9055 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9056 %{
9057   match(Set dst (LShiftL dst shift));
9058   effect(KILL cr);
9059 
9060   format %{ "salq    $dst, $shift" %}
9061   opcode(0xD1, 0x4); /* D1 /4 */
9062   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9063   ins_pipe(ialu_reg);
9064 %}
9065 
9066 // Shift Left by one
9067 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9068 %{
9069   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9070   effect(KILL cr);
9071 
9072   format %{ "salq    $dst, $shift" %}
9073   opcode(0xD1, 0x4); /* D1 /4 */
9074   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9075   ins_pipe(ialu_mem_imm);
9076 %}
9077 
9078 // Shift Left by 8-bit immediate
9079 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9080 %{
9081   match(Set dst (LShiftL dst shift));
9082   effect(KILL cr);
9083 
9084   format %{ "salq    $dst, $shift" %}
9085   opcode(0xC1, 0x4); /* C1 /4 ib */
9086   ins_encode(reg_opc_imm_wide(dst, shift));
9087   ins_pipe(ialu_reg);
9088 %}
9089 
9090 // Shift Left by 8-bit immediate
9091 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9092 %{
9093   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9094   effect(KILL cr);
9095 
9096   format %{ "salq    $dst, $shift" %}
9097   opcode(0xC1, 0x4); /* C1 /4 ib */
9098   ins_encode(REX_mem_wide(dst), OpcP,
9099              RM_opc_mem(secondary, dst), Con8or32(shift));
9100   ins_pipe(ialu_mem_imm);
9101 %}
9102 
9103 // Shift Left by variable
9104 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9105 %{
9106   match(Set dst (LShiftL dst shift));
9107   effect(KILL cr);
9108 
9109   format %{ "salq    $dst, $shift" %}
9110   opcode(0xD3, 0x4); /* D3 /4 */
9111   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9112   ins_pipe(ialu_reg_reg);
9113 %}
9114 
9115 // Shift Left by variable
9116 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9117 %{
9118   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9119   effect(KILL cr);
9120 
9121   format %{ "salq    $dst, $shift" %}
9122   opcode(0xD3, 0x4); /* D3 /4 */
9123   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9124   ins_pipe(ialu_mem_reg);
9125 %}
9126 
9127 // Arithmetic shift right by one
9128 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9129 %{
9130   match(Set dst (RShiftL dst shift));
9131   effect(KILL cr);
9132 
9133   format %{ "sarq    $dst, $shift" %}
9134   opcode(0xD1, 0x7); /* D1 /7 */
9135   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9136   ins_pipe(ialu_reg);
9137 %}
9138 
9139 // Arithmetic shift right by one
9140 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9141 %{
9142   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9143   effect(KILL cr);
9144 
9145   format %{ "sarq    $dst, $shift" %}
9146   opcode(0xD1, 0x7); /* D1 /7 */
9147   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9148   ins_pipe(ialu_mem_imm);
9149 %}
9150 
9151 // Arithmetic Shift Right by 8-bit immediate
9152 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9153 %{
9154   match(Set dst (RShiftL dst shift));
9155   effect(KILL cr);
9156 
9157   format %{ "sarq    $dst, $shift" %}
9158   opcode(0xC1, 0x7); /* C1 /7 ib */
9159   ins_encode(reg_opc_imm_wide(dst, shift));
9160   ins_pipe(ialu_mem_imm);
9161 %}
9162 
9163 // Arithmetic Shift Right by 8-bit immediate
9164 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9165 %{
9166   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9167   effect(KILL cr);
9168 
9169   format %{ "sarq    $dst, $shift" %}
9170   opcode(0xC1, 0x7); /* C1 /7 ib */
9171   ins_encode(REX_mem_wide(dst), OpcP,
9172              RM_opc_mem(secondary, dst), Con8or32(shift));
9173   ins_pipe(ialu_mem_imm);
9174 %}
9175 
9176 // Arithmetic Shift Right by variable
9177 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9178 %{
9179   match(Set dst (RShiftL dst shift));
9180   effect(KILL cr);
9181 
9182   format %{ "sarq    $dst, $shift" %}
9183   opcode(0xD3, 0x7); /* D3 /7 */
9184   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9185   ins_pipe(ialu_reg_reg);
9186 %}
9187 
9188 // Arithmetic Shift Right by variable
9189 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9190 %{
9191   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9192   effect(KILL cr);
9193 
9194   format %{ "sarq    $dst, $shift" %}
9195   opcode(0xD3, 0x7); /* D3 /7 */
9196   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9197   ins_pipe(ialu_mem_reg);
9198 %}
9199 
9200 // Logical shift right by one
9201 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9202 %{
9203   match(Set dst (URShiftL dst shift));
9204   effect(KILL cr);
9205 
9206   format %{ "shrq    $dst, $shift" %}
9207   opcode(0xD1, 0x5); /* D1 /5 */
9208   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9209   ins_pipe(ialu_reg);
9210 %}
9211 
9212 // Logical shift right by one
9213 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9214 %{
9215   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9216   effect(KILL cr);
9217 
9218   format %{ "shrq    $dst, $shift" %}
9219   opcode(0xD1, 0x5); /* D1 /5 */
9220   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9221   ins_pipe(ialu_mem_imm);
9222 %}
9223 
9224 // Logical Shift Right by 8-bit immediate
9225 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9226 %{
9227   match(Set dst (URShiftL dst shift));
9228   effect(KILL cr);
9229 
9230   format %{ "shrq    $dst, $shift" %}
9231   opcode(0xC1, 0x5); /* C1 /5 ib */
9232   ins_encode(reg_opc_imm_wide(dst, shift));
9233   ins_pipe(ialu_reg);
9234 %}
9235 
9236 
9237 // Logical Shift Right by 8-bit immediate
9238 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9239 %{
9240   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9241   effect(KILL cr);
9242 
9243   format %{ "shrq    $dst, $shift" %}
9244   opcode(0xC1, 0x5); /* C1 /5 ib */
9245   ins_encode(REX_mem_wide(dst), OpcP,
9246              RM_opc_mem(secondary, dst), Con8or32(shift));
9247   ins_pipe(ialu_mem_imm);
9248 %}
9249 
9250 // Logical Shift Right by variable
9251 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9252 %{
9253   match(Set dst (URShiftL dst shift));
9254   effect(KILL cr);
9255 
9256   format %{ "shrq    $dst, $shift" %}
9257   opcode(0xD3, 0x5); /* D3 /5 */
9258   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9259   ins_pipe(ialu_reg_reg);
9260 %}
9261 
9262 // Logical Shift Right by variable
9263 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9264 %{
9265   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9266   effect(KILL cr);
9267 
9268   format %{ "shrq    $dst, $shift" %}
9269   opcode(0xD3, 0x5); /* D3 /5 */
9270   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9271   ins_pipe(ialu_mem_reg);
9272 %}
9273 
9274 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9275 // This idiom is used by the compiler for the i2b bytecode.
9276 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9277 %{
9278   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9279 
9280   format %{ "movsbl  $dst, $src\t# i2b" %}
9281   opcode(0x0F, 0xBE);
9282   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9283   ins_pipe(ialu_reg_reg);
9284 %}
9285 
9286 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9287 // This idiom is used by the compiler the i2s bytecode.
9288 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9289 %{
9290   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9291 
9292   format %{ "movswl  $dst, $src\t# i2s" %}
9293   opcode(0x0F, 0xBF);
9294   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9295   ins_pipe(ialu_reg_reg);
9296 %}
9297 
9298 // ROL/ROR instructions
9299 
9300 // ROL expand
9301 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9302   effect(KILL cr, USE_DEF dst);
9303 
9304   format %{ "roll    $dst" %}
9305   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9306   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9307   ins_pipe(ialu_reg);
9308 %}
9309 
9310 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9311   effect(USE_DEF dst, USE shift, KILL cr);
9312 
9313   format %{ "roll    $dst, $shift" %}
9314   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9315   ins_encode( reg_opc_imm(dst, shift) );
9316   ins_pipe(ialu_reg);
9317 %}
9318 
9319 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9320 %{
9321   effect(USE_DEF dst, USE shift, KILL cr);
9322 
9323   format %{ "roll    $dst, $shift" %}
9324   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9325   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9326   ins_pipe(ialu_reg_reg);
9327 %}
9328 // end of ROL expand
9329 
9330 // Rotate Left by one
9331 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9332 %{
9333   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9334 
9335   expand %{
9336     rolI_rReg_imm1(dst, cr);
9337   %}
9338 %}
9339 
9340 // Rotate Left by 8-bit immediate
9341 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9342 %{
9343   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9344   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9345 
9346   expand %{
9347     rolI_rReg_imm8(dst, lshift, cr);
9348   %}
9349 %}
9350 
9351 // Rotate Left by variable
9352 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9353 %{
9354   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9355 
9356   expand %{
9357     rolI_rReg_CL(dst, shift, cr);
9358   %}
9359 %}
9360 
9361 // Rotate Left by variable
9362 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9363 %{
9364   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9365 
9366   expand %{
9367     rolI_rReg_CL(dst, shift, cr);
9368   %}
9369 %}
9370 
9371 // ROR expand
9372 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9373 %{
9374   effect(USE_DEF dst, KILL cr);
9375 
9376   format %{ "rorl    $dst" %}
9377   opcode(0xD1, 0x1); /* D1 /1 */
9378   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9379   ins_pipe(ialu_reg);
9380 %}
9381 
9382 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9383 %{
9384   effect(USE_DEF dst, USE shift, KILL cr);
9385 
9386   format %{ "rorl    $dst, $shift" %}
9387   opcode(0xC1, 0x1); /* C1 /1 ib */
9388   ins_encode(reg_opc_imm(dst, shift));
9389   ins_pipe(ialu_reg);
9390 %}
9391 
9392 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9393 %{
9394   effect(USE_DEF dst, USE shift, KILL cr);
9395 
9396   format %{ "rorl    $dst, $shift" %}
9397   opcode(0xD3, 0x1); /* D3 /1 */
9398   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9399   ins_pipe(ialu_reg_reg);
9400 %}
9401 // end of ROR expand
9402 
9403 // Rotate Right by one
9404 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9405 %{
9406   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9407 
9408   expand %{
9409     rorI_rReg_imm1(dst, cr);
9410   %}
9411 %}
9412 
9413 // Rotate Right by 8-bit immediate
9414 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9415 %{
9416   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9417   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9418 
9419   expand %{
9420     rorI_rReg_imm8(dst, rshift, cr);
9421   %}
9422 %}
9423 
9424 // Rotate Right by variable
9425 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9426 %{
9427   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9428 
9429   expand %{
9430     rorI_rReg_CL(dst, shift, cr);
9431   %}
9432 %}
9433 
9434 // Rotate Right by variable
9435 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9436 %{
9437   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9438 
9439   expand %{
9440     rorI_rReg_CL(dst, shift, cr);
9441   %}
9442 %}
9443 
9444 // for long rotate
9445 // ROL expand
9446 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9447   effect(USE_DEF dst, KILL cr);
9448 
9449   format %{ "rolq    $dst" %}
9450   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9451   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9452   ins_pipe(ialu_reg);
9453 %}
9454 
9455 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9456   effect(USE_DEF dst, USE shift, KILL cr);
9457 
9458   format %{ "rolq    $dst, $shift" %}
9459   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9460   ins_encode( reg_opc_imm_wide(dst, shift) );
9461   ins_pipe(ialu_reg);
9462 %}
9463 
9464 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9465 %{
9466   effect(USE_DEF dst, USE shift, KILL cr);
9467 
9468   format %{ "rolq    $dst, $shift" %}
9469   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9470   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9471   ins_pipe(ialu_reg_reg);
9472 %}
9473 // end of ROL expand
9474 
9475 // Rotate Left by one
9476 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9477 %{
9478   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9479 
9480   expand %{
9481     rolL_rReg_imm1(dst, cr);
9482   %}
9483 %}
9484 
9485 // Rotate Left by 8-bit immediate
9486 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9487 %{
9488   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9489   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9490 
9491   expand %{
9492     rolL_rReg_imm8(dst, lshift, cr);
9493   %}
9494 %}
9495 
9496 // Rotate Left by variable
9497 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9498 %{
9499   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9500 
9501   expand %{
9502     rolL_rReg_CL(dst, shift, cr);
9503   %}
9504 %}
9505 
9506 // Rotate Left by variable
9507 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9508 %{
9509   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9510 
9511   expand %{
9512     rolL_rReg_CL(dst, shift, cr);
9513   %}
9514 %}
9515 
9516 // ROR expand
9517 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9518 %{
9519   effect(USE_DEF dst, KILL cr);
9520 
9521   format %{ "rorq    $dst" %}
9522   opcode(0xD1, 0x1); /* D1 /1 */
9523   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9524   ins_pipe(ialu_reg);
9525 %}
9526 
9527 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9528 %{
9529   effect(USE_DEF dst, USE shift, KILL cr);
9530 
9531   format %{ "rorq    $dst, $shift" %}
9532   opcode(0xC1, 0x1); /* C1 /1 ib */
9533   ins_encode(reg_opc_imm_wide(dst, shift));
9534   ins_pipe(ialu_reg);
9535 %}
9536 
9537 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9538 %{
9539   effect(USE_DEF dst, USE shift, KILL cr);
9540 
9541   format %{ "rorq    $dst, $shift" %}
9542   opcode(0xD3, 0x1); /* D3 /1 */
9543   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9544   ins_pipe(ialu_reg_reg);
9545 %}
9546 // end of ROR expand
9547 
9548 // Rotate Right by one
9549 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9550 %{
9551   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9552 
9553   expand %{
9554     rorL_rReg_imm1(dst, cr);
9555   %}
9556 %}
9557 
9558 // Rotate Right by 8-bit immediate
9559 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9560 %{
9561   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9562   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9563 
9564   expand %{
9565     rorL_rReg_imm8(dst, rshift, cr);
9566   %}
9567 %}
9568 
9569 // Rotate Right by variable
9570 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9571 %{
9572   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9573 
9574   expand %{
9575     rorL_rReg_CL(dst, shift, cr);
9576   %}
9577 %}
9578 
9579 // Rotate Right by variable
9580 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9581 %{
9582   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9583 
9584   expand %{
9585     rorL_rReg_CL(dst, shift, cr);
9586   %}
9587 %}
9588 
9589 // Logical Instructions
9590 
9591 // Integer Logical Instructions
9592 
9593 // And Instructions
9594 // And Register with Register
9595 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9596 %{
9597   match(Set dst (AndI dst src));
9598   effect(KILL cr);
9599 
9600   format %{ "andl    $dst, $src\t# int" %}
9601   opcode(0x23);
9602   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9603   ins_pipe(ialu_reg_reg);
9604 %}
9605 
9606 // And Register with Immediate 255
9607 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9608 %{
9609   match(Set dst (AndI dst src));
9610 
9611   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9612   opcode(0x0F, 0xB6);
9613   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9614   ins_pipe(ialu_reg);
9615 %}
9616 
9617 // And Register with Immediate 255 and promote to long
9618 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9619 %{
9620   match(Set dst (ConvI2L (AndI src mask)));
9621 
9622   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9623   opcode(0x0F, 0xB6);
9624   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9625   ins_pipe(ialu_reg);
9626 %}
9627 
9628 // And Register with Immediate 65535
9629 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9630 %{
9631   match(Set dst (AndI dst src));
9632 
9633   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9634   opcode(0x0F, 0xB7);
9635   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9636   ins_pipe(ialu_reg);
9637 %}
9638 
9639 // And Register with Immediate 65535 and promote to long
9640 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9641 %{
9642   match(Set dst (ConvI2L (AndI src mask)));
9643 
9644   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9645   opcode(0x0F, 0xB7);
9646   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9647   ins_pipe(ialu_reg);
9648 %}
9649 
9650 // And Register with Immediate
9651 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9652 %{
9653   match(Set dst (AndI dst src));
9654   effect(KILL cr);
9655 
9656   format %{ "andl    $dst, $src\t# int" %}
9657   opcode(0x81, 0x04); /* Opcode 81 /4 */
9658   ins_encode(OpcSErm(dst, src), Con8or32(src));
9659   ins_pipe(ialu_reg);
9660 %}
9661 
9662 // And Register with Memory
9663 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9664 %{
9665   match(Set dst (AndI dst (LoadI src)));
9666   effect(KILL cr);
9667 
9668   ins_cost(125);
9669   format %{ "andl    $dst, $src\t# int" %}
9670   opcode(0x23);
9671   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9672   ins_pipe(ialu_reg_mem);
9673 %}
9674 
9675 // And Memory with Register
9676 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9677 %{
9678   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9679   effect(KILL cr);
9680 
9681   ins_cost(150);
9682   format %{ "andb    $dst, $src\t# byte" %}
9683   opcode(0x20);
9684   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9685   ins_pipe(ialu_mem_reg);
9686 %}
9687 
9688 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9689 %{
9690   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9691   effect(KILL cr);
9692 
9693   ins_cost(150);
9694   format %{ "andl    $dst, $src\t# int" %}
9695   opcode(0x21); /* Opcode 21 /r */
9696   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9697   ins_pipe(ialu_mem_reg);
9698 %}
9699 
9700 // And Memory with Immediate
9701 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9702 %{
9703   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9704   effect(KILL cr);
9705 
9706   ins_cost(125);
9707   format %{ "andl    $dst, $src\t# int" %}
9708   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9709   ins_encode(REX_mem(dst), OpcSE(src),
9710              RM_opc_mem(secondary, dst), Con8or32(src));
9711   ins_pipe(ialu_mem_imm);
9712 %}
9713 
9714 // BMI1 instructions
9715 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9716   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9717   predicate(UseBMI1Instructions);
9718   effect(KILL cr);
9719 
9720   ins_cost(125);
9721   format %{ "andnl  $dst, $src1, $src2" %}
9722 
9723   ins_encode %{
9724     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9725   %}
9726   ins_pipe(ialu_reg_mem);
9727 %}
9728 
9729 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9730   match(Set dst (AndI (XorI src1 minus_1) src2));
9731   predicate(UseBMI1Instructions);
9732   effect(KILL cr);
9733 
9734   format %{ "andnl  $dst, $src1, $src2" %}
9735 
9736   ins_encode %{
9737     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9738   %}
9739   ins_pipe(ialu_reg);
9740 %}
9741 
9742 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9743   match(Set dst (AndI (SubI imm_zero src) src));
9744   predicate(UseBMI1Instructions);
9745   effect(KILL cr);
9746 
9747   format %{ "blsil  $dst, $src" %}
9748 
9749   ins_encode %{
9750     __ blsil($dst$$Register, $src$$Register);
9751   %}
9752   ins_pipe(ialu_reg);
9753 %}
9754 
9755 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9756   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9757   predicate(UseBMI1Instructions);
9758   effect(KILL cr);
9759 
9760   ins_cost(125);
9761   format %{ "blsil  $dst, $src" %}
9762 
9763   ins_encode %{
9764     __ blsil($dst$$Register, $src$$Address);
9765   %}
9766   ins_pipe(ialu_reg_mem);
9767 %}
9768 
9769 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9770 %{
9771   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9772   predicate(UseBMI1Instructions);
9773   effect(KILL cr);
9774 
9775   ins_cost(125);
9776   format %{ "blsmskl $dst, $src" %}
9777 
9778   ins_encode %{
9779     __ blsmskl($dst$$Register, $src$$Address);
9780   %}
9781   ins_pipe(ialu_reg_mem);
9782 %}
9783 
9784 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9785 %{
9786   match(Set dst (XorI (AddI src minus_1) src));
9787   predicate(UseBMI1Instructions);
9788   effect(KILL cr);
9789 
9790   format %{ "blsmskl $dst, $src" %}
9791 
9792   ins_encode %{
9793     __ blsmskl($dst$$Register, $src$$Register);
9794   %}
9795 
9796   ins_pipe(ialu_reg);
9797 %}
9798 
9799 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9800 %{
9801   match(Set dst (AndI (AddI src minus_1) src) );
9802   predicate(UseBMI1Instructions);
9803   effect(KILL cr);
9804 
9805   format %{ "blsrl  $dst, $src" %}
9806 
9807   ins_encode %{
9808     __ blsrl($dst$$Register, $src$$Register);
9809   %}
9810 
9811   ins_pipe(ialu_reg_mem);
9812 %}
9813 
9814 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9815 %{
9816   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9817   predicate(UseBMI1Instructions);
9818   effect(KILL cr);
9819 
9820   ins_cost(125);
9821   format %{ "blsrl  $dst, $src" %}
9822 
9823   ins_encode %{
9824     __ blsrl($dst$$Register, $src$$Address);
9825   %}
9826 
9827   ins_pipe(ialu_reg);
9828 %}
9829 
9830 // Or Instructions
9831 // Or Register with Register
9832 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9833 %{
9834   match(Set dst (OrI dst src));
9835   effect(KILL cr);
9836 
9837   format %{ "orl     $dst, $src\t# int" %}
9838   opcode(0x0B);
9839   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9840   ins_pipe(ialu_reg_reg);
9841 %}
9842 
9843 // Or Register with Immediate
9844 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9845 %{
9846   match(Set dst (OrI dst src));
9847   effect(KILL cr);
9848 
9849   format %{ "orl     $dst, $src\t# int" %}
9850   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9851   ins_encode(OpcSErm(dst, src), Con8or32(src));
9852   ins_pipe(ialu_reg);
9853 %}
9854 
9855 // Or Register with Memory
9856 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9857 %{
9858   match(Set dst (OrI dst (LoadI src)));
9859   effect(KILL cr);
9860 
9861   ins_cost(125);
9862   format %{ "orl     $dst, $src\t# int" %}
9863   opcode(0x0B);
9864   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9865   ins_pipe(ialu_reg_mem);
9866 %}
9867 
9868 // Or Memory with Register
9869 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9870 %{
9871   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9872   effect(KILL cr);
9873 
9874   ins_cost(150);
9875   format %{ "orb    $dst, $src\t# byte" %}
9876   opcode(0x08);
9877   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9878   ins_pipe(ialu_mem_reg);
9879 %}
9880 
9881 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9882 %{
9883   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9884   effect(KILL cr);
9885 
9886   ins_cost(150);
9887   format %{ "orl     $dst, $src\t# int" %}
9888   opcode(0x09); /* Opcode 09 /r */
9889   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9890   ins_pipe(ialu_mem_reg);
9891 %}
9892 
9893 // Or Memory with Immediate
9894 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9895 %{
9896   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9897   effect(KILL cr);
9898 
9899   ins_cost(125);
9900   format %{ "orl     $dst, $src\t# int" %}
9901   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9902   ins_encode(REX_mem(dst), OpcSE(src),
9903              RM_opc_mem(secondary, dst), Con8or32(src));
9904   ins_pipe(ialu_mem_imm);
9905 %}
9906 
9907 // Xor Instructions
9908 // Xor Register with Register
9909 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9910 %{
9911   match(Set dst (XorI dst src));
9912   effect(KILL cr);
9913 
9914   format %{ "xorl    $dst, $src\t# int" %}
9915   opcode(0x33);
9916   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9917   ins_pipe(ialu_reg_reg);
9918 %}
9919 
9920 // Xor Register with Immediate -1
9921 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9922   match(Set dst (XorI dst imm));
9923 
9924   format %{ "not    $dst" %}
9925   ins_encode %{
9926      __ notl($dst$$Register);
9927   %}
9928   ins_pipe(ialu_reg);
9929 %}
9930 
9931 // Xor Register with Immediate
9932 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9933 %{
9934   match(Set dst (XorI dst src));
9935   effect(KILL cr);
9936 
9937   format %{ "xorl    $dst, $src\t# int" %}
9938   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9939   ins_encode(OpcSErm(dst, src), Con8or32(src));
9940   ins_pipe(ialu_reg);
9941 %}
9942 
9943 // Xor Register with Memory
9944 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9945 %{
9946   match(Set dst (XorI dst (LoadI src)));
9947   effect(KILL cr);
9948 
9949   ins_cost(125);
9950   format %{ "xorl    $dst, $src\t# int" %}
9951   opcode(0x33);
9952   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9953   ins_pipe(ialu_reg_mem);
9954 %}
9955 
9956 // Xor Memory with Register
9957 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9958 %{
9959   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9960   effect(KILL cr);
9961 
9962   ins_cost(150);
9963   format %{ "xorb    $dst, $src\t# byte" %}
9964   opcode(0x30);
9965   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9966   ins_pipe(ialu_mem_reg);
9967 %}
9968 
9969 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9970 %{
9971   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9972   effect(KILL cr);
9973 
9974   ins_cost(150);
9975   format %{ "xorl    $dst, $src\t# int" %}
9976   opcode(0x31); /* Opcode 31 /r */
9977   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9978   ins_pipe(ialu_mem_reg);
9979 %}
9980 
9981 // Xor Memory with Immediate
9982 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9983 %{
9984   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9985   effect(KILL cr);
9986 
9987   ins_cost(125);
9988   format %{ "xorl    $dst, $src\t# int" %}
9989   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9990   ins_encode(REX_mem(dst), OpcSE(src),
9991              RM_opc_mem(secondary, dst), Con8or32(src));
9992   ins_pipe(ialu_mem_imm);
9993 %}
9994 
9995 
9996 // Long Logical Instructions
9997 
9998 // And Instructions
9999 // And Register with Register
10000 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10001 %{
10002   match(Set dst (AndL dst src));
10003   effect(KILL cr);
10004 
10005   format %{ "andq    $dst, $src\t# long" %}
10006   opcode(0x23);
10007   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10008   ins_pipe(ialu_reg_reg);
10009 %}
10010 
10011 // And Register with Immediate 255
10012 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
10013 %{
10014   match(Set dst (AndL dst src));
10015 
10016   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
10017   opcode(0x0F, 0xB6);
10018   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
10019   ins_pipe(ialu_reg);
10020 %}
10021 
10022 // And Register with Immediate 65535
10023 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
10024 %{
10025   match(Set dst (AndL dst src));
10026 
10027   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
10028   opcode(0x0F, 0xB7);
10029   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
10030   ins_pipe(ialu_reg);
10031 %}
10032 
10033 // And Register with Immediate
10034 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10035 %{
10036   match(Set dst (AndL dst src));
10037   effect(KILL cr);
10038 
10039   format %{ "andq    $dst, $src\t# long" %}
10040   opcode(0x81, 0x04); /* Opcode 81 /4 */
10041   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10042   ins_pipe(ialu_reg);
10043 %}
10044 
10045 // And Register with Memory
10046 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10047 %{
10048   match(Set dst (AndL dst (LoadL src)));
10049   effect(KILL cr);
10050 
10051   ins_cost(125);
10052   format %{ "andq    $dst, $src\t# long" %}
10053   opcode(0x23);
10054   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10055   ins_pipe(ialu_reg_mem);
10056 %}
10057 
10058 // And Memory with Register
10059 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10060 %{
10061   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10062   effect(KILL cr);
10063 
10064   ins_cost(150);
10065   format %{ "andq    $dst, $src\t# long" %}
10066   opcode(0x21); /* Opcode 21 /r */
10067   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10068   ins_pipe(ialu_mem_reg);
10069 %}
10070 
10071 // And Memory with Immediate
10072 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10073 %{
10074   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
10075   effect(KILL cr);
10076 
10077   ins_cost(125);
10078   format %{ "andq    $dst, $src\t# long" %}
10079   opcode(0x81, 0x4); /* Opcode 81 /4 id */
10080   ins_encode(REX_mem_wide(dst), OpcSE(src),
10081              RM_opc_mem(secondary, dst), Con8or32(src));
10082   ins_pipe(ialu_mem_imm);
10083 %}
10084 
10085 // BMI1 instructions
10086 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
10087   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
10088   predicate(UseBMI1Instructions);
10089   effect(KILL cr);
10090 
10091   ins_cost(125);
10092   format %{ "andnq  $dst, $src1, $src2" %}
10093 
10094   ins_encode %{
10095     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
10096   %}
10097   ins_pipe(ialu_reg_mem);
10098 %}
10099 
10100 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
10101   match(Set dst (AndL (XorL src1 minus_1) src2));
10102   predicate(UseBMI1Instructions);
10103   effect(KILL cr);
10104 
10105   format %{ "andnq  $dst, $src1, $src2" %}
10106 
10107   ins_encode %{
10108   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
10109   %}
10110   ins_pipe(ialu_reg_mem);
10111 %}
10112 
10113 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
10114   match(Set dst (AndL (SubL imm_zero src) src));
10115   predicate(UseBMI1Instructions);
10116   effect(KILL cr);
10117 
10118   format %{ "blsiq  $dst, $src" %}
10119 
10120   ins_encode %{
10121     __ blsiq($dst$$Register, $src$$Register);
10122   %}
10123   ins_pipe(ialu_reg);
10124 %}
10125 
10126 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
10127   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
10128   predicate(UseBMI1Instructions);
10129   effect(KILL cr);
10130 
10131   ins_cost(125);
10132   format %{ "blsiq  $dst, $src" %}
10133 
10134   ins_encode %{
10135     __ blsiq($dst$$Register, $src$$Address);
10136   %}
10137   ins_pipe(ialu_reg_mem);
10138 %}
10139 
10140 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10141 %{
10142   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10143   predicate(UseBMI1Instructions);
10144   effect(KILL cr);
10145 
10146   ins_cost(125);
10147   format %{ "blsmskq $dst, $src" %}
10148 
10149   ins_encode %{
10150     __ blsmskq($dst$$Register, $src$$Address);
10151   %}
10152   ins_pipe(ialu_reg_mem);
10153 %}
10154 
10155 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10156 %{
10157   match(Set dst (XorL (AddL src minus_1) src));
10158   predicate(UseBMI1Instructions);
10159   effect(KILL cr);
10160 
10161   format %{ "blsmskq $dst, $src" %}
10162 
10163   ins_encode %{
10164     __ blsmskq($dst$$Register, $src$$Register);
10165   %}
10166 
10167   ins_pipe(ialu_reg);
10168 %}
10169 
10170 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10171 %{
10172   match(Set dst (AndL (AddL src minus_1) src) );
10173   predicate(UseBMI1Instructions);
10174   effect(KILL cr);
10175 
10176   format %{ "blsrq  $dst, $src" %}
10177 
10178   ins_encode %{
10179     __ blsrq($dst$$Register, $src$$Register);
10180   %}
10181 
10182   ins_pipe(ialu_reg);
10183 %}
10184 
10185 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10186 %{
10187   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10188   predicate(UseBMI1Instructions);
10189   effect(KILL cr);
10190 
10191   ins_cost(125);
10192   format %{ "blsrq  $dst, $src" %}
10193 
10194   ins_encode %{
10195     __ blsrq($dst$$Register, $src$$Address);
10196   %}
10197 
10198   ins_pipe(ialu_reg);
10199 %}
10200 
10201 // Or Instructions
10202 // Or Register with Register
10203 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10204 %{
10205   match(Set dst (OrL dst src));
10206   effect(KILL cr);
10207 
10208   format %{ "orq     $dst, $src\t# long" %}
10209   opcode(0x0B);
10210   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10211   ins_pipe(ialu_reg_reg);
10212 %}
10213 
10214 // Use any_RegP to match R15 (TLS register) without spilling.
10215 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10216   match(Set dst (OrL dst (CastP2X src)));
10217   effect(KILL cr);
10218 
10219   format %{ "orq     $dst, $src\t# long" %}
10220   opcode(0x0B);
10221   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10222   ins_pipe(ialu_reg_reg);
10223 %}
10224 
10225 
10226 // Or Register with Immediate
10227 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10228 %{
10229   match(Set dst (OrL dst src));
10230   effect(KILL cr);
10231 
10232   format %{ "orq     $dst, $src\t# long" %}
10233   opcode(0x81, 0x01); /* Opcode 81 /1 id */
10234   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10235   ins_pipe(ialu_reg);
10236 %}
10237 
10238 // Or Register with Memory
10239 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10240 %{
10241   match(Set dst (OrL dst (LoadL src)));
10242   effect(KILL cr);
10243 
10244   ins_cost(125);
10245   format %{ "orq     $dst, $src\t# long" %}
10246   opcode(0x0B);
10247   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10248   ins_pipe(ialu_reg_mem);
10249 %}
10250 
10251 // Or Memory with Register
10252 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10253 %{
10254   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10255   effect(KILL cr);
10256 
10257   ins_cost(150);
10258   format %{ "orq     $dst, $src\t# long" %}
10259   opcode(0x09); /* Opcode 09 /r */
10260   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10261   ins_pipe(ialu_mem_reg);
10262 %}
10263 
10264 // Or Memory with Immediate
10265 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10266 %{
10267   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10268   effect(KILL cr);
10269 
10270   ins_cost(125);
10271   format %{ "orq     $dst, $src\t# long" %}
10272   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10273   ins_encode(REX_mem_wide(dst), OpcSE(src),
10274              RM_opc_mem(secondary, dst), Con8or32(src));
10275   ins_pipe(ialu_mem_imm);
10276 %}
10277 
10278 // Xor Instructions
10279 // Xor Register with Register
10280 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10281 %{
10282   match(Set dst (XorL dst src));
10283   effect(KILL cr);
10284 
10285   format %{ "xorq    $dst, $src\t# long" %}
10286   opcode(0x33);
10287   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10288   ins_pipe(ialu_reg_reg);
10289 %}
10290 
10291 // Xor Register with Immediate -1
10292 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10293   match(Set dst (XorL dst imm));
10294 
10295   format %{ "notq   $dst" %}
10296   ins_encode %{
10297      __ notq($dst$$Register);
10298   %}
10299   ins_pipe(ialu_reg);
10300 %}
10301 
10302 // Xor Register with Immediate
10303 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10304 %{
10305   match(Set dst (XorL dst src));
10306   effect(KILL cr);
10307 
10308   format %{ "xorq    $dst, $src\t# long" %}
10309   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10310   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10311   ins_pipe(ialu_reg);
10312 %}
10313 
10314 // Xor Register with Memory
10315 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10316 %{
10317   match(Set dst (XorL dst (LoadL src)));
10318   effect(KILL cr);
10319 
10320   ins_cost(125);
10321   format %{ "xorq    $dst, $src\t# long" %}
10322   opcode(0x33);
10323   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10324   ins_pipe(ialu_reg_mem);
10325 %}
10326 
10327 // Xor Memory with Register
10328 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10329 %{
10330   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10331   effect(KILL cr);
10332 
10333   ins_cost(150);
10334   format %{ "xorq    $dst, $src\t# long" %}
10335   opcode(0x31); /* Opcode 31 /r */
10336   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10337   ins_pipe(ialu_mem_reg);
10338 %}
10339 
10340 // Xor Memory with Immediate
10341 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10342 %{
10343   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10344   effect(KILL cr);
10345 
10346   ins_cost(125);
10347   format %{ "xorq    $dst, $src\t# long" %}
10348   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10349   ins_encode(REX_mem_wide(dst), OpcSE(src),
10350              RM_opc_mem(secondary, dst), Con8or32(src));
10351   ins_pipe(ialu_mem_imm);
10352 %}
10353 
10354 // Convert Int to Boolean
10355 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10356 %{
10357   match(Set dst (Conv2B src));
10358   effect(KILL cr);
10359 
10360   format %{ "testl   $src, $src\t# ci2b\n\t"
10361             "setnz   $dst\n\t"
10362             "movzbl  $dst, $dst" %}
10363   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10364              setNZ_reg(dst),
10365              REX_reg_breg(dst, dst), // movzbl
10366              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10367   ins_pipe(pipe_slow); // XXX
10368 %}
10369 
10370 // Convert Pointer to Boolean
10371 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10372 %{
10373   match(Set dst (Conv2B src));
10374   effect(KILL cr);
10375 
10376   format %{ "testq   $src, $src\t# cp2b\n\t"
10377             "setnz   $dst\n\t"
10378             "movzbl  $dst, $dst" %}
10379   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10380              setNZ_reg(dst),
10381              REX_reg_breg(dst, dst), // movzbl
10382              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10383   ins_pipe(pipe_slow); // XXX
10384 %}
10385 
10386 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10387 %{
10388   match(Set dst (CmpLTMask p q));
10389   effect(KILL cr);
10390 
10391   ins_cost(400);
10392   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10393             "setlt   $dst\n\t"
10394             "movzbl  $dst, $dst\n\t"
10395             "negl    $dst" %}
10396   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10397              setLT_reg(dst),
10398              REX_reg_breg(dst, dst), // movzbl
10399              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10400              neg_reg(dst));
10401   ins_pipe(pipe_slow);
10402 %}
10403 
10404 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10405 %{
10406   match(Set dst (CmpLTMask dst zero));
10407   effect(KILL cr);
10408 
10409   ins_cost(100);
10410   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10411   ins_encode %{
10412   __ sarl($dst$$Register, 31);
10413   %}
10414   ins_pipe(ialu_reg);
10415 %}
10416 
10417 /* Better to save a register than avoid a branch */
10418 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10419 %{
10420   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10421   effect(KILL cr);
10422   ins_cost(300);
10423   format %{ "subl    $p,$q\t# cadd_cmpLTMask\n\t"
10424             "jge     done\n\t"
10425             "addl    $p,$y\n"
10426             "done:   " %}
10427   ins_encode %{
10428     Register Rp = $p$$Register;
10429     Register Rq = $q$$Register;
10430     Register Ry = $y$$Register;
10431     Label done;
10432     __ subl(Rp, Rq);
10433     __ jccb(Assembler::greaterEqual, done);
10434     __ addl(Rp, Ry);
10435     __ bind(done);
10436   %}
10437   ins_pipe(pipe_cmplt);
10438 %}
10439 
10440 /* Better to save a register than avoid a branch */
10441 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10442 %{
10443   match(Set y (AndI (CmpLTMask p q) y));
10444   effect(KILL cr);
10445 
10446   ins_cost(300);
10447 
10448   format %{ "cmpl    $p, $q\t# and_cmpLTMask\n\t"
10449             "jlt     done\n\t"
10450             "xorl    $y, $y\n"
10451             "done:   " %}
10452   ins_encode %{
10453     Register Rp = $p$$Register;
10454     Register Rq = $q$$Register;
10455     Register Ry = $y$$Register;
10456     Label done;
10457     __ cmpl(Rp, Rq);
10458     __ jccb(Assembler::less, done);
10459     __ xorl(Ry, Ry);
10460     __ bind(done);
10461   %}
10462   ins_pipe(pipe_cmplt);
10463 %}
10464 
10465 
10466 //---------- FP Instructions------------------------------------------------
10467 
10468 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10469 %{
10470   match(Set cr (CmpF src1 src2));
10471 
10472   ins_cost(145);
10473   format %{ "ucomiss $src1, $src2\n\t"
10474             "jnp,s   exit\n\t"
10475             "pushfq\t# saw NaN, set CF\n\t"
10476             "andq    [rsp], #0xffffff2b\n\t"
10477             "popfq\n"
10478     "exit:" %}
10479   ins_encode %{
10480     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10481     emit_cmpfp_fixup(_masm);
10482   %}
10483   ins_pipe(pipe_slow);
10484 %}
10485 
10486 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10487   match(Set cr (CmpF src1 src2));
10488 
10489   ins_cost(100);
10490   format %{ "ucomiss $src1, $src2" %}
10491   ins_encode %{
10492     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10493   %}
10494   ins_pipe(pipe_slow);
10495 %}
10496 
10497 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10498 %{
10499   match(Set cr (CmpF src1 (LoadF src2)));
10500 
10501   ins_cost(145);
10502   format %{ "ucomiss $src1, $src2\n\t"
10503             "jnp,s   exit\n\t"
10504             "pushfq\t# saw NaN, set CF\n\t"
10505             "andq    [rsp], #0xffffff2b\n\t"
10506             "popfq\n"
10507     "exit:" %}
10508   ins_encode %{
10509     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10510     emit_cmpfp_fixup(_masm);
10511   %}
10512   ins_pipe(pipe_slow);
10513 %}
10514 
10515 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10516   match(Set cr (CmpF src1 (LoadF src2)));
10517 
10518   ins_cost(100);
10519   format %{ "ucomiss $src1, $src2" %}
10520   ins_encode %{
10521     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10522   %}
10523   ins_pipe(pipe_slow);
10524 %}
10525 
10526 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10527   match(Set cr (CmpF src con));
10528 
10529   ins_cost(145);
10530   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10531             "jnp,s   exit\n\t"
10532             "pushfq\t# saw NaN, set CF\n\t"
10533             "andq    [rsp], #0xffffff2b\n\t"
10534             "popfq\n"
10535     "exit:" %}
10536   ins_encode %{
10537     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10538     emit_cmpfp_fixup(_masm);
10539   %}
10540   ins_pipe(pipe_slow);
10541 %}
10542 
10543 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10544   match(Set cr (CmpF src con));
10545   ins_cost(100);
10546   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10547   ins_encode %{
10548     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10549   %}
10550   ins_pipe(pipe_slow);
10551 %}
10552 
10553 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10554 %{
10555   match(Set cr (CmpD src1 src2));
10556 
10557   ins_cost(145);
10558   format %{ "ucomisd $src1, $src2\n\t"
10559             "jnp,s   exit\n\t"
10560             "pushfq\t# saw NaN, set CF\n\t"
10561             "andq    [rsp], #0xffffff2b\n\t"
10562             "popfq\n"
10563     "exit:" %}
10564   ins_encode %{
10565     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10566     emit_cmpfp_fixup(_masm);
10567   %}
10568   ins_pipe(pipe_slow);
10569 %}
10570 
10571 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10572   match(Set cr (CmpD src1 src2));
10573 
10574   ins_cost(100);
10575   format %{ "ucomisd $src1, $src2 test" %}
10576   ins_encode %{
10577     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10578   %}
10579   ins_pipe(pipe_slow);
10580 %}
10581 
10582 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10583 %{
10584   match(Set cr (CmpD src1 (LoadD src2)));
10585 
10586   ins_cost(145);
10587   format %{ "ucomisd $src1, $src2\n\t"
10588             "jnp,s   exit\n\t"
10589             "pushfq\t# saw NaN, set CF\n\t"
10590             "andq    [rsp], #0xffffff2b\n\t"
10591             "popfq\n"
10592     "exit:" %}
10593   ins_encode %{
10594     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10595     emit_cmpfp_fixup(_masm);
10596   %}
10597   ins_pipe(pipe_slow);
10598 %}
10599 
10600 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10601   match(Set cr (CmpD src1 (LoadD src2)));
10602 
10603   ins_cost(100);
10604   format %{ "ucomisd $src1, $src2" %}
10605   ins_encode %{
10606     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10607   %}
10608   ins_pipe(pipe_slow);
10609 %}
10610 
10611 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10612   match(Set cr (CmpD src con));
10613 
10614   ins_cost(145);
10615   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10616             "jnp,s   exit\n\t"
10617             "pushfq\t# saw NaN, set CF\n\t"
10618             "andq    [rsp], #0xffffff2b\n\t"
10619             "popfq\n"
10620     "exit:" %}
10621   ins_encode %{
10622     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10623     emit_cmpfp_fixup(_masm);
10624   %}
10625   ins_pipe(pipe_slow);
10626 %}
10627 
10628 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10629   match(Set cr (CmpD src con));
10630   ins_cost(100);
10631   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10632   ins_encode %{
10633     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10634   %}
10635   ins_pipe(pipe_slow);
10636 %}
10637 
10638 // Compare into -1,0,1
10639 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10640 %{
10641   match(Set dst (CmpF3 src1 src2));
10642   effect(KILL cr);
10643 
10644   ins_cost(275);
10645   format %{ "ucomiss $src1, $src2\n\t"
10646             "movl    $dst, #-1\n\t"
10647             "jp,s    done\n\t"
10648             "jb,s    done\n\t"
10649             "setne   $dst\n\t"
10650             "movzbl  $dst, $dst\n"
10651     "done:" %}
10652   ins_encode %{
10653     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10654     emit_cmpfp3(_masm, $dst$$Register);
10655   %}
10656   ins_pipe(pipe_slow);
10657 %}
10658 
10659 // Compare into -1,0,1
10660 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10661 %{
10662   match(Set dst (CmpF3 src1 (LoadF src2)));
10663   effect(KILL cr);
10664 
10665   ins_cost(275);
10666   format %{ "ucomiss $src1, $src2\n\t"
10667             "movl    $dst, #-1\n\t"
10668             "jp,s    done\n\t"
10669             "jb,s    done\n\t"
10670             "setne   $dst\n\t"
10671             "movzbl  $dst, $dst\n"
10672     "done:" %}
10673   ins_encode %{
10674     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10675     emit_cmpfp3(_masm, $dst$$Register);
10676   %}
10677   ins_pipe(pipe_slow);
10678 %}
10679 
10680 // Compare into -1,0,1
10681 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10682   match(Set dst (CmpF3 src con));
10683   effect(KILL cr);
10684 
10685   ins_cost(275);
10686   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10687             "movl    $dst, #-1\n\t"
10688             "jp,s    done\n\t"
10689             "jb,s    done\n\t"
10690             "setne   $dst\n\t"
10691             "movzbl  $dst, $dst\n"
10692     "done:" %}
10693   ins_encode %{
10694     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10695     emit_cmpfp3(_masm, $dst$$Register);
10696   %}
10697   ins_pipe(pipe_slow);
10698 %}
10699 
10700 // Compare into -1,0,1
10701 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10702 %{
10703   match(Set dst (CmpD3 src1 src2));
10704   effect(KILL cr);
10705 
10706   ins_cost(275);
10707   format %{ "ucomisd $src1, $src2\n\t"
10708             "movl    $dst, #-1\n\t"
10709             "jp,s    done\n\t"
10710             "jb,s    done\n\t"
10711             "setne   $dst\n\t"
10712             "movzbl  $dst, $dst\n"
10713     "done:" %}
10714   ins_encode %{
10715     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10716     emit_cmpfp3(_masm, $dst$$Register);
10717   %}
10718   ins_pipe(pipe_slow);
10719 %}
10720 
10721 // Compare into -1,0,1
10722 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10723 %{
10724   match(Set dst (CmpD3 src1 (LoadD src2)));
10725   effect(KILL cr);
10726 
10727   ins_cost(275);
10728   format %{ "ucomisd $src1, $src2\n\t"
10729             "movl    $dst, #-1\n\t"
10730             "jp,s    done\n\t"
10731             "jb,s    done\n\t"
10732             "setne   $dst\n\t"
10733             "movzbl  $dst, $dst\n"
10734     "done:" %}
10735   ins_encode %{
10736     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10737     emit_cmpfp3(_masm, $dst$$Register);
10738   %}
10739   ins_pipe(pipe_slow);
10740 %}
10741 
10742 // Compare into -1,0,1
10743 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10744   match(Set dst (CmpD3 src con));
10745   effect(KILL cr);
10746 
10747   ins_cost(275);
10748   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10749             "movl    $dst, #-1\n\t"
10750             "jp,s    done\n\t"
10751             "jb,s    done\n\t"
10752             "setne   $dst\n\t"
10753             "movzbl  $dst, $dst\n"
10754     "done:" %}
10755   ins_encode %{
10756     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10757     emit_cmpfp3(_masm, $dst$$Register);
10758   %}
10759   ins_pipe(pipe_slow);
10760 %}
10761 
10762 //----------Arithmetic Conversion Instructions---------------------------------
10763 
10764 instruct roundFloat_nop(regF dst)
10765 %{
10766   match(Set dst (RoundFloat dst));
10767 
10768   ins_cost(0);
10769   ins_encode();
10770   ins_pipe(empty);
10771 %}
10772 
10773 instruct roundDouble_nop(regD dst)
10774 %{
10775   match(Set dst (RoundDouble dst));
10776 
10777   ins_cost(0);
10778   ins_encode();
10779   ins_pipe(empty);
10780 %}
10781 
10782 instruct convF2D_reg_reg(regD dst, regF src)
10783 %{
10784   match(Set dst (ConvF2D src));
10785 
10786   format %{ "cvtss2sd $dst, $src" %}
10787   ins_encode %{
10788     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10789   %}
10790   ins_pipe(pipe_slow); // XXX
10791 %}
10792 
10793 instruct convF2D_reg_mem(regD dst, memory src)
10794 %{
10795   match(Set dst (ConvF2D (LoadF src)));
10796 
10797   format %{ "cvtss2sd $dst, $src" %}
10798   ins_encode %{
10799     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10800   %}
10801   ins_pipe(pipe_slow); // XXX
10802 %}
10803 
10804 instruct convD2F_reg_reg(regF dst, regD src)
10805 %{
10806   match(Set dst (ConvD2F src));
10807 
10808   format %{ "cvtsd2ss $dst, $src" %}
10809   ins_encode %{
10810     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10811   %}
10812   ins_pipe(pipe_slow); // XXX
10813 %}
10814 
10815 instruct convD2F_reg_mem(regF dst, memory src)
10816 %{
10817   match(Set dst (ConvD2F (LoadD src)));
10818 
10819   format %{ "cvtsd2ss $dst, $src" %}
10820   ins_encode %{
10821     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10822   %}
10823   ins_pipe(pipe_slow); // XXX
10824 %}
10825 
10826 // XXX do mem variants
10827 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10828 %{
10829   match(Set dst (ConvF2I src));
10830   effect(KILL cr);
10831 
10832   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10833             "cmpl    $dst, #0x80000000\n\t"
10834             "jne,s   done\n\t"
10835             "subq    rsp, #8\n\t"
10836             "movss   [rsp], $src\n\t"
10837             "call    f2i_fixup\n\t"
10838             "popq    $dst\n"
10839     "done:   "%}
10840   ins_encode %{
10841     Label done;
10842     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10843     __ cmpl($dst$$Register, 0x80000000);
10844     __ jccb(Assembler::notEqual, done);
10845     __ subptr(rsp, 8);
10846     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10847     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10848     __ pop($dst$$Register);
10849     __ bind(done);
10850   %}
10851   ins_pipe(pipe_slow);
10852 %}
10853 
10854 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10855 %{
10856   match(Set dst (ConvF2L src));
10857   effect(KILL cr);
10858 
10859   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10860             "cmpq    $dst, [0x8000000000000000]\n\t"
10861             "jne,s   done\n\t"
10862             "subq    rsp, #8\n\t"
10863             "movss   [rsp], $src\n\t"
10864             "call    f2l_fixup\n\t"
10865             "popq    $dst\n"
10866     "done:   "%}
10867   ins_encode %{
10868     Label done;
10869     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10870     __ cmp64($dst$$Register,
10871              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10872     __ jccb(Assembler::notEqual, done);
10873     __ subptr(rsp, 8);
10874     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10875     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10876     __ pop($dst$$Register);
10877     __ bind(done);
10878   %}
10879   ins_pipe(pipe_slow);
10880 %}
10881 
10882 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10883 %{
10884   match(Set dst (ConvD2I src));
10885   effect(KILL cr);
10886 
10887   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10888             "cmpl    $dst, #0x80000000\n\t"
10889             "jne,s   done\n\t"
10890             "subq    rsp, #8\n\t"
10891             "movsd   [rsp], $src\n\t"
10892             "call    d2i_fixup\n\t"
10893             "popq    $dst\n"
10894     "done:   "%}
10895   ins_encode %{
10896     Label done;
10897     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10898     __ cmpl($dst$$Register, 0x80000000);
10899     __ jccb(Assembler::notEqual, done);
10900     __ subptr(rsp, 8);
10901     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10902     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10903     __ pop($dst$$Register);
10904     __ bind(done);
10905   %}
10906   ins_pipe(pipe_slow);
10907 %}
10908 
10909 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10910 %{
10911   match(Set dst (ConvD2L src));
10912   effect(KILL cr);
10913 
10914   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10915             "cmpq    $dst, [0x8000000000000000]\n\t"
10916             "jne,s   done\n\t"
10917             "subq    rsp, #8\n\t"
10918             "movsd   [rsp], $src\n\t"
10919             "call    d2l_fixup\n\t"
10920             "popq    $dst\n"
10921     "done:   "%}
10922   ins_encode %{
10923     Label done;
10924     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10925     __ cmp64($dst$$Register,
10926              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10927     __ jccb(Assembler::notEqual, done);
10928     __ subptr(rsp, 8);
10929     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10930     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10931     __ pop($dst$$Register);
10932     __ bind(done);
10933   %}
10934   ins_pipe(pipe_slow);
10935 %}
10936 
10937 instruct convI2F_reg_reg(regF dst, rRegI src)
10938 %{
10939   predicate(!UseXmmI2F);
10940   match(Set dst (ConvI2F src));
10941 
10942   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10943   ins_encode %{
10944     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10945   %}
10946   ins_pipe(pipe_slow); // XXX
10947 %}
10948 
10949 instruct convI2F_reg_mem(regF dst, memory src)
10950 %{
10951   match(Set dst (ConvI2F (LoadI src)));
10952 
10953   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10954   ins_encode %{
10955     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10956   %}
10957   ins_pipe(pipe_slow); // XXX
10958 %}
10959 
10960 instruct convI2D_reg_reg(regD dst, rRegI src)
10961 %{
10962   predicate(!UseXmmI2D);
10963   match(Set dst (ConvI2D src));
10964 
10965   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10966   ins_encode %{
10967     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10968   %}
10969   ins_pipe(pipe_slow); // XXX
10970 %}
10971 
10972 instruct convI2D_reg_mem(regD dst, memory src)
10973 %{
10974   match(Set dst (ConvI2D (LoadI src)));
10975 
10976   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10977   ins_encode %{
10978     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10979   %}
10980   ins_pipe(pipe_slow); // XXX
10981 %}
10982 
10983 instruct convXI2F_reg(regF dst, rRegI src)
10984 %{
10985   predicate(UseXmmI2F);
10986   match(Set dst (ConvI2F src));
10987 
10988   format %{ "movdl $dst, $src\n\t"
10989             "cvtdq2psl $dst, $dst\t# i2f" %}
10990   ins_encode %{
10991     __ movdl($dst$$XMMRegister, $src$$Register);
10992     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10993   %}
10994   ins_pipe(pipe_slow); // XXX
10995 %}
10996 
10997 instruct convXI2D_reg(regD dst, rRegI src)
10998 %{
10999   predicate(UseXmmI2D);
11000   match(Set dst (ConvI2D src));
11001 
11002   format %{ "movdl $dst, $src\n\t"
11003             "cvtdq2pdl $dst, $dst\t# i2d" %}
11004   ins_encode %{
11005     __ movdl($dst$$XMMRegister, $src$$Register);
11006     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11007   %}
11008   ins_pipe(pipe_slow); // XXX
11009 %}
11010 
11011 instruct convL2F_reg_reg(regF dst, rRegL src)
11012 %{
11013   match(Set dst (ConvL2F src));
11014 
11015   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11016   ins_encode %{
11017     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
11018   %}
11019   ins_pipe(pipe_slow); // XXX
11020 %}
11021 
11022 instruct convL2F_reg_mem(regF dst, memory src)
11023 %{
11024   match(Set dst (ConvL2F (LoadL src)));
11025 
11026   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11027   ins_encode %{
11028     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
11029   %}
11030   ins_pipe(pipe_slow); // XXX
11031 %}
11032 
11033 instruct convL2D_reg_reg(regD dst, rRegL src)
11034 %{
11035   match(Set dst (ConvL2D src));
11036 
11037   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11038   ins_encode %{
11039     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
11040   %}
11041   ins_pipe(pipe_slow); // XXX
11042 %}
11043 
11044 instruct convL2D_reg_mem(regD dst, memory src)
11045 %{
11046   match(Set dst (ConvL2D (LoadL src)));
11047 
11048   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11049   ins_encode %{
11050     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
11051   %}
11052   ins_pipe(pipe_slow); // XXX
11053 %}
11054 
11055 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11056 %{
11057   match(Set dst (ConvI2L src));
11058 
11059   ins_cost(125);
11060   format %{ "movslq  $dst, $src\t# i2l" %}
11061   ins_encode %{
11062     __ movslq($dst$$Register, $src$$Register);
11063   %}
11064   ins_pipe(ialu_reg_reg);
11065 %}
11066 
11067 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11068 // %{
11069 //   match(Set dst (ConvI2L src));
11070 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11071 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11072 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11073 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11074 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11075 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11076 
11077 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11078 //   ins_encode(enc_copy(dst, src));
11079 // //   opcode(0x63); // needs REX.W
11080 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11081 //   ins_pipe(ialu_reg_reg);
11082 // %}
11083 
11084 // Zero-extend convert int to long
11085 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11086 %{
11087   match(Set dst (AndL (ConvI2L src) mask));
11088 
11089   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11090   ins_encode %{
11091     if ($dst$$reg != $src$$reg) {
11092       __ movl($dst$$Register, $src$$Register);
11093     }
11094   %}
11095   ins_pipe(ialu_reg_reg);
11096 %}
11097 
11098 // Zero-extend convert int to long
11099 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11100 %{
11101   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11102 
11103   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11104   ins_encode %{
11105     __ movl($dst$$Register, $src$$Address);
11106   %}
11107   ins_pipe(ialu_reg_mem);
11108 %}
11109 
11110 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11111 %{
11112   match(Set dst (AndL src mask));
11113 
11114   format %{ "movl    $dst, $src\t# zero-extend long" %}
11115   ins_encode %{
11116     __ movl($dst$$Register, $src$$Register);
11117   %}
11118   ins_pipe(ialu_reg_reg);
11119 %}
11120 
11121 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11122 %{
11123   match(Set dst (ConvL2I src));
11124 
11125   format %{ "movl    $dst, $src\t# l2i" %}
11126   ins_encode %{
11127     __ movl($dst$$Register, $src$$Register);
11128   %}
11129   ins_pipe(ialu_reg_reg);
11130 %}
11131 
11132 
11133 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11134   match(Set dst (MoveF2I src));
11135   effect(DEF dst, USE src);
11136 
11137   ins_cost(125);
11138   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11139   ins_encode %{
11140     __ movl($dst$$Register, Address(rsp, $src$$disp));
11141   %}
11142   ins_pipe(ialu_reg_mem);
11143 %}
11144 
11145 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11146   match(Set dst (MoveI2F src));
11147   effect(DEF dst, USE src);
11148 
11149   ins_cost(125);
11150   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11151   ins_encode %{
11152     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11153   %}
11154   ins_pipe(pipe_slow);
11155 %}
11156 
11157 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11158   match(Set dst (MoveD2L src));
11159   effect(DEF dst, USE src);
11160 
11161   ins_cost(125);
11162   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11163   ins_encode %{
11164     __ movq($dst$$Register, Address(rsp, $src$$disp));
11165   %}
11166   ins_pipe(ialu_reg_mem);
11167 %}
11168 
11169 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11170   predicate(!UseXmmLoadAndClearUpper);
11171   match(Set dst (MoveL2D src));
11172   effect(DEF dst, USE src);
11173 
11174   ins_cost(125);
11175   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11176   ins_encode %{
11177     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11178   %}
11179   ins_pipe(pipe_slow);
11180 %}
11181 
11182 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11183   predicate(UseXmmLoadAndClearUpper);
11184   match(Set dst (MoveL2D src));
11185   effect(DEF dst, USE src);
11186 
11187   ins_cost(125);
11188   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11189   ins_encode %{
11190     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11191   %}
11192   ins_pipe(pipe_slow);
11193 %}
11194 
11195 
11196 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11197   match(Set dst (MoveF2I src));
11198   effect(DEF dst, USE src);
11199 
11200   ins_cost(95); // XXX
11201   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11202   ins_encode %{
11203     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11204   %}
11205   ins_pipe(pipe_slow);
11206 %}
11207 
11208 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11209   match(Set dst (MoveI2F src));
11210   effect(DEF dst, USE src);
11211 
11212   ins_cost(100);
11213   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11214   ins_encode %{
11215     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11216   %}
11217   ins_pipe( ialu_mem_reg );
11218 %}
11219 
11220 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11221   match(Set dst (MoveD2L src));
11222   effect(DEF dst, USE src);
11223 
11224   ins_cost(95); // XXX
11225   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11226   ins_encode %{
11227     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11228   %}
11229   ins_pipe(pipe_slow);
11230 %}
11231 
11232 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11233   match(Set dst (MoveL2D src));
11234   effect(DEF dst, USE src);
11235 
11236   ins_cost(100);
11237   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11238   ins_encode %{
11239     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11240   %}
11241   ins_pipe(ialu_mem_reg);
11242 %}
11243 
11244 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11245   match(Set dst (MoveF2I src));
11246   effect(DEF dst, USE src);
11247   ins_cost(85);
11248   format %{ "movd    $dst,$src\t# MoveF2I" %}
11249   ins_encode %{
11250     __ movdl($dst$$Register, $src$$XMMRegister);
11251   %}
11252   ins_pipe( pipe_slow );
11253 %}
11254 
11255 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11256   match(Set dst (MoveD2L src));
11257   effect(DEF dst, USE src);
11258   ins_cost(85);
11259   format %{ "movd    $dst,$src\t# MoveD2L" %}
11260   ins_encode %{
11261     __ movdq($dst$$Register, $src$$XMMRegister);
11262   %}
11263   ins_pipe( pipe_slow );
11264 %}
11265 
11266 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11267   match(Set dst (MoveI2F src));
11268   effect(DEF dst, USE src);
11269   ins_cost(100);
11270   format %{ "movd    $dst,$src\t# MoveI2F" %}
11271   ins_encode %{
11272     __ movdl($dst$$XMMRegister, $src$$Register);
11273   %}
11274   ins_pipe( pipe_slow );
11275 %}
11276 
11277 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11278   match(Set dst (MoveL2D src));
11279   effect(DEF dst, USE src);
11280   ins_cost(100);
11281   format %{ "movd    $dst,$src\t# MoveL2D" %}
11282   ins_encode %{
11283      __ movdq($dst$$XMMRegister, $src$$Register);
11284   %}
11285   ins_pipe( pipe_slow );
11286 %}
11287 
11288 
11289 // =======================================================================
11290 // fast clearing of an array
11291 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11292                   Universe dummy, rFlagsReg cr)
11293 %{
11294   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only());
11295   match(Set dummy (ClearArray (Binary cnt base) val));
11296   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11297 
11298   format %{ $$template
11299     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11300     $$emit$$"jg      LARGE\n\t"
11301     $$emit$$"dec     rcx\n\t"
11302     $$emit$$"js      DONE\t# Zero length\n\t"
11303     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11304     $$emit$$"dec     rcx\n\t"
11305     $$emit$$"jge     LOOP\n\t"
11306     $$emit$$"jmp     DONE\n\t"
11307     $$emit$$"# LARGE:\n\t"
11308     if (UseFastStosb) {
11309        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11310        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11311     } else if (UseXMMForObjInit) {
11312        $$emit$$"movdq   $tmp, $val\n\t"
11313        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11314        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11315        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11316        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11317        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11318        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11319        $$emit$$"add     0x40,rax\n\t"
11320        $$emit$$"# L_zero_64_bytes:\n\t"
11321        $$emit$$"sub     0x8,rcx\n\t"
11322        $$emit$$"jge     L_loop\n\t"
11323        $$emit$$"add     0x4,rcx\n\t"
11324        $$emit$$"jl      L_tail\n\t"
11325        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11326        $$emit$$"add     0x20,rax\n\t"
11327        $$emit$$"sub     0x4,rcx\n\t"
11328        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11329        $$emit$$"add     0x4,rcx\n\t"
11330        $$emit$$"jle     L_end\n\t"
11331        $$emit$$"dec     rcx\n\t"
11332        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11333        $$emit$$"vmovq   xmm0,(rax)\n\t"
11334        $$emit$$"add     0x8,rax\n\t"
11335        $$emit$$"dec     rcx\n\t"
11336        $$emit$$"jge     L_sloop\n\t"
11337        $$emit$$"# L_end:\n\t"
11338     } else {
11339        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11340     }
11341     $$emit$$"# DONE"
11342   %}
11343   ins_encode %{
11344     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11345                  $tmp$$XMMRegister, false, false);
11346   %}
11347   ins_pipe(pipe_slow);
11348 %}
11349 
11350 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11351                   Universe dummy, rFlagsReg cr)
11352 %{
11353   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only());
11354   match(Set dummy (ClearArray (Binary cnt base) val));
11355   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11356 
11357   format %{ $$template
11358     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11359     $$emit$$"jg      LARGE\n\t"
11360     $$emit$$"dec     rcx\n\t"
11361     $$emit$$"js      DONE\t# Zero length\n\t"
11362     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11363     $$emit$$"dec     rcx\n\t"
11364     $$emit$$"jge     LOOP\n\t"
11365     $$emit$$"jmp     DONE\n\t"
11366     $$emit$$"# LARGE:\n\t"
11367     if (UseXMMForObjInit) {
11368        $$emit$$"movdq   $tmp, $val\n\t"
11369        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11370        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11371        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11372        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11373        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11374        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11375        $$emit$$"add     0x40,rax\n\t"
11376        $$emit$$"# L_zero_64_bytes:\n\t"
11377        $$emit$$"sub     0x8,rcx\n\t"
11378        $$emit$$"jge     L_loop\n\t"
11379        $$emit$$"add     0x4,rcx\n\t"
11380        $$emit$$"jl      L_tail\n\t"
11381        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11382        $$emit$$"add     0x20,rax\n\t"
11383        $$emit$$"sub     0x4,rcx\n\t"
11384        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11385        $$emit$$"add     0x4,rcx\n\t"
11386        $$emit$$"jle     L_end\n\t"
11387        $$emit$$"dec     rcx\n\t"
11388        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11389        $$emit$$"vmovq   xmm0,(rax)\n\t"
11390        $$emit$$"add     0x8,rax\n\t"
11391        $$emit$$"dec     rcx\n\t"
11392        $$emit$$"jge     L_sloop\n\t"
11393        $$emit$$"# L_end:\n\t"
11394     } else {
11395        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11396     }
11397     $$emit$$"# DONE"
11398   %}
11399   ins_encode %{
11400     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11401                  $tmp$$XMMRegister, false, true);
11402   %}
11403   ins_pipe(pipe_slow);
11404 %}
11405 
11406 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11407                         Universe dummy, rFlagsReg cr)
11408 %{
11409   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only());
11410   match(Set dummy (ClearArray (Binary cnt base) val));
11411   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11412 
11413   format %{ $$template
11414     if (UseFastStosb) {
11415        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11416        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11417     } else if (UseXMMForObjInit) {
11418        $$emit$$"movdq   $tmp, $val\n\t"
11419        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11420        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11421        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11422        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11423        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11424        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11425        $$emit$$"add     0x40,rax\n\t"
11426        $$emit$$"# L_zero_64_bytes:\n\t"
11427        $$emit$$"sub     0x8,rcx\n\t"
11428        $$emit$$"jge     L_loop\n\t"
11429        $$emit$$"add     0x4,rcx\n\t"
11430        $$emit$$"jl      L_tail\n\t"
11431        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11432        $$emit$$"add     0x20,rax\n\t"
11433        $$emit$$"sub     0x4,rcx\n\t"
11434        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11435        $$emit$$"add     0x4,rcx\n\t"
11436        $$emit$$"jle     L_end\n\t"
11437        $$emit$$"dec     rcx\n\t"
11438        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11439        $$emit$$"vmovq   xmm0,(rax)\n\t"
11440        $$emit$$"add     0x8,rax\n\t"
11441        $$emit$$"dec     rcx\n\t"
11442        $$emit$$"jge     L_sloop\n\t"
11443        $$emit$$"# L_end:\n\t"
11444     } else {
11445        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11446     }
11447   %}
11448   ins_encode %{
11449     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11450                  $tmp$$XMMRegister, true, false);
11451   %}
11452   ins_pipe(pipe_slow);
11453 %}
11454 
11455 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, 
11456                         Universe dummy, rFlagsReg cr)
11457 %{
11458   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only());
11459   match(Set dummy (ClearArray (Binary cnt base) val));
11460   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11461 
11462   format %{ $$template
11463     if (UseXMMForObjInit) {
11464        $$emit$$"movdq   $tmp, $val\n\t"
11465        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11466        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11467        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11468        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11469        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11470        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11471        $$emit$$"add     0x40,rax\n\t"
11472        $$emit$$"# L_zero_64_bytes:\n\t"
11473        $$emit$$"sub     0x8,rcx\n\t"
11474        $$emit$$"jge     L_loop\n\t"
11475        $$emit$$"add     0x4,rcx\n\t"
11476        $$emit$$"jl      L_tail\n\t"
11477        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11478        $$emit$$"add     0x20,rax\n\t"
11479        $$emit$$"sub     0x4,rcx\n\t"
11480        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11481        $$emit$$"add     0x4,rcx\n\t"
11482        $$emit$$"jle     L_end\n\t"
11483        $$emit$$"dec     rcx\n\t"
11484        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11485        $$emit$$"vmovq   xmm0,(rax)\n\t"
11486        $$emit$$"add     0x8,rax\n\t"
11487        $$emit$$"dec     rcx\n\t"
11488        $$emit$$"jge     L_sloop\n\t"
11489        $$emit$$"# L_end:\n\t"
11490     } else {
11491        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11492     }
11493   %}
11494   ins_encode %{
11495     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register, 
11496                  $tmp$$XMMRegister, true, true);
11497   %}
11498   ins_pipe(pipe_slow);
11499 %}
11500 
11501 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11502                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11503 %{
11504   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11505   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11506   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11507 
11508   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11509   ins_encode %{
11510     __ string_compare($str1$$Register, $str2$$Register,
11511                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11512                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11513   %}
11514   ins_pipe( pipe_slow );
11515 %}
11516 
11517 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11518                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11519 %{
11520   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11521   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11522   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11523 
11524   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11525   ins_encode %{
11526     __ string_compare($str1$$Register, $str2$$Register,
11527                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11528                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11529   %}
11530   ins_pipe( pipe_slow );
11531 %}
11532 
11533 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11534                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11535 %{
11536   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11537   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11538   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11539 
11540   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11541   ins_encode %{
11542     __ string_compare($str1$$Register, $str2$$Register,
11543                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11544                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11545   %}
11546   ins_pipe( pipe_slow );
11547 %}
11548 
11549 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11550                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11551 %{
11552   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11553   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11554   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11555 
11556   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11557   ins_encode %{
11558     __ string_compare($str2$$Register, $str1$$Register,
11559                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11560                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11561   %}
11562   ins_pipe( pipe_slow );
11563 %}
11564 
11565 // fast search of substring with known size.
11566 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11567                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11568 %{
11569   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11570   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11571   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11572 
11573   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11574   ins_encode %{
11575     int icnt2 = (int)$int_cnt2$$constant;
11576     if (icnt2 >= 16) {
11577       // IndexOf for constant substrings with size >= 16 elements
11578       // which don't need to be loaded through stack.
11579       __ string_indexofC8($str1$$Register, $str2$$Register,
11580                           $cnt1$$Register, $cnt2$$Register,
11581                           icnt2, $result$$Register,
11582                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11583     } else {
11584       // Small strings are loaded through stack if they cross page boundary.
11585       __ string_indexof($str1$$Register, $str2$$Register,
11586                         $cnt1$$Register, $cnt2$$Register,
11587                         icnt2, $result$$Register,
11588                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11589     }
11590   %}
11591   ins_pipe( pipe_slow );
11592 %}
11593 
11594 // fast search of substring with known size.
11595 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11596                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11597 %{
11598   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11599   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11600   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11601 
11602   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11603   ins_encode %{
11604     int icnt2 = (int)$int_cnt2$$constant;
11605     if (icnt2 >= 8) {
11606       // IndexOf for constant substrings with size >= 8 elements
11607       // which don't need to be loaded through stack.
11608       __ string_indexofC8($str1$$Register, $str2$$Register,
11609                           $cnt1$$Register, $cnt2$$Register,
11610                           icnt2, $result$$Register,
11611                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11612     } else {
11613       // Small strings are loaded through stack if they cross page boundary.
11614       __ string_indexof($str1$$Register, $str2$$Register,
11615                         $cnt1$$Register, $cnt2$$Register,
11616                         icnt2, $result$$Register,
11617                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11618     }
11619   %}
11620   ins_pipe( pipe_slow );
11621 %}
11622 
11623 // fast search of substring with known size.
11624 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11625                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11626 %{
11627   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11628   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11629   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11630 
11631   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11632   ins_encode %{
11633     int icnt2 = (int)$int_cnt2$$constant;
11634     if (icnt2 >= 8) {
11635       // IndexOf for constant substrings with size >= 8 elements
11636       // which don't need to be loaded through stack.
11637       __ string_indexofC8($str1$$Register, $str2$$Register,
11638                           $cnt1$$Register, $cnt2$$Register,
11639                           icnt2, $result$$Register,
11640                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11641     } else {
11642       // Small strings are loaded through stack if they cross page boundary.
11643       __ string_indexof($str1$$Register, $str2$$Register,
11644                         $cnt1$$Register, $cnt2$$Register,
11645                         icnt2, $result$$Register,
11646                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11647     }
11648   %}
11649   ins_pipe( pipe_slow );
11650 %}
11651 
11652 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11653                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11654 %{
11655   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11656   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11657   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11658 
11659   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11660   ins_encode %{
11661     __ string_indexof($str1$$Register, $str2$$Register,
11662                       $cnt1$$Register, $cnt2$$Register,
11663                       (-1), $result$$Register,
11664                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11665   %}
11666   ins_pipe( pipe_slow );
11667 %}
11668 
11669 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11670                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11671 %{
11672   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11673   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11674   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11675 
11676   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11677   ins_encode %{
11678     __ string_indexof($str1$$Register, $str2$$Register,
11679                       $cnt1$$Register, $cnt2$$Register,
11680                       (-1), $result$$Register,
11681                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11682   %}
11683   ins_pipe( pipe_slow );
11684 %}
11685 
11686 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11687                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11688 %{
11689   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11690   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11691   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11692 
11693   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11694   ins_encode %{
11695     __ string_indexof($str1$$Register, $str2$$Register,
11696                       $cnt1$$Register, $cnt2$$Register,
11697                       (-1), $result$$Register,
11698                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11699   %}
11700   ins_pipe( pipe_slow );
11701 %}
11702 
11703 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11704                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11705 %{
11706   predicate(UseSSE42Intrinsics);
11707   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11708   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11709   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11710   ins_encode %{
11711     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11712                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11713   %}
11714   ins_pipe( pipe_slow );
11715 %}
11716 
11717 // fast string equals
11718 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11719                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11720 %{
11721   match(Set result (StrEquals (Binary str1 str2) cnt));
11722   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11723 
11724   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11725   ins_encode %{
11726     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11727                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11728                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11729   %}
11730   ins_pipe( pipe_slow );
11731 %}
11732 
11733 // fast array equals
11734 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11735                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11736 %{
11737   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11738   match(Set result (AryEq ary1 ary2));
11739   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11740 
11741   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11742   ins_encode %{
11743     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11744                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11745                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11746   %}
11747   ins_pipe( pipe_slow );
11748 %}
11749 
11750 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11751                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11752 %{
11753   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11754   match(Set result (AryEq ary1 ary2));
11755   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11756 
11757   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11758   ins_encode %{
11759     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11760                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11761                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11762   %}
11763   ins_pipe( pipe_slow );
11764 %}
11765 
11766 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11767                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11768 %{
11769   match(Set result (HasNegatives ary1 len));
11770   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11771 
11772   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11773   ins_encode %{
11774     __ has_negatives($ary1$$Register, $len$$Register,
11775                      $result$$Register, $tmp3$$Register,
11776                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11777   %}
11778   ins_pipe( pipe_slow );
11779 %}
11780 
11781 // fast char[] to byte[] compression
11782 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11783                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11784   match(Set result (StrCompressedCopy src (Binary dst len)));
11785   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11786 
11787   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11788   ins_encode %{
11789     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11790                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11791                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11792   %}
11793   ins_pipe( pipe_slow );
11794 %}
11795 
11796 // fast byte[] to char[] inflation
11797 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11798                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11799   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11800   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11801 
11802   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11803   ins_encode %{
11804     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11805                           $tmp1$$XMMRegister, $tmp2$$Register);
11806   %}
11807   ins_pipe( pipe_slow );
11808 %}
11809 
11810 // encode char[] to byte[] in ISO_8859_1
11811 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11812                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11813                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11814   match(Set result (EncodeISOArray src (Binary dst len)));
11815   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11816 
11817   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11818   ins_encode %{
11819     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11820                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11821                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11822   %}
11823   ins_pipe( pipe_slow );
11824 %}
11825 
11826 //----------Overflow Math Instructions-----------------------------------------
11827 
11828 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11829 %{
11830   match(Set cr (OverflowAddI op1 op2));
11831   effect(DEF cr, USE_KILL op1, USE op2);
11832 
11833   format %{ "addl    $op1, $op2\t# overflow check int" %}
11834 
11835   ins_encode %{
11836     __ addl($op1$$Register, $op2$$Register);
11837   %}
11838   ins_pipe(ialu_reg_reg);
11839 %}
11840 
11841 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11842 %{
11843   match(Set cr (OverflowAddI op1 op2));
11844   effect(DEF cr, USE_KILL op1, USE op2);
11845 
11846   format %{ "addl    $op1, $op2\t# overflow check int" %}
11847 
11848   ins_encode %{
11849     __ addl($op1$$Register, $op2$$constant);
11850   %}
11851   ins_pipe(ialu_reg_reg);
11852 %}
11853 
11854 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11855 %{
11856   match(Set cr (OverflowAddL op1 op2));
11857   effect(DEF cr, USE_KILL op1, USE op2);
11858 
11859   format %{ "addq    $op1, $op2\t# overflow check long" %}
11860   ins_encode %{
11861     __ addq($op1$$Register, $op2$$Register);
11862   %}
11863   ins_pipe(ialu_reg_reg);
11864 %}
11865 
11866 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11867 %{
11868   match(Set cr (OverflowAddL op1 op2));
11869   effect(DEF cr, USE_KILL op1, USE op2);
11870 
11871   format %{ "addq    $op1, $op2\t# overflow check long" %}
11872   ins_encode %{
11873     __ addq($op1$$Register, $op2$$constant);
11874   %}
11875   ins_pipe(ialu_reg_reg);
11876 %}
11877 
11878 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11879 %{
11880   match(Set cr (OverflowSubI op1 op2));
11881 
11882   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11883   ins_encode %{
11884     __ cmpl($op1$$Register, $op2$$Register);
11885   %}
11886   ins_pipe(ialu_reg_reg);
11887 %}
11888 
11889 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11890 %{
11891   match(Set cr (OverflowSubI op1 op2));
11892 
11893   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11894   ins_encode %{
11895     __ cmpl($op1$$Register, $op2$$constant);
11896   %}
11897   ins_pipe(ialu_reg_reg);
11898 %}
11899 
11900 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11901 %{
11902   match(Set cr (OverflowSubL op1 op2));
11903 
11904   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11905   ins_encode %{
11906     __ cmpq($op1$$Register, $op2$$Register);
11907   %}
11908   ins_pipe(ialu_reg_reg);
11909 %}
11910 
11911 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11912 %{
11913   match(Set cr (OverflowSubL op1 op2));
11914 
11915   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11916   ins_encode %{
11917     __ cmpq($op1$$Register, $op2$$constant);
11918   %}
11919   ins_pipe(ialu_reg_reg);
11920 %}
11921 
11922 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11923 %{
11924   match(Set cr (OverflowSubI zero op2));
11925   effect(DEF cr, USE_KILL op2);
11926 
11927   format %{ "negl    $op2\t# overflow check int" %}
11928   ins_encode %{
11929     __ negl($op2$$Register);
11930   %}
11931   ins_pipe(ialu_reg_reg);
11932 %}
11933 
11934 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11935 %{
11936   match(Set cr (OverflowSubL zero op2));
11937   effect(DEF cr, USE_KILL op2);
11938 
11939   format %{ "negq    $op2\t# overflow check long" %}
11940   ins_encode %{
11941     __ negq($op2$$Register);
11942   %}
11943   ins_pipe(ialu_reg_reg);
11944 %}
11945 
11946 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11947 %{
11948   match(Set cr (OverflowMulI op1 op2));
11949   effect(DEF cr, USE_KILL op1, USE op2);
11950 
11951   format %{ "imull    $op1, $op2\t# overflow check int" %}
11952   ins_encode %{
11953     __ imull($op1$$Register, $op2$$Register);
11954   %}
11955   ins_pipe(ialu_reg_reg_alu0);
11956 %}
11957 
11958 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11959 %{
11960   match(Set cr (OverflowMulI op1 op2));
11961   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11962 
11963   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11964   ins_encode %{
11965     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11966   %}
11967   ins_pipe(ialu_reg_reg_alu0);
11968 %}
11969 
11970 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11971 %{
11972   match(Set cr (OverflowMulL op1 op2));
11973   effect(DEF cr, USE_KILL op1, USE op2);
11974 
11975   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11976   ins_encode %{
11977     __ imulq($op1$$Register, $op2$$Register);
11978   %}
11979   ins_pipe(ialu_reg_reg_alu0);
11980 %}
11981 
11982 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11983 %{
11984   match(Set cr (OverflowMulL op1 op2));
11985   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11986 
11987   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11988   ins_encode %{
11989     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11990   %}
11991   ins_pipe(ialu_reg_reg_alu0);
11992 %}
11993 
11994 
11995 //----------Control Flow Instructions------------------------------------------
11996 // Signed compare Instructions
11997 
11998 // XXX more variants!!
11999 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
12000 %{
12001   match(Set cr (CmpI op1 op2));
12002   effect(DEF cr, USE op1, USE op2);
12003 
12004   format %{ "cmpl    $op1, $op2" %}
12005   opcode(0x3B);  /* Opcode 3B /r */
12006   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
12007   ins_pipe(ialu_cr_reg_reg);
12008 %}
12009 
12010 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
12011 %{
12012   match(Set cr (CmpI op1 op2));
12013 
12014   format %{ "cmpl    $op1, $op2" %}
12015   opcode(0x81, 0x07); /* Opcode 81 /7 */
12016   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
12017   ins_pipe(ialu_cr_reg_imm);
12018 %}
12019 
12020 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
12021 %{
12022   match(Set cr (CmpI op1 (LoadI op2)));
12023 
12024   ins_cost(500); // XXX
12025   format %{ "cmpl    $op1, $op2" %}
12026   opcode(0x3B); /* Opcode 3B /r */
12027   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
12028   ins_pipe(ialu_cr_reg_mem);
12029 %}
12030 
12031 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
12032 %{
12033   match(Set cr (CmpI src zero));
12034 
12035   format %{ "testl   $src, $src" %}
12036   opcode(0x85);
12037   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
12038   ins_pipe(ialu_cr_reg_imm);
12039 %}
12040 
12041 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
12042 %{
12043   match(Set cr (CmpI (AndI src con) zero));
12044 
12045   format %{ "testl   $src, $con" %}
12046   opcode(0xF7, 0x00);
12047   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
12048   ins_pipe(ialu_cr_reg_imm);
12049 %}
12050 
12051 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
12052 %{
12053   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
12054 
12055   format %{ "testl   $src, $mem" %}
12056   opcode(0x85);
12057   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
12058   ins_pipe(ialu_cr_reg_mem);
12059 %}
12060 
12061 // Fold array properties check
12062 instruct testI_mem_imm(rFlagsReg cr, memory mem, immI con, immI0 zero)
12063 %{
12064   match(Set cr (CmpI (AndI (CastN2I (LoadNKlass mem)) con) zero));
12065 
12066   format %{ "testl   $mem, $con" %}
12067   opcode(0xF7, 0x00);
12068   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(con));
12069   ins_pipe(ialu_mem_imm);
12070 %}
12071 















12072 // Unsigned compare Instructions; really, same as signed except they
12073 // produce an rFlagsRegU instead of rFlagsReg.
12074 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
12075 %{
12076   match(Set cr (CmpU op1 op2));
12077 
12078   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12079   opcode(0x3B); /* Opcode 3B /r */
12080   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
12081   ins_pipe(ialu_cr_reg_reg);
12082 %}
12083 
12084 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
12085 %{
12086   match(Set cr (CmpU op1 op2));
12087 
12088   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12089   opcode(0x81,0x07); /* Opcode 81 /7 */
12090   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
12091   ins_pipe(ialu_cr_reg_imm);
12092 %}
12093 
12094 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
12095 %{
12096   match(Set cr (CmpU op1 (LoadI op2)));
12097 
12098   ins_cost(500); // XXX
12099   format %{ "cmpl    $op1, $op2\t# unsigned" %}
12100   opcode(0x3B); /* Opcode 3B /r */
12101   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
12102   ins_pipe(ialu_cr_reg_mem);
12103 %}
12104 
12105 // // // Cisc-spilled version of cmpU_rReg
12106 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
12107 // //%{
12108 // //  match(Set cr (CmpU (LoadI op1) op2));
12109 // //
12110 // //  format %{ "CMPu   $op1,$op2" %}
12111 // //  ins_cost(500);
12112 // //  opcode(0x39);  /* Opcode 39 /r */
12113 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12114 // //%}
12115 
12116 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
12117 %{
12118   match(Set cr (CmpU src zero));
12119 
12120   format %{ "testl   $src, $src\t# unsigned" %}
12121   opcode(0x85);
12122   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
12123   ins_pipe(ialu_cr_reg_imm);
12124 %}
12125 
12126 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
12127 %{
12128   match(Set cr (CmpP op1 op2));
12129 
12130   format %{ "cmpq    $op1, $op2\t# ptr" %}
12131   opcode(0x3B); /* Opcode 3B /r */
12132   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12133   ins_pipe(ialu_cr_reg_reg);
12134 %}
12135 
12136 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
12137 %{
12138   match(Set cr (CmpP op1 (LoadP op2)));
12139 
12140   ins_cost(500); // XXX
12141   format %{ "cmpq    $op1, $op2\t# ptr" %}
12142   opcode(0x3B); /* Opcode 3B /r */
12143   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12144   ins_pipe(ialu_cr_reg_mem);
12145 %}
12146 
12147 // // // Cisc-spilled version of cmpP_rReg
12148 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
12149 // //%{
12150 // //  match(Set cr (CmpP (LoadP op1) op2));
12151 // //
12152 // //  format %{ "CMPu   $op1,$op2" %}
12153 // //  ins_cost(500);
12154 // //  opcode(0x39);  /* Opcode 39 /r */
12155 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12156 // //%}
12157 
12158 // XXX this is generalized by compP_rReg_mem???
12159 // Compare raw pointer (used in out-of-heap check).
12160 // Only works because non-oop pointers must be raw pointers
12161 // and raw pointers have no anti-dependencies.
12162 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12163 %{
12164   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
12165   match(Set cr (CmpP op1 (LoadP op2)));
12166 
12167   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12168   opcode(0x3B); /* Opcode 3B /r */
12169   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12170   ins_pipe(ialu_cr_reg_mem);
12171 %}
12172 
12173 // This will generate a signed flags result. This should be OK since
12174 // any compare to a zero should be eq/neq.
12175 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12176 %{
12177   match(Set cr (CmpP src zero));
12178 
12179   format %{ "testq   $src, $src\t# ptr" %}
12180   opcode(0x85);
12181   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12182   ins_pipe(ialu_cr_reg_imm);
12183 %}
12184 
12185 // This will generate a signed flags result. This should be OK since
12186 // any compare to a zero should be eq/neq.
12187 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12188 %{
12189   predicate(!UseCompressedOops || (CompressedOops::base() != NULL));
12190   match(Set cr (CmpP (LoadP op) zero));
12191 
12192   ins_cost(500); // XXX
12193   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12194   opcode(0xF7); /* Opcode F7 /0 */
12195   ins_encode(REX_mem_wide(op),
12196              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
12197   ins_pipe(ialu_cr_reg_imm);
12198 %}
12199 
12200 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12201 %{
12202   predicate(UseCompressedOops && (CompressedOops::base() == NULL) && (CompressedKlassPointers::base() == NULL));
12203   match(Set cr (CmpP (LoadP mem) zero));
12204 
12205   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12206   ins_encode %{
12207     __ cmpq(r12, $mem$$Address);
12208   %}
12209   ins_pipe(ialu_cr_reg_mem);
12210 %}
12211 
12212 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12213 %{
12214   match(Set cr (CmpN op1 op2));
12215 
12216   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12217   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12218   ins_pipe(ialu_cr_reg_reg);
12219 %}
12220 
12221 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12222 %{
12223   match(Set cr (CmpN src (LoadN mem)));
12224 
12225   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12226   ins_encode %{
12227     __ cmpl($src$$Register, $mem$$Address);
12228   %}
12229   ins_pipe(ialu_cr_reg_mem);
12230 %}
12231 
12232 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12233   match(Set cr (CmpN op1 op2));
12234 
12235   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12236   ins_encode %{
12237     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12238   %}
12239   ins_pipe(ialu_cr_reg_imm);
12240 %}
12241 
12242 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12243 %{
12244   match(Set cr (CmpN src (LoadN mem)));
12245 
12246   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12247   ins_encode %{
12248     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12249   %}
12250   ins_pipe(ialu_cr_reg_mem);
12251 %}
12252 
12253 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12254   match(Set cr (CmpN op1 op2));
12255 
12256   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12257   ins_encode %{
12258     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12259   %}
12260   ins_pipe(ialu_cr_reg_imm);
12261 %}
12262 
12263 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12264 %{
12265   match(Set cr (CmpN src (LoadNKlass mem)));
12266 
12267   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12268   ins_encode %{
12269     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12270   %}
12271   ins_pipe(ialu_cr_reg_mem);
12272 %}
12273 
12274 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12275   match(Set cr (CmpN src zero));
12276 
12277   format %{ "testl   $src, $src\t# compressed ptr" %}
12278   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12279   ins_pipe(ialu_cr_reg_imm);
12280 %}
12281 
12282 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12283 %{
12284   predicate(CompressedOops::base() != NULL);
12285   match(Set cr (CmpN (LoadN mem) zero));
12286 
12287   ins_cost(500); // XXX
12288   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12289   ins_encode %{
12290     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12291   %}
12292   ins_pipe(ialu_cr_reg_mem);
12293 %}
12294 
12295 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12296 %{
12297   predicate(CompressedOops::base() == NULL && (CompressedKlassPointers::base() == NULL));
12298   match(Set cr (CmpN (LoadN mem) zero));
12299 
12300   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12301   ins_encode %{
12302     __ cmpl(r12, $mem$$Address);
12303   %}
12304   ins_pipe(ialu_cr_reg_mem);
12305 %}
12306 
12307 // Yanked all unsigned pointer compare operations.
12308 // Pointer compares are done with CmpP which is already unsigned.
12309 
12310 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12311 %{
12312   match(Set cr (CmpL op1 op2));
12313 
12314   format %{ "cmpq    $op1, $op2" %}
12315   opcode(0x3B);  /* Opcode 3B /r */
12316   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12317   ins_pipe(ialu_cr_reg_reg);
12318 %}
12319 
12320 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12321 %{
12322   match(Set cr (CmpL op1 op2));
12323 
12324   format %{ "cmpq    $op1, $op2" %}
12325   opcode(0x81, 0x07); /* Opcode 81 /7 */
12326   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12327   ins_pipe(ialu_cr_reg_imm);
12328 %}
12329 
12330 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12331 %{
12332   match(Set cr (CmpL op1 (LoadL op2)));
12333 
12334   format %{ "cmpq    $op1, $op2" %}
12335   opcode(0x3B); /* Opcode 3B /r */
12336   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12337   ins_pipe(ialu_cr_reg_mem);
12338 %}
12339 
12340 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12341 %{
12342   match(Set cr (CmpL src zero));
12343 
12344   format %{ "testq   $src, $src" %}
12345   opcode(0x85);
12346   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12347   ins_pipe(ialu_cr_reg_imm);
12348 %}
12349 
12350 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12351 %{
12352   match(Set cr (CmpL (AndL src con) zero));
12353 
12354   format %{ "testq   $src, $con\t# long" %}
12355   opcode(0xF7, 0x00);
12356   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
12357   ins_pipe(ialu_cr_reg_imm);
12358 %}
12359 
12360 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12361 %{
12362   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12363 
12364   format %{ "testq   $src, $mem" %}
12365   opcode(0x85);
12366   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12367   ins_pipe(ialu_cr_reg_mem);
12368 %}
12369 
12370 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12371 %{
12372   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12373 
12374   format %{ "testq   $src, $mem" %}
12375   opcode(0x85);
12376   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12377   ins_pipe(ialu_cr_reg_mem);
12378 %}
12379 
12380 // Fold array properties check
12381 instruct testL_reg_mem3(rFlagsReg cr, memory mem, rRegL src, immL0 zero)
12382 %{
12383   match(Set cr (CmpL (AndL (CastP2X (LoadKlass mem)) src) zero));
12384 
12385   format %{ "testq   $src, $mem" %}
12386   opcode(0x85);
12387   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12388   ins_pipe(ialu_cr_reg_mem);
12389 %}
12390 
12391 // Manifest a CmpL result in an integer register.  Very painful.
12392 // This is the test to avoid.
12393 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12394 %{
12395   match(Set dst (CmpL3 src1 src2));
12396   effect(KILL flags);
12397 
12398   ins_cost(275); // XXX
12399   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12400             "movl    $dst, -1\n\t"
12401             "jl,s    done\n\t"
12402             "setne   $dst\n\t"
12403             "movzbl  $dst, $dst\n\t"
12404     "done:" %}
12405   ins_encode(cmpl3_flag(src1, src2, dst));
12406   ins_pipe(pipe_slow);
12407 %}
12408 
12409 // Unsigned long compare Instructions; really, same as signed long except they
12410 // produce an rFlagsRegU instead of rFlagsReg.
12411 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12412 %{
12413   match(Set cr (CmpUL op1 op2));
12414 
12415   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12416   opcode(0x3B);  /* Opcode 3B /r */
12417   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12418   ins_pipe(ialu_cr_reg_reg);
12419 %}
12420 
12421 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12422 %{
12423   match(Set cr (CmpUL op1 op2));
12424 
12425   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12426   opcode(0x81, 0x07); /* Opcode 81 /7 */
12427   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12428   ins_pipe(ialu_cr_reg_imm);
12429 %}
12430 
12431 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12432 %{
12433   match(Set cr (CmpUL op1 (LoadL op2)));
12434 
12435   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12436   opcode(0x3B); /* Opcode 3B /r */
12437   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12438   ins_pipe(ialu_cr_reg_mem);
12439 %}
12440 
12441 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12442 %{
12443   match(Set cr (CmpUL src zero));
12444 
12445   format %{ "testq   $src, $src\t# unsigned" %}
12446   opcode(0x85);
12447   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12448   ins_pipe(ialu_cr_reg_imm);
12449 %}
12450 
12451 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12452 %{
12453   match(Set cr (CmpI (LoadB mem) imm));
12454 
12455   ins_cost(125);
12456   format %{ "cmpb    $mem, $imm" %}
12457   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12458   ins_pipe(ialu_cr_reg_mem);
12459 %}
12460 
12461 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
12462 %{
12463   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12464 
12465   ins_cost(125);
12466   format %{ "testb   $mem, $imm\t# ubyte" %}
12467   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12468   ins_pipe(ialu_cr_reg_mem);
12469 %}
12470 
12471 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12472 %{
12473   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12474 
12475   ins_cost(125);
12476   format %{ "testb   $mem, $imm\t# byte" %}
12477   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12478   ins_pipe(ialu_cr_reg_mem);
12479 %}
12480 
12481 //----------Max and Min--------------------------------------------------------
12482 // Min Instructions
12483 
12484 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12485 %{
12486   effect(USE_DEF dst, USE src, USE cr);
12487 
12488   format %{ "cmovlgt $dst, $src\t# min" %}
12489   opcode(0x0F, 0x4F);
12490   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12491   ins_pipe(pipe_cmov_reg);
12492 %}
12493 
12494 
12495 instruct minI_rReg(rRegI dst, rRegI src)
12496 %{
12497   match(Set dst (MinI dst src));
12498 
12499   ins_cost(200);
12500   expand %{
12501     rFlagsReg cr;
12502     compI_rReg(cr, dst, src);
12503     cmovI_reg_g(dst, src, cr);
12504   %}
12505 %}
12506 
12507 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12508 %{
12509   effect(USE_DEF dst, USE src, USE cr);
12510 
12511   format %{ "cmovllt $dst, $src\t# max" %}
12512   opcode(0x0F, 0x4C);
12513   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12514   ins_pipe(pipe_cmov_reg);
12515 %}
12516 
12517 
12518 instruct maxI_rReg(rRegI dst, rRegI src)
12519 %{
12520   match(Set dst (MaxI dst src));
12521 
12522   ins_cost(200);
12523   expand %{
12524     rFlagsReg cr;
12525     compI_rReg(cr, dst, src);
12526     cmovI_reg_l(dst, src, cr);
12527   %}
12528 %}
12529 
12530 // ============================================================================
12531 // Branch Instructions
12532 
12533 // Jump Direct - Label defines a relative address from JMP+1
12534 instruct jmpDir(label labl)
12535 %{
12536   match(Goto);
12537   effect(USE labl);
12538 
12539   ins_cost(300);
12540   format %{ "jmp     $labl" %}
12541   size(5);
12542   ins_encode %{
12543     Label* L = $labl$$label;
12544     __ jmp(*L, false); // Always long jump
12545   %}
12546   ins_pipe(pipe_jmp);
12547 %}
12548 
12549 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12550 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12551 %{
12552   match(If cop cr);
12553   effect(USE labl);
12554 
12555   ins_cost(300);
12556   format %{ "j$cop     $labl" %}
12557   size(6);
12558   ins_encode %{
12559     Label* L = $labl$$label;
12560     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12561   %}
12562   ins_pipe(pipe_jcc);
12563 %}
12564 
12565 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12566 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12567 %{
12568   predicate(!n->has_vector_mask_set());
12569   match(CountedLoopEnd cop cr);
12570   effect(USE labl);
12571 
12572   ins_cost(300);
12573   format %{ "j$cop     $labl\t# loop end" %}
12574   size(6);
12575   ins_encode %{
12576     Label* L = $labl$$label;
12577     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12578   %}
12579   ins_pipe(pipe_jcc);
12580 %}
12581 
12582 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12583 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12584   predicate(!n->has_vector_mask_set());
12585   match(CountedLoopEnd cop cmp);
12586   effect(USE labl);
12587 
12588   ins_cost(300);
12589   format %{ "j$cop,u   $labl\t# loop end" %}
12590   size(6);
12591   ins_encode %{
12592     Label* L = $labl$$label;
12593     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12594   %}
12595   ins_pipe(pipe_jcc);
12596 %}
12597 
12598 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12599   predicate(!n->has_vector_mask_set());
12600   match(CountedLoopEnd cop cmp);
12601   effect(USE labl);
12602 
12603   ins_cost(200);
12604   format %{ "j$cop,u   $labl\t# loop end" %}
12605   size(6);
12606   ins_encode %{
12607     Label* L = $labl$$label;
12608     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12609   %}
12610   ins_pipe(pipe_jcc);
12611 %}
12612 
12613 // mask version
12614 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12615 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12616 %{
12617   predicate(n->has_vector_mask_set());
12618   match(CountedLoopEnd cop cr);
12619   effect(USE labl);
12620 
12621   ins_cost(400);
12622   format %{ "j$cop     $labl\t# loop end\n\t"
12623             "restorevectmask \t# vector mask restore for loops" %}
12624   size(10);
12625   ins_encode %{
12626     Label* L = $labl$$label;
12627     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12628     __ restorevectmask();
12629   %}
12630   ins_pipe(pipe_jcc);
12631 %}
12632 
12633 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12634 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12635   predicate(n->has_vector_mask_set());
12636   match(CountedLoopEnd cop cmp);
12637   effect(USE labl);
12638 
12639   ins_cost(400);
12640   format %{ "j$cop,u   $labl\t# loop end\n\t"
12641             "restorevectmask \t# vector mask restore for loops" %}
12642   size(10);
12643   ins_encode %{
12644     Label* L = $labl$$label;
12645     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12646     __ restorevectmask();
12647   %}
12648   ins_pipe(pipe_jcc);
12649 %}
12650 
12651 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12652   predicate(n->has_vector_mask_set());
12653   match(CountedLoopEnd cop cmp);
12654   effect(USE labl);
12655 
12656   ins_cost(300);
12657   format %{ "j$cop,u   $labl\t# loop end\n\t"
12658             "restorevectmask \t# vector mask restore for loops" %}
12659   size(10);
12660   ins_encode %{
12661     Label* L = $labl$$label;
12662     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12663     __ restorevectmask();
12664   %}
12665   ins_pipe(pipe_jcc);
12666 %}
12667 
12668 // Jump Direct Conditional - using unsigned comparison
12669 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12670   match(If cop cmp);
12671   effect(USE labl);
12672 
12673   ins_cost(300);
12674   format %{ "j$cop,u   $labl" %}
12675   size(6);
12676   ins_encode %{
12677     Label* L = $labl$$label;
12678     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12679   %}
12680   ins_pipe(pipe_jcc);
12681 %}
12682 
12683 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12684   match(If cop cmp);
12685   effect(USE labl);
12686 
12687   ins_cost(200);
12688   format %{ "j$cop,u   $labl" %}
12689   size(6);
12690   ins_encode %{
12691     Label* L = $labl$$label;
12692     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12693   %}
12694   ins_pipe(pipe_jcc);
12695 %}
12696 
12697 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12698   match(If cop cmp);
12699   effect(USE labl);
12700 
12701   ins_cost(200);
12702   format %{ $$template
12703     if ($cop$$cmpcode == Assembler::notEqual) {
12704       $$emit$$"jp,u    $labl\n\t"
12705       $$emit$$"j$cop,u   $labl"
12706     } else {
12707       $$emit$$"jp,u    done\n\t"
12708       $$emit$$"j$cop,u   $labl\n\t"
12709       $$emit$$"done:"
12710     }
12711   %}
12712   ins_encode %{
12713     Label* l = $labl$$label;
12714     if ($cop$$cmpcode == Assembler::notEqual) {
12715       __ jcc(Assembler::parity, *l, false);
12716       __ jcc(Assembler::notEqual, *l, false);
12717     } else if ($cop$$cmpcode == Assembler::equal) {
12718       Label done;
12719       __ jccb(Assembler::parity, done);
12720       __ jcc(Assembler::equal, *l, false);
12721       __ bind(done);
12722     } else {
12723        ShouldNotReachHere();
12724     }
12725   %}
12726   ins_pipe(pipe_jcc);
12727 %}
12728 
12729 // ============================================================================
12730 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12731 // superklass array for an instance of the superklass.  Set a hidden
12732 // internal cache on a hit (cache is checked with exposed code in
12733 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12734 // encoding ALSO sets flags.
12735 
12736 instruct partialSubtypeCheck(rdi_RegP result,
12737                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12738                              rFlagsReg cr)
12739 %{
12740   match(Set result (PartialSubtypeCheck sub super));
12741   effect(KILL rcx, KILL cr);
12742 
12743   ins_cost(1100);  // slightly larger than the next version
12744   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12745             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12746             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12747             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12748             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12749             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12750             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12751     "miss:\t" %}
12752 
12753   opcode(0x1); // Force a XOR of RDI
12754   ins_encode(enc_PartialSubtypeCheck());
12755   ins_pipe(pipe_slow);
12756 %}
12757 
12758 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12759                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12760                                      immP0 zero,
12761                                      rdi_RegP result)
12762 %{
12763   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12764   effect(KILL rcx, KILL result);
12765 
12766   ins_cost(1000);
12767   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12768             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12769             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12770             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12771             "jne,s   miss\t\t# Missed: flags nz\n\t"
12772             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12773     "miss:\t" %}
12774 
12775   opcode(0x0); // No need to XOR RDI
12776   ins_encode(enc_PartialSubtypeCheck());
12777   ins_pipe(pipe_slow);
12778 %}
12779 
12780 // ============================================================================
12781 // Branch Instructions -- short offset versions
12782 //
12783 // These instructions are used to replace jumps of a long offset (the default
12784 // match) with jumps of a shorter offset.  These instructions are all tagged
12785 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12786 // match rules in general matching.  Instead, the ADLC generates a conversion
12787 // method in the MachNode which can be used to do in-place replacement of the
12788 // long variant with the shorter variant.  The compiler will determine if a
12789 // branch can be taken by the is_short_branch_offset() predicate in the machine
12790 // specific code section of the file.
12791 
12792 // Jump Direct - Label defines a relative address from JMP+1
12793 instruct jmpDir_short(label labl) %{
12794   match(Goto);
12795   effect(USE labl);
12796 
12797   ins_cost(300);
12798   format %{ "jmp,s   $labl" %}
12799   size(2);
12800   ins_encode %{
12801     Label* L = $labl$$label;
12802     __ jmpb(*L);
12803   %}
12804   ins_pipe(pipe_jmp);
12805   ins_short_branch(1);
12806 %}
12807 
12808 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12809 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12810   match(If cop cr);
12811   effect(USE labl);
12812 
12813   ins_cost(300);
12814   format %{ "j$cop,s   $labl" %}
12815   size(2);
12816   ins_encode %{
12817     Label* L = $labl$$label;
12818     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12819   %}
12820   ins_pipe(pipe_jcc);
12821   ins_short_branch(1);
12822 %}
12823 
12824 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12825 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12826   match(CountedLoopEnd cop cr);
12827   effect(USE labl);
12828 
12829   ins_cost(300);
12830   format %{ "j$cop,s   $labl\t# loop end" %}
12831   size(2);
12832   ins_encode %{
12833     Label* L = $labl$$label;
12834     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12835   %}
12836   ins_pipe(pipe_jcc);
12837   ins_short_branch(1);
12838 %}
12839 
12840 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12841 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12842   match(CountedLoopEnd cop cmp);
12843   effect(USE labl);
12844 
12845   ins_cost(300);
12846   format %{ "j$cop,us  $labl\t# loop end" %}
12847   size(2);
12848   ins_encode %{
12849     Label* L = $labl$$label;
12850     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12851   %}
12852   ins_pipe(pipe_jcc);
12853   ins_short_branch(1);
12854 %}
12855 
12856 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12857   match(CountedLoopEnd cop cmp);
12858   effect(USE labl);
12859 
12860   ins_cost(300);
12861   format %{ "j$cop,us  $labl\t# loop end" %}
12862   size(2);
12863   ins_encode %{
12864     Label* L = $labl$$label;
12865     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12866   %}
12867   ins_pipe(pipe_jcc);
12868   ins_short_branch(1);
12869 %}
12870 
12871 // Jump Direct Conditional - using unsigned comparison
12872 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12873   match(If cop cmp);
12874   effect(USE labl);
12875 
12876   ins_cost(300);
12877   format %{ "j$cop,us  $labl" %}
12878   size(2);
12879   ins_encode %{
12880     Label* L = $labl$$label;
12881     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12882   %}
12883   ins_pipe(pipe_jcc);
12884   ins_short_branch(1);
12885 %}
12886 
12887 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12888   match(If cop cmp);
12889   effect(USE labl);
12890 
12891   ins_cost(300);
12892   format %{ "j$cop,us  $labl" %}
12893   size(2);
12894   ins_encode %{
12895     Label* L = $labl$$label;
12896     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12897   %}
12898   ins_pipe(pipe_jcc);
12899   ins_short_branch(1);
12900 %}
12901 
12902 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12903   match(If cop cmp);
12904   effect(USE labl);
12905 
12906   ins_cost(300);
12907   format %{ $$template
12908     if ($cop$$cmpcode == Assembler::notEqual) {
12909       $$emit$$"jp,u,s  $labl\n\t"
12910       $$emit$$"j$cop,u,s  $labl"
12911     } else {
12912       $$emit$$"jp,u,s  done\n\t"
12913       $$emit$$"j$cop,u,s  $labl\n\t"
12914       $$emit$$"done:"
12915     }
12916   %}
12917   size(4);
12918   ins_encode %{
12919     Label* l = $labl$$label;
12920     if ($cop$$cmpcode == Assembler::notEqual) {
12921       __ jccb(Assembler::parity, *l);
12922       __ jccb(Assembler::notEqual, *l);
12923     } else if ($cop$$cmpcode == Assembler::equal) {
12924       Label done;
12925       __ jccb(Assembler::parity, done);
12926       __ jccb(Assembler::equal, *l);
12927       __ bind(done);
12928     } else {
12929        ShouldNotReachHere();
12930     }
12931   %}
12932   ins_pipe(pipe_jcc);
12933   ins_short_branch(1);
12934 %}
12935 
12936 // ============================================================================
12937 // inlined locking and unlocking
12938 
12939 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12940   predicate(Compile::current()->use_rtm());
12941   match(Set cr (FastLock object box));
12942   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12943   ins_cost(300);
12944   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12945   ins_encode %{
12946     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12947                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12948                  _counters, _rtm_counters, _stack_rtm_counters,
12949                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12950                  true, ra_->C->profile_rtm());
12951   %}
12952   ins_pipe(pipe_slow);
12953 %}
12954 
12955 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12956   predicate(!Compile::current()->use_rtm());
12957   match(Set cr (FastLock object box));
12958   effect(TEMP tmp, TEMP scr, USE_KILL box);
12959   ins_cost(300);
12960   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12961   ins_encode %{
12962     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12963                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12964   %}
12965   ins_pipe(pipe_slow);
12966 %}
12967 
12968 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12969   match(Set cr (FastUnlock object box));
12970   effect(TEMP tmp, USE_KILL box);
12971   ins_cost(300);
12972   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12973   ins_encode %{
12974     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12975   %}
12976   ins_pipe(pipe_slow);
12977 %}
12978 
12979 
12980 // ============================================================================
12981 // Safepoint Instructions
12982 instruct safePoint_poll(rFlagsReg cr)
12983 %{
12984   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12985   match(SafePoint);
12986   effect(KILL cr);
12987 
12988   format %{ "testl   rax, [rip + #offset_to_poll_page]\t"
12989             "# Safepoint: poll for GC" %}
12990   ins_cost(125);
12991   ins_encode %{
12992     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12993     __ testl(rax, addr);
12994   %}
12995   ins_pipe(ialu_reg_mem);
12996 %}
12997 
12998 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12999 %{
13000   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
13001   match(SafePoint poll);
13002   effect(KILL cr, USE poll);
13003 
13004   format %{ "testl   rax, [$poll]\t"
13005             "# Safepoint: poll for GC" %}
13006   ins_cost(125);
13007   ins_encode %{
13008     __ relocate(relocInfo::poll_type);
13009     __ testl(rax, Address($poll$$Register, 0));
13010   %}
13011   ins_pipe(ialu_reg_mem);
13012 %}
13013 
13014 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
13015 %{
13016   predicate(SafepointMechanism::uses_thread_local_poll());
13017   match(SafePoint poll);
13018   effect(KILL cr, USE poll);
13019 
13020   format %{ "testl   rax, [$poll]\t"
13021             "# Safepoint: poll for GC" %}
13022   ins_cost(125);
13023   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
13024   ins_encode %{
13025     __ relocate(relocInfo::poll_type);
13026     address pre_pc = __ pc();
13027     __ testl(rax, Address($poll$$Register, 0));
13028     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
13029   %}
13030   ins_pipe(ialu_reg_mem);
13031 %}
13032 
13033 // ============================================================================
13034 // Procedure Call/Return Instructions
13035 // Call Java Static Instruction
13036 // Note: If this code changes, the corresponding ret_addr_offset() and
13037 //       compute_padding() functions will have to be adjusted.
13038 instruct CallStaticJavaDirect(method meth) %{
13039   match(CallStaticJava);
13040   effect(USE meth);
13041 
13042   ins_cost(300);
13043   format %{ "call,static " %}
13044   opcode(0xE8); /* E8 cd */
13045   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
13046   ins_pipe(pipe_slow);
13047   ins_alignment(4);
13048 %}
13049 
13050 // Call Java Dynamic Instruction
13051 // Note: If this code changes, the corresponding ret_addr_offset() and
13052 //       compute_padding() functions will have to be adjusted.
13053 instruct CallDynamicJavaDirect(method meth)
13054 %{
13055   match(CallDynamicJava);
13056   effect(USE meth);
13057 
13058   ins_cost(300);
13059   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
13060             "call,dynamic " %}
13061   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
13062   ins_pipe(pipe_slow);
13063   ins_alignment(4);
13064 %}
13065 
13066 // Call Runtime Instruction
13067 instruct CallRuntimeDirect(method meth)
13068 %{
13069   match(CallRuntime);
13070   effect(USE meth);
13071 
13072   ins_cost(300);
13073   format %{ "call,runtime " %}
13074   ins_encode(clear_avx, Java_To_Runtime(meth));
13075   ins_pipe(pipe_slow);
13076 %}
13077 
13078 // Call runtime without safepoint
13079 instruct CallLeafDirect(method meth)
13080 %{
13081   match(CallLeaf);
13082   effect(USE meth);
13083 
13084   ins_cost(300);
13085   format %{ "call_leaf,runtime " %}
13086   ins_encode(clear_avx, Java_To_Runtime(meth));
13087   ins_pipe(pipe_slow);
13088 %}
13089 
13090 // Call runtime without safepoint
13091 // entry point is null, target holds the address to call
13092 instruct CallLeafNoFPInDirect(rRegP target)
13093 %{
13094   predicate(n->as_Call()->entry_point() == NULL);
13095   match(CallLeafNoFP target);
13096 
13097   ins_cost(300);
13098   format %{ "call_leaf_nofp,runtime indirect " %}
13099   ins_encode %{
13100      __ call($target$$Register);
13101   %}
13102 
13103   ins_pipe(pipe_slow);
13104 %}
13105 
13106 instruct CallLeafNoFPDirect(method meth)
13107 %{
13108   predicate(n->as_Call()->entry_point() != NULL);
13109   match(CallLeafNoFP);
13110   effect(USE meth);
13111 
13112   ins_cost(300);
13113   format %{ "call_leaf_nofp,runtime " %}
13114   ins_encode(clear_avx, Java_To_Runtime(meth));
13115   ins_pipe(pipe_slow);
13116 %}
13117 
13118 // Return Instruction
13119 // Remove the return address & jump to it.
13120 // Notice: We always emit a nop after a ret to make sure there is room
13121 // for safepoint patching
13122 instruct Ret()
13123 %{
13124   match(Return);
13125 
13126   format %{ "ret" %}
13127   opcode(0xC3);
13128   ins_encode(OpcP);
13129   ins_pipe(pipe_jmp);
13130 %}
13131 
13132 // Tail Call; Jump from runtime stub to Java code.
13133 // Also known as an 'interprocedural jump'.
13134 // Target of jump will eventually return to caller.
13135 // TailJump below removes the return address.
13136 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
13137 %{
13138   match(TailCall jump_target method_oop);
13139 
13140   ins_cost(300);
13141   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
13142   opcode(0xFF, 0x4); /* Opcode FF /4 */
13143   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
13144   ins_pipe(pipe_jmp);
13145 %}
13146 
13147 // Tail Jump; remove the return address; jump to target.
13148 // TailCall above leaves the return address around.
13149 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
13150 %{
13151   match(TailJump jump_target ex_oop);
13152 
13153   ins_cost(300);
13154   format %{ "popq    rdx\t# pop return address\n\t"
13155             "jmp     $jump_target" %}
13156   opcode(0xFF, 0x4); /* Opcode FF /4 */
13157   ins_encode(Opcode(0x5a), // popq rdx
13158              REX_reg(jump_target), OpcP, reg_opc(jump_target));
13159   ins_pipe(pipe_jmp);
13160 %}
13161 
13162 // Create exception oop: created by stack-crawling runtime code.
13163 // Created exception is now available to this handler, and is setup
13164 // just prior to jumping to this handler.  No code emitted.
13165 instruct CreateException(rax_RegP ex_oop)
13166 %{
13167   match(Set ex_oop (CreateEx));
13168 
13169   size(0);
13170   // use the following format syntax
13171   format %{ "# exception oop is in rax; no code emitted" %}
13172   ins_encode();
13173   ins_pipe(empty);
13174 %}
13175 
13176 // Rethrow exception:
13177 // The exception oop will come in the first argument position.
13178 // Then JUMP (not call) to the rethrow stub code.
13179 instruct RethrowException()
13180 %{
13181   match(Rethrow);
13182 
13183   // use the following format syntax
13184   format %{ "jmp     rethrow_stub" %}
13185   ins_encode(enc_rethrow);
13186   ins_pipe(pipe_jmp);
13187 %}
13188 
13189 // ============================================================================
13190 // This name is KNOWN by the ADLC and cannot be changed.
13191 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13192 // for this guy.
13193 instruct tlsLoadP(r15_RegP dst) %{
13194   match(Set dst (ThreadLocal));
13195   effect(DEF dst);
13196 
13197   size(0);
13198   format %{ "# TLS is in R15" %}
13199   ins_encode( /*empty encoding*/ );
13200   ins_pipe(ialu_reg_reg);
13201 %}
13202 
13203 
13204 //----------PEEPHOLE RULES-----------------------------------------------------
13205 // These must follow all instruction definitions as they use the names
13206 // defined in the instructions definitions.
13207 //
13208 // peepmatch ( root_instr_name [preceding_instruction]* );
13209 //
13210 // peepconstraint %{
13211 // (instruction_number.operand_name relational_op instruction_number.operand_name
13212 //  [, ...] );
13213 // // instruction numbers are zero-based using left to right order in peepmatch
13214 //
13215 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13216 // // provide an instruction_number.operand_name for each operand that appears
13217 // // in the replacement instruction's match rule
13218 //
13219 // ---------VM FLAGS---------------------------------------------------------
13220 //
13221 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13222 //
13223 // Each peephole rule is given an identifying number starting with zero and
13224 // increasing by one in the order seen by the parser.  An individual peephole
13225 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13226 // on the command-line.
13227 //
13228 // ---------CURRENT LIMITATIONS----------------------------------------------
13229 //
13230 // Only match adjacent instructions in same basic block
13231 // Only equality constraints
13232 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13233 // Only one replacement instruction
13234 //
13235 // ---------EXAMPLE----------------------------------------------------------
13236 //
13237 // // pertinent parts of existing instructions in architecture description
13238 // instruct movI(rRegI dst, rRegI src)
13239 // %{
13240 //   match(Set dst (CopyI src));
13241 // %}
13242 //
13243 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
13244 // %{
13245 //   match(Set dst (AddI dst src));
13246 //   effect(KILL cr);
13247 // %}
13248 //
13249 // // Change (inc mov) to lea
13250 // peephole %{
13251 //   // increment preceeded by register-register move
13252 //   peepmatch ( incI_rReg movI );
13253 //   // require that the destination register of the increment
13254 //   // match the destination register of the move
13255 //   peepconstraint ( 0.dst == 1.dst );
13256 //   // construct a replacement instruction that sets
13257 //   // the destination to ( move's source register + one )
13258 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13259 // %}
13260 //
13261 
13262 // Implementation no longer uses movX instructions since
13263 // machine-independent system no longer uses CopyX nodes.
13264 //
13265 // peephole
13266 // %{
13267 //   peepmatch (incI_rReg movI);
13268 //   peepconstraint (0.dst == 1.dst);
13269 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13270 // %}
13271 
13272 // peephole
13273 // %{
13274 //   peepmatch (decI_rReg movI);
13275 //   peepconstraint (0.dst == 1.dst);
13276 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13277 // %}
13278 
13279 // peephole
13280 // %{
13281 //   peepmatch (addI_rReg_imm movI);
13282 //   peepconstraint (0.dst == 1.dst);
13283 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13284 // %}
13285 
13286 // peephole
13287 // %{
13288 //   peepmatch (incL_rReg movL);
13289 //   peepconstraint (0.dst == 1.dst);
13290 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13291 // %}
13292 
13293 // peephole
13294 // %{
13295 //   peepmatch (decL_rReg movL);
13296 //   peepconstraint (0.dst == 1.dst);
13297 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13298 // %}
13299 
13300 // peephole
13301 // %{
13302 //   peepmatch (addL_rReg_imm movL);
13303 //   peepconstraint (0.dst == 1.dst);
13304 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13305 // %}
13306 
13307 // peephole
13308 // %{
13309 //   peepmatch (addP_rReg_imm movP);
13310 //   peepconstraint (0.dst == 1.dst);
13311 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13312 // %}
13313 
13314 // // Change load of spilled value to only a spill
13315 // instruct storeI(memory mem, rRegI src)
13316 // %{
13317 //   match(Set mem (StoreI mem src));
13318 // %}
13319 //
13320 // instruct loadI(rRegI dst, memory mem)
13321 // %{
13322 //   match(Set dst (LoadI mem));
13323 // %}
13324 //
13325 
13326 peephole
13327 %{
13328   peepmatch (loadI storeI);
13329   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13330   peepreplace (storeI(1.mem 1.mem 1.src));
13331 %}
13332 
13333 peephole
13334 %{
13335   peepmatch (loadL storeL);
13336   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13337   peepreplace (storeL(1.mem 1.mem 1.src));
13338 %}
13339 
13340 //----------SMARTSPILL RULES---------------------------------------------------
13341 // These must follow all instruction definitions as they use the names
13342 // defined in the instructions definitions.
--- EOF ---