1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Singleton class for RAX pointer register
 262 reg_class ptr_rax_reg(RAX, RAX_H);
 263 
 264 // Singleton class for RBX pointer register
 265 reg_class ptr_rbx_reg(RBX, RBX_H);
 266 
 267 // Singleton class for RSI pointer register
 268 reg_class ptr_rsi_reg(RSI, RSI_H);
 269 
 270 // Singleton class for RDI pointer register
 271 reg_class ptr_rdi_reg(RDI, RDI_H);
 272 
 273 // Singleton class for stack pointer
 274 reg_class ptr_rsp_reg(RSP, RSP_H);
 275 
 276 // Singleton class for TLS pointer
 277 reg_class ptr_r15_reg(R15, R15_H);
 278 
 279 // Singleton class for RAX long register
 280 reg_class long_rax_reg(RAX, RAX_H);
 281 
 282 // Singleton class for RCX long register
 283 reg_class long_rcx_reg(RCX, RCX_H);
 284 
 285 // Singleton class for RDX long register
 286 reg_class long_rdx_reg(RDX, RDX_H);
 287 
 288 // Singleton class for RAX int register
 289 reg_class int_rax_reg(RAX);
 290 
 291 // Singleton class for RBX int register
 292 reg_class int_rbx_reg(RBX);
 293 
 294 // Singleton class for RCX int register
 295 reg_class int_rcx_reg(RCX);
 296 
 297 // Singleton class for RCX int register
 298 reg_class int_rdx_reg(RDX);
 299 
 300 // Singleton class for RCX int register
 301 reg_class int_rdi_reg(RDI);
 302 
 303 // Singleton class for instruction pointer
 304 // reg_class ip_reg(RIP);
 305 
 306 %}
 307 
 308 //----------SOURCE BLOCK-------------------------------------------------------
 309 // This is a block of C++ code which provides values, functions, and
 310 // definitions necessary in the rest of the architecture description
 311 source_hpp %{
 312 
 313 extern RegMask _ANY_REG_mask;
 314 extern RegMask _PTR_REG_mask;
 315 extern RegMask _PTR_REG_NO_RBP_mask;
 316 extern RegMask _PTR_NO_RAX_REG_mask;
 317 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 318 extern RegMask _LONG_REG_mask;
 319 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 320 extern RegMask _LONG_NO_RCX_REG_mask;
 321 extern RegMask _INT_REG_mask;
 322 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 323 extern RegMask _INT_NO_RCX_REG_mask;
 324 
 325 extern RegMask _STACK_OR_PTR_REG_mask;
 326 extern RegMask _STACK_OR_LONG_REG_mask;
 327 extern RegMask _STACK_OR_INT_REG_mask;
 328 
 329 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 330 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 331 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 332 
 333 %}
 334 
 335 source %{
 336 #define   RELOC_IMM64    Assembler::imm_operand
 337 #define   RELOC_DISP32   Assembler::disp32_operand
 338 
 339 #define __ _masm.
 340 
 341 RegMask _ANY_REG_mask;
 342 RegMask _PTR_REG_mask;
 343 RegMask _PTR_REG_NO_RBP_mask;
 344 RegMask _PTR_NO_RAX_REG_mask;
 345 RegMask _PTR_NO_RAX_RBX_REG_mask;
 346 RegMask _LONG_REG_mask;
 347 RegMask _LONG_NO_RAX_RDX_REG_mask;
 348 RegMask _LONG_NO_RCX_REG_mask;
 349 RegMask _INT_REG_mask;
 350 RegMask _INT_NO_RAX_RDX_REG_mask;
 351 RegMask _INT_NO_RCX_REG_mask;
 352 RegMask _STACK_OR_PTR_REG_mask;
 353 RegMask _STACK_OR_LONG_REG_mask;
 354 RegMask _STACK_OR_INT_REG_mask;
 355 
 356 static bool need_r12_heapbase() {
 357   return UseCompressedOops || UseCompressedClassPointers;
 358 }
 359 
 360 void reg_mask_init() {
 361   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 362   // We derive a number of subsets from it.
 363   _ANY_REG_mask = _ALL_REG_mask;
 364 
 365   if (PreserveFramePointer) {
 366     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 367     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 368   }
 369   if (need_r12_heapbase()) {
 370     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 371     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 372   }
 373 
 374   _PTR_REG_mask = _ANY_REG_mask;
 375   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 376   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 377   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 378   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 379 
 380   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 381   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 382 
 383   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 384   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 385   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 386 
 387   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 388   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 389   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 390 
 391   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 392   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 393   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 394 
 395   _LONG_REG_mask = _PTR_REG_mask;
 396   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 397   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 398 
 399   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 400   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 401   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 402   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 403   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 404 
 405   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 406   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 407   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 408 
 409   _INT_REG_mask = _ALL_INT_REG_mask;
 410   if (PreserveFramePointer) {
 411     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 412   }
 413   if (need_r12_heapbase()) {
 414     _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 415   }
 416 
 417   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 418   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 419 
 420   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 421   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 422   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 423 
 424   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 425   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 426 }
 427 
 428 static bool generate_vzeroupper(Compile* C) {
 429   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 430 }
 431 
 432 static int clear_avx_size() {
 433   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 434 }
 435 
 436 // !!!!! Special hack to get all types of calls to specify the byte offset
 437 //       from the start of the call to the point where the return address
 438 //       will point.
 439 int MachCallStaticJavaNode::ret_addr_offset()
 440 {
 441   int offset = 5; // 5 bytes from start of call to where return address points
 442   offset += clear_avx_size();
 443   return offset;
 444 }
 445 
 446 int MachCallDynamicJavaNode::ret_addr_offset()
 447 {
 448   int offset = 15; // 15 bytes from start of call to where return address points
 449   offset += clear_avx_size();
 450   return offset;
 451 }
 452 
 453 int MachCallRuntimeNode::ret_addr_offset() {
 454   int offset = 13; // movq r10,#addr; callq (r10)
 455   offset += clear_avx_size();
 456   return offset;
 457 }
 458 
 459 // Indicate if the safepoint node needs the polling page as an input,
 460 // it does if the polling page is more than disp32 away.
 461 bool SafePointNode::needs_polling_address_input()
 462 {
 463   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 464 }
 465 
 466 //
 467 // Compute padding required for nodes which need alignment
 468 //
 469 
 470 // The address of the call instruction needs to be 4-byte aligned to
 471 // ensure that it does not span a cache line so that it can be patched.
 472 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 473 {
 474   current_offset += clear_avx_size(); // skip vzeroupper
 475   current_offset += 1; // skip call opcode byte
 476   return align_up(current_offset, alignment_required()) - current_offset;
 477 }
 478 
 479 // The address of the call instruction needs to be 4-byte aligned to
 480 // ensure that it does not span a cache line so that it can be patched.
 481 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 482 {
 483   current_offset += clear_avx_size(); // skip vzeroupper
 484   current_offset += 11; // skip movq instruction + call opcode byte
 485   return align_up(current_offset, alignment_required()) - current_offset;
 486 }
 487 
 488 // EMIT_RM()
 489 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 490   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 491   cbuf.insts()->emit_int8(c);
 492 }
 493 
 494 // EMIT_CC()
 495 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 496   unsigned char c = (unsigned char) (f1 | f2);
 497   cbuf.insts()->emit_int8(c);
 498 }
 499 
 500 // EMIT_OPCODE()
 501 void emit_opcode(CodeBuffer &cbuf, int code) {
 502   cbuf.insts()->emit_int8((unsigned char) code);
 503 }
 504 
 505 // EMIT_OPCODE() w/ relocation information
 506 void emit_opcode(CodeBuffer &cbuf,
 507                  int code, relocInfo::relocType reloc, int offset, int format)
 508 {
 509   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 510   emit_opcode(cbuf, code);
 511 }
 512 
 513 // EMIT_D8()
 514 void emit_d8(CodeBuffer &cbuf, int d8) {
 515   cbuf.insts()->emit_int8((unsigned char) d8);
 516 }
 517 
 518 // EMIT_D16()
 519 void emit_d16(CodeBuffer &cbuf, int d16) {
 520   cbuf.insts()->emit_int16(d16);
 521 }
 522 
 523 // EMIT_D32()
 524 void emit_d32(CodeBuffer &cbuf, int d32) {
 525   cbuf.insts()->emit_int32(d32);
 526 }
 527 
 528 // EMIT_D64()
 529 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 530   cbuf.insts()->emit_int64(d64);
 531 }
 532 
 533 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 534 void emit_d32_reloc(CodeBuffer& cbuf,
 535                     int d32,
 536                     relocInfo::relocType reloc,
 537                     int format)
 538 {
 539   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 540   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 541   cbuf.insts()->emit_int32(d32);
 542 }
 543 
 544 // emit 32 bit value and construct relocation entry from RelocationHolder
 545 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 546 #ifdef ASSERT
 547   if (rspec.reloc()->type() == relocInfo::oop_type &&
 548       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 549     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 550     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 551   }
 552 #endif
 553   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 554   cbuf.insts()->emit_int32(d32);
 555 }
 556 
 557 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 558   address next_ip = cbuf.insts_end() + 4;
 559   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 560                  external_word_Relocation::spec(addr),
 561                  RELOC_DISP32);
 562 }
 563 
 564 
 565 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 566 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 567   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 568   cbuf.insts()->emit_int64(d64);
 569 }
 570 
 571 // emit 64 bit value and construct relocation entry from RelocationHolder
 572 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 573 #ifdef ASSERT
 574   if (rspec.reloc()->type() == relocInfo::oop_type &&
 575       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 576     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 577     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 578            "cannot embed scavengable oops in code");
 579   }
 580 #endif
 581   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 582   cbuf.insts()->emit_int64(d64);
 583 }
 584 
 585 // Access stack slot for load or store
 586 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 587 {
 588   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 589   if (-0x80 <= disp && disp < 0x80) {
 590     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 591     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 592     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 593   } else {
 594     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 595     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 596     emit_d32(cbuf, disp);     // Displacement // R/M byte
 597   }
 598 }
 599 
 600    // rRegI ereg, memory mem) %{    // emit_reg_mem
 601 void encode_RegMem(CodeBuffer &cbuf,
 602                    int reg,
 603                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 604 {
 605   assert(disp_reloc == relocInfo::none, "cannot have disp");
 606   int regenc = reg & 7;
 607   int baseenc = base & 7;
 608   int indexenc = index & 7;
 609 
 610   // There is no index & no scale, use form without SIB byte
 611   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 612     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 613     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 614       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 615     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 616       // If 8-bit displacement, mode 0x1
 617       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 618       emit_d8(cbuf, disp);
 619     } else {
 620       // If 32-bit displacement
 621       if (base == -1) { // Special flag for absolute address
 622         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 623         if (disp_reloc != relocInfo::none) {
 624           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 625         } else {
 626           emit_d32(cbuf, disp);
 627         }
 628       } else {
 629         // Normal base + offset
 630         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 631         if (disp_reloc != relocInfo::none) {
 632           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 633         } else {
 634           emit_d32(cbuf, disp);
 635         }
 636       }
 637     }
 638   } else {
 639     // Else, encode with the SIB byte
 640     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 641     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 642       // If no displacement
 643       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 644       emit_rm(cbuf, scale, indexenc, baseenc);
 645     } else {
 646       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 647         // If 8-bit displacement, mode 0x1
 648         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 649         emit_rm(cbuf, scale, indexenc, baseenc);
 650         emit_d8(cbuf, disp);
 651       } else {
 652         // If 32-bit displacement
 653         if (base == 0x04 ) {
 654           emit_rm(cbuf, 0x2, regenc, 0x4);
 655           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 656         } else {
 657           emit_rm(cbuf, 0x2, regenc, 0x4);
 658           emit_rm(cbuf, scale, indexenc, baseenc); // *
 659         }
 660         if (disp_reloc != relocInfo::none) {
 661           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 662         } else {
 663           emit_d32(cbuf, disp);
 664         }
 665       }
 666     }
 667   }
 668 }
 669 
 670 // This could be in MacroAssembler but it's fairly C2 specific
 671 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 672   Label exit;
 673   __ jccb(Assembler::noParity, exit);
 674   __ pushf();
 675   //
 676   // comiss/ucomiss instructions set ZF,PF,CF flags and
 677   // zero OF,AF,SF for NaN values.
 678   // Fixup flags by zeroing ZF,PF so that compare of NaN
 679   // values returns 'less than' result (CF is set).
 680   // Leave the rest of flags unchanged.
 681   //
 682   //    7 6 5 4 3 2 1 0
 683   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 684   //    0 0 1 0 1 0 1 1   (0x2B)
 685   //
 686   __ andq(Address(rsp, 0), 0xffffff2b);
 687   __ popf();
 688   __ bind(exit);
 689 }
 690 
 691 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 692   Label done;
 693   __ movl(dst, -1);
 694   __ jcc(Assembler::parity, done);
 695   __ jcc(Assembler::below, done);
 696   __ setb(Assembler::notEqual, dst);
 697   __ movzbl(dst, dst);
 698   __ bind(done);
 699 }
 700 
 701 // Math.min()    # Math.max()
 702 // --------------------------
 703 // ucomis[s/d]   #
 704 // ja   -> b     # a
 705 // jp   -> NaN   # NaN
 706 // jb   -> a     # b
 707 // je            #
 708 // |-jz -> a | b # a & b
 709 // |    -> a     #
 710 void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
 711                      XMMRegister a, XMMRegister b,
 712                      XMMRegister xmmt, Register rt,
 713                      bool min, bool single) {
 714 
 715   Label nan, zero, below, above, done;
 716 
 717   if (single)
 718     __ ucomiss(a, b);
 719   else
 720     __ ucomisd(a, b);
 721 
 722   if (dst->encoding() != (min ? b : a)->encoding())
 723     __ jccb(Assembler::above, above); // CF=0 & ZF=0
 724   else
 725     __ jccb(Assembler::above, done);
 726 
 727   __ jccb(Assembler::parity, nan);  // PF=1
 728   __ jccb(Assembler::below, below); // CF=1
 729 
 730   // equal
 731   __ vpxor(xmmt, xmmt, xmmt, Assembler::AVX_128bit);
 732   if (single) {
 733     __ ucomiss(a, xmmt);
 734     __ jccb(Assembler::equal, zero);
 735 
 736     __ movflt(dst, a);
 737     __ jmp(done);
 738   }
 739   else {
 740     __ ucomisd(a, xmmt);
 741     __ jccb(Assembler::equal, zero);
 742 
 743     __ movdbl(dst, a);
 744     __ jmp(done);
 745   }
 746 
 747   __ bind(zero);
 748   if (min)
 749     __ vpor(dst, a, b, Assembler::AVX_128bit);
 750   else
 751     __ vpand(dst, a, b, Assembler::AVX_128bit);
 752 
 753   __ jmp(done);
 754 
 755   __ bind(above);
 756   if (single)
 757     __ movflt(dst, min ? b : a);
 758   else
 759     __ movdbl(dst, min ? b : a);
 760 
 761   __ jmp(done);
 762 
 763   __ bind(nan);
 764   if (single) {
 765     __ movl(rt, 0x7fc00000); // Float.NaN
 766     __ movdl(dst, rt);
 767   }
 768   else {
 769     __ mov64(rt, 0x7ff8000000000000L); // Double.NaN
 770     __ movdq(dst, rt);
 771   }
 772   __ jmp(done);
 773 
 774   __ bind(below);
 775   if (single)
 776     __ movflt(dst, min ? a : b);
 777   else
 778     __ movdbl(dst, min ? a : b);
 779 
 780   __ bind(done);
 781 }
 782 
 783 //=============================================================================
 784 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 785 
 786 int Compile::ConstantTable::calculate_table_base_offset() const {
 787   return 0;  // absolute addressing, no offset
 788 }
 789 
 790 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 791 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 792   ShouldNotReachHere();
 793 }
 794 
 795 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 796   // Empty encoding
 797 }
 798 
 799 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 800   return 0;
 801 }
 802 
 803 #ifndef PRODUCT
 804 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 805   st->print("# MachConstantBaseNode (empty encoding)");
 806 }
 807 #endif
 808 
 809 
 810 //=============================================================================
 811 #ifndef PRODUCT
 812 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 813   Compile* C = ra_->C;
 814 
 815   int framesize = C->frame_size_in_bytes();
 816   int bangsize = C->bang_size_in_bytes();
 817   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 818   // Remove wordSize for return addr which is already pushed.
 819   framesize -= wordSize;
 820 
 821   if (C->need_stack_bang(bangsize)) {
 822     framesize -= wordSize;
 823     st->print("# stack bang (%d bytes)", bangsize);
 824     st->print("\n\t");
 825     st->print("pushq   rbp\t# Save rbp");
 826     if (PreserveFramePointer) {
 827         st->print("\n\t");
 828         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 829     }
 830     if (framesize) {
 831       st->print("\n\t");
 832       st->print("subq    rsp, #%d\t# Create frame",framesize);
 833     }
 834   } else {
 835     st->print("subq    rsp, #%d\t# Create frame",framesize);
 836     st->print("\n\t");
 837     framesize -= wordSize;
 838     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 839     if (PreserveFramePointer) {
 840       st->print("\n\t");
 841       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 842       if (framesize > 0) {
 843         st->print("\n\t");
 844         st->print("addq    rbp, #%d", framesize);
 845       }
 846     }
 847   }
 848 
 849   if (VerifyStackAtCalls) {
 850     st->print("\n\t");
 851     framesize -= wordSize;
 852     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 853 #ifdef ASSERT
 854     st->print("\n\t");
 855     st->print("# stack alignment check");
 856 #endif
 857   }
 858   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 859     st->print("\n\t");
 860     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 861     st->print("\n\t");
 862     st->print("je      fast_entry\t");
 863     st->print("\n\t");
 864     st->print("call    #nmethod_entry_barrier_stub\t");
 865     st->print("\n\tfast_entry:");
 866   }
 867   st->cr();
 868 }
 869 #endif
 870 
 871 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 872   Compile* C = ra_->C;
 873   MacroAssembler _masm(&cbuf);
 874 
 875   __ verified_entry(C);
 876   __ bind(*_verified_entry);
 877 
 878   C->set_frame_complete(cbuf.insts_size());
 879 
 880   if (C->has_mach_constant_base_node()) {
 881     // NOTE: We set the table base offset here because users might be
 882     // emitted before MachConstantBaseNode.
 883     Compile::ConstantTable& constant_table = C->constant_table();
 884     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 885   }
 886 }
 887 
 888 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 889 {
 890   return MachNode::size(ra_); // too many variables; just compute it
 891                               // the hard way
 892 }
 893 
 894 int MachPrologNode::reloc() const
 895 {
 896   return 0; // a large enough number
 897 }
 898 
 899 //=============================================================================
 900 #ifndef PRODUCT
 901 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 902 {
 903   Compile* C = ra_->C;
 904   if (generate_vzeroupper(C)) {
 905     st->print("vzeroupper");
 906     st->cr(); st->print("\t");
 907   }
 908 
 909   int framesize = C->frame_size_in_bytes();
 910   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 911   // Remove word for return adr already pushed
 912   // and RBP
 913   framesize -= 2*wordSize;
 914 
 915   if (framesize) {
 916     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 917     st->print("\t");
 918   }
 919 
 920   st->print_cr("popq   rbp");
 921   if (do_polling() && C->is_method_compilation()) {
 922     st->print("\t");
 923     if (SafepointMechanism::uses_thread_local_poll()) {
 924       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 925                    "testl  rax, [rscratch1]\t"
 926                    "# Safepoint: poll for GC");
 927     } else if (Assembler::is_polling_page_far()) {
 928       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 929                    "testl  rax, [rscratch1]\t"
 930                    "# Safepoint: poll for GC");
 931     } else {
 932       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 933                    "# Safepoint: poll for GC");
 934     }
 935   }
 936 }
 937 #endif
 938 
 939 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 940 {
 941   Compile* C = ra_->C;
 942   MacroAssembler _masm(&cbuf);
 943 
 944   if (generate_vzeroupper(C)) {
 945     // Clear upper bits of YMM registers when current compiled code uses
 946     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 947     __ vzeroupper();
 948   }
 949 
 950   __ restore_stack(C);
 951 
 952 
 953   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 954     __ reserved_stack_check();
 955   }
 956 
 957   if (do_polling() && C->is_method_compilation()) {
 958     MacroAssembler _masm(&cbuf);
 959     if (SafepointMechanism::uses_thread_local_poll()) {
 960       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
 961       __ relocate(relocInfo::poll_return_type);
 962       __ testl(rax, Address(rscratch1, 0));
 963     } else {
 964       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 965       if (Assembler::is_polling_page_far()) {
 966         __ lea(rscratch1, polling_page);
 967         __ relocate(relocInfo::poll_return_type);
 968         __ testl(rax, Address(rscratch1, 0));
 969       } else {
 970         __ testl(rax, polling_page);
 971       }
 972     }
 973   }
 974 }
 975 
 976 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 977 {
 978   return MachNode::size(ra_); // too many variables; just compute it
 979                               // the hard way
 980 }
 981 
 982 int MachEpilogNode::reloc() const
 983 {
 984   return 2; // a large enough number
 985 }
 986 
 987 const Pipeline* MachEpilogNode::pipeline() const
 988 {
 989   return MachNode::pipeline_class();
 990 }
 991 
 992 int MachEpilogNode::safepoint_offset() const
 993 {
 994   return 0;
 995 }
 996 
 997 //=============================================================================
 998 
 999 enum RC {
1000   rc_bad,
1001   rc_int,
1002   rc_float,
1003   rc_stack
1004 };
1005 
1006 static enum RC rc_class(OptoReg::Name reg)
1007 {
1008   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1009 
1010   if (OptoReg::is_stack(reg)) return rc_stack;
1011 
1012   VMReg r = OptoReg::as_VMReg(reg);
1013 
1014   if (r->is_Register()) return rc_int;
1015 
1016   assert(r->is_XMMRegister(), "must be");
1017   return rc_float;
1018 }
1019 
1020 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1021 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1022                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1023 
1024 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1025                             int stack_offset, int reg, uint ireg, outputStream* st);
1026 
1027 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1028                                       int dst_offset, uint ireg, outputStream* st) {
1029   if (cbuf) {
1030     MacroAssembler _masm(cbuf);
1031     switch (ireg) {
1032     case Op_VecS:
1033       __ movq(Address(rsp, -8), rax);
1034       __ movl(rax, Address(rsp, src_offset));
1035       __ movl(Address(rsp, dst_offset), rax);
1036       __ movq(rax, Address(rsp, -8));
1037       break;
1038     case Op_VecD:
1039       __ pushq(Address(rsp, src_offset));
1040       __ popq (Address(rsp, dst_offset));
1041       break;
1042     case Op_VecX:
1043       __ pushq(Address(rsp, src_offset));
1044       __ popq (Address(rsp, dst_offset));
1045       __ pushq(Address(rsp, src_offset+8));
1046       __ popq (Address(rsp, dst_offset+8));
1047       break;
1048     case Op_VecY:
1049       __ vmovdqu(Address(rsp, -32), xmm0);
1050       __ vmovdqu(xmm0, Address(rsp, src_offset));
1051       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1052       __ vmovdqu(xmm0, Address(rsp, -32));
1053       break;
1054     case Op_VecZ:
1055       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1056       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1057       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1058       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1059       break;
1060     default:
1061       ShouldNotReachHere();
1062     }
1063 #ifndef PRODUCT
1064   } else {
1065     switch (ireg) {
1066     case Op_VecS:
1067       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1068                 "movl    rax, [rsp + #%d]\n\t"
1069                 "movl    [rsp + #%d], rax\n\t"
1070                 "movq    rax, [rsp - #8]",
1071                 src_offset, dst_offset);
1072       break;
1073     case Op_VecD:
1074       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1075                 "popq    [rsp + #%d]",
1076                 src_offset, dst_offset);
1077       break;
1078      case Op_VecX:
1079       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1080                 "popq    [rsp + #%d]\n\t"
1081                 "pushq   [rsp + #%d]\n\t"
1082                 "popq    [rsp + #%d]",
1083                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1084       break;
1085     case Op_VecY:
1086       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1087                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1088                 "vmovdqu [rsp + #%d], xmm0\n\t"
1089                 "vmovdqu xmm0, [rsp - #32]",
1090                 src_offset, dst_offset);
1091       break;
1092     case Op_VecZ:
1093       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1094                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1095                 "vmovdqu [rsp + #%d], xmm0\n\t"
1096                 "vmovdqu xmm0, [rsp - #64]",
1097                 src_offset, dst_offset);
1098       break;
1099     default:
1100       ShouldNotReachHere();
1101     }
1102 #endif
1103   }
1104 }
1105 
1106 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1107                                        PhaseRegAlloc* ra_,
1108                                        bool do_size,
1109                                        outputStream* st) const {
1110   assert(cbuf != NULL || st  != NULL, "sanity");
1111   // Get registers to move
1112   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1113   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1114   OptoReg::Name dst_second = ra_->get_reg_second(this);
1115   OptoReg::Name dst_first = ra_->get_reg_first(this);
1116 
1117   enum RC src_second_rc = rc_class(src_second);
1118   enum RC src_first_rc = rc_class(src_first);
1119   enum RC dst_second_rc = rc_class(dst_second);
1120   enum RC dst_first_rc = rc_class(dst_first);
1121 
1122   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1123          "must move at least 1 register" );
1124 
1125   if (src_first == dst_first && src_second == dst_second) {
1126     // Self copy, no move
1127     return 0;
1128   }
1129   if (bottom_type()->isa_vect() != NULL) {
1130     uint ireg = ideal_reg();
1131     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1132     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1133     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1134       // mem -> mem
1135       int src_offset = ra_->reg2offset(src_first);
1136       int dst_offset = ra_->reg2offset(dst_first);
1137       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1138     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1139       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1140     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1141       int stack_offset = ra_->reg2offset(dst_first);
1142       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1143     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1144       int stack_offset = ra_->reg2offset(src_first);
1145       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1146     } else {
1147       ShouldNotReachHere();
1148     }
1149     return 0;
1150   }
1151   if (src_first_rc == rc_stack) {
1152     // mem ->
1153     if (dst_first_rc == rc_stack) {
1154       // mem -> mem
1155       assert(src_second != dst_first, "overlap");
1156       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1157           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1158         // 64-bit
1159         int src_offset = ra_->reg2offset(src_first);
1160         int dst_offset = ra_->reg2offset(dst_first);
1161         if (cbuf) {
1162           MacroAssembler _masm(cbuf);
1163           __ pushq(Address(rsp, src_offset));
1164           __ popq (Address(rsp, dst_offset));
1165 #ifndef PRODUCT
1166         } else {
1167           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1168                     "popq    [rsp + #%d]",
1169                      src_offset, dst_offset);
1170 #endif
1171         }
1172       } else {
1173         // 32-bit
1174         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1175         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1176         // No pushl/popl, so:
1177         int src_offset = ra_->reg2offset(src_first);
1178         int dst_offset = ra_->reg2offset(dst_first);
1179         if (cbuf) {
1180           MacroAssembler _masm(cbuf);
1181           __ movq(Address(rsp, -8), rax);
1182           __ movl(rax, Address(rsp, src_offset));
1183           __ movl(Address(rsp, dst_offset), rax);
1184           __ movq(rax, Address(rsp, -8));
1185 #ifndef PRODUCT
1186         } else {
1187           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1188                     "movl    rax, [rsp + #%d]\n\t"
1189                     "movl    [rsp + #%d], rax\n\t"
1190                     "movq    rax, [rsp - #8]",
1191                      src_offset, dst_offset);
1192 #endif
1193         }
1194       }
1195       return 0;
1196     } else if (dst_first_rc == rc_int) {
1197       // mem -> gpr
1198       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1199           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1200         // 64-bit
1201         int offset = ra_->reg2offset(src_first);
1202         if (cbuf) {
1203           MacroAssembler _masm(cbuf);
1204           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1205 #ifndef PRODUCT
1206         } else {
1207           st->print("movq    %s, [rsp + #%d]\t# spill",
1208                      Matcher::regName[dst_first],
1209                      offset);
1210 #endif
1211         }
1212       } else {
1213         // 32-bit
1214         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1215         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1216         int offset = ra_->reg2offset(src_first);
1217         if (cbuf) {
1218           MacroAssembler _masm(cbuf);
1219           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1220 #ifndef PRODUCT
1221         } else {
1222           st->print("movl    %s, [rsp + #%d]\t# spill",
1223                      Matcher::regName[dst_first],
1224                      offset);
1225 #endif
1226         }
1227       }
1228       return 0;
1229     } else if (dst_first_rc == rc_float) {
1230       // mem-> xmm
1231       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1232           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1233         // 64-bit
1234         int offset = ra_->reg2offset(src_first);
1235         if (cbuf) {
1236           MacroAssembler _masm(cbuf);
1237           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1238 #ifndef PRODUCT
1239         } else {
1240           st->print("%s  %s, [rsp + #%d]\t# spill",
1241                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1242                      Matcher::regName[dst_first],
1243                      offset);
1244 #endif
1245         }
1246       } else {
1247         // 32-bit
1248         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1249         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1250         int offset = ra_->reg2offset(src_first);
1251         if (cbuf) {
1252           MacroAssembler _masm(cbuf);
1253           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1254 #ifndef PRODUCT
1255         } else {
1256           st->print("movss   %s, [rsp + #%d]\t# spill",
1257                      Matcher::regName[dst_first],
1258                      offset);
1259 #endif
1260         }
1261       }
1262       return 0;
1263     }
1264   } else if (src_first_rc == rc_int) {
1265     // gpr ->
1266     if (dst_first_rc == rc_stack) {
1267       // gpr -> mem
1268       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1269           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1270         // 64-bit
1271         int offset = ra_->reg2offset(dst_first);
1272         if (cbuf) {
1273           MacroAssembler _masm(cbuf);
1274           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1275 #ifndef PRODUCT
1276         } else {
1277           st->print("movq    [rsp + #%d], %s\t# spill",
1278                      offset,
1279                      Matcher::regName[src_first]);
1280 #endif
1281         }
1282       } else {
1283         // 32-bit
1284         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1285         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1286         int offset = ra_->reg2offset(dst_first);
1287         if (cbuf) {
1288           MacroAssembler _masm(cbuf);
1289           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1290 #ifndef PRODUCT
1291         } else {
1292           st->print("movl    [rsp + #%d], %s\t# spill",
1293                      offset,
1294                      Matcher::regName[src_first]);
1295 #endif
1296         }
1297       }
1298       return 0;
1299     } else if (dst_first_rc == rc_int) {
1300       // gpr -> gpr
1301       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1302           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1303         // 64-bit
1304         if (cbuf) {
1305           MacroAssembler _masm(cbuf);
1306           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1307                   as_Register(Matcher::_regEncode[src_first]));
1308 #ifndef PRODUCT
1309         } else {
1310           st->print("movq    %s, %s\t# spill",
1311                      Matcher::regName[dst_first],
1312                      Matcher::regName[src_first]);
1313 #endif
1314         }
1315         return 0;
1316       } else {
1317         // 32-bit
1318         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1319         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1320         if (cbuf) {
1321           MacroAssembler _masm(cbuf);
1322           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1323                   as_Register(Matcher::_regEncode[src_first]));
1324 #ifndef PRODUCT
1325         } else {
1326           st->print("movl    %s, %s\t# spill",
1327                      Matcher::regName[dst_first],
1328                      Matcher::regName[src_first]);
1329 #endif
1330         }
1331         return 0;
1332       }
1333     } else if (dst_first_rc == rc_float) {
1334       // gpr -> xmm
1335       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1336           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1337         // 64-bit
1338         if (cbuf) {
1339           MacroAssembler _masm(cbuf);
1340           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1341 #ifndef PRODUCT
1342         } else {
1343           st->print("movdq   %s, %s\t# spill",
1344                      Matcher::regName[dst_first],
1345                      Matcher::regName[src_first]);
1346 #endif
1347         }
1348       } else {
1349         // 32-bit
1350         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1351         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1352         if (cbuf) {
1353           MacroAssembler _masm(cbuf);
1354           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1355 #ifndef PRODUCT
1356         } else {
1357           st->print("movdl   %s, %s\t# spill",
1358                      Matcher::regName[dst_first],
1359                      Matcher::regName[src_first]);
1360 #endif
1361         }
1362       }
1363       return 0;
1364     }
1365   } else if (src_first_rc == rc_float) {
1366     // xmm ->
1367     if (dst_first_rc == rc_stack) {
1368       // xmm -> mem
1369       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1370           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1371         // 64-bit
1372         int offset = ra_->reg2offset(dst_first);
1373         if (cbuf) {
1374           MacroAssembler _masm(cbuf);
1375           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1376 #ifndef PRODUCT
1377         } else {
1378           st->print("movsd   [rsp + #%d], %s\t# spill",
1379                      offset,
1380                      Matcher::regName[src_first]);
1381 #endif
1382         }
1383       } else {
1384         // 32-bit
1385         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1386         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1387         int offset = ra_->reg2offset(dst_first);
1388         if (cbuf) {
1389           MacroAssembler _masm(cbuf);
1390           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1391 #ifndef PRODUCT
1392         } else {
1393           st->print("movss   [rsp + #%d], %s\t# spill",
1394                      offset,
1395                      Matcher::regName[src_first]);
1396 #endif
1397         }
1398       }
1399       return 0;
1400     } else if (dst_first_rc == rc_int) {
1401       // xmm -> gpr
1402       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1403           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1404         // 64-bit
1405         if (cbuf) {
1406           MacroAssembler _masm(cbuf);
1407           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1408 #ifndef PRODUCT
1409         } else {
1410           st->print("movdq   %s, %s\t# spill",
1411                      Matcher::regName[dst_first],
1412                      Matcher::regName[src_first]);
1413 #endif
1414         }
1415       } else {
1416         // 32-bit
1417         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1418         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1419         if (cbuf) {
1420           MacroAssembler _masm(cbuf);
1421           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1422 #ifndef PRODUCT
1423         } else {
1424           st->print("movdl   %s, %s\t# spill",
1425                      Matcher::regName[dst_first],
1426                      Matcher::regName[src_first]);
1427 #endif
1428         }
1429       }
1430       return 0;
1431     } else if (dst_first_rc == rc_float) {
1432       // xmm -> xmm
1433       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1434           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1435         // 64-bit
1436         if (cbuf) {
1437           MacroAssembler _masm(cbuf);
1438           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1439 #ifndef PRODUCT
1440         } else {
1441           st->print("%s  %s, %s\t# spill",
1442                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1443                      Matcher::regName[dst_first],
1444                      Matcher::regName[src_first]);
1445 #endif
1446         }
1447       } else {
1448         // 32-bit
1449         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1450         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1451         if (cbuf) {
1452           MacroAssembler _masm(cbuf);
1453           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1454 #ifndef PRODUCT
1455         } else {
1456           st->print("%s  %s, %s\t# spill",
1457                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1458                      Matcher::regName[dst_first],
1459                      Matcher::regName[src_first]);
1460 #endif
1461         }
1462       }
1463       return 0;
1464     }
1465   }
1466 
1467   assert(0," foo ");
1468   Unimplemented();
1469   return 0;
1470 }
1471 
1472 #ifndef PRODUCT
1473 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1474   implementation(NULL, ra_, false, st);
1475 }
1476 #endif
1477 
1478 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1479   implementation(&cbuf, ra_, false, NULL);
1480 }
1481 
1482 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1483   return MachNode::size(ra_);
1484 }
1485 
1486 //=============================================================================
1487 #ifndef PRODUCT
1488 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1489 {
1490   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1491   int reg = ra_->get_reg_first(this);
1492   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1493             Matcher::regName[reg], offset);
1494 }
1495 #endif
1496 
1497 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1498 {
1499   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1500   int reg = ra_->get_encode(this);
1501   if (offset >= 0x80) {
1502     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1503     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1504     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1505     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1506     emit_d32(cbuf, offset);
1507   } else {
1508     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1509     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1510     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1511     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1512     emit_d8(cbuf, offset);
1513   }
1514 }
1515 
1516 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1517 {
1518   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1519   return (offset < 0x80) ? 5 : 8; // REX
1520 }
1521 
1522 //=============================================================================
1523 #ifndef PRODUCT
1524 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1525 {
1526   st->print_cr("MachVEPNode");
1527 }
1528 #endif
1529 
1530 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1531 {
1532   MacroAssembler masm(&cbuf);
1533   if (!_verified) {  
1534     uint insts_size = cbuf.insts_size();
1535     if (UseCompressedClassPointers) {
1536       masm.load_klass(rscratch1, j_rarg0);
1537       masm.cmpptr(rax, rscratch1);
1538     } else {
1539       masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1540     }
1541     masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1542   } else {
1543     // Unpack value type args passed as oop and then jump to
1544     // the verified entry point (skipping the unverified entry).
1545     masm.unpack_value_args(ra_->C, _receiver_only);
1546     masm.jmp(*_verified_entry);
1547   }
1548 }
1549 
1550 uint MachVEPNode::size(PhaseRegAlloc* ra_) const
1551 {
1552   return MachNode::size(ra_); // too many variables; just compute it the hard way
1553 }
1554 
1555 //=============================================================================
1556 #ifndef PRODUCT
1557 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1558 {
1559   if (UseCompressedClassPointers) {
1560     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1561     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1562     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1563   } else {
1564     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1565                  "# Inline cache check");
1566   }
1567   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1568   st->print_cr("\tnop\t# nops to align entry point");
1569 }
1570 #endif
1571 
1572 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1573 {
1574   MacroAssembler masm(&cbuf);
1575   uint insts_size = cbuf.insts_size();
1576   if (UseCompressedClassPointers) {
1577     masm.load_klass(rscratch1, j_rarg0);
1578     masm.cmpptr(rax, rscratch1);
1579   } else {
1580     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1581   }
1582 
1583   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1584 
1585   /* WARNING these NOPs are critical so that verified entry point is properly
1586      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1587   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1588   if (OptoBreakpoint) {
1589     // Leave space for int3
1590     nops_cnt -= 1;
1591   }
1592   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1593   if (nops_cnt > 0)
1594     masm.nop(nops_cnt);
1595 }
1596 
1597 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1598 {
1599   return MachNode::size(ra_); // too many variables; just compute it
1600                               // the hard way
1601 }
1602 
1603 
1604 //=============================================================================
1605 
1606 int Matcher::regnum_to_fpu_offset(int regnum)
1607 {
1608   return regnum - 32; // The FP registers are in the second chunk
1609 }
1610 
1611 // This is UltraSparc specific, true just means we have fast l2f conversion
1612 const bool Matcher::convL2FSupported(void) {
1613   return true;
1614 }
1615 
1616 // Is this branch offset short enough that a short branch can be used?
1617 //
1618 // NOTE: If the platform does not provide any short branch variants, then
1619 //       this method should return false for offset 0.
1620 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1621   // The passed offset is relative to address of the branch.
1622   // On 86 a branch displacement is calculated relative to address
1623   // of a next instruction.
1624   offset -= br_size;
1625 
1626   // the short version of jmpConUCF2 contains multiple branches,
1627   // making the reach slightly less
1628   if (rule == jmpConUCF2_rule)
1629     return (-126 <= offset && offset <= 125);
1630   return (-128 <= offset && offset <= 127);
1631 }
1632 
1633 const bool Matcher::isSimpleConstant64(jlong value) {
1634   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1635   //return value == (int) value;  // Cf. storeImmL and immL32.
1636 
1637   // Probably always true, even if a temp register is required.
1638   return true;
1639 }
1640 
1641 // The ecx parameter to rep stosq for the ClearArray node is in words.
1642 const bool Matcher::init_array_count_is_in_bytes = false;
1643 
1644 // No additional cost for CMOVL.
1645 const int Matcher::long_cmove_cost() { return 0; }
1646 
1647 // No CMOVF/CMOVD with SSE2
1648 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1649 
1650 // Does the CPU require late expand (see block.cpp for description of late expand)?
1651 const bool Matcher::require_postalloc_expand = false;
1652 
1653 // Do we need to mask the count passed to shift instructions or does
1654 // the cpu only look at the lower 5/6 bits anyway?
1655 const bool Matcher::need_masked_shift_count = false;
1656 
1657 bool Matcher::narrow_oop_use_complex_address() {
1658   assert(UseCompressedOops, "only for compressed oops code");
1659   return (LogMinObjAlignmentInBytes <= 3);
1660 }
1661 
1662 bool Matcher::narrow_klass_use_complex_address() {
1663   assert(UseCompressedClassPointers, "only for compressed klass code");
1664   return (LogKlassAlignmentInBytes <= 3);
1665 }
1666 
1667 bool Matcher::const_oop_prefer_decode() {
1668   // Prefer ConN+DecodeN over ConP.
1669   return true;
1670 }
1671 
1672 bool Matcher::const_klass_prefer_decode() {
1673   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1674   //       or condisider the following:
1675   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1676   //return Universe::narrow_klass_base() == NULL;
1677   return true;
1678 }
1679 
1680 // Is it better to copy float constants, or load them directly from
1681 // memory?  Intel can load a float constant from a direct address,
1682 // requiring no extra registers.  Most RISCs will have to materialize
1683 // an address into a register first, so they would do better to copy
1684 // the constant from stack.
1685 const bool Matcher::rematerialize_float_constants = true; // XXX
1686 
1687 // If CPU can load and store mis-aligned doubles directly then no
1688 // fixup is needed.  Else we split the double into 2 integer pieces
1689 // and move it piece-by-piece.  Only happens when passing doubles into
1690 // C code as the Java calling convention forces doubles to be aligned.
1691 const bool Matcher::misaligned_doubles_ok = true;
1692 
1693 // No-op on amd64
1694 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1695 
1696 // Advertise here if the CPU requires explicit rounding operations to
1697 // implement the UseStrictFP mode.
1698 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1699 
1700 // Are floats conerted to double when stored to stack during deoptimization?
1701 // On x64 it is stored without convertion so we can use normal access.
1702 bool Matcher::float_in_double() { return false; }
1703 
1704 // Do ints take an entire long register or just half?
1705 const bool Matcher::int_in_long = true;
1706 
1707 // Return whether or not this register is ever used as an argument.
1708 // This function is used on startup to build the trampoline stubs in
1709 // generateOptoStub.  Registers not mentioned will be killed by the VM
1710 // call in the trampoline, and arguments in those registers not be
1711 // available to the callee.
1712 bool Matcher::can_be_java_arg(int reg)
1713 {
1714   return
1715     reg ==  RDI_num || reg == RDI_H_num ||
1716     reg ==  RSI_num || reg == RSI_H_num ||
1717     reg ==  RDX_num || reg == RDX_H_num ||
1718     reg ==  RCX_num || reg == RCX_H_num ||
1719     reg ==   R8_num || reg ==  R8_H_num ||
1720     reg ==   R9_num || reg ==  R9_H_num ||
1721     reg ==  R12_num || reg == R12_H_num ||
1722     reg == XMM0_num || reg == XMM0b_num ||
1723     reg == XMM1_num || reg == XMM1b_num ||
1724     reg == XMM2_num || reg == XMM2b_num ||
1725     reg == XMM3_num || reg == XMM3b_num ||
1726     reg == XMM4_num || reg == XMM4b_num ||
1727     reg == XMM5_num || reg == XMM5b_num ||
1728     reg == XMM6_num || reg == XMM6b_num ||
1729     reg == XMM7_num || reg == XMM7b_num;
1730 }
1731 
1732 bool Matcher::is_spillable_arg(int reg)
1733 {
1734   return can_be_java_arg(reg);
1735 }
1736 
1737 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1738   // In 64 bit mode a code which use multiply when
1739   // devisor is constant is faster than hardware
1740   // DIV instruction (it uses MulHiL).
1741   return false;
1742 }
1743 
1744 // Register for DIVI projection of divmodI
1745 RegMask Matcher::divI_proj_mask() {
1746   return INT_RAX_REG_mask();
1747 }
1748 
1749 // Register for MODI projection of divmodI
1750 RegMask Matcher::modI_proj_mask() {
1751   return INT_RDX_REG_mask();
1752 }
1753 
1754 // Register for DIVL projection of divmodL
1755 RegMask Matcher::divL_proj_mask() {
1756   return LONG_RAX_REG_mask();
1757 }
1758 
1759 // Register for MODL projection of divmodL
1760 RegMask Matcher::modL_proj_mask() {
1761   return LONG_RDX_REG_mask();
1762 }
1763 
1764 // Register for saving SP into on method handle invokes. Not used on x86_64.
1765 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1766     return NO_REG_mask();
1767 }
1768 
1769 %}
1770 
1771 //----------ENCODING BLOCK-----------------------------------------------------
1772 // This block specifies the encoding classes used by the compiler to
1773 // output byte streams.  Encoding classes are parameterized macros
1774 // used by Machine Instruction Nodes in order to generate the bit
1775 // encoding of the instruction.  Operands specify their base encoding
1776 // interface with the interface keyword.  There are currently
1777 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1778 // COND_INTER.  REG_INTER causes an operand to generate a function
1779 // which returns its register number when queried.  CONST_INTER causes
1780 // an operand to generate a function which returns the value of the
1781 // constant when queried.  MEMORY_INTER causes an operand to generate
1782 // four functions which return the Base Register, the Index Register,
1783 // the Scale Value, and the Offset Value of the operand when queried.
1784 // COND_INTER causes an operand to generate six functions which return
1785 // the encoding code (ie - encoding bits for the instruction)
1786 // associated with each basic boolean condition for a conditional
1787 // instruction.
1788 //
1789 // Instructions specify two basic values for encoding.  Again, a
1790 // function is available to check if the constant displacement is an
1791 // oop. They use the ins_encode keyword to specify their encoding
1792 // classes (which must be a sequence of enc_class names, and their
1793 // parameters, specified in the encoding block), and they use the
1794 // opcode keyword to specify, in order, their primary, secondary, and
1795 // tertiary opcode.  Only the opcode sections which a particular
1796 // instruction needs for encoding need to be specified.
1797 encode %{
1798   // Build emit functions for each basic byte or larger field in the
1799   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1800   // from C++ code in the enc_class source block.  Emit functions will
1801   // live in the main source block for now.  In future, we can
1802   // generalize this by adding a syntax that specifies the sizes of
1803   // fields in an order, so that the adlc can build the emit functions
1804   // automagically
1805 
1806   // Emit primary opcode
1807   enc_class OpcP
1808   %{
1809     emit_opcode(cbuf, $primary);
1810   %}
1811 
1812   // Emit secondary opcode
1813   enc_class OpcS
1814   %{
1815     emit_opcode(cbuf, $secondary);
1816   %}
1817 
1818   // Emit tertiary opcode
1819   enc_class OpcT
1820   %{
1821     emit_opcode(cbuf, $tertiary);
1822   %}
1823 
1824   // Emit opcode directly
1825   enc_class Opcode(immI d8)
1826   %{
1827     emit_opcode(cbuf, $d8$$constant);
1828   %}
1829 
1830   // Emit size prefix
1831   enc_class SizePrefix
1832   %{
1833     emit_opcode(cbuf, 0x66);
1834   %}
1835 
1836   enc_class reg(rRegI reg)
1837   %{
1838     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1839   %}
1840 
1841   enc_class reg_reg(rRegI dst, rRegI src)
1842   %{
1843     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1844   %}
1845 
1846   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1847   %{
1848     emit_opcode(cbuf, $opcode$$constant);
1849     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1850   %}
1851 
1852   enc_class cdql_enc(no_rax_rdx_RegI div)
1853   %{
1854     // Full implementation of Java idiv and irem; checks for
1855     // special case as described in JVM spec., p.243 & p.271.
1856     //
1857     //         normal case                           special case
1858     //
1859     // input : rax: dividend                         min_int
1860     //         reg: divisor                          -1
1861     //
1862     // output: rax: quotient  (= rax idiv reg)       min_int
1863     //         rdx: remainder (= rax irem reg)       0
1864     //
1865     //  Code sequnce:
1866     //
1867     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1868     //    5:   75 07/08                jne    e <normal>
1869     //    7:   33 d2                   xor    %edx,%edx
1870     //  [div >= 8 -> offset + 1]
1871     //  [REX_B]
1872     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1873     //    c:   74 03/04                je     11 <done>
1874     // 000000000000000e <normal>:
1875     //    e:   99                      cltd
1876     //  [div >= 8 -> offset + 1]
1877     //  [REX_B]
1878     //    f:   f7 f9                   idiv   $div
1879     // 0000000000000011 <done>:
1880 
1881     // cmp    $0x80000000,%eax
1882     emit_opcode(cbuf, 0x3d);
1883     emit_d8(cbuf, 0x00);
1884     emit_d8(cbuf, 0x00);
1885     emit_d8(cbuf, 0x00);
1886     emit_d8(cbuf, 0x80);
1887 
1888     // jne    e <normal>
1889     emit_opcode(cbuf, 0x75);
1890     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1891 
1892     // xor    %edx,%edx
1893     emit_opcode(cbuf, 0x33);
1894     emit_d8(cbuf, 0xD2);
1895 
1896     // cmp    $0xffffffffffffffff,%ecx
1897     if ($div$$reg >= 8) {
1898       emit_opcode(cbuf, Assembler::REX_B);
1899     }
1900     emit_opcode(cbuf, 0x83);
1901     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1902     emit_d8(cbuf, 0xFF);
1903 
1904     // je     11 <done>
1905     emit_opcode(cbuf, 0x74);
1906     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1907 
1908     // <normal>
1909     // cltd
1910     emit_opcode(cbuf, 0x99);
1911 
1912     // idivl (note: must be emitted by the user of this rule)
1913     // <done>
1914   %}
1915 
1916   enc_class cdqq_enc(no_rax_rdx_RegL div)
1917   %{
1918     // Full implementation of Java ldiv and lrem; checks for
1919     // special case as described in JVM spec., p.243 & p.271.
1920     //
1921     //         normal case                           special case
1922     //
1923     // input : rax: dividend                         min_long
1924     //         reg: divisor                          -1
1925     //
1926     // output: rax: quotient  (= rax idiv reg)       min_long
1927     //         rdx: remainder (= rax irem reg)       0
1928     //
1929     //  Code sequnce:
1930     //
1931     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1932     //    7:   00 00 80
1933     //    a:   48 39 d0                cmp    %rdx,%rax
1934     //    d:   75 08                   jne    17 <normal>
1935     //    f:   33 d2                   xor    %edx,%edx
1936     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1937     //   15:   74 05                   je     1c <done>
1938     // 0000000000000017 <normal>:
1939     //   17:   48 99                   cqto
1940     //   19:   48 f7 f9                idiv   $div
1941     // 000000000000001c <done>:
1942 
1943     // mov    $0x8000000000000000,%rdx
1944     emit_opcode(cbuf, Assembler::REX_W);
1945     emit_opcode(cbuf, 0xBA);
1946     emit_d8(cbuf, 0x00);
1947     emit_d8(cbuf, 0x00);
1948     emit_d8(cbuf, 0x00);
1949     emit_d8(cbuf, 0x00);
1950     emit_d8(cbuf, 0x00);
1951     emit_d8(cbuf, 0x00);
1952     emit_d8(cbuf, 0x00);
1953     emit_d8(cbuf, 0x80);
1954 
1955     // cmp    %rdx,%rax
1956     emit_opcode(cbuf, Assembler::REX_W);
1957     emit_opcode(cbuf, 0x39);
1958     emit_d8(cbuf, 0xD0);
1959 
1960     // jne    17 <normal>
1961     emit_opcode(cbuf, 0x75);
1962     emit_d8(cbuf, 0x08);
1963 
1964     // xor    %edx,%edx
1965     emit_opcode(cbuf, 0x33);
1966     emit_d8(cbuf, 0xD2);
1967 
1968     // cmp    $0xffffffffffffffff,$div
1969     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1970     emit_opcode(cbuf, 0x83);
1971     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1972     emit_d8(cbuf, 0xFF);
1973 
1974     // je     1e <done>
1975     emit_opcode(cbuf, 0x74);
1976     emit_d8(cbuf, 0x05);
1977 
1978     // <normal>
1979     // cqto
1980     emit_opcode(cbuf, Assembler::REX_W);
1981     emit_opcode(cbuf, 0x99);
1982 
1983     // idivq (note: must be emitted by the user of this rule)
1984     // <done>
1985   %}
1986 
1987   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1988   enc_class OpcSE(immI imm)
1989   %{
1990     // Emit primary opcode and set sign-extend bit
1991     // Check for 8-bit immediate, and set sign extend bit in opcode
1992     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1993       emit_opcode(cbuf, $primary | 0x02);
1994     } else {
1995       // 32-bit immediate
1996       emit_opcode(cbuf, $primary);
1997     }
1998   %}
1999 
2000   enc_class OpcSErm(rRegI dst, immI imm)
2001   %{
2002     // OpcSEr/m
2003     int dstenc = $dst$$reg;
2004     if (dstenc >= 8) {
2005       emit_opcode(cbuf, Assembler::REX_B);
2006       dstenc -= 8;
2007     }
2008     // Emit primary opcode and set sign-extend bit
2009     // Check for 8-bit immediate, and set sign extend bit in opcode
2010     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2011       emit_opcode(cbuf, $primary | 0x02);
2012     } else {
2013       // 32-bit immediate
2014       emit_opcode(cbuf, $primary);
2015     }
2016     // Emit r/m byte with secondary opcode, after primary opcode.
2017     emit_rm(cbuf, 0x3, $secondary, dstenc);
2018   %}
2019 
2020   enc_class OpcSErm_wide(rRegL dst, immI imm)
2021   %{
2022     // OpcSEr/m
2023     int dstenc = $dst$$reg;
2024     if (dstenc < 8) {
2025       emit_opcode(cbuf, Assembler::REX_W);
2026     } else {
2027       emit_opcode(cbuf, Assembler::REX_WB);
2028       dstenc -= 8;
2029     }
2030     // Emit primary opcode and set sign-extend bit
2031     // Check for 8-bit immediate, and set sign extend bit in opcode
2032     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2033       emit_opcode(cbuf, $primary | 0x02);
2034     } else {
2035       // 32-bit immediate
2036       emit_opcode(cbuf, $primary);
2037     }
2038     // Emit r/m byte with secondary opcode, after primary opcode.
2039     emit_rm(cbuf, 0x3, $secondary, dstenc);
2040   %}
2041 
2042   enc_class Con8or32(immI imm)
2043   %{
2044     // Check for 8-bit immediate, and set sign extend bit in opcode
2045     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2046       $$$emit8$imm$$constant;
2047     } else {
2048       // 32-bit immediate
2049       $$$emit32$imm$$constant;
2050     }
2051   %}
2052 
2053   enc_class opc2_reg(rRegI dst)
2054   %{
2055     // BSWAP
2056     emit_cc(cbuf, $secondary, $dst$$reg);
2057   %}
2058 
2059   enc_class opc3_reg(rRegI dst)
2060   %{
2061     // BSWAP
2062     emit_cc(cbuf, $tertiary, $dst$$reg);
2063   %}
2064 
2065   enc_class reg_opc(rRegI div)
2066   %{
2067     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2068     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2069   %}
2070 
2071   enc_class enc_cmov(cmpOp cop)
2072   %{
2073     // CMOV
2074     $$$emit8$primary;
2075     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2076   %}
2077 
2078   enc_class enc_PartialSubtypeCheck()
2079   %{
2080     Register Rrdi = as_Register(RDI_enc); // result register
2081     Register Rrax = as_Register(RAX_enc); // super class
2082     Register Rrcx = as_Register(RCX_enc); // killed
2083     Register Rrsi = as_Register(RSI_enc); // sub class
2084     Label miss;
2085     const bool set_cond_codes = true;
2086 
2087     MacroAssembler _masm(&cbuf);
2088     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2089                                      NULL, &miss,
2090                                      /*set_cond_codes:*/ true);
2091     if ($primary) {
2092       __ xorptr(Rrdi, Rrdi);
2093     }
2094     __ bind(miss);
2095   %}
2096 
2097   enc_class clear_avx %{
2098     debug_only(int off0 = cbuf.insts_size());
2099     if (generate_vzeroupper(Compile::current())) {
2100       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2101       // Clear upper bits of YMM registers when current compiled code uses
2102       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2103       MacroAssembler _masm(&cbuf);
2104       __ vzeroupper();
2105     }
2106     debug_only(int off1 = cbuf.insts_size());
2107     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2108   %}
2109 
2110   enc_class Java_To_Runtime(method meth) %{
2111     // No relocation needed
2112     MacroAssembler _masm(&cbuf);
2113     __ mov64(r10, (int64_t) $meth$$method);
2114     __ call(r10);
2115   %}
2116 
2117   enc_class Java_To_Interpreter(method meth)
2118   %{
2119     // CALL Java_To_Interpreter
2120     // This is the instruction starting address for relocation info.
2121     cbuf.set_insts_mark();
2122     $$$emit8$primary;
2123     // CALL directly to the runtime
2124     emit_d32_reloc(cbuf,
2125                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2126                    runtime_call_Relocation::spec(),
2127                    RELOC_DISP32);
2128   %}
2129 
2130   enc_class Java_Static_Call(method meth)
2131   %{
2132     // JAVA STATIC CALL
2133     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2134     // determine who we intended to call.
2135     cbuf.set_insts_mark();
2136     $$$emit8$primary;
2137 
2138     if (!_method) {
2139       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2140                      runtime_call_Relocation::spec(),
2141                      RELOC_DISP32);
2142     } else {
2143       int method_index = resolved_method_index(cbuf);
2144       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2145                                                   : static_call_Relocation::spec(method_index);
2146       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2147                      rspec, RELOC_DISP32);
2148       // Emit stubs for static call.
2149       address mark = cbuf.insts_mark();
2150       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2151       if (stub == NULL) {
2152         ciEnv::current()->record_failure("CodeCache is full");
2153         return;
2154       }
2155 #if INCLUDE_AOT
2156       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2157 #endif
2158     }
2159   %}
2160 
2161   enc_class Java_Dynamic_Call(method meth) %{
2162     MacroAssembler _masm(&cbuf);
2163     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2164   %}
2165 
2166   enc_class Java_Compiled_Call(method meth)
2167   %{
2168     // JAVA COMPILED CALL
2169     int disp = in_bytes(Method:: from_compiled_offset());
2170 
2171     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2172     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2173 
2174     // callq *disp(%rax)
2175     cbuf.set_insts_mark();
2176     $$$emit8$primary;
2177     if (disp < 0x80) {
2178       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2179       emit_d8(cbuf, disp); // Displacement
2180     } else {
2181       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2182       emit_d32(cbuf, disp); // Displacement
2183     }
2184   %}
2185 
2186   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2187   %{
2188     // SAL, SAR, SHR
2189     int dstenc = $dst$$reg;
2190     if (dstenc >= 8) {
2191       emit_opcode(cbuf, Assembler::REX_B);
2192       dstenc -= 8;
2193     }
2194     $$$emit8$primary;
2195     emit_rm(cbuf, 0x3, $secondary, dstenc);
2196     $$$emit8$shift$$constant;
2197   %}
2198 
2199   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2200   %{
2201     // SAL, SAR, SHR
2202     int dstenc = $dst$$reg;
2203     if (dstenc < 8) {
2204       emit_opcode(cbuf, Assembler::REX_W);
2205     } else {
2206       emit_opcode(cbuf, Assembler::REX_WB);
2207       dstenc -= 8;
2208     }
2209     $$$emit8$primary;
2210     emit_rm(cbuf, 0x3, $secondary, dstenc);
2211     $$$emit8$shift$$constant;
2212   %}
2213 
2214   enc_class load_immI(rRegI dst, immI src)
2215   %{
2216     int dstenc = $dst$$reg;
2217     if (dstenc >= 8) {
2218       emit_opcode(cbuf, Assembler::REX_B);
2219       dstenc -= 8;
2220     }
2221     emit_opcode(cbuf, 0xB8 | dstenc);
2222     $$$emit32$src$$constant;
2223   %}
2224 
2225   enc_class load_immL(rRegL dst, immL src)
2226   %{
2227     int dstenc = $dst$$reg;
2228     if (dstenc < 8) {
2229       emit_opcode(cbuf, Assembler::REX_W);
2230     } else {
2231       emit_opcode(cbuf, Assembler::REX_WB);
2232       dstenc -= 8;
2233     }
2234     emit_opcode(cbuf, 0xB8 | dstenc);
2235     emit_d64(cbuf, $src$$constant);
2236   %}
2237 
2238   enc_class load_immUL32(rRegL dst, immUL32 src)
2239   %{
2240     // same as load_immI, but this time we care about zeroes in the high word
2241     int dstenc = $dst$$reg;
2242     if (dstenc >= 8) {
2243       emit_opcode(cbuf, Assembler::REX_B);
2244       dstenc -= 8;
2245     }
2246     emit_opcode(cbuf, 0xB8 | dstenc);
2247     $$$emit32$src$$constant;
2248   %}
2249 
2250   enc_class load_immL32(rRegL dst, immL32 src)
2251   %{
2252     int dstenc = $dst$$reg;
2253     if (dstenc < 8) {
2254       emit_opcode(cbuf, Assembler::REX_W);
2255     } else {
2256       emit_opcode(cbuf, Assembler::REX_WB);
2257       dstenc -= 8;
2258     }
2259     emit_opcode(cbuf, 0xC7);
2260     emit_rm(cbuf, 0x03, 0x00, dstenc);
2261     $$$emit32$src$$constant;
2262   %}
2263 
2264   enc_class load_immP31(rRegP dst, immP32 src)
2265   %{
2266     // same as load_immI, but this time we care about zeroes in the high word
2267     int dstenc = $dst$$reg;
2268     if (dstenc >= 8) {
2269       emit_opcode(cbuf, Assembler::REX_B);
2270       dstenc -= 8;
2271     }
2272     emit_opcode(cbuf, 0xB8 | dstenc);
2273     $$$emit32$src$$constant;
2274   %}
2275 
2276   enc_class load_immP(rRegP dst, immP src)
2277   %{
2278     int dstenc = $dst$$reg;
2279     if (dstenc < 8) {
2280       emit_opcode(cbuf, Assembler::REX_W);
2281     } else {
2282       emit_opcode(cbuf, Assembler::REX_WB);
2283       dstenc -= 8;
2284     }
2285     emit_opcode(cbuf, 0xB8 | dstenc);
2286     // This next line should be generated from ADLC
2287     if ($src->constant_reloc() != relocInfo::none) {
2288       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2289     } else {
2290       emit_d64(cbuf, $src$$constant);
2291     }
2292   %}
2293 
2294   enc_class Con32(immI src)
2295   %{
2296     // Output immediate
2297     $$$emit32$src$$constant;
2298   %}
2299 
2300   enc_class Con32F_as_bits(immF src)
2301   %{
2302     // Output Float immediate bits
2303     jfloat jf = $src$$constant;
2304     jint jf_as_bits = jint_cast(jf);
2305     emit_d32(cbuf, jf_as_bits);
2306   %}
2307 
2308   enc_class Con16(immI src)
2309   %{
2310     // Output immediate
2311     $$$emit16$src$$constant;
2312   %}
2313 
2314   // How is this different from Con32??? XXX
2315   enc_class Con_d32(immI src)
2316   %{
2317     emit_d32(cbuf,$src$$constant);
2318   %}
2319 
2320   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2321     // Output immediate memory reference
2322     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2323     emit_d32(cbuf, 0x00);
2324   %}
2325 
2326   enc_class lock_prefix()
2327   %{
2328     emit_opcode(cbuf, 0xF0); // lock
2329   %}
2330 
2331   enc_class REX_mem(memory mem)
2332   %{
2333     if ($mem$$base >= 8) {
2334       if ($mem$$index < 8) {
2335         emit_opcode(cbuf, Assembler::REX_B);
2336       } else {
2337         emit_opcode(cbuf, Assembler::REX_XB);
2338       }
2339     } else {
2340       if ($mem$$index >= 8) {
2341         emit_opcode(cbuf, Assembler::REX_X);
2342       }
2343     }
2344   %}
2345 
2346   enc_class REX_mem_wide(memory mem)
2347   %{
2348     if ($mem$$base >= 8) {
2349       if ($mem$$index < 8) {
2350         emit_opcode(cbuf, Assembler::REX_WB);
2351       } else {
2352         emit_opcode(cbuf, Assembler::REX_WXB);
2353       }
2354     } else {
2355       if ($mem$$index < 8) {
2356         emit_opcode(cbuf, Assembler::REX_W);
2357       } else {
2358         emit_opcode(cbuf, Assembler::REX_WX);
2359       }
2360     }
2361   %}
2362 
2363   // for byte regs
2364   enc_class REX_breg(rRegI reg)
2365   %{
2366     if ($reg$$reg >= 4) {
2367       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2368     }
2369   %}
2370 
2371   // for byte regs
2372   enc_class REX_reg_breg(rRegI dst, rRegI src)
2373   %{
2374     if ($dst$$reg < 8) {
2375       if ($src$$reg >= 4) {
2376         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2377       }
2378     } else {
2379       if ($src$$reg < 8) {
2380         emit_opcode(cbuf, Assembler::REX_R);
2381       } else {
2382         emit_opcode(cbuf, Assembler::REX_RB);
2383       }
2384     }
2385   %}
2386 
2387   // for byte regs
2388   enc_class REX_breg_mem(rRegI reg, memory mem)
2389   %{
2390     if ($reg$$reg < 8) {
2391       if ($mem$$base < 8) {
2392         if ($mem$$index >= 8) {
2393           emit_opcode(cbuf, Assembler::REX_X);
2394         } else if ($reg$$reg >= 4) {
2395           emit_opcode(cbuf, Assembler::REX);
2396         }
2397       } else {
2398         if ($mem$$index < 8) {
2399           emit_opcode(cbuf, Assembler::REX_B);
2400         } else {
2401           emit_opcode(cbuf, Assembler::REX_XB);
2402         }
2403       }
2404     } else {
2405       if ($mem$$base < 8) {
2406         if ($mem$$index < 8) {
2407           emit_opcode(cbuf, Assembler::REX_R);
2408         } else {
2409           emit_opcode(cbuf, Assembler::REX_RX);
2410         }
2411       } else {
2412         if ($mem$$index < 8) {
2413           emit_opcode(cbuf, Assembler::REX_RB);
2414         } else {
2415           emit_opcode(cbuf, Assembler::REX_RXB);
2416         }
2417       }
2418     }
2419   %}
2420 
2421   enc_class REX_reg(rRegI reg)
2422   %{
2423     if ($reg$$reg >= 8) {
2424       emit_opcode(cbuf, Assembler::REX_B);
2425     }
2426   %}
2427 
2428   enc_class REX_reg_wide(rRegI reg)
2429   %{
2430     if ($reg$$reg < 8) {
2431       emit_opcode(cbuf, Assembler::REX_W);
2432     } else {
2433       emit_opcode(cbuf, Assembler::REX_WB);
2434     }
2435   %}
2436 
2437   enc_class REX_reg_reg(rRegI dst, rRegI src)
2438   %{
2439     if ($dst$$reg < 8) {
2440       if ($src$$reg >= 8) {
2441         emit_opcode(cbuf, Assembler::REX_B);
2442       }
2443     } else {
2444       if ($src$$reg < 8) {
2445         emit_opcode(cbuf, Assembler::REX_R);
2446       } else {
2447         emit_opcode(cbuf, Assembler::REX_RB);
2448       }
2449     }
2450   %}
2451 
2452   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2453   %{
2454     if ($dst$$reg < 8) {
2455       if ($src$$reg < 8) {
2456         emit_opcode(cbuf, Assembler::REX_W);
2457       } else {
2458         emit_opcode(cbuf, Assembler::REX_WB);
2459       }
2460     } else {
2461       if ($src$$reg < 8) {
2462         emit_opcode(cbuf, Assembler::REX_WR);
2463       } else {
2464         emit_opcode(cbuf, Assembler::REX_WRB);
2465       }
2466     }
2467   %}
2468 
2469   enc_class REX_reg_mem(rRegI reg, memory mem)
2470   %{
2471     if ($reg$$reg < 8) {
2472       if ($mem$$base < 8) {
2473         if ($mem$$index >= 8) {
2474           emit_opcode(cbuf, Assembler::REX_X);
2475         }
2476       } else {
2477         if ($mem$$index < 8) {
2478           emit_opcode(cbuf, Assembler::REX_B);
2479         } else {
2480           emit_opcode(cbuf, Assembler::REX_XB);
2481         }
2482       }
2483     } else {
2484       if ($mem$$base < 8) {
2485         if ($mem$$index < 8) {
2486           emit_opcode(cbuf, Assembler::REX_R);
2487         } else {
2488           emit_opcode(cbuf, Assembler::REX_RX);
2489         }
2490       } else {
2491         if ($mem$$index < 8) {
2492           emit_opcode(cbuf, Assembler::REX_RB);
2493         } else {
2494           emit_opcode(cbuf, Assembler::REX_RXB);
2495         }
2496       }
2497     }
2498   %}
2499 
2500   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2501   %{
2502     if ($reg$$reg < 8) {
2503       if ($mem$$base < 8) {
2504         if ($mem$$index < 8) {
2505           emit_opcode(cbuf, Assembler::REX_W);
2506         } else {
2507           emit_opcode(cbuf, Assembler::REX_WX);
2508         }
2509       } else {
2510         if ($mem$$index < 8) {
2511           emit_opcode(cbuf, Assembler::REX_WB);
2512         } else {
2513           emit_opcode(cbuf, Assembler::REX_WXB);
2514         }
2515       }
2516     } else {
2517       if ($mem$$base < 8) {
2518         if ($mem$$index < 8) {
2519           emit_opcode(cbuf, Assembler::REX_WR);
2520         } else {
2521           emit_opcode(cbuf, Assembler::REX_WRX);
2522         }
2523       } else {
2524         if ($mem$$index < 8) {
2525           emit_opcode(cbuf, Assembler::REX_WRB);
2526         } else {
2527           emit_opcode(cbuf, Assembler::REX_WRXB);
2528         }
2529       }
2530     }
2531   %}
2532 
2533   enc_class reg_mem(rRegI ereg, memory mem)
2534   %{
2535     // High registers handle in encode_RegMem
2536     int reg = $ereg$$reg;
2537     int base = $mem$$base;
2538     int index = $mem$$index;
2539     int scale = $mem$$scale;
2540     int disp = $mem$$disp;
2541     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2542 
2543     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2544   %}
2545 
2546   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2547   %{
2548     int rm_byte_opcode = $rm_opcode$$constant;
2549 
2550     // High registers handle in encode_RegMem
2551     int base = $mem$$base;
2552     int index = $mem$$index;
2553     int scale = $mem$$scale;
2554     int displace = $mem$$disp;
2555 
2556     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2557                                             // working with static
2558                                             // globals
2559     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2560                   disp_reloc);
2561   %}
2562 
2563   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2564   %{
2565     int reg_encoding = $dst$$reg;
2566     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2567     int index        = 0x04;            // 0x04 indicates no index
2568     int scale        = 0x00;            // 0x00 indicates no scale
2569     int displace     = $src1$$constant; // 0x00 indicates no displacement
2570     relocInfo::relocType disp_reloc = relocInfo::none;
2571     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2572                   disp_reloc);
2573   %}
2574 
2575   enc_class neg_reg(rRegI dst)
2576   %{
2577     int dstenc = $dst$$reg;
2578     if (dstenc >= 8) {
2579       emit_opcode(cbuf, Assembler::REX_B);
2580       dstenc -= 8;
2581     }
2582     // NEG $dst
2583     emit_opcode(cbuf, 0xF7);
2584     emit_rm(cbuf, 0x3, 0x03, dstenc);
2585   %}
2586 
2587   enc_class neg_reg_wide(rRegI dst)
2588   %{
2589     int dstenc = $dst$$reg;
2590     if (dstenc < 8) {
2591       emit_opcode(cbuf, Assembler::REX_W);
2592     } else {
2593       emit_opcode(cbuf, Assembler::REX_WB);
2594       dstenc -= 8;
2595     }
2596     // NEG $dst
2597     emit_opcode(cbuf, 0xF7);
2598     emit_rm(cbuf, 0x3, 0x03, dstenc);
2599   %}
2600 
2601   enc_class setLT_reg(rRegI dst)
2602   %{
2603     int dstenc = $dst$$reg;
2604     if (dstenc >= 8) {
2605       emit_opcode(cbuf, Assembler::REX_B);
2606       dstenc -= 8;
2607     } else if (dstenc >= 4) {
2608       emit_opcode(cbuf, Assembler::REX);
2609     }
2610     // SETLT $dst
2611     emit_opcode(cbuf, 0x0F);
2612     emit_opcode(cbuf, 0x9C);
2613     emit_rm(cbuf, 0x3, 0x0, dstenc);
2614   %}
2615 
2616   enc_class setNZ_reg(rRegI dst)
2617   %{
2618     int dstenc = $dst$$reg;
2619     if (dstenc >= 8) {
2620       emit_opcode(cbuf, Assembler::REX_B);
2621       dstenc -= 8;
2622     } else if (dstenc >= 4) {
2623       emit_opcode(cbuf, Assembler::REX);
2624     }
2625     // SETNZ $dst
2626     emit_opcode(cbuf, 0x0F);
2627     emit_opcode(cbuf, 0x95);
2628     emit_rm(cbuf, 0x3, 0x0, dstenc);
2629   %}
2630 
2631 
2632   // Compare the lonogs and set -1, 0, or 1 into dst
2633   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2634   %{
2635     int src1enc = $src1$$reg;
2636     int src2enc = $src2$$reg;
2637     int dstenc = $dst$$reg;
2638 
2639     // cmpq $src1, $src2
2640     if (src1enc < 8) {
2641       if (src2enc < 8) {
2642         emit_opcode(cbuf, Assembler::REX_W);
2643       } else {
2644         emit_opcode(cbuf, Assembler::REX_WB);
2645       }
2646     } else {
2647       if (src2enc < 8) {
2648         emit_opcode(cbuf, Assembler::REX_WR);
2649       } else {
2650         emit_opcode(cbuf, Assembler::REX_WRB);
2651       }
2652     }
2653     emit_opcode(cbuf, 0x3B);
2654     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2655 
2656     // movl $dst, -1
2657     if (dstenc >= 8) {
2658       emit_opcode(cbuf, Assembler::REX_B);
2659     }
2660     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2661     emit_d32(cbuf, -1);
2662 
2663     // jl,s done
2664     emit_opcode(cbuf, 0x7C);
2665     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2666 
2667     // setne $dst
2668     if (dstenc >= 4) {
2669       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2670     }
2671     emit_opcode(cbuf, 0x0F);
2672     emit_opcode(cbuf, 0x95);
2673     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2674 
2675     // movzbl $dst, $dst
2676     if (dstenc >= 4) {
2677       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2678     }
2679     emit_opcode(cbuf, 0x0F);
2680     emit_opcode(cbuf, 0xB6);
2681     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2682   %}
2683 
2684   enc_class Push_ResultXD(regD dst) %{
2685     MacroAssembler _masm(&cbuf);
2686     __ fstp_d(Address(rsp, 0));
2687     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2688     __ addptr(rsp, 8);
2689   %}
2690 
2691   enc_class Push_SrcXD(regD src) %{
2692     MacroAssembler _masm(&cbuf);
2693     __ subptr(rsp, 8);
2694     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2695     __ fld_d(Address(rsp, 0));
2696   %}
2697 
2698 
2699   enc_class enc_rethrow()
2700   %{
2701     cbuf.set_insts_mark();
2702     emit_opcode(cbuf, 0xE9); // jmp entry
2703     emit_d32_reloc(cbuf,
2704                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2705                    runtime_call_Relocation::spec(),
2706                    RELOC_DISP32);
2707   %}
2708 
2709 %}
2710 
2711 
2712 
2713 //----------FRAME--------------------------------------------------------------
2714 // Definition of frame structure and management information.
2715 //
2716 //  S T A C K   L A Y O U T    Allocators stack-slot number
2717 //                             |   (to get allocators register number
2718 //  G  Owned by    |        |  v    add OptoReg::stack0())
2719 //  r   CALLER     |        |
2720 //  o     |        +--------+      pad to even-align allocators stack-slot
2721 //  w     V        |  pad0  |        numbers; owned by CALLER
2722 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2723 //  h     ^        |   in   |  5
2724 //        |        |  args  |  4   Holes in incoming args owned by SELF
2725 //  |     |        |        |  3
2726 //  |     |        +--------+
2727 //  V     |        | old out|      Empty on Intel, window on Sparc
2728 //        |    old |preserve|      Must be even aligned.
2729 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2730 //        |        |   in   |  3   area for Intel ret address
2731 //     Owned by    |preserve|      Empty on Sparc.
2732 //       SELF      +--------+
2733 //        |        |  pad2  |  2   pad to align old SP
2734 //        |        +--------+  1
2735 //        |        | locks  |  0
2736 //        |        +--------+----> OptoReg::stack0(), even aligned
2737 //        |        |  pad1  | 11   pad to align new SP
2738 //        |        +--------+
2739 //        |        |        | 10
2740 //        |        | spills |  9   spills
2741 //        V        |        |  8   (pad0 slot for callee)
2742 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2743 //        ^        |  out   |  7
2744 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2745 //     Owned by    +--------+
2746 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2747 //        |    new |preserve|      Must be even-aligned.
2748 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2749 //        |        |        |
2750 //
2751 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2752 //         known from SELF's arguments and the Java calling convention.
2753 //         Region 6-7 is determined per call site.
2754 // Note 2: If the calling convention leaves holes in the incoming argument
2755 //         area, those holes are owned by SELF.  Holes in the outgoing area
2756 //         are owned by the CALLEE.  Holes should not be nessecary in the
2757 //         incoming area, as the Java calling convention is completely under
2758 //         the control of the AD file.  Doubles can be sorted and packed to
2759 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2760 //         varargs C calling conventions.
2761 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2762 //         even aligned with pad0 as needed.
2763 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2764 //         region 6-11 is even aligned; it may be padded out more so that
2765 //         the region from SP to FP meets the minimum stack alignment.
2766 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2767 //         alignment.  Region 11, pad1, may be dynamically extended so that
2768 //         SP meets the minimum alignment.
2769 
2770 frame
2771 %{
2772   // What direction does stack grow in (assumed to be same for C & Java)
2773   stack_direction(TOWARDS_LOW);
2774 
2775   // These three registers define part of the calling convention
2776   // between compiled code and the interpreter.
2777   inline_cache_reg(RAX);                // Inline Cache Register
2778   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2779                                         // calling interpreter
2780 
2781   // Optional: name the operand used by cisc-spilling to access
2782   // [stack_pointer + offset]
2783   cisc_spilling_operand_name(indOffset32);
2784 
2785   // Number of stack slots consumed by locking an object
2786   sync_stack_slots(2);
2787 
2788   // Compiled code's Frame Pointer
2789   frame_pointer(RSP);
2790 
2791   // Interpreter stores its frame pointer in a register which is
2792   // stored to the stack by I2CAdaptors.
2793   // I2CAdaptors convert from interpreted java to compiled java.
2794   interpreter_frame_pointer(RBP);
2795 
2796   // Stack alignment requirement
2797   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2798 
2799   // Number of stack slots between incoming argument block and the start of
2800   // a new frame.  The PROLOG must add this many slots to the stack.  The
2801   // EPILOG must remove this many slots.  amd64 needs two slots for
2802   // return address.
2803   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2804 
2805   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2806   // for calls to C.  Supports the var-args backing area for register parms.
2807   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2808 
2809   // The after-PROLOG location of the return address.  Location of
2810   // return address specifies a type (REG or STACK) and a number
2811   // representing the register number (i.e. - use a register name) or
2812   // stack slot.
2813   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2814   // Otherwise, it is above the locks and verification slot and alignment word
2815   return_addr(STACK - 2 +
2816               align_up((Compile::current()->in_preserve_stack_slots() +
2817                         Compile::current()->fixed_slots()),
2818                        stack_alignment_in_slots()));
2819 
2820   // Body of function which returns an integer array locating
2821   // arguments either in registers or in stack slots.  Passed an array
2822   // of ideal registers called "sig" and a "length" count.  Stack-slot
2823   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2824   // arguments for a CALLEE.  Incoming stack arguments are
2825   // automatically biased by the preserve_stack_slots field above.
2826 
2827   calling_convention
2828   %{
2829     // No difference between ingoing/outgoing just pass false
2830     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2831   %}
2832 
2833   c_calling_convention
2834   %{
2835     // This is obviously always outgoing
2836     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2837   %}
2838 
2839   // Location of compiled Java return values.  Same as C for now.
2840   return_value
2841   %{
2842     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2843            "only return normal values");
2844 
2845     static const int lo[Op_RegL + 1] = {
2846       0,
2847       0,
2848       RAX_num,  // Op_RegN
2849       RAX_num,  // Op_RegI
2850       RAX_num,  // Op_RegP
2851       XMM0_num, // Op_RegF
2852       XMM0_num, // Op_RegD
2853       RAX_num   // Op_RegL
2854     };
2855     static const int hi[Op_RegL + 1] = {
2856       0,
2857       0,
2858       OptoReg::Bad, // Op_RegN
2859       OptoReg::Bad, // Op_RegI
2860       RAX_H_num,    // Op_RegP
2861       OptoReg::Bad, // Op_RegF
2862       XMM0b_num,    // Op_RegD
2863       RAX_H_num     // Op_RegL
2864     };
2865     // Excluded flags and vector registers.
2866     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2867     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2868   %}
2869 %}
2870 
2871 //----------ATTRIBUTES---------------------------------------------------------
2872 //----------Operand Attributes-------------------------------------------------
2873 op_attrib op_cost(0);        // Required cost attribute
2874 
2875 //----------Instruction Attributes---------------------------------------------
2876 ins_attrib ins_cost(100);       // Required cost attribute
2877 ins_attrib ins_size(8);         // Required size attribute (in bits)
2878 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2879                                 // a non-matching short branch variant
2880                                 // of some long branch?
2881 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2882                                 // be a power of 2) specifies the
2883                                 // alignment that some part of the
2884                                 // instruction (not necessarily the
2885                                 // start) requires.  If > 1, a
2886                                 // compute_padding() function must be
2887                                 // provided for the instruction
2888 
2889 //----------OPERANDS-----------------------------------------------------------
2890 // Operand definitions must precede instruction definitions for correct parsing
2891 // in the ADLC because operands constitute user defined types which are used in
2892 // instruction definitions.
2893 
2894 //----------Simple Operands----------------------------------------------------
2895 // Immediate Operands
2896 // Integer Immediate
2897 operand immI()
2898 %{
2899   match(ConI);
2900 
2901   op_cost(10);
2902   format %{ %}
2903   interface(CONST_INTER);
2904 %}
2905 
2906 // Constant for test vs zero
2907 operand immI0()
2908 %{
2909   predicate(n->get_int() == 0);
2910   match(ConI);
2911 
2912   op_cost(0);
2913   format %{ %}
2914   interface(CONST_INTER);
2915 %}
2916 
2917 // Constant for increment
2918 operand immI1()
2919 %{
2920   predicate(n->get_int() == 1);
2921   match(ConI);
2922 
2923   op_cost(0);
2924   format %{ %}
2925   interface(CONST_INTER);
2926 %}
2927 
2928 // Constant for decrement
2929 operand immI_M1()
2930 %{
2931   predicate(n->get_int() == -1);
2932   match(ConI);
2933 
2934   op_cost(0);
2935   format %{ %}
2936   interface(CONST_INTER);
2937 %}
2938 
2939 // Valid scale values for addressing modes
2940 operand immI2()
2941 %{
2942   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2943   match(ConI);
2944 
2945   format %{ %}
2946   interface(CONST_INTER);
2947 %}
2948 
2949 operand immI8()
2950 %{
2951   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2952   match(ConI);
2953 
2954   op_cost(5);
2955   format %{ %}
2956   interface(CONST_INTER);
2957 %}
2958 
2959 operand immU8()
2960 %{
2961   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2962   match(ConI);
2963 
2964   op_cost(5);
2965   format %{ %}
2966   interface(CONST_INTER);
2967 %}
2968 
2969 operand immI16()
2970 %{
2971   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2972   match(ConI);
2973 
2974   op_cost(10);
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 // Int Immediate non-negative
2980 operand immU31()
2981 %{
2982   predicate(n->get_int() >= 0);
2983   match(ConI);
2984 
2985   op_cost(0);
2986   format %{ %}
2987   interface(CONST_INTER);
2988 %}
2989 
2990 // Constant for long shifts
2991 operand immI_32()
2992 %{
2993   predicate( n->get_int() == 32 );
2994   match(ConI);
2995 
2996   op_cost(0);
2997   format %{ %}
2998   interface(CONST_INTER);
2999 %}
3000 
3001 // Constant for long shifts
3002 operand immI_64()
3003 %{
3004   predicate( n->get_int() == 64 );
3005   match(ConI);
3006 
3007   op_cost(0);
3008   format %{ %}
3009   interface(CONST_INTER);
3010 %}
3011 
3012 // Pointer Immediate
3013 operand immP()
3014 %{
3015   match(ConP);
3016 
3017   op_cost(10);
3018   format %{ %}
3019   interface(CONST_INTER);
3020 %}
3021 
3022 // NULL Pointer Immediate
3023 operand immP0()
3024 %{
3025   predicate(n->get_ptr() == 0);
3026   match(ConP);
3027 
3028   op_cost(5);
3029   format %{ %}
3030   interface(CONST_INTER);
3031 %}
3032 
3033 // Pointer Immediate
3034 operand immN() %{
3035   match(ConN);
3036 
3037   op_cost(10);
3038   format %{ %}
3039   interface(CONST_INTER);
3040 %}
3041 
3042 operand immNKlass() %{
3043   match(ConNKlass);
3044 
3045   op_cost(10);
3046   format %{ %}
3047   interface(CONST_INTER);
3048 %}
3049 
3050 // NULL Pointer Immediate
3051 operand immN0() %{
3052   predicate(n->get_narrowcon() == 0);
3053   match(ConN);
3054 
3055   op_cost(5);
3056   format %{ %}
3057   interface(CONST_INTER);
3058 %}
3059 
3060 operand immP31()
3061 %{
3062   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3063             && (n->get_ptr() >> 31) == 0);
3064   match(ConP);
3065 
3066   op_cost(5);
3067   format %{ %}
3068   interface(CONST_INTER);
3069 %}
3070 
3071 
3072 // Long Immediate
3073 operand immL()
3074 %{
3075   match(ConL);
3076 
3077   op_cost(20);
3078   format %{ %}
3079   interface(CONST_INTER);
3080 %}
3081 
3082 // Long Immediate 8-bit
3083 operand immL8()
3084 %{
3085   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3086   match(ConL);
3087 
3088   op_cost(5);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Long Immediate 32-bit unsigned
3094 operand immUL32()
3095 %{
3096   predicate(n->get_long() == (unsigned int) (n->get_long()));
3097   match(ConL);
3098 
3099   op_cost(10);
3100   format %{ %}
3101   interface(CONST_INTER);
3102 %}
3103 
3104 // Long Immediate 32-bit signed
3105 operand immL32()
3106 %{
3107   predicate(n->get_long() == (int) (n->get_long()));
3108   match(ConL);
3109 
3110   op_cost(15);
3111   format %{ %}
3112   interface(CONST_INTER);
3113 %}
3114 
3115 // Long Immediate zero
3116 operand immL0()
3117 %{
3118   predicate(n->get_long() == 0L);
3119   match(ConL);
3120 
3121   op_cost(10);
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 // Constant for increment
3127 operand immL1()
3128 %{
3129   predicate(n->get_long() == 1);
3130   match(ConL);
3131 
3132   format %{ %}
3133   interface(CONST_INTER);
3134 %}
3135 
3136 // Constant for decrement
3137 operand immL_M1()
3138 %{
3139   predicate(n->get_long() == -1);
3140   match(ConL);
3141 
3142   format %{ %}
3143   interface(CONST_INTER);
3144 %}
3145 
3146 // Long Immediate: the value 10
3147 operand immL10()
3148 %{
3149   predicate(n->get_long() == 10);
3150   match(ConL);
3151 
3152   format %{ %}
3153   interface(CONST_INTER);
3154 %}
3155 
3156 // Long immediate from 0 to 127.
3157 // Used for a shorter form of long mul by 10.
3158 operand immL_127()
3159 %{
3160   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3161   match(ConL);
3162 
3163   op_cost(10);
3164   format %{ %}
3165   interface(CONST_INTER);
3166 %}
3167 
3168 // Long Immediate: low 32-bit mask
3169 operand immL_32bits()
3170 %{
3171   predicate(n->get_long() == 0xFFFFFFFFL);
3172   match(ConL);
3173   op_cost(20);
3174 
3175   format %{ %}
3176   interface(CONST_INTER);
3177 %}
3178 
3179 // Float Immediate zero
3180 operand immF0()
3181 %{
3182   predicate(jint_cast(n->getf()) == 0);
3183   match(ConF);
3184 
3185   op_cost(5);
3186   format %{ %}
3187   interface(CONST_INTER);
3188 %}
3189 
3190 // Float Immediate
3191 operand immF()
3192 %{
3193   match(ConF);
3194 
3195   op_cost(15);
3196   format %{ %}
3197   interface(CONST_INTER);
3198 %}
3199 
3200 // Double Immediate zero
3201 operand immD0()
3202 %{
3203   predicate(jlong_cast(n->getd()) == 0);
3204   match(ConD);
3205 
3206   op_cost(5);
3207   format %{ %}
3208   interface(CONST_INTER);
3209 %}
3210 
3211 // Double Immediate
3212 operand immD()
3213 %{
3214   match(ConD);
3215 
3216   op_cost(15);
3217   format %{ %}
3218   interface(CONST_INTER);
3219 %}
3220 
3221 // Immediates for special shifts (sign extend)
3222 
3223 // Constants for increment
3224 operand immI_16()
3225 %{
3226   predicate(n->get_int() == 16);
3227   match(ConI);
3228 
3229   format %{ %}
3230   interface(CONST_INTER);
3231 %}
3232 
3233 operand immI_24()
3234 %{
3235   predicate(n->get_int() == 24);
3236   match(ConI);
3237 
3238   format %{ %}
3239   interface(CONST_INTER);
3240 %}
3241 
3242 // Constant for byte-wide masking
3243 operand immI_255()
3244 %{
3245   predicate(n->get_int() == 255);
3246   match(ConI);
3247 
3248   format %{ %}
3249   interface(CONST_INTER);
3250 %}
3251 
3252 // Constant for short-wide masking
3253 operand immI_65535()
3254 %{
3255   predicate(n->get_int() == 65535);
3256   match(ConI);
3257 
3258   format %{ %}
3259   interface(CONST_INTER);
3260 %}
3261 
3262 // Constant for byte-wide masking
3263 operand immL_255()
3264 %{
3265   predicate(n->get_long() == 255);
3266   match(ConL);
3267 
3268   format %{ %}
3269   interface(CONST_INTER);
3270 %}
3271 
3272 // Constant for short-wide masking
3273 operand immL_65535()
3274 %{
3275   predicate(n->get_long() == 65535);
3276   match(ConL);
3277 
3278   format %{ %}
3279   interface(CONST_INTER);
3280 %}
3281 
3282 // Register Operands
3283 // Integer Register
3284 operand rRegI()
3285 %{
3286   constraint(ALLOC_IN_RC(int_reg));
3287   match(RegI);
3288 
3289   match(rax_RegI);
3290   match(rbx_RegI);
3291   match(rcx_RegI);
3292   match(rdx_RegI);
3293   match(rdi_RegI);
3294 
3295   format %{ %}
3296   interface(REG_INTER);
3297 %}
3298 
3299 // Special Registers
3300 operand rax_RegI()
3301 %{
3302   constraint(ALLOC_IN_RC(int_rax_reg));
3303   match(RegI);
3304   match(rRegI);
3305 
3306   format %{ "RAX" %}
3307   interface(REG_INTER);
3308 %}
3309 
3310 // Special Registers
3311 operand rbx_RegI()
3312 %{
3313   constraint(ALLOC_IN_RC(int_rbx_reg));
3314   match(RegI);
3315   match(rRegI);
3316 
3317   format %{ "RBX" %}
3318   interface(REG_INTER);
3319 %}
3320 
3321 operand rcx_RegI()
3322 %{
3323   constraint(ALLOC_IN_RC(int_rcx_reg));
3324   match(RegI);
3325   match(rRegI);
3326 
3327   format %{ "RCX" %}
3328   interface(REG_INTER);
3329 %}
3330 
3331 operand rdx_RegI()
3332 %{
3333   constraint(ALLOC_IN_RC(int_rdx_reg));
3334   match(RegI);
3335   match(rRegI);
3336 
3337   format %{ "RDX" %}
3338   interface(REG_INTER);
3339 %}
3340 
3341 operand rdi_RegI()
3342 %{
3343   constraint(ALLOC_IN_RC(int_rdi_reg));
3344   match(RegI);
3345   match(rRegI);
3346 
3347   format %{ "RDI" %}
3348   interface(REG_INTER);
3349 %}
3350 
3351 operand no_rcx_RegI()
3352 %{
3353   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3354   match(RegI);
3355   match(rax_RegI);
3356   match(rbx_RegI);
3357   match(rdx_RegI);
3358   match(rdi_RegI);
3359 
3360   format %{ %}
3361   interface(REG_INTER);
3362 %}
3363 
3364 operand no_rax_rdx_RegI()
3365 %{
3366   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3367   match(RegI);
3368   match(rbx_RegI);
3369   match(rcx_RegI);
3370   match(rdi_RegI);
3371 
3372   format %{ %}
3373   interface(REG_INTER);
3374 %}
3375 
3376 // Pointer Register
3377 operand any_RegP()
3378 %{
3379   constraint(ALLOC_IN_RC(any_reg));
3380   match(RegP);
3381   match(rax_RegP);
3382   match(rbx_RegP);
3383   match(rdi_RegP);
3384   match(rsi_RegP);
3385   match(rbp_RegP);
3386   match(r15_RegP);
3387   match(rRegP);
3388 
3389   format %{ %}
3390   interface(REG_INTER);
3391 %}
3392 
3393 operand rRegP()
3394 %{
3395   constraint(ALLOC_IN_RC(ptr_reg));
3396   match(RegP);
3397   match(rax_RegP);
3398   match(rbx_RegP);
3399   match(rdi_RegP);
3400   match(rsi_RegP);
3401   match(rbp_RegP);  // See Q&A below about
3402   match(r15_RegP);  // r15_RegP and rbp_RegP.
3403 
3404   format %{ %}
3405   interface(REG_INTER);
3406 %}
3407 
3408 operand rRegN() %{
3409   constraint(ALLOC_IN_RC(int_reg));
3410   match(RegN);
3411 
3412   format %{ %}
3413   interface(REG_INTER);
3414 %}
3415 
3416 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3417 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3418 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3419 // The output of an instruction is controlled by the allocator, which respects
3420 // register class masks, not match rules.  Unless an instruction mentions
3421 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3422 // by the allocator as an input.
3423 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3424 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3425 // result, RBP is not included in the output of the instruction either.
3426 
3427 operand no_rax_RegP()
3428 %{
3429   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3430   match(RegP);
3431   match(rbx_RegP);
3432   match(rsi_RegP);
3433   match(rdi_RegP);
3434 
3435   format %{ %}
3436   interface(REG_INTER);
3437 %}
3438 
3439 // This operand is not allowed to use RBP even if
3440 // RBP is not used to hold the frame pointer.
3441 operand no_rbp_RegP()
3442 %{
3443   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3444   match(RegP);
3445   match(rbx_RegP);
3446   match(rsi_RegP);
3447   match(rdi_RegP);
3448 
3449   format %{ %}
3450   interface(REG_INTER);
3451 %}
3452 
3453 operand no_rax_rbx_RegP()
3454 %{
3455   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3456   match(RegP);
3457   match(rsi_RegP);
3458   match(rdi_RegP);
3459 
3460   format %{ %}
3461   interface(REG_INTER);
3462 %}
3463 
3464 // Special Registers
3465 // Return a pointer value
3466 operand rax_RegP()
3467 %{
3468   constraint(ALLOC_IN_RC(ptr_rax_reg));
3469   match(RegP);
3470   match(rRegP);
3471 
3472   format %{ %}
3473   interface(REG_INTER);
3474 %}
3475 
3476 // Special Registers
3477 // Return a compressed pointer value
3478 operand rax_RegN()
3479 %{
3480   constraint(ALLOC_IN_RC(int_rax_reg));
3481   match(RegN);
3482   match(rRegN);
3483 
3484   format %{ %}
3485   interface(REG_INTER);
3486 %}
3487 
3488 // Used in AtomicAdd
3489 operand rbx_RegP()
3490 %{
3491   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3492   match(RegP);
3493   match(rRegP);
3494 
3495   format %{ %}
3496   interface(REG_INTER);
3497 %}
3498 
3499 operand rsi_RegP()
3500 %{
3501   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3502   match(RegP);
3503   match(rRegP);
3504 
3505   format %{ %}
3506   interface(REG_INTER);
3507 %}
3508 
3509 // Used in rep stosq
3510 operand rdi_RegP()
3511 %{
3512   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3513   match(RegP);
3514   match(rRegP);
3515 
3516   format %{ %}
3517   interface(REG_INTER);
3518 %}
3519 
3520 operand r15_RegP()
3521 %{
3522   constraint(ALLOC_IN_RC(ptr_r15_reg));
3523   match(RegP);
3524   match(rRegP);
3525 
3526   format %{ %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 operand rRegL()
3531 %{
3532   constraint(ALLOC_IN_RC(long_reg));
3533   match(RegL);
3534   match(rax_RegL);
3535   match(rdx_RegL);
3536 
3537   format %{ %}
3538   interface(REG_INTER);
3539 %}
3540 
3541 // Special Registers
3542 operand no_rax_rdx_RegL()
3543 %{
3544   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3545   match(RegL);
3546   match(rRegL);
3547 
3548   format %{ %}
3549   interface(REG_INTER);
3550 %}
3551 
3552 operand no_rax_RegL()
3553 %{
3554   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3555   match(RegL);
3556   match(rRegL);
3557   match(rdx_RegL);
3558 
3559   format %{ %}
3560   interface(REG_INTER);
3561 %}
3562 
3563 operand no_rcx_RegL()
3564 %{
3565   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3566   match(RegL);
3567   match(rRegL);
3568 
3569   format %{ %}
3570   interface(REG_INTER);
3571 %}
3572 
3573 operand rax_RegL()
3574 %{
3575   constraint(ALLOC_IN_RC(long_rax_reg));
3576   match(RegL);
3577   match(rRegL);
3578 
3579   format %{ "RAX" %}
3580   interface(REG_INTER);
3581 %}
3582 
3583 operand rcx_RegL()
3584 %{
3585   constraint(ALLOC_IN_RC(long_rcx_reg));
3586   match(RegL);
3587   match(rRegL);
3588 
3589   format %{ %}
3590   interface(REG_INTER);
3591 %}
3592 
3593 operand rdx_RegL()
3594 %{
3595   constraint(ALLOC_IN_RC(long_rdx_reg));
3596   match(RegL);
3597   match(rRegL);
3598 
3599   format %{ %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 // Flags register, used as output of compare instructions
3604 operand rFlagsReg()
3605 %{
3606   constraint(ALLOC_IN_RC(int_flags));
3607   match(RegFlags);
3608 
3609   format %{ "RFLAGS" %}
3610   interface(REG_INTER);
3611 %}
3612 
3613 // Flags register, used as output of FLOATING POINT compare instructions
3614 operand rFlagsRegU()
3615 %{
3616   constraint(ALLOC_IN_RC(int_flags));
3617   match(RegFlags);
3618 
3619   format %{ "RFLAGS_U" %}
3620   interface(REG_INTER);
3621 %}
3622 
3623 operand rFlagsRegUCF() %{
3624   constraint(ALLOC_IN_RC(int_flags));
3625   match(RegFlags);
3626   predicate(false);
3627 
3628   format %{ "RFLAGS_U_CF" %}
3629   interface(REG_INTER);
3630 %}
3631 
3632 // Float register operands
3633 operand regF() %{
3634    constraint(ALLOC_IN_RC(float_reg));
3635    match(RegF);
3636 
3637    format %{ %}
3638    interface(REG_INTER);
3639 %}
3640 
3641 // Float register operands
3642 operand legRegF() %{
3643    constraint(ALLOC_IN_RC(float_reg_legacy));
3644    match(RegF);
3645 
3646    format %{ %}
3647    interface(REG_INTER);
3648 %}
3649 
3650 // Float register operands
3651 operand vlRegF() %{
3652    constraint(ALLOC_IN_RC(float_reg_vl));
3653    match(RegF);
3654 
3655    format %{ %}
3656    interface(REG_INTER);
3657 %}
3658 
3659 // Double register operands
3660 operand regD() %{
3661    constraint(ALLOC_IN_RC(double_reg));
3662    match(RegD);
3663 
3664    format %{ %}
3665    interface(REG_INTER);
3666 %}
3667 
3668 // Double register operands
3669 operand legRegD() %{
3670    constraint(ALLOC_IN_RC(double_reg_legacy));
3671    match(RegD);
3672 
3673    format %{ %}
3674    interface(REG_INTER);
3675 %}
3676 
3677 // Double register operands
3678 operand vlRegD() %{
3679    constraint(ALLOC_IN_RC(double_reg_vl));
3680    match(RegD);
3681 
3682    format %{ %}
3683    interface(REG_INTER);
3684 %}
3685 
3686 // Vectors
3687 operand vecS() %{
3688   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3689   match(VecS);
3690 
3691   format %{ %}
3692   interface(REG_INTER);
3693 %}
3694 
3695 // Vectors
3696 operand legVecS() %{
3697   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3698   match(VecS);
3699 
3700   format %{ %}
3701   interface(REG_INTER);
3702 %}
3703 
3704 operand vecD() %{
3705   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3706   match(VecD);
3707 
3708   format %{ %}
3709   interface(REG_INTER);
3710 %}
3711 
3712 operand legVecD() %{
3713   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3714   match(VecD);
3715 
3716   format %{ %}
3717   interface(REG_INTER);
3718 %}
3719 
3720 operand vecX() %{
3721   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3722   match(VecX);
3723 
3724   format %{ %}
3725   interface(REG_INTER);
3726 %}
3727 
3728 operand legVecX() %{
3729   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3730   match(VecX);
3731 
3732   format %{ %}
3733   interface(REG_INTER);
3734 %}
3735 
3736 operand vecY() %{
3737   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3738   match(VecY);
3739 
3740   format %{ %}
3741   interface(REG_INTER);
3742 %}
3743 
3744 operand legVecY() %{
3745   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3746   match(VecY);
3747 
3748   format %{ %}
3749   interface(REG_INTER);
3750 %}
3751 
3752 //----------Memory Operands----------------------------------------------------
3753 // Direct Memory Operand
3754 // operand direct(immP addr)
3755 // %{
3756 //   match(addr);
3757 
3758 //   format %{ "[$addr]" %}
3759 //   interface(MEMORY_INTER) %{
3760 //     base(0xFFFFFFFF);
3761 //     index(0x4);
3762 //     scale(0x0);
3763 //     disp($addr);
3764 //   %}
3765 // %}
3766 
3767 // Indirect Memory Operand
3768 operand indirect(any_RegP reg)
3769 %{
3770   constraint(ALLOC_IN_RC(ptr_reg));
3771   match(reg);
3772 
3773   format %{ "[$reg]" %}
3774   interface(MEMORY_INTER) %{
3775     base($reg);
3776     index(0x4);
3777     scale(0x0);
3778     disp(0x0);
3779   %}
3780 %}
3781 
3782 // Indirect Memory Plus Short Offset Operand
3783 operand indOffset8(any_RegP reg, immL8 off)
3784 %{
3785   constraint(ALLOC_IN_RC(ptr_reg));
3786   match(AddP reg off);
3787 
3788   format %{ "[$reg + $off (8-bit)]" %}
3789   interface(MEMORY_INTER) %{
3790     base($reg);
3791     index(0x4);
3792     scale(0x0);
3793     disp($off);
3794   %}
3795 %}
3796 
3797 // Indirect Memory Plus Long Offset Operand
3798 operand indOffset32(any_RegP reg, immL32 off)
3799 %{
3800   constraint(ALLOC_IN_RC(ptr_reg));
3801   match(AddP reg off);
3802 
3803   format %{ "[$reg + $off (32-bit)]" %}
3804   interface(MEMORY_INTER) %{
3805     base($reg);
3806     index(0x4);
3807     scale(0x0);
3808     disp($off);
3809   %}
3810 %}
3811 
3812 // Indirect Memory Plus Index Register Plus Offset Operand
3813 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3814 %{
3815   constraint(ALLOC_IN_RC(ptr_reg));
3816   match(AddP (AddP reg lreg) off);
3817 
3818   op_cost(10);
3819   format %{"[$reg + $off + $lreg]" %}
3820   interface(MEMORY_INTER) %{
3821     base($reg);
3822     index($lreg);
3823     scale(0x0);
3824     disp($off);
3825   %}
3826 %}
3827 
3828 // Indirect Memory Plus Index Register Plus Offset Operand
3829 operand indIndex(any_RegP reg, rRegL lreg)
3830 %{
3831   constraint(ALLOC_IN_RC(ptr_reg));
3832   match(AddP reg lreg);
3833 
3834   op_cost(10);
3835   format %{"[$reg + $lreg]" %}
3836   interface(MEMORY_INTER) %{
3837     base($reg);
3838     index($lreg);
3839     scale(0x0);
3840     disp(0x0);
3841   %}
3842 %}
3843 
3844 // Indirect Memory Times Scale Plus Index Register
3845 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3846 %{
3847   constraint(ALLOC_IN_RC(ptr_reg));
3848   match(AddP reg (LShiftL lreg scale));
3849 
3850   op_cost(10);
3851   format %{"[$reg + $lreg << $scale]" %}
3852   interface(MEMORY_INTER) %{
3853     base($reg);
3854     index($lreg);
3855     scale($scale);
3856     disp(0x0);
3857   %}
3858 %}
3859 
3860 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3861 %{
3862   constraint(ALLOC_IN_RC(ptr_reg));
3863   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3864   match(AddP reg (LShiftL (ConvI2L idx) scale));
3865 
3866   op_cost(10);
3867   format %{"[$reg + pos $idx << $scale]" %}
3868   interface(MEMORY_INTER) %{
3869     base($reg);
3870     index($idx);
3871     scale($scale);
3872     disp(0x0);
3873   %}
3874 %}
3875 
3876 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3877 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3878 %{
3879   constraint(ALLOC_IN_RC(ptr_reg));
3880   match(AddP (AddP reg (LShiftL lreg scale)) off);
3881 
3882   op_cost(10);
3883   format %{"[$reg + $off + $lreg << $scale]" %}
3884   interface(MEMORY_INTER) %{
3885     base($reg);
3886     index($lreg);
3887     scale($scale);
3888     disp($off);
3889   %}
3890 %}
3891 
3892 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3893 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3894 %{
3895   constraint(ALLOC_IN_RC(ptr_reg));
3896   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3897   match(AddP (AddP reg (ConvI2L idx)) off);
3898 
3899   op_cost(10);
3900   format %{"[$reg + $off + $idx]" %}
3901   interface(MEMORY_INTER) %{
3902     base($reg);
3903     index($idx);
3904     scale(0x0);
3905     disp($off);
3906   %}
3907 %}
3908 
3909 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3910 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3911 %{
3912   constraint(ALLOC_IN_RC(ptr_reg));
3913   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3914   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3915 
3916   op_cost(10);
3917   format %{"[$reg + $off + $idx << $scale]" %}
3918   interface(MEMORY_INTER) %{
3919     base($reg);
3920     index($idx);
3921     scale($scale);
3922     disp($off);
3923   %}
3924 %}
3925 
3926 // Indirect Narrow Oop Plus Offset Operand
3927 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3928 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3929 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3930   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3931   constraint(ALLOC_IN_RC(ptr_reg));
3932   match(AddP (DecodeN reg) off);
3933 
3934   op_cost(10);
3935   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3936   interface(MEMORY_INTER) %{
3937     base(0xc); // R12
3938     index($reg);
3939     scale(0x3);
3940     disp($off);
3941   %}
3942 %}
3943 
3944 // Indirect Memory Operand
3945 operand indirectNarrow(rRegN reg)
3946 %{
3947   predicate(Universe::narrow_oop_shift() == 0);
3948   constraint(ALLOC_IN_RC(ptr_reg));
3949   match(DecodeN reg);
3950 
3951   format %{ "[$reg]" %}
3952   interface(MEMORY_INTER) %{
3953     base($reg);
3954     index(0x4);
3955     scale(0x0);
3956     disp(0x0);
3957   %}
3958 %}
3959 
3960 // Indirect Memory Plus Short Offset Operand
3961 operand indOffset8Narrow(rRegN reg, immL8 off)
3962 %{
3963   predicate(Universe::narrow_oop_shift() == 0);
3964   constraint(ALLOC_IN_RC(ptr_reg));
3965   match(AddP (DecodeN reg) off);
3966 
3967   format %{ "[$reg + $off (8-bit)]" %}
3968   interface(MEMORY_INTER) %{
3969     base($reg);
3970     index(0x4);
3971     scale(0x0);
3972     disp($off);
3973   %}
3974 %}
3975 
3976 // Indirect Memory Plus Long Offset Operand
3977 operand indOffset32Narrow(rRegN reg, immL32 off)
3978 %{
3979   predicate(Universe::narrow_oop_shift() == 0);
3980   constraint(ALLOC_IN_RC(ptr_reg));
3981   match(AddP (DecodeN reg) off);
3982 
3983   format %{ "[$reg + $off (32-bit)]" %}
3984   interface(MEMORY_INTER) %{
3985     base($reg);
3986     index(0x4);
3987     scale(0x0);
3988     disp($off);
3989   %}
3990 %}
3991 
3992 // Indirect Memory Plus Index Register Plus Offset Operand
3993 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3994 %{
3995   predicate(Universe::narrow_oop_shift() == 0);
3996   constraint(ALLOC_IN_RC(ptr_reg));
3997   match(AddP (AddP (DecodeN reg) lreg) off);
3998 
3999   op_cost(10);
4000   format %{"[$reg + $off + $lreg]" %}
4001   interface(MEMORY_INTER) %{
4002     base($reg);
4003     index($lreg);
4004     scale(0x0);
4005     disp($off);
4006   %}
4007 %}
4008 
4009 // Indirect Memory Plus Index Register Plus Offset Operand
4010 operand indIndexNarrow(rRegN reg, rRegL lreg)
4011 %{
4012   predicate(Universe::narrow_oop_shift() == 0);
4013   constraint(ALLOC_IN_RC(ptr_reg));
4014   match(AddP (DecodeN reg) lreg);
4015 
4016   op_cost(10);
4017   format %{"[$reg + $lreg]" %}
4018   interface(MEMORY_INTER) %{
4019     base($reg);
4020     index($lreg);
4021     scale(0x0);
4022     disp(0x0);
4023   %}
4024 %}
4025 
4026 // Indirect Memory Times Scale Plus Index Register
4027 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4028 %{
4029   predicate(Universe::narrow_oop_shift() == 0);
4030   constraint(ALLOC_IN_RC(ptr_reg));
4031   match(AddP (DecodeN reg) (LShiftL lreg scale));
4032 
4033   op_cost(10);
4034   format %{"[$reg + $lreg << $scale]" %}
4035   interface(MEMORY_INTER) %{
4036     base($reg);
4037     index($lreg);
4038     scale($scale);
4039     disp(0x0);
4040   %}
4041 %}
4042 
4043 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4044 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4045 %{
4046   predicate(Universe::narrow_oop_shift() == 0);
4047   constraint(ALLOC_IN_RC(ptr_reg));
4048   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4049 
4050   op_cost(10);
4051   format %{"[$reg + $off + $lreg << $scale]" %}
4052   interface(MEMORY_INTER) %{
4053     base($reg);
4054     index($lreg);
4055     scale($scale);
4056     disp($off);
4057   %}
4058 %}
4059 
4060 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4061 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4062 %{
4063   constraint(ALLOC_IN_RC(ptr_reg));
4064   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4065   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4066 
4067   op_cost(10);
4068   format %{"[$reg + $off + $idx]" %}
4069   interface(MEMORY_INTER) %{
4070     base($reg);
4071     index($idx);
4072     scale(0x0);
4073     disp($off);
4074   %}
4075 %}
4076 
4077 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4078 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4079 %{
4080   constraint(ALLOC_IN_RC(ptr_reg));
4081   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4082   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4083 
4084   op_cost(10);
4085   format %{"[$reg + $off + $idx << $scale]" %}
4086   interface(MEMORY_INTER) %{
4087     base($reg);
4088     index($idx);
4089     scale($scale);
4090     disp($off);
4091   %}
4092 %}
4093 
4094 //----------Special Memory Operands--------------------------------------------
4095 // Stack Slot Operand - This operand is used for loading and storing temporary
4096 //                      values on the stack where a match requires a value to
4097 //                      flow through memory.
4098 operand stackSlotP(sRegP reg)
4099 %{
4100   constraint(ALLOC_IN_RC(stack_slots));
4101   // No match rule because this operand is only generated in matching
4102 
4103   format %{ "[$reg]" %}
4104   interface(MEMORY_INTER) %{
4105     base(0x4);   // RSP
4106     index(0x4);  // No Index
4107     scale(0x0);  // No Scale
4108     disp($reg);  // Stack Offset
4109   %}
4110 %}
4111 
4112 operand stackSlotI(sRegI reg)
4113 %{
4114   constraint(ALLOC_IN_RC(stack_slots));
4115   // No match rule because this operand is only generated in matching
4116 
4117   format %{ "[$reg]" %}
4118   interface(MEMORY_INTER) %{
4119     base(0x4);   // RSP
4120     index(0x4);  // No Index
4121     scale(0x0);  // No Scale
4122     disp($reg);  // Stack Offset
4123   %}
4124 %}
4125 
4126 operand stackSlotF(sRegF reg)
4127 %{
4128   constraint(ALLOC_IN_RC(stack_slots));
4129   // No match rule because this operand is only generated in matching
4130 
4131   format %{ "[$reg]" %}
4132   interface(MEMORY_INTER) %{
4133     base(0x4);   // RSP
4134     index(0x4);  // No Index
4135     scale(0x0);  // No Scale
4136     disp($reg);  // Stack Offset
4137   %}
4138 %}
4139 
4140 operand stackSlotD(sRegD reg)
4141 %{
4142   constraint(ALLOC_IN_RC(stack_slots));
4143   // No match rule because this operand is only generated in matching
4144 
4145   format %{ "[$reg]" %}
4146   interface(MEMORY_INTER) %{
4147     base(0x4);   // RSP
4148     index(0x4);  // No Index
4149     scale(0x0);  // No Scale
4150     disp($reg);  // Stack Offset
4151   %}
4152 %}
4153 operand stackSlotL(sRegL reg)
4154 %{
4155   constraint(ALLOC_IN_RC(stack_slots));
4156   // No match rule because this operand is only generated in matching
4157 
4158   format %{ "[$reg]" %}
4159   interface(MEMORY_INTER) %{
4160     base(0x4);   // RSP
4161     index(0x4);  // No Index
4162     scale(0x0);  // No Scale
4163     disp($reg);  // Stack Offset
4164   %}
4165 %}
4166 
4167 //----------Conditional Branch Operands----------------------------------------
4168 // Comparison Op  - This is the operation of the comparison, and is limited to
4169 //                  the following set of codes:
4170 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4171 //
4172 // Other attributes of the comparison, such as unsignedness, are specified
4173 // by the comparison instruction that sets a condition code flags register.
4174 // That result is represented by a flags operand whose subtype is appropriate
4175 // to the unsignedness (etc.) of the comparison.
4176 //
4177 // Later, the instruction which matches both the Comparison Op (a Bool) and
4178 // the flags (produced by the Cmp) specifies the coding of the comparison op
4179 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4180 
4181 // Comparision Code
4182 operand cmpOp()
4183 %{
4184   match(Bool);
4185 
4186   format %{ "" %}
4187   interface(COND_INTER) %{
4188     equal(0x4, "e");
4189     not_equal(0x5, "ne");
4190     less(0xC, "l");
4191     greater_equal(0xD, "ge");
4192     less_equal(0xE, "le");
4193     greater(0xF, "g");
4194     overflow(0x0, "o");
4195     no_overflow(0x1, "no");
4196   %}
4197 %}
4198 
4199 // Comparison Code, unsigned compare.  Used by FP also, with
4200 // C2 (unordered) turned into GT or LT already.  The other bits
4201 // C0 and C3 are turned into Carry & Zero flags.
4202 operand cmpOpU()
4203 %{
4204   match(Bool);
4205 
4206   format %{ "" %}
4207   interface(COND_INTER) %{
4208     equal(0x4, "e");
4209     not_equal(0x5, "ne");
4210     less(0x2, "b");
4211     greater_equal(0x3, "nb");
4212     less_equal(0x6, "be");
4213     greater(0x7, "nbe");
4214     overflow(0x0, "o");
4215     no_overflow(0x1, "no");
4216   %}
4217 %}
4218 
4219 
4220 // Floating comparisons that don't require any fixup for the unordered case
4221 operand cmpOpUCF() %{
4222   match(Bool);
4223   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4224             n->as_Bool()->_test._test == BoolTest::ge ||
4225             n->as_Bool()->_test._test == BoolTest::le ||
4226             n->as_Bool()->_test._test == BoolTest::gt);
4227   format %{ "" %}
4228   interface(COND_INTER) %{
4229     equal(0x4, "e");
4230     not_equal(0x5, "ne");
4231     less(0x2, "b");
4232     greater_equal(0x3, "nb");
4233     less_equal(0x6, "be");
4234     greater(0x7, "nbe");
4235     overflow(0x0, "o");
4236     no_overflow(0x1, "no");
4237   %}
4238 %}
4239 
4240 
4241 // Floating comparisons that can be fixed up with extra conditional jumps
4242 operand cmpOpUCF2() %{
4243   match(Bool);
4244   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4245             n->as_Bool()->_test._test == BoolTest::eq);
4246   format %{ "" %}
4247   interface(COND_INTER) %{
4248     equal(0x4, "e");
4249     not_equal(0x5, "ne");
4250     less(0x2, "b");
4251     greater_equal(0x3, "nb");
4252     less_equal(0x6, "be");
4253     greater(0x7, "nbe");
4254     overflow(0x0, "o");
4255     no_overflow(0x1, "no");
4256   %}
4257 %}
4258 
4259 // Operands for bound floating pointer register arguments
4260 operand rxmm0() %{
4261   constraint(ALLOC_IN_RC(xmm0_reg));
4262   match(VecX);
4263   format%{%}
4264   interface(REG_INTER);
4265 %}
4266 operand rxmm1() %{
4267   constraint(ALLOC_IN_RC(xmm1_reg));
4268   match(VecX);
4269   format%{%}
4270   interface(REG_INTER);
4271 %}
4272 operand rxmm2() %{
4273   constraint(ALLOC_IN_RC(xmm2_reg));
4274   match(VecX);
4275   format%{%}
4276   interface(REG_INTER);
4277 %}
4278 operand rxmm3() %{
4279   constraint(ALLOC_IN_RC(xmm3_reg));
4280   match(VecX);
4281   format%{%}
4282   interface(REG_INTER);
4283 %}
4284 operand rxmm4() %{
4285   constraint(ALLOC_IN_RC(xmm4_reg));
4286   match(VecX);
4287   format%{%}
4288   interface(REG_INTER);
4289 %}
4290 operand rxmm5() %{
4291   constraint(ALLOC_IN_RC(xmm5_reg));
4292   match(VecX);
4293   format%{%}
4294   interface(REG_INTER);
4295 %}
4296 operand rxmm6() %{
4297   constraint(ALLOC_IN_RC(xmm6_reg));
4298   match(VecX);
4299   format%{%}
4300   interface(REG_INTER);
4301 %}
4302 operand rxmm7() %{
4303   constraint(ALLOC_IN_RC(xmm7_reg));
4304   match(VecX);
4305   format%{%}
4306   interface(REG_INTER);
4307 %}
4308 operand rxmm8() %{
4309   constraint(ALLOC_IN_RC(xmm8_reg));
4310   match(VecX);
4311   format%{%}
4312   interface(REG_INTER);
4313 %}
4314 operand rxmm9() %{
4315   constraint(ALLOC_IN_RC(xmm9_reg));
4316   match(VecX);
4317   format%{%}
4318   interface(REG_INTER);
4319 %}
4320 operand rxmm10() %{
4321   constraint(ALLOC_IN_RC(xmm10_reg));
4322   match(VecX);
4323   format%{%}
4324   interface(REG_INTER);
4325 %}
4326 operand rxmm11() %{
4327   constraint(ALLOC_IN_RC(xmm11_reg));
4328   match(VecX);
4329   format%{%}
4330   interface(REG_INTER);
4331 %}
4332 operand rxmm12() %{
4333   constraint(ALLOC_IN_RC(xmm12_reg));
4334   match(VecX);
4335   format%{%}
4336   interface(REG_INTER);
4337 %}
4338 operand rxmm13() %{
4339   constraint(ALLOC_IN_RC(xmm13_reg));
4340   match(VecX);
4341   format%{%}
4342   interface(REG_INTER);
4343 %}
4344 operand rxmm14() %{
4345   constraint(ALLOC_IN_RC(xmm14_reg));
4346   match(VecX);
4347   format%{%}
4348   interface(REG_INTER);
4349 %}
4350 operand rxmm15() %{
4351   constraint(ALLOC_IN_RC(xmm15_reg));
4352   match(VecX);
4353   format%{%}
4354   interface(REG_INTER);
4355 %}
4356 operand rxmm16() %{
4357   constraint(ALLOC_IN_RC(xmm16_reg));
4358   match(VecX);
4359   format%{%}
4360   interface(REG_INTER);
4361 %}
4362 operand rxmm17() %{
4363   constraint(ALLOC_IN_RC(xmm17_reg));
4364   match(VecX);
4365   format%{%}
4366   interface(REG_INTER);
4367 %}
4368 operand rxmm18() %{
4369   constraint(ALLOC_IN_RC(xmm18_reg));
4370   match(VecX);
4371   format%{%}
4372   interface(REG_INTER);
4373 %}
4374 operand rxmm19() %{
4375   constraint(ALLOC_IN_RC(xmm19_reg));
4376   match(VecX);
4377   format%{%}
4378   interface(REG_INTER);
4379 %}
4380 operand rxmm20() %{
4381   constraint(ALLOC_IN_RC(xmm20_reg));
4382   match(VecX);
4383   format%{%}
4384   interface(REG_INTER);
4385 %}
4386 operand rxmm21() %{
4387   constraint(ALLOC_IN_RC(xmm21_reg));
4388   match(VecX);
4389   format%{%}
4390   interface(REG_INTER);
4391 %}
4392 operand rxmm22() %{
4393   constraint(ALLOC_IN_RC(xmm22_reg));
4394   match(VecX);
4395   format%{%}
4396   interface(REG_INTER);
4397 %}
4398 operand rxmm23() %{
4399   constraint(ALLOC_IN_RC(xmm23_reg));
4400   match(VecX);
4401   format%{%}
4402   interface(REG_INTER);
4403 %}
4404 operand rxmm24() %{
4405   constraint(ALLOC_IN_RC(xmm24_reg));
4406   match(VecX);
4407   format%{%}
4408   interface(REG_INTER);
4409 %}
4410 operand rxmm25() %{
4411   constraint(ALLOC_IN_RC(xmm25_reg));
4412   match(VecX);
4413   format%{%}
4414   interface(REG_INTER);
4415 %}
4416 operand rxmm26() %{
4417   constraint(ALLOC_IN_RC(xmm26_reg));
4418   match(VecX);
4419   format%{%}
4420   interface(REG_INTER);
4421 %}
4422 operand rxmm27() %{
4423   constraint(ALLOC_IN_RC(xmm27_reg));
4424   match(VecX);
4425   format%{%}
4426   interface(REG_INTER);
4427 %}
4428 operand rxmm28() %{
4429   constraint(ALLOC_IN_RC(xmm28_reg));
4430   match(VecX);
4431   format%{%}
4432   interface(REG_INTER);
4433 %}
4434 operand rxmm29() %{
4435   constraint(ALLOC_IN_RC(xmm29_reg));
4436   match(VecX);
4437   format%{%}
4438   interface(REG_INTER);
4439 %}
4440 operand rxmm30() %{
4441   constraint(ALLOC_IN_RC(xmm30_reg));
4442   match(VecX);
4443   format%{%}
4444   interface(REG_INTER);
4445 %}
4446 operand rxmm31() %{
4447   constraint(ALLOC_IN_RC(xmm31_reg));
4448   match(VecX);
4449   format%{%}
4450   interface(REG_INTER);
4451 %}
4452 
4453 //----------OPERAND CLASSES----------------------------------------------------
4454 // Operand Classes are groups of operands that are used as to simplify
4455 // instruction definitions by not requiring the AD writer to specify separate
4456 // instructions for every form of operand when the instruction accepts
4457 // multiple operand types with the same basic encoding and format.  The classic
4458 // case of this is memory operands.
4459 
4460 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4461                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4462                indCompressedOopOffset,
4463                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4464                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4465                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4466 
4467 //----------PIPELINE-----------------------------------------------------------
4468 // Rules which define the behavior of the target architectures pipeline.
4469 pipeline %{
4470 
4471 //----------ATTRIBUTES---------------------------------------------------------
4472 attributes %{
4473   variable_size_instructions;        // Fixed size instructions
4474   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4475   instruction_unit_size = 1;         // An instruction is 1 bytes long
4476   instruction_fetch_unit_size = 16;  // The processor fetches one line
4477   instruction_fetch_units = 1;       // of 16 bytes
4478 
4479   // List of nop instructions
4480   nops( MachNop );
4481 %}
4482 
4483 //----------RESOURCES----------------------------------------------------------
4484 // Resources are the functional units available to the machine
4485 
4486 // Generic P2/P3 pipeline
4487 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4488 // 3 instructions decoded per cycle.
4489 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4490 // 3 ALU op, only ALU0 handles mul instructions.
4491 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4492            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4493            BR, FPU,
4494            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4495 
4496 //----------PIPELINE DESCRIPTION-----------------------------------------------
4497 // Pipeline Description specifies the stages in the machine's pipeline
4498 
4499 // Generic P2/P3 pipeline
4500 pipe_desc(S0, S1, S2, S3, S4, S5);
4501 
4502 //----------PIPELINE CLASSES---------------------------------------------------
4503 // Pipeline Classes describe the stages in which input and output are
4504 // referenced by the hardware pipeline.
4505 
4506 // Naming convention: ialu or fpu
4507 // Then: _reg
4508 // Then: _reg if there is a 2nd register
4509 // Then: _long if it's a pair of instructions implementing a long
4510 // Then: _fat if it requires the big decoder
4511 //   Or: _mem if it requires the big decoder and a memory unit.
4512 
4513 // Integer ALU reg operation
4514 pipe_class ialu_reg(rRegI dst)
4515 %{
4516     single_instruction;
4517     dst    : S4(write);
4518     dst    : S3(read);
4519     DECODE : S0;        // any decoder
4520     ALU    : S3;        // any alu
4521 %}
4522 
4523 // Long ALU reg operation
4524 pipe_class ialu_reg_long(rRegL dst)
4525 %{
4526     instruction_count(2);
4527     dst    : S4(write);
4528     dst    : S3(read);
4529     DECODE : S0(2);     // any 2 decoders
4530     ALU    : S3(2);     // both alus
4531 %}
4532 
4533 // Integer ALU reg operation using big decoder
4534 pipe_class ialu_reg_fat(rRegI dst)
4535 %{
4536     single_instruction;
4537     dst    : S4(write);
4538     dst    : S3(read);
4539     D0     : S0;        // big decoder only
4540     ALU    : S3;        // any alu
4541 %}
4542 
4543 // Long ALU reg operation using big decoder
4544 pipe_class ialu_reg_long_fat(rRegL dst)
4545 %{
4546     instruction_count(2);
4547     dst    : S4(write);
4548     dst    : S3(read);
4549     D0     : S0(2);     // big decoder only; twice
4550     ALU    : S3(2);     // any 2 alus
4551 %}
4552 
4553 // Integer ALU reg-reg operation
4554 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4555 %{
4556     single_instruction;
4557     dst    : S4(write);
4558     src    : S3(read);
4559     DECODE : S0;        // any decoder
4560     ALU    : S3;        // any alu
4561 %}
4562 
4563 // Long ALU reg-reg operation
4564 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4565 %{
4566     instruction_count(2);
4567     dst    : S4(write);
4568     src    : S3(read);
4569     DECODE : S0(2);     // any 2 decoders
4570     ALU    : S3(2);     // both alus
4571 %}
4572 
4573 // Integer ALU reg-reg operation
4574 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4575 %{
4576     single_instruction;
4577     dst    : S4(write);
4578     src    : S3(read);
4579     D0     : S0;        // big decoder only
4580     ALU    : S3;        // any alu
4581 %}
4582 
4583 // Long ALU reg-reg operation
4584 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4585 %{
4586     instruction_count(2);
4587     dst    : S4(write);
4588     src    : S3(read);
4589     D0     : S0(2);     // big decoder only; twice
4590     ALU    : S3(2);     // both alus
4591 %}
4592 
4593 // Integer ALU reg-mem operation
4594 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4595 %{
4596     single_instruction;
4597     dst    : S5(write);
4598     mem    : S3(read);
4599     D0     : S0;        // big decoder only
4600     ALU    : S4;        // any alu
4601     MEM    : S3;        // any mem
4602 %}
4603 
4604 // Integer mem operation (prefetch)
4605 pipe_class ialu_mem(memory mem)
4606 %{
4607     single_instruction;
4608     mem    : S3(read);
4609     D0     : S0;        // big decoder only
4610     MEM    : S3;        // any mem
4611 %}
4612 
4613 // Integer Store to Memory
4614 pipe_class ialu_mem_reg(memory mem, rRegI src)
4615 %{
4616     single_instruction;
4617     mem    : S3(read);
4618     src    : S5(read);
4619     D0     : S0;        // big decoder only
4620     ALU    : S4;        // any alu
4621     MEM    : S3;
4622 %}
4623 
4624 // // Long Store to Memory
4625 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4626 // %{
4627 //     instruction_count(2);
4628 //     mem    : S3(read);
4629 //     src    : S5(read);
4630 //     D0     : S0(2);          // big decoder only; twice
4631 //     ALU    : S4(2);     // any 2 alus
4632 //     MEM    : S3(2);  // Both mems
4633 // %}
4634 
4635 // Integer Store to Memory
4636 pipe_class ialu_mem_imm(memory mem)
4637 %{
4638     single_instruction;
4639     mem    : S3(read);
4640     D0     : S0;        // big decoder only
4641     ALU    : S4;        // any alu
4642     MEM    : S3;
4643 %}
4644 
4645 // Integer ALU0 reg-reg operation
4646 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4647 %{
4648     single_instruction;
4649     dst    : S4(write);
4650     src    : S3(read);
4651     D0     : S0;        // Big decoder only
4652     ALU0   : S3;        // only alu0
4653 %}
4654 
4655 // Integer ALU0 reg-mem operation
4656 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4657 %{
4658     single_instruction;
4659     dst    : S5(write);
4660     mem    : S3(read);
4661     D0     : S0;        // big decoder only
4662     ALU0   : S4;        // ALU0 only
4663     MEM    : S3;        // any mem
4664 %}
4665 
4666 // Integer ALU reg-reg operation
4667 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4668 %{
4669     single_instruction;
4670     cr     : S4(write);
4671     src1   : S3(read);
4672     src2   : S3(read);
4673     DECODE : S0;        // any decoder
4674     ALU    : S3;        // any alu
4675 %}
4676 
4677 // Integer ALU reg-imm operation
4678 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4679 %{
4680     single_instruction;
4681     cr     : S4(write);
4682     src1   : S3(read);
4683     DECODE : S0;        // any decoder
4684     ALU    : S3;        // any alu
4685 %}
4686 
4687 // Integer ALU reg-mem operation
4688 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4689 %{
4690     single_instruction;
4691     cr     : S4(write);
4692     src1   : S3(read);
4693     src2   : S3(read);
4694     D0     : S0;        // big decoder only
4695     ALU    : S4;        // any alu
4696     MEM    : S3;
4697 %}
4698 
4699 // Conditional move reg-reg
4700 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4701 %{
4702     instruction_count(4);
4703     y      : S4(read);
4704     q      : S3(read);
4705     p      : S3(read);
4706     DECODE : S0(4);     // any decoder
4707 %}
4708 
4709 // Conditional move reg-reg
4710 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4711 %{
4712     single_instruction;
4713     dst    : S4(write);
4714     src    : S3(read);
4715     cr     : S3(read);
4716     DECODE : S0;        // any decoder
4717 %}
4718 
4719 // Conditional move reg-mem
4720 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4721 %{
4722     single_instruction;
4723     dst    : S4(write);
4724     src    : S3(read);
4725     cr     : S3(read);
4726     DECODE : S0;        // any decoder
4727     MEM    : S3;
4728 %}
4729 
4730 // Conditional move reg-reg long
4731 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4732 %{
4733     single_instruction;
4734     dst    : S4(write);
4735     src    : S3(read);
4736     cr     : S3(read);
4737     DECODE : S0(2);     // any 2 decoders
4738 %}
4739 
4740 // XXX
4741 // // Conditional move double reg-reg
4742 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4743 // %{
4744 //     single_instruction;
4745 //     dst    : S4(write);
4746 //     src    : S3(read);
4747 //     cr     : S3(read);
4748 //     DECODE : S0;     // any decoder
4749 // %}
4750 
4751 // Float reg-reg operation
4752 pipe_class fpu_reg(regD dst)
4753 %{
4754     instruction_count(2);
4755     dst    : S3(read);
4756     DECODE : S0(2);     // any 2 decoders
4757     FPU    : S3;
4758 %}
4759 
4760 // Float reg-reg operation
4761 pipe_class fpu_reg_reg(regD dst, regD src)
4762 %{
4763     instruction_count(2);
4764     dst    : S4(write);
4765     src    : S3(read);
4766     DECODE : S0(2);     // any 2 decoders
4767     FPU    : S3;
4768 %}
4769 
4770 // Float reg-reg operation
4771 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4772 %{
4773     instruction_count(3);
4774     dst    : S4(write);
4775     src1   : S3(read);
4776     src2   : S3(read);
4777     DECODE : S0(3);     // any 3 decoders
4778     FPU    : S3(2);
4779 %}
4780 
4781 // Float reg-reg operation
4782 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4783 %{
4784     instruction_count(4);
4785     dst    : S4(write);
4786     src1   : S3(read);
4787     src2   : S3(read);
4788     src3   : S3(read);
4789     DECODE : S0(4);     // any 3 decoders
4790     FPU    : S3(2);
4791 %}
4792 
4793 // Float reg-reg operation
4794 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4795 %{
4796     instruction_count(4);
4797     dst    : S4(write);
4798     src1   : S3(read);
4799     src2   : S3(read);
4800     src3   : S3(read);
4801     DECODE : S1(3);     // any 3 decoders
4802     D0     : S0;        // Big decoder only
4803     FPU    : S3(2);
4804     MEM    : S3;
4805 %}
4806 
4807 // Float reg-mem operation
4808 pipe_class fpu_reg_mem(regD dst, memory mem)
4809 %{
4810     instruction_count(2);
4811     dst    : S5(write);
4812     mem    : S3(read);
4813     D0     : S0;        // big decoder only
4814     DECODE : S1;        // any decoder for FPU POP
4815     FPU    : S4;
4816     MEM    : S3;        // any mem
4817 %}
4818 
4819 // Float reg-mem operation
4820 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4821 %{
4822     instruction_count(3);
4823     dst    : S5(write);
4824     src1   : S3(read);
4825     mem    : S3(read);
4826     D0     : S0;        // big decoder only
4827     DECODE : S1(2);     // any decoder for FPU POP
4828     FPU    : S4;
4829     MEM    : S3;        // any mem
4830 %}
4831 
4832 // Float mem-reg operation
4833 pipe_class fpu_mem_reg(memory mem, regD src)
4834 %{
4835     instruction_count(2);
4836     src    : S5(read);
4837     mem    : S3(read);
4838     DECODE : S0;        // any decoder for FPU PUSH
4839     D0     : S1;        // big decoder only
4840     FPU    : S4;
4841     MEM    : S3;        // any mem
4842 %}
4843 
4844 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4845 %{
4846     instruction_count(3);
4847     src1   : S3(read);
4848     src2   : S3(read);
4849     mem    : S3(read);
4850     DECODE : S0(2);     // any decoder for FPU PUSH
4851     D0     : S1;        // big decoder only
4852     FPU    : S4;
4853     MEM    : S3;        // any mem
4854 %}
4855 
4856 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4857 %{
4858     instruction_count(3);
4859     src1   : S3(read);
4860     src2   : S3(read);
4861     mem    : S4(read);
4862     DECODE : S0;        // any decoder for FPU PUSH
4863     D0     : S0(2);     // big decoder only
4864     FPU    : S4;
4865     MEM    : S3(2);     // any mem
4866 %}
4867 
4868 pipe_class fpu_mem_mem(memory dst, memory src1)
4869 %{
4870     instruction_count(2);
4871     src1   : S3(read);
4872     dst    : S4(read);
4873     D0     : S0(2);     // big decoder only
4874     MEM    : S3(2);     // any mem
4875 %}
4876 
4877 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4878 %{
4879     instruction_count(3);
4880     src1   : S3(read);
4881     src2   : S3(read);
4882     dst    : S4(read);
4883     D0     : S0(3);     // big decoder only
4884     FPU    : S4;
4885     MEM    : S3(3);     // any mem
4886 %}
4887 
4888 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4889 %{
4890     instruction_count(3);
4891     src1   : S4(read);
4892     mem    : S4(read);
4893     DECODE : S0;        // any decoder for FPU PUSH
4894     D0     : S0(2);     // big decoder only
4895     FPU    : S4;
4896     MEM    : S3(2);     // any mem
4897 %}
4898 
4899 // Float load constant
4900 pipe_class fpu_reg_con(regD dst)
4901 %{
4902     instruction_count(2);
4903     dst    : S5(write);
4904     D0     : S0;        // big decoder only for the load
4905     DECODE : S1;        // any decoder for FPU POP
4906     FPU    : S4;
4907     MEM    : S3;        // any mem
4908 %}
4909 
4910 // Float load constant
4911 pipe_class fpu_reg_reg_con(regD dst, regD src)
4912 %{
4913     instruction_count(3);
4914     dst    : S5(write);
4915     src    : S3(read);
4916     D0     : S0;        // big decoder only for the load
4917     DECODE : S1(2);     // any decoder for FPU POP
4918     FPU    : S4;
4919     MEM    : S3;        // any mem
4920 %}
4921 
4922 // UnConditional branch
4923 pipe_class pipe_jmp(label labl)
4924 %{
4925     single_instruction;
4926     BR   : S3;
4927 %}
4928 
4929 // Conditional branch
4930 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4931 %{
4932     single_instruction;
4933     cr    : S1(read);
4934     BR    : S3;
4935 %}
4936 
4937 // Allocation idiom
4938 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4939 %{
4940     instruction_count(1); force_serialization;
4941     fixed_latency(6);
4942     heap_ptr : S3(read);
4943     DECODE   : S0(3);
4944     D0       : S2;
4945     MEM      : S3;
4946     ALU      : S3(2);
4947     dst      : S5(write);
4948     BR       : S5;
4949 %}
4950 
4951 // Generic big/slow expanded idiom
4952 pipe_class pipe_slow()
4953 %{
4954     instruction_count(10); multiple_bundles; force_serialization;
4955     fixed_latency(100);
4956     D0  : S0(2);
4957     MEM : S3(2);
4958 %}
4959 
4960 // The real do-nothing guy
4961 pipe_class empty()
4962 %{
4963     instruction_count(0);
4964 %}
4965 
4966 // Define the class for the Nop node
4967 define
4968 %{
4969    MachNop = empty;
4970 %}
4971 
4972 %}
4973 
4974 //----------INSTRUCTIONS-------------------------------------------------------
4975 //
4976 // match      -- States which machine-independent subtree may be replaced
4977 //               by this instruction.
4978 // ins_cost   -- The estimated cost of this instruction is used by instruction
4979 //               selection to identify a minimum cost tree of machine
4980 //               instructions that matches a tree of machine-independent
4981 //               instructions.
4982 // format     -- A string providing the disassembly for this instruction.
4983 //               The value of an instruction's operand may be inserted
4984 //               by referring to it with a '$' prefix.
4985 // opcode     -- Three instruction opcodes may be provided.  These are referred
4986 //               to within an encode class as $primary, $secondary, and $tertiary
4987 //               rrspectively.  The primary opcode is commonly used to
4988 //               indicate the type of machine instruction, while secondary
4989 //               and tertiary are often used for prefix options or addressing
4990 //               modes.
4991 // ins_encode -- A list of encode classes with parameters. The encode class
4992 //               name must have been defined in an 'enc_class' specification
4993 //               in the encode section of the architecture description.
4994 
4995 
4996 //----------Load/Store/Move Instructions---------------------------------------
4997 //----------Load Instructions--------------------------------------------------
4998 
4999 // Load Byte (8 bit signed)
5000 instruct loadB(rRegI dst, memory mem)
5001 %{
5002   match(Set dst (LoadB mem));
5003 
5004   ins_cost(125);
5005   format %{ "movsbl  $dst, $mem\t# byte" %}
5006 
5007   ins_encode %{
5008     __ movsbl($dst$$Register, $mem$$Address);
5009   %}
5010 
5011   ins_pipe(ialu_reg_mem);
5012 %}
5013 
5014 // Load Byte (8 bit signed) into Long Register
5015 instruct loadB2L(rRegL dst, memory mem)
5016 %{
5017   match(Set dst (ConvI2L (LoadB mem)));
5018 
5019   ins_cost(125);
5020   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5021 
5022   ins_encode %{
5023     __ movsbq($dst$$Register, $mem$$Address);
5024   %}
5025 
5026   ins_pipe(ialu_reg_mem);
5027 %}
5028 
5029 // Load Unsigned Byte (8 bit UNsigned)
5030 instruct loadUB(rRegI dst, memory mem)
5031 %{
5032   match(Set dst (LoadUB mem));
5033 
5034   ins_cost(125);
5035   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5036 
5037   ins_encode %{
5038     __ movzbl($dst$$Register, $mem$$Address);
5039   %}
5040 
5041   ins_pipe(ialu_reg_mem);
5042 %}
5043 
5044 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5045 instruct loadUB2L(rRegL dst, memory mem)
5046 %{
5047   match(Set dst (ConvI2L (LoadUB mem)));
5048 
5049   ins_cost(125);
5050   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5051 
5052   ins_encode %{
5053     __ movzbq($dst$$Register, $mem$$Address);
5054   %}
5055 
5056   ins_pipe(ialu_reg_mem);
5057 %}
5058 
5059 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5060 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5061   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5062   effect(KILL cr);
5063 
5064   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5065             "andl    $dst, right_n_bits($mask, 8)" %}
5066   ins_encode %{
5067     Register Rdst = $dst$$Register;
5068     __ movzbq(Rdst, $mem$$Address);
5069     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5070   %}
5071   ins_pipe(ialu_reg_mem);
5072 %}
5073 
5074 // Load Short (16 bit signed)
5075 instruct loadS(rRegI dst, memory mem)
5076 %{
5077   match(Set dst (LoadS mem));
5078 
5079   ins_cost(125);
5080   format %{ "movswl $dst, $mem\t# short" %}
5081 
5082   ins_encode %{
5083     __ movswl($dst$$Register, $mem$$Address);
5084   %}
5085 
5086   ins_pipe(ialu_reg_mem);
5087 %}
5088 
5089 // Load Short (16 bit signed) to Byte (8 bit signed)
5090 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5091   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5092 
5093   ins_cost(125);
5094   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5095   ins_encode %{
5096     __ movsbl($dst$$Register, $mem$$Address);
5097   %}
5098   ins_pipe(ialu_reg_mem);
5099 %}
5100 
5101 // Load Short (16 bit signed) into Long Register
5102 instruct loadS2L(rRegL dst, memory mem)
5103 %{
5104   match(Set dst (ConvI2L (LoadS mem)));
5105 
5106   ins_cost(125);
5107   format %{ "movswq $dst, $mem\t# short -> long" %}
5108 
5109   ins_encode %{
5110     __ movswq($dst$$Register, $mem$$Address);
5111   %}
5112 
5113   ins_pipe(ialu_reg_mem);
5114 %}
5115 
5116 // Load Unsigned Short/Char (16 bit UNsigned)
5117 instruct loadUS(rRegI dst, memory mem)
5118 %{
5119   match(Set dst (LoadUS mem));
5120 
5121   ins_cost(125);
5122   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5123 
5124   ins_encode %{
5125     __ movzwl($dst$$Register, $mem$$Address);
5126   %}
5127 
5128   ins_pipe(ialu_reg_mem);
5129 %}
5130 
5131 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5132 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5133   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5134 
5135   ins_cost(125);
5136   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5137   ins_encode %{
5138     __ movsbl($dst$$Register, $mem$$Address);
5139   %}
5140   ins_pipe(ialu_reg_mem);
5141 %}
5142 
5143 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5144 instruct loadUS2L(rRegL dst, memory mem)
5145 %{
5146   match(Set dst (ConvI2L (LoadUS mem)));
5147 
5148   ins_cost(125);
5149   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5150 
5151   ins_encode %{
5152     __ movzwq($dst$$Register, $mem$$Address);
5153   %}
5154 
5155   ins_pipe(ialu_reg_mem);
5156 %}
5157 
5158 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5159 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5160   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5161 
5162   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5163   ins_encode %{
5164     __ movzbq($dst$$Register, $mem$$Address);
5165   %}
5166   ins_pipe(ialu_reg_mem);
5167 %}
5168 
5169 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5170 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5171   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5172   effect(KILL cr);
5173 
5174   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5175             "andl    $dst, right_n_bits($mask, 16)" %}
5176   ins_encode %{
5177     Register Rdst = $dst$$Register;
5178     __ movzwq(Rdst, $mem$$Address);
5179     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5180   %}
5181   ins_pipe(ialu_reg_mem);
5182 %}
5183 
5184 // Load Integer
5185 instruct loadI(rRegI dst, memory mem)
5186 %{
5187   match(Set dst (LoadI mem));
5188 
5189   ins_cost(125);
5190   format %{ "movl    $dst, $mem\t# int" %}
5191 
5192   ins_encode %{
5193     __ movl($dst$$Register, $mem$$Address);
5194   %}
5195 
5196   ins_pipe(ialu_reg_mem);
5197 %}
5198 
5199 // Load Integer (32 bit signed) to Byte (8 bit signed)
5200 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5201   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5202 
5203   ins_cost(125);
5204   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5205   ins_encode %{
5206     __ movsbl($dst$$Register, $mem$$Address);
5207   %}
5208   ins_pipe(ialu_reg_mem);
5209 %}
5210 
5211 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5212 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5213   match(Set dst (AndI (LoadI mem) mask));
5214 
5215   ins_cost(125);
5216   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5217   ins_encode %{
5218     __ movzbl($dst$$Register, $mem$$Address);
5219   %}
5220   ins_pipe(ialu_reg_mem);
5221 %}
5222 
5223 // Load Integer (32 bit signed) to Short (16 bit signed)
5224 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5225   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5226 
5227   ins_cost(125);
5228   format %{ "movswl  $dst, $mem\t# int -> short" %}
5229   ins_encode %{
5230     __ movswl($dst$$Register, $mem$$Address);
5231   %}
5232   ins_pipe(ialu_reg_mem);
5233 %}
5234 
5235 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5236 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5237   match(Set dst (AndI (LoadI mem) mask));
5238 
5239   ins_cost(125);
5240   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5241   ins_encode %{
5242     __ movzwl($dst$$Register, $mem$$Address);
5243   %}
5244   ins_pipe(ialu_reg_mem);
5245 %}
5246 
5247 // Load Integer into Long Register
5248 instruct loadI2L(rRegL dst, memory mem)
5249 %{
5250   match(Set dst (ConvI2L (LoadI mem)));
5251 
5252   ins_cost(125);
5253   format %{ "movslq  $dst, $mem\t# int -> long" %}
5254 
5255   ins_encode %{
5256     __ movslq($dst$$Register, $mem$$Address);
5257   %}
5258 
5259   ins_pipe(ialu_reg_mem);
5260 %}
5261 
5262 // Load Integer with mask 0xFF into Long Register
5263 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5264   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5265 
5266   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5267   ins_encode %{
5268     __ movzbq($dst$$Register, $mem$$Address);
5269   %}
5270   ins_pipe(ialu_reg_mem);
5271 %}
5272 
5273 // Load Integer with mask 0xFFFF into Long Register
5274 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5275   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5276 
5277   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5278   ins_encode %{
5279     __ movzwq($dst$$Register, $mem$$Address);
5280   %}
5281   ins_pipe(ialu_reg_mem);
5282 %}
5283 
5284 // Load Integer with a 31-bit mask into Long Register
5285 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5286   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5287   effect(KILL cr);
5288 
5289   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5290             "andl    $dst, $mask" %}
5291   ins_encode %{
5292     Register Rdst = $dst$$Register;
5293     __ movl(Rdst, $mem$$Address);
5294     __ andl(Rdst, $mask$$constant);
5295   %}
5296   ins_pipe(ialu_reg_mem);
5297 %}
5298 
5299 // Load Unsigned Integer into Long Register
5300 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5301 %{
5302   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5303 
5304   ins_cost(125);
5305   format %{ "movl    $dst, $mem\t# uint -> long" %}
5306 
5307   ins_encode %{
5308     __ movl($dst$$Register, $mem$$Address);
5309   %}
5310 
5311   ins_pipe(ialu_reg_mem);
5312 %}
5313 
5314 // Load Long
5315 instruct loadL(rRegL dst, memory mem)
5316 %{
5317   match(Set dst (LoadL mem));
5318 
5319   ins_cost(125);
5320   format %{ "movq    $dst, $mem\t# long" %}
5321 
5322   ins_encode %{
5323     __ movq($dst$$Register, $mem$$Address);
5324   %}
5325 
5326   ins_pipe(ialu_reg_mem); // XXX
5327 %}
5328 
5329 // Load Range
5330 instruct loadRange(rRegI dst, memory mem)
5331 %{
5332   match(Set dst (LoadRange mem));
5333 
5334   ins_cost(125); // XXX
5335   format %{ "movl    $dst, $mem\t# range" %}
5336   opcode(0x8B);
5337   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5338   ins_pipe(ialu_reg_mem);
5339 %}
5340 
5341 // Load Pointer
5342 instruct loadP(rRegP dst, memory mem)
5343 %{
5344   match(Set dst (LoadP mem));
5345 
5346   ins_cost(125); // XXX
5347   format %{ "movq    $dst, $mem\t# ptr" %}
5348   opcode(0x8B);
5349   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5350   ins_pipe(ialu_reg_mem); // XXX
5351 %}
5352 
5353 // Load Compressed Pointer
5354 instruct loadN(rRegN dst, memory mem)
5355 %{
5356    match(Set dst (LoadN mem));
5357 
5358    ins_cost(125); // XXX
5359    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5360    ins_encode %{
5361      __ movl($dst$$Register, $mem$$Address);
5362    %}
5363    ins_pipe(ialu_reg_mem); // XXX
5364 %}
5365 
5366 
5367 // Load Klass Pointer
5368 instruct loadKlass(rRegP dst, memory mem)
5369 %{
5370   match(Set dst (LoadKlass mem));
5371 
5372   ins_cost(125); // XXX
5373   format %{ "movq    $dst, $mem\t# class" %}
5374   opcode(0x8B);
5375   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5376   ins_pipe(ialu_reg_mem); // XXX
5377 %}
5378 
5379 // Load narrow Klass Pointer
5380 instruct loadNKlass(rRegN dst, memory mem)
5381 %{
5382   match(Set dst (LoadNKlass mem));
5383 
5384   ins_cost(125); // XXX
5385   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5386   ins_encode %{
5387     __ movl($dst$$Register, $mem$$Address);
5388   %}
5389   ins_pipe(ialu_reg_mem); // XXX
5390 %}
5391 
5392 // Load Float
5393 instruct loadF(regF dst, memory mem)
5394 %{
5395   match(Set dst (LoadF mem));
5396 
5397   ins_cost(145); // XXX
5398   format %{ "movss   $dst, $mem\t# float" %}
5399   ins_encode %{
5400     __ movflt($dst$$XMMRegister, $mem$$Address);
5401   %}
5402   ins_pipe(pipe_slow); // XXX
5403 %}
5404 
5405 // Load Float
5406 instruct MoveF2VL(vlRegF dst, regF src) %{
5407   match(Set dst src);
5408   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5409   ins_encode %{
5410     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5411   %}
5412   ins_pipe( fpu_reg_reg );
5413 %}
5414 
5415 // Load Float
5416 instruct MoveF2LEG(legRegF dst, regF src) %{
5417   match(Set dst src);
5418   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5419   ins_encode %{
5420     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5421   %}
5422   ins_pipe( fpu_reg_reg );
5423 %}
5424 
5425 // Load Float
5426 instruct MoveVL2F(regF dst, vlRegF src) %{
5427   match(Set dst src);
5428   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5429   ins_encode %{
5430     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5431   %}
5432   ins_pipe( fpu_reg_reg );
5433 %}
5434 
5435 // Load Float
5436 instruct MoveLEG2F(regF dst, legRegF src) %{
5437   match(Set dst src);
5438   format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
5439   ins_encode %{
5440     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5441   %}
5442   ins_pipe( fpu_reg_reg );
5443 %}
5444 
5445 // Load Double
5446 instruct loadD_partial(regD dst, memory mem)
5447 %{
5448   predicate(!UseXmmLoadAndClearUpper);
5449   match(Set dst (LoadD mem));
5450 
5451   ins_cost(145); // XXX
5452   format %{ "movlpd  $dst, $mem\t# double" %}
5453   ins_encode %{
5454     __ movdbl($dst$$XMMRegister, $mem$$Address);
5455   %}
5456   ins_pipe(pipe_slow); // XXX
5457 %}
5458 
5459 instruct loadD(regD dst, memory mem)
5460 %{
5461   predicate(UseXmmLoadAndClearUpper);
5462   match(Set dst (LoadD mem));
5463 
5464   ins_cost(145); // XXX
5465   format %{ "movsd   $dst, $mem\t# double" %}
5466   ins_encode %{
5467     __ movdbl($dst$$XMMRegister, $mem$$Address);
5468   %}
5469   ins_pipe(pipe_slow); // XXX
5470 %}
5471 
5472 // Load Double
5473 instruct MoveD2VL(vlRegD dst, regD src) %{
5474   match(Set dst src);
5475   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5476   ins_encode %{
5477     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5478   %}
5479   ins_pipe( fpu_reg_reg );
5480 %}
5481 
5482 // Load Double
5483 instruct MoveD2LEG(legRegD dst, regD src) %{
5484   match(Set dst src);
5485   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5486   ins_encode %{
5487     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5488   %}
5489   ins_pipe( fpu_reg_reg );
5490 %}
5491 
5492 // Load Double
5493 instruct MoveVL2D(regD dst, vlRegD src) %{
5494   match(Set dst src);
5495   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5496   ins_encode %{
5497     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5498   %}
5499   ins_pipe( fpu_reg_reg );
5500 %}
5501 
5502 // Load Double
5503 instruct MoveLEG2D(regD dst, legRegD src) %{
5504   match(Set dst src);
5505   format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
5506   ins_encode %{
5507     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5508   %}
5509   ins_pipe( fpu_reg_reg );
5510 %}
5511 
5512 // Following pseudo code describes the algorithm for max[FD]:
5513 // Min algorithm is on similar lines
5514 //  btmp = (b < +0.0) ? a : b
5515 //  atmp = (b < +0.0) ? b : a
5516 //  Tmp  = Max_Float(atmp , btmp)
5517 //  Res  = (atmp == NaN) ? atmp : Tmp
5518 
5519 // max = java.lang.Math.max(float a, float b)
5520 instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5521   predicate(UseAVX > 0 && !n->is_reduction());
5522   match(Set dst (MaxF a b));
5523   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5524   format %{
5525      "blendvps         $btmp,$b,$a,$b           \n\t"
5526      "blendvps         $atmp,$a,$b,$b           \n\t"
5527      "vmaxss           $tmp,$atmp,$btmp         \n\t"
5528      "cmpps.unordered  $btmp,$atmp,$atmp        \n\t"
5529      "blendvps         $dst,$tmp,$atmp,$btmp    \n\t"
5530   %}
5531   ins_encode %{
5532     int vector_len = Assembler::AVX_128bit;
5533     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5534     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5535     __ vmaxss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5536     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5537     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5538  %}
5539   ins_pipe( pipe_slow );
5540 %}
5541 
5542 instruct maxF_reduction_reg(regF dst, regF a, regF b, regF xmmt, rRegI tmp, rFlagsReg cr) %{
5543   predicate(UseAVX > 0 && n->is_reduction());
5544   match(Set dst (MaxF a b));
5545   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5546 
5547   format %{ "$dst = max($a, $b)\t# intrinsic (float)" %}
5548   ins_encode %{
5549     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5550                     false /*min*/, true /*single*/);
5551   %}
5552   ins_pipe( pipe_slow );
5553 %}
5554 
5555 // max = java.lang.Math.max(double a, double b)
5556 instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5557   predicate(UseAVX > 0 && !n->is_reduction());
5558   match(Set dst (MaxD a b));
5559   effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
5560   format %{
5561      "blendvpd         $btmp,$b,$a,$b            \n\t"
5562      "blendvpd         $atmp,$a,$b,$b            \n\t"
5563      "vmaxsd           $tmp,$atmp,$btmp          \n\t"
5564      "cmppd.unordered  $btmp,$atmp,$atmp         \n\t"
5565      "blendvpd         $dst,$tmp,$atmp,$btmp     \n\t"
5566   %}
5567   ins_encode %{
5568     int vector_len = Assembler::AVX_128bit;
5569     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, vector_len);
5570     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $b$$XMMRegister, vector_len);
5571     __ vmaxsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5572     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5573     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5574   %}
5575   ins_pipe( pipe_slow );
5576 %}
5577 
5578 instruct maxD_reduction_reg(regD dst, regD a, regD b, regD xmmt, rRegL tmp, rFlagsReg cr) %{
5579   predicate(UseAVX > 0 && n->is_reduction());
5580   match(Set dst (MaxD a b));
5581   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5582 
5583   format %{ "$dst = max($a, $b)\t# intrinsic (double)" %}
5584   ins_encode %{
5585     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5586                     false /*min*/, false /*single*/);
5587   %}
5588   ins_pipe( pipe_slow );
5589 %}
5590 
5591 // min = java.lang.Math.min(float a, float b)
5592 instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
5593   predicate(UseAVX > 0 && !n->is_reduction());
5594   match(Set dst (MinF a b));
5595   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5596   format %{
5597      "blendvps         $atmp,$a,$b,$a             \n\t"
5598      "blendvps         $btmp,$b,$a,$a             \n\t"
5599      "vminss           $tmp,$atmp,$btmp           \n\t"
5600      "cmpps.unordered  $btmp,$atmp,$atmp          \n\t"
5601      "blendvps         $dst,$tmp,$atmp,$btmp      \n\t"
5602   %}
5603   ins_encode %{
5604     int vector_len = Assembler::AVX_128bit;
5605     __ blendvps($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5606     __ blendvps($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5607     __ vminss($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5608     __ cmpps($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5609     __ blendvps($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5610   %}
5611   ins_pipe( pipe_slow );
5612 %}
5613 
5614 instruct minF_reduction_reg(regF dst, regF a, regF b, regF xmmt, rRegI tmp, rFlagsReg cr) %{
5615   predicate(UseAVX > 0 && n->is_reduction());
5616   match(Set dst (MinF a b));
5617   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5618 
5619   format %{ "$dst = min($a, $b)\t# intrinsic (float)" %}
5620   ins_encode %{
5621     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5622                     true /*min*/, true /*single*/);
5623   %}
5624   ins_pipe( pipe_slow );
5625 %}
5626 
5627 // min = java.lang.Math.min(double a, double b)
5628 instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
5629   predicate(UseAVX > 0 && !n->is_reduction());
5630   match(Set dst (MinD a b));
5631   effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
5632   format %{
5633      "blendvpd         $atmp,$a,$b,$a           \n\t"
5634      "blendvpd         $btmp,$b,$a,$a           \n\t"
5635      "vminsd           $tmp,$atmp,$btmp         \n\t"
5636      "cmppd.unordered  $btmp,$atmp,$atmp        \n\t"
5637      "blendvpd         $dst,$tmp,$atmp,$btmp    \n\t"
5638   %}
5639   ins_encode %{
5640     int vector_len = Assembler::AVX_128bit;
5641     __ blendvpd($atmp$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, vector_len);
5642     __ blendvpd($btmp$$XMMRegister, $b$$XMMRegister, $a$$XMMRegister, $a$$XMMRegister, vector_len);
5643     __ vminsd($tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister);
5644     __ cmppd($btmp$$XMMRegister, $atmp$$XMMRegister, $atmp$$XMMRegister, Assembler::_false, vector_len);
5645     __ blendvpd($dst$$XMMRegister, $tmp$$XMMRegister, $atmp$$XMMRegister, $btmp$$XMMRegister, vector_len);
5646   %}
5647   ins_pipe( pipe_slow );
5648 %}
5649 
5650 instruct minD_reduction_reg(regD dst, regD a, regD b, regD xmmt, rRegL tmp, rFlagsReg cr) %{
5651   predicate(UseAVX > 0 && n->is_reduction());
5652   match(Set dst (MinD a b));
5653   effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
5654 
5655   format %{ "$dst = min($a, $b)\t# intrinsic (double)" %}
5656   ins_encode %{
5657     emit_fp_min_max(_masm, $dst$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister, $xmmt$$XMMRegister, $tmp$$Register,
5658                     true /*min*/, false /*single*/);
5659   %}
5660   ins_pipe( pipe_slow );
5661 %}
5662 
5663 // Load Effective Address
5664 instruct leaP8(rRegP dst, indOffset8 mem)
5665 %{
5666   match(Set dst mem);
5667 
5668   ins_cost(110); // XXX
5669   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5670   opcode(0x8D);
5671   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5672   ins_pipe(ialu_reg_reg_fat);
5673 %}
5674 
5675 instruct leaP32(rRegP dst, indOffset32 mem)
5676 %{
5677   match(Set dst mem);
5678 
5679   ins_cost(110);
5680   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5681   opcode(0x8D);
5682   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5683   ins_pipe(ialu_reg_reg_fat);
5684 %}
5685 
5686 // instruct leaPIdx(rRegP dst, indIndex mem)
5687 // %{
5688 //   match(Set dst mem);
5689 
5690 //   ins_cost(110);
5691 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5692 //   opcode(0x8D);
5693 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5694 //   ins_pipe(ialu_reg_reg_fat);
5695 // %}
5696 
5697 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5698 %{
5699   match(Set dst mem);
5700 
5701   ins_cost(110);
5702   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5703   opcode(0x8D);
5704   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5705   ins_pipe(ialu_reg_reg_fat);
5706 %}
5707 
5708 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5709 %{
5710   match(Set dst mem);
5711 
5712   ins_cost(110);
5713   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5714   opcode(0x8D);
5715   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5716   ins_pipe(ialu_reg_reg_fat);
5717 %}
5718 
5719 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5720 %{
5721   match(Set dst mem);
5722 
5723   ins_cost(110);
5724   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5725   opcode(0x8D);
5726   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5727   ins_pipe(ialu_reg_reg_fat);
5728 %}
5729 
5730 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5731 %{
5732   match(Set dst mem);
5733 
5734   ins_cost(110);
5735   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5736   opcode(0x8D);
5737   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5738   ins_pipe(ialu_reg_reg_fat);
5739 %}
5740 
5741 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5742 %{
5743   match(Set dst mem);
5744 
5745   ins_cost(110);
5746   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5747   opcode(0x8D);
5748   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5749   ins_pipe(ialu_reg_reg_fat);
5750 %}
5751 
5752 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5753 %{
5754   match(Set dst mem);
5755 
5756   ins_cost(110);
5757   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5758   opcode(0x8D);
5759   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5760   ins_pipe(ialu_reg_reg_fat);
5761 %}
5762 
5763 // Load Effective Address which uses Narrow (32-bits) oop
5764 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5765 %{
5766   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5767   match(Set dst mem);
5768 
5769   ins_cost(110);
5770   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5771   opcode(0x8D);
5772   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5773   ins_pipe(ialu_reg_reg_fat);
5774 %}
5775 
5776 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5777 %{
5778   predicate(Universe::narrow_oop_shift() == 0);
5779   match(Set dst mem);
5780 
5781   ins_cost(110); // XXX
5782   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5783   opcode(0x8D);
5784   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5785   ins_pipe(ialu_reg_reg_fat);
5786 %}
5787 
5788 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5789 %{
5790   predicate(Universe::narrow_oop_shift() == 0);
5791   match(Set dst mem);
5792 
5793   ins_cost(110);
5794   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5795   opcode(0x8D);
5796   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5797   ins_pipe(ialu_reg_reg_fat);
5798 %}
5799 
5800 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5801 %{
5802   predicate(Universe::narrow_oop_shift() == 0);
5803   match(Set dst mem);
5804 
5805   ins_cost(110);
5806   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5807   opcode(0x8D);
5808   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5809   ins_pipe(ialu_reg_reg_fat);
5810 %}
5811 
5812 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5813 %{
5814   predicate(Universe::narrow_oop_shift() == 0);
5815   match(Set dst mem);
5816 
5817   ins_cost(110);
5818   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5819   opcode(0x8D);
5820   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5821   ins_pipe(ialu_reg_reg_fat);
5822 %}
5823 
5824 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5825 %{
5826   predicate(Universe::narrow_oop_shift() == 0);
5827   match(Set dst mem);
5828 
5829   ins_cost(110);
5830   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5831   opcode(0x8D);
5832   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5833   ins_pipe(ialu_reg_reg_fat);
5834 %}
5835 
5836 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5837 %{
5838   predicate(Universe::narrow_oop_shift() == 0);
5839   match(Set dst mem);
5840 
5841   ins_cost(110);
5842   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5843   opcode(0x8D);
5844   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5845   ins_pipe(ialu_reg_reg_fat);
5846 %}
5847 
5848 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5849 %{
5850   predicate(Universe::narrow_oop_shift() == 0);
5851   match(Set dst mem);
5852 
5853   ins_cost(110);
5854   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5855   opcode(0x8D);
5856   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5857   ins_pipe(ialu_reg_reg_fat);
5858 %}
5859 
5860 instruct loadConI(rRegI dst, immI src)
5861 %{
5862   match(Set dst src);
5863 
5864   format %{ "movl    $dst, $src\t# int" %}
5865   ins_encode(load_immI(dst, src));
5866   ins_pipe(ialu_reg_fat); // XXX
5867 %}
5868 
5869 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5870 %{
5871   match(Set dst src);
5872   effect(KILL cr);
5873 
5874   ins_cost(50);
5875   format %{ "xorl    $dst, $dst\t# int" %}
5876   opcode(0x33); /* + rd */
5877   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5878   ins_pipe(ialu_reg);
5879 %}
5880 
5881 instruct loadConL(rRegL dst, immL src)
5882 %{
5883   match(Set dst src);
5884 
5885   ins_cost(150);
5886   format %{ "movq    $dst, $src\t# long" %}
5887   ins_encode(load_immL(dst, src));
5888   ins_pipe(ialu_reg);
5889 %}
5890 
5891 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5892 %{
5893   match(Set dst src);
5894   effect(KILL cr);
5895 
5896   ins_cost(50);
5897   format %{ "xorl    $dst, $dst\t# long" %}
5898   opcode(0x33); /* + rd */
5899   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5900   ins_pipe(ialu_reg); // XXX
5901 %}
5902 
5903 instruct loadConUL32(rRegL dst, immUL32 src)
5904 %{
5905   match(Set dst src);
5906 
5907   ins_cost(60);
5908   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5909   ins_encode(load_immUL32(dst, src));
5910   ins_pipe(ialu_reg);
5911 %}
5912 
5913 instruct loadConL32(rRegL dst, immL32 src)
5914 %{
5915   match(Set dst src);
5916 
5917   ins_cost(70);
5918   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5919   ins_encode(load_immL32(dst, src));
5920   ins_pipe(ialu_reg);
5921 %}
5922 
5923 instruct loadConP(rRegP dst, immP con) %{
5924   match(Set dst con);
5925 
5926   format %{ "movq    $dst, $con\t# ptr" %}
5927   ins_encode(load_immP(dst, con));
5928   ins_pipe(ialu_reg_fat); // XXX
5929 %}
5930 
5931 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5932 %{
5933   match(Set dst src);
5934   effect(KILL cr);
5935 
5936   ins_cost(50);
5937   format %{ "xorl    $dst, $dst\t# ptr" %}
5938   opcode(0x33); /* + rd */
5939   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5940   ins_pipe(ialu_reg);
5941 %}
5942 
5943 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5944 %{
5945   match(Set dst src);
5946   effect(KILL cr);
5947 
5948   ins_cost(60);
5949   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5950   ins_encode(load_immP31(dst, src));
5951   ins_pipe(ialu_reg);
5952 %}
5953 
5954 instruct loadConF(regF dst, immF con) %{
5955   match(Set dst con);
5956   ins_cost(125);
5957   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5958   ins_encode %{
5959     __ movflt($dst$$XMMRegister, $constantaddress($con));
5960   %}
5961   ins_pipe(pipe_slow);
5962 %}
5963 
5964 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5965   match(Set dst src);
5966   effect(KILL cr);
5967   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5968   ins_encode %{
5969     __ xorq($dst$$Register, $dst$$Register);
5970   %}
5971   ins_pipe(ialu_reg);
5972 %}
5973 
5974 instruct loadConN(rRegN dst, immN src) %{
5975   match(Set dst src);
5976 
5977   ins_cost(125);
5978   format %{ "movl    $dst, $src\t# compressed ptr" %}
5979   ins_encode %{
5980     address con = (address)$src$$constant;
5981     if (con == NULL) {
5982       ShouldNotReachHere();
5983     } else {
5984       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5985     }
5986   %}
5987   ins_pipe(ialu_reg_fat); // XXX
5988 %}
5989 
5990 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5991   match(Set dst src);
5992 
5993   ins_cost(125);
5994   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5995   ins_encode %{
5996     address con = (address)$src$$constant;
5997     if (con == NULL) {
5998       ShouldNotReachHere();
5999     } else {
6000       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
6001     }
6002   %}
6003   ins_pipe(ialu_reg_fat); // XXX
6004 %}
6005 
6006 instruct loadConF0(regF dst, immF0 src)
6007 %{
6008   match(Set dst src);
6009   ins_cost(100);
6010 
6011   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6012   ins_encode %{
6013     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6014   %}
6015   ins_pipe(pipe_slow);
6016 %}
6017 
6018 // Use the same format since predicate() can not be used here.
6019 instruct loadConD(regD dst, immD con) %{
6020   match(Set dst con);
6021   ins_cost(125);
6022   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6023   ins_encode %{
6024     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6025   %}
6026   ins_pipe(pipe_slow);
6027 %}
6028 
6029 instruct loadConD0(regD dst, immD0 src)
6030 %{
6031   match(Set dst src);
6032   ins_cost(100);
6033 
6034   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6035   ins_encode %{
6036     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6037   %}
6038   ins_pipe(pipe_slow);
6039 %}
6040 
6041 instruct loadSSI(rRegI dst, stackSlotI src)
6042 %{
6043   match(Set dst src);
6044 
6045   ins_cost(125);
6046   format %{ "movl    $dst, $src\t# int stk" %}
6047   opcode(0x8B);
6048   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6049   ins_pipe(ialu_reg_mem);
6050 %}
6051 
6052 instruct loadSSL(rRegL dst, stackSlotL src)
6053 %{
6054   match(Set dst src);
6055 
6056   ins_cost(125);
6057   format %{ "movq    $dst, $src\t# long stk" %}
6058   opcode(0x8B);
6059   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6060   ins_pipe(ialu_reg_mem);
6061 %}
6062 
6063 instruct loadSSP(rRegP dst, stackSlotP src)
6064 %{
6065   match(Set dst src);
6066 
6067   ins_cost(125);
6068   format %{ "movq    $dst, $src\t# ptr stk" %}
6069   opcode(0x8B);
6070   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6071   ins_pipe(ialu_reg_mem);
6072 %}
6073 
6074 instruct loadSSF(regF dst, stackSlotF src)
6075 %{
6076   match(Set dst src);
6077 
6078   ins_cost(125);
6079   format %{ "movss   $dst, $src\t# float stk" %}
6080   ins_encode %{
6081     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
6082   %}
6083   ins_pipe(pipe_slow); // XXX
6084 %}
6085 
6086 // Use the same format since predicate() can not be used here.
6087 instruct loadSSD(regD dst, stackSlotD src)
6088 %{
6089   match(Set dst src);
6090 
6091   ins_cost(125);
6092   format %{ "movsd   $dst, $src\t# double stk" %}
6093   ins_encode  %{
6094     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6095   %}
6096   ins_pipe(pipe_slow); // XXX
6097 %}
6098 
6099 // Prefetch instructions for allocation.
6100 // Must be safe to execute with invalid address (cannot fault).
6101 
6102 instruct prefetchAlloc( memory mem ) %{
6103   predicate(AllocatePrefetchInstr==3);
6104   match(PrefetchAllocation mem);
6105   ins_cost(125);
6106 
6107   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6108   ins_encode %{
6109     __ prefetchw($mem$$Address);
6110   %}
6111   ins_pipe(ialu_mem);
6112 %}
6113 
6114 instruct prefetchAllocNTA( memory mem ) %{
6115   predicate(AllocatePrefetchInstr==0);
6116   match(PrefetchAllocation mem);
6117   ins_cost(125);
6118 
6119   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6120   ins_encode %{
6121     __ prefetchnta($mem$$Address);
6122   %}
6123   ins_pipe(ialu_mem);
6124 %}
6125 
6126 instruct prefetchAllocT0( memory mem ) %{
6127   predicate(AllocatePrefetchInstr==1);
6128   match(PrefetchAllocation mem);
6129   ins_cost(125);
6130 
6131   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6132   ins_encode %{
6133     __ prefetcht0($mem$$Address);
6134   %}
6135   ins_pipe(ialu_mem);
6136 %}
6137 
6138 instruct prefetchAllocT2( memory mem ) %{
6139   predicate(AllocatePrefetchInstr==2);
6140   match(PrefetchAllocation mem);
6141   ins_cost(125);
6142 
6143   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6144   ins_encode %{
6145     __ prefetcht2($mem$$Address);
6146   %}
6147   ins_pipe(ialu_mem);
6148 %}
6149 
6150 //----------Store Instructions-------------------------------------------------
6151 
6152 // Store Byte
6153 instruct storeB(memory mem, rRegI src)
6154 %{
6155   match(Set mem (StoreB mem src));
6156 
6157   ins_cost(125); // XXX
6158   format %{ "movb    $mem, $src\t# byte" %}
6159   opcode(0x88);
6160   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6161   ins_pipe(ialu_mem_reg);
6162 %}
6163 
6164 // Store Char/Short
6165 instruct storeC(memory mem, rRegI src)
6166 %{
6167   match(Set mem (StoreC mem src));
6168 
6169   ins_cost(125); // XXX
6170   format %{ "movw    $mem, $src\t# char/short" %}
6171   opcode(0x89);
6172   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6173   ins_pipe(ialu_mem_reg);
6174 %}
6175 
6176 // Store Integer
6177 instruct storeI(memory mem, rRegI src)
6178 %{
6179   match(Set mem (StoreI mem src));
6180 
6181   ins_cost(125); // XXX
6182   format %{ "movl    $mem, $src\t# int" %}
6183   opcode(0x89);
6184   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6185   ins_pipe(ialu_mem_reg);
6186 %}
6187 
6188 // Store Long
6189 instruct storeL(memory mem, rRegL src)
6190 %{
6191   match(Set mem (StoreL mem src));
6192 
6193   ins_cost(125); // XXX
6194   format %{ "movq    $mem, $src\t# long" %}
6195   opcode(0x89);
6196   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6197   ins_pipe(ialu_mem_reg); // XXX
6198 %}
6199 
6200 // Store Pointer
6201 instruct storeP(memory mem, any_RegP src)
6202 %{
6203   match(Set mem (StoreP mem src));
6204 
6205   ins_cost(125); // XXX
6206   format %{ "movq    $mem, $src\t# ptr" %}
6207   opcode(0x89);
6208   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6209   ins_pipe(ialu_mem_reg);
6210 %}
6211 
6212 instruct storeImmP0(memory mem, immP0 zero)
6213 %{
6214   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6215   match(Set mem (StoreP mem zero));
6216 
6217   ins_cost(125); // XXX
6218   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6219   ins_encode %{
6220     __ movq($mem$$Address, r12);
6221   %}
6222   ins_pipe(ialu_mem_reg);
6223 %}
6224 
6225 // Store NULL Pointer, mark word, or other simple pointer constant.
6226 instruct storeImmP(memory mem, immP31 src)
6227 %{
6228   match(Set mem (StoreP mem src));
6229 
6230   ins_cost(150); // XXX
6231   format %{ "movq    $mem, $src\t# ptr" %}
6232   opcode(0xC7); /* C7 /0 */
6233   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6234   ins_pipe(ialu_mem_imm);
6235 %}
6236 
6237 // Store Compressed Pointer
6238 instruct storeN(memory mem, rRegN src)
6239 %{
6240   match(Set mem (StoreN mem src));
6241 
6242   ins_cost(125); // XXX
6243   format %{ "movl    $mem, $src\t# compressed ptr" %}
6244   ins_encode %{
6245     __ movl($mem$$Address, $src$$Register);
6246   %}
6247   ins_pipe(ialu_mem_reg);
6248 %}
6249 
6250 instruct storeNKlass(memory mem, rRegN src)
6251 %{
6252   match(Set mem (StoreNKlass mem src));
6253 
6254   ins_cost(125); // XXX
6255   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6256   ins_encode %{
6257     __ movl($mem$$Address, $src$$Register);
6258   %}
6259   ins_pipe(ialu_mem_reg);
6260 %}
6261 
6262 instruct storeImmN0(memory mem, immN0 zero)
6263 %{
6264   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
6265   match(Set mem (StoreN mem zero));
6266 
6267   ins_cost(125); // XXX
6268   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6269   ins_encode %{
6270     __ movl($mem$$Address, r12);
6271   %}
6272   ins_pipe(ialu_mem_reg);
6273 %}
6274 
6275 instruct storeImmN(memory mem, immN src)
6276 %{
6277   match(Set mem (StoreN mem src));
6278 
6279   ins_cost(150); // XXX
6280   format %{ "movl    $mem, $src\t# compressed ptr" %}
6281   ins_encode %{
6282     address con = (address)$src$$constant;
6283     if (con == NULL) {
6284       __ movl($mem$$Address, (int32_t)0);
6285     } else {
6286       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6287     }
6288   %}
6289   ins_pipe(ialu_mem_imm);
6290 %}
6291 
6292 instruct storeImmNKlass(memory mem, immNKlass src)
6293 %{
6294   match(Set mem (StoreNKlass mem src));
6295 
6296   ins_cost(150); // XXX
6297   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6298   ins_encode %{
6299     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6300   %}
6301   ins_pipe(ialu_mem_imm);
6302 %}
6303 
6304 // Store Integer Immediate
6305 instruct storeImmI0(memory mem, immI0 zero)
6306 %{
6307   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6308   match(Set mem (StoreI mem zero));
6309 
6310   ins_cost(125); // XXX
6311   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6312   ins_encode %{
6313     __ movl($mem$$Address, r12);
6314   %}
6315   ins_pipe(ialu_mem_reg);
6316 %}
6317 
6318 instruct storeImmI(memory mem, immI src)
6319 %{
6320   match(Set mem (StoreI mem src));
6321 
6322   ins_cost(150);
6323   format %{ "movl    $mem, $src\t# int" %}
6324   opcode(0xC7); /* C7 /0 */
6325   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6326   ins_pipe(ialu_mem_imm);
6327 %}
6328 
6329 // Store Long Immediate
6330 instruct storeImmL0(memory mem, immL0 zero)
6331 %{
6332   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6333   match(Set mem (StoreL mem zero));
6334 
6335   ins_cost(125); // XXX
6336   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6337   ins_encode %{
6338     __ movq($mem$$Address, r12);
6339   %}
6340   ins_pipe(ialu_mem_reg);
6341 %}
6342 
6343 instruct storeImmL(memory mem, immL32 src)
6344 %{
6345   match(Set mem (StoreL mem src));
6346 
6347   ins_cost(150);
6348   format %{ "movq    $mem, $src\t# long" %}
6349   opcode(0xC7); /* C7 /0 */
6350   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6351   ins_pipe(ialu_mem_imm);
6352 %}
6353 
6354 // Store Short/Char Immediate
6355 instruct storeImmC0(memory mem, immI0 zero)
6356 %{
6357   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6358   match(Set mem (StoreC mem zero));
6359 
6360   ins_cost(125); // XXX
6361   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6362   ins_encode %{
6363     __ movw($mem$$Address, r12);
6364   %}
6365   ins_pipe(ialu_mem_reg);
6366 %}
6367 
6368 instruct storeImmI16(memory mem, immI16 src)
6369 %{
6370   predicate(UseStoreImmI16);
6371   match(Set mem (StoreC mem src));
6372 
6373   ins_cost(150);
6374   format %{ "movw    $mem, $src\t# short/char" %}
6375   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6376   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6377   ins_pipe(ialu_mem_imm);
6378 %}
6379 
6380 // Store Byte Immediate
6381 instruct storeImmB0(memory mem, immI0 zero)
6382 %{
6383   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6384   match(Set mem (StoreB mem zero));
6385 
6386   ins_cost(125); // XXX
6387   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6388   ins_encode %{
6389     __ movb($mem$$Address, r12);
6390   %}
6391   ins_pipe(ialu_mem_reg);
6392 %}
6393 
6394 instruct storeImmB(memory mem, immI8 src)
6395 %{
6396   match(Set mem (StoreB mem src));
6397 
6398   ins_cost(150); // XXX
6399   format %{ "movb    $mem, $src\t# byte" %}
6400   opcode(0xC6); /* C6 /0 */
6401   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6402   ins_pipe(ialu_mem_imm);
6403 %}
6404 
6405 // Store CMS card-mark Immediate
6406 instruct storeImmCM0_reg(memory mem, immI0 zero)
6407 %{
6408   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6409   match(Set mem (StoreCM mem zero));
6410 
6411   ins_cost(125); // XXX
6412   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6413   ins_encode %{
6414     __ movb($mem$$Address, r12);
6415   %}
6416   ins_pipe(ialu_mem_reg);
6417 %}
6418 
6419 instruct storeImmCM0(memory mem, immI0 src)
6420 %{
6421   match(Set mem (StoreCM mem src));
6422 
6423   ins_cost(150); // XXX
6424   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6425   opcode(0xC6); /* C6 /0 */
6426   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6427   ins_pipe(ialu_mem_imm);
6428 %}
6429 
6430 // Store Float
6431 instruct storeF(memory mem, regF src)
6432 %{
6433   match(Set mem (StoreF mem src));
6434 
6435   ins_cost(95); // XXX
6436   format %{ "movss   $mem, $src\t# float" %}
6437   ins_encode %{
6438     __ movflt($mem$$Address, $src$$XMMRegister);
6439   %}
6440   ins_pipe(pipe_slow); // XXX
6441 %}
6442 
6443 // Store immediate Float value (it is faster than store from XMM register)
6444 instruct storeF0(memory mem, immF0 zero)
6445 %{
6446   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6447   match(Set mem (StoreF mem zero));
6448 
6449   ins_cost(25); // XXX
6450   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6451   ins_encode %{
6452     __ movl($mem$$Address, r12);
6453   %}
6454   ins_pipe(ialu_mem_reg);
6455 %}
6456 
6457 instruct storeF_imm(memory mem, immF src)
6458 %{
6459   match(Set mem (StoreF mem src));
6460 
6461   ins_cost(50);
6462   format %{ "movl    $mem, $src\t# float" %}
6463   opcode(0xC7); /* C7 /0 */
6464   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6465   ins_pipe(ialu_mem_imm);
6466 %}
6467 
6468 // Store Double
6469 instruct storeD(memory mem, regD src)
6470 %{
6471   match(Set mem (StoreD mem src));
6472 
6473   ins_cost(95); // XXX
6474   format %{ "movsd   $mem, $src\t# double" %}
6475   ins_encode %{
6476     __ movdbl($mem$$Address, $src$$XMMRegister);
6477   %}
6478   ins_pipe(pipe_slow); // XXX
6479 %}
6480 
6481 // Store immediate double 0.0 (it is faster than store from XMM register)
6482 instruct storeD0_imm(memory mem, immD0 src)
6483 %{
6484   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6485   match(Set mem (StoreD mem src));
6486 
6487   ins_cost(50);
6488   format %{ "movq    $mem, $src\t# double 0." %}
6489   opcode(0xC7); /* C7 /0 */
6490   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6491   ins_pipe(ialu_mem_imm);
6492 %}
6493 
6494 instruct storeD0(memory mem, immD0 zero)
6495 %{
6496   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6497   match(Set mem (StoreD mem zero));
6498 
6499   ins_cost(25); // XXX
6500   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6501   ins_encode %{
6502     __ movq($mem$$Address, r12);
6503   %}
6504   ins_pipe(ialu_mem_reg);
6505 %}
6506 
6507 instruct storeSSI(stackSlotI dst, rRegI src)
6508 %{
6509   match(Set dst src);
6510 
6511   ins_cost(100);
6512   format %{ "movl    $dst, $src\t# int stk" %}
6513   opcode(0x89);
6514   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6515   ins_pipe( ialu_mem_reg );
6516 %}
6517 
6518 instruct storeSSL(stackSlotL dst, rRegL src)
6519 %{
6520   match(Set dst src);
6521 
6522   ins_cost(100);
6523   format %{ "movq    $dst, $src\t# long stk" %}
6524   opcode(0x89);
6525   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6526   ins_pipe(ialu_mem_reg);
6527 %}
6528 
6529 instruct storeSSP(stackSlotP dst, rRegP src)
6530 %{
6531   match(Set dst src);
6532 
6533   ins_cost(100);
6534   format %{ "movq    $dst, $src\t# ptr stk" %}
6535   opcode(0x89);
6536   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6537   ins_pipe(ialu_mem_reg);
6538 %}
6539 
6540 instruct storeSSF(stackSlotF dst, regF src)
6541 %{
6542   match(Set dst src);
6543 
6544   ins_cost(95); // XXX
6545   format %{ "movss   $dst, $src\t# float stk" %}
6546   ins_encode %{
6547     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6548   %}
6549   ins_pipe(pipe_slow); // XXX
6550 %}
6551 
6552 instruct storeSSD(stackSlotD dst, regD src)
6553 %{
6554   match(Set dst src);
6555 
6556   ins_cost(95); // XXX
6557   format %{ "movsd   $dst, $src\t# double stk" %}
6558   ins_encode %{
6559     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6560   %}
6561   ins_pipe(pipe_slow); // XXX
6562 %}
6563 
6564 //----------BSWAP Instructions-------------------------------------------------
6565 instruct bytes_reverse_int(rRegI dst) %{
6566   match(Set dst (ReverseBytesI dst));
6567 
6568   format %{ "bswapl  $dst" %}
6569   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6570   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6571   ins_pipe( ialu_reg );
6572 %}
6573 
6574 instruct bytes_reverse_long(rRegL dst) %{
6575   match(Set dst (ReverseBytesL dst));
6576 
6577   format %{ "bswapq  $dst" %}
6578   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6579   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6580   ins_pipe( ialu_reg);
6581 %}
6582 
6583 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6584   match(Set dst (ReverseBytesUS dst));
6585   effect(KILL cr);
6586 
6587   format %{ "bswapl  $dst\n\t"
6588             "shrl    $dst,16\n\t" %}
6589   ins_encode %{
6590     __ bswapl($dst$$Register);
6591     __ shrl($dst$$Register, 16);
6592   %}
6593   ins_pipe( ialu_reg );
6594 %}
6595 
6596 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6597   match(Set dst (ReverseBytesS dst));
6598   effect(KILL cr);
6599 
6600   format %{ "bswapl  $dst\n\t"
6601             "sar     $dst,16\n\t" %}
6602   ins_encode %{
6603     __ bswapl($dst$$Register);
6604     __ sarl($dst$$Register, 16);
6605   %}
6606   ins_pipe( ialu_reg );
6607 %}
6608 
6609 //---------- Zeros Count Instructions ------------------------------------------
6610 
6611 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6612   predicate(UseCountLeadingZerosInstruction);
6613   match(Set dst (CountLeadingZerosI src));
6614   effect(KILL cr);
6615 
6616   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6617   ins_encode %{
6618     __ lzcntl($dst$$Register, $src$$Register);
6619   %}
6620   ins_pipe(ialu_reg);
6621 %}
6622 
6623 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6624   predicate(!UseCountLeadingZerosInstruction);
6625   match(Set dst (CountLeadingZerosI src));
6626   effect(KILL cr);
6627 
6628   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6629             "jnz     skip\n\t"
6630             "movl    $dst, -1\n"
6631       "skip:\n\t"
6632             "negl    $dst\n\t"
6633             "addl    $dst, 31" %}
6634   ins_encode %{
6635     Register Rdst = $dst$$Register;
6636     Register Rsrc = $src$$Register;
6637     Label skip;
6638     __ bsrl(Rdst, Rsrc);
6639     __ jccb(Assembler::notZero, skip);
6640     __ movl(Rdst, -1);
6641     __ bind(skip);
6642     __ negl(Rdst);
6643     __ addl(Rdst, BitsPerInt - 1);
6644   %}
6645   ins_pipe(ialu_reg);
6646 %}
6647 
6648 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6649   predicate(UseCountLeadingZerosInstruction);
6650   match(Set dst (CountLeadingZerosL src));
6651   effect(KILL cr);
6652 
6653   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6654   ins_encode %{
6655     __ lzcntq($dst$$Register, $src$$Register);
6656   %}
6657   ins_pipe(ialu_reg);
6658 %}
6659 
6660 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6661   predicate(!UseCountLeadingZerosInstruction);
6662   match(Set dst (CountLeadingZerosL src));
6663   effect(KILL cr);
6664 
6665   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6666             "jnz     skip\n\t"
6667             "movl    $dst, -1\n"
6668       "skip:\n\t"
6669             "negl    $dst\n\t"
6670             "addl    $dst, 63" %}
6671   ins_encode %{
6672     Register Rdst = $dst$$Register;
6673     Register Rsrc = $src$$Register;
6674     Label skip;
6675     __ bsrq(Rdst, Rsrc);
6676     __ jccb(Assembler::notZero, skip);
6677     __ movl(Rdst, -1);
6678     __ bind(skip);
6679     __ negl(Rdst);
6680     __ addl(Rdst, BitsPerLong - 1);
6681   %}
6682   ins_pipe(ialu_reg);
6683 %}
6684 
6685 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6686   predicate(UseCountTrailingZerosInstruction);
6687   match(Set dst (CountTrailingZerosI src));
6688   effect(KILL cr);
6689 
6690   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6691   ins_encode %{
6692     __ tzcntl($dst$$Register, $src$$Register);
6693   %}
6694   ins_pipe(ialu_reg);
6695 %}
6696 
6697 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6698   predicate(!UseCountTrailingZerosInstruction);
6699   match(Set dst (CountTrailingZerosI src));
6700   effect(KILL cr);
6701 
6702   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6703             "jnz     done\n\t"
6704             "movl    $dst, 32\n"
6705       "done:" %}
6706   ins_encode %{
6707     Register Rdst = $dst$$Register;
6708     Label done;
6709     __ bsfl(Rdst, $src$$Register);
6710     __ jccb(Assembler::notZero, done);
6711     __ movl(Rdst, BitsPerInt);
6712     __ bind(done);
6713   %}
6714   ins_pipe(ialu_reg);
6715 %}
6716 
6717 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6718   predicate(UseCountTrailingZerosInstruction);
6719   match(Set dst (CountTrailingZerosL src));
6720   effect(KILL cr);
6721 
6722   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6723   ins_encode %{
6724     __ tzcntq($dst$$Register, $src$$Register);
6725   %}
6726   ins_pipe(ialu_reg);
6727 %}
6728 
6729 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6730   predicate(!UseCountTrailingZerosInstruction);
6731   match(Set dst (CountTrailingZerosL src));
6732   effect(KILL cr);
6733 
6734   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6735             "jnz     done\n\t"
6736             "movl    $dst, 64\n"
6737       "done:" %}
6738   ins_encode %{
6739     Register Rdst = $dst$$Register;
6740     Label done;
6741     __ bsfq(Rdst, $src$$Register);
6742     __ jccb(Assembler::notZero, done);
6743     __ movl(Rdst, BitsPerLong);
6744     __ bind(done);
6745   %}
6746   ins_pipe(ialu_reg);
6747 %}
6748 
6749 
6750 //---------- Population Count Instructions -------------------------------------
6751 
6752 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6753   predicate(UsePopCountInstruction);
6754   match(Set dst (PopCountI src));
6755   effect(KILL cr);
6756 
6757   format %{ "popcnt  $dst, $src" %}
6758   ins_encode %{
6759     __ popcntl($dst$$Register, $src$$Register);
6760   %}
6761   ins_pipe(ialu_reg);
6762 %}
6763 
6764 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6765   predicate(UsePopCountInstruction);
6766   match(Set dst (PopCountI (LoadI mem)));
6767   effect(KILL cr);
6768 
6769   format %{ "popcnt  $dst, $mem" %}
6770   ins_encode %{
6771     __ popcntl($dst$$Register, $mem$$Address);
6772   %}
6773   ins_pipe(ialu_reg);
6774 %}
6775 
6776 // Note: Long.bitCount(long) returns an int.
6777 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6778   predicate(UsePopCountInstruction);
6779   match(Set dst (PopCountL src));
6780   effect(KILL cr);
6781 
6782   format %{ "popcnt  $dst, $src" %}
6783   ins_encode %{
6784     __ popcntq($dst$$Register, $src$$Register);
6785   %}
6786   ins_pipe(ialu_reg);
6787 %}
6788 
6789 // Note: Long.bitCount(long) returns an int.
6790 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6791   predicate(UsePopCountInstruction);
6792   match(Set dst (PopCountL (LoadL mem)));
6793   effect(KILL cr);
6794 
6795   format %{ "popcnt  $dst, $mem" %}
6796   ins_encode %{
6797     __ popcntq($dst$$Register, $mem$$Address);
6798   %}
6799   ins_pipe(ialu_reg);
6800 %}
6801 
6802 
6803 //----------MemBar Instructions-----------------------------------------------
6804 // Memory barrier flavors
6805 
6806 instruct membar_acquire()
6807 %{
6808   match(MemBarAcquire);
6809   match(LoadFence);
6810   ins_cost(0);
6811 
6812   size(0);
6813   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6814   ins_encode();
6815   ins_pipe(empty);
6816 %}
6817 
6818 instruct membar_acquire_lock()
6819 %{
6820   match(MemBarAcquireLock);
6821   ins_cost(0);
6822 
6823   size(0);
6824   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6825   ins_encode();
6826   ins_pipe(empty);
6827 %}
6828 
6829 instruct membar_release()
6830 %{
6831   match(MemBarRelease);
6832   match(StoreFence);
6833   ins_cost(0);
6834 
6835   size(0);
6836   format %{ "MEMBAR-release ! (empty encoding)" %}
6837   ins_encode();
6838   ins_pipe(empty);
6839 %}
6840 
6841 instruct membar_release_lock()
6842 %{
6843   match(MemBarReleaseLock);
6844   ins_cost(0);
6845 
6846   size(0);
6847   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6848   ins_encode();
6849   ins_pipe(empty);
6850 %}
6851 
6852 instruct membar_volatile(rFlagsReg cr) %{
6853   match(MemBarVolatile);
6854   effect(KILL cr);
6855   ins_cost(400);
6856 
6857   format %{
6858     $$template
6859     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6860   %}
6861   ins_encode %{
6862     __ membar(Assembler::StoreLoad);
6863   %}
6864   ins_pipe(pipe_slow);
6865 %}
6866 
6867 instruct unnecessary_membar_volatile()
6868 %{
6869   match(MemBarVolatile);
6870   predicate(Matcher::post_store_load_barrier(n));
6871   ins_cost(0);
6872 
6873   size(0);
6874   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6875   ins_encode();
6876   ins_pipe(empty);
6877 %}
6878 
6879 instruct membar_storestore() %{
6880   match(MemBarStoreStore);
6881   ins_cost(0);
6882 
6883   size(0);
6884   format %{ "MEMBAR-storestore (empty encoding)" %}
6885   ins_encode( );
6886   ins_pipe(empty);
6887 %}
6888 
6889 //----------Move Instructions--------------------------------------------------
6890 
6891 instruct castX2P(rRegP dst, rRegL src)
6892 %{
6893   match(Set dst (CastX2P src));
6894 
6895   format %{ "movq    $dst, $src\t# long->ptr" %}
6896   ins_encode %{
6897     if ($dst$$reg != $src$$reg) {
6898       __ movptr($dst$$Register, $src$$Register);
6899     }
6900   %}
6901   ins_pipe(ialu_reg_reg); // XXX
6902 %}
6903 
6904 instruct castN2X(rRegL dst, rRegN src)
6905 %{
6906   match(Set dst (CastP2X src));
6907 
6908   format %{ "movq    $dst, $src\t# ptr -> long" %}
6909   ins_encode %{
6910     if ($dst$$reg != $src$$reg) {
6911       __ movptr($dst$$Register, $src$$Register);
6912     }
6913   %}
6914   ins_pipe(ialu_reg_reg); // XXX
6915 %}
6916 
6917 instruct castP2X(rRegL dst, rRegP src)
6918 %{
6919   match(Set dst (CastP2X src));
6920 
6921   format %{ "movq    $dst, $src\t# ptr -> long" %}
6922   ins_encode %{
6923     if ($dst$$reg != $src$$reg) {
6924       __ movptr($dst$$Register, $src$$Register);
6925     }
6926   %}
6927   ins_pipe(ialu_reg_reg); // XXX
6928 %}
6929 
6930 // Convert oop into int for vectors alignment masking
6931 instruct convP2I(rRegI dst, rRegP src)
6932 %{
6933   match(Set dst (ConvL2I (CastP2X src)));
6934 
6935   format %{ "movl    $dst, $src\t# ptr -> int" %}
6936   ins_encode %{
6937     __ movl($dst$$Register, $src$$Register);
6938   %}
6939   ins_pipe(ialu_reg_reg); // XXX
6940 %}
6941 
6942 // Convert compressed oop into int for vectors alignment masking
6943 // in case of 32bit oops (heap < 4Gb).
6944 instruct convN2I(rRegI dst, rRegN src)
6945 %{
6946   predicate(Universe::narrow_oop_shift() == 0);
6947   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6948 
6949   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6950   ins_encode %{
6951     __ movl($dst$$Register, $src$$Register);
6952   %}
6953   ins_pipe(ialu_reg_reg); // XXX
6954 %}
6955 
6956 // Convert oop pointer into compressed form
6957 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6958   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6959   match(Set dst (EncodeP src));
6960   effect(KILL cr);
6961   format %{ "encode_heap_oop $dst,$src" %}
6962   ins_encode %{
6963     Register s = $src$$Register;
6964     Register d = $dst$$Register;
6965     if (s != d) {
6966       __ movq(d, s);
6967     }
6968     __ encode_heap_oop(d);
6969   %}
6970   ins_pipe(ialu_reg_long);
6971 %}
6972 
6973 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6974   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6975   match(Set dst (EncodeP src));
6976   effect(KILL cr);
6977   format %{ "encode_heap_oop_not_null $dst,$src" %}
6978   ins_encode %{
6979     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6980   %}
6981   ins_pipe(ialu_reg_long);
6982 %}
6983 
6984 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6985   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6986             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6987   match(Set dst (DecodeN src));
6988   effect(KILL cr);
6989   format %{ "decode_heap_oop $dst,$src" %}
6990   ins_encode %{
6991     Register s = $src$$Register;
6992     Register d = $dst$$Register;
6993     if (s != d) {
6994       __ movq(d, s);
6995     }
6996     __ decode_heap_oop(d);
6997   %}
6998   ins_pipe(ialu_reg_long);
6999 %}
7000 
7001 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7002   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7003             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7004   match(Set dst (DecodeN src));
7005   effect(KILL cr);
7006   format %{ "decode_heap_oop_not_null $dst,$src" %}
7007   ins_encode %{
7008     Register s = $src$$Register;
7009     Register d = $dst$$Register;
7010     if (s != d) {
7011       __ decode_heap_oop_not_null(d, s);
7012     } else {
7013       __ decode_heap_oop_not_null(d);
7014     }
7015   %}
7016   ins_pipe(ialu_reg_long);
7017 %}
7018 
7019 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7020   match(Set dst (EncodePKlass src));
7021   effect(KILL cr);
7022   format %{ "encode_klass_not_null $dst,$src" %}
7023   ins_encode %{
7024     __ encode_klass_not_null($dst$$Register, $src$$Register);
7025   %}
7026   ins_pipe(ialu_reg_long);
7027 %}
7028 
7029 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7030   match(Set dst (DecodeNKlass src));
7031   effect(KILL cr);
7032   format %{ "decode_klass_not_null $dst,$src" %}
7033   ins_encode %{
7034     Register s = $src$$Register;
7035     Register d = $dst$$Register;
7036     if (s != d) {
7037       __ decode_klass_not_null(d, s);
7038     } else {
7039       __ decode_klass_not_null(d);
7040     }
7041   %}
7042   ins_pipe(ialu_reg_long);
7043 %}
7044 
7045 
7046 //----------Conditional Move---------------------------------------------------
7047 // Jump
7048 // dummy instruction for generating temp registers
7049 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7050   match(Jump (LShiftL switch_val shift));
7051   ins_cost(350);
7052   predicate(false);
7053   effect(TEMP dest);
7054 
7055   format %{ "leaq    $dest, [$constantaddress]\n\t"
7056             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7057   ins_encode %{
7058     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7059     // to do that and the compiler is using that register as one it can allocate.
7060     // So we build it all by hand.
7061     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7062     // ArrayAddress dispatch(table, index);
7063     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7064     __ lea($dest$$Register, $constantaddress);
7065     __ jmp(dispatch);
7066   %}
7067   ins_pipe(pipe_jmp);
7068 %}
7069 
7070 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7071   match(Jump (AddL (LShiftL switch_val shift) offset));
7072   ins_cost(350);
7073   effect(TEMP dest);
7074 
7075   format %{ "leaq    $dest, [$constantaddress]\n\t"
7076             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7077   ins_encode %{
7078     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7079     // to do that and the compiler is using that register as one it can allocate.
7080     // So we build it all by hand.
7081     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7082     // ArrayAddress dispatch(table, index);
7083     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7084     __ lea($dest$$Register, $constantaddress);
7085     __ jmp(dispatch);
7086   %}
7087   ins_pipe(pipe_jmp);
7088 %}
7089 
7090 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7091   match(Jump switch_val);
7092   ins_cost(350);
7093   effect(TEMP dest);
7094 
7095   format %{ "leaq    $dest, [$constantaddress]\n\t"
7096             "jmp     [$dest + $switch_val]\n\t" %}
7097   ins_encode %{
7098     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7099     // to do that and the compiler is using that register as one it can allocate.
7100     // So we build it all by hand.
7101     // Address index(noreg, switch_reg, Address::times_1);
7102     // ArrayAddress dispatch(table, index);
7103     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7104     __ lea($dest$$Register, $constantaddress);
7105     __ jmp(dispatch);
7106   %}
7107   ins_pipe(pipe_jmp);
7108 %}
7109 
7110 // Conditional move
7111 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7112 %{
7113   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7114 
7115   ins_cost(200); // XXX
7116   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7117   opcode(0x0F, 0x40);
7118   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7119   ins_pipe(pipe_cmov_reg);
7120 %}
7121 
7122 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7123   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7124 
7125   ins_cost(200); // XXX
7126   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7127   opcode(0x0F, 0x40);
7128   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7129   ins_pipe(pipe_cmov_reg);
7130 %}
7131 
7132 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7133   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7134   ins_cost(200);
7135   expand %{
7136     cmovI_regU(cop, cr, dst, src);
7137   %}
7138 %}
7139 
7140 // Conditional move
7141 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7142   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7143 
7144   ins_cost(250); // XXX
7145   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7146   opcode(0x0F, 0x40);
7147   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7148   ins_pipe(pipe_cmov_mem);
7149 %}
7150 
7151 // Conditional move
7152 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7153 %{
7154   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7155 
7156   ins_cost(250); // XXX
7157   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7158   opcode(0x0F, 0x40);
7159   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7160   ins_pipe(pipe_cmov_mem);
7161 %}
7162 
7163 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7164   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7165   ins_cost(250);
7166   expand %{
7167     cmovI_memU(cop, cr, dst, src);
7168   %}
7169 %}
7170 
7171 // Conditional move
7172 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7173 %{
7174   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7175 
7176   ins_cost(200); // XXX
7177   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7178   opcode(0x0F, 0x40);
7179   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7180   ins_pipe(pipe_cmov_reg);
7181 %}
7182 
7183 // Conditional move
7184 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7185 %{
7186   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7187 
7188   ins_cost(200); // XXX
7189   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7190   opcode(0x0F, 0x40);
7191   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7192   ins_pipe(pipe_cmov_reg);
7193 %}
7194 
7195 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7196   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7197   ins_cost(200);
7198   expand %{
7199     cmovN_regU(cop, cr, dst, src);
7200   %}
7201 %}
7202 
7203 // Conditional move
7204 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7205 %{
7206   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7207 
7208   ins_cost(200); // XXX
7209   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7210   opcode(0x0F, 0x40);
7211   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7212   ins_pipe(pipe_cmov_reg);  // XXX
7213 %}
7214 
7215 // Conditional move
7216 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7217 %{
7218   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7219 
7220   ins_cost(200); // XXX
7221   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7222   opcode(0x0F, 0x40);
7223   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7224   ins_pipe(pipe_cmov_reg); // XXX
7225 %}
7226 
7227 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7228   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7229   ins_cost(200);
7230   expand %{
7231     cmovP_regU(cop, cr, dst, src);
7232   %}
7233 %}
7234 
7235 // DISABLED: Requires the ADLC to emit a bottom_type call that
7236 // correctly meets the two pointer arguments; one is an incoming
7237 // register but the other is a memory operand.  ALSO appears to
7238 // be buggy with implicit null checks.
7239 //
7240 //// Conditional move
7241 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7242 //%{
7243 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7244 //  ins_cost(250);
7245 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7246 //  opcode(0x0F,0x40);
7247 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7248 //  ins_pipe( pipe_cmov_mem );
7249 //%}
7250 //
7251 //// Conditional move
7252 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7253 //%{
7254 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7255 //  ins_cost(250);
7256 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7257 //  opcode(0x0F,0x40);
7258 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7259 //  ins_pipe( pipe_cmov_mem );
7260 //%}
7261 
7262 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7263 %{
7264   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7265 
7266   ins_cost(200); // XXX
7267   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7268   opcode(0x0F, 0x40);
7269   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7270   ins_pipe(pipe_cmov_reg);  // XXX
7271 %}
7272 
7273 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7274 %{
7275   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7276 
7277   ins_cost(200); // XXX
7278   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7279   opcode(0x0F, 0x40);
7280   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7281   ins_pipe(pipe_cmov_mem);  // XXX
7282 %}
7283 
7284 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7285 %{
7286   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7287 
7288   ins_cost(200); // XXX
7289   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7290   opcode(0x0F, 0x40);
7291   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7292   ins_pipe(pipe_cmov_reg); // XXX
7293 %}
7294 
7295 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7296   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7297   ins_cost(200);
7298   expand %{
7299     cmovL_regU(cop, cr, dst, src);
7300   %}
7301 %}
7302 
7303 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7304 %{
7305   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7306 
7307   ins_cost(200); // XXX
7308   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7309   opcode(0x0F, 0x40);
7310   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7311   ins_pipe(pipe_cmov_mem); // XXX
7312 %}
7313 
7314 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7315   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7316   ins_cost(200);
7317   expand %{
7318     cmovL_memU(cop, cr, dst, src);
7319   %}
7320 %}
7321 
7322 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7323 %{
7324   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7325 
7326   ins_cost(200); // XXX
7327   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7328             "movss     $dst, $src\n"
7329     "skip:" %}
7330   ins_encode %{
7331     Label Lskip;
7332     // Invert sense of branch from sense of CMOV
7333     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7334     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7335     __ bind(Lskip);
7336   %}
7337   ins_pipe(pipe_slow);
7338 %}
7339 
7340 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7341 // %{
7342 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7343 
7344 //   ins_cost(200); // XXX
7345 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7346 //             "movss     $dst, $src\n"
7347 //     "skip:" %}
7348 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7349 //   ins_pipe(pipe_slow);
7350 // %}
7351 
7352 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7353 %{
7354   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7355 
7356   ins_cost(200); // XXX
7357   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7358             "movss     $dst, $src\n"
7359     "skip:" %}
7360   ins_encode %{
7361     Label Lskip;
7362     // Invert sense of branch from sense of CMOV
7363     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7364     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7365     __ bind(Lskip);
7366   %}
7367   ins_pipe(pipe_slow);
7368 %}
7369 
7370 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7371   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7372   ins_cost(200);
7373   expand %{
7374     cmovF_regU(cop, cr, dst, src);
7375   %}
7376 %}
7377 
7378 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7379 %{
7380   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7381 
7382   ins_cost(200); // XXX
7383   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7384             "movsd     $dst, $src\n"
7385     "skip:" %}
7386   ins_encode %{
7387     Label Lskip;
7388     // Invert sense of branch from sense of CMOV
7389     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7390     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7391     __ bind(Lskip);
7392   %}
7393   ins_pipe(pipe_slow);
7394 %}
7395 
7396 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7397 %{
7398   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7399 
7400   ins_cost(200); // XXX
7401   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7402             "movsd     $dst, $src\n"
7403     "skip:" %}
7404   ins_encode %{
7405     Label Lskip;
7406     // Invert sense of branch from sense of CMOV
7407     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7408     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7409     __ bind(Lskip);
7410   %}
7411   ins_pipe(pipe_slow);
7412 %}
7413 
7414 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7415   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7416   ins_cost(200);
7417   expand %{
7418     cmovD_regU(cop, cr, dst, src);
7419   %}
7420 %}
7421 
7422 //----------Arithmetic Instructions--------------------------------------------
7423 //----------Addition Instructions----------------------------------------------
7424 
7425 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7426 %{
7427   match(Set dst (AddI dst src));
7428   effect(KILL cr);
7429 
7430   format %{ "addl    $dst, $src\t# int" %}
7431   opcode(0x03);
7432   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7433   ins_pipe(ialu_reg_reg);
7434 %}
7435 
7436 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7437 %{
7438   match(Set dst (AddI dst src));
7439   effect(KILL cr);
7440 
7441   format %{ "addl    $dst, $src\t# int" %}
7442   opcode(0x81, 0x00); /* /0 id */
7443   ins_encode(OpcSErm(dst, src), Con8or32(src));
7444   ins_pipe( ialu_reg );
7445 %}
7446 
7447 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7448 %{
7449   match(Set dst (AddI dst (LoadI src)));
7450   effect(KILL cr);
7451 
7452   ins_cost(125); // XXX
7453   format %{ "addl    $dst, $src\t# int" %}
7454   opcode(0x03);
7455   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7456   ins_pipe(ialu_reg_mem);
7457 %}
7458 
7459 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7460 %{
7461   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7462   effect(KILL cr);
7463 
7464   ins_cost(150); // XXX
7465   format %{ "addl    $dst, $src\t# int" %}
7466   opcode(0x01); /* Opcode 01 /r */
7467   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7468   ins_pipe(ialu_mem_reg);
7469 %}
7470 
7471 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7472 %{
7473   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7474   effect(KILL cr);
7475 
7476   ins_cost(125); // XXX
7477   format %{ "addl    $dst, $src\t# int" %}
7478   opcode(0x81); /* Opcode 81 /0 id */
7479   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7480   ins_pipe(ialu_mem_imm);
7481 %}
7482 
7483 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7484 %{
7485   predicate(UseIncDec);
7486   match(Set dst (AddI dst src));
7487   effect(KILL cr);
7488 
7489   format %{ "incl    $dst\t# int" %}
7490   opcode(0xFF, 0x00); // FF /0
7491   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7492   ins_pipe(ialu_reg);
7493 %}
7494 
7495 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7496 %{
7497   predicate(UseIncDec);
7498   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7499   effect(KILL cr);
7500 
7501   ins_cost(125); // XXX
7502   format %{ "incl    $dst\t# int" %}
7503   opcode(0xFF); /* Opcode FF /0 */
7504   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7505   ins_pipe(ialu_mem_imm);
7506 %}
7507 
7508 // XXX why does that use AddI
7509 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7510 %{
7511   predicate(UseIncDec);
7512   match(Set dst (AddI dst src));
7513   effect(KILL cr);
7514 
7515   format %{ "decl    $dst\t# int" %}
7516   opcode(0xFF, 0x01); // FF /1
7517   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7518   ins_pipe(ialu_reg);
7519 %}
7520 
7521 // XXX why does that use AddI
7522 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7523 %{
7524   predicate(UseIncDec);
7525   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7526   effect(KILL cr);
7527 
7528   ins_cost(125); // XXX
7529   format %{ "decl    $dst\t# int" %}
7530   opcode(0xFF); /* Opcode FF /1 */
7531   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7532   ins_pipe(ialu_mem_imm);
7533 %}
7534 
7535 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7536 %{
7537   match(Set dst (AddI src0 src1));
7538 
7539   ins_cost(110);
7540   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7541   opcode(0x8D); /* 0x8D /r */
7542   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7543   ins_pipe(ialu_reg_reg);
7544 %}
7545 
7546 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7547 %{
7548   match(Set dst (AddL dst src));
7549   effect(KILL cr);
7550 
7551   format %{ "addq    $dst, $src\t# long" %}
7552   opcode(0x03);
7553   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7554   ins_pipe(ialu_reg_reg);
7555 %}
7556 
7557 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7558 %{
7559   match(Set dst (AddL dst src));
7560   effect(KILL cr);
7561 
7562   format %{ "addq    $dst, $src\t# long" %}
7563   opcode(0x81, 0x00); /* /0 id */
7564   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7565   ins_pipe( ialu_reg );
7566 %}
7567 
7568 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7569 %{
7570   match(Set dst (AddL dst (LoadL src)));
7571   effect(KILL cr);
7572 
7573   ins_cost(125); // XXX
7574   format %{ "addq    $dst, $src\t# long" %}
7575   opcode(0x03);
7576   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7577   ins_pipe(ialu_reg_mem);
7578 %}
7579 
7580 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7581 %{
7582   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7583   effect(KILL cr);
7584 
7585   ins_cost(150); // XXX
7586   format %{ "addq    $dst, $src\t# long" %}
7587   opcode(0x01); /* Opcode 01 /r */
7588   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7589   ins_pipe(ialu_mem_reg);
7590 %}
7591 
7592 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7593 %{
7594   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7595   effect(KILL cr);
7596 
7597   ins_cost(125); // XXX
7598   format %{ "addq    $dst, $src\t# long" %}
7599   opcode(0x81); /* Opcode 81 /0 id */
7600   ins_encode(REX_mem_wide(dst),
7601              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7602   ins_pipe(ialu_mem_imm);
7603 %}
7604 
7605 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7606 %{
7607   predicate(UseIncDec);
7608   match(Set dst (AddL dst src));
7609   effect(KILL cr);
7610 
7611   format %{ "incq    $dst\t# long" %}
7612   opcode(0xFF, 0x00); // FF /0
7613   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7614   ins_pipe(ialu_reg);
7615 %}
7616 
7617 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7618 %{
7619   predicate(UseIncDec);
7620   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7621   effect(KILL cr);
7622 
7623   ins_cost(125); // XXX
7624   format %{ "incq    $dst\t# long" %}
7625   opcode(0xFF); /* Opcode FF /0 */
7626   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7627   ins_pipe(ialu_mem_imm);
7628 %}
7629 
7630 // XXX why does that use AddL
7631 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7632 %{
7633   predicate(UseIncDec);
7634   match(Set dst (AddL dst src));
7635   effect(KILL cr);
7636 
7637   format %{ "decq    $dst\t# long" %}
7638   opcode(0xFF, 0x01); // FF /1
7639   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7640   ins_pipe(ialu_reg);
7641 %}
7642 
7643 // XXX why does that use AddL
7644 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7645 %{
7646   predicate(UseIncDec);
7647   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7648   effect(KILL cr);
7649 
7650   ins_cost(125); // XXX
7651   format %{ "decq    $dst\t# long" %}
7652   opcode(0xFF); /* Opcode FF /1 */
7653   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7654   ins_pipe(ialu_mem_imm);
7655 %}
7656 
7657 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7658 %{
7659   match(Set dst (AddL src0 src1));
7660 
7661   ins_cost(110);
7662   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7663   opcode(0x8D); /* 0x8D /r */
7664   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7665   ins_pipe(ialu_reg_reg);
7666 %}
7667 
7668 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7669 %{
7670   match(Set dst (AddP dst src));
7671   effect(KILL cr);
7672 
7673   format %{ "addq    $dst, $src\t# ptr" %}
7674   opcode(0x03);
7675   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7676   ins_pipe(ialu_reg_reg);
7677 %}
7678 
7679 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7680 %{
7681   match(Set dst (AddP dst src));
7682   effect(KILL cr);
7683 
7684   format %{ "addq    $dst, $src\t# ptr" %}
7685   opcode(0x81, 0x00); /* /0 id */
7686   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7687   ins_pipe( ialu_reg );
7688 %}
7689 
7690 // XXX addP mem ops ????
7691 
7692 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7693 %{
7694   match(Set dst (AddP src0 src1));
7695 
7696   ins_cost(110);
7697   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7698   opcode(0x8D); /* 0x8D /r */
7699   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7700   ins_pipe(ialu_reg_reg);
7701 %}
7702 
7703 instruct checkCastPP(rRegP dst)
7704 %{
7705   match(Set dst (CheckCastPP dst));
7706 
7707   size(0);
7708   format %{ "# checkcastPP of $dst" %}
7709   ins_encode(/* empty encoding */);
7710   ins_pipe(empty);
7711 %}
7712 
7713 instruct castPP(rRegP dst)
7714 %{
7715   match(Set dst (CastPP dst));
7716 
7717   size(0);
7718   format %{ "# castPP of $dst" %}
7719   ins_encode(/* empty encoding */);
7720   ins_pipe(empty);
7721 %}
7722 
7723 instruct castII(rRegI dst)
7724 %{
7725   match(Set dst (CastII dst));
7726 
7727   size(0);
7728   format %{ "# castII of $dst" %}
7729   ins_encode(/* empty encoding */);
7730   ins_cost(0);
7731   ins_pipe(empty);
7732 %}
7733 
7734 // LoadP-locked same as a regular LoadP when used with compare-swap
7735 instruct loadPLocked(rRegP dst, memory mem)
7736 %{
7737   match(Set dst (LoadPLocked mem));
7738 
7739   ins_cost(125); // XXX
7740   format %{ "movq    $dst, $mem\t# ptr locked" %}
7741   opcode(0x8B);
7742   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7743   ins_pipe(ialu_reg_mem); // XXX
7744 %}
7745 
7746 // Conditional-store of the updated heap-top.
7747 // Used during allocation of the shared heap.
7748 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7749 
7750 instruct storePConditional(memory heap_top_ptr,
7751                            rax_RegP oldval, rRegP newval,
7752                            rFlagsReg cr)
7753 %{
7754   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7755 
7756   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7757             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7758   opcode(0x0F, 0xB1);
7759   ins_encode(lock_prefix,
7760              REX_reg_mem_wide(newval, heap_top_ptr),
7761              OpcP, OpcS,
7762              reg_mem(newval, heap_top_ptr));
7763   ins_pipe(pipe_cmpxchg);
7764 %}
7765 
7766 // Conditional-store of an int value.
7767 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7768 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7769 %{
7770   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7771   effect(KILL oldval);
7772 
7773   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7774   opcode(0x0F, 0xB1);
7775   ins_encode(lock_prefix,
7776              REX_reg_mem(newval, mem),
7777              OpcP, OpcS,
7778              reg_mem(newval, mem));
7779   ins_pipe(pipe_cmpxchg);
7780 %}
7781 
7782 // Conditional-store of a long value.
7783 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7784 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7785 %{
7786   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7787   effect(KILL oldval);
7788 
7789   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7790   opcode(0x0F, 0xB1);
7791   ins_encode(lock_prefix,
7792              REX_reg_mem_wide(newval, mem),
7793              OpcP, OpcS,
7794              reg_mem(newval, mem));
7795   ins_pipe(pipe_cmpxchg);
7796 %}
7797 
7798 
7799 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7800 instruct compareAndSwapP(rRegI res,
7801                          memory mem_ptr,
7802                          rax_RegP oldval, rRegP newval,
7803                          rFlagsReg cr)
7804 %{
7805   predicate(VM_Version::supports_cx8());
7806   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7807   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7808   effect(KILL cr, KILL oldval);
7809 
7810   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7811             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7812             "sete    $res\n\t"
7813             "movzbl  $res, $res" %}
7814   opcode(0x0F, 0xB1);
7815   ins_encode(lock_prefix,
7816              REX_reg_mem_wide(newval, mem_ptr),
7817              OpcP, OpcS,
7818              reg_mem(newval, mem_ptr),
7819              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7820              REX_reg_breg(res, res), // movzbl
7821              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7822   ins_pipe( pipe_cmpxchg );
7823 %}
7824 
7825 instruct compareAndSwapL(rRegI res,
7826                          memory mem_ptr,
7827                          rax_RegL oldval, rRegL newval,
7828                          rFlagsReg cr)
7829 %{
7830   predicate(VM_Version::supports_cx8());
7831   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7832   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7833   effect(KILL cr, KILL oldval);
7834 
7835   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7836             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7837             "sete    $res\n\t"
7838             "movzbl  $res, $res" %}
7839   opcode(0x0F, 0xB1);
7840   ins_encode(lock_prefix,
7841              REX_reg_mem_wide(newval, mem_ptr),
7842              OpcP, OpcS,
7843              reg_mem(newval, mem_ptr),
7844              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7845              REX_reg_breg(res, res), // movzbl
7846              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7847   ins_pipe( pipe_cmpxchg );
7848 %}
7849 
7850 instruct compareAndSwapI(rRegI res,
7851                          memory mem_ptr,
7852                          rax_RegI oldval, rRegI newval,
7853                          rFlagsReg cr)
7854 %{
7855   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7856   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7857   effect(KILL cr, KILL oldval);
7858 
7859   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7860             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7861             "sete    $res\n\t"
7862             "movzbl  $res, $res" %}
7863   opcode(0x0F, 0xB1);
7864   ins_encode(lock_prefix,
7865              REX_reg_mem(newval, mem_ptr),
7866              OpcP, OpcS,
7867              reg_mem(newval, mem_ptr),
7868              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7869              REX_reg_breg(res, res), // movzbl
7870              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7871   ins_pipe( pipe_cmpxchg );
7872 %}
7873 
7874 instruct compareAndSwapB(rRegI res,
7875                          memory mem_ptr,
7876                          rax_RegI oldval, rRegI newval,
7877                          rFlagsReg cr)
7878 %{
7879   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7880   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7881   effect(KILL cr, KILL oldval);
7882 
7883   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7884             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7885             "sete    $res\n\t"
7886             "movzbl  $res, $res" %}
7887   opcode(0x0F, 0xB0);
7888   ins_encode(lock_prefix,
7889              REX_breg_mem(newval, mem_ptr),
7890              OpcP, OpcS,
7891              reg_mem(newval, mem_ptr),
7892              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7893              REX_reg_breg(res, res), // movzbl
7894              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7895   ins_pipe( pipe_cmpxchg );
7896 %}
7897 
7898 instruct compareAndSwapS(rRegI res,
7899                          memory mem_ptr,
7900                          rax_RegI oldval, rRegI newval,
7901                          rFlagsReg cr)
7902 %{
7903   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7904   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7905   effect(KILL cr, KILL oldval);
7906 
7907   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7908             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7909             "sete    $res\n\t"
7910             "movzbl  $res, $res" %}
7911   opcode(0x0F, 0xB1);
7912   ins_encode(lock_prefix,
7913              SizePrefix,
7914              REX_reg_mem(newval, mem_ptr),
7915              OpcP, OpcS,
7916              reg_mem(newval, mem_ptr),
7917              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7918              REX_reg_breg(res, res), // movzbl
7919              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7920   ins_pipe( pipe_cmpxchg );
7921 %}
7922 
7923 instruct compareAndSwapN(rRegI res,
7924                           memory mem_ptr,
7925                           rax_RegN oldval, rRegN newval,
7926                           rFlagsReg cr) %{
7927   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7928   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7929   effect(KILL cr, KILL oldval);
7930 
7931   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7932             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7933             "sete    $res\n\t"
7934             "movzbl  $res, $res" %}
7935   opcode(0x0F, 0xB1);
7936   ins_encode(lock_prefix,
7937              REX_reg_mem(newval, mem_ptr),
7938              OpcP, OpcS,
7939              reg_mem(newval, mem_ptr),
7940              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7941              REX_reg_breg(res, res), // movzbl
7942              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7943   ins_pipe( pipe_cmpxchg );
7944 %}
7945 
7946 instruct compareAndExchangeB(
7947                          memory mem_ptr,
7948                          rax_RegI oldval, rRegI newval,
7949                          rFlagsReg cr)
7950 %{
7951   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7952   effect(KILL cr);
7953 
7954   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7955             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7956   opcode(0x0F, 0xB0);
7957   ins_encode(lock_prefix,
7958              REX_breg_mem(newval, mem_ptr),
7959              OpcP, OpcS,
7960              reg_mem(newval, mem_ptr) // lock cmpxchg
7961              );
7962   ins_pipe( pipe_cmpxchg );
7963 %}
7964 
7965 instruct compareAndExchangeS(
7966                          memory mem_ptr,
7967                          rax_RegI oldval, rRegI newval,
7968                          rFlagsReg cr)
7969 %{
7970   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7971   effect(KILL cr);
7972 
7973   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7974             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7975   opcode(0x0F, 0xB1);
7976   ins_encode(lock_prefix,
7977              SizePrefix,
7978              REX_reg_mem(newval, mem_ptr),
7979              OpcP, OpcS,
7980              reg_mem(newval, mem_ptr) // lock cmpxchg
7981              );
7982   ins_pipe( pipe_cmpxchg );
7983 %}
7984 
7985 instruct compareAndExchangeI(
7986                          memory mem_ptr,
7987                          rax_RegI oldval, rRegI newval,
7988                          rFlagsReg cr)
7989 %{
7990   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7991   effect(KILL cr);
7992 
7993   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7994             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7995   opcode(0x0F, 0xB1);
7996   ins_encode(lock_prefix,
7997              REX_reg_mem(newval, mem_ptr),
7998              OpcP, OpcS,
7999              reg_mem(newval, mem_ptr) // lock cmpxchg
8000              );
8001   ins_pipe( pipe_cmpxchg );
8002 %}
8003 
8004 instruct compareAndExchangeL(
8005                          memory mem_ptr,
8006                          rax_RegL oldval, rRegL newval,
8007                          rFlagsReg cr)
8008 %{
8009   predicate(VM_Version::supports_cx8());
8010   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
8011   effect(KILL cr);
8012 
8013   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8014             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8015   opcode(0x0F, 0xB1);
8016   ins_encode(lock_prefix,
8017              REX_reg_mem_wide(newval, mem_ptr),
8018              OpcP, OpcS,
8019              reg_mem(newval, mem_ptr)  // lock cmpxchg
8020             );
8021   ins_pipe( pipe_cmpxchg );
8022 %}
8023 
8024 instruct compareAndExchangeN(
8025                           memory mem_ptr,
8026                           rax_RegN oldval, rRegN newval,
8027                           rFlagsReg cr) %{
8028   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
8029   effect(KILL cr);
8030 
8031   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8032             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8033   opcode(0x0F, 0xB1);
8034   ins_encode(lock_prefix,
8035              REX_reg_mem(newval, mem_ptr),
8036              OpcP, OpcS,
8037              reg_mem(newval, mem_ptr)  // lock cmpxchg
8038           );
8039   ins_pipe( pipe_cmpxchg );
8040 %}
8041 
8042 instruct compareAndExchangeP(
8043                          memory mem_ptr,
8044                          rax_RegP oldval, rRegP newval,
8045                          rFlagsReg cr)
8046 %{
8047   predicate(VM_Version::supports_cx8());
8048   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
8049   effect(KILL cr);
8050 
8051   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8052             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8053   opcode(0x0F, 0xB1);
8054   ins_encode(lock_prefix,
8055              REX_reg_mem_wide(newval, mem_ptr),
8056              OpcP, OpcS,
8057              reg_mem(newval, mem_ptr)  // lock cmpxchg
8058           );
8059   ins_pipe( pipe_cmpxchg );
8060 %}
8061 
8062 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8063   predicate(n->as_LoadStore()->result_not_used());
8064   match(Set dummy (GetAndAddB mem add));
8065   effect(KILL cr);
8066   format %{ "ADDB  [$mem],$add" %}
8067   ins_encode %{
8068     __ lock();
8069     __ addb($mem$$Address, $add$$constant);
8070   %}
8071   ins_pipe( pipe_cmpxchg );
8072 %}
8073 
8074 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
8075   match(Set newval (GetAndAddB mem newval));
8076   effect(KILL cr);
8077   format %{ "XADDB  [$mem],$newval" %}
8078   ins_encode %{
8079     __ lock();
8080     __ xaddb($mem$$Address, $newval$$Register);
8081   %}
8082   ins_pipe( pipe_cmpxchg );
8083 %}
8084 
8085 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8086   predicate(n->as_LoadStore()->result_not_used());
8087   match(Set dummy (GetAndAddS mem add));
8088   effect(KILL cr);
8089   format %{ "ADDW  [$mem],$add" %}
8090   ins_encode %{
8091     __ lock();
8092     __ addw($mem$$Address, $add$$constant);
8093   %}
8094   ins_pipe( pipe_cmpxchg );
8095 %}
8096 
8097 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
8098   match(Set newval (GetAndAddS mem newval));
8099   effect(KILL cr);
8100   format %{ "XADDW  [$mem],$newval" %}
8101   ins_encode %{
8102     __ lock();
8103     __ xaddw($mem$$Address, $newval$$Register);
8104   %}
8105   ins_pipe( pipe_cmpxchg );
8106 %}
8107 
8108 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8109   predicate(n->as_LoadStore()->result_not_used());
8110   match(Set dummy (GetAndAddI mem add));
8111   effect(KILL cr);
8112   format %{ "ADDL  [$mem],$add" %}
8113   ins_encode %{
8114     __ lock();
8115     __ addl($mem$$Address, $add$$constant);
8116   %}
8117   ins_pipe( pipe_cmpxchg );
8118 %}
8119 
8120 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
8121   match(Set newval (GetAndAddI mem newval));
8122   effect(KILL cr);
8123   format %{ "XADDL  [$mem],$newval" %}
8124   ins_encode %{
8125     __ lock();
8126     __ xaddl($mem$$Address, $newval$$Register);
8127   %}
8128   ins_pipe( pipe_cmpxchg );
8129 %}
8130 
8131 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
8132   predicate(n->as_LoadStore()->result_not_used());
8133   match(Set dummy (GetAndAddL mem add));
8134   effect(KILL cr);
8135   format %{ "ADDQ  [$mem],$add" %}
8136   ins_encode %{
8137     __ lock();
8138     __ addq($mem$$Address, $add$$constant);
8139   %}
8140   ins_pipe( pipe_cmpxchg );
8141 %}
8142 
8143 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8144   match(Set newval (GetAndAddL mem newval));
8145   effect(KILL cr);
8146   format %{ "XADDQ  [$mem],$newval" %}
8147   ins_encode %{
8148     __ lock();
8149     __ xaddq($mem$$Address, $newval$$Register);
8150   %}
8151   ins_pipe( pipe_cmpxchg );
8152 %}
8153 
8154 instruct xchgB( memory mem, rRegI newval) %{
8155   match(Set newval (GetAndSetB mem newval));
8156   format %{ "XCHGB  $newval,[$mem]" %}
8157   ins_encode %{
8158     __ xchgb($newval$$Register, $mem$$Address);
8159   %}
8160   ins_pipe( pipe_cmpxchg );
8161 %}
8162 
8163 instruct xchgS( memory mem, rRegI newval) %{
8164   match(Set newval (GetAndSetS mem newval));
8165   format %{ "XCHGW  $newval,[$mem]" %}
8166   ins_encode %{
8167     __ xchgw($newval$$Register, $mem$$Address);
8168   %}
8169   ins_pipe( pipe_cmpxchg );
8170 %}
8171 
8172 instruct xchgI( memory mem, rRegI newval) %{
8173   match(Set newval (GetAndSetI mem newval));
8174   format %{ "XCHGL  $newval,[$mem]" %}
8175   ins_encode %{
8176     __ xchgl($newval$$Register, $mem$$Address);
8177   %}
8178   ins_pipe( pipe_cmpxchg );
8179 %}
8180 
8181 instruct xchgL( memory mem, rRegL newval) %{
8182   match(Set newval (GetAndSetL mem newval));
8183   format %{ "XCHGL  $newval,[$mem]" %}
8184   ins_encode %{
8185     __ xchgq($newval$$Register, $mem$$Address);
8186   %}
8187   ins_pipe( pipe_cmpxchg );
8188 %}
8189 
8190 instruct xchgP( memory mem, rRegP newval) %{
8191   match(Set newval (GetAndSetP mem newval));
8192   format %{ "XCHGQ  $newval,[$mem]" %}
8193   ins_encode %{
8194     __ xchgq($newval$$Register, $mem$$Address);
8195   %}
8196   ins_pipe( pipe_cmpxchg );
8197 %}
8198 
8199 instruct xchgN( memory mem, rRegN newval) %{
8200   match(Set newval (GetAndSetN mem newval));
8201   format %{ "XCHGL  $newval,$mem]" %}
8202   ins_encode %{
8203     __ xchgl($newval$$Register, $mem$$Address);
8204   %}
8205   ins_pipe( pipe_cmpxchg );
8206 %}
8207 
8208 //----------Subtraction Instructions-------------------------------------------
8209 
8210 // Integer Subtraction Instructions
8211 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8212 %{
8213   match(Set dst (SubI dst src));
8214   effect(KILL cr);
8215 
8216   format %{ "subl    $dst, $src\t# int" %}
8217   opcode(0x2B);
8218   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8219   ins_pipe(ialu_reg_reg);
8220 %}
8221 
8222 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8223 %{
8224   match(Set dst (SubI dst src));
8225   effect(KILL cr);
8226 
8227   format %{ "subl    $dst, $src\t# int" %}
8228   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8229   ins_encode(OpcSErm(dst, src), Con8or32(src));
8230   ins_pipe(ialu_reg);
8231 %}
8232 
8233 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8234 %{
8235   match(Set dst (SubI dst (LoadI src)));
8236   effect(KILL cr);
8237 
8238   ins_cost(125);
8239   format %{ "subl    $dst, $src\t# int" %}
8240   opcode(0x2B);
8241   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8242   ins_pipe(ialu_reg_mem);
8243 %}
8244 
8245 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8246 %{
8247   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8248   effect(KILL cr);
8249 
8250   ins_cost(150);
8251   format %{ "subl    $dst, $src\t# int" %}
8252   opcode(0x29); /* Opcode 29 /r */
8253   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8254   ins_pipe(ialu_mem_reg);
8255 %}
8256 
8257 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8258 %{
8259   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8260   effect(KILL cr);
8261 
8262   ins_cost(125); // XXX
8263   format %{ "subl    $dst, $src\t# int" %}
8264   opcode(0x81); /* Opcode 81 /5 id */
8265   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8266   ins_pipe(ialu_mem_imm);
8267 %}
8268 
8269 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8270 %{
8271   match(Set dst (SubL dst src));
8272   effect(KILL cr);
8273 
8274   format %{ "subq    $dst, $src\t# long" %}
8275   opcode(0x2B);
8276   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8277   ins_pipe(ialu_reg_reg);
8278 %}
8279 
8280 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8281 %{
8282   match(Set dst (SubL dst src));
8283   effect(KILL cr);
8284 
8285   format %{ "subq    $dst, $src\t# long" %}
8286   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8287   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8288   ins_pipe(ialu_reg);
8289 %}
8290 
8291 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8292 %{
8293   match(Set dst (SubL dst (LoadL src)));
8294   effect(KILL cr);
8295 
8296   ins_cost(125);
8297   format %{ "subq    $dst, $src\t# long" %}
8298   opcode(0x2B);
8299   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8300   ins_pipe(ialu_reg_mem);
8301 %}
8302 
8303 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8304 %{
8305   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8306   effect(KILL cr);
8307 
8308   ins_cost(150);
8309   format %{ "subq    $dst, $src\t# long" %}
8310   opcode(0x29); /* Opcode 29 /r */
8311   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8312   ins_pipe(ialu_mem_reg);
8313 %}
8314 
8315 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8316 %{
8317   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8318   effect(KILL cr);
8319 
8320   ins_cost(125); // XXX
8321   format %{ "subq    $dst, $src\t# long" %}
8322   opcode(0x81); /* Opcode 81 /5 id */
8323   ins_encode(REX_mem_wide(dst),
8324              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8325   ins_pipe(ialu_mem_imm);
8326 %}
8327 
8328 // Subtract from a pointer
8329 // XXX hmpf???
8330 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8331 %{
8332   match(Set dst (AddP dst (SubI zero src)));
8333   effect(KILL cr);
8334 
8335   format %{ "subq    $dst, $src\t# ptr - int" %}
8336   opcode(0x2B);
8337   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8338   ins_pipe(ialu_reg_reg);
8339 %}
8340 
8341 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8342 %{
8343   match(Set dst (SubI zero dst));
8344   effect(KILL cr);
8345 
8346   format %{ "negl    $dst\t# int" %}
8347   opcode(0xF7, 0x03);  // Opcode F7 /3
8348   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8349   ins_pipe(ialu_reg);
8350 %}
8351 
8352 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8353 %{
8354   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8355   effect(KILL cr);
8356 
8357   format %{ "negl    $dst\t# int" %}
8358   opcode(0xF7, 0x03);  // Opcode F7 /3
8359   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8360   ins_pipe(ialu_reg);
8361 %}
8362 
8363 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8364 %{
8365   match(Set dst (SubL zero dst));
8366   effect(KILL cr);
8367 
8368   format %{ "negq    $dst\t# long" %}
8369   opcode(0xF7, 0x03);  // Opcode F7 /3
8370   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8371   ins_pipe(ialu_reg);
8372 %}
8373 
8374 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8375 %{
8376   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8377   effect(KILL cr);
8378 
8379   format %{ "negq    $dst\t# long" %}
8380   opcode(0xF7, 0x03);  // Opcode F7 /3
8381   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8382   ins_pipe(ialu_reg);
8383 %}
8384 
8385 //----------Multiplication/Division Instructions-------------------------------
8386 // Integer Multiplication Instructions
8387 // Multiply Register
8388 
8389 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8390 %{
8391   match(Set dst (MulI dst src));
8392   effect(KILL cr);
8393 
8394   ins_cost(300);
8395   format %{ "imull   $dst, $src\t# int" %}
8396   opcode(0x0F, 0xAF);
8397   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8398   ins_pipe(ialu_reg_reg_alu0);
8399 %}
8400 
8401 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8402 %{
8403   match(Set dst (MulI src imm));
8404   effect(KILL cr);
8405 
8406   ins_cost(300);
8407   format %{ "imull   $dst, $src, $imm\t# int" %}
8408   opcode(0x69); /* 69 /r id */
8409   ins_encode(REX_reg_reg(dst, src),
8410              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8411   ins_pipe(ialu_reg_reg_alu0);
8412 %}
8413 
8414 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8415 %{
8416   match(Set dst (MulI dst (LoadI src)));
8417   effect(KILL cr);
8418 
8419   ins_cost(350);
8420   format %{ "imull   $dst, $src\t# int" %}
8421   opcode(0x0F, 0xAF);
8422   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8423   ins_pipe(ialu_reg_mem_alu0);
8424 %}
8425 
8426 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8427 %{
8428   match(Set dst (MulI (LoadI src) imm));
8429   effect(KILL cr);
8430 
8431   ins_cost(300);
8432   format %{ "imull   $dst, $src, $imm\t# int" %}
8433   opcode(0x69); /* 69 /r id */
8434   ins_encode(REX_reg_mem(dst, src),
8435              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8436   ins_pipe(ialu_reg_mem_alu0);
8437 %}
8438 
8439 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8440 %{
8441   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8442   effect(KILL cr, KILL src2);
8443 
8444   expand %{ mulI_rReg(dst, src1, cr);
8445            mulI_rReg(src2, src3, cr);
8446            addI_rReg(dst, src2, cr); %}
8447 %}
8448 
8449 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8450 %{
8451   match(Set dst (MulL dst src));
8452   effect(KILL cr);
8453 
8454   ins_cost(300);
8455   format %{ "imulq   $dst, $src\t# long" %}
8456   opcode(0x0F, 0xAF);
8457   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8458   ins_pipe(ialu_reg_reg_alu0);
8459 %}
8460 
8461 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8462 %{
8463   match(Set dst (MulL src imm));
8464   effect(KILL cr);
8465 
8466   ins_cost(300);
8467   format %{ "imulq   $dst, $src, $imm\t# long" %}
8468   opcode(0x69); /* 69 /r id */
8469   ins_encode(REX_reg_reg_wide(dst, src),
8470              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8471   ins_pipe(ialu_reg_reg_alu0);
8472 %}
8473 
8474 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8475 %{
8476   match(Set dst (MulL dst (LoadL src)));
8477   effect(KILL cr);
8478 
8479   ins_cost(350);
8480   format %{ "imulq   $dst, $src\t# long" %}
8481   opcode(0x0F, 0xAF);
8482   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8483   ins_pipe(ialu_reg_mem_alu0);
8484 %}
8485 
8486 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8487 %{
8488   match(Set dst (MulL (LoadL src) imm));
8489   effect(KILL cr);
8490 
8491   ins_cost(300);
8492   format %{ "imulq   $dst, $src, $imm\t# long" %}
8493   opcode(0x69); /* 69 /r id */
8494   ins_encode(REX_reg_mem_wide(dst, src),
8495              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8496   ins_pipe(ialu_reg_mem_alu0);
8497 %}
8498 
8499 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8500 %{
8501   match(Set dst (MulHiL src rax));
8502   effect(USE_KILL rax, KILL cr);
8503 
8504   ins_cost(300);
8505   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8506   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8507   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8508   ins_pipe(ialu_reg_reg_alu0);
8509 %}
8510 
8511 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8512                    rFlagsReg cr)
8513 %{
8514   match(Set rax (DivI rax div));
8515   effect(KILL rdx, KILL cr);
8516 
8517   ins_cost(30*100+10*100); // XXX
8518   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8519             "jne,s   normal\n\t"
8520             "xorl    rdx, rdx\n\t"
8521             "cmpl    $div, -1\n\t"
8522             "je,s    done\n"
8523     "normal: cdql\n\t"
8524             "idivl   $div\n"
8525     "done:"        %}
8526   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8527   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8528   ins_pipe(ialu_reg_reg_alu0);
8529 %}
8530 
8531 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8532                    rFlagsReg cr)
8533 %{
8534   match(Set rax (DivL rax div));
8535   effect(KILL rdx, KILL cr);
8536 
8537   ins_cost(30*100+10*100); // XXX
8538   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8539             "cmpq    rax, rdx\n\t"
8540             "jne,s   normal\n\t"
8541             "xorl    rdx, rdx\n\t"
8542             "cmpq    $div, -1\n\t"
8543             "je,s    done\n"
8544     "normal: cdqq\n\t"
8545             "idivq   $div\n"
8546     "done:"        %}
8547   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8548   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8549   ins_pipe(ialu_reg_reg_alu0);
8550 %}
8551 
8552 // Integer DIVMOD with Register, both quotient and mod results
8553 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8554                              rFlagsReg cr)
8555 %{
8556   match(DivModI rax div);
8557   effect(KILL cr);
8558 
8559   ins_cost(30*100+10*100); // XXX
8560   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8561             "jne,s   normal\n\t"
8562             "xorl    rdx, rdx\n\t"
8563             "cmpl    $div, -1\n\t"
8564             "je,s    done\n"
8565     "normal: cdql\n\t"
8566             "idivl   $div\n"
8567     "done:"        %}
8568   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8569   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8570   ins_pipe(pipe_slow);
8571 %}
8572 
8573 // Long DIVMOD with Register, both quotient and mod results
8574 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8575                              rFlagsReg cr)
8576 %{
8577   match(DivModL rax div);
8578   effect(KILL cr);
8579 
8580   ins_cost(30*100+10*100); // XXX
8581   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8582             "cmpq    rax, rdx\n\t"
8583             "jne,s   normal\n\t"
8584             "xorl    rdx, rdx\n\t"
8585             "cmpq    $div, -1\n\t"
8586             "je,s    done\n"
8587     "normal: cdqq\n\t"
8588             "idivq   $div\n"
8589     "done:"        %}
8590   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8591   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8592   ins_pipe(pipe_slow);
8593 %}
8594 
8595 //----------- DivL-By-Constant-Expansions--------------------------------------
8596 // DivI cases are handled by the compiler
8597 
8598 // Magic constant, reciprocal of 10
8599 instruct loadConL_0x6666666666666667(rRegL dst)
8600 %{
8601   effect(DEF dst);
8602 
8603   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8604   ins_encode(load_immL(dst, 0x6666666666666667));
8605   ins_pipe(ialu_reg);
8606 %}
8607 
8608 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8609 %{
8610   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8611 
8612   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8613   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8614   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8615   ins_pipe(ialu_reg_reg_alu0);
8616 %}
8617 
8618 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8619 %{
8620   effect(USE_DEF dst, KILL cr);
8621 
8622   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8623   opcode(0xC1, 0x7); /* C1 /7 ib */
8624   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8625   ins_pipe(ialu_reg);
8626 %}
8627 
8628 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8629 %{
8630   effect(USE_DEF dst, KILL cr);
8631 
8632   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8633   opcode(0xC1, 0x7); /* C1 /7 ib */
8634   ins_encode(reg_opc_imm_wide(dst, 0x2));
8635   ins_pipe(ialu_reg);
8636 %}
8637 
8638 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8639 %{
8640   match(Set dst (DivL src div));
8641 
8642   ins_cost((5+8)*100);
8643   expand %{
8644     rax_RegL rax;                     // Killed temp
8645     rFlagsReg cr;                     // Killed
8646     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8647     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8648     sarL_rReg_63(src, cr);            // sarq  src, 63
8649     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8650     subL_rReg(dst, src, cr);          // subl  rdx, src
8651   %}
8652 %}
8653 
8654 //-----------------------------------------------------------------------------
8655 
8656 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8657                    rFlagsReg cr)
8658 %{
8659   match(Set rdx (ModI rax div));
8660   effect(KILL rax, KILL cr);
8661 
8662   ins_cost(300); // XXX
8663   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8664             "jne,s   normal\n\t"
8665             "xorl    rdx, rdx\n\t"
8666             "cmpl    $div, -1\n\t"
8667             "je,s    done\n"
8668     "normal: cdql\n\t"
8669             "idivl   $div\n"
8670     "done:"        %}
8671   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8672   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8673   ins_pipe(ialu_reg_reg_alu0);
8674 %}
8675 
8676 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8677                    rFlagsReg cr)
8678 %{
8679   match(Set rdx (ModL rax div));
8680   effect(KILL rax, KILL cr);
8681 
8682   ins_cost(300); // XXX
8683   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8684             "cmpq    rax, rdx\n\t"
8685             "jne,s   normal\n\t"
8686             "xorl    rdx, rdx\n\t"
8687             "cmpq    $div, -1\n\t"
8688             "je,s    done\n"
8689     "normal: cdqq\n\t"
8690             "idivq   $div\n"
8691     "done:"        %}
8692   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8693   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8694   ins_pipe(ialu_reg_reg_alu0);
8695 %}
8696 
8697 // Integer Shift Instructions
8698 // Shift Left by one
8699 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8700 %{
8701   match(Set dst (LShiftI dst shift));
8702   effect(KILL cr);
8703 
8704   format %{ "sall    $dst, $shift" %}
8705   opcode(0xD1, 0x4); /* D1 /4 */
8706   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8707   ins_pipe(ialu_reg);
8708 %}
8709 
8710 // Shift Left by one
8711 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8712 %{
8713   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8714   effect(KILL cr);
8715 
8716   format %{ "sall    $dst, $shift\t" %}
8717   opcode(0xD1, 0x4); /* D1 /4 */
8718   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8719   ins_pipe(ialu_mem_imm);
8720 %}
8721 
8722 // Shift Left by 8-bit immediate
8723 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8724 %{
8725   match(Set dst (LShiftI dst shift));
8726   effect(KILL cr);
8727 
8728   format %{ "sall    $dst, $shift" %}
8729   opcode(0xC1, 0x4); /* C1 /4 ib */
8730   ins_encode(reg_opc_imm(dst, shift));
8731   ins_pipe(ialu_reg);
8732 %}
8733 
8734 // Shift Left by 8-bit immediate
8735 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8736 %{
8737   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8738   effect(KILL cr);
8739 
8740   format %{ "sall    $dst, $shift" %}
8741   opcode(0xC1, 0x4); /* C1 /4 ib */
8742   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8743   ins_pipe(ialu_mem_imm);
8744 %}
8745 
8746 // Shift Left by variable
8747 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8748 %{
8749   match(Set dst (LShiftI dst shift));
8750   effect(KILL cr);
8751 
8752   format %{ "sall    $dst, $shift" %}
8753   opcode(0xD3, 0x4); /* D3 /4 */
8754   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8755   ins_pipe(ialu_reg_reg);
8756 %}
8757 
8758 // Shift Left by variable
8759 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8760 %{
8761   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8762   effect(KILL cr);
8763 
8764   format %{ "sall    $dst, $shift" %}
8765   opcode(0xD3, 0x4); /* D3 /4 */
8766   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8767   ins_pipe(ialu_mem_reg);
8768 %}
8769 
8770 // Arithmetic shift right by one
8771 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8772 %{
8773   match(Set dst (RShiftI dst shift));
8774   effect(KILL cr);
8775 
8776   format %{ "sarl    $dst, $shift" %}
8777   opcode(0xD1, 0x7); /* D1 /7 */
8778   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8779   ins_pipe(ialu_reg);
8780 %}
8781 
8782 // Arithmetic shift right by one
8783 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8784 %{
8785   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8786   effect(KILL cr);
8787 
8788   format %{ "sarl    $dst, $shift" %}
8789   opcode(0xD1, 0x7); /* D1 /7 */
8790   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8791   ins_pipe(ialu_mem_imm);
8792 %}
8793 
8794 // Arithmetic Shift Right by 8-bit immediate
8795 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8796 %{
8797   match(Set dst (RShiftI dst shift));
8798   effect(KILL cr);
8799 
8800   format %{ "sarl    $dst, $shift" %}
8801   opcode(0xC1, 0x7); /* C1 /7 ib */
8802   ins_encode(reg_opc_imm(dst, shift));
8803   ins_pipe(ialu_mem_imm);
8804 %}
8805 
8806 // Arithmetic Shift Right by 8-bit immediate
8807 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8808 %{
8809   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8810   effect(KILL cr);
8811 
8812   format %{ "sarl    $dst, $shift" %}
8813   opcode(0xC1, 0x7); /* C1 /7 ib */
8814   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8815   ins_pipe(ialu_mem_imm);
8816 %}
8817 
8818 // Arithmetic Shift Right by variable
8819 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8820 %{
8821   match(Set dst (RShiftI dst shift));
8822   effect(KILL cr);
8823 
8824   format %{ "sarl    $dst, $shift" %}
8825   opcode(0xD3, 0x7); /* D3 /7 */
8826   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8827   ins_pipe(ialu_reg_reg);
8828 %}
8829 
8830 // Arithmetic Shift Right by variable
8831 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8832 %{
8833   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8834   effect(KILL cr);
8835 
8836   format %{ "sarl    $dst, $shift" %}
8837   opcode(0xD3, 0x7); /* D3 /7 */
8838   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8839   ins_pipe(ialu_mem_reg);
8840 %}
8841 
8842 // Logical shift right by one
8843 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8844 %{
8845   match(Set dst (URShiftI dst shift));
8846   effect(KILL cr);
8847 
8848   format %{ "shrl    $dst, $shift" %}
8849   opcode(0xD1, 0x5); /* D1 /5 */
8850   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8851   ins_pipe(ialu_reg);
8852 %}
8853 
8854 // Logical shift right by one
8855 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8856 %{
8857   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8858   effect(KILL cr);
8859 
8860   format %{ "shrl    $dst, $shift" %}
8861   opcode(0xD1, 0x5); /* D1 /5 */
8862   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8863   ins_pipe(ialu_mem_imm);
8864 %}
8865 
8866 // Logical Shift Right by 8-bit immediate
8867 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8868 %{
8869   match(Set dst (URShiftI dst shift));
8870   effect(KILL cr);
8871 
8872   format %{ "shrl    $dst, $shift" %}
8873   opcode(0xC1, 0x5); /* C1 /5 ib */
8874   ins_encode(reg_opc_imm(dst, shift));
8875   ins_pipe(ialu_reg);
8876 %}
8877 
8878 // Logical Shift Right by 8-bit immediate
8879 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8880 %{
8881   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8882   effect(KILL cr);
8883 
8884   format %{ "shrl    $dst, $shift" %}
8885   opcode(0xC1, 0x5); /* C1 /5 ib */
8886   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8887   ins_pipe(ialu_mem_imm);
8888 %}
8889 
8890 // Logical Shift Right by variable
8891 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8892 %{
8893   match(Set dst (URShiftI dst shift));
8894   effect(KILL cr);
8895 
8896   format %{ "shrl    $dst, $shift" %}
8897   opcode(0xD3, 0x5); /* D3 /5 */
8898   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8899   ins_pipe(ialu_reg_reg);
8900 %}
8901 
8902 // Logical Shift Right by variable
8903 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8904 %{
8905   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8906   effect(KILL cr);
8907 
8908   format %{ "shrl    $dst, $shift" %}
8909   opcode(0xD3, 0x5); /* D3 /5 */
8910   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8911   ins_pipe(ialu_mem_reg);
8912 %}
8913 
8914 // Long Shift Instructions
8915 // Shift Left by one
8916 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8917 %{
8918   match(Set dst (LShiftL dst shift));
8919   effect(KILL cr);
8920 
8921   format %{ "salq    $dst, $shift" %}
8922   opcode(0xD1, 0x4); /* D1 /4 */
8923   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8924   ins_pipe(ialu_reg);
8925 %}
8926 
8927 // Shift Left by one
8928 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8929 %{
8930   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8931   effect(KILL cr);
8932 
8933   format %{ "salq    $dst, $shift" %}
8934   opcode(0xD1, 0x4); /* D1 /4 */
8935   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8936   ins_pipe(ialu_mem_imm);
8937 %}
8938 
8939 // Shift Left by 8-bit immediate
8940 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8941 %{
8942   match(Set dst (LShiftL dst shift));
8943   effect(KILL cr);
8944 
8945   format %{ "salq    $dst, $shift" %}
8946   opcode(0xC1, 0x4); /* C1 /4 ib */
8947   ins_encode(reg_opc_imm_wide(dst, shift));
8948   ins_pipe(ialu_reg);
8949 %}
8950 
8951 // Shift Left by 8-bit immediate
8952 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8953 %{
8954   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8955   effect(KILL cr);
8956 
8957   format %{ "salq    $dst, $shift" %}
8958   opcode(0xC1, 0x4); /* C1 /4 ib */
8959   ins_encode(REX_mem_wide(dst), OpcP,
8960              RM_opc_mem(secondary, dst), Con8or32(shift));
8961   ins_pipe(ialu_mem_imm);
8962 %}
8963 
8964 // Shift Left by variable
8965 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8966 %{
8967   match(Set dst (LShiftL dst shift));
8968   effect(KILL cr);
8969 
8970   format %{ "salq    $dst, $shift" %}
8971   opcode(0xD3, 0x4); /* D3 /4 */
8972   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8973   ins_pipe(ialu_reg_reg);
8974 %}
8975 
8976 // Shift Left by variable
8977 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8978 %{
8979   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8980   effect(KILL cr);
8981 
8982   format %{ "salq    $dst, $shift" %}
8983   opcode(0xD3, 0x4); /* D3 /4 */
8984   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8985   ins_pipe(ialu_mem_reg);
8986 %}
8987 
8988 // Arithmetic shift right by one
8989 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8990 %{
8991   match(Set dst (RShiftL dst shift));
8992   effect(KILL cr);
8993 
8994   format %{ "sarq    $dst, $shift" %}
8995   opcode(0xD1, 0x7); /* D1 /7 */
8996   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8997   ins_pipe(ialu_reg);
8998 %}
8999 
9000 // Arithmetic shift right by one
9001 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9002 %{
9003   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9004   effect(KILL cr);
9005 
9006   format %{ "sarq    $dst, $shift" %}
9007   opcode(0xD1, 0x7); /* D1 /7 */
9008   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9009   ins_pipe(ialu_mem_imm);
9010 %}
9011 
9012 // Arithmetic Shift Right by 8-bit immediate
9013 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9014 %{
9015   match(Set dst (RShiftL dst shift));
9016   effect(KILL cr);
9017 
9018   format %{ "sarq    $dst, $shift" %}
9019   opcode(0xC1, 0x7); /* C1 /7 ib */
9020   ins_encode(reg_opc_imm_wide(dst, shift));
9021   ins_pipe(ialu_mem_imm);
9022 %}
9023 
9024 // Arithmetic Shift Right by 8-bit immediate
9025 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9026 %{
9027   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9028   effect(KILL cr);
9029 
9030   format %{ "sarq    $dst, $shift" %}
9031   opcode(0xC1, 0x7); /* C1 /7 ib */
9032   ins_encode(REX_mem_wide(dst), OpcP,
9033              RM_opc_mem(secondary, dst), Con8or32(shift));
9034   ins_pipe(ialu_mem_imm);
9035 %}
9036 
9037 // Arithmetic Shift Right by variable
9038 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9039 %{
9040   match(Set dst (RShiftL dst shift));
9041   effect(KILL cr);
9042 
9043   format %{ "sarq    $dst, $shift" %}
9044   opcode(0xD3, 0x7); /* D3 /7 */
9045   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9046   ins_pipe(ialu_reg_reg);
9047 %}
9048 
9049 // Arithmetic Shift Right by variable
9050 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9051 %{
9052   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9053   effect(KILL cr);
9054 
9055   format %{ "sarq    $dst, $shift" %}
9056   opcode(0xD3, 0x7); /* D3 /7 */
9057   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9058   ins_pipe(ialu_mem_reg);
9059 %}
9060 
9061 // Logical shift right by one
9062 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9063 %{
9064   match(Set dst (URShiftL dst shift));
9065   effect(KILL cr);
9066 
9067   format %{ "shrq    $dst, $shift" %}
9068   opcode(0xD1, 0x5); /* D1 /5 */
9069   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9070   ins_pipe(ialu_reg);
9071 %}
9072 
9073 // Logical shift right by one
9074 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9075 %{
9076   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9077   effect(KILL cr);
9078 
9079   format %{ "shrq    $dst, $shift" %}
9080   opcode(0xD1, 0x5); /* D1 /5 */
9081   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9082   ins_pipe(ialu_mem_imm);
9083 %}
9084 
9085 // Logical Shift Right by 8-bit immediate
9086 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9087 %{
9088   match(Set dst (URShiftL dst shift));
9089   effect(KILL cr);
9090 
9091   format %{ "shrq    $dst, $shift" %}
9092   opcode(0xC1, 0x5); /* C1 /5 ib */
9093   ins_encode(reg_opc_imm_wide(dst, shift));
9094   ins_pipe(ialu_reg);
9095 %}
9096 
9097 
9098 // Logical Shift Right by 8-bit immediate
9099 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9100 %{
9101   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9102   effect(KILL cr);
9103 
9104   format %{ "shrq    $dst, $shift" %}
9105   opcode(0xC1, 0x5); /* C1 /5 ib */
9106   ins_encode(REX_mem_wide(dst), OpcP,
9107              RM_opc_mem(secondary, dst), Con8or32(shift));
9108   ins_pipe(ialu_mem_imm);
9109 %}
9110 
9111 // Logical Shift Right by variable
9112 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9113 %{
9114   match(Set dst (URShiftL dst shift));
9115   effect(KILL cr);
9116 
9117   format %{ "shrq    $dst, $shift" %}
9118   opcode(0xD3, 0x5); /* D3 /5 */
9119   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9120   ins_pipe(ialu_reg_reg);
9121 %}
9122 
9123 // Logical Shift Right by variable
9124 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9125 %{
9126   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9127   effect(KILL cr);
9128 
9129   format %{ "shrq    $dst, $shift" %}
9130   opcode(0xD3, 0x5); /* D3 /5 */
9131   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9132   ins_pipe(ialu_mem_reg);
9133 %}
9134 
9135 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9136 // This idiom is used by the compiler for the i2b bytecode.
9137 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9138 %{
9139   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9140 
9141   format %{ "movsbl  $dst, $src\t# i2b" %}
9142   opcode(0x0F, 0xBE);
9143   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9144   ins_pipe(ialu_reg_reg);
9145 %}
9146 
9147 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9148 // This idiom is used by the compiler the i2s bytecode.
9149 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9150 %{
9151   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9152 
9153   format %{ "movswl  $dst, $src\t# i2s" %}
9154   opcode(0x0F, 0xBF);
9155   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9156   ins_pipe(ialu_reg_reg);
9157 %}
9158 
9159 // ROL/ROR instructions
9160 
9161 // ROL expand
9162 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9163   effect(KILL cr, USE_DEF dst);
9164 
9165   format %{ "roll    $dst" %}
9166   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9167   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9168   ins_pipe(ialu_reg);
9169 %}
9170 
9171 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9172   effect(USE_DEF dst, USE shift, KILL cr);
9173 
9174   format %{ "roll    $dst, $shift" %}
9175   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9176   ins_encode( reg_opc_imm(dst, shift) );
9177   ins_pipe(ialu_reg);
9178 %}
9179 
9180 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9181 %{
9182   effect(USE_DEF dst, USE shift, KILL cr);
9183 
9184   format %{ "roll    $dst, $shift" %}
9185   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9186   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9187   ins_pipe(ialu_reg_reg);
9188 %}
9189 // end of ROL expand
9190 
9191 // Rotate Left by one
9192 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9193 %{
9194   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9195 
9196   expand %{
9197     rolI_rReg_imm1(dst, cr);
9198   %}
9199 %}
9200 
9201 // Rotate Left by 8-bit immediate
9202 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9203 %{
9204   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9205   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9206 
9207   expand %{
9208     rolI_rReg_imm8(dst, lshift, cr);
9209   %}
9210 %}
9211 
9212 // Rotate Left by variable
9213 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9214 %{
9215   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9216 
9217   expand %{
9218     rolI_rReg_CL(dst, shift, cr);
9219   %}
9220 %}
9221 
9222 // Rotate Left by variable
9223 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9224 %{
9225   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9226 
9227   expand %{
9228     rolI_rReg_CL(dst, shift, cr);
9229   %}
9230 %}
9231 
9232 // ROR expand
9233 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9234 %{
9235   effect(USE_DEF dst, KILL cr);
9236 
9237   format %{ "rorl    $dst" %}
9238   opcode(0xD1, 0x1); /* D1 /1 */
9239   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9240   ins_pipe(ialu_reg);
9241 %}
9242 
9243 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9244 %{
9245   effect(USE_DEF dst, USE shift, KILL cr);
9246 
9247   format %{ "rorl    $dst, $shift" %}
9248   opcode(0xC1, 0x1); /* C1 /1 ib */
9249   ins_encode(reg_opc_imm(dst, shift));
9250   ins_pipe(ialu_reg);
9251 %}
9252 
9253 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9254 %{
9255   effect(USE_DEF dst, USE shift, KILL cr);
9256 
9257   format %{ "rorl    $dst, $shift" %}
9258   opcode(0xD3, 0x1); /* D3 /1 */
9259   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9260   ins_pipe(ialu_reg_reg);
9261 %}
9262 // end of ROR expand
9263 
9264 // Rotate Right by one
9265 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9266 %{
9267   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9268 
9269   expand %{
9270     rorI_rReg_imm1(dst, cr);
9271   %}
9272 %}
9273 
9274 // Rotate Right by 8-bit immediate
9275 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9276 %{
9277   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9278   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9279 
9280   expand %{
9281     rorI_rReg_imm8(dst, rshift, cr);
9282   %}
9283 %}
9284 
9285 // Rotate Right by variable
9286 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9287 %{
9288   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9289 
9290   expand %{
9291     rorI_rReg_CL(dst, shift, cr);
9292   %}
9293 %}
9294 
9295 // Rotate Right by variable
9296 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9297 %{
9298   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9299 
9300   expand %{
9301     rorI_rReg_CL(dst, shift, cr);
9302   %}
9303 %}
9304 
9305 // for long rotate
9306 // ROL expand
9307 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9308   effect(USE_DEF dst, KILL cr);
9309 
9310   format %{ "rolq    $dst" %}
9311   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9312   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9313   ins_pipe(ialu_reg);
9314 %}
9315 
9316 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9317   effect(USE_DEF dst, USE shift, KILL cr);
9318 
9319   format %{ "rolq    $dst, $shift" %}
9320   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9321   ins_encode( reg_opc_imm_wide(dst, shift) );
9322   ins_pipe(ialu_reg);
9323 %}
9324 
9325 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9326 %{
9327   effect(USE_DEF dst, USE shift, KILL cr);
9328 
9329   format %{ "rolq    $dst, $shift" %}
9330   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9331   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9332   ins_pipe(ialu_reg_reg);
9333 %}
9334 // end of ROL expand
9335 
9336 // Rotate Left by one
9337 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9338 %{
9339   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9340 
9341   expand %{
9342     rolL_rReg_imm1(dst, cr);
9343   %}
9344 %}
9345 
9346 // Rotate Left by 8-bit immediate
9347 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9348 %{
9349   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9350   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9351 
9352   expand %{
9353     rolL_rReg_imm8(dst, lshift, cr);
9354   %}
9355 %}
9356 
9357 // Rotate Left by variable
9358 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9359 %{
9360   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9361 
9362   expand %{
9363     rolL_rReg_CL(dst, shift, cr);
9364   %}
9365 %}
9366 
9367 // Rotate Left by variable
9368 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9369 %{
9370   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9371 
9372   expand %{
9373     rolL_rReg_CL(dst, shift, cr);
9374   %}
9375 %}
9376 
9377 // ROR expand
9378 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9379 %{
9380   effect(USE_DEF dst, KILL cr);
9381 
9382   format %{ "rorq    $dst" %}
9383   opcode(0xD1, 0x1); /* D1 /1 */
9384   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9385   ins_pipe(ialu_reg);
9386 %}
9387 
9388 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9389 %{
9390   effect(USE_DEF dst, USE shift, KILL cr);
9391 
9392   format %{ "rorq    $dst, $shift" %}
9393   opcode(0xC1, 0x1); /* C1 /1 ib */
9394   ins_encode(reg_opc_imm_wide(dst, shift));
9395   ins_pipe(ialu_reg);
9396 %}
9397 
9398 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9399 %{
9400   effect(USE_DEF dst, USE shift, KILL cr);
9401 
9402   format %{ "rorq    $dst, $shift" %}
9403   opcode(0xD3, 0x1); /* D3 /1 */
9404   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9405   ins_pipe(ialu_reg_reg);
9406 %}
9407 // end of ROR expand
9408 
9409 // Rotate Right by one
9410 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9411 %{
9412   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9413 
9414   expand %{
9415     rorL_rReg_imm1(dst, cr);
9416   %}
9417 %}
9418 
9419 // Rotate Right by 8-bit immediate
9420 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9421 %{
9422   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9423   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9424 
9425   expand %{
9426     rorL_rReg_imm8(dst, rshift, cr);
9427   %}
9428 %}
9429 
9430 // Rotate Right by variable
9431 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9432 %{
9433   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9434 
9435   expand %{
9436     rorL_rReg_CL(dst, shift, cr);
9437   %}
9438 %}
9439 
9440 // Rotate Right by variable
9441 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9442 %{
9443   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9444 
9445   expand %{
9446     rorL_rReg_CL(dst, shift, cr);
9447   %}
9448 %}
9449 
9450 // Logical Instructions
9451 
9452 // Integer Logical Instructions
9453 
9454 // And Instructions
9455 // And Register with Register
9456 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9457 %{
9458   match(Set dst (AndI dst src));
9459   effect(KILL cr);
9460 
9461   format %{ "andl    $dst, $src\t# int" %}
9462   opcode(0x23);
9463   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9464   ins_pipe(ialu_reg_reg);
9465 %}
9466 
9467 // And Register with Immediate 255
9468 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9469 %{
9470   match(Set dst (AndI dst src));
9471 
9472   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9473   opcode(0x0F, 0xB6);
9474   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9475   ins_pipe(ialu_reg);
9476 %}
9477 
9478 // And Register with Immediate 255 and promote to long
9479 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9480 %{
9481   match(Set dst (ConvI2L (AndI src mask)));
9482 
9483   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9484   opcode(0x0F, 0xB6);
9485   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9486   ins_pipe(ialu_reg);
9487 %}
9488 
9489 // And Register with Immediate 65535
9490 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9491 %{
9492   match(Set dst (AndI dst src));
9493 
9494   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9495   opcode(0x0F, 0xB7);
9496   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9497   ins_pipe(ialu_reg);
9498 %}
9499 
9500 // And Register with Immediate 65535 and promote to long
9501 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9502 %{
9503   match(Set dst (ConvI2L (AndI src mask)));
9504 
9505   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9506   opcode(0x0F, 0xB7);
9507   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9508   ins_pipe(ialu_reg);
9509 %}
9510 
9511 // And Register with Immediate
9512 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9513 %{
9514   match(Set dst (AndI dst src));
9515   effect(KILL cr);
9516 
9517   format %{ "andl    $dst, $src\t# int" %}
9518   opcode(0x81, 0x04); /* Opcode 81 /4 */
9519   ins_encode(OpcSErm(dst, src), Con8or32(src));
9520   ins_pipe(ialu_reg);
9521 %}
9522 
9523 // And Register with Memory
9524 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9525 %{
9526   match(Set dst (AndI dst (LoadI src)));
9527   effect(KILL cr);
9528 
9529   ins_cost(125);
9530   format %{ "andl    $dst, $src\t# int" %}
9531   opcode(0x23);
9532   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9533   ins_pipe(ialu_reg_mem);
9534 %}
9535 
9536 // And Memory with Register
9537 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9538 %{
9539   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9540   effect(KILL cr);
9541 
9542   ins_cost(150);
9543   format %{ "andb    $dst, $src\t# byte" %}
9544   opcode(0x20);
9545   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9546   ins_pipe(ialu_mem_reg);
9547 %}
9548 
9549 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9550 %{
9551   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9552   effect(KILL cr);
9553 
9554   ins_cost(150);
9555   format %{ "andl    $dst, $src\t# int" %}
9556   opcode(0x21); /* Opcode 21 /r */
9557   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9558   ins_pipe(ialu_mem_reg);
9559 %}
9560 
9561 // And Memory with Immediate
9562 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9563 %{
9564   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9565   effect(KILL cr);
9566 
9567   ins_cost(125);
9568   format %{ "andl    $dst, $src\t# int" %}
9569   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9570   ins_encode(REX_mem(dst), OpcSE(src),
9571              RM_opc_mem(secondary, dst), Con8or32(src));
9572   ins_pipe(ialu_mem_imm);
9573 %}
9574 
9575 // BMI1 instructions
9576 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9577   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9578   predicate(UseBMI1Instructions);
9579   effect(KILL cr);
9580 
9581   ins_cost(125);
9582   format %{ "andnl  $dst, $src1, $src2" %}
9583 
9584   ins_encode %{
9585     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9586   %}
9587   ins_pipe(ialu_reg_mem);
9588 %}
9589 
9590 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9591   match(Set dst (AndI (XorI src1 minus_1) src2));
9592   predicate(UseBMI1Instructions);
9593   effect(KILL cr);
9594 
9595   format %{ "andnl  $dst, $src1, $src2" %}
9596 
9597   ins_encode %{
9598     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9599   %}
9600   ins_pipe(ialu_reg);
9601 %}
9602 
9603 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9604   match(Set dst (AndI (SubI imm_zero src) src));
9605   predicate(UseBMI1Instructions);
9606   effect(KILL cr);
9607 
9608   format %{ "blsil  $dst, $src" %}
9609 
9610   ins_encode %{
9611     __ blsil($dst$$Register, $src$$Register);
9612   %}
9613   ins_pipe(ialu_reg);
9614 %}
9615 
9616 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9617   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9618   predicate(UseBMI1Instructions);
9619   effect(KILL cr);
9620 
9621   ins_cost(125);
9622   format %{ "blsil  $dst, $src" %}
9623 
9624   ins_encode %{
9625     __ blsil($dst$$Register, $src$$Address);
9626   %}
9627   ins_pipe(ialu_reg_mem);
9628 %}
9629 
9630 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9631 %{
9632   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9633   predicate(UseBMI1Instructions);
9634   effect(KILL cr);
9635 
9636   ins_cost(125);
9637   format %{ "blsmskl $dst, $src" %}
9638 
9639   ins_encode %{
9640     __ blsmskl($dst$$Register, $src$$Address);
9641   %}
9642   ins_pipe(ialu_reg_mem);
9643 %}
9644 
9645 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9646 %{
9647   match(Set dst (XorI (AddI src minus_1) src));
9648   predicate(UseBMI1Instructions);
9649   effect(KILL cr);
9650 
9651   format %{ "blsmskl $dst, $src" %}
9652 
9653   ins_encode %{
9654     __ blsmskl($dst$$Register, $src$$Register);
9655   %}
9656 
9657   ins_pipe(ialu_reg);
9658 %}
9659 
9660 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9661 %{
9662   match(Set dst (AndI (AddI src minus_1) src) );
9663   predicate(UseBMI1Instructions);
9664   effect(KILL cr);
9665 
9666   format %{ "blsrl  $dst, $src" %}
9667 
9668   ins_encode %{
9669     __ blsrl($dst$$Register, $src$$Register);
9670   %}
9671 
9672   ins_pipe(ialu_reg_mem);
9673 %}
9674 
9675 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9676 %{
9677   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9678   predicate(UseBMI1Instructions);
9679   effect(KILL cr);
9680 
9681   ins_cost(125);
9682   format %{ "blsrl  $dst, $src" %}
9683 
9684   ins_encode %{
9685     __ blsrl($dst$$Register, $src$$Address);
9686   %}
9687 
9688   ins_pipe(ialu_reg);
9689 %}
9690 
9691 // Or Instructions
9692 // Or Register with Register
9693 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9694 %{
9695   match(Set dst (OrI dst src));
9696   effect(KILL cr);
9697 
9698   format %{ "orl     $dst, $src\t# int" %}
9699   opcode(0x0B);
9700   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9701   ins_pipe(ialu_reg_reg);
9702 %}
9703 
9704 // Or Register with Immediate
9705 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9706 %{
9707   match(Set dst (OrI dst src));
9708   effect(KILL cr);
9709 
9710   format %{ "orl     $dst, $src\t# int" %}
9711   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9712   ins_encode(OpcSErm(dst, src), Con8or32(src));
9713   ins_pipe(ialu_reg);
9714 %}
9715 
9716 // Or Register with Memory
9717 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9718 %{
9719   match(Set dst (OrI dst (LoadI src)));
9720   effect(KILL cr);
9721 
9722   ins_cost(125);
9723   format %{ "orl     $dst, $src\t# int" %}
9724   opcode(0x0B);
9725   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9726   ins_pipe(ialu_reg_mem);
9727 %}
9728 
9729 // Or Memory with Register
9730 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9731 %{
9732   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9733   effect(KILL cr);
9734 
9735   ins_cost(150);
9736   format %{ "orb    $dst, $src\t# byte" %}
9737   opcode(0x08);
9738   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9739   ins_pipe(ialu_mem_reg);
9740 %}
9741 
9742 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9743 %{
9744   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9745   effect(KILL cr);
9746 
9747   ins_cost(150);
9748   format %{ "orl     $dst, $src\t# int" %}
9749   opcode(0x09); /* Opcode 09 /r */
9750   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9751   ins_pipe(ialu_mem_reg);
9752 %}
9753 
9754 // Or Memory with Immediate
9755 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9756 %{
9757   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9758   effect(KILL cr);
9759 
9760   ins_cost(125);
9761   format %{ "orl     $dst, $src\t# int" %}
9762   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9763   ins_encode(REX_mem(dst), OpcSE(src),
9764              RM_opc_mem(secondary, dst), Con8or32(src));
9765   ins_pipe(ialu_mem_imm);
9766 %}
9767 
9768 // Xor Instructions
9769 // Xor Register with Register
9770 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9771 %{
9772   match(Set dst (XorI dst src));
9773   effect(KILL cr);
9774 
9775   format %{ "xorl    $dst, $src\t# int" %}
9776   opcode(0x33);
9777   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9778   ins_pipe(ialu_reg_reg);
9779 %}
9780 
9781 // Xor Register with Immediate -1
9782 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9783   match(Set dst (XorI dst imm));
9784 
9785   format %{ "not    $dst" %}
9786   ins_encode %{
9787      __ notl($dst$$Register);
9788   %}
9789   ins_pipe(ialu_reg);
9790 %}
9791 
9792 // Xor Register with Immediate
9793 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9794 %{
9795   match(Set dst (XorI dst src));
9796   effect(KILL cr);
9797 
9798   format %{ "xorl    $dst, $src\t# int" %}
9799   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9800   ins_encode(OpcSErm(dst, src), Con8or32(src));
9801   ins_pipe(ialu_reg);
9802 %}
9803 
9804 // Xor Register with Memory
9805 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9806 %{
9807   match(Set dst (XorI dst (LoadI src)));
9808   effect(KILL cr);
9809 
9810   ins_cost(125);
9811   format %{ "xorl    $dst, $src\t# int" %}
9812   opcode(0x33);
9813   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9814   ins_pipe(ialu_reg_mem);
9815 %}
9816 
9817 // Xor Memory with Register
9818 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9819 %{
9820   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9821   effect(KILL cr);
9822 
9823   ins_cost(150);
9824   format %{ "xorb    $dst, $src\t# byte" %}
9825   opcode(0x30);
9826   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9827   ins_pipe(ialu_mem_reg);
9828 %}
9829 
9830 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9831 %{
9832   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9833   effect(KILL cr);
9834 
9835   ins_cost(150);
9836   format %{ "xorl    $dst, $src\t# int" %}
9837   opcode(0x31); /* Opcode 31 /r */
9838   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9839   ins_pipe(ialu_mem_reg);
9840 %}
9841 
9842 // Xor Memory with Immediate
9843 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9844 %{
9845   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9846   effect(KILL cr);
9847 
9848   ins_cost(125);
9849   format %{ "xorl    $dst, $src\t# int" %}
9850   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9851   ins_encode(REX_mem(dst), OpcSE(src),
9852              RM_opc_mem(secondary, dst), Con8or32(src));
9853   ins_pipe(ialu_mem_imm);
9854 %}
9855 
9856 
9857 // Long Logical Instructions
9858 
9859 // And Instructions
9860 // And Register with Register
9861 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9862 %{
9863   match(Set dst (AndL dst src));
9864   effect(KILL cr);
9865 
9866   format %{ "andq    $dst, $src\t# long" %}
9867   opcode(0x23);
9868   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9869   ins_pipe(ialu_reg_reg);
9870 %}
9871 
9872 // And Register with Immediate 255
9873 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9874 %{
9875   match(Set dst (AndL dst src));
9876 
9877   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9878   opcode(0x0F, 0xB6);
9879   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9880   ins_pipe(ialu_reg);
9881 %}
9882 
9883 // And Register with Immediate 65535
9884 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9885 %{
9886   match(Set dst (AndL dst src));
9887 
9888   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9889   opcode(0x0F, 0xB7);
9890   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9891   ins_pipe(ialu_reg);
9892 %}
9893 
9894 // And Register with Immediate
9895 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9896 %{
9897   match(Set dst (AndL dst src));
9898   effect(KILL cr);
9899 
9900   format %{ "andq    $dst, $src\t# long" %}
9901   opcode(0x81, 0x04); /* Opcode 81 /4 */
9902   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9903   ins_pipe(ialu_reg);
9904 %}
9905 
9906 // And Register with Memory
9907 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9908 %{
9909   match(Set dst (AndL dst (LoadL src)));
9910   effect(KILL cr);
9911 
9912   ins_cost(125);
9913   format %{ "andq    $dst, $src\t# long" %}
9914   opcode(0x23);
9915   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9916   ins_pipe(ialu_reg_mem);
9917 %}
9918 
9919 // And Memory with Register
9920 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9921 %{
9922   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9923   effect(KILL cr);
9924 
9925   ins_cost(150);
9926   format %{ "andq    $dst, $src\t# long" %}
9927   opcode(0x21); /* Opcode 21 /r */
9928   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9929   ins_pipe(ialu_mem_reg);
9930 %}
9931 
9932 // And Memory with Immediate
9933 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9934 %{
9935   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9936   effect(KILL cr);
9937 
9938   ins_cost(125);
9939   format %{ "andq    $dst, $src\t# long" %}
9940   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9941   ins_encode(REX_mem_wide(dst), OpcSE(src),
9942              RM_opc_mem(secondary, dst), Con8or32(src));
9943   ins_pipe(ialu_mem_imm);
9944 %}
9945 
9946 // BMI1 instructions
9947 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9948   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9949   predicate(UseBMI1Instructions);
9950   effect(KILL cr);
9951 
9952   ins_cost(125);
9953   format %{ "andnq  $dst, $src1, $src2" %}
9954 
9955   ins_encode %{
9956     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9957   %}
9958   ins_pipe(ialu_reg_mem);
9959 %}
9960 
9961 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9962   match(Set dst (AndL (XorL src1 minus_1) src2));
9963   predicate(UseBMI1Instructions);
9964   effect(KILL cr);
9965 
9966   format %{ "andnq  $dst, $src1, $src2" %}
9967 
9968   ins_encode %{
9969   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9970   %}
9971   ins_pipe(ialu_reg_mem);
9972 %}
9973 
9974 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9975   match(Set dst (AndL (SubL imm_zero src) src));
9976   predicate(UseBMI1Instructions);
9977   effect(KILL cr);
9978 
9979   format %{ "blsiq  $dst, $src" %}
9980 
9981   ins_encode %{
9982     __ blsiq($dst$$Register, $src$$Register);
9983   %}
9984   ins_pipe(ialu_reg);
9985 %}
9986 
9987 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9988   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9989   predicate(UseBMI1Instructions);
9990   effect(KILL cr);
9991 
9992   ins_cost(125);
9993   format %{ "blsiq  $dst, $src" %}
9994 
9995   ins_encode %{
9996     __ blsiq($dst$$Register, $src$$Address);
9997   %}
9998   ins_pipe(ialu_reg_mem);
9999 %}
10000 
10001 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10002 %{
10003   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10004   predicate(UseBMI1Instructions);
10005   effect(KILL cr);
10006 
10007   ins_cost(125);
10008   format %{ "blsmskq $dst, $src" %}
10009 
10010   ins_encode %{
10011     __ blsmskq($dst$$Register, $src$$Address);
10012   %}
10013   ins_pipe(ialu_reg_mem);
10014 %}
10015 
10016 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10017 %{
10018   match(Set dst (XorL (AddL src minus_1) src));
10019   predicate(UseBMI1Instructions);
10020   effect(KILL cr);
10021 
10022   format %{ "blsmskq $dst, $src" %}
10023 
10024   ins_encode %{
10025     __ blsmskq($dst$$Register, $src$$Register);
10026   %}
10027 
10028   ins_pipe(ialu_reg);
10029 %}
10030 
10031 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10032 %{
10033   match(Set dst (AndL (AddL src minus_1) src) );
10034   predicate(UseBMI1Instructions);
10035   effect(KILL cr);
10036 
10037   format %{ "blsrq  $dst, $src" %}
10038 
10039   ins_encode %{
10040     __ blsrq($dst$$Register, $src$$Register);
10041   %}
10042 
10043   ins_pipe(ialu_reg);
10044 %}
10045 
10046 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10047 %{
10048   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10049   predicate(UseBMI1Instructions);
10050   effect(KILL cr);
10051 
10052   ins_cost(125);
10053   format %{ "blsrq  $dst, $src" %}
10054 
10055   ins_encode %{
10056     __ blsrq($dst$$Register, $src$$Address);
10057   %}
10058 
10059   ins_pipe(ialu_reg);
10060 %}
10061 
10062 // Or Instructions
10063 // Or Register with Register
10064 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10065 %{
10066   match(Set dst (OrL dst src));
10067   effect(KILL cr);
10068 
10069   format %{ "orq     $dst, $src\t# long" %}
10070   opcode(0x0B);
10071   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10072   ins_pipe(ialu_reg_reg);
10073 %}
10074 
10075 // Use any_RegP to match R15 (TLS register) without spilling.
10076 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10077   match(Set dst (OrL dst (CastP2X src)));
10078   effect(KILL cr);
10079 
10080   format %{ "orq     $dst, $src\t# long" %}
10081   opcode(0x0B);
10082   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10083   ins_pipe(ialu_reg_reg);
10084 %}
10085 
10086 
10087 // Or Register with Immediate
10088 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10089 %{
10090   match(Set dst (OrL dst src));
10091   effect(KILL cr);
10092 
10093   format %{ "orq     $dst, $src\t# long" %}
10094   opcode(0x81, 0x01); /* Opcode 81 /1 id */
10095   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10096   ins_pipe(ialu_reg);
10097 %}
10098 
10099 // Or Register with Memory
10100 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10101 %{
10102   match(Set dst (OrL dst (LoadL src)));
10103   effect(KILL cr);
10104 
10105   ins_cost(125);
10106   format %{ "orq     $dst, $src\t# long" %}
10107   opcode(0x0B);
10108   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10109   ins_pipe(ialu_reg_mem);
10110 %}
10111 
10112 // Or Memory with Register
10113 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10114 %{
10115   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10116   effect(KILL cr);
10117 
10118   ins_cost(150);
10119   format %{ "orq     $dst, $src\t# long" %}
10120   opcode(0x09); /* Opcode 09 /r */
10121   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10122   ins_pipe(ialu_mem_reg);
10123 %}
10124 
10125 // Or Memory with Immediate
10126 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10127 %{
10128   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10129   effect(KILL cr);
10130 
10131   ins_cost(125);
10132   format %{ "orq     $dst, $src\t# long" %}
10133   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10134   ins_encode(REX_mem_wide(dst), OpcSE(src),
10135              RM_opc_mem(secondary, dst), Con8or32(src));
10136   ins_pipe(ialu_mem_imm);
10137 %}
10138 
10139 // Xor Instructions
10140 // Xor Register with Register
10141 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10142 %{
10143   match(Set dst (XorL dst src));
10144   effect(KILL cr);
10145 
10146   format %{ "xorq    $dst, $src\t# long" %}
10147   opcode(0x33);
10148   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10149   ins_pipe(ialu_reg_reg);
10150 %}
10151 
10152 // Xor Register with Immediate -1
10153 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10154   match(Set dst (XorL dst imm));
10155 
10156   format %{ "notq   $dst" %}
10157   ins_encode %{
10158      __ notq($dst$$Register);
10159   %}
10160   ins_pipe(ialu_reg);
10161 %}
10162 
10163 // Xor Register with Immediate
10164 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10165 %{
10166   match(Set dst (XorL dst src));
10167   effect(KILL cr);
10168 
10169   format %{ "xorq    $dst, $src\t# long" %}
10170   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10171   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10172   ins_pipe(ialu_reg);
10173 %}
10174 
10175 // Xor Register with Memory
10176 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10177 %{
10178   match(Set dst (XorL dst (LoadL src)));
10179   effect(KILL cr);
10180 
10181   ins_cost(125);
10182   format %{ "xorq    $dst, $src\t# long" %}
10183   opcode(0x33);
10184   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10185   ins_pipe(ialu_reg_mem);
10186 %}
10187 
10188 // Xor Memory with Register
10189 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10190 %{
10191   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10192   effect(KILL cr);
10193 
10194   ins_cost(150);
10195   format %{ "xorq    $dst, $src\t# long" %}
10196   opcode(0x31); /* Opcode 31 /r */
10197   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10198   ins_pipe(ialu_mem_reg);
10199 %}
10200 
10201 // Xor Memory with Immediate
10202 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10203 %{
10204   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10205   effect(KILL cr);
10206 
10207   ins_cost(125);
10208   format %{ "xorq    $dst, $src\t# long" %}
10209   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10210   ins_encode(REX_mem_wide(dst), OpcSE(src),
10211              RM_opc_mem(secondary, dst), Con8or32(src));
10212   ins_pipe(ialu_mem_imm);
10213 %}
10214 
10215 // Convert Int to Boolean
10216 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10217 %{
10218   match(Set dst (Conv2B src));
10219   effect(KILL cr);
10220 
10221   format %{ "testl   $src, $src\t# ci2b\n\t"
10222             "setnz   $dst\n\t"
10223             "movzbl  $dst, $dst" %}
10224   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10225              setNZ_reg(dst),
10226              REX_reg_breg(dst, dst), // movzbl
10227              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10228   ins_pipe(pipe_slow); // XXX
10229 %}
10230 
10231 // Convert Pointer to Boolean
10232 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10233 %{
10234   match(Set dst (Conv2B src));
10235   effect(KILL cr);
10236 
10237   format %{ "testq   $src, $src\t# cp2b\n\t"
10238             "setnz   $dst\n\t"
10239             "movzbl  $dst, $dst" %}
10240   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10241              setNZ_reg(dst),
10242              REX_reg_breg(dst, dst), // movzbl
10243              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10244   ins_pipe(pipe_slow); // XXX
10245 %}
10246 
10247 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10248 %{
10249   match(Set dst (CmpLTMask p q));
10250   effect(KILL cr);
10251 
10252   ins_cost(400);
10253   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10254             "setlt   $dst\n\t"
10255             "movzbl  $dst, $dst\n\t"
10256             "negl    $dst" %}
10257   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10258              setLT_reg(dst),
10259              REX_reg_breg(dst, dst), // movzbl
10260              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10261              neg_reg(dst));
10262   ins_pipe(pipe_slow);
10263 %}
10264 
10265 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10266 %{
10267   match(Set dst (CmpLTMask dst zero));
10268   effect(KILL cr);
10269 
10270   ins_cost(100);
10271   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10272   ins_encode %{
10273   __ sarl($dst$$Register, 31);
10274   %}
10275   ins_pipe(ialu_reg);
10276 %}
10277 
10278 /* Better to save a register than avoid a branch */
10279 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10280 %{
10281   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10282   effect(KILL cr);
10283   ins_cost(300);
10284   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
10285             "jge    done\n\t"
10286             "addl   $p,$y\n"
10287             "done:  " %}
10288   ins_encode %{
10289     Register Rp = $p$$Register;
10290     Register Rq = $q$$Register;
10291     Register Ry = $y$$Register;
10292     Label done;
10293     __ subl(Rp, Rq);
10294     __ jccb(Assembler::greaterEqual, done);
10295     __ addl(Rp, Ry);
10296     __ bind(done);
10297   %}
10298   ins_pipe(pipe_cmplt);
10299 %}
10300 
10301 /* Better to save a register than avoid a branch */
10302 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10303 %{
10304   match(Set y (AndI (CmpLTMask p q) y));
10305   effect(KILL cr);
10306 
10307   ins_cost(300);
10308 
10309   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
10310             "jlt      done\n\t"
10311             "xorl     $y, $y\n"
10312             "done:  " %}
10313   ins_encode %{
10314     Register Rp = $p$$Register;
10315     Register Rq = $q$$Register;
10316     Register Ry = $y$$Register;
10317     Label done;
10318     __ cmpl(Rp, Rq);
10319     __ jccb(Assembler::less, done);
10320     __ xorl(Ry, Ry);
10321     __ bind(done);
10322   %}
10323   ins_pipe(pipe_cmplt);
10324 %}
10325 
10326 
10327 //---------- FP Instructions------------------------------------------------
10328 
10329 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10330 %{
10331   match(Set cr (CmpF src1 src2));
10332 
10333   ins_cost(145);
10334   format %{ "ucomiss $src1, $src2\n\t"
10335             "jnp,s   exit\n\t"
10336             "pushfq\t# saw NaN, set CF\n\t"
10337             "andq    [rsp], #0xffffff2b\n\t"
10338             "popfq\n"
10339     "exit:" %}
10340   ins_encode %{
10341     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10342     emit_cmpfp_fixup(_masm);
10343   %}
10344   ins_pipe(pipe_slow);
10345 %}
10346 
10347 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10348   match(Set cr (CmpF src1 src2));
10349 
10350   ins_cost(100);
10351   format %{ "ucomiss $src1, $src2" %}
10352   ins_encode %{
10353     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10354   %}
10355   ins_pipe(pipe_slow);
10356 %}
10357 
10358 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10359 %{
10360   match(Set cr (CmpF src1 (LoadF src2)));
10361 
10362   ins_cost(145);
10363   format %{ "ucomiss $src1, $src2\n\t"
10364             "jnp,s   exit\n\t"
10365             "pushfq\t# saw NaN, set CF\n\t"
10366             "andq    [rsp], #0xffffff2b\n\t"
10367             "popfq\n"
10368     "exit:" %}
10369   ins_encode %{
10370     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10371     emit_cmpfp_fixup(_masm);
10372   %}
10373   ins_pipe(pipe_slow);
10374 %}
10375 
10376 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10377   match(Set cr (CmpF src1 (LoadF src2)));
10378 
10379   ins_cost(100);
10380   format %{ "ucomiss $src1, $src2" %}
10381   ins_encode %{
10382     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10383   %}
10384   ins_pipe(pipe_slow);
10385 %}
10386 
10387 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10388   match(Set cr (CmpF src con));
10389 
10390   ins_cost(145);
10391   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10392             "jnp,s   exit\n\t"
10393             "pushfq\t# saw NaN, set CF\n\t"
10394             "andq    [rsp], #0xffffff2b\n\t"
10395             "popfq\n"
10396     "exit:" %}
10397   ins_encode %{
10398     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10399     emit_cmpfp_fixup(_masm);
10400   %}
10401   ins_pipe(pipe_slow);
10402 %}
10403 
10404 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10405   match(Set cr (CmpF src con));
10406   ins_cost(100);
10407   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10408   ins_encode %{
10409     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10410   %}
10411   ins_pipe(pipe_slow);
10412 %}
10413 
10414 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10415 %{
10416   match(Set cr (CmpD src1 src2));
10417 
10418   ins_cost(145);
10419   format %{ "ucomisd $src1, $src2\n\t"
10420             "jnp,s   exit\n\t"
10421             "pushfq\t# saw NaN, set CF\n\t"
10422             "andq    [rsp], #0xffffff2b\n\t"
10423             "popfq\n"
10424     "exit:" %}
10425   ins_encode %{
10426     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10427     emit_cmpfp_fixup(_masm);
10428   %}
10429   ins_pipe(pipe_slow);
10430 %}
10431 
10432 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10433   match(Set cr (CmpD src1 src2));
10434 
10435   ins_cost(100);
10436   format %{ "ucomisd $src1, $src2 test" %}
10437   ins_encode %{
10438     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10439   %}
10440   ins_pipe(pipe_slow);
10441 %}
10442 
10443 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10444 %{
10445   match(Set cr (CmpD src1 (LoadD src2)));
10446 
10447   ins_cost(145);
10448   format %{ "ucomisd $src1, $src2\n\t"
10449             "jnp,s   exit\n\t"
10450             "pushfq\t# saw NaN, set CF\n\t"
10451             "andq    [rsp], #0xffffff2b\n\t"
10452             "popfq\n"
10453     "exit:" %}
10454   ins_encode %{
10455     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10456     emit_cmpfp_fixup(_masm);
10457   %}
10458   ins_pipe(pipe_slow);
10459 %}
10460 
10461 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10462   match(Set cr (CmpD src1 (LoadD src2)));
10463 
10464   ins_cost(100);
10465   format %{ "ucomisd $src1, $src2" %}
10466   ins_encode %{
10467     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10468   %}
10469   ins_pipe(pipe_slow);
10470 %}
10471 
10472 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10473   match(Set cr (CmpD src con));
10474 
10475   ins_cost(145);
10476   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10477             "jnp,s   exit\n\t"
10478             "pushfq\t# saw NaN, set CF\n\t"
10479             "andq    [rsp], #0xffffff2b\n\t"
10480             "popfq\n"
10481     "exit:" %}
10482   ins_encode %{
10483     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10484     emit_cmpfp_fixup(_masm);
10485   %}
10486   ins_pipe(pipe_slow);
10487 %}
10488 
10489 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10490   match(Set cr (CmpD src con));
10491   ins_cost(100);
10492   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10493   ins_encode %{
10494     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10495   %}
10496   ins_pipe(pipe_slow);
10497 %}
10498 
10499 // Compare into -1,0,1
10500 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10501 %{
10502   match(Set dst (CmpF3 src1 src2));
10503   effect(KILL cr);
10504 
10505   ins_cost(275);
10506   format %{ "ucomiss $src1, $src2\n\t"
10507             "movl    $dst, #-1\n\t"
10508             "jp,s    done\n\t"
10509             "jb,s    done\n\t"
10510             "setne   $dst\n\t"
10511             "movzbl  $dst, $dst\n"
10512     "done:" %}
10513   ins_encode %{
10514     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10515     emit_cmpfp3(_masm, $dst$$Register);
10516   %}
10517   ins_pipe(pipe_slow);
10518 %}
10519 
10520 // Compare into -1,0,1
10521 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10522 %{
10523   match(Set dst (CmpF3 src1 (LoadF src2)));
10524   effect(KILL cr);
10525 
10526   ins_cost(275);
10527   format %{ "ucomiss $src1, $src2\n\t"
10528             "movl    $dst, #-1\n\t"
10529             "jp,s    done\n\t"
10530             "jb,s    done\n\t"
10531             "setne   $dst\n\t"
10532             "movzbl  $dst, $dst\n"
10533     "done:" %}
10534   ins_encode %{
10535     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10536     emit_cmpfp3(_masm, $dst$$Register);
10537   %}
10538   ins_pipe(pipe_slow);
10539 %}
10540 
10541 // Compare into -1,0,1
10542 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10543   match(Set dst (CmpF3 src con));
10544   effect(KILL cr);
10545 
10546   ins_cost(275);
10547   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10548             "movl    $dst, #-1\n\t"
10549             "jp,s    done\n\t"
10550             "jb,s    done\n\t"
10551             "setne   $dst\n\t"
10552             "movzbl  $dst, $dst\n"
10553     "done:" %}
10554   ins_encode %{
10555     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10556     emit_cmpfp3(_masm, $dst$$Register);
10557   %}
10558   ins_pipe(pipe_slow);
10559 %}
10560 
10561 // Compare into -1,0,1
10562 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10563 %{
10564   match(Set dst (CmpD3 src1 src2));
10565   effect(KILL cr);
10566 
10567   ins_cost(275);
10568   format %{ "ucomisd $src1, $src2\n\t"
10569             "movl    $dst, #-1\n\t"
10570             "jp,s    done\n\t"
10571             "jb,s    done\n\t"
10572             "setne   $dst\n\t"
10573             "movzbl  $dst, $dst\n"
10574     "done:" %}
10575   ins_encode %{
10576     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10577     emit_cmpfp3(_masm, $dst$$Register);
10578   %}
10579   ins_pipe(pipe_slow);
10580 %}
10581 
10582 // Compare into -1,0,1
10583 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10584 %{
10585   match(Set dst (CmpD3 src1 (LoadD src2)));
10586   effect(KILL cr);
10587 
10588   ins_cost(275);
10589   format %{ "ucomisd $src1, $src2\n\t"
10590             "movl    $dst, #-1\n\t"
10591             "jp,s    done\n\t"
10592             "jb,s    done\n\t"
10593             "setne   $dst\n\t"
10594             "movzbl  $dst, $dst\n"
10595     "done:" %}
10596   ins_encode %{
10597     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10598     emit_cmpfp3(_masm, $dst$$Register);
10599   %}
10600   ins_pipe(pipe_slow);
10601 %}
10602 
10603 // Compare into -1,0,1
10604 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10605   match(Set dst (CmpD3 src con));
10606   effect(KILL cr);
10607 
10608   ins_cost(275);
10609   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10610             "movl    $dst, #-1\n\t"
10611             "jp,s    done\n\t"
10612             "jb,s    done\n\t"
10613             "setne   $dst\n\t"
10614             "movzbl  $dst, $dst\n"
10615     "done:" %}
10616   ins_encode %{
10617     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10618     emit_cmpfp3(_masm, $dst$$Register);
10619   %}
10620   ins_pipe(pipe_slow);
10621 %}
10622 
10623 //----------Arithmetic Conversion Instructions---------------------------------
10624 
10625 instruct roundFloat_nop(regF dst)
10626 %{
10627   match(Set dst (RoundFloat dst));
10628 
10629   ins_cost(0);
10630   ins_encode();
10631   ins_pipe(empty);
10632 %}
10633 
10634 instruct roundDouble_nop(regD dst)
10635 %{
10636   match(Set dst (RoundDouble dst));
10637 
10638   ins_cost(0);
10639   ins_encode();
10640   ins_pipe(empty);
10641 %}
10642 
10643 instruct convF2D_reg_reg(regD dst, regF src)
10644 %{
10645   match(Set dst (ConvF2D src));
10646 
10647   format %{ "cvtss2sd $dst, $src" %}
10648   ins_encode %{
10649     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10650   %}
10651   ins_pipe(pipe_slow); // XXX
10652 %}
10653 
10654 instruct convF2D_reg_mem(regD dst, memory src)
10655 %{
10656   match(Set dst (ConvF2D (LoadF src)));
10657 
10658   format %{ "cvtss2sd $dst, $src" %}
10659   ins_encode %{
10660     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10661   %}
10662   ins_pipe(pipe_slow); // XXX
10663 %}
10664 
10665 instruct convD2F_reg_reg(regF dst, regD src)
10666 %{
10667   match(Set dst (ConvD2F src));
10668 
10669   format %{ "cvtsd2ss $dst, $src" %}
10670   ins_encode %{
10671     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10672   %}
10673   ins_pipe(pipe_slow); // XXX
10674 %}
10675 
10676 instruct convD2F_reg_mem(regF dst, memory src)
10677 %{
10678   match(Set dst (ConvD2F (LoadD src)));
10679 
10680   format %{ "cvtsd2ss $dst, $src" %}
10681   ins_encode %{
10682     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10683   %}
10684   ins_pipe(pipe_slow); // XXX
10685 %}
10686 
10687 // XXX do mem variants
10688 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10689 %{
10690   match(Set dst (ConvF2I src));
10691   effect(KILL cr);
10692 
10693   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10694             "cmpl    $dst, #0x80000000\n\t"
10695             "jne,s   done\n\t"
10696             "subq    rsp, #8\n\t"
10697             "movss   [rsp], $src\n\t"
10698             "call    f2i_fixup\n\t"
10699             "popq    $dst\n"
10700     "done:   "%}
10701   ins_encode %{
10702     Label done;
10703     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10704     __ cmpl($dst$$Register, 0x80000000);
10705     __ jccb(Assembler::notEqual, done);
10706     __ subptr(rsp, 8);
10707     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10708     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10709     __ pop($dst$$Register);
10710     __ bind(done);
10711   %}
10712   ins_pipe(pipe_slow);
10713 %}
10714 
10715 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10716 %{
10717   match(Set dst (ConvF2L src));
10718   effect(KILL cr);
10719 
10720   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10721             "cmpq    $dst, [0x8000000000000000]\n\t"
10722             "jne,s   done\n\t"
10723             "subq    rsp, #8\n\t"
10724             "movss   [rsp], $src\n\t"
10725             "call    f2l_fixup\n\t"
10726             "popq    $dst\n"
10727     "done:   "%}
10728   ins_encode %{
10729     Label done;
10730     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10731     __ cmp64($dst$$Register,
10732              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10733     __ jccb(Assembler::notEqual, done);
10734     __ subptr(rsp, 8);
10735     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10736     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10737     __ pop($dst$$Register);
10738     __ bind(done);
10739   %}
10740   ins_pipe(pipe_slow);
10741 %}
10742 
10743 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10744 %{
10745   match(Set dst (ConvD2I src));
10746   effect(KILL cr);
10747 
10748   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10749             "cmpl    $dst, #0x80000000\n\t"
10750             "jne,s   done\n\t"
10751             "subq    rsp, #8\n\t"
10752             "movsd   [rsp], $src\n\t"
10753             "call    d2i_fixup\n\t"
10754             "popq    $dst\n"
10755     "done:   "%}
10756   ins_encode %{
10757     Label done;
10758     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10759     __ cmpl($dst$$Register, 0x80000000);
10760     __ jccb(Assembler::notEqual, done);
10761     __ subptr(rsp, 8);
10762     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10763     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10764     __ pop($dst$$Register);
10765     __ bind(done);
10766   %}
10767   ins_pipe(pipe_slow);
10768 %}
10769 
10770 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10771 %{
10772   match(Set dst (ConvD2L src));
10773   effect(KILL cr);
10774 
10775   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10776             "cmpq    $dst, [0x8000000000000000]\n\t"
10777             "jne,s   done\n\t"
10778             "subq    rsp, #8\n\t"
10779             "movsd   [rsp], $src\n\t"
10780             "call    d2l_fixup\n\t"
10781             "popq    $dst\n"
10782     "done:   "%}
10783   ins_encode %{
10784     Label done;
10785     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10786     __ cmp64($dst$$Register,
10787              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10788     __ jccb(Assembler::notEqual, done);
10789     __ subptr(rsp, 8);
10790     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10791     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10792     __ pop($dst$$Register);
10793     __ bind(done);
10794   %}
10795   ins_pipe(pipe_slow);
10796 %}
10797 
10798 instruct convI2F_reg_reg(regF dst, rRegI src)
10799 %{
10800   predicate(!UseXmmI2F);
10801   match(Set dst (ConvI2F src));
10802 
10803   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10804   ins_encode %{
10805     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10806   %}
10807   ins_pipe(pipe_slow); // XXX
10808 %}
10809 
10810 instruct convI2F_reg_mem(regF dst, memory src)
10811 %{
10812   match(Set dst (ConvI2F (LoadI src)));
10813 
10814   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10815   ins_encode %{
10816     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10817   %}
10818   ins_pipe(pipe_slow); // XXX
10819 %}
10820 
10821 instruct convI2D_reg_reg(regD dst, rRegI src)
10822 %{
10823   predicate(!UseXmmI2D);
10824   match(Set dst (ConvI2D src));
10825 
10826   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10827   ins_encode %{
10828     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10829   %}
10830   ins_pipe(pipe_slow); // XXX
10831 %}
10832 
10833 instruct convI2D_reg_mem(regD dst, memory src)
10834 %{
10835   match(Set dst (ConvI2D (LoadI src)));
10836 
10837   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10838   ins_encode %{
10839     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10840   %}
10841   ins_pipe(pipe_slow); // XXX
10842 %}
10843 
10844 instruct convXI2F_reg(regF dst, rRegI src)
10845 %{
10846   predicate(UseXmmI2F);
10847   match(Set dst (ConvI2F src));
10848 
10849   format %{ "movdl $dst, $src\n\t"
10850             "cvtdq2psl $dst, $dst\t# i2f" %}
10851   ins_encode %{
10852     __ movdl($dst$$XMMRegister, $src$$Register);
10853     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10854   %}
10855   ins_pipe(pipe_slow); // XXX
10856 %}
10857 
10858 instruct convXI2D_reg(regD dst, rRegI src)
10859 %{
10860   predicate(UseXmmI2D);
10861   match(Set dst (ConvI2D src));
10862 
10863   format %{ "movdl $dst, $src\n\t"
10864             "cvtdq2pdl $dst, $dst\t# i2d" %}
10865   ins_encode %{
10866     __ movdl($dst$$XMMRegister, $src$$Register);
10867     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10868   %}
10869   ins_pipe(pipe_slow); // XXX
10870 %}
10871 
10872 instruct convL2F_reg_reg(regF dst, rRegL src)
10873 %{
10874   match(Set dst (ConvL2F src));
10875 
10876   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10877   ins_encode %{
10878     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10879   %}
10880   ins_pipe(pipe_slow); // XXX
10881 %}
10882 
10883 instruct convL2F_reg_mem(regF dst, memory src)
10884 %{
10885   match(Set dst (ConvL2F (LoadL src)));
10886 
10887   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10888   ins_encode %{
10889     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10890   %}
10891   ins_pipe(pipe_slow); // XXX
10892 %}
10893 
10894 instruct convL2D_reg_reg(regD dst, rRegL src)
10895 %{
10896   match(Set dst (ConvL2D src));
10897 
10898   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10899   ins_encode %{
10900     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10901   %}
10902   ins_pipe(pipe_slow); // XXX
10903 %}
10904 
10905 instruct convL2D_reg_mem(regD dst, memory src)
10906 %{
10907   match(Set dst (ConvL2D (LoadL src)));
10908 
10909   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10910   ins_encode %{
10911     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10912   %}
10913   ins_pipe(pipe_slow); // XXX
10914 %}
10915 
10916 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10917 %{
10918   match(Set dst (ConvI2L src));
10919 
10920   ins_cost(125);
10921   format %{ "movslq  $dst, $src\t# i2l" %}
10922   ins_encode %{
10923     __ movslq($dst$$Register, $src$$Register);
10924   %}
10925   ins_pipe(ialu_reg_reg);
10926 %}
10927 
10928 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10929 // %{
10930 //   match(Set dst (ConvI2L src));
10931 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10932 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10933 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10934 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10935 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10936 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10937 
10938 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10939 //   ins_encode(enc_copy(dst, src));
10940 // //   opcode(0x63); // needs REX.W
10941 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10942 //   ins_pipe(ialu_reg_reg);
10943 // %}
10944 
10945 // Zero-extend convert int to long
10946 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10947 %{
10948   match(Set dst (AndL (ConvI2L src) mask));
10949 
10950   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10951   ins_encode %{
10952     if ($dst$$reg != $src$$reg) {
10953       __ movl($dst$$Register, $src$$Register);
10954     }
10955   %}
10956   ins_pipe(ialu_reg_reg);
10957 %}
10958 
10959 // Zero-extend convert int to long
10960 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10961 %{
10962   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10963 
10964   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10965   ins_encode %{
10966     __ movl($dst$$Register, $src$$Address);
10967   %}
10968   ins_pipe(ialu_reg_mem);
10969 %}
10970 
10971 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10972 %{
10973   match(Set dst (AndL src mask));
10974 
10975   format %{ "movl    $dst, $src\t# zero-extend long" %}
10976   ins_encode %{
10977     __ movl($dst$$Register, $src$$Register);
10978   %}
10979   ins_pipe(ialu_reg_reg);
10980 %}
10981 
10982 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10983 %{
10984   match(Set dst (ConvL2I src));
10985 
10986   format %{ "movl    $dst, $src\t# l2i" %}
10987   ins_encode %{
10988     __ movl($dst$$Register, $src$$Register);
10989   %}
10990   ins_pipe(ialu_reg_reg);
10991 %}
10992 
10993 
10994 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10995   match(Set dst (MoveF2I src));
10996   effect(DEF dst, USE src);
10997 
10998   ins_cost(125);
10999   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11000   ins_encode %{
11001     __ movl($dst$$Register, Address(rsp, $src$$disp));
11002   %}
11003   ins_pipe(ialu_reg_mem);
11004 %}
11005 
11006 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11007   match(Set dst (MoveI2F src));
11008   effect(DEF dst, USE src);
11009 
11010   ins_cost(125);
11011   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11012   ins_encode %{
11013     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11014   %}
11015   ins_pipe(pipe_slow);
11016 %}
11017 
11018 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11019   match(Set dst (MoveD2L src));
11020   effect(DEF dst, USE src);
11021 
11022   ins_cost(125);
11023   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11024   ins_encode %{
11025     __ movq($dst$$Register, Address(rsp, $src$$disp));
11026   %}
11027   ins_pipe(ialu_reg_mem);
11028 %}
11029 
11030 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11031   predicate(!UseXmmLoadAndClearUpper);
11032   match(Set dst (MoveL2D src));
11033   effect(DEF dst, USE src);
11034 
11035   ins_cost(125);
11036   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11037   ins_encode %{
11038     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11039   %}
11040   ins_pipe(pipe_slow);
11041 %}
11042 
11043 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11044   predicate(UseXmmLoadAndClearUpper);
11045   match(Set dst (MoveL2D src));
11046   effect(DEF dst, USE src);
11047 
11048   ins_cost(125);
11049   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11050   ins_encode %{
11051     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11052   %}
11053   ins_pipe(pipe_slow);
11054 %}
11055 
11056 
11057 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11058   match(Set dst (MoveF2I src));
11059   effect(DEF dst, USE src);
11060 
11061   ins_cost(95); // XXX
11062   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11063   ins_encode %{
11064     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11065   %}
11066   ins_pipe(pipe_slow);
11067 %}
11068 
11069 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11070   match(Set dst (MoveI2F src));
11071   effect(DEF dst, USE src);
11072 
11073   ins_cost(100);
11074   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11075   ins_encode %{
11076     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11077   %}
11078   ins_pipe( ialu_mem_reg );
11079 %}
11080 
11081 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11082   match(Set dst (MoveD2L src));
11083   effect(DEF dst, USE src);
11084 
11085   ins_cost(95); // XXX
11086   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11087   ins_encode %{
11088     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11089   %}
11090   ins_pipe(pipe_slow);
11091 %}
11092 
11093 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11094   match(Set dst (MoveL2D src));
11095   effect(DEF dst, USE src);
11096 
11097   ins_cost(100);
11098   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11099   ins_encode %{
11100     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11101   %}
11102   ins_pipe(ialu_mem_reg);
11103 %}
11104 
11105 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11106   match(Set dst (MoveF2I src));
11107   effect(DEF dst, USE src);
11108   ins_cost(85);
11109   format %{ "movd    $dst,$src\t# MoveF2I" %}
11110   ins_encode %{
11111     __ movdl($dst$$Register, $src$$XMMRegister);
11112   %}
11113   ins_pipe( pipe_slow );
11114 %}
11115 
11116 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11117   match(Set dst (MoveD2L src));
11118   effect(DEF dst, USE src);
11119   ins_cost(85);
11120   format %{ "movd    $dst,$src\t# MoveD2L" %}
11121   ins_encode %{
11122     __ movdq($dst$$Register, $src$$XMMRegister);
11123   %}
11124   ins_pipe( pipe_slow );
11125 %}
11126 
11127 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11128   match(Set dst (MoveI2F src));
11129   effect(DEF dst, USE src);
11130   ins_cost(100);
11131   format %{ "movd    $dst,$src\t# MoveI2F" %}
11132   ins_encode %{
11133     __ movdl($dst$$XMMRegister, $src$$Register);
11134   %}
11135   ins_pipe( pipe_slow );
11136 %}
11137 
11138 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11139   match(Set dst (MoveL2D src));
11140   effect(DEF dst, USE src);
11141   ins_cost(100);
11142   format %{ "movd    $dst,$src\t# MoveL2D" %}
11143   ins_encode %{
11144      __ movdq($dst$$XMMRegister, $src$$Register);
11145   %}
11146   ins_pipe( pipe_slow );
11147 %}
11148 
11149 
11150 // =======================================================================
11151 // fast clearing of an array
11152 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11153                   Universe dummy, rFlagsReg cr)
11154 %{
11155   predicate(!((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only());
11156   match(Set dummy (ClearArray (Binary cnt base) val));
11157   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11158 
11159   format %{ $$template
11160     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11161     $$emit$$"jg      LARGE\n\t"
11162     $$emit$$"dec     rcx\n\t"
11163     $$emit$$"js      DONE\t# Zero length\n\t"
11164     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11165     $$emit$$"dec     rcx\n\t"
11166     $$emit$$"jge     LOOP\n\t"
11167     $$emit$$"jmp     DONE\n\t"
11168     $$emit$$"# LARGE:\n\t"
11169     if (UseFastStosb) {
11170        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11171        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11172     } else if (UseXMMForObjInit) {
11173        $$emit$$"movdq   $tmp, $val\n\t"
11174        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11175        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11176        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11177        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11178        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11179        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11180        $$emit$$"add     0x40,rax\n\t"
11181        $$emit$$"# L_zero_64_bytes:\n\t"
11182        $$emit$$"sub     0x8,rcx\n\t"
11183        $$emit$$"jge     L_loop\n\t"
11184        $$emit$$"add     0x4,rcx\n\t"
11185        $$emit$$"jl      L_tail\n\t"
11186        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11187        $$emit$$"add     0x20,rax\n\t"
11188        $$emit$$"sub     0x4,rcx\n\t"
11189        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11190        $$emit$$"add     0x4,rcx\n\t"
11191        $$emit$$"jle     L_end\n\t"
11192        $$emit$$"dec     rcx\n\t"
11193        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11194        $$emit$$"vmovq   xmm0,(rax)\n\t"
11195        $$emit$$"add     0x8,rax\n\t"
11196        $$emit$$"dec     rcx\n\t"
11197        $$emit$$"jge     L_sloop\n\t"
11198        $$emit$$"# L_end:\n\t"
11199     } else {
11200        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11201     }
11202     $$emit$$"# DONE"
11203   %}
11204   ins_encode %{
11205     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11206                  $tmp$$XMMRegister, false, false);
11207   %}
11208   ins_pipe(pipe_slow);
11209 %}
11210 
11211 instruct rep_stos_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11212                   Universe dummy, rFlagsReg cr)
11213 %{
11214   predicate(!((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only());
11215   match(Set dummy (ClearArray (Binary cnt base) val));
11216   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11217 
11218   format %{ $$template
11219     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11220     $$emit$$"jg      LARGE\n\t"
11221     $$emit$$"dec     rcx\n\t"
11222     $$emit$$"js      DONE\t# Zero length\n\t"
11223     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11224     $$emit$$"dec     rcx\n\t"
11225     $$emit$$"jge     LOOP\n\t"
11226     $$emit$$"jmp     DONE\n\t"
11227     $$emit$$"# LARGE:\n\t"
11228     if (UseXMMForObjInit) {
11229        $$emit$$"movdq   $tmp, $val\n\t"
11230        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11231        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11232        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11233        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11234        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11235        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11236        $$emit$$"add     0x40,rax\n\t"
11237        $$emit$$"# L_zero_64_bytes:\n\t"
11238        $$emit$$"sub     0x8,rcx\n\t"
11239        $$emit$$"jge     L_loop\n\t"
11240        $$emit$$"add     0x4,rcx\n\t"
11241        $$emit$$"jl      L_tail\n\t"
11242        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11243        $$emit$$"add     0x20,rax\n\t"
11244        $$emit$$"sub     0x4,rcx\n\t"
11245        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11246        $$emit$$"add     0x4,rcx\n\t"
11247        $$emit$$"jle     L_end\n\t"
11248        $$emit$$"dec     rcx\n\t"
11249        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11250        $$emit$$"vmovq   xmm0,(rax)\n\t"
11251        $$emit$$"add     0x8,rax\n\t"
11252        $$emit$$"dec     rcx\n\t"
11253        $$emit$$"jge     L_sloop\n\t"
11254        $$emit$$"# L_end:\n\t"
11255     } else {
11256        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11257     }
11258     $$emit$$"# DONE"
11259   %}
11260   ins_encode %{
11261     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11262                  $tmp$$XMMRegister, false, true);
11263   %}
11264   ins_pipe(pipe_slow);
11265 %}
11266 
11267 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val,
11268                         Universe dummy, rFlagsReg cr)
11269 %{
11270   predicate(((ClearArrayNode*)n)->is_large() && !((ClearArrayNode*)n)->word_copy_only());
11271   match(Set dummy (ClearArray (Binary cnt base) val));
11272   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11273 
11274   format %{ $$template
11275     if (UseFastStosb) {
11276        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11277        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11278     } else if (UseXMMForObjInit) {
11279        $$emit$$"movdq   $tmp, $val\n\t"
11280        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11281        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11282        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11283        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11284        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11285        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11286        $$emit$$"add     0x40,rax\n\t"
11287        $$emit$$"# L_zero_64_bytes:\n\t"
11288        $$emit$$"sub     0x8,rcx\n\t"
11289        $$emit$$"jge     L_loop\n\t"
11290        $$emit$$"add     0x4,rcx\n\t"
11291        $$emit$$"jl      L_tail\n\t"
11292        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11293        $$emit$$"add     0x20,rax\n\t"
11294        $$emit$$"sub     0x4,rcx\n\t"
11295        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11296        $$emit$$"add     0x4,rcx\n\t"
11297        $$emit$$"jle     L_end\n\t"
11298        $$emit$$"dec     rcx\n\t"
11299        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11300        $$emit$$"vmovq   xmm0,(rax)\n\t"
11301        $$emit$$"add     0x8,rax\n\t"
11302        $$emit$$"dec     rcx\n\t"
11303        $$emit$$"jge     L_sloop\n\t"
11304        $$emit$$"# L_end:\n\t"
11305     } else {
11306        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11307     }
11308   %}
11309   ins_encode %{
11310     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register,
11311                  $tmp$$XMMRegister, true, false);
11312   %}
11313   ins_pipe(pipe_slow);
11314 %}
11315 
11316 instruct rep_stos_large_word_copy(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegL val, 
11317                         Universe dummy, rFlagsReg cr)
11318 %{
11319   predicate(((ClearArrayNode*)n)->is_large() && ((ClearArrayNode*)n)->word_copy_only());
11320   match(Set dummy (ClearArray (Binary cnt base) val));
11321   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL cr);
11322 
11323   format %{ $$template
11324     if (UseXMMForObjInit) {
11325        $$emit$$"movdq   $tmp, $val\n\t"
11326        $$emit$$"punpcklqdq $tmp, $tmp\n\t"
11327        $$emit$$"vinserti128_high $tmp, $tmp\n\t"
11328        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11329        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11330        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11331        $$emit$$"vmovdqu $tmp,0x20(rax)\n\t"
11332        $$emit$$"add     0x40,rax\n\t"
11333        $$emit$$"# L_zero_64_bytes:\n\t"
11334        $$emit$$"sub     0x8,rcx\n\t"
11335        $$emit$$"jge     L_loop\n\t"
11336        $$emit$$"add     0x4,rcx\n\t"
11337        $$emit$$"jl      L_tail\n\t"
11338        $$emit$$"vmovdqu $tmp,(rax)\n\t"
11339        $$emit$$"add     0x20,rax\n\t"
11340        $$emit$$"sub     0x4,rcx\n\t"
11341        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11342        $$emit$$"add     0x4,rcx\n\t"
11343        $$emit$$"jle     L_end\n\t"
11344        $$emit$$"dec     rcx\n\t"
11345        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11346        $$emit$$"vmovq   xmm0,(rax)\n\t"
11347        $$emit$$"add     0x8,rax\n\t"
11348        $$emit$$"dec     rcx\n\t"
11349        $$emit$$"jge     L_sloop\n\t"
11350        $$emit$$"# L_end:\n\t"
11351     } else {
11352        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11353     }
11354   %}
11355   ins_encode %{
11356     __ clear_mem($base$$Register, $cnt$$Register, $val$$Register, 
11357                  $tmp$$XMMRegister, true, true);
11358   %}
11359   ins_pipe(pipe_slow);
11360 %}
11361 
11362 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11363                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11364 %{
11365   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11366   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11367   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11368 
11369   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11370   ins_encode %{
11371     __ string_compare($str1$$Register, $str2$$Register,
11372                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11373                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11374   %}
11375   ins_pipe( pipe_slow );
11376 %}
11377 
11378 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11379                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11380 %{
11381   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11382   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11383   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11384 
11385   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11386   ins_encode %{
11387     __ string_compare($str1$$Register, $str2$$Register,
11388                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11389                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11390   %}
11391   ins_pipe( pipe_slow );
11392 %}
11393 
11394 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11395                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11396 %{
11397   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11398   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11399   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11400 
11401   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11402   ins_encode %{
11403     __ string_compare($str1$$Register, $str2$$Register,
11404                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11405                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11406   %}
11407   ins_pipe( pipe_slow );
11408 %}
11409 
11410 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11411                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11412 %{
11413   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11414   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11415   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11416 
11417   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11418   ins_encode %{
11419     __ string_compare($str2$$Register, $str1$$Register,
11420                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11421                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11422   %}
11423   ins_pipe( pipe_slow );
11424 %}
11425 
11426 // fast search of substring with known size.
11427 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11428                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11429 %{
11430   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11431   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11432   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11433 
11434   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11435   ins_encode %{
11436     int icnt2 = (int)$int_cnt2$$constant;
11437     if (icnt2 >= 16) {
11438       // IndexOf for constant substrings with size >= 16 elements
11439       // which don't need to be loaded through stack.
11440       __ string_indexofC8($str1$$Register, $str2$$Register,
11441                           $cnt1$$Register, $cnt2$$Register,
11442                           icnt2, $result$$Register,
11443                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11444     } else {
11445       // Small strings are loaded through stack if they cross page boundary.
11446       __ string_indexof($str1$$Register, $str2$$Register,
11447                         $cnt1$$Register, $cnt2$$Register,
11448                         icnt2, $result$$Register,
11449                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11450     }
11451   %}
11452   ins_pipe( pipe_slow );
11453 %}
11454 
11455 // fast search of substring with known size.
11456 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11457                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11458 %{
11459   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11460   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11461   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11462 
11463   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11464   ins_encode %{
11465     int icnt2 = (int)$int_cnt2$$constant;
11466     if (icnt2 >= 8) {
11467       // IndexOf for constant substrings with size >= 8 elements
11468       // which don't need to be loaded through stack.
11469       __ string_indexofC8($str1$$Register, $str2$$Register,
11470                           $cnt1$$Register, $cnt2$$Register,
11471                           icnt2, $result$$Register,
11472                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11473     } else {
11474       // Small strings are loaded through stack if they cross page boundary.
11475       __ string_indexof($str1$$Register, $str2$$Register,
11476                         $cnt1$$Register, $cnt2$$Register,
11477                         icnt2, $result$$Register,
11478                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11479     }
11480   %}
11481   ins_pipe( pipe_slow );
11482 %}
11483 
11484 // fast search of substring with known size.
11485 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11486                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11487 %{
11488   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11489   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11490   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11491 
11492   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11493   ins_encode %{
11494     int icnt2 = (int)$int_cnt2$$constant;
11495     if (icnt2 >= 8) {
11496       // IndexOf for constant substrings with size >= 8 elements
11497       // which don't need to be loaded through stack.
11498       __ string_indexofC8($str1$$Register, $str2$$Register,
11499                           $cnt1$$Register, $cnt2$$Register,
11500                           icnt2, $result$$Register,
11501                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11502     } else {
11503       // Small strings are loaded through stack if they cross page boundary.
11504       __ string_indexof($str1$$Register, $str2$$Register,
11505                         $cnt1$$Register, $cnt2$$Register,
11506                         icnt2, $result$$Register,
11507                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11508     }
11509   %}
11510   ins_pipe( pipe_slow );
11511 %}
11512 
11513 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11514                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11515 %{
11516   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11517   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11518   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11519 
11520   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11521   ins_encode %{
11522     __ string_indexof($str1$$Register, $str2$$Register,
11523                       $cnt1$$Register, $cnt2$$Register,
11524                       (-1), $result$$Register,
11525                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11526   %}
11527   ins_pipe( pipe_slow );
11528 %}
11529 
11530 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11531                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11532 %{
11533   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11534   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11535   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11536 
11537   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11538   ins_encode %{
11539     __ string_indexof($str1$$Register, $str2$$Register,
11540                       $cnt1$$Register, $cnt2$$Register,
11541                       (-1), $result$$Register,
11542                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11543   %}
11544   ins_pipe( pipe_slow );
11545 %}
11546 
11547 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11548                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11549 %{
11550   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11551   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11552   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11553 
11554   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11555   ins_encode %{
11556     __ string_indexof($str1$$Register, $str2$$Register,
11557                       $cnt1$$Register, $cnt2$$Register,
11558                       (-1), $result$$Register,
11559                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11560   %}
11561   ins_pipe( pipe_slow );
11562 %}
11563 
11564 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11565                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11566 %{
11567   predicate(UseSSE42Intrinsics);
11568   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11569   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11570   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11571   ins_encode %{
11572     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11573                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11574   %}
11575   ins_pipe( pipe_slow );
11576 %}
11577 
11578 // fast string equals
11579 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11580                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11581 %{
11582   match(Set result (StrEquals (Binary str1 str2) cnt));
11583   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11584 
11585   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11586   ins_encode %{
11587     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11588                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11589                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11590   %}
11591   ins_pipe( pipe_slow );
11592 %}
11593 
11594 // fast array equals
11595 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11596                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11597 %{
11598   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11599   match(Set result (AryEq ary1 ary2));
11600   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11601 
11602   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11603   ins_encode %{
11604     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11605                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11606                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11607   %}
11608   ins_pipe( pipe_slow );
11609 %}
11610 
11611 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11612                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11613 %{
11614   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11615   match(Set result (AryEq ary1 ary2));
11616   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11617 
11618   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11619   ins_encode %{
11620     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11621                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11622                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11623   %}
11624   ins_pipe( pipe_slow );
11625 %}
11626 
11627 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11628                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11629 %{
11630   match(Set result (HasNegatives ary1 len));
11631   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11632 
11633   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11634   ins_encode %{
11635     __ has_negatives($ary1$$Register, $len$$Register,
11636                      $result$$Register, $tmp3$$Register,
11637                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11638   %}
11639   ins_pipe( pipe_slow );
11640 %}
11641 
11642 // fast char[] to byte[] compression
11643 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11644                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11645   match(Set result (StrCompressedCopy src (Binary dst len)));
11646   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11647 
11648   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11649   ins_encode %{
11650     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11651                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11652                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11653   %}
11654   ins_pipe( pipe_slow );
11655 %}
11656 
11657 // fast byte[] to char[] inflation
11658 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11659                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11660   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11661   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11662 
11663   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11664   ins_encode %{
11665     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11666                           $tmp1$$XMMRegister, $tmp2$$Register);
11667   %}
11668   ins_pipe( pipe_slow );
11669 %}
11670 
11671 // encode char[] to byte[] in ISO_8859_1
11672 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11673                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11674                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11675   match(Set result (EncodeISOArray src (Binary dst len)));
11676   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11677 
11678   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11679   ins_encode %{
11680     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11681                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11682                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11683   %}
11684   ins_pipe( pipe_slow );
11685 %}
11686 
11687 //----------Overflow Math Instructions-----------------------------------------
11688 
11689 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11690 %{
11691   match(Set cr (OverflowAddI op1 op2));
11692   effect(DEF cr, USE_KILL op1, USE op2);
11693 
11694   format %{ "addl    $op1, $op2\t# overflow check int" %}
11695 
11696   ins_encode %{
11697     __ addl($op1$$Register, $op2$$Register);
11698   %}
11699   ins_pipe(ialu_reg_reg);
11700 %}
11701 
11702 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11703 %{
11704   match(Set cr (OverflowAddI op1 op2));
11705   effect(DEF cr, USE_KILL op1, USE op2);
11706 
11707   format %{ "addl    $op1, $op2\t# overflow check int" %}
11708 
11709   ins_encode %{
11710     __ addl($op1$$Register, $op2$$constant);
11711   %}
11712   ins_pipe(ialu_reg_reg);
11713 %}
11714 
11715 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11716 %{
11717   match(Set cr (OverflowAddL op1 op2));
11718   effect(DEF cr, USE_KILL op1, USE op2);
11719 
11720   format %{ "addq    $op1, $op2\t# overflow check long" %}
11721   ins_encode %{
11722     __ addq($op1$$Register, $op2$$Register);
11723   %}
11724   ins_pipe(ialu_reg_reg);
11725 %}
11726 
11727 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11728 %{
11729   match(Set cr (OverflowAddL op1 op2));
11730   effect(DEF cr, USE_KILL op1, USE op2);
11731 
11732   format %{ "addq    $op1, $op2\t# overflow check long" %}
11733   ins_encode %{
11734     __ addq($op1$$Register, $op2$$constant);
11735   %}
11736   ins_pipe(ialu_reg_reg);
11737 %}
11738 
11739 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11740 %{
11741   match(Set cr (OverflowSubI op1 op2));
11742 
11743   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11744   ins_encode %{
11745     __ cmpl($op1$$Register, $op2$$Register);
11746   %}
11747   ins_pipe(ialu_reg_reg);
11748 %}
11749 
11750 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11751 %{
11752   match(Set cr (OverflowSubI op1 op2));
11753 
11754   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11755   ins_encode %{
11756     __ cmpl($op1$$Register, $op2$$constant);
11757   %}
11758   ins_pipe(ialu_reg_reg);
11759 %}
11760 
11761 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11762 %{
11763   match(Set cr (OverflowSubL op1 op2));
11764 
11765   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11766   ins_encode %{
11767     __ cmpq($op1$$Register, $op2$$Register);
11768   %}
11769   ins_pipe(ialu_reg_reg);
11770 %}
11771 
11772 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11773 %{
11774   match(Set cr (OverflowSubL op1 op2));
11775 
11776   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11777   ins_encode %{
11778     __ cmpq($op1$$Register, $op2$$constant);
11779   %}
11780   ins_pipe(ialu_reg_reg);
11781 %}
11782 
11783 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11784 %{
11785   match(Set cr (OverflowSubI zero op2));
11786   effect(DEF cr, USE_KILL op2);
11787 
11788   format %{ "negl    $op2\t# overflow check int" %}
11789   ins_encode %{
11790     __ negl($op2$$Register);
11791   %}
11792   ins_pipe(ialu_reg_reg);
11793 %}
11794 
11795 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11796 %{
11797   match(Set cr (OverflowSubL zero op2));
11798   effect(DEF cr, USE_KILL op2);
11799 
11800   format %{ "negq    $op2\t# overflow check long" %}
11801   ins_encode %{
11802     __ negq($op2$$Register);
11803   %}
11804   ins_pipe(ialu_reg_reg);
11805 %}
11806 
11807 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11808 %{
11809   match(Set cr (OverflowMulI op1 op2));
11810   effect(DEF cr, USE_KILL op1, USE op2);
11811 
11812   format %{ "imull    $op1, $op2\t# overflow check int" %}
11813   ins_encode %{
11814     __ imull($op1$$Register, $op2$$Register);
11815   %}
11816   ins_pipe(ialu_reg_reg_alu0);
11817 %}
11818 
11819 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11820 %{
11821   match(Set cr (OverflowMulI op1 op2));
11822   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11823 
11824   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11825   ins_encode %{
11826     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11827   %}
11828   ins_pipe(ialu_reg_reg_alu0);
11829 %}
11830 
11831 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11832 %{
11833   match(Set cr (OverflowMulL op1 op2));
11834   effect(DEF cr, USE_KILL op1, USE op2);
11835 
11836   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11837   ins_encode %{
11838     __ imulq($op1$$Register, $op2$$Register);
11839   %}
11840   ins_pipe(ialu_reg_reg_alu0);
11841 %}
11842 
11843 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11844 %{
11845   match(Set cr (OverflowMulL op1 op2));
11846   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11847 
11848   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11849   ins_encode %{
11850     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11851   %}
11852   ins_pipe(ialu_reg_reg_alu0);
11853 %}
11854 
11855 
11856 //----------Control Flow Instructions------------------------------------------
11857 // Signed compare Instructions
11858 
11859 // XXX more variants!!
11860 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11861 %{
11862   match(Set cr (CmpI op1 op2));
11863   effect(DEF cr, USE op1, USE op2);
11864 
11865   format %{ "cmpl    $op1, $op2" %}
11866   opcode(0x3B);  /* Opcode 3B /r */
11867   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11868   ins_pipe(ialu_cr_reg_reg);
11869 %}
11870 
11871 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11872 %{
11873   match(Set cr (CmpI op1 op2));
11874 
11875   format %{ "cmpl    $op1, $op2" %}
11876   opcode(0x81, 0x07); /* Opcode 81 /7 */
11877   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11878   ins_pipe(ialu_cr_reg_imm);
11879 %}
11880 
11881 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11882 %{
11883   match(Set cr (CmpI op1 (LoadI op2)));
11884 
11885   ins_cost(500); // XXX
11886   format %{ "cmpl    $op1, $op2" %}
11887   opcode(0x3B); /* Opcode 3B /r */
11888   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11889   ins_pipe(ialu_cr_reg_mem);
11890 %}
11891 
11892 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11893 %{
11894   match(Set cr (CmpI src zero));
11895 
11896   format %{ "testl   $src, $src" %}
11897   opcode(0x85);
11898   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11899   ins_pipe(ialu_cr_reg_imm);
11900 %}
11901 
11902 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11903 %{
11904   match(Set cr (CmpI (AndI src con) zero));
11905 
11906   format %{ "testl   $src, $con" %}
11907   opcode(0xF7, 0x00);
11908   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11909   ins_pipe(ialu_cr_reg_imm);
11910 %}
11911 
11912 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11913 %{
11914   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11915 
11916   format %{ "testl   $src, $mem" %}
11917   opcode(0x85);
11918   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11919   ins_pipe(ialu_cr_reg_mem);
11920 %}
11921 
11922 // Unsigned compare Instructions; really, same as signed except they
11923 // produce an rFlagsRegU instead of rFlagsReg.
11924 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11925 %{
11926   match(Set cr (CmpU op1 op2));
11927 
11928   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11929   opcode(0x3B); /* Opcode 3B /r */
11930   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11931   ins_pipe(ialu_cr_reg_reg);
11932 %}
11933 
11934 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11935 %{
11936   match(Set cr (CmpU op1 op2));
11937 
11938   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11939   opcode(0x81,0x07); /* Opcode 81 /7 */
11940   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11941   ins_pipe(ialu_cr_reg_imm);
11942 %}
11943 
11944 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11945 %{
11946   match(Set cr (CmpU op1 (LoadI op2)));
11947 
11948   ins_cost(500); // XXX
11949   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11950   opcode(0x3B); /* Opcode 3B /r */
11951   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11952   ins_pipe(ialu_cr_reg_mem);
11953 %}
11954 
11955 // // // Cisc-spilled version of cmpU_rReg
11956 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11957 // //%{
11958 // //  match(Set cr (CmpU (LoadI op1) op2));
11959 // //
11960 // //  format %{ "CMPu   $op1,$op2" %}
11961 // //  ins_cost(500);
11962 // //  opcode(0x39);  /* Opcode 39 /r */
11963 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11964 // //%}
11965 
11966 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11967 %{
11968   match(Set cr (CmpU src zero));
11969 
11970   format %{ "testl  $src, $src\t# unsigned" %}
11971   opcode(0x85);
11972   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11973   ins_pipe(ialu_cr_reg_imm);
11974 %}
11975 
11976 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11977 %{
11978   match(Set cr (CmpP op1 op2));
11979 
11980   format %{ "cmpq    $op1, $op2\t# ptr" %}
11981   opcode(0x3B); /* Opcode 3B /r */
11982   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11983   ins_pipe(ialu_cr_reg_reg);
11984 %}
11985 
11986 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11987 %{
11988   match(Set cr (CmpP op1 (LoadP op2)));
11989 
11990   ins_cost(500); // XXX
11991   format %{ "cmpq    $op1, $op2\t# ptr" %}
11992   opcode(0x3B); /* Opcode 3B /r */
11993   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11994   ins_pipe(ialu_cr_reg_mem);
11995 %}
11996 
11997 // // // Cisc-spilled version of cmpP_rReg
11998 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11999 // //%{
12000 // //  match(Set cr (CmpP (LoadP op1) op2));
12001 // //
12002 // //  format %{ "CMPu   $op1,$op2" %}
12003 // //  ins_cost(500);
12004 // //  opcode(0x39);  /* Opcode 39 /r */
12005 // //  ins_encode( OpcP, reg_mem( op1, op2) );
12006 // //%}
12007 
12008 // XXX this is generalized by compP_rReg_mem???
12009 // Compare raw pointer (used in out-of-heap check).
12010 // Only works because non-oop pointers must be raw pointers
12011 // and raw pointers have no anti-dependencies.
12012 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
12013 %{
12014   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
12015   match(Set cr (CmpP op1 (LoadP op2)));
12016 
12017   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
12018   opcode(0x3B); /* Opcode 3B /r */
12019   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12020   ins_pipe(ialu_cr_reg_mem);
12021 %}
12022 
12023 // This will generate a signed flags result. This should be OK since
12024 // any compare to a zero should be eq/neq.
12025 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
12026 %{
12027   match(Set cr (CmpP src zero));
12028 
12029   format %{ "testq   $src, $src\t# ptr" %}
12030   opcode(0x85);
12031   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12032   ins_pipe(ialu_cr_reg_imm);
12033 %}
12034 
12035 // This will generate a signed flags result. This should be OK since
12036 // any compare to a zero should be eq/neq.
12037 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
12038 %{
12039   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
12040   match(Set cr (CmpP (LoadP op) zero));
12041 
12042   ins_cost(500); // XXX
12043   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
12044   opcode(0xF7); /* Opcode F7 /0 */
12045   ins_encode(REX_mem_wide(op),
12046              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
12047   ins_pipe(ialu_cr_reg_imm);
12048 %}
12049 
12050 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
12051 %{
12052   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
12053   match(Set cr (CmpP (LoadP mem) zero));
12054 
12055   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
12056   ins_encode %{
12057     __ cmpq(r12, $mem$$Address);
12058   %}
12059   ins_pipe(ialu_cr_reg_mem);
12060 %}
12061 
12062 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
12063 %{
12064   match(Set cr (CmpN op1 op2));
12065 
12066   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12067   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
12068   ins_pipe(ialu_cr_reg_reg);
12069 %}
12070 
12071 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
12072 %{
12073   match(Set cr (CmpN src (LoadN mem)));
12074 
12075   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
12076   ins_encode %{
12077     __ cmpl($src$$Register, $mem$$Address);
12078   %}
12079   ins_pipe(ialu_cr_reg_mem);
12080 %}
12081 
12082 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
12083   match(Set cr (CmpN op1 op2));
12084 
12085   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
12086   ins_encode %{
12087     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
12088   %}
12089   ins_pipe(ialu_cr_reg_imm);
12090 %}
12091 
12092 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12093 %{
12094   match(Set cr (CmpN src (LoadN mem)));
12095 
12096   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12097   ins_encode %{
12098     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12099   %}
12100   ins_pipe(ialu_cr_reg_mem);
12101 %}
12102 
12103 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12104   match(Set cr (CmpN op1 op2));
12105 
12106   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12107   ins_encode %{
12108     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12109   %}
12110   ins_pipe(ialu_cr_reg_imm);
12111 %}
12112 
12113 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12114 %{
12115   match(Set cr (CmpN src (LoadNKlass mem)));
12116 
12117   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12118   ins_encode %{
12119     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12120   %}
12121   ins_pipe(ialu_cr_reg_mem);
12122 %}
12123 
12124 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12125   match(Set cr (CmpN src zero));
12126 
12127   format %{ "testl   $src, $src\t# compressed ptr" %}
12128   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12129   ins_pipe(ialu_cr_reg_imm);
12130 %}
12131 
12132 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12133 %{
12134   predicate(Universe::narrow_oop_base() != NULL);
12135   match(Set cr (CmpN (LoadN mem) zero));
12136 
12137   ins_cost(500); // XXX
12138   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12139   ins_encode %{
12140     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12141   %}
12142   ins_pipe(ialu_cr_reg_mem);
12143 %}
12144 
12145 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12146 %{
12147   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
12148   match(Set cr (CmpN (LoadN mem) zero));
12149 
12150   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12151   ins_encode %{
12152     __ cmpl(r12, $mem$$Address);
12153   %}
12154   ins_pipe(ialu_cr_reg_mem);
12155 %}
12156 
12157 // Yanked all unsigned pointer compare operations.
12158 // Pointer compares are done with CmpP which is already unsigned.
12159 
12160 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12161 %{
12162   match(Set cr (CmpL op1 op2));
12163 
12164   format %{ "cmpq    $op1, $op2" %}
12165   opcode(0x3B);  /* Opcode 3B /r */
12166   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12167   ins_pipe(ialu_cr_reg_reg);
12168 %}
12169 
12170 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12171 %{
12172   match(Set cr (CmpL op1 op2));
12173 
12174   format %{ "cmpq    $op1, $op2" %}
12175   opcode(0x81, 0x07); /* Opcode 81 /7 */
12176   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12177   ins_pipe(ialu_cr_reg_imm);
12178 %}
12179 
12180 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12181 %{
12182   match(Set cr (CmpL op1 (LoadL op2)));
12183 
12184   format %{ "cmpq    $op1, $op2" %}
12185   opcode(0x3B); /* Opcode 3B /r */
12186   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12187   ins_pipe(ialu_cr_reg_mem);
12188 %}
12189 
12190 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12191 %{
12192   match(Set cr (CmpL src zero));
12193 
12194   format %{ "testq   $src, $src" %}
12195   opcode(0x85);
12196   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12197   ins_pipe(ialu_cr_reg_imm);
12198 %}
12199 
12200 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12201 %{
12202   match(Set cr (CmpL (AndL src con) zero));
12203 
12204   format %{ "testq   $src, $con\t# long" %}
12205   opcode(0xF7, 0x00);
12206   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
12207   ins_pipe(ialu_cr_reg_imm);
12208 %}
12209 
12210 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12211 %{
12212   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12213 
12214   format %{ "testq   $src, $mem" %}
12215   opcode(0x85);
12216   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12217   ins_pipe(ialu_cr_reg_mem);
12218 %}
12219 
12220 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12221 %{
12222   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12223 
12224   format %{ "testq   $src, $mem" %}
12225   opcode(0x85);
12226   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12227   ins_pipe(ialu_cr_reg_mem);
12228 %}
12229 
12230 // Manifest a CmpL result in an integer register.  Very painful.
12231 // This is the test to avoid.
12232 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12233 %{
12234   match(Set dst (CmpL3 src1 src2));
12235   effect(KILL flags);
12236 
12237   ins_cost(275); // XXX
12238   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12239             "movl    $dst, -1\n\t"
12240             "jl,s    done\n\t"
12241             "setne   $dst\n\t"
12242             "movzbl  $dst, $dst\n\t"
12243     "done:" %}
12244   ins_encode(cmpl3_flag(src1, src2, dst));
12245   ins_pipe(pipe_slow);
12246 %}
12247 
12248 // Unsigned long compare Instructions; really, same as signed long except they
12249 // produce an rFlagsRegU instead of rFlagsReg.
12250 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12251 %{
12252   match(Set cr (CmpUL op1 op2));
12253 
12254   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12255   opcode(0x3B);  /* Opcode 3B /r */
12256   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12257   ins_pipe(ialu_cr_reg_reg);
12258 %}
12259 
12260 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12261 %{
12262   match(Set cr (CmpUL op1 op2));
12263 
12264   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12265   opcode(0x81, 0x07); /* Opcode 81 /7 */
12266   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12267   ins_pipe(ialu_cr_reg_imm);
12268 %}
12269 
12270 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12271 %{
12272   match(Set cr (CmpUL op1 (LoadL op2)));
12273 
12274   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12275   opcode(0x3B); /* Opcode 3B /r */
12276   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12277   ins_pipe(ialu_cr_reg_mem);
12278 %}
12279 
12280 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12281 %{
12282   match(Set cr (CmpUL src zero));
12283 
12284   format %{ "testq   $src, $src\t# unsigned" %}
12285   opcode(0x85);
12286   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12287   ins_pipe(ialu_cr_reg_imm);
12288 %}
12289 
12290 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12291 %{
12292   match(Set cr (CmpI (LoadB mem) imm));
12293 
12294   ins_cost(125);
12295   format %{ "cmpb    $mem, $imm" %}
12296   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12297   ins_pipe(ialu_cr_reg_mem);
12298 %}
12299 
12300 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
12301 %{
12302   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12303 
12304   ins_cost(125);
12305   format %{ "testb   $mem, $imm\t# ubyte" %}
12306   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12307   ins_pipe(ialu_cr_reg_mem);
12308 %}
12309 
12310 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12311 %{
12312   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12313 
12314   ins_cost(125);
12315   format %{ "testb   $mem, $imm\t# byte" %}
12316   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12317   ins_pipe(ialu_cr_reg_mem);
12318 %}
12319 
12320 //----------Max and Min--------------------------------------------------------
12321 // Min Instructions
12322 
12323 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12324 %{
12325   effect(USE_DEF dst, USE src, USE cr);
12326 
12327   format %{ "cmovlgt $dst, $src\t# min" %}
12328   opcode(0x0F, 0x4F);
12329   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12330   ins_pipe(pipe_cmov_reg);
12331 %}
12332 
12333 
12334 instruct minI_rReg(rRegI dst, rRegI src)
12335 %{
12336   match(Set dst (MinI dst src));
12337 
12338   ins_cost(200);
12339   expand %{
12340     rFlagsReg cr;
12341     compI_rReg(cr, dst, src);
12342     cmovI_reg_g(dst, src, cr);
12343   %}
12344 %}
12345 
12346 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12347 %{
12348   effect(USE_DEF dst, USE src, USE cr);
12349 
12350   format %{ "cmovllt $dst, $src\t# max" %}
12351   opcode(0x0F, 0x4C);
12352   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12353   ins_pipe(pipe_cmov_reg);
12354 %}
12355 
12356 
12357 instruct maxI_rReg(rRegI dst, rRegI src)
12358 %{
12359   match(Set dst (MaxI dst src));
12360 
12361   ins_cost(200);
12362   expand %{
12363     rFlagsReg cr;
12364     compI_rReg(cr, dst, src);
12365     cmovI_reg_l(dst, src, cr);
12366   %}
12367 %}
12368 
12369 // ============================================================================
12370 // Branch Instructions
12371 
12372 // Jump Direct - Label defines a relative address from JMP+1
12373 instruct jmpDir(label labl)
12374 %{
12375   match(Goto);
12376   effect(USE labl);
12377 
12378   ins_cost(300);
12379   format %{ "jmp     $labl" %}
12380   size(5);
12381   ins_encode %{
12382     Label* L = $labl$$label;
12383     __ jmp(*L, false); // Always long jump
12384   %}
12385   ins_pipe(pipe_jmp);
12386 %}
12387 
12388 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12389 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12390 %{
12391   match(If cop cr);
12392   effect(USE labl);
12393 
12394   ins_cost(300);
12395   format %{ "j$cop     $labl" %}
12396   size(6);
12397   ins_encode %{
12398     Label* L = $labl$$label;
12399     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12400   %}
12401   ins_pipe(pipe_jcc);
12402 %}
12403 
12404 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12405 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12406 %{
12407   predicate(!n->has_vector_mask_set());
12408   match(CountedLoopEnd cop cr);
12409   effect(USE labl);
12410 
12411   ins_cost(300);
12412   format %{ "j$cop     $labl\t# loop end" %}
12413   size(6);
12414   ins_encode %{
12415     Label* L = $labl$$label;
12416     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12417   %}
12418   ins_pipe(pipe_jcc);
12419 %}
12420 
12421 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12422 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12423   predicate(!n->has_vector_mask_set());
12424   match(CountedLoopEnd cop cmp);
12425   effect(USE labl);
12426 
12427   ins_cost(300);
12428   format %{ "j$cop,u   $labl\t# loop end" %}
12429   size(6);
12430   ins_encode %{
12431     Label* L = $labl$$label;
12432     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12433   %}
12434   ins_pipe(pipe_jcc);
12435 %}
12436 
12437 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12438   predicate(!n->has_vector_mask_set());
12439   match(CountedLoopEnd cop cmp);
12440   effect(USE labl);
12441 
12442   ins_cost(200);
12443   format %{ "j$cop,u   $labl\t# loop end" %}
12444   size(6);
12445   ins_encode %{
12446     Label* L = $labl$$label;
12447     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12448   %}
12449   ins_pipe(pipe_jcc);
12450 %}
12451 
12452 // mask version
12453 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12454 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12455 %{
12456   predicate(n->has_vector_mask_set());
12457   match(CountedLoopEnd cop cr);
12458   effect(USE labl);
12459 
12460   ins_cost(400);
12461   format %{ "j$cop     $labl\t# loop end\n\t"
12462             "restorevectmask \t# vector mask restore for loops" %}
12463   size(10);
12464   ins_encode %{
12465     Label* L = $labl$$label;
12466     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12467     __ restorevectmask();
12468   %}
12469   ins_pipe(pipe_jcc);
12470 %}
12471 
12472 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12473 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12474   predicate(n->has_vector_mask_set());
12475   match(CountedLoopEnd cop cmp);
12476   effect(USE labl);
12477 
12478   ins_cost(400);
12479   format %{ "j$cop,u   $labl\t# loop end\n\t"
12480             "restorevectmask \t# vector mask restore for loops" %}
12481   size(10);
12482   ins_encode %{
12483     Label* L = $labl$$label;
12484     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12485     __ restorevectmask();
12486   %}
12487   ins_pipe(pipe_jcc);
12488 %}
12489 
12490 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12491   predicate(n->has_vector_mask_set());
12492   match(CountedLoopEnd cop cmp);
12493   effect(USE labl);
12494 
12495   ins_cost(300);
12496   format %{ "j$cop,u   $labl\t# loop end\n\t"
12497             "restorevectmask \t# vector mask restore for loops" %}
12498   size(10);
12499   ins_encode %{
12500     Label* L = $labl$$label;
12501     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12502     __ restorevectmask();
12503   %}
12504   ins_pipe(pipe_jcc);
12505 %}
12506 
12507 // Jump Direct Conditional - using unsigned comparison
12508 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12509   match(If cop cmp);
12510   effect(USE labl);
12511 
12512   ins_cost(300);
12513   format %{ "j$cop,u  $labl" %}
12514   size(6);
12515   ins_encode %{
12516     Label* L = $labl$$label;
12517     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12518   %}
12519   ins_pipe(pipe_jcc);
12520 %}
12521 
12522 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12523   match(If cop cmp);
12524   effect(USE labl);
12525 
12526   ins_cost(200);
12527   format %{ "j$cop,u  $labl" %}
12528   size(6);
12529   ins_encode %{
12530     Label* L = $labl$$label;
12531     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12532   %}
12533   ins_pipe(pipe_jcc);
12534 %}
12535 
12536 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12537   match(If cop cmp);
12538   effect(USE labl);
12539 
12540   ins_cost(200);
12541   format %{ $$template
12542     if ($cop$$cmpcode == Assembler::notEqual) {
12543       $$emit$$"jp,u   $labl\n\t"
12544       $$emit$$"j$cop,u   $labl"
12545     } else {
12546       $$emit$$"jp,u   done\n\t"
12547       $$emit$$"j$cop,u   $labl\n\t"
12548       $$emit$$"done:"
12549     }
12550   %}
12551   ins_encode %{
12552     Label* l = $labl$$label;
12553     if ($cop$$cmpcode == Assembler::notEqual) {
12554       __ jcc(Assembler::parity, *l, false);
12555       __ jcc(Assembler::notEqual, *l, false);
12556     } else if ($cop$$cmpcode == Assembler::equal) {
12557       Label done;
12558       __ jccb(Assembler::parity, done);
12559       __ jcc(Assembler::equal, *l, false);
12560       __ bind(done);
12561     } else {
12562        ShouldNotReachHere();
12563     }
12564   %}
12565   ins_pipe(pipe_jcc);
12566 %}
12567 
12568 // ============================================================================
12569 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12570 // superklass array for an instance of the superklass.  Set a hidden
12571 // internal cache on a hit (cache is checked with exposed code in
12572 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12573 // encoding ALSO sets flags.
12574 
12575 instruct partialSubtypeCheck(rdi_RegP result,
12576                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12577                              rFlagsReg cr)
12578 %{
12579   match(Set result (PartialSubtypeCheck sub super));
12580   effect(KILL rcx, KILL cr);
12581 
12582   ins_cost(1100);  // slightly larger than the next version
12583   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12584             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12585             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12586             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12587             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12588             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12589             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12590     "miss:\t" %}
12591 
12592   opcode(0x1); // Force a XOR of RDI
12593   ins_encode(enc_PartialSubtypeCheck());
12594   ins_pipe(pipe_slow);
12595 %}
12596 
12597 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12598                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12599                                      immP0 zero,
12600                                      rdi_RegP result)
12601 %{
12602   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12603   effect(KILL rcx, KILL result);
12604 
12605   ins_cost(1000);
12606   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12607             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12608             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12609             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12610             "jne,s   miss\t\t# Missed: flags nz\n\t"
12611             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12612     "miss:\t" %}
12613 
12614   opcode(0x0); // No need to XOR RDI
12615   ins_encode(enc_PartialSubtypeCheck());
12616   ins_pipe(pipe_slow);
12617 %}
12618 
12619 // ============================================================================
12620 // Branch Instructions -- short offset versions
12621 //
12622 // These instructions are used to replace jumps of a long offset (the default
12623 // match) with jumps of a shorter offset.  These instructions are all tagged
12624 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12625 // match rules in general matching.  Instead, the ADLC generates a conversion
12626 // method in the MachNode which can be used to do in-place replacement of the
12627 // long variant with the shorter variant.  The compiler will determine if a
12628 // branch can be taken by the is_short_branch_offset() predicate in the machine
12629 // specific code section of the file.
12630 
12631 // Jump Direct - Label defines a relative address from JMP+1
12632 instruct jmpDir_short(label labl) %{
12633   match(Goto);
12634   effect(USE labl);
12635 
12636   ins_cost(300);
12637   format %{ "jmp,s   $labl" %}
12638   size(2);
12639   ins_encode %{
12640     Label* L = $labl$$label;
12641     __ jmpb(*L);
12642   %}
12643   ins_pipe(pipe_jmp);
12644   ins_short_branch(1);
12645 %}
12646 
12647 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12648 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12649   match(If cop cr);
12650   effect(USE labl);
12651 
12652   ins_cost(300);
12653   format %{ "j$cop,s   $labl" %}
12654   size(2);
12655   ins_encode %{
12656     Label* L = $labl$$label;
12657     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12658   %}
12659   ins_pipe(pipe_jcc);
12660   ins_short_branch(1);
12661 %}
12662 
12663 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12664 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12665   match(CountedLoopEnd cop cr);
12666   effect(USE labl);
12667 
12668   ins_cost(300);
12669   format %{ "j$cop,s   $labl\t# loop end" %}
12670   size(2);
12671   ins_encode %{
12672     Label* L = $labl$$label;
12673     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12674   %}
12675   ins_pipe(pipe_jcc);
12676   ins_short_branch(1);
12677 %}
12678 
12679 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12680 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12681   match(CountedLoopEnd cop cmp);
12682   effect(USE labl);
12683 
12684   ins_cost(300);
12685   format %{ "j$cop,us  $labl\t# loop end" %}
12686   size(2);
12687   ins_encode %{
12688     Label* L = $labl$$label;
12689     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12690   %}
12691   ins_pipe(pipe_jcc);
12692   ins_short_branch(1);
12693 %}
12694 
12695 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12696   match(CountedLoopEnd cop cmp);
12697   effect(USE labl);
12698 
12699   ins_cost(300);
12700   format %{ "j$cop,us  $labl\t# loop end" %}
12701   size(2);
12702   ins_encode %{
12703     Label* L = $labl$$label;
12704     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12705   %}
12706   ins_pipe(pipe_jcc);
12707   ins_short_branch(1);
12708 %}
12709 
12710 // Jump Direct Conditional - using unsigned comparison
12711 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12712   match(If cop cmp);
12713   effect(USE labl);
12714 
12715   ins_cost(300);
12716   format %{ "j$cop,us  $labl" %}
12717   size(2);
12718   ins_encode %{
12719     Label* L = $labl$$label;
12720     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12721   %}
12722   ins_pipe(pipe_jcc);
12723   ins_short_branch(1);
12724 %}
12725 
12726 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12727   match(If cop cmp);
12728   effect(USE labl);
12729 
12730   ins_cost(300);
12731   format %{ "j$cop,us  $labl" %}
12732   size(2);
12733   ins_encode %{
12734     Label* L = $labl$$label;
12735     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12736   %}
12737   ins_pipe(pipe_jcc);
12738   ins_short_branch(1);
12739 %}
12740 
12741 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12742   match(If cop cmp);
12743   effect(USE labl);
12744 
12745   ins_cost(300);
12746   format %{ $$template
12747     if ($cop$$cmpcode == Assembler::notEqual) {
12748       $$emit$$"jp,u,s   $labl\n\t"
12749       $$emit$$"j$cop,u,s   $labl"
12750     } else {
12751       $$emit$$"jp,u,s   done\n\t"
12752       $$emit$$"j$cop,u,s  $labl\n\t"
12753       $$emit$$"done:"
12754     }
12755   %}
12756   size(4);
12757   ins_encode %{
12758     Label* l = $labl$$label;
12759     if ($cop$$cmpcode == Assembler::notEqual) {
12760       __ jccb(Assembler::parity, *l);
12761       __ jccb(Assembler::notEqual, *l);
12762     } else if ($cop$$cmpcode == Assembler::equal) {
12763       Label done;
12764       __ jccb(Assembler::parity, done);
12765       __ jccb(Assembler::equal, *l);
12766       __ bind(done);
12767     } else {
12768        ShouldNotReachHere();
12769     }
12770   %}
12771   ins_pipe(pipe_jcc);
12772   ins_short_branch(1);
12773 %}
12774 
12775 // ============================================================================
12776 // inlined locking and unlocking
12777 
12778 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12779   predicate(Compile::current()->use_rtm());
12780   match(Set cr (FastLock object box));
12781   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12782   ins_cost(300);
12783   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12784   ins_encode %{
12785     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12786                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12787                  _counters, _rtm_counters, _stack_rtm_counters,
12788                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12789                  true, ra_->C->profile_rtm());
12790   %}
12791   ins_pipe(pipe_slow);
12792 %}
12793 
12794 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12795   predicate(!Compile::current()->use_rtm());
12796   match(Set cr (FastLock object box));
12797   effect(TEMP tmp, TEMP scr, USE_KILL box);
12798   ins_cost(300);
12799   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12800   ins_encode %{
12801     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12802                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12803   %}
12804   ins_pipe(pipe_slow);
12805 %}
12806 
12807 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12808   match(Set cr (FastUnlock object box));
12809   effect(TEMP tmp, USE_KILL box);
12810   ins_cost(300);
12811   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12812   ins_encode %{
12813     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12814   %}
12815   ins_pipe(pipe_slow);
12816 %}
12817 
12818 
12819 // ============================================================================
12820 // Safepoint Instructions
12821 instruct safePoint_poll(rFlagsReg cr)
12822 %{
12823   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12824   match(SafePoint);
12825   effect(KILL cr);
12826 
12827   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12828             "# Safepoint: poll for GC" %}
12829   ins_cost(125);
12830   ins_encode %{
12831     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12832     __ testl(rax, addr);
12833   %}
12834   ins_pipe(ialu_reg_mem);
12835 %}
12836 
12837 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12838 %{
12839   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12840   match(SafePoint poll);
12841   effect(KILL cr, USE poll);
12842 
12843   format %{ "testl  rax, [$poll]\t"
12844             "# Safepoint: poll for GC" %}
12845   ins_cost(125);
12846   ins_encode %{
12847     __ relocate(relocInfo::poll_type);
12848     __ testl(rax, Address($poll$$Register, 0));
12849   %}
12850   ins_pipe(ialu_reg_mem);
12851 %}
12852 
12853 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12854 %{
12855   predicate(SafepointMechanism::uses_thread_local_poll());
12856   match(SafePoint poll);
12857   effect(KILL cr, USE poll);
12858 
12859   format %{ "testl  rax, [$poll]\t"
12860             "# Safepoint: poll for GC" %}
12861   ins_cost(125);
12862   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12863   ins_encode %{
12864     __ relocate(relocInfo::poll_type);
12865     address pre_pc = __ pc();
12866     __ testl(rax, Address($poll$$Register, 0));
12867     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12868   %}
12869   ins_pipe(ialu_reg_mem);
12870 %}
12871 
12872 // ============================================================================
12873 // Procedure Call/Return Instructions
12874 // Call Java Static Instruction
12875 // Note: If this code changes, the corresponding ret_addr_offset() and
12876 //       compute_padding() functions will have to be adjusted.
12877 instruct CallStaticJavaDirect(method meth) %{
12878   match(CallStaticJava);
12879   effect(USE meth);
12880 
12881   ins_cost(300);
12882   format %{ "call,static " %}
12883   opcode(0xE8); /* E8 cd */
12884   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12885   ins_pipe(pipe_slow);
12886   ins_alignment(4);
12887 %}
12888 
12889 // Call Java Dynamic Instruction
12890 // Note: If this code changes, the corresponding ret_addr_offset() and
12891 //       compute_padding() functions will have to be adjusted.
12892 instruct CallDynamicJavaDirect(method meth)
12893 %{
12894   match(CallDynamicJava);
12895   effect(USE meth);
12896 
12897   ins_cost(300);
12898   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12899             "call,dynamic " %}
12900   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12901   ins_pipe(pipe_slow);
12902   ins_alignment(4);
12903 %}
12904 
12905 // Call Runtime Instruction
12906 instruct CallRuntimeDirect(method meth)
12907 %{
12908   match(CallRuntime);
12909   effect(USE meth);
12910 
12911   ins_cost(300);
12912   format %{ "call,runtime " %}
12913   ins_encode(clear_avx, Java_To_Runtime(meth));
12914   ins_pipe(pipe_slow);
12915 %}
12916 
12917 // Call runtime without safepoint
12918 instruct CallLeafDirect(method meth)
12919 %{
12920   match(CallLeaf);
12921   effect(USE meth);
12922 
12923   ins_cost(300);
12924   format %{ "call_leaf,runtime " %}
12925   ins_encode(clear_avx, Java_To_Runtime(meth));
12926   ins_pipe(pipe_slow);
12927 %}
12928 
12929 // Call runtime without safepoint
12930 // entry point is null, target holds the address to call
12931 instruct CallLeafNoFPInDirect(rRegP target)
12932 %{
12933   predicate(n->as_Call()->entry_point() == NULL);
12934   match(CallLeafNoFP target);
12935 
12936   ins_cost(300);
12937   format %{ "call_leaf_nofp,runtime indirect " %}
12938   ins_encode %{
12939      __ call($target$$Register);
12940   %}
12941 
12942   ins_pipe(pipe_slow);
12943 %}
12944 
12945 instruct CallLeafNoFPDirect(method meth)
12946 %{
12947   predicate(n->as_Call()->entry_point() != NULL);
12948   match(CallLeafNoFP);
12949   effect(USE meth);
12950 
12951   ins_cost(300);
12952   format %{ "call_leaf_nofp,runtime " %}
12953   ins_encode(clear_avx, Java_To_Runtime(meth));
12954   ins_pipe(pipe_slow);
12955 %}
12956 
12957 // Return Instruction
12958 // Remove the return address & jump to it.
12959 // Notice: We always emit a nop after a ret to make sure there is room
12960 // for safepoint patching
12961 instruct Ret()
12962 %{
12963   match(Return);
12964 
12965   format %{ "ret" %}
12966   opcode(0xC3);
12967   ins_encode(OpcP);
12968   ins_pipe(pipe_jmp);
12969 %}
12970 
12971 // Tail Call; Jump from runtime stub to Java code.
12972 // Also known as an 'interprocedural jump'.
12973 // Target of jump will eventually return to caller.
12974 // TailJump below removes the return address.
12975 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12976 %{
12977   match(TailCall jump_target method_oop);
12978 
12979   ins_cost(300);
12980   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12981   opcode(0xFF, 0x4); /* Opcode FF /4 */
12982   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12983   ins_pipe(pipe_jmp);
12984 %}
12985 
12986 // Tail Jump; remove the return address; jump to target.
12987 // TailCall above leaves the return address around.
12988 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12989 %{
12990   match(TailJump jump_target ex_oop);
12991 
12992   ins_cost(300);
12993   format %{ "popq    rdx\t# pop return address\n\t"
12994             "jmp     $jump_target" %}
12995   opcode(0xFF, 0x4); /* Opcode FF /4 */
12996   ins_encode(Opcode(0x5a), // popq rdx
12997              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12998   ins_pipe(pipe_jmp);
12999 %}
13000 
13001 // Create exception oop: created by stack-crawling runtime code.
13002 // Created exception is now available to this handler, and is setup
13003 // just prior to jumping to this handler.  No code emitted.
13004 instruct CreateException(rax_RegP ex_oop)
13005 %{
13006   match(Set ex_oop (CreateEx));
13007 
13008   size(0);
13009   // use the following format syntax
13010   format %{ "# exception oop is in rax; no code emitted" %}
13011   ins_encode();
13012   ins_pipe(empty);
13013 %}
13014 
13015 // Rethrow exception:
13016 // The exception oop will come in the first argument position.
13017 // Then JUMP (not call) to the rethrow stub code.
13018 instruct RethrowException()
13019 %{
13020   match(Rethrow);
13021 
13022   // use the following format syntax
13023   format %{ "jmp     rethrow_stub" %}
13024   ins_encode(enc_rethrow);
13025   ins_pipe(pipe_jmp);
13026 %}
13027 
13028 // ============================================================================
13029 // This name is KNOWN by the ADLC and cannot be changed.
13030 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
13031 // for this guy.
13032 instruct tlsLoadP(r15_RegP dst) %{
13033   match(Set dst (ThreadLocal));
13034   effect(DEF dst);
13035 
13036   size(0);
13037   format %{ "# TLS is in R15" %}
13038   ins_encode( /*empty encoding*/ );
13039   ins_pipe(ialu_reg_reg);
13040 %}
13041 
13042 
13043 //----------PEEPHOLE RULES-----------------------------------------------------
13044 // These must follow all instruction definitions as they use the names
13045 // defined in the instructions definitions.
13046 //
13047 // peepmatch ( root_instr_name [preceding_instruction]* );
13048 //
13049 // peepconstraint %{
13050 // (instruction_number.operand_name relational_op instruction_number.operand_name
13051 //  [, ...] );
13052 // // instruction numbers are zero-based using left to right order in peepmatch
13053 //
13054 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
13055 // // provide an instruction_number.operand_name for each operand that appears
13056 // // in the replacement instruction's match rule
13057 //
13058 // ---------VM FLAGS---------------------------------------------------------
13059 //
13060 // All peephole optimizations can be turned off using -XX:-OptoPeephole
13061 //
13062 // Each peephole rule is given an identifying number starting with zero and
13063 // increasing by one in the order seen by the parser.  An individual peephole
13064 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
13065 // on the command-line.
13066 //
13067 // ---------CURRENT LIMITATIONS----------------------------------------------
13068 //
13069 // Only match adjacent instructions in same basic block
13070 // Only equality constraints
13071 // Only constraints between operands, not (0.dest_reg == RAX_enc)
13072 // Only one replacement instruction
13073 //
13074 // ---------EXAMPLE----------------------------------------------------------
13075 //
13076 // // pertinent parts of existing instructions in architecture description
13077 // instruct movI(rRegI dst, rRegI src)
13078 // %{
13079 //   match(Set dst (CopyI src));
13080 // %}
13081 //
13082 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
13083 // %{
13084 //   match(Set dst (AddI dst src));
13085 //   effect(KILL cr);
13086 // %}
13087 //
13088 // // Change (inc mov) to lea
13089 // peephole %{
13090 //   // increment preceeded by register-register move
13091 //   peepmatch ( incI_rReg movI );
13092 //   // require that the destination register of the increment
13093 //   // match the destination register of the move
13094 //   peepconstraint ( 0.dst == 1.dst );
13095 //   // construct a replacement instruction that sets
13096 //   // the destination to ( move's source register + one )
13097 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
13098 // %}
13099 //
13100 
13101 // Implementation no longer uses movX instructions since
13102 // machine-independent system no longer uses CopyX nodes.
13103 //
13104 // peephole
13105 // %{
13106 //   peepmatch (incI_rReg movI);
13107 //   peepconstraint (0.dst == 1.dst);
13108 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13109 // %}
13110 
13111 // peephole
13112 // %{
13113 //   peepmatch (decI_rReg movI);
13114 //   peepconstraint (0.dst == 1.dst);
13115 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13116 // %}
13117 
13118 // peephole
13119 // %{
13120 //   peepmatch (addI_rReg_imm movI);
13121 //   peepconstraint (0.dst == 1.dst);
13122 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13123 // %}
13124 
13125 // peephole
13126 // %{
13127 //   peepmatch (incL_rReg movL);
13128 //   peepconstraint (0.dst == 1.dst);
13129 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13130 // %}
13131 
13132 // peephole
13133 // %{
13134 //   peepmatch (decL_rReg movL);
13135 //   peepconstraint (0.dst == 1.dst);
13136 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13137 // %}
13138 
13139 // peephole
13140 // %{
13141 //   peepmatch (addL_rReg_imm movL);
13142 //   peepconstraint (0.dst == 1.dst);
13143 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13144 // %}
13145 
13146 // peephole
13147 // %{
13148 //   peepmatch (addP_rReg_imm movP);
13149 //   peepconstraint (0.dst == 1.dst);
13150 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13151 // %}
13152 
13153 // // Change load of spilled value to only a spill
13154 // instruct storeI(memory mem, rRegI src)
13155 // %{
13156 //   match(Set mem (StoreI mem src));
13157 // %}
13158 //
13159 // instruct loadI(rRegI dst, memory mem)
13160 // %{
13161 //   match(Set dst (LoadI mem));
13162 // %}
13163 //
13164 
13165 peephole
13166 %{
13167   peepmatch (loadI storeI);
13168   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13169   peepreplace (storeI(1.mem 1.mem 1.src));
13170 %}
13171 
13172 peephole
13173 %{
13174   peepmatch (loadL storeL);
13175   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13176   peepreplace (storeL(1.mem 1.mem 1.src));
13177 %}
13178 
13179 //----------SMARTSPILL RULES---------------------------------------------------
13180 // These must follow all instruction definitions as they use the names
13181 // defined in the instructions definitions.