rev 5968 : 8031320: Use Intel RTM instructions for locks
Summary: Use RTM for inflated locks and stack locks.
Reviewed-by: iveresov, twisti, roland, dcubed

   1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Class for all pointer registers (including RSP)
 170 reg_class any_reg(RAX, RAX_H,
 171                   RDX, RDX_H,
 172                   RBP, RBP_H,
 173                   RDI, RDI_H,
 174                   RSI, RSI_H,
 175                   RCX, RCX_H,
 176                   RBX, RBX_H,
 177                   RSP, RSP_H,
 178                   R8,  R8_H,
 179                   R9,  R9_H,
 180                   R10, R10_H,
 181                   R11, R11_H,
 182                   R12, R12_H,
 183                   R13, R13_H,
 184                   R14, R14_H,
 185                   R15, R15_H);
 186 
 187 // Class for all pointer registers except RSP
 188 reg_class ptr_reg(RAX, RAX_H,
 189                   RDX, RDX_H,
 190                   RBP, RBP_H,
 191                   RDI, RDI_H,
 192                   RSI, RSI_H,
 193                   RCX, RCX_H,
 194                   RBX, RBX_H,
 195                   R8,  R8_H,
 196                   R9,  R9_H,
 197                   R10, R10_H,
 198                   R11, R11_H,
 199                   R13, R13_H,
 200                   R14, R14_H);
 201 
 202 // Class for all pointer registers except RAX and RSP
 203 reg_class ptr_no_rax_reg(RDX, RDX_H,
 204                          RBP, RBP_H,
 205                          RDI, RDI_H,
 206                          RSI, RSI_H,
 207                          RCX, RCX_H,
 208                          RBX, RBX_H,
 209                          R8,  R8_H,
 210                          R9,  R9_H,
 211                          R10, R10_H,
 212                          R11, R11_H,
 213                          R13, R13_H,
 214                          R14, R14_H);
 215 
 216 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 217                          RAX, RAX_H,
 218                          RDI, RDI_H,
 219                          RSI, RSI_H,
 220                          RCX, RCX_H,
 221                          RBX, RBX_H,
 222                          R8,  R8_H,
 223                          R9,  R9_H,
 224                          R10, R10_H,
 225                          R11, R11_H,
 226                          R13, R13_H,
 227                          R14, R14_H);
 228 
 229 // Class for all pointer registers except RAX, RBX and RSP
 230 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 231                              RBP, RBP_H,
 232                              RDI, RDI_H,
 233                              RSI, RSI_H,
 234                              RCX, RCX_H,
 235                              R8,  R8_H,
 236                              R9,  R9_H,
 237                              R10, R10_H,
 238                              R11, R11_H,
 239                              R13, R13_H,
 240                              R14, R14_H);
 241 
 242 // Singleton class for RAX pointer register
 243 reg_class ptr_rax_reg(RAX, RAX_H);
 244 
 245 // Singleton class for RBX pointer register
 246 reg_class ptr_rbx_reg(RBX, RBX_H);
 247 
 248 // Singleton class for RSI pointer register
 249 reg_class ptr_rsi_reg(RSI, RSI_H);
 250 
 251 // Singleton class for RDI pointer register
 252 reg_class ptr_rdi_reg(RDI, RDI_H);
 253 
 254 // Singleton class for RBP pointer register
 255 reg_class ptr_rbp_reg(RBP, RBP_H);
 256 
 257 // Singleton class for stack pointer
 258 reg_class ptr_rsp_reg(RSP, RSP_H);
 259 
 260 // Singleton class for TLS pointer
 261 reg_class ptr_r15_reg(R15, R15_H);
 262 
 263 // Class for all long registers (except RSP)
 264 reg_class long_reg(RAX, RAX_H,
 265                    RDX, RDX_H,
 266                    RBP, RBP_H,
 267                    RDI, RDI_H,
 268                    RSI, RSI_H,
 269                    RCX, RCX_H,
 270                    RBX, RBX_H,
 271                    R8,  R8_H,
 272                    R9,  R9_H,
 273                    R10, R10_H,
 274                    R11, R11_H,
 275                    R13, R13_H,
 276                    R14, R14_H);
 277 
 278 // Class for all long registers except RAX, RDX (and RSP)
 279 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 280                               RDI, RDI_H,
 281                               RSI, RSI_H,
 282                               RCX, RCX_H,
 283                               RBX, RBX_H,
 284                               R8,  R8_H,
 285                               R9,  R9_H,
 286                               R10, R10_H,
 287                               R11, R11_H,
 288                               R13, R13_H,
 289                               R14, R14_H);
 290 
 291 // Class for all long registers except RCX (and RSP)
 292 reg_class long_no_rcx_reg(RBP, RBP_H,
 293                           RDI, RDI_H,
 294                           RSI, RSI_H,
 295                           RAX, RAX_H,
 296                           RDX, RDX_H,
 297                           RBX, RBX_H,
 298                           R8,  R8_H,
 299                           R9,  R9_H,
 300                           R10, R10_H,
 301                           R11, R11_H,
 302                           R13, R13_H,
 303                           R14, R14_H);
 304 
 305 // Class for all long registers except RAX (and RSP)
 306 reg_class long_no_rax_reg(RBP, RBP_H,
 307                           RDX, RDX_H,
 308                           RDI, RDI_H,
 309                           RSI, RSI_H,
 310                           RCX, RCX_H,
 311                           RBX, RBX_H,
 312                           R8,  R8_H,
 313                           R9,  R9_H,
 314                           R10, R10_H,
 315                           R11, R11_H,
 316                           R13, R13_H,
 317                           R14, R14_H);
 318 
 319 // Singleton class for RAX long register
 320 reg_class long_rax_reg(RAX, RAX_H);
 321 
 322 // Singleton class for RCX long register
 323 reg_class long_rcx_reg(RCX, RCX_H);
 324 
 325 // Singleton class for RDX long register
 326 reg_class long_rdx_reg(RDX, RDX_H);
 327 
 328 // Class for all int registers (except RSP)
 329 reg_class int_reg(RAX,
 330                   RDX,
 331                   RBP,
 332                   RDI,
 333                   RSI,
 334                   RCX,
 335                   RBX,
 336                   R8,
 337                   R9,
 338                   R10,
 339                   R11,
 340                   R13,
 341                   R14);
 342 
 343 // Class for all int registers except RCX (and RSP)
 344 reg_class int_no_rcx_reg(RAX,
 345                          RDX,
 346                          RBP,
 347                          RDI,
 348                          RSI,
 349                          RBX,
 350                          R8,
 351                          R9,
 352                          R10,
 353                          R11,
 354                          R13,
 355                          R14);
 356 
 357 // Class for all int registers except RAX, RDX (and RSP)
 358 reg_class int_no_rax_rdx_reg(RBP,
 359                              RDI,
 360                              RSI,
 361                              RCX,
 362                              RBX,
 363                              R8,
 364                              R9,
 365                              R10,
 366                              R11,
 367                              R13,
 368                              R14);
 369 
 370 // Singleton class for RAX int register
 371 reg_class int_rax_reg(RAX);
 372 
 373 // Singleton class for RBX int register
 374 reg_class int_rbx_reg(RBX);
 375 
 376 // Singleton class for RCX int register
 377 reg_class int_rcx_reg(RCX);
 378 
 379 // Singleton class for RCX int register
 380 reg_class int_rdx_reg(RDX);
 381 
 382 // Singleton class for RCX int register
 383 reg_class int_rdi_reg(RDI);
 384 
 385 // Singleton class for instruction pointer
 386 // reg_class ip_reg(RIP);
 387 
 388 %}
 389 
 390 //----------SOURCE BLOCK-------------------------------------------------------
 391 // This is a block of C++ code which provides values, functions, and
 392 // definitions necessary in the rest of the architecture description
 393 source %{
 394 #define   RELOC_IMM64    Assembler::imm_operand
 395 #define   RELOC_DISP32   Assembler::disp32_operand
 396 
 397 #define __ _masm.
 398 
 399 static int preserve_SP_size() {
 400   return 3;  // rex.w, op, rm(reg/reg)
 401 }
 402 static int clear_avx_size() {
 403   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 404 }
 405 
 406 // !!!!! Special hack to get all types of calls to specify the byte offset
 407 //       from the start of the call to the point where the return address
 408 //       will point.
 409 int MachCallStaticJavaNode::ret_addr_offset()
 410 {
 411   int offset = 5; // 5 bytes from start of call to where return address points
 412   offset += clear_avx_size();
 413   if (_method_handle_invoke)
 414     offset += preserve_SP_size();
 415   return offset;
 416 }
 417 
 418 int MachCallDynamicJavaNode::ret_addr_offset()
 419 {
 420   int offset = 15; // 15 bytes from start of call to where return address points
 421   offset += clear_avx_size();
 422   return offset;
 423 }
 424 
 425 int MachCallRuntimeNode::ret_addr_offset() {
 426   int offset = 13; // movq r10,#addr; callq (r10)
 427   offset += clear_avx_size();
 428   return offset;
 429 }
 430 
 431 // Indicate if the safepoint node needs the polling page as an input,
 432 // it does if the polling page is more than disp32 away.
 433 bool SafePointNode::needs_polling_address_input()
 434 {
 435   return Assembler::is_polling_page_far();
 436 }
 437 
 438 //
 439 // Compute padding required for nodes which need alignment
 440 //
 441 
 442 // The address of the call instruction needs to be 4-byte aligned to
 443 // ensure that it does not span a cache line so that it can be patched.
 444 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 445 {
 446   current_offset += clear_avx_size(); // skip vzeroupper
 447   current_offset += 1; // skip call opcode byte
 448   return round_to(current_offset, alignment_required()) - current_offset;
 449 }
 450 
 451 // The address of the call instruction needs to be 4-byte aligned to
 452 // ensure that it does not span a cache line so that it can be patched.
 453 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 454 {
 455   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 456   current_offset += clear_avx_size(); // skip vzeroupper
 457   current_offset += 1; // skip call opcode byte
 458   return round_to(current_offset, alignment_required()) - current_offset;
 459 }
 460 
 461 // The address of the call instruction needs to be 4-byte aligned to
 462 // ensure that it does not span a cache line so that it can be patched.
 463 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 464 {
 465   current_offset += clear_avx_size(); // skip vzeroupper
 466   current_offset += 11; // skip movq instruction + call opcode byte
 467   return round_to(current_offset, alignment_required()) - current_offset;
 468 }
 469 
 470 // EMIT_RM()
 471 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 472   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 473   cbuf.insts()->emit_int8(c);
 474 }
 475 
 476 // EMIT_CC()
 477 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 478   unsigned char c = (unsigned char) (f1 | f2);
 479   cbuf.insts()->emit_int8(c);
 480 }
 481 
 482 // EMIT_OPCODE()
 483 void emit_opcode(CodeBuffer &cbuf, int code) {
 484   cbuf.insts()->emit_int8((unsigned char) code);
 485 }
 486 
 487 // EMIT_OPCODE() w/ relocation information
 488 void emit_opcode(CodeBuffer &cbuf,
 489                  int code, relocInfo::relocType reloc, int offset, int format)
 490 {
 491   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 492   emit_opcode(cbuf, code);
 493 }
 494 
 495 // EMIT_D8()
 496 void emit_d8(CodeBuffer &cbuf, int d8) {
 497   cbuf.insts()->emit_int8((unsigned char) d8);
 498 }
 499 
 500 // EMIT_D16()
 501 void emit_d16(CodeBuffer &cbuf, int d16) {
 502   cbuf.insts()->emit_int16(d16);
 503 }
 504 
 505 // EMIT_D32()
 506 void emit_d32(CodeBuffer &cbuf, int d32) {
 507   cbuf.insts()->emit_int32(d32);
 508 }
 509 
 510 // EMIT_D64()
 511 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 512   cbuf.insts()->emit_int64(d64);
 513 }
 514 
 515 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 516 void emit_d32_reloc(CodeBuffer& cbuf,
 517                     int d32,
 518                     relocInfo::relocType reloc,
 519                     int format)
 520 {
 521   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 522   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 523   cbuf.insts()->emit_int32(d32);
 524 }
 525 
 526 // emit 32 bit value and construct relocation entry from RelocationHolder
 527 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 528 #ifdef ASSERT
 529   if (rspec.reloc()->type() == relocInfo::oop_type &&
 530       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 531     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 532     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 533   }
 534 #endif
 535   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 536   cbuf.insts()->emit_int32(d32);
 537 }
 538 
 539 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 540   address next_ip = cbuf.insts_end() + 4;
 541   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 542                  external_word_Relocation::spec(addr),
 543                  RELOC_DISP32);
 544 }
 545 
 546 
 547 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 548 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 549   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 550   cbuf.insts()->emit_int64(d64);
 551 }
 552 
 553 // emit 64 bit value and construct relocation entry from RelocationHolder
 554 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 555 #ifdef ASSERT
 556   if (rspec.reloc()->type() == relocInfo::oop_type &&
 557       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 558     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 559     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 560            "cannot embed scavengable oops in code");
 561   }
 562 #endif
 563   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 564   cbuf.insts()->emit_int64(d64);
 565 }
 566 
 567 // Access stack slot for load or store
 568 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 569 {
 570   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 571   if (-0x80 <= disp && disp < 0x80) {
 572     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 573     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 574     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 575   } else {
 576     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 577     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 578     emit_d32(cbuf, disp);     // Displacement // R/M byte
 579   }
 580 }
 581 
 582    // rRegI ereg, memory mem) %{    // emit_reg_mem
 583 void encode_RegMem(CodeBuffer &cbuf,
 584                    int reg,
 585                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 586 {
 587   assert(disp_reloc == relocInfo::none, "cannot have disp");
 588   int regenc = reg & 7;
 589   int baseenc = base & 7;
 590   int indexenc = index & 7;
 591 
 592   // There is no index & no scale, use form without SIB byte
 593   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 594     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 595     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 596       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 597     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 598       // If 8-bit displacement, mode 0x1
 599       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 600       emit_d8(cbuf, disp);
 601     } else {
 602       // If 32-bit displacement
 603       if (base == -1) { // Special flag for absolute address
 604         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 605         if (disp_reloc != relocInfo::none) {
 606           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 607         } else {
 608           emit_d32(cbuf, disp);
 609         }
 610       } else {
 611         // Normal base + offset
 612         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 613         if (disp_reloc != relocInfo::none) {
 614           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 615         } else {
 616           emit_d32(cbuf, disp);
 617         }
 618       }
 619     }
 620   } else {
 621     // Else, encode with the SIB byte
 622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 624       // If no displacement
 625       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 626       emit_rm(cbuf, scale, indexenc, baseenc);
 627     } else {
 628       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 629         // If 8-bit displacement, mode 0x1
 630         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 631         emit_rm(cbuf, scale, indexenc, baseenc);
 632         emit_d8(cbuf, disp);
 633       } else {
 634         // If 32-bit displacement
 635         if (base == 0x04 ) {
 636           emit_rm(cbuf, 0x2, regenc, 0x4);
 637           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 638         } else {
 639           emit_rm(cbuf, 0x2, regenc, 0x4);
 640           emit_rm(cbuf, scale, indexenc, baseenc); // *
 641         }
 642         if (disp_reloc != relocInfo::none) {
 643           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 644         } else {
 645           emit_d32(cbuf, disp);
 646         }
 647       }
 648     }
 649   }
 650 }
 651 
 652 // This could be in MacroAssembler but it's fairly C2 specific
 653 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 654   Label exit;
 655   __ jccb(Assembler::noParity, exit);
 656   __ pushf();
 657   //
 658   // comiss/ucomiss instructions set ZF,PF,CF flags and
 659   // zero OF,AF,SF for NaN values.
 660   // Fixup flags by zeroing ZF,PF so that compare of NaN
 661   // values returns 'less than' result (CF is set).
 662   // Leave the rest of flags unchanged.
 663   //
 664   //    7 6 5 4 3 2 1 0
 665   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 666   //    0 0 1 0 1 0 1 1   (0x2B)
 667   //
 668   __ andq(Address(rsp, 0), 0xffffff2b);
 669   __ popf();
 670   __ bind(exit);
 671 }
 672 
 673 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 674   Label done;
 675   __ movl(dst, -1);
 676   __ jcc(Assembler::parity, done);
 677   __ jcc(Assembler::below, done);
 678   __ setb(Assembler::notEqual, dst);
 679   __ movzbl(dst, dst);
 680   __ bind(done);
 681 }
 682 
 683 
 684 //=============================================================================
 685 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 686 
 687 int Compile::ConstantTable::calculate_table_base_offset() const {
 688   return 0;  // absolute addressing, no offset
 689 }
 690 
 691 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 692   // Empty encoding
 693 }
 694 
 695 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 696   return 0;
 697 }
 698 
 699 #ifndef PRODUCT
 700 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 701   st->print("# MachConstantBaseNode (empty encoding)");
 702 }
 703 #endif
 704 
 705 
 706 //=============================================================================
 707 #ifndef PRODUCT
 708 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 709   Compile* C = ra_->C;
 710 
 711   int framesize = C->frame_slots() << LogBytesPerInt;
 712   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 713   // Remove wordSize for return addr which is already pushed.
 714   framesize -= wordSize;
 715 
 716   if (C->need_stack_bang(framesize)) {
 717     framesize -= wordSize;
 718     st->print("# stack bang");
 719     st->print("\n\t");
 720     st->print("pushq   rbp\t# Save rbp");
 721     if (framesize) {
 722       st->print("\n\t");
 723       st->print("subq    rsp, #%d\t# Create frame",framesize);
 724     }
 725   } else {
 726     st->print("subq    rsp, #%d\t# Create frame",framesize);
 727     st->print("\n\t");
 728     framesize -= wordSize;
 729     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 730   }
 731 
 732   if (VerifyStackAtCalls) {
 733     st->print("\n\t");
 734     framesize -= wordSize;
 735     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 736 #ifdef ASSERT
 737     st->print("\n\t");
 738     st->print("# stack alignment check");
 739 #endif
 740   }
 741   st->cr();
 742 }
 743 #endif
 744 
 745 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 746   Compile* C = ra_->C;
 747   MacroAssembler _masm(&cbuf);
 748 
 749   int framesize = C->frame_slots() << LogBytesPerInt;
 750 
 751   __ verified_entry(framesize, C->need_stack_bang(framesize), false);
 752 
 753   C->set_frame_complete(cbuf.insts_size());
 754 
 755   if (C->has_mach_constant_base_node()) {
 756     // NOTE: We set the table base offset here because users might be
 757     // emitted before MachConstantBaseNode.
 758     Compile::ConstantTable& constant_table = C->constant_table();
 759     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 760   }
 761 }
 762 
 763 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 764 {
 765   return MachNode::size(ra_); // too many variables; just compute it
 766                               // the hard way
 767 }
 768 
 769 int MachPrologNode::reloc() const
 770 {
 771   return 0; // a large enough number
 772 }
 773 
 774 //=============================================================================
 775 #ifndef PRODUCT
 776 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 777 {
 778   Compile* C = ra_->C;
 779   if (C->max_vector_size() > 16) {
 780     st->print("vzeroupper");
 781     st->cr(); st->print("\t");
 782   }
 783 
 784   int framesize = C->frame_slots() << LogBytesPerInt;
 785   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 786   // Remove word for return adr already pushed
 787   // and RBP
 788   framesize -= 2*wordSize;
 789 
 790   if (framesize) {
 791     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 792     st->print("\t");
 793   }
 794 
 795   st->print_cr("popq   rbp");
 796   if (do_polling() && C->is_method_compilation()) {
 797     st->print("\t");
 798     if (Assembler::is_polling_page_far()) {
 799       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 800                    "testl  rax, [rscratch1]\t"
 801                    "# Safepoint: poll for GC");
 802     } else {
 803       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 804                    "# Safepoint: poll for GC");
 805     }
 806   }
 807 }
 808 #endif
 809 
 810 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 811 {
 812   Compile* C = ra_->C;
 813   if (C->max_vector_size() > 16) {
 814     // Clear upper bits of YMM registers when current compiled code uses
 815     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 816     MacroAssembler _masm(&cbuf);
 817     __ vzeroupper();
 818   }
 819 
 820   int framesize = C->frame_slots() << LogBytesPerInt;
 821   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 822   // Remove word for return adr already pushed
 823   // and RBP
 824   framesize -= 2*wordSize;
 825 
 826   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 827 
 828   if (framesize) {
 829     emit_opcode(cbuf, Assembler::REX_W);
 830     if (framesize < 0x80) {
 831       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 832       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 833       emit_d8(cbuf, framesize);
 834     } else {
 835       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 836       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 837       emit_d32(cbuf, framesize);
 838     }
 839   }
 840 
 841   // popq rbp
 842   emit_opcode(cbuf, 0x58 | RBP_enc);
 843 
 844   if (do_polling() && C->is_method_compilation()) {
 845     MacroAssembler _masm(&cbuf);
 846     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 847     if (Assembler::is_polling_page_far()) {
 848       __ lea(rscratch1, polling_page);
 849       __ relocate(relocInfo::poll_return_type);
 850       __ testl(rax, Address(rscratch1, 0));
 851     } else {
 852       __ testl(rax, polling_page);
 853     }
 854   }
 855 }
 856 
 857 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 858 {
 859   return MachNode::size(ra_); // too many variables; just compute it
 860                               // the hard way
 861 }
 862 
 863 int MachEpilogNode::reloc() const
 864 {
 865   return 2; // a large enough number
 866 }
 867 
 868 const Pipeline* MachEpilogNode::pipeline() const
 869 {
 870   return MachNode::pipeline_class();
 871 }
 872 
 873 int MachEpilogNode::safepoint_offset() const
 874 {
 875   return 0;
 876 }
 877 
 878 //=============================================================================
 879 
 880 enum RC {
 881   rc_bad,
 882   rc_int,
 883   rc_float,
 884   rc_stack
 885 };
 886 
 887 static enum RC rc_class(OptoReg::Name reg)
 888 {
 889   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 890 
 891   if (OptoReg::is_stack(reg)) return rc_stack;
 892 
 893   VMReg r = OptoReg::as_VMReg(reg);
 894 
 895   if (r->is_Register()) return rc_int;
 896 
 897   assert(r->is_XMMRegister(), "must be");
 898   return rc_float;
 899 }
 900 
 901 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 902 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 903                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 904 
 905 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 906                             int stack_offset, int reg, uint ireg, outputStream* st);
 907 
 908 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 909                                       int dst_offset, uint ireg, outputStream* st) {
 910   if (cbuf) {
 911     MacroAssembler _masm(cbuf);
 912     switch (ireg) {
 913     case Op_VecS:
 914       __ movq(Address(rsp, -8), rax);
 915       __ movl(rax, Address(rsp, src_offset));
 916       __ movl(Address(rsp, dst_offset), rax);
 917       __ movq(rax, Address(rsp, -8));
 918       break;
 919     case Op_VecD:
 920       __ pushq(Address(rsp, src_offset));
 921       __ popq (Address(rsp, dst_offset));
 922       break;
 923     case Op_VecX:
 924       __ pushq(Address(rsp, src_offset));
 925       __ popq (Address(rsp, dst_offset));
 926       __ pushq(Address(rsp, src_offset+8));
 927       __ popq (Address(rsp, dst_offset+8));
 928       break;
 929     case Op_VecY:
 930       __ vmovdqu(Address(rsp, -32), xmm0);
 931       __ vmovdqu(xmm0, Address(rsp, src_offset));
 932       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 933       __ vmovdqu(xmm0, Address(rsp, -32));
 934       break;
 935     default:
 936       ShouldNotReachHere();
 937     }
 938 #ifndef PRODUCT
 939   } else {
 940     switch (ireg) {
 941     case Op_VecS:
 942       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 943                 "movl    rax, [rsp + #%d]\n\t"
 944                 "movl    [rsp + #%d], rax\n\t"
 945                 "movq    rax, [rsp - #8]",
 946                 src_offset, dst_offset);
 947       break;
 948     case Op_VecD:
 949       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 950                 "popq    [rsp + #%d]",
 951                 src_offset, dst_offset);
 952       break;
 953      case Op_VecX:
 954       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 955                 "popq    [rsp + #%d]\n\t"
 956                 "pushq   [rsp + #%d]\n\t"
 957                 "popq    [rsp + #%d]",
 958                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 959       break;
 960     case Op_VecY:
 961       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 962                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 963                 "vmovdqu [rsp + #%d], xmm0\n\t"
 964                 "vmovdqu xmm0, [rsp - #32]",
 965                 src_offset, dst_offset);
 966       break;
 967     default:
 968       ShouldNotReachHere();
 969     }
 970 #endif
 971   }
 972 }
 973 
 974 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 975                                        PhaseRegAlloc* ra_,
 976                                        bool do_size,
 977                                        outputStream* st) const {
 978   assert(cbuf != NULL || st  != NULL, "sanity");
 979   // Get registers to move
 980   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 981   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 982   OptoReg::Name dst_second = ra_->get_reg_second(this);
 983   OptoReg::Name dst_first = ra_->get_reg_first(this);
 984 
 985   enum RC src_second_rc = rc_class(src_second);
 986   enum RC src_first_rc = rc_class(src_first);
 987   enum RC dst_second_rc = rc_class(dst_second);
 988   enum RC dst_first_rc = rc_class(dst_first);
 989 
 990   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 991          "must move at least 1 register" );
 992 
 993   if (src_first == dst_first && src_second == dst_second) {
 994     // Self copy, no move
 995     return 0;
 996   }
 997   if (bottom_type()->isa_vect() != NULL) {
 998     uint ireg = ideal_reg();
 999     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1000     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1001     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1002       // mem -> mem
1003       int src_offset = ra_->reg2offset(src_first);
1004       int dst_offset = ra_->reg2offset(dst_first);
1005       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1006     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1007       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1008     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1009       int stack_offset = ra_->reg2offset(dst_first);
1010       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1011     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1012       int stack_offset = ra_->reg2offset(src_first);
1013       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1014     } else {
1015       ShouldNotReachHere();
1016     }
1017     return 0;
1018   }
1019   if (src_first_rc == rc_stack) {
1020     // mem ->
1021     if (dst_first_rc == rc_stack) {
1022       // mem -> mem
1023       assert(src_second != dst_first, "overlap");
1024       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1025           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1026         // 64-bit
1027         int src_offset = ra_->reg2offset(src_first);
1028         int dst_offset = ra_->reg2offset(dst_first);
1029         if (cbuf) {
1030           MacroAssembler _masm(cbuf);
1031           __ pushq(Address(rsp, src_offset));
1032           __ popq (Address(rsp, dst_offset));
1033 #ifndef PRODUCT
1034         } else {
1035           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1036                     "popq    [rsp + #%d]",
1037                      src_offset, dst_offset);
1038 #endif
1039         }
1040       } else {
1041         // 32-bit
1042         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1043         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1044         // No pushl/popl, so:
1045         int src_offset = ra_->reg2offset(src_first);
1046         int dst_offset = ra_->reg2offset(dst_first);
1047         if (cbuf) {
1048           MacroAssembler _masm(cbuf);
1049           __ movq(Address(rsp, -8), rax);
1050           __ movl(rax, Address(rsp, src_offset));
1051           __ movl(Address(rsp, dst_offset), rax);
1052           __ movq(rax, Address(rsp, -8));
1053 #ifndef PRODUCT
1054         } else {
1055           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1056                     "movl    rax, [rsp + #%d]\n\t"
1057                     "movl    [rsp + #%d], rax\n\t"
1058                     "movq    rax, [rsp - #8]",
1059                      src_offset, dst_offset);
1060 #endif
1061         }
1062       }
1063       return 0;
1064     } else if (dst_first_rc == rc_int) {
1065       // mem -> gpr
1066       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1067           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1068         // 64-bit
1069         int offset = ra_->reg2offset(src_first);
1070         if (cbuf) {
1071           MacroAssembler _masm(cbuf);
1072           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1073 #ifndef PRODUCT
1074         } else {
1075           st->print("movq    %s, [rsp + #%d]\t# spill",
1076                      Matcher::regName[dst_first],
1077                      offset);
1078 #endif
1079         }
1080       } else {
1081         // 32-bit
1082         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1083         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1084         int offset = ra_->reg2offset(src_first);
1085         if (cbuf) {
1086           MacroAssembler _masm(cbuf);
1087           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1088 #ifndef PRODUCT
1089         } else {
1090           st->print("movl    %s, [rsp + #%d]\t# spill",
1091                      Matcher::regName[dst_first],
1092                      offset);
1093 #endif
1094         }
1095       }
1096       return 0;
1097     } else if (dst_first_rc == rc_float) {
1098       // mem-> xmm
1099       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1100           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1101         // 64-bit
1102         int offset = ra_->reg2offset(src_first);
1103         if (cbuf) {
1104           MacroAssembler _masm(cbuf);
1105           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1106 #ifndef PRODUCT
1107         } else {
1108           st->print("%s  %s, [rsp + #%d]\t# spill",
1109                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1110                      Matcher::regName[dst_first],
1111                      offset);
1112 #endif
1113         }
1114       } else {
1115         // 32-bit
1116         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1117         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1118         int offset = ra_->reg2offset(src_first);
1119         if (cbuf) {
1120           MacroAssembler _masm(cbuf);
1121           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1122 #ifndef PRODUCT
1123         } else {
1124           st->print("movss   %s, [rsp + #%d]\t# spill",
1125                      Matcher::regName[dst_first],
1126                      offset);
1127 #endif
1128         }
1129       }
1130       return 0;
1131     }
1132   } else if (src_first_rc == rc_int) {
1133     // gpr ->
1134     if (dst_first_rc == rc_stack) {
1135       // gpr -> mem
1136       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1137           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1138         // 64-bit
1139         int offset = ra_->reg2offset(dst_first);
1140         if (cbuf) {
1141           MacroAssembler _masm(cbuf);
1142           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1143 #ifndef PRODUCT
1144         } else {
1145           st->print("movq    [rsp + #%d], %s\t# spill",
1146                      offset,
1147                      Matcher::regName[src_first]);
1148 #endif
1149         }
1150       } else {
1151         // 32-bit
1152         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1153         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1154         int offset = ra_->reg2offset(dst_first);
1155         if (cbuf) {
1156           MacroAssembler _masm(cbuf);
1157           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1158 #ifndef PRODUCT
1159         } else {
1160           st->print("movl    [rsp + #%d], %s\t# spill",
1161                      offset,
1162                      Matcher::regName[src_first]);
1163 #endif
1164         }
1165       }
1166       return 0;
1167     } else if (dst_first_rc == rc_int) {
1168       // gpr -> gpr
1169       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1170           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1171         // 64-bit
1172         if (cbuf) {
1173           MacroAssembler _masm(cbuf);
1174           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1175                   as_Register(Matcher::_regEncode[src_first]));
1176 #ifndef PRODUCT
1177         } else {
1178           st->print("movq    %s, %s\t# spill",
1179                      Matcher::regName[dst_first],
1180                      Matcher::regName[src_first]);
1181 #endif
1182         }
1183         return 0;
1184       } else {
1185         // 32-bit
1186         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1187         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1188         if (cbuf) {
1189           MacroAssembler _masm(cbuf);
1190           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1191                   as_Register(Matcher::_regEncode[src_first]));
1192 #ifndef PRODUCT
1193         } else {
1194           st->print("movl    %s, %s\t# spill",
1195                      Matcher::regName[dst_first],
1196                      Matcher::regName[src_first]);
1197 #endif
1198         }
1199         return 0;
1200       }
1201     } else if (dst_first_rc == rc_float) {
1202       // gpr -> xmm
1203       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1204           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1205         // 64-bit
1206         if (cbuf) {
1207           MacroAssembler _masm(cbuf);
1208           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1209 #ifndef PRODUCT
1210         } else {
1211           st->print("movdq   %s, %s\t# spill",
1212                      Matcher::regName[dst_first],
1213                      Matcher::regName[src_first]);
1214 #endif
1215         }
1216       } else {
1217         // 32-bit
1218         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1219         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1220         if (cbuf) {
1221           MacroAssembler _masm(cbuf);
1222           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1223 #ifndef PRODUCT
1224         } else {
1225           st->print("movdl   %s, %s\t# spill",
1226                      Matcher::regName[dst_first],
1227                      Matcher::regName[src_first]);
1228 #endif
1229         }
1230       }
1231       return 0;
1232     }
1233   } else if (src_first_rc == rc_float) {
1234     // xmm ->
1235     if (dst_first_rc == rc_stack) {
1236       // xmm -> mem
1237       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1238           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1239         // 64-bit
1240         int offset = ra_->reg2offset(dst_first);
1241         if (cbuf) {
1242           MacroAssembler _masm(cbuf);
1243           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1244 #ifndef PRODUCT
1245         } else {
1246           st->print("movsd   [rsp + #%d], %s\t# spill",
1247                      offset,
1248                      Matcher::regName[src_first]);
1249 #endif
1250         }
1251       } else {
1252         // 32-bit
1253         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1254         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1255         int offset = ra_->reg2offset(dst_first);
1256         if (cbuf) {
1257           MacroAssembler _masm(cbuf);
1258           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1259 #ifndef PRODUCT
1260         } else {
1261           st->print("movss   [rsp + #%d], %s\t# spill",
1262                      offset,
1263                      Matcher::regName[src_first]);
1264 #endif
1265         }
1266       }
1267       return 0;
1268     } else if (dst_first_rc == rc_int) {
1269       // xmm -> gpr
1270       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1271           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1272         // 64-bit
1273         if (cbuf) {
1274           MacroAssembler _masm(cbuf);
1275           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1276 #ifndef PRODUCT
1277         } else {
1278           st->print("movdq   %s, %s\t# spill",
1279                      Matcher::regName[dst_first],
1280                      Matcher::regName[src_first]);
1281 #endif
1282         }
1283       } else {
1284         // 32-bit
1285         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1286         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1287         if (cbuf) {
1288           MacroAssembler _masm(cbuf);
1289           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1290 #ifndef PRODUCT
1291         } else {
1292           st->print("movdl   %s, %s\t# spill",
1293                      Matcher::regName[dst_first],
1294                      Matcher::regName[src_first]);
1295 #endif
1296         }
1297       }
1298       return 0;
1299     } else if (dst_first_rc == rc_float) {
1300       // xmm -> xmm
1301       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1302           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1303         // 64-bit
1304         if (cbuf) {
1305           MacroAssembler _masm(cbuf);
1306           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1307 #ifndef PRODUCT
1308         } else {
1309           st->print("%s  %s, %s\t# spill",
1310                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1311                      Matcher::regName[dst_first],
1312                      Matcher::regName[src_first]);
1313 #endif
1314         }
1315       } else {
1316         // 32-bit
1317         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1318         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1319         if (cbuf) {
1320           MacroAssembler _masm(cbuf);
1321           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1322 #ifndef PRODUCT
1323         } else {
1324           st->print("%s  %s, %s\t# spill",
1325                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1326                      Matcher::regName[dst_first],
1327                      Matcher::regName[src_first]);
1328 #endif
1329         }
1330       }
1331       return 0;
1332     }
1333   }
1334 
1335   assert(0," foo ");
1336   Unimplemented();
1337   return 0;
1338 }
1339 
1340 #ifndef PRODUCT
1341 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1342   implementation(NULL, ra_, false, st);
1343 }
1344 #endif
1345 
1346 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1347   implementation(&cbuf, ra_, false, NULL);
1348 }
1349 
1350 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1351   return MachNode::size(ra_);
1352 }
1353 
1354 //=============================================================================
1355 #ifndef PRODUCT
1356 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1357 {
1358   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1359   int reg = ra_->get_reg_first(this);
1360   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1361             Matcher::regName[reg], offset);
1362 }
1363 #endif
1364 
1365 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1366 {
1367   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1368   int reg = ra_->get_encode(this);
1369   if (offset >= 0x80) {
1370     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1371     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1372     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1373     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1374     emit_d32(cbuf, offset);
1375   } else {
1376     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1377     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1378     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1379     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1380     emit_d8(cbuf, offset);
1381   }
1382 }
1383 
1384 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1385 {
1386   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1387   return (offset < 0x80) ? 5 : 8; // REX
1388 }
1389 
1390 //=============================================================================
1391 #ifndef PRODUCT
1392 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1393 {
1394   if (UseCompressedClassPointers) {
1395     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1396     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1397     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1398   } else {
1399     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1400                  "# Inline cache check");
1401   }
1402   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1403   st->print_cr("\tnop\t# nops to align entry point");
1404 }
1405 #endif
1406 
1407 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1408 {
1409   MacroAssembler masm(&cbuf);
1410   uint insts_size = cbuf.insts_size();
1411   if (UseCompressedClassPointers) {
1412     masm.load_klass(rscratch1, j_rarg0);
1413     masm.cmpptr(rax, rscratch1);
1414   } else {
1415     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1416   }
1417 
1418   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1419 
1420   /* WARNING these NOPs are critical so that verified entry point is properly
1421      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1422   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1423   if (OptoBreakpoint) {
1424     // Leave space for int3
1425     nops_cnt -= 1;
1426   }
1427   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1428   if (nops_cnt > 0)
1429     masm.nop(nops_cnt);
1430 }
1431 
1432 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1433 {
1434   return MachNode::size(ra_); // too many variables; just compute it
1435                               // the hard way
1436 }
1437 
1438 
1439 //=============================================================================
1440 uint size_exception_handler()
1441 {
1442   // NativeCall instruction size is the same as NativeJump.
1443   // Note that this value is also credited (in output.cpp) to
1444   // the size of the code section.
1445   return NativeJump::instruction_size;
1446 }
1447 
1448 // Emit exception handler code.
1449 int emit_exception_handler(CodeBuffer& cbuf)
1450 {
1451 
1452   // Note that the code buffer's insts_mark is always relative to insts.
1453   // That's why we must use the macroassembler to generate a handler.
1454   MacroAssembler _masm(&cbuf);
1455   address base =
1456   __ start_a_stub(size_exception_handler());
1457   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1458   int offset = __ offset();
1459   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1460   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1461   __ end_a_stub();
1462   return offset;
1463 }
1464 
1465 uint size_deopt_handler()
1466 {
1467   // three 5 byte instructions
1468   return 15;
1469 }
1470 
1471 // Emit deopt handler code.
1472 int emit_deopt_handler(CodeBuffer& cbuf)
1473 {
1474 
1475   // Note that the code buffer's insts_mark is always relative to insts.
1476   // That's why we must use the macroassembler to generate a handler.
1477   MacroAssembler _masm(&cbuf);
1478   address base =
1479   __ start_a_stub(size_deopt_handler());
1480   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1481   int offset = __ offset();
1482   address the_pc = (address) __ pc();
1483   Label next;
1484   // push a "the_pc" on the stack without destroying any registers
1485   // as they all may be live.
1486 
1487   // push address of "next"
1488   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1489   __ bind(next);
1490   // adjust it so it matches "the_pc"
1491   __ subptr(Address(rsp, 0), __ offset() - offset);
1492   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1493   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1494   __ end_a_stub();
1495   return offset;
1496 }
1497 
1498 int Matcher::regnum_to_fpu_offset(int regnum)
1499 {
1500   return regnum - 32; // The FP registers are in the second chunk
1501 }
1502 
1503 // This is UltraSparc specific, true just means we have fast l2f conversion
1504 const bool Matcher::convL2FSupported(void) {
1505   return true;
1506 }
1507 
1508 // Is this branch offset short enough that a short branch can be used?
1509 //
1510 // NOTE: If the platform does not provide any short branch variants, then
1511 //       this method should return false for offset 0.
1512 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1513   // The passed offset is relative to address of the branch.
1514   // On 86 a branch displacement is calculated relative to address
1515   // of a next instruction.
1516   offset -= br_size;
1517 
1518   // the short version of jmpConUCF2 contains multiple branches,
1519   // making the reach slightly less
1520   if (rule == jmpConUCF2_rule)
1521     return (-126 <= offset && offset <= 125);
1522   return (-128 <= offset && offset <= 127);
1523 }
1524 
1525 const bool Matcher::isSimpleConstant64(jlong value) {
1526   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1527   //return value == (int) value;  // Cf. storeImmL and immL32.
1528 
1529   // Probably always true, even if a temp register is required.
1530   return true;
1531 }
1532 
1533 // The ecx parameter to rep stosq for the ClearArray node is in words.
1534 const bool Matcher::init_array_count_is_in_bytes = false;
1535 
1536 // Threshold size for cleararray.
1537 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1538 
1539 // No additional cost for CMOVL.
1540 const int Matcher::long_cmove_cost() { return 0; }
1541 
1542 // No CMOVF/CMOVD with SSE2
1543 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1544 
1545 // Should the Matcher clone shifts on addressing modes, expecting them
1546 // to be subsumed into complex addressing expressions or compute them
1547 // into registers?  True for Intel but false for most RISCs
1548 const bool Matcher::clone_shift_expressions = true;
1549 
1550 // Do we need to mask the count passed to shift instructions or does
1551 // the cpu only look at the lower 5/6 bits anyway?
1552 const bool Matcher::need_masked_shift_count = false;
1553 
1554 bool Matcher::narrow_oop_use_complex_address() {
1555   assert(UseCompressedOops, "only for compressed oops code");
1556   return (LogMinObjAlignmentInBytes <= 3);
1557 }
1558 
1559 bool Matcher::narrow_klass_use_complex_address() {
1560   assert(UseCompressedClassPointers, "only for compressed klass code");
1561   return (LogKlassAlignmentInBytes <= 3);
1562 }
1563 
1564 // Is it better to copy float constants, or load them directly from
1565 // memory?  Intel can load a float constant from a direct address,
1566 // requiring no extra registers.  Most RISCs will have to materialize
1567 // an address into a register first, so they would do better to copy
1568 // the constant from stack.
1569 const bool Matcher::rematerialize_float_constants = true; // XXX
1570 
1571 // If CPU can load and store mis-aligned doubles directly then no
1572 // fixup is needed.  Else we split the double into 2 integer pieces
1573 // and move it piece-by-piece.  Only happens when passing doubles into
1574 // C code as the Java calling convention forces doubles to be aligned.
1575 const bool Matcher::misaligned_doubles_ok = true;
1576 
1577 // No-op on amd64
1578 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1579 
1580 // Advertise here if the CPU requires explicit rounding operations to
1581 // implement the UseStrictFP mode.
1582 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1583 
1584 // Are floats conerted to double when stored to stack during deoptimization?
1585 // On x64 it is stored without convertion so we can use normal access.
1586 bool Matcher::float_in_double() { return false; }
1587 
1588 // Do ints take an entire long register or just half?
1589 const bool Matcher::int_in_long = true;
1590 
1591 // Return whether or not this register is ever used as an argument.
1592 // This function is used on startup to build the trampoline stubs in
1593 // generateOptoStub.  Registers not mentioned will be killed by the VM
1594 // call in the trampoline, and arguments in those registers not be
1595 // available to the callee.
1596 bool Matcher::can_be_java_arg(int reg)
1597 {
1598   return
1599     reg ==  RDI_num || reg == RDI_H_num ||
1600     reg ==  RSI_num || reg == RSI_H_num ||
1601     reg ==  RDX_num || reg == RDX_H_num ||
1602     reg ==  RCX_num || reg == RCX_H_num ||
1603     reg ==   R8_num || reg ==  R8_H_num ||
1604     reg ==   R9_num || reg ==  R9_H_num ||
1605     reg ==  R12_num || reg == R12_H_num ||
1606     reg == XMM0_num || reg == XMM0b_num ||
1607     reg == XMM1_num || reg == XMM1b_num ||
1608     reg == XMM2_num || reg == XMM2b_num ||
1609     reg == XMM3_num || reg == XMM3b_num ||
1610     reg == XMM4_num || reg == XMM4b_num ||
1611     reg == XMM5_num || reg == XMM5b_num ||
1612     reg == XMM6_num || reg == XMM6b_num ||
1613     reg == XMM7_num || reg == XMM7b_num;
1614 }
1615 
1616 bool Matcher::is_spillable_arg(int reg)
1617 {
1618   return can_be_java_arg(reg);
1619 }
1620 
1621 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1622   // In 64 bit mode a code which use multiply when
1623   // devisor is constant is faster than hardware
1624   // DIV instruction (it uses MulHiL).
1625   return false;
1626 }
1627 
1628 // Register for DIVI projection of divmodI
1629 RegMask Matcher::divI_proj_mask() {
1630   return INT_RAX_REG_mask();
1631 }
1632 
1633 // Register for MODI projection of divmodI
1634 RegMask Matcher::modI_proj_mask() {
1635   return INT_RDX_REG_mask();
1636 }
1637 
1638 // Register for DIVL projection of divmodL
1639 RegMask Matcher::divL_proj_mask() {
1640   return LONG_RAX_REG_mask();
1641 }
1642 
1643 // Register for MODL projection of divmodL
1644 RegMask Matcher::modL_proj_mask() {
1645   return LONG_RDX_REG_mask();
1646 }
1647 
1648 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1649   return PTR_RBP_REG_mask();
1650 }
1651 
1652 %}
1653 
1654 //----------ENCODING BLOCK-----------------------------------------------------
1655 // This block specifies the encoding classes used by the compiler to
1656 // output byte streams.  Encoding classes are parameterized macros
1657 // used by Machine Instruction Nodes in order to generate the bit
1658 // encoding of the instruction.  Operands specify their base encoding
1659 // interface with the interface keyword.  There are currently
1660 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1661 // COND_INTER.  REG_INTER causes an operand to generate a function
1662 // which returns its register number when queried.  CONST_INTER causes
1663 // an operand to generate a function which returns the value of the
1664 // constant when queried.  MEMORY_INTER causes an operand to generate
1665 // four functions which return the Base Register, the Index Register,
1666 // the Scale Value, and the Offset Value of the operand when queried.
1667 // COND_INTER causes an operand to generate six functions which return
1668 // the encoding code (ie - encoding bits for the instruction)
1669 // associated with each basic boolean condition for a conditional
1670 // instruction.
1671 //
1672 // Instructions specify two basic values for encoding.  Again, a
1673 // function is available to check if the constant displacement is an
1674 // oop. They use the ins_encode keyword to specify their encoding
1675 // classes (which must be a sequence of enc_class names, and their
1676 // parameters, specified in the encoding block), and they use the
1677 // opcode keyword to specify, in order, their primary, secondary, and
1678 // tertiary opcode.  Only the opcode sections which a particular
1679 // instruction needs for encoding need to be specified.
1680 encode %{
1681   // Build emit functions for each basic byte or larger field in the
1682   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1683   // from C++ code in the enc_class source block.  Emit functions will
1684   // live in the main source block for now.  In future, we can
1685   // generalize this by adding a syntax that specifies the sizes of
1686   // fields in an order, so that the adlc can build the emit functions
1687   // automagically
1688 
1689   // Emit primary opcode
1690   enc_class OpcP
1691   %{
1692     emit_opcode(cbuf, $primary);
1693   %}
1694 
1695   // Emit secondary opcode
1696   enc_class OpcS
1697   %{
1698     emit_opcode(cbuf, $secondary);
1699   %}
1700 
1701   // Emit tertiary opcode
1702   enc_class OpcT
1703   %{
1704     emit_opcode(cbuf, $tertiary);
1705   %}
1706 
1707   // Emit opcode directly
1708   enc_class Opcode(immI d8)
1709   %{
1710     emit_opcode(cbuf, $d8$$constant);
1711   %}
1712 
1713   // Emit size prefix
1714   enc_class SizePrefix
1715   %{
1716     emit_opcode(cbuf, 0x66);
1717   %}
1718 
1719   enc_class reg(rRegI reg)
1720   %{
1721     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1722   %}
1723 
1724   enc_class reg_reg(rRegI dst, rRegI src)
1725   %{
1726     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1727   %}
1728 
1729   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1730   %{
1731     emit_opcode(cbuf, $opcode$$constant);
1732     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1733   %}
1734 
1735   enc_class cdql_enc(no_rax_rdx_RegI div)
1736   %{
1737     // Full implementation of Java idiv and irem; checks for
1738     // special case as described in JVM spec., p.243 & p.271.
1739     //
1740     //         normal case                           special case
1741     //
1742     // input : rax: dividend                         min_int
1743     //         reg: divisor                          -1
1744     //
1745     // output: rax: quotient  (= rax idiv reg)       min_int
1746     //         rdx: remainder (= rax irem reg)       0
1747     //
1748     //  Code sequnce:
1749     //
1750     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1751     //    5:   75 07/08                jne    e <normal>
1752     //    7:   33 d2                   xor    %edx,%edx
1753     //  [div >= 8 -> offset + 1]
1754     //  [REX_B]
1755     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1756     //    c:   74 03/04                je     11 <done>
1757     // 000000000000000e <normal>:
1758     //    e:   99                      cltd
1759     //  [div >= 8 -> offset + 1]
1760     //  [REX_B]
1761     //    f:   f7 f9                   idiv   $div
1762     // 0000000000000011 <done>:
1763 
1764     // cmp    $0x80000000,%eax
1765     emit_opcode(cbuf, 0x3d);
1766     emit_d8(cbuf, 0x00);
1767     emit_d8(cbuf, 0x00);
1768     emit_d8(cbuf, 0x00);
1769     emit_d8(cbuf, 0x80);
1770 
1771     // jne    e <normal>
1772     emit_opcode(cbuf, 0x75);
1773     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1774 
1775     // xor    %edx,%edx
1776     emit_opcode(cbuf, 0x33);
1777     emit_d8(cbuf, 0xD2);
1778 
1779     // cmp    $0xffffffffffffffff,%ecx
1780     if ($div$$reg >= 8) {
1781       emit_opcode(cbuf, Assembler::REX_B);
1782     }
1783     emit_opcode(cbuf, 0x83);
1784     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1785     emit_d8(cbuf, 0xFF);
1786 
1787     // je     11 <done>
1788     emit_opcode(cbuf, 0x74);
1789     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1790 
1791     // <normal>
1792     // cltd
1793     emit_opcode(cbuf, 0x99);
1794 
1795     // idivl (note: must be emitted by the user of this rule)
1796     // <done>
1797   %}
1798 
1799   enc_class cdqq_enc(no_rax_rdx_RegL div)
1800   %{
1801     // Full implementation of Java ldiv and lrem; checks for
1802     // special case as described in JVM spec., p.243 & p.271.
1803     //
1804     //         normal case                           special case
1805     //
1806     // input : rax: dividend                         min_long
1807     //         reg: divisor                          -1
1808     //
1809     // output: rax: quotient  (= rax idiv reg)       min_long
1810     //         rdx: remainder (= rax irem reg)       0
1811     //
1812     //  Code sequnce:
1813     //
1814     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1815     //    7:   00 00 80
1816     //    a:   48 39 d0                cmp    %rdx,%rax
1817     //    d:   75 08                   jne    17 <normal>
1818     //    f:   33 d2                   xor    %edx,%edx
1819     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1820     //   15:   74 05                   je     1c <done>
1821     // 0000000000000017 <normal>:
1822     //   17:   48 99                   cqto
1823     //   19:   48 f7 f9                idiv   $div
1824     // 000000000000001c <done>:
1825 
1826     // mov    $0x8000000000000000,%rdx
1827     emit_opcode(cbuf, Assembler::REX_W);
1828     emit_opcode(cbuf, 0xBA);
1829     emit_d8(cbuf, 0x00);
1830     emit_d8(cbuf, 0x00);
1831     emit_d8(cbuf, 0x00);
1832     emit_d8(cbuf, 0x00);
1833     emit_d8(cbuf, 0x00);
1834     emit_d8(cbuf, 0x00);
1835     emit_d8(cbuf, 0x00);
1836     emit_d8(cbuf, 0x80);
1837 
1838     // cmp    %rdx,%rax
1839     emit_opcode(cbuf, Assembler::REX_W);
1840     emit_opcode(cbuf, 0x39);
1841     emit_d8(cbuf, 0xD0);
1842 
1843     // jne    17 <normal>
1844     emit_opcode(cbuf, 0x75);
1845     emit_d8(cbuf, 0x08);
1846 
1847     // xor    %edx,%edx
1848     emit_opcode(cbuf, 0x33);
1849     emit_d8(cbuf, 0xD2);
1850 
1851     // cmp    $0xffffffffffffffff,$div
1852     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1853     emit_opcode(cbuf, 0x83);
1854     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1855     emit_d8(cbuf, 0xFF);
1856 
1857     // je     1e <done>
1858     emit_opcode(cbuf, 0x74);
1859     emit_d8(cbuf, 0x05);
1860 
1861     // <normal>
1862     // cqto
1863     emit_opcode(cbuf, Assembler::REX_W);
1864     emit_opcode(cbuf, 0x99);
1865 
1866     // idivq (note: must be emitted by the user of this rule)
1867     // <done>
1868   %}
1869 
1870   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1871   enc_class OpcSE(immI imm)
1872   %{
1873     // Emit primary opcode and set sign-extend bit
1874     // Check for 8-bit immediate, and set sign extend bit in opcode
1875     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1876       emit_opcode(cbuf, $primary | 0x02);
1877     } else {
1878       // 32-bit immediate
1879       emit_opcode(cbuf, $primary);
1880     }
1881   %}
1882 
1883   enc_class OpcSErm(rRegI dst, immI imm)
1884   %{
1885     // OpcSEr/m
1886     int dstenc = $dst$$reg;
1887     if (dstenc >= 8) {
1888       emit_opcode(cbuf, Assembler::REX_B);
1889       dstenc -= 8;
1890     }
1891     // Emit primary opcode and set sign-extend bit
1892     // Check for 8-bit immediate, and set sign extend bit in opcode
1893     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1894       emit_opcode(cbuf, $primary | 0x02);
1895     } else {
1896       // 32-bit immediate
1897       emit_opcode(cbuf, $primary);
1898     }
1899     // Emit r/m byte with secondary opcode, after primary opcode.
1900     emit_rm(cbuf, 0x3, $secondary, dstenc);
1901   %}
1902 
1903   enc_class OpcSErm_wide(rRegL dst, immI imm)
1904   %{
1905     // OpcSEr/m
1906     int dstenc = $dst$$reg;
1907     if (dstenc < 8) {
1908       emit_opcode(cbuf, Assembler::REX_W);
1909     } else {
1910       emit_opcode(cbuf, Assembler::REX_WB);
1911       dstenc -= 8;
1912     }
1913     // Emit primary opcode and set sign-extend bit
1914     // Check for 8-bit immediate, and set sign extend bit in opcode
1915     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1916       emit_opcode(cbuf, $primary | 0x02);
1917     } else {
1918       // 32-bit immediate
1919       emit_opcode(cbuf, $primary);
1920     }
1921     // Emit r/m byte with secondary opcode, after primary opcode.
1922     emit_rm(cbuf, 0x3, $secondary, dstenc);
1923   %}
1924 
1925   enc_class Con8or32(immI imm)
1926   %{
1927     // Check for 8-bit immediate, and set sign extend bit in opcode
1928     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1929       $$$emit8$imm$$constant;
1930     } else {
1931       // 32-bit immediate
1932       $$$emit32$imm$$constant;
1933     }
1934   %}
1935 
1936   enc_class opc2_reg(rRegI dst)
1937   %{
1938     // BSWAP
1939     emit_cc(cbuf, $secondary, $dst$$reg);
1940   %}
1941 
1942   enc_class opc3_reg(rRegI dst)
1943   %{
1944     // BSWAP
1945     emit_cc(cbuf, $tertiary, $dst$$reg);
1946   %}
1947 
1948   enc_class reg_opc(rRegI div)
1949   %{
1950     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1951     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1952   %}
1953 
1954   enc_class enc_cmov(cmpOp cop)
1955   %{
1956     // CMOV
1957     $$$emit8$primary;
1958     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1959   %}
1960 
1961   enc_class enc_PartialSubtypeCheck()
1962   %{
1963     Register Rrdi = as_Register(RDI_enc); // result register
1964     Register Rrax = as_Register(RAX_enc); // super class
1965     Register Rrcx = as_Register(RCX_enc); // killed
1966     Register Rrsi = as_Register(RSI_enc); // sub class
1967     Label miss;
1968     const bool set_cond_codes = true;
1969 
1970     MacroAssembler _masm(&cbuf);
1971     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1972                                      NULL, &miss,
1973                                      /*set_cond_codes:*/ true);
1974     if ($primary) {
1975       __ xorptr(Rrdi, Rrdi);
1976     }
1977     __ bind(miss);
1978   %}
1979 
1980   enc_class clear_avx %{
1981     debug_only(int off0 = cbuf.insts_size());
1982     if (ra_->C->max_vector_size() > 16) {
1983       // Clear upper bits of YMM registers when current compiled code uses
1984       // wide vectors to avoid AVX <-> SSE transition penalty during call.
1985       MacroAssembler _masm(&cbuf);
1986       __ vzeroupper();
1987     }
1988     debug_only(int off1 = cbuf.insts_size());
1989     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1990   %}
1991 
1992   enc_class Java_To_Runtime(method meth) %{
1993     // No relocation needed
1994     MacroAssembler _masm(&cbuf);
1995     __ mov64(r10, (int64_t) $meth$$method);
1996     __ call(r10);
1997   %}
1998 
1999   enc_class Java_To_Interpreter(method meth)
2000   %{
2001     // CALL Java_To_Interpreter
2002     // This is the instruction starting address for relocation info.
2003     cbuf.set_insts_mark();
2004     $$$emit8$primary;
2005     // CALL directly to the runtime
2006     emit_d32_reloc(cbuf,
2007                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2008                    runtime_call_Relocation::spec(),
2009                    RELOC_DISP32);
2010   %}
2011 
2012   enc_class Java_Static_Call(method meth)
2013   %{
2014     // JAVA STATIC CALL
2015     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2016     // determine who we intended to call.
2017     cbuf.set_insts_mark();
2018     $$$emit8$primary;
2019 
2020     if (!_method) {
2021       emit_d32_reloc(cbuf,
2022                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2023                      runtime_call_Relocation::spec(),
2024                      RELOC_DISP32);
2025     } else if (_optimized_virtual) {
2026       emit_d32_reloc(cbuf,
2027                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2028                      opt_virtual_call_Relocation::spec(),
2029                      RELOC_DISP32);
2030     } else {
2031       emit_d32_reloc(cbuf,
2032                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2033                      static_call_Relocation::spec(),
2034                      RELOC_DISP32);
2035     }
2036     if (_method) {
2037       // Emit stub for static call.
2038       CompiledStaticCall::emit_to_interp_stub(cbuf);
2039     }
2040   %}
2041 
2042   enc_class Java_Dynamic_Call(method meth) %{
2043     MacroAssembler _masm(&cbuf);
2044     __ ic_call((address)$meth$$method);
2045   %}
2046 
2047   enc_class Java_Compiled_Call(method meth)
2048   %{
2049     // JAVA COMPILED CALL
2050     int disp = in_bytes(Method:: from_compiled_offset());
2051 
2052     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2053     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2054 
2055     // callq *disp(%rax)
2056     cbuf.set_insts_mark();
2057     $$$emit8$primary;
2058     if (disp < 0x80) {
2059       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2060       emit_d8(cbuf, disp); // Displacement
2061     } else {
2062       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2063       emit_d32(cbuf, disp); // Displacement
2064     }
2065   %}
2066 
2067   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2068   %{
2069     // SAL, SAR, SHR
2070     int dstenc = $dst$$reg;
2071     if (dstenc >= 8) {
2072       emit_opcode(cbuf, Assembler::REX_B);
2073       dstenc -= 8;
2074     }
2075     $$$emit8$primary;
2076     emit_rm(cbuf, 0x3, $secondary, dstenc);
2077     $$$emit8$shift$$constant;
2078   %}
2079 
2080   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2081   %{
2082     // SAL, SAR, SHR
2083     int dstenc = $dst$$reg;
2084     if (dstenc < 8) {
2085       emit_opcode(cbuf, Assembler::REX_W);
2086     } else {
2087       emit_opcode(cbuf, Assembler::REX_WB);
2088       dstenc -= 8;
2089     }
2090     $$$emit8$primary;
2091     emit_rm(cbuf, 0x3, $secondary, dstenc);
2092     $$$emit8$shift$$constant;
2093   %}
2094 
2095   enc_class load_immI(rRegI dst, immI src)
2096   %{
2097     int dstenc = $dst$$reg;
2098     if (dstenc >= 8) {
2099       emit_opcode(cbuf, Assembler::REX_B);
2100       dstenc -= 8;
2101     }
2102     emit_opcode(cbuf, 0xB8 | dstenc);
2103     $$$emit32$src$$constant;
2104   %}
2105 
2106   enc_class load_immL(rRegL dst, immL src)
2107   %{
2108     int dstenc = $dst$$reg;
2109     if (dstenc < 8) {
2110       emit_opcode(cbuf, Assembler::REX_W);
2111     } else {
2112       emit_opcode(cbuf, Assembler::REX_WB);
2113       dstenc -= 8;
2114     }
2115     emit_opcode(cbuf, 0xB8 | dstenc);
2116     emit_d64(cbuf, $src$$constant);
2117   %}
2118 
2119   enc_class load_immUL32(rRegL dst, immUL32 src)
2120   %{
2121     // same as load_immI, but this time we care about zeroes in the high word
2122     int dstenc = $dst$$reg;
2123     if (dstenc >= 8) {
2124       emit_opcode(cbuf, Assembler::REX_B);
2125       dstenc -= 8;
2126     }
2127     emit_opcode(cbuf, 0xB8 | dstenc);
2128     $$$emit32$src$$constant;
2129   %}
2130 
2131   enc_class load_immL32(rRegL dst, immL32 src)
2132   %{
2133     int dstenc = $dst$$reg;
2134     if (dstenc < 8) {
2135       emit_opcode(cbuf, Assembler::REX_W);
2136     } else {
2137       emit_opcode(cbuf, Assembler::REX_WB);
2138       dstenc -= 8;
2139     }
2140     emit_opcode(cbuf, 0xC7);
2141     emit_rm(cbuf, 0x03, 0x00, dstenc);
2142     $$$emit32$src$$constant;
2143   %}
2144 
2145   enc_class load_immP31(rRegP dst, immP32 src)
2146   %{
2147     // same as load_immI, but this time we care about zeroes in the high word
2148     int dstenc = $dst$$reg;
2149     if (dstenc >= 8) {
2150       emit_opcode(cbuf, Assembler::REX_B);
2151       dstenc -= 8;
2152     }
2153     emit_opcode(cbuf, 0xB8 | dstenc);
2154     $$$emit32$src$$constant;
2155   %}
2156 
2157   enc_class load_immP(rRegP dst, immP src)
2158   %{
2159     int dstenc = $dst$$reg;
2160     if (dstenc < 8) {
2161       emit_opcode(cbuf, Assembler::REX_W);
2162     } else {
2163       emit_opcode(cbuf, Assembler::REX_WB);
2164       dstenc -= 8;
2165     }
2166     emit_opcode(cbuf, 0xB8 | dstenc);
2167     // This next line should be generated from ADLC
2168     if ($src->constant_reloc() != relocInfo::none) {
2169       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2170     } else {
2171       emit_d64(cbuf, $src$$constant);
2172     }
2173   %}
2174 
2175   enc_class Con32(immI src)
2176   %{
2177     // Output immediate
2178     $$$emit32$src$$constant;
2179   %}
2180 
2181   enc_class Con32F_as_bits(immF src)
2182   %{
2183     // Output Float immediate bits
2184     jfloat jf = $src$$constant;
2185     jint jf_as_bits = jint_cast(jf);
2186     emit_d32(cbuf, jf_as_bits);
2187   %}
2188 
2189   enc_class Con16(immI src)
2190   %{
2191     // Output immediate
2192     $$$emit16$src$$constant;
2193   %}
2194 
2195   // How is this different from Con32??? XXX
2196   enc_class Con_d32(immI src)
2197   %{
2198     emit_d32(cbuf,$src$$constant);
2199   %}
2200 
2201   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2202     // Output immediate memory reference
2203     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2204     emit_d32(cbuf, 0x00);
2205   %}
2206 
2207   enc_class lock_prefix()
2208   %{
2209     if (os::is_MP()) {
2210       emit_opcode(cbuf, 0xF0); // lock
2211     }
2212   %}
2213 
2214   enc_class REX_mem(memory mem)
2215   %{
2216     if ($mem$$base >= 8) {
2217       if ($mem$$index < 8) {
2218         emit_opcode(cbuf, Assembler::REX_B);
2219       } else {
2220         emit_opcode(cbuf, Assembler::REX_XB);
2221       }
2222     } else {
2223       if ($mem$$index >= 8) {
2224         emit_opcode(cbuf, Assembler::REX_X);
2225       }
2226     }
2227   %}
2228 
2229   enc_class REX_mem_wide(memory mem)
2230   %{
2231     if ($mem$$base >= 8) {
2232       if ($mem$$index < 8) {
2233         emit_opcode(cbuf, Assembler::REX_WB);
2234       } else {
2235         emit_opcode(cbuf, Assembler::REX_WXB);
2236       }
2237     } else {
2238       if ($mem$$index < 8) {
2239         emit_opcode(cbuf, Assembler::REX_W);
2240       } else {
2241         emit_opcode(cbuf, Assembler::REX_WX);
2242       }
2243     }
2244   %}
2245 
2246   // for byte regs
2247   enc_class REX_breg(rRegI reg)
2248   %{
2249     if ($reg$$reg >= 4) {
2250       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2251     }
2252   %}
2253 
2254   // for byte regs
2255   enc_class REX_reg_breg(rRegI dst, rRegI src)
2256   %{
2257     if ($dst$$reg < 8) {
2258       if ($src$$reg >= 4) {
2259         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2260       }
2261     } else {
2262       if ($src$$reg < 8) {
2263         emit_opcode(cbuf, Assembler::REX_R);
2264       } else {
2265         emit_opcode(cbuf, Assembler::REX_RB);
2266       }
2267     }
2268   %}
2269 
2270   // for byte regs
2271   enc_class REX_breg_mem(rRegI reg, memory mem)
2272   %{
2273     if ($reg$$reg < 8) {
2274       if ($mem$$base < 8) {
2275         if ($mem$$index >= 8) {
2276           emit_opcode(cbuf, Assembler::REX_X);
2277         } else if ($reg$$reg >= 4) {
2278           emit_opcode(cbuf, Assembler::REX);
2279         }
2280       } else {
2281         if ($mem$$index < 8) {
2282           emit_opcode(cbuf, Assembler::REX_B);
2283         } else {
2284           emit_opcode(cbuf, Assembler::REX_XB);
2285         }
2286       }
2287     } else {
2288       if ($mem$$base < 8) {
2289         if ($mem$$index < 8) {
2290           emit_opcode(cbuf, Assembler::REX_R);
2291         } else {
2292           emit_opcode(cbuf, Assembler::REX_RX);
2293         }
2294       } else {
2295         if ($mem$$index < 8) {
2296           emit_opcode(cbuf, Assembler::REX_RB);
2297         } else {
2298           emit_opcode(cbuf, Assembler::REX_RXB);
2299         }
2300       }
2301     }
2302   %}
2303 
2304   enc_class REX_reg(rRegI reg)
2305   %{
2306     if ($reg$$reg >= 8) {
2307       emit_opcode(cbuf, Assembler::REX_B);
2308     }
2309   %}
2310 
2311   enc_class REX_reg_wide(rRegI reg)
2312   %{
2313     if ($reg$$reg < 8) {
2314       emit_opcode(cbuf, Assembler::REX_W);
2315     } else {
2316       emit_opcode(cbuf, Assembler::REX_WB);
2317     }
2318   %}
2319 
2320   enc_class REX_reg_reg(rRegI dst, rRegI src)
2321   %{
2322     if ($dst$$reg < 8) {
2323       if ($src$$reg >= 8) {
2324         emit_opcode(cbuf, Assembler::REX_B);
2325       }
2326     } else {
2327       if ($src$$reg < 8) {
2328         emit_opcode(cbuf, Assembler::REX_R);
2329       } else {
2330         emit_opcode(cbuf, Assembler::REX_RB);
2331       }
2332     }
2333   %}
2334 
2335   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2336   %{
2337     if ($dst$$reg < 8) {
2338       if ($src$$reg < 8) {
2339         emit_opcode(cbuf, Assembler::REX_W);
2340       } else {
2341         emit_opcode(cbuf, Assembler::REX_WB);
2342       }
2343     } else {
2344       if ($src$$reg < 8) {
2345         emit_opcode(cbuf, Assembler::REX_WR);
2346       } else {
2347         emit_opcode(cbuf, Assembler::REX_WRB);
2348       }
2349     }
2350   %}
2351 
2352   enc_class REX_reg_mem(rRegI reg, memory mem)
2353   %{
2354     if ($reg$$reg < 8) {
2355       if ($mem$$base < 8) {
2356         if ($mem$$index >= 8) {
2357           emit_opcode(cbuf, Assembler::REX_X);
2358         }
2359       } else {
2360         if ($mem$$index < 8) {
2361           emit_opcode(cbuf, Assembler::REX_B);
2362         } else {
2363           emit_opcode(cbuf, Assembler::REX_XB);
2364         }
2365       }
2366     } else {
2367       if ($mem$$base < 8) {
2368         if ($mem$$index < 8) {
2369           emit_opcode(cbuf, Assembler::REX_R);
2370         } else {
2371           emit_opcode(cbuf, Assembler::REX_RX);
2372         }
2373       } else {
2374         if ($mem$$index < 8) {
2375           emit_opcode(cbuf, Assembler::REX_RB);
2376         } else {
2377           emit_opcode(cbuf, Assembler::REX_RXB);
2378         }
2379       }
2380     }
2381   %}
2382 
2383   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2384   %{
2385     if ($reg$$reg < 8) {
2386       if ($mem$$base < 8) {
2387         if ($mem$$index < 8) {
2388           emit_opcode(cbuf, Assembler::REX_W);
2389         } else {
2390           emit_opcode(cbuf, Assembler::REX_WX);
2391         }
2392       } else {
2393         if ($mem$$index < 8) {
2394           emit_opcode(cbuf, Assembler::REX_WB);
2395         } else {
2396           emit_opcode(cbuf, Assembler::REX_WXB);
2397         }
2398       }
2399     } else {
2400       if ($mem$$base < 8) {
2401         if ($mem$$index < 8) {
2402           emit_opcode(cbuf, Assembler::REX_WR);
2403         } else {
2404           emit_opcode(cbuf, Assembler::REX_WRX);
2405         }
2406       } else {
2407         if ($mem$$index < 8) {
2408           emit_opcode(cbuf, Assembler::REX_WRB);
2409         } else {
2410           emit_opcode(cbuf, Assembler::REX_WRXB);
2411         }
2412       }
2413     }
2414   %}
2415 
2416   enc_class reg_mem(rRegI ereg, memory mem)
2417   %{
2418     // High registers handle in encode_RegMem
2419     int reg = $ereg$$reg;
2420     int base = $mem$$base;
2421     int index = $mem$$index;
2422     int scale = $mem$$scale;
2423     int disp = $mem$$disp;
2424     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2425 
2426     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2427   %}
2428 
2429   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2430   %{
2431     int rm_byte_opcode = $rm_opcode$$constant;
2432 
2433     // High registers handle in encode_RegMem
2434     int base = $mem$$base;
2435     int index = $mem$$index;
2436     int scale = $mem$$scale;
2437     int displace = $mem$$disp;
2438 
2439     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2440                                             // working with static
2441                                             // globals
2442     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2443                   disp_reloc);
2444   %}
2445 
2446   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2447   %{
2448     int reg_encoding = $dst$$reg;
2449     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2450     int index        = 0x04;            // 0x04 indicates no index
2451     int scale        = 0x00;            // 0x00 indicates no scale
2452     int displace     = $src1$$constant; // 0x00 indicates no displacement
2453     relocInfo::relocType disp_reloc = relocInfo::none;
2454     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2455                   disp_reloc);
2456   %}
2457 
2458   enc_class neg_reg(rRegI dst)
2459   %{
2460     int dstenc = $dst$$reg;
2461     if (dstenc >= 8) {
2462       emit_opcode(cbuf, Assembler::REX_B);
2463       dstenc -= 8;
2464     }
2465     // NEG $dst
2466     emit_opcode(cbuf, 0xF7);
2467     emit_rm(cbuf, 0x3, 0x03, dstenc);
2468   %}
2469 
2470   enc_class neg_reg_wide(rRegI dst)
2471   %{
2472     int dstenc = $dst$$reg;
2473     if (dstenc < 8) {
2474       emit_opcode(cbuf, Assembler::REX_W);
2475     } else {
2476       emit_opcode(cbuf, Assembler::REX_WB);
2477       dstenc -= 8;
2478     }
2479     // NEG $dst
2480     emit_opcode(cbuf, 0xF7);
2481     emit_rm(cbuf, 0x3, 0x03, dstenc);
2482   %}
2483 
2484   enc_class setLT_reg(rRegI dst)
2485   %{
2486     int dstenc = $dst$$reg;
2487     if (dstenc >= 8) {
2488       emit_opcode(cbuf, Assembler::REX_B);
2489       dstenc -= 8;
2490     } else if (dstenc >= 4) {
2491       emit_opcode(cbuf, Assembler::REX);
2492     }
2493     // SETLT $dst
2494     emit_opcode(cbuf, 0x0F);
2495     emit_opcode(cbuf, 0x9C);
2496     emit_rm(cbuf, 0x3, 0x0, dstenc);
2497   %}
2498 
2499   enc_class setNZ_reg(rRegI dst)
2500   %{
2501     int dstenc = $dst$$reg;
2502     if (dstenc >= 8) {
2503       emit_opcode(cbuf, Assembler::REX_B);
2504       dstenc -= 8;
2505     } else if (dstenc >= 4) {
2506       emit_opcode(cbuf, Assembler::REX);
2507     }
2508     // SETNZ $dst
2509     emit_opcode(cbuf, 0x0F);
2510     emit_opcode(cbuf, 0x95);
2511     emit_rm(cbuf, 0x3, 0x0, dstenc);
2512   %}
2513 
2514 
2515   // Compare the lonogs and set -1, 0, or 1 into dst
2516   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2517   %{
2518     int src1enc = $src1$$reg;
2519     int src2enc = $src2$$reg;
2520     int dstenc = $dst$$reg;
2521 
2522     // cmpq $src1, $src2
2523     if (src1enc < 8) {
2524       if (src2enc < 8) {
2525         emit_opcode(cbuf, Assembler::REX_W);
2526       } else {
2527         emit_opcode(cbuf, Assembler::REX_WB);
2528       }
2529     } else {
2530       if (src2enc < 8) {
2531         emit_opcode(cbuf, Assembler::REX_WR);
2532       } else {
2533         emit_opcode(cbuf, Assembler::REX_WRB);
2534       }
2535     }
2536     emit_opcode(cbuf, 0x3B);
2537     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2538 
2539     // movl $dst, -1
2540     if (dstenc >= 8) {
2541       emit_opcode(cbuf, Assembler::REX_B);
2542     }
2543     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2544     emit_d32(cbuf, -1);
2545 
2546     // jl,s done
2547     emit_opcode(cbuf, 0x7C);
2548     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2549 
2550     // setne $dst
2551     if (dstenc >= 4) {
2552       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2553     }
2554     emit_opcode(cbuf, 0x0F);
2555     emit_opcode(cbuf, 0x95);
2556     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2557 
2558     // movzbl $dst, $dst
2559     if (dstenc >= 4) {
2560       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2561     }
2562     emit_opcode(cbuf, 0x0F);
2563     emit_opcode(cbuf, 0xB6);
2564     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2565   %}
2566 
2567   enc_class Push_ResultXD(regD dst) %{
2568     MacroAssembler _masm(&cbuf);
2569     __ fstp_d(Address(rsp, 0));
2570     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2571     __ addptr(rsp, 8);
2572   %}
2573 
2574   enc_class Push_SrcXD(regD src) %{
2575     MacroAssembler _masm(&cbuf);
2576     __ subptr(rsp, 8);
2577     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2578     __ fld_d(Address(rsp, 0));
2579   %}
2580 
2581 
2582   enc_class enc_rethrow()
2583   %{
2584     cbuf.set_insts_mark();
2585     emit_opcode(cbuf, 0xE9); // jmp entry
2586     emit_d32_reloc(cbuf,
2587                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2588                    runtime_call_Relocation::spec(),
2589                    RELOC_DISP32);
2590   %}
2591 
2592 %}
2593 
2594 
2595 
2596 //----------FRAME--------------------------------------------------------------
2597 // Definition of frame structure and management information.
2598 //
2599 //  S T A C K   L A Y O U T    Allocators stack-slot number
2600 //                             |   (to get allocators register number
2601 //  G  Owned by    |        |  v    add OptoReg::stack0())
2602 //  r   CALLER     |        |
2603 //  o     |        +--------+      pad to even-align allocators stack-slot
2604 //  w     V        |  pad0  |        numbers; owned by CALLER
2605 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2606 //  h     ^        |   in   |  5
2607 //        |        |  args  |  4   Holes in incoming args owned by SELF
2608 //  |     |        |        |  3
2609 //  |     |        +--------+
2610 //  V     |        | old out|      Empty on Intel, window on Sparc
2611 //        |    old |preserve|      Must be even aligned.
2612 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2613 //        |        |   in   |  3   area for Intel ret address
2614 //     Owned by    |preserve|      Empty on Sparc.
2615 //       SELF      +--------+
2616 //        |        |  pad2  |  2   pad to align old SP
2617 //        |        +--------+  1
2618 //        |        | locks  |  0
2619 //        |        +--------+----> OptoReg::stack0(), even aligned
2620 //        |        |  pad1  | 11   pad to align new SP
2621 //        |        +--------+
2622 //        |        |        | 10
2623 //        |        | spills |  9   spills
2624 //        V        |        |  8   (pad0 slot for callee)
2625 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2626 //        ^        |  out   |  7
2627 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2628 //     Owned by    +--------+
2629 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2630 //        |    new |preserve|      Must be even-aligned.
2631 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2632 //        |        |        |
2633 //
2634 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2635 //         known from SELF's arguments and the Java calling convention.
2636 //         Region 6-7 is determined per call site.
2637 // Note 2: If the calling convention leaves holes in the incoming argument
2638 //         area, those holes are owned by SELF.  Holes in the outgoing area
2639 //         are owned by the CALLEE.  Holes should not be nessecary in the
2640 //         incoming area, as the Java calling convention is completely under
2641 //         the control of the AD file.  Doubles can be sorted and packed to
2642 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2643 //         varargs C calling conventions.
2644 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2645 //         even aligned with pad0 as needed.
2646 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2647 //         region 6-11 is even aligned; it may be padded out more so that
2648 //         the region from SP to FP meets the minimum stack alignment.
2649 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2650 //         alignment.  Region 11, pad1, may be dynamically extended so that
2651 //         SP meets the minimum alignment.
2652 
2653 frame
2654 %{
2655   // What direction does stack grow in (assumed to be same for C & Java)
2656   stack_direction(TOWARDS_LOW);
2657 
2658   // These three registers define part of the calling convention
2659   // between compiled code and the interpreter.
2660   inline_cache_reg(RAX);                // Inline Cache Register
2661   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2662                                         // calling interpreter
2663 
2664   // Optional: name the operand used by cisc-spilling to access
2665   // [stack_pointer + offset]
2666   cisc_spilling_operand_name(indOffset32);
2667 
2668   // Number of stack slots consumed by locking an object
2669   sync_stack_slots(2);
2670 
2671   // Compiled code's Frame Pointer
2672   frame_pointer(RSP);
2673 
2674   // Interpreter stores its frame pointer in a register which is
2675   // stored to the stack by I2CAdaptors.
2676   // I2CAdaptors convert from interpreted java to compiled java.
2677   interpreter_frame_pointer(RBP);
2678 
2679   // Stack alignment requirement
2680   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2681 
2682   // Number of stack slots between incoming argument block and the start of
2683   // a new frame.  The PROLOG must add this many slots to the stack.  The
2684   // EPILOG must remove this many slots.  amd64 needs two slots for
2685   // return address.
2686   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2687 
2688   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2689   // for calls to C.  Supports the var-args backing area for register parms.
2690   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2691 
2692   // The after-PROLOG location of the return address.  Location of
2693   // return address specifies a type (REG or STACK) and a number
2694   // representing the register number (i.e. - use a register name) or
2695   // stack slot.
2696   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2697   // Otherwise, it is above the locks and verification slot and alignment word
2698   return_addr(STACK - 2 +
2699               round_to((Compile::current()->in_preserve_stack_slots() +
2700                         Compile::current()->fixed_slots()),
2701                        stack_alignment_in_slots()));
2702 
2703   // Body of function which returns an integer array locating
2704   // arguments either in registers or in stack slots.  Passed an array
2705   // of ideal registers called "sig" and a "length" count.  Stack-slot
2706   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2707   // arguments for a CALLEE.  Incoming stack arguments are
2708   // automatically biased by the preserve_stack_slots field above.
2709 
2710   calling_convention
2711   %{
2712     // No difference between ingoing/outgoing just pass false
2713     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2714   %}
2715 
2716   c_calling_convention
2717   %{
2718     // This is obviously always outgoing
2719     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
2720   %}
2721 
2722   // Location of compiled Java return values.  Same as C for now.
2723   return_value
2724   %{
2725     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2726            "only return normal values");
2727 
2728     static const int lo[Op_RegL + 1] = {
2729       0,
2730       0,
2731       RAX_num,  // Op_RegN
2732       RAX_num,  // Op_RegI
2733       RAX_num,  // Op_RegP
2734       XMM0_num, // Op_RegF
2735       XMM0_num, // Op_RegD
2736       RAX_num   // Op_RegL
2737     };
2738     static const int hi[Op_RegL + 1] = {
2739       0,
2740       0,
2741       OptoReg::Bad, // Op_RegN
2742       OptoReg::Bad, // Op_RegI
2743       RAX_H_num,    // Op_RegP
2744       OptoReg::Bad, // Op_RegF
2745       XMM0b_num,    // Op_RegD
2746       RAX_H_num     // Op_RegL
2747     };
2748     // Excluded flags and vector registers.
2749     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2750     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2751   %}
2752 %}
2753 
2754 //----------ATTRIBUTES---------------------------------------------------------
2755 //----------Operand Attributes-------------------------------------------------
2756 op_attrib op_cost(0);        // Required cost attribute
2757 
2758 //----------Instruction Attributes---------------------------------------------
2759 ins_attrib ins_cost(100);       // Required cost attribute
2760 ins_attrib ins_size(8);         // Required size attribute (in bits)
2761 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2762                                 // a non-matching short branch variant
2763                                 // of some long branch?
2764 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2765                                 // be a power of 2) specifies the
2766                                 // alignment that some part of the
2767                                 // instruction (not necessarily the
2768                                 // start) requires.  If > 1, a
2769                                 // compute_padding() function must be
2770                                 // provided for the instruction
2771 
2772 //----------OPERANDS-----------------------------------------------------------
2773 // Operand definitions must precede instruction definitions for correct parsing
2774 // in the ADLC because operands constitute user defined types which are used in
2775 // instruction definitions.
2776 
2777 //----------Simple Operands----------------------------------------------------
2778 // Immediate Operands
2779 // Integer Immediate
2780 operand immI()
2781 %{
2782   match(ConI);
2783 
2784   op_cost(10);
2785   format %{ %}
2786   interface(CONST_INTER);
2787 %}
2788 
2789 // Constant for test vs zero
2790 operand immI0()
2791 %{
2792   predicate(n->get_int() == 0);
2793   match(ConI);
2794 
2795   op_cost(0);
2796   format %{ %}
2797   interface(CONST_INTER);
2798 %}
2799 
2800 // Constant for increment
2801 operand immI1()
2802 %{
2803   predicate(n->get_int() == 1);
2804   match(ConI);
2805 
2806   op_cost(0);
2807   format %{ %}
2808   interface(CONST_INTER);
2809 %}
2810 
2811 // Constant for decrement
2812 operand immI_M1()
2813 %{
2814   predicate(n->get_int() == -1);
2815   match(ConI);
2816 
2817   op_cost(0);
2818   format %{ %}
2819   interface(CONST_INTER);
2820 %}
2821 
2822 // Valid scale values for addressing modes
2823 operand immI2()
2824 %{
2825   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2826   match(ConI);
2827 
2828   format %{ %}
2829   interface(CONST_INTER);
2830 %}
2831 
2832 operand immI8()
2833 %{
2834   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2835   match(ConI);
2836 
2837   op_cost(5);
2838   format %{ %}
2839   interface(CONST_INTER);
2840 %}
2841 
2842 operand immI16()
2843 %{
2844   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2845   match(ConI);
2846 
2847   op_cost(10);
2848   format %{ %}
2849   interface(CONST_INTER);
2850 %}
2851 
2852 // Int Immediate non-negative
2853 operand immU31()
2854 %{
2855   predicate(n->get_int() >= 0);
2856   match(ConI);
2857 
2858   op_cost(0);
2859   format %{ %}
2860   interface(CONST_INTER);
2861 %}
2862 
2863 // Constant for long shifts
2864 operand immI_32()
2865 %{
2866   predicate( n->get_int() == 32 );
2867   match(ConI);
2868 
2869   op_cost(0);
2870   format %{ %}
2871   interface(CONST_INTER);
2872 %}
2873 
2874 // Constant for long shifts
2875 operand immI_64()
2876 %{
2877   predicate( n->get_int() == 64 );
2878   match(ConI);
2879 
2880   op_cost(0);
2881   format %{ %}
2882   interface(CONST_INTER);
2883 %}
2884 
2885 // Pointer Immediate
2886 operand immP()
2887 %{
2888   match(ConP);
2889 
2890   op_cost(10);
2891   format %{ %}
2892   interface(CONST_INTER);
2893 %}
2894 
2895 // NULL Pointer Immediate
2896 operand immP0()
2897 %{
2898   predicate(n->get_ptr() == 0);
2899   match(ConP);
2900 
2901   op_cost(5);
2902   format %{ %}
2903   interface(CONST_INTER);
2904 %}
2905 
2906 // Pointer Immediate
2907 operand immN() %{
2908   match(ConN);
2909 
2910   op_cost(10);
2911   format %{ %}
2912   interface(CONST_INTER);
2913 %}
2914 
2915 operand immNKlass() %{
2916   match(ConNKlass);
2917 
2918   op_cost(10);
2919   format %{ %}
2920   interface(CONST_INTER);
2921 %}
2922 
2923 // NULL Pointer Immediate
2924 operand immN0() %{
2925   predicate(n->get_narrowcon() == 0);
2926   match(ConN);
2927 
2928   op_cost(5);
2929   format %{ %}
2930   interface(CONST_INTER);
2931 %}
2932 
2933 operand immP31()
2934 %{
2935   predicate(n->as_Type()->type()->reloc() == relocInfo::none
2936             && (n->get_ptr() >> 31) == 0);
2937   match(ConP);
2938 
2939   op_cost(5);
2940   format %{ %}
2941   interface(CONST_INTER);
2942 %}
2943 
2944 
2945 // Long Immediate
2946 operand immL()
2947 %{
2948   match(ConL);
2949 
2950   op_cost(20);
2951   format %{ %}
2952   interface(CONST_INTER);
2953 %}
2954 
2955 // Long Immediate 8-bit
2956 operand immL8()
2957 %{
2958   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2959   match(ConL);
2960 
2961   op_cost(5);
2962   format %{ %}
2963   interface(CONST_INTER);
2964 %}
2965 
2966 // Long Immediate 32-bit unsigned
2967 operand immUL32()
2968 %{
2969   predicate(n->get_long() == (unsigned int) (n->get_long()));
2970   match(ConL);
2971 
2972   op_cost(10);
2973   format %{ %}
2974   interface(CONST_INTER);
2975 %}
2976 
2977 // Long Immediate 32-bit signed
2978 operand immL32()
2979 %{
2980   predicate(n->get_long() == (int) (n->get_long()));
2981   match(ConL);
2982 
2983   op_cost(15);
2984   format %{ %}
2985   interface(CONST_INTER);
2986 %}
2987 
2988 // Long Immediate zero
2989 operand immL0()
2990 %{
2991   predicate(n->get_long() == 0L);
2992   match(ConL);
2993 
2994   op_cost(10);
2995   format %{ %}
2996   interface(CONST_INTER);
2997 %}
2998 
2999 // Constant for increment
3000 operand immL1()
3001 %{
3002   predicate(n->get_long() == 1);
3003   match(ConL);
3004 
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 // Constant for decrement
3010 operand immL_M1()
3011 %{
3012   predicate(n->get_long() == -1);
3013   match(ConL);
3014 
3015   format %{ %}
3016   interface(CONST_INTER);
3017 %}
3018 
3019 // Long Immediate: the value 10
3020 operand immL10()
3021 %{
3022   predicate(n->get_long() == 10);
3023   match(ConL);
3024 
3025   format %{ %}
3026   interface(CONST_INTER);
3027 %}
3028 
3029 // Long immediate from 0 to 127.
3030 // Used for a shorter form of long mul by 10.
3031 operand immL_127()
3032 %{
3033   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3034   match(ConL);
3035 
3036   op_cost(10);
3037   format %{ %}
3038   interface(CONST_INTER);
3039 %}
3040 
3041 // Long Immediate: low 32-bit mask
3042 operand immL_32bits()
3043 %{
3044   predicate(n->get_long() == 0xFFFFFFFFL);
3045   match(ConL);
3046   op_cost(20);
3047 
3048   format %{ %}
3049   interface(CONST_INTER);
3050 %}
3051 
3052 // Float Immediate zero
3053 operand immF0()
3054 %{
3055   predicate(jint_cast(n->getf()) == 0);
3056   match(ConF);
3057 
3058   op_cost(5);
3059   format %{ %}
3060   interface(CONST_INTER);
3061 %}
3062 
3063 // Float Immediate
3064 operand immF()
3065 %{
3066   match(ConF);
3067 
3068   op_cost(15);
3069   format %{ %}
3070   interface(CONST_INTER);
3071 %}
3072 
3073 // Double Immediate zero
3074 operand immD0()
3075 %{
3076   predicate(jlong_cast(n->getd()) == 0);
3077   match(ConD);
3078 
3079   op_cost(5);
3080   format %{ %}
3081   interface(CONST_INTER);
3082 %}
3083 
3084 // Double Immediate
3085 operand immD()
3086 %{
3087   match(ConD);
3088 
3089   op_cost(15);
3090   format %{ %}
3091   interface(CONST_INTER);
3092 %}
3093 
3094 // Immediates for special shifts (sign extend)
3095 
3096 // Constants for increment
3097 operand immI_16()
3098 %{
3099   predicate(n->get_int() == 16);
3100   match(ConI);
3101 
3102   format %{ %}
3103   interface(CONST_INTER);
3104 %}
3105 
3106 operand immI_24()
3107 %{
3108   predicate(n->get_int() == 24);
3109   match(ConI);
3110 
3111   format %{ %}
3112   interface(CONST_INTER);
3113 %}
3114 
3115 // Constant for byte-wide masking
3116 operand immI_255()
3117 %{
3118   predicate(n->get_int() == 255);
3119   match(ConI);
3120 
3121   format %{ %}
3122   interface(CONST_INTER);
3123 %}
3124 
3125 // Constant for short-wide masking
3126 operand immI_65535()
3127 %{
3128   predicate(n->get_int() == 65535);
3129   match(ConI);
3130 
3131   format %{ %}
3132   interface(CONST_INTER);
3133 %}
3134 
3135 // Constant for byte-wide masking
3136 operand immL_255()
3137 %{
3138   predicate(n->get_long() == 255);
3139   match(ConL);
3140 
3141   format %{ %}
3142   interface(CONST_INTER);
3143 %}
3144 
3145 // Constant for short-wide masking
3146 operand immL_65535()
3147 %{
3148   predicate(n->get_long() == 65535);
3149   match(ConL);
3150 
3151   format %{ %}
3152   interface(CONST_INTER);
3153 %}
3154 
3155 // Register Operands
3156 // Integer Register
3157 operand rRegI()
3158 %{
3159   constraint(ALLOC_IN_RC(int_reg));
3160   match(RegI);
3161 
3162   match(rax_RegI);
3163   match(rbx_RegI);
3164   match(rcx_RegI);
3165   match(rdx_RegI);
3166   match(rdi_RegI);
3167 
3168   format %{ %}
3169   interface(REG_INTER);
3170 %}
3171 
3172 // Special Registers
3173 operand rax_RegI()
3174 %{
3175   constraint(ALLOC_IN_RC(int_rax_reg));
3176   match(RegI);
3177   match(rRegI);
3178 
3179   format %{ "RAX" %}
3180   interface(REG_INTER);
3181 %}
3182 
3183 // Special Registers
3184 operand rbx_RegI()
3185 %{
3186   constraint(ALLOC_IN_RC(int_rbx_reg));
3187   match(RegI);
3188   match(rRegI);
3189 
3190   format %{ "RBX" %}
3191   interface(REG_INTER);
3192 %}
3193 
3194 operand rcx_RegI()
3195 %{
3196   constraint(ALLOC_IN_RC(int_rcx_reg));
3197   match(RegI);
3198   match(rRegI);
3199 
3200   format %{ "RCX" %}
3201   interface(REG_INTER);
3202 %}
3203 
3204 operand rdx_RegI()
3205 %{
3206   constraint(ALLOC_IN_RC(int_rdx_reg));
3207   match(RegI);
3208   match(rRegI);
3209 
3210   format %{ "RDX" %}
3211   interface(REG_INTER);
3212 %}
3213 
3214 operand rdi_RegI()
3215 %{
3216   constraint(ALLOC_IN_RC(int_rdi_reg));
3217   match(RegI);
3218   match(rRegI);
3219 
3220   format %{ "RDI" %}
3221   interface(REG_INTER);
3222 %}
3223 
3224 operand no_rcx_RegI()
3225 %{
3226   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3227   match(RegI);
3228   match(rax_RegI);
3229   match(rbx_RegI);
3230   match(rdx_RegI);
3231   match(rdi_RegI);
3232 
3233   format %{ %}
3234   interface(REG_INTER);
3235 %}
3236 
3237 operand no_rax_rdx_RegI()
3238 %{
3239   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3240   match(RegI);
3241   match(rbx_RegI);
3242   match(rcx_RegI);
3243   match(rdi_RegI);
3244 
3245   format %{ %}
3246   interface(REG_INTER);
3247 %}
3248 
3249 // Pointer Register
3250 operand any_RegP()
3251 %{
3252   constraint(ALLOC_IN_RC(any_reg));
3253   match(RegP);
3254   match(rax_RegP);
3255   match(rbx_RegP);
3256   match(rdi_RegP);
3257   match(rsi_RegP);
3258   match(rbp_RegP);
3259   match(r15_RegP);
3260   match(rRegP);
3261 
3262   format %{ %}
3263   interface(REG_INTER);
3264 %}
3265 
3266 operand rRegP()
3267 %{
3268   constraint(ALLOC_IN_RC(ptr_reg));
3269   match(RegP);
3270   match(rax_RegP);
3271   match(rbx_RegP);
3272   match(rdi_RegP);
3273   match(rsi_RegP);
3274   match(rbp_RegP);
3275   match(r15_RegP);  // See Q&A below about r15_RegP.
3276 
3277   format %{ %}
3278   interface(REG_INTER);
3279 %}
3280 
3281 operand rRegN() %{
3282   constraint(ALLOC_IN_RC(int_reg));
3283   match(RegN);
3284 
3285   format %{ %}
3286   interface(REG_INTER);
3287 %}
3288 
3289 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3290 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3291 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
3292 // The output of an instruction is controlled by the allocator, which respects
3293 // register class masks, not match rules.  Unless an instruction mentions
3294 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3295 // by the allocator as an input.
3296 
3297 operand no_rax_RegP()
3298 %{
3299   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3300   match(RegP);
3301   match(rbx_RegP);
3302   match(rsi_RegP);
3303   match(rdi_RegP);
3304 
3305   format %{ %}
3306   interface(REG_INTER);
3307 %}
3308 
3309 operand no_rbp_RegP()
3310 %{
3311   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
3312   match(RegP);
3313   match(rbx_RegP);
3314   match(rsi_RegP);
3315   match(rdi_RegP);
3316 
3317   format %{ %}
3318   interface(REG_INTER);
3319 %}
3320 
3321 operand no_rax_rbx_RegP()
3322 %{
3323   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3324   match(RegP);
3325   match(rsi_RegP);
3326   match(rdi_RegP);
3327 
3328   format %{ %}
3329   interface(REG_INTER);
3330 %}
3331 
3332 // Special Registers
3333 // Return a pointer value
3334 operand rax_RegP()
3335 %{
3336   constraint(ALLOC_IN_RC(ptr_rax_reg));
3337   match(RegP);
3338   match(rRegP);
3339 
3340   format %{ %}
3341   interface(REG_INTER);
3342 %}
3343 
3344 // Special Registers
3345 // Return a compressed pointer value
3346 operand rax_RegN()
3347 %{
3348   constraint(ALLOC_IN_RC(int_rax_reg));
3349   match(RegN);
3350   match(rRegN);
3351 
3352   format %{ %}
3353   interface(REG_INTER);
3354 %}
3355 
3356 // Used in AtomicAdd
3357 operand rbx_RegP()
3358 %{
3359   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3360   match(RegP);
3361   match(rRegP);
3362 
3363   format %{ %}
3364   interface(REG_INTER);
3365 %}
3366 
3367 operand rsi_RegP()
3368 %{
3369   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3370   match(RegP);
3371   match(rRegP);
3372 
3373   format %{ %}
3374   interface(REG_INTER);
3375 %}
3376 
3377 // Used in rep stosq
3378 operand rdi_RegP()
3379 %{
3380   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3381   match(RegP);
3382   match(rRegP);
3383 
3384   format %{ %}
3385   interface(REG_INTER);
3386 %}
3387 
3388 operand rbp_RegP()
3389 %{
3390   constraint(ALLOC_IN_RC(ptr_rbp_reg));
3391   match(RegP);
3392   match(rRegP);
3393 
3394   format %{ %}
3395   interface(REG_INTER);
3396 %}
3397 
3398 operand r15_RegP()
3399 %{
3400   constraint(ALLOC_IN_RC(ptr_r15_reg));
3401   match(RegP);
3402   match(rRegP);
3403 
3404   format %{ %}
3405   interface(REG_INTER);
3406 %}
3407 
3408 operand rRegL()
3409 %{
3410   constraint(ALLOC_IN_RC(long_reg));
3411   match(RegL);
3412   match(rax_RegL);
3413   match(rdx_RegL);
3414 
3415   format %{ %}
3416   interface(REG_INTER);
3417 %}
3418 
3419 // Special Registers
3420 operand no_rax_rdx_RegL()
3421 %{
3422   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3423   match(RegL);
3424   match(rRegL);
3425 
3426   format %{ %}
3427   interface(REG_INTER);
3428 %}
3429 
3430 operand no_rax_RegL()
3431 %{
3432   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3433   match(RegL);
3434   match(rRegL);
3435   match(rdx_RegL);
3436 
3437   format %{ %}
3438   interface(REG_INTER);
3439 %}
3440 
3441 operand no_rcx_RegL()
3442 %{
3443   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3444   match(RegL);
3445   match(rRegL);
3446 
3447   format %{ %}
3448   interface(REG_INTER);
3449 %}
3450 
3451 operand rax_RegL()
3452 %{
3453   constraint(ALLOC_IN_RC(long_rax_reg));
3454   match(RegL);
3455   match(rRegL);
3456 
3457   format %{ "RAX" %}
3458   interface(REG_INTER);
3459 %}
3460 
3461 operand rcx_RegL()
3462 %{
3463   constraint(ALLOC_IN_RC(long_rcx_reg));
3464   match(RegL);
3465   match(rRegL);
3466 
3467   format %{ %}
3468   interface(REG_INTER);
3469 %}
3470 
3471 operand rdx_RegL()
3472 %{
3473   constraint(ALLOC_IN_RC(long_rdx_reg));
3474   match(RegL);
3475   match(rRegL);
3476 
3477   format %{ %}
3478   interface(REG_INTER);
3479 %}
3480 
3481 // Flags register, used as output of compare instructions
3482 operand rFlagsReg()
3483 %{
3484   constraint(ALLOC_IN_RC(int_flags));
3485   match(RegFlags);
3486 
3487   format %{ "RFLAGS" %}
3488   interface(REG_INTER);
3489 %}
3490 
3491 // Flags register, used as output of FLOATING POINT compare instructions
3492 operand rFlagsRegU()
3493 %{
3494   constraint(ALLOC_IN_RC(int_flags));
3495   match(RegFlags);
3496 
3497   format %{ "RFLAGS_U" %}
3498   interface(REG_INTER);
3499 %}
3500 
3501 operand rFlagsRegUCF() %{
3502   constraint(ALLOC_IN_RC(int_flags));
3503   match(RegFlags);
3504   predicate(false);
3505 
3506   format %{ "RFLAGS_U_CF" %}
3507   interface(REG_INTER);
3508 %}
3509 
3510 // Float register operands
3511 operand regF()
3512 %{
3513   constraint(ALLOC_IN_RC(float_reg));
3514   match(RegF);
3515 
3516   format %{ %}
3517   interface(REG_INTER);
3518 %}
3519 
3520 // Double register operands
3521 operand regD()
3522 %{
3523   constraint(ALLOC_IN_RC(double_reg));
3524   match(RegD);
3525 
3526   format %{ %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 //----------Memory Operands----------------------------------------------------
3531 // Direct Memory Operand
3532 // operand direct(immP addr)
3533 // %{
3534 //   match(addr);
3535 
3536 //   format %{ "[$addr]" %}
3537 //   interface(MEMORY_INTER) %{
3538 //     base(0xFFFFFFFF);
3539 //     index(0x4);
3540 //     scale(0x0);
3541 //     disp($addr);
3542 //   %}
3543 // %}
3544 
3545 // Indirect Memory Operand
3546 operand indirect(any_RegP reg)
3547 %{
3548   constraint(ALLOC_IN_RC(ptr_reg));
3549   match(reg);
3550 
3551   format %{ "[$reg]" %}
3552   interface(MEMORY_INTER) %{
3553     base($reg);
3554     index(0x4);
3555     scale(0x0);
3556     disp(0x0);
3557   %}
3558 %}
3559 
3560 // Indirect Memory Plus Short Offset Operand
3561 operand indOffset8(any_RegP reg, immL8 off)
3562 %{
3563   constraint(ALLOC_IN_RC(ptr_reg));
3564   match(AddP reg off);
3565 
3566   format %{ "[$reg + $off (8-bit)]" %}
3567   interface(MEMORY_INTER) %{
3568     base($reg);
3569     index(0x4);
3570     scale(0x0);
3571     disp($off);
3572   %}
3573 %}
3574 
3575 // Indirect Memory Plus Long Offset Operand
3576 operand indOffset32(any_RegP reg, immL32 off)
3577 %{
3578   constraint(ALLOC_IN_RC(ptr_reg));
3579   match(AddP reg off);
3580 
3581   format %{ "[$reg + $off (32-bit)]" %}
3582   interface(MEMORY_INTER) %{
3583     base($reg);
3584     index(0x4);
3585     scale(0x0);
3586     disp($off);
3587   %}
3588 %}
3589 
3590 // Indirect Memory Plus Index Register Plus Offset Operand
3591 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3592 %{
3593   constraint(ALLOC_IN_RC(ptr_reg));
3594   match(AddP (AddP reg lreg) off);
3595 
3596   op_cost(10);
3597   format %{"[$reg + $off + $lreg]" %}
3598   interface(MEMORY_INTER) %{
3599     base($reg);
3600     index($lreg);
3601     scale(0x0);
3602     disp($off);
3603   %}
3604 %}
3605 
3606 // Indirect Memory Plus Index Register Plus Offset Operand
3607 operand indIndex(any_RegP reg, rRegL lreg)
3608 %{
3609   constraint(ALLOC_IN_RC(ptr_reg));
3610   match(AddP reg lreg);
3611 
3612   op_cost(10);
3613   format %{"[$reg + $lreg]" %}
3614   interface(MEMORY_INTER) %{
3615     base($reg);
3616     index($lreg);
3617     scale(0x0);
3618     disp(0x0);
3619   %}
3620 %}
3621 
3622 // Indirect Memory Times Scale Plus Index Register
3623 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3624 %{
3625   constraint(ALLOC_IN_RC(ptr_reg));
3626   match(AddP reg (LShiftL lreg scale));
3627 
3628   op_cost(10);
3629   format %{"[$reg + $lreg << $scale]" %}
3630   interface(MEMORY_INTER) %{
3631     base($reg);
3632     index($lreg);
3633     scale($scale);
3634     disp(0x0);
3635   %}
3636 %}
3637 
3638 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3639 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3640 %{
3641   constraint(ALLOC_IN_RC(ptr_reg));
3642   match(AddP (AddP reg (LShiftL lreg scale)) off);
3643 
3644   op_cost(10);
3645   format %{"[$reg + $off + $lreg << $scale]" %}
3646   interface(MEMORY_INTER) %{
3647     base($reg);
3648     index($lreg);
3649     scale($scale);
3650     disp($off);
3651   %}
3652 %}
3653 
3654 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3655 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3656 %{
3657   constraint(ALLOC_IN_RC(ptr_reg));
3658   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3659   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3660 
3661   op_cost(10);
3662   format %{"[$reg + $off + $idx << $scale]" %}
3663   interface(MEMORY_INTER) %{
3664     base($reg);
3665     index($idx);
3666     scale($scale);
3667     disp($off);
3668   %}
3669 %}
3670 
3671 // Indirect Narrow Oop Plus Offset Operand
3672 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3673 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3674 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3675   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3676   constraint(ALLOC_IN_RC(ptr_reg));
3677   match(AddP (DecodeN reg) off);
3678 
3679   op_cost(10);
3680   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3681   interface(MEMORY_INTER) %{
3682     base(0xc); // R12
3683     index($reg);
3684     scale(0x3);
3685     disp($off);
3686   %}
3687 %}
3688 
3689 // Indirect Memory Operand
3690 operand indirectNarrow(rRegN reg)
3691 %{
3692   predicate(Universe::narrow_oop_shift() == 0);
3693   constraint(ALLOC_IN_RC(ptr_reg));
3694   match(DecodeN reg);
3695 
3696   format %{ "[$reg]" %}
3697   interface(MEMORY_INTER) %{
3698     base($reg);
3699     index(0x4);
3700     scale(0x0);
3701     disp(0x0);
3702   %}
3703 %}
3704 
3705 // Indirect Memory Plus Short Offset Operand
3706 operand indOffset8Narrow(rRegN reg, immL8 off)
3707 %{
3708   predicate(Universe::narrow_oop_shift() == 0);
3709   constraint(ALLOC_IN_RC(ptr_reg));
3710   match(AddP (DecodeN reg) off);
3711 
3712   format %{ "[$reg + $off (8-bit)]" %}
3713   interface(MEMORY_INTER) %{
3714     base($reg);
3715     index(0x4);
3716     scale(0x0);
3717     disp($off);
3718   %}
3719 %}
3720 
3721 // Indirect Memory Plus Long Offset Operand
3722 operand indOffset32Narrow(rRegN reg, immL32 off)
3723 %{
3724   predicate(Universe::narrow_oop_shift() == 0);
3725   constraint(ALLOC_IN_RC(ptr_reg));
3726   match(AddP (DecodeN reg) off);
3727 
3728   format %{ "[$reg + $off (32-bit)]" %}
3729   interface(MEMORY_INTER) %{
3730     base($reg);
3731     index(0x4);
3732     scale(0x0);
3733     disp($off);
3734   %}
3735 %}
3736 
3737 // Indirect Memory Plus Index Register Plus Offset Operand
3738 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3739 %{
3740   predicate(Universe::narrow_oop_shift() == 0);
3741   constraint(ALLOC_IN_RC(ptr_reg));
3742   match(AddP (AddP (DecodeN reg) lreg) off);
3743 
3744   op_cost(10);
3745   format %{"[$reg + $off + $lreg]" %}
3746   interface(MEMORY_INTER) %{
3747     base($reg);
3748     index($lreg);
3749     scale(0x0);
3750     disp($off);
3751   %}
3752 %}
3753 
3754 // Indirect Memory Plus Index Register Plus Offset Operand
3755 operand indIndexNarrow(rRegN reg, rRegL lreg)
3756 %{
3757   predicate(Universe::narrow_oop_shift() == 0);
3758   constraint(ALLOC_IN_RC(ptr_reg));
3759   match(AddP (DecodeN reg) lreg);
3760 
3761   op_cost(10);
3762   format %{"[$reg + $lreg]" %}
3763   interface(MEMORY_INTER) %{
3764     base($reg);
3765     index($lreg);
3766     scale(0x0);
3767     disp(0x0);
3768   %}
3769 %}
3770 
3771 // Indirect Memory Times Scale Plus Index Register
3772 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3773 %{
3774   predicate(Universe::narrow_oop_shift() == 0);
3775   constraint(ALLOC_IN_RC(ptr_reg));
3776   match(AddP (DecodeN reg) (LShiftL lreg scale));
3777 
3778   op_cost(10);
3779   format %{"[$reg + $lreg << $scale]" %}
3780   interface(MEMORY_INTER) %{
3781     base($reg);
3782     index($lreg);
3783     scale($scale);
3784     disp(0x0);
3785   %}
3786 %}
3787 
3788 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3789 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3790 %{
3791   predicate(Universe::narrow_oop_shift() == 0);
3792   constraint(ALLOC_IN_RC(ptr_reg));
3793   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3794 
3795   op_cost(10);
3796   format %{"[$reg + $off + $lreg << $scale]" %}
3797   interface(MEMORY_INTER) %{
3798     base($reg);
3799     index($lreg);
3800     scale($scale);
3801     disp($off);
3802   %}
3803 %}
3804 
3805 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3806 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3807 %{
3808   constraint(ALLOC_IN_RC(ptr_reg));
3809   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3810   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3811 
3812   op_cost(10);
3813   format %{"[$reg + $off + $idx << $scale]" %}
3814   interface(MEMORY_INTER) %{
3815     base($reg);
3816     index($idx);
3817     scale($scale);
3818     disp($off);
3819   %}
3820 %}
3821 
3822 //----------Special Memory Operands--------------------------------------------
3823 // Stack Slot Operand - This operand is used for loading and storing temporary
3824 //                      values on the stack where a match requires a value to
3825 //                      flow through memory.
3826 operand stackSlotP(sRegP reg)
3827 %{
3828   constraint(ALLOC_IN_RC(stack_slots));
3829   // No match rule because this operand is only generated in matching
3830 
3831   format %{ "[$reg]" %}
3832   interface(MEMORY_INTER) %{
3833     base(0x4);   // RSP
3834     index(0x4);  // No Index
3835     scale(0x0);  // No Scale
3836     disp($reg);  // Stack Offset
3837   %}
3838 %}
3839 
3840 operand stackSlotI(sRegI reg)
3841 %{
3842   constraint(ALLOC_IN_RC(stack_slots));
3843   // No match rule because this operand is only generated in matching
3844 
3845   format %{ "[$reg]" %}
3846   interface(MEMORY_INTER) %{
3847     base(0x4);   // RSP
3848     index(0x4);  // No Index
3849     scale(0x0);  // No Scale
3850     disp($reg);  // Stack Offset
3851   %}
3852 %}
3853 
3854 operand stackSlotF(sRegF reg)
3855 %{
3856   constraint(ALLOC_IN_RC(stack_slots));
3857   // No match rule because this operand is only generated in matching
3858 
3859   format %{ "[$reg]" %}
3860   interface(MEMORY_INTER) %{
3861     base(0x4);   // RSP
3862     index(0x4);  // No Index
3863     scale(0x0);  // No Scale
3864     disp($reg);  // Stack Offset
3865   %}
3866 %}
3867 
3868 operand stackSlotD(sRegD reg)
3869 %{
3870   constraint(ALLOC_IN_RC(stack_slots));
3871   // No match rule because this operand is only generated in matching
3872 
3873   format %{ "[$reg]" %}
3874   interface(MEMORY_INTER) %{
3875     base(0x4);   // RSP
3876     index(0x4);  // No Index
3877     scale(0x0);  // No Scale
3878     disp($reg);  // Stack Offset
3879   %}
3880 %}
3881 operand stackSlotL(sRegL reg)
3882 %{
3883   constraint(ALLOC_IN_RC(stack_slots));
3884   // No match rule because this operand is only generated in matching
3885 
3886   format %{ "[$reg]" %}
3887   interface(MEMORY_INTER) %{
3888     base(0x4);   // RSP
3889     index(0x4);  // No Index
3890     scale(0x0);  // No Scale
3891     disp($reg);  // Stack Offset
3892   %}
3893 %}
3894 
3895 //----------Conditional Branch Operands----------------------------------------
3896 // Comparison Op  - This is the operation of the comparison, and is limited to
3897 //                  the following set of codes:
3898 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3899 //
3900 // Other attributes of the comparison, such as unsignedness, are specified
3901 // by the comparison instruction that sets a condition code flags register.
3902 // That result is represented by a flags operand whose subtype is appropriate
3903 // to the unsignedness (etc.) of the comparison.
3904 //
3905 // Later, the instruction which matches both the Comparison Op (a Bool) and
3906 // the flags (produced by the Cmp) specifies the coding of the comparison op
3907 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3908 
3909 // Comparision Code
3910 operand cmpOp()
3911 %{
3912   match(Bool);
3913 
3914   format %{ "" %}
3915   interface(COND_INTER) %{
3916     equal(0x4, "e");
3917     not_equal(0x5, "ne");
3918     less(0xC, "l");
3919     greater_equal(0xD, "ge");
3920     less_equal(0xE, "le");
3921     greater(0xF, "g");
3922     overflow(0x0, "o");
3923     no_overflow(0x1, "no");
3924   %}
3925 %}
3926 
3927 // Comparison Code, unsigned compare.  Used by FP also, with
3928 // C2 (unordered) turned into GT or LT already.  The other bits
3929 // C0 and C3 are turned into Carry & Zero flags.
3930 operand cmpOpU()
3931 %{
3932   match(Bool);
3933 
3934   format %{ "" %}
3935   interface(COND_INTER) %{
3936     equal(0x4, "e");
3937     not_equal(0x5, "ne");
3938     less(0x2, "b");
3939     greater_equal(0x3, "nb");
3940     less_equal(0x6, "be");
3941     greater(0x7, "nbe");
3942     overflow(0x0, "o");
3943     no_overflow(0x1, "no");
3944   %}
3945 %}
3946 
3947 
3948 // Floating comparisons that don't require any fixup for the unordered case
3949 operand cmpOpUCF() %{
3950   match(Bool);
3951   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
3952             n->as_Bool()->_test._test == BoolTest::ge ||
3953             n->as_Bool()->_test._test == BoolTest::le ||
3954             n->as_Bool()->_test._test == BoolTest::gt);
3955   format %{ "" %}
3956   interface(COND_INTER) %{
3957     equal(0x4, "e");
3958     not_equal(0x5, "ne");
3959     less(0x2, "b");
3960     greater_equal(0x3, "nb");
3961     less_equal(0x6, "be");
3962     greater(0x7, "nbe");
3963     overflow(0x0, "o");
3964     no_overflow(0x1, "no");
3965   %}
3966 %}
3967 
3968 
3969 // Floating comparisons that can be fixed up with extra conditional jumps
3970 operand cmpOpUCF2() %{
3971   match(Bool);
3972   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
3973             n->as_Bool()->_test._test == BoolTest::eq);
3974   format %{ "" %}
3975   interface(COND_INTER) %{
3976     equal(0x4, "e");
3977     not_equal(0x5, "ne");
3978     less(0x2, "b");
3979     greater_equal(0x3, "nb");
3980     less_equal(0x6, "be");
3981     greater(0x7, "nbe");
3982     overflow(0x0, "o");
3983     no_overflow(0x1, "no");
3984   %}
3985 %}
3986 
3987 
3988 //----------OPERAND CLASSES----------------------------------------------------
3989 // Operand Classes are groups of operands that are used as to simplify
3990 // instruction definitions by not requiring the AD writer to specify separate
3991 // instructions for every form of operand when the instruction accepts
3992 // multiple operand types with the same basic encoding and format.  The classic
3993 // case of this is memory operands.
3994 
3995 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
3996                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
3997                indCompressedOopOffset,
3998                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
3999                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4000                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
4001 
4002 //----------PIPELINE-----------------------------------------------------------
4003 // Rules which define the behavior of the target architectures pipeline.
4004 pipeline %{
4005 
4006 //----------ATTRIBUTES---------------------------------------------------------
4007 attributes %{
4008   variable_size_instructions;        // Fixed size instructions
4009   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4010   instruction_unit_size = 1;         // An instruction is 1 bytes long
4011   instruction_fetch_unit_size = 16;  // The processor fetches one line
4012   instruction_fetch_units = 1;       // of 16 bytes
4013 
4014   // List of nop instructions
4015   nops( MachNop );
4016 %}
4017 
4018 //----------RESOURCES----------------------------------------------------------
4019 // Resources are the functional units available to the machine
4020 
4021 // Generic P2/P3 pipeline
4022 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4023 // 3 instructions decoded per cycle.
4024 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4025 // 3 ALU op, only ALU0 handles mul instructions.
4026 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4027            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4028            BR, FPU,
4029            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4030 
4031 //----------PIPELINE DESCRIPTION-----------------------------------------------
4032 // Pipeline Description specifies the stages in the machine's pipeline
4033 
4034 // Generic P2/P3 pipeline
4035 pipe_desc(S0, S1, S2, S3, S4, S5);
4036 
4037 //----------PIPELINE CLASSES---------------------------------------------------
4038 // Pipeline Classes describe the stages in which input and output are
4039 // referenced by the hardware pipeline.
4040 
4041 // Naming convention: ialu or fpu
4042 // Then: _reg
4043 // Then: _reg if there is a 2nd register
4044 // Then: _long if it's a pair of instructions implementing a long
4045 // Then: _fat if it requires the big decoder
4046 //   Or: _mem if it requires the big decoder and a memory unit.
4047 
4048 // Integer ALU reg operation
4049 pipe_class ialu_reg(rRegI dst)
4050 %{
4051     single_instruction;
4052     dst    : S4(write);
4053     dst    : S3(read);
4054     DECODE : S0;        // any decoder
4055     ALU    : S3;        // any alu
4056 %}
4057 
4058 // Long ALU reg operation
4059 pipe_class ialu_reg_long(rRegL dst)
4060 %{
4061     instruction_count(2);
4062     dst    : S4(write);
4063     dst    : S3(read);
4064     DECODE : S0(2);     // any 2 decoders
4065     ALU    : S3(2);     // both alus
4066 %}
4067 
4068 // Integer ALU reg operation using big decoder
4069 pipe_class ialu_reg_fat(rRegI dst)
4070 %{
4071     single_instruction;
4072     dst    : S4(write);
4073     dst    : S3(read);
4074     D0     : S0;        // big decoder only
4075     ALU    : S3;        // any alu
4076 %}
4077 
4078 // Long ALU reg operation using big decoder
4079 pipe_class ialu_reg_long_fat(rRegL dst)
4080 %{
4081     instruction_count(2);
4082     dst    : S4(write);
4083     dst    : S3(read);
4084     D0     : S0(2);     // big decoder only; twice
4085     ALU    : S3(2);     // any 2 alus
4086 %}
4087 
4088 // Integer ALU reg-reg operation
4089 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4090 %{
4091     single_instruction;
4092     dst    : S4(write);
4093     src    : S3(read);
4094     DECODE : S0;        // any decoder
4095     ALU    : S3;        // any alu
4096 %}
4097 
4098 // Long ALU reg-reg operation
4099 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4100 %{
4101     instruction_count(2);
4102     dst    : S4(write);
4103     src    : S3(read);
4104     DECODE : S0(2);     // any 2 decoders
4105     ALU    : S3(2);     // both alus
4106 %}
4107 
4108 // Integer ALU reg-reg operation
4109 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4110 %{
4111     single_instruction;
4112     dst    : S4(write);
4113     src    : S3(read);
4114     D0     : S0;        // big decoder only
4115     ALU    : S3;        // any alu
4116 %}
4117 
4118 // Long ALU reg-reg operation
4119 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4120 %{
4121     instruction_count(2);
4122     dst    : S4(write);
4123     src    : S3(read);
4124     D0     : S0(2);     // big decoder only; twice
4125     ALU    : S3(2);     // both alus
4126 %}
4127 
4128 // Integer ALU reg-mem operation
4129 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4130 %{
4131     single_instruction;
4132     dst    : S5(write);
4133     mem    : S3(read);
4134     D0     : S0;        // big decoder only
4135     ALU    : S4;        // any alu
4136     MEM    : S3;        // any mem
4137 %}
4138 
4139 // Integer mem operation (prefetch)
4140 pipe_class ialu_mem(memory mem)
4141 %{
4142     single_instruction;
4143     mem    : S3(read);
4144     D0     : S0;        // big decoder only
4145     MEM    : S3;        // any mem
4146 %}
4147 
4148 // Integer Store to Memory
4149 pipe_class ialu_mem_reg(memory mem, rRegI src)
4150 %{
4151     single_instruction;
4152     mem    : S3(read);
4153     src    : S5(read);
4154     D0     : S0;        // big decoder only
4155     ALU    : S4;        // any alu
4156     MEM    : S3;
4157 %}
4158 
4159 // // Long Store to Memory
4160 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4161 // %{
4162 //     instruction_count(2);
4163 //     mem    : S3(read);
4164 //     src    : S5(read);
4165 //     D0     : S0(2);          // big decoder only; twice
4166 //     ALU    : S4(2);     // any 2 alus
4167 //     MEM    : S3(2);  // Both mems
4168 // %}
4169 
4170 // Integer Store to Memory
4171 pipe_class ialu_mem_imm(memory mem)
4172 %{
4173     single_instruction;
4174     mem    : S3(read);
4175     D0     : S0;        // big decoder only
4176     ALU    : S4;        // any alu
4177     MEM    : S3;
4178 %}
4179 
4180 // Integer ALU0 reg-reg operation
4181 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4182 %{
4183     single_instruction;
4184     dst    : S4(write);
4185     src    : S3(read);
4186     D0     : S0;        // Big decoder only
4187     ALU0   : S3;        // only alu0
4188 %}
4189 
4190 // Integer ALU0 reg-mem operation
4191 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4192 %{
4193     single_instruction;
4194     dst    : S5(write);
4195     mem    : S3(read);
4196     D0     : S0;        // big decoder only
4197     ALU0   : S4;        // ALU0 only
4198     MEM    : S3;        // any mem
4199 %}
4200 
4201 // Integer ALU reg-reg operation
4202 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4203 %{
4204     single_instruction;
4205     cr     : S4(write);
4206     src1   : S3(read);
4207     src2   : S3(read);
4208     DECODE : S0;        // any decoder
4209     ALU    : S3;        // any alu
4210 %}
4211 
4212 // Integer ALU reg-imm operation
4213 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4214 %{
4215     single_instruction;
4216     cr     : S4(write);
4217     src1   : S3(read);
4218     DECODE : S0;        // any decoder
4219     ALU    : S3;        // any alu
4220 %}
4221 
4222 // Integer ALU reg-mem operation
4223 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4224 %{
4225     single_instruction;
4226     cr     : S4(write);
4227     src1   : S3(read);
4228     src2   : S3(read);
4229     D0     : S0;        // big decoder only
4230     ALU    : S4;        // any alu
4231     MEM    : S3;
4232 %}
4233 
4234 // Conditional move reg-reg
4235 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4236 %{
4237     instruction_count(4);
4238     y      : S4(read);
4239     q      : S3(read);
4240     p      : S3(read);
4241     DECODE : S0(4);     // any decoder
4242 %}
4243 
4244 // Conditional move reg-reg
4245 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4246 %{
4247     single_instruction;
4248     dst    : S4(write);
4249     src    : S3(read);
4250     cr     : S3(read);
4251     DECODE : S0;        // any decoder
4252 %}
4253 
4254 // Conditional move reg-mem
4255 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4256 %{
4257     single_instruction;
4258     dst    : S4(write);
4259     src    : S3(read);
4260     cr     : S3(read);
4261     DECODE : S0;        // any decoder
4262     MEM    : S3;
4263 %}
4264 
4265 // Conditional move reg-reg long
4266 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4267 %{
4268     single_instruction;
4269     dst    : S4(write);
4270     src    : S3(read);
4271     cr     : S3(read);
4272     DECODE : S0(2);     // any 2 decoders
4273 %}
4274 
4275 // XXX
4276 // // Conditional move double reg-reg
4277 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4278 // %{
4279 //     single_instruction;
4280 //     dst    : S4(write);
4281 //     src    : S3(read);
4282 //     cr     : S3(read);
4283 //     DECODE : S0;     // any decoder
4284 // %}
4285 
4286 // Float reg-reg operation
4287 pipe_class fpu_reg(regD dst)
4288 %{
4289     instruction_count(2);
4290     dst    : S3(read);
4291     DECODE : S0(2);     // any 2 decoders
4292     FPU    : S3;
4293 %}
4294 
4295 // Float reg-reg operation
4296 pipe_class fpu_reg_reg(regD dst, regD src)
4297 %{
4298     instruction_count(2);
4299     dst    : S4(write);
4300     src    : S3(read);
4301     DECODE : S0(2);     // any 2 decoders
4302     FPU    : S3;
4303 %}
4304 
4305 // Float reg-reg operation
4306 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4307 %{
4308     instruction_count(3);
4309     dst    : S4(write);
4310     src1   : S3(read);
4311     src2   : S3(read);
4312     DECODE : S0(3);     // any 3 decoders
4313     FPU    : S3(2);
4314 %}
4315 
4316 // Float reg-reg operation
4317 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4318 %{
4319     instruction_count(4);
4320     dst    : S4(write);
4321     src1   : S3(read);
4322     src2   : S3(read);
4323     src3   : S3(read);
4324     DECODE : S0(4);     // any 3 decoders
4325     FPU    : S3(2);
4326 %}
4327 
4328 // Float reg-reg operation
4329 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4330 %{
4331     instruction_count(4);
4332     dst    : S4(write);
4333     src1   : S3(read);
4334     src2   : S3(read);
4335     src3   : S3(read);
4336     DECODE : S1(3);     // any 3 decoders
4337     D0     : S0;        // Big decoder only
4338     FPU    : S3(2);
4339     MEM    : S3;
4340 %}
4341 
4342 // Float reg-mem operation
4343 pipe_class fpu_reg_mem(regD dst, memory mem)
4344 %{
4345     instruction_count(2);
4346     dst    : S5(write);
4347     mem    : S3(read);
4348     D0     : S0;        // big decoder only
4349     DECODE : S1;        // any decoder for FPU POP
4350     FPU    : S4;
4351     MEM    : S3;        // any mem
4352 %}
4353 
4354 // Float reg-mem operation
4355 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4356 %{
4357     instruction_count(3);
4358     dst    : S5(write);
4359     src1   : S3(read);
4360     mem    : S3(read);
4361     D0     : S0;        // big decoder only
4362     DECODE : S1(2);     // any decoder for FPU POP
4363     FPU    : S4;
4364     MEM    : S3;        // any mem
4365 %}
4366 
4367 // Float mem-reg operation
4368 pipe_class fpu_mem_reg(memory mem, regD src)
4369 %{
4370     instruction_count(2);
4371     src    : S5(read);
4372     mem    : S3(read);
4373     DECODE : S0;        // any decoder for FPU PUSH
4374     D0     : S1;        // big decoder only
4375     FPU    : S4;
4376     MEM    : S3;        // any mem
4377 %}
4378 
4379 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4380 %{
4381     instruction_count(3);
4382     src1   : S3(read);
4383     src2   : S3(read);
4384     mem    : S3(read);
4385     DECODE : S0(2);     // any decoder for FPU PUSH
4386     D0     : S1;        // big decoder only
4387     FPU    : S4;
4388     MEM    : S3;        // any mem
4389 %}
4390 
4391 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4392 %{
4393     instruction_count(3);
4394     src1   : S3(read);
4395     src2   : S3(read);
4396     mem    : S4(read);
4397     DECODE : S0;        // any decoder for FPU PUSH
4398     D0     : S0(2);     // big decoder only
4399     FPU    : S4;
4400     MEM    : S3(2);     // any mem
4401 %}
4402 
4403 pipe_class fpu_mem_mem(memory dst, memory src1)
4404 %{
4405     instruction_count(2);
4406     src1   : S3(read);
4407     dst    : S4(read);
4408     D0     : S0(2);     // big decoder only
4409     MEM    : S3(2);     // any mem
4410 %}
4411 
4412 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4413 %{
4414     instruction_count(3);
4415     src1   : S3(read);
4416     src2   : S3(read);
4417     dst    : S4(read);
4418     D0     : S0(3);     // big decoder only
4419     FPU    : S4;
4420     MEM    : S3(3);     // any mem
4421 %}
4422 
4423 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4424 %{
4425     instruction_count(3);
4426     src1   : S4(read);
4427     mem    : S4(read);
4428     DECODE : S0;        // any decoder for FPU PUSH
4429     D0     : S0(2);     // big decoder only
4430     FPU    : S4;
4431     MEM    : S3(2);     // any mem
4432 %}
4433 
4434 // Float load constant
4435 pipe_class fpu_reg_con(regD dst)
4436 %{
4437     instruction_count(2);
4438     dst    : S5(write);
4439     D0     : S0;        // big decoder only for the load
4440     DECODE : S1;        // any decoder for FPU POP
4441     FPU    : S4;
4442     MEM    : S3;        // any mem
4443 %}
4444 
4445 // Float load constant
4446 pipe_class fpu_reg_reg_con(regD dst, regD src)
4447 %{
4448     instruction_count(3);
4449     dst    : S5(write);
4450     src    : S3(read);
4451     D0     : S0;        // big decoder only for the load
4452     DECODE : S1(2);     // any decoder for FPU POP
4453     FPU    : S4;
4454     MEM    : S3;        // any mem
4455 %}
4456 
4457 // UnConditional branch
4458 pipe_class pipe_jmp(label labl)
4459 %{
4460     single_instruction;
4461     BR   : S3;
4462 %}
4463 
4464 // Conditional branch
4465 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4466 %{
4467     single_instruction;
4468     cr    : S1(read);
4469     BR    : S3;
4470 %}
4471 
4472 // Allocation idiom
4473 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4474 %{
4475     instruction_count(1); force_serialization;
4476     fixed_latency(6);
4477     heap_ptr : S3(read);
4478     DECODE   : S0(3);
4479     D0       : S2;
4480     MEM      : S3;
4481     ALU      : S3(2);
4482     dst      : S5(write);
4483     BR       : S5;
4484 %}
4485 
4486 // Generic big/slow expanded idiom
4487 pipe_class pipe_slow()
4488 %{
4489     instruction_count(10); multiple_bundles; force_serialization;
4490     fixed_latency(100);
4491     D0  : S0(2);
4492     MEM : S3(2);
4493 %}
4494 
4495 // The real do-nothing guy
4496 pipe_class empty()
4497 %{
4498     instruction_count(0);
4499 %}
4500 
4501 // Define the class for the Nop node
4502 define
4503 %{
4504    MachNop = empty;
4505 %}
4506 
4507 %}
4508 
4509 //----------INSTRUCTIONS-------------------------------------------------------
4510 //
4511 // match      -- States which machine-independent subtree may be replaced
4512 //               by this instruction.
4513 // ins_cost   -- The estimated cost of this instruction is used by instruction
4514 //               selection to identify a minimum cost tree of machine
4515 //               instructions that matches a tree of machine-independent
4516 //               instructions.
4517 // format     -- A string providing the disassembly for this instruction.
4518 //               The value of an instruction's operand may be inserted
4519 //               by referring to it with a '$' prefix.
4520 // opcode     -- Three instruction opcodes may be provided.  These are referred
4521 //               to within an encode class as $primary, $secondary, and $tertiary
4522 //               rrspectively.  The primary opcode is commonly used to
4523 //               indicate the type of machine instruction, while secondary
4524 //               and tertiary are often used for prefix options or addressing
4525 //               modes.
4526 // ins_encode -- A list of encode classes with parameters. The encode class
4527 //               name must have been defined in an 'enc_class' specification
4528 //               in the encode section of the architecture description.
4529 
4530 
4531 //----------Load/Store/Move Instructions---------------------------------------
4532 //----------Load Instructions--------------------------------------------------
4533 
4534 // Load Byte (8 bit signed)
4535 instruct loadB(rRegI dst, memory mem)
4536 %{
4537   match(Set dst (LoadB mem));
4538 
4539   ins_cost(125);
4540   format %{ "movsbl  $dst, $mem\t# byte" %}
4541 
4542   ins_encode %{
4543     __ movsbl($dst$$Register, $mem$$Address);
4544   %}
4545 
4546   ins_pipe(ialu_reg_mem);
4547 %}
4548 
4549 // Load Byte (8 bit signed) into Long Register
4550 instruct loadB2L(rRegL dst, memory mem)
4551 %{
4552   match(Set dst (ConvI2L (LoadB mem)));
4553 
4554   ins_cost(125);
4555   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4556 
4557   ins_encode %{
4558     __ movsbq($dst$$Register, $mem$$Address);
4559   %}
4560 
4561   ins_pipe(ialu_reg_mem);
4562 %}
4563 
4564 // Load Unsigned Byte (8 bit UNsigned)
4565 instruct loadUB(rRegI dst, memory mem)
4566 %{
4567   match(Set dst (LoadUB mem));
4568 
4569   ins_cost(125);
4570   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4571 
4572   ins_encode %{
4573     __ movzbl($dst$$Register, $mem$$Address);
4574   %}
4575 
4576   ins_pipe(ialu_reg_mem);
4577 %}
4578 
4579 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4580 instruct loadUB2L(rRegL dst, memory mem)
4581 %{
4582   match(Set dst (ConvI2L (LoadUB mem)));
4583 
4584   ins_cost(125);
4585   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4586 
4587   ins_encode %{
4588     __ movzbq($dst$$Register, $mem$$Address);
4589   %}
4590 
4591   ins_pipe(ialu_reg_mem);
4592 %}
4593 
4594 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4595 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4596   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4597   effect(KILL cr);
4598 
4599   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4600             "andl    $dst, $mask" %}
4601   ins_encode %{
4602     Register Rdst = $dst$$Register;
4603     __ movzbq(Rdst, $mem$$Address);
4604     __ andl(Rdst, $mask$$constant);
4605   %}
4606   ins_pipe(ialu_reg_mem);
4607 %}
4608 
4609 // Load Short (16 bit signed)
4610 instruct loadS(rRegI dst, memory mem)
4611 %{
4612   match(Set dst (LoadS mem));
4613 
4614   ins_cost(125);
4615   format %{ "movswl $dst, $mem\t# short" %}
4616 
4617   ins_encode %{
4618     __ movswl($dst$$Register, $mem$$Address);
4619   %}
4620 
4621   ins_pipe(ialu_reg_mem);
4622 %}
4623 
4624 // Load Short (16 bit signed) to Byte (8 bit signed)
4625 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4626   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4627 
4628   ins_cost(125);
4629   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4630   ins_encode %{
4631     __ movsbl($dst$$Register, $mem$$Address);
4632   %}
4633   ins_pipe(ialu_reg_mem);
4634 %}
4635 
4636 // Load Short (16 bit signed) into Long Register
4637 instruct loadS2L(rRegL dst, memory mem)
4638 %{
4639   match(Set dst (ConvI2L (LoadS mem)));
4640 
4641   ins_cost(125);
4642   format %{ "movswq $dst, $mem\t# short -> long" %}
4643 
4644   ins_encode %{
4645     __ movswq($dst$$Register, $mem$$Address);
4646   %}
4647 
4648   ins_pipe(ialu_reg_mem);
4649 %}
4650 
4651 // Load Unsigned Short/Char (16 bit UNsigned)
4652 instruct loadUS(rRegI dst, memory mem)
4653 %{
4654   match(Set dst (LoadUS mem));
4655 
4656   ins_cost(125);
4657   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4658 
4659   ins_encode %{
4660     __ movzwl($dst$$Register, $mem$$Address);
4661   %}
4662 
4663   ins_pipe(ialu_reg_mem);
4664 %}
4665 
4666 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4667 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4668   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4669 
4670   ins_cost(125);
4671   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4672   ins_encode %{
4673     __ movsbl($dst$$Register, $mem$$Address);
4674   %}
4675   ins_pipe(ialu_reg_mem);
4676 %}
4677 
4678 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4679 instruct loadUS2L(rRegL dst, memory mem)
4680 %{
4681   match(Set dst (ConvI2L (LoadUS mem)));
4682 
4683   ins_cost(125);
4684   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4685 
4686   ins_encode %{
4687     __ movzwq($dst$$Register, $mem$$Address);
4688   %}
4689 
4690   ins_pipe(ialu_reg_mem);
4691 %}
4692 
4693 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4694 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4695   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4696 
4697   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4698   ins_encode %{
4699     __ movzbq($dst$$Register, $mem$$Address);
4700   %}
4701   ins_pipe(ialu_reg_mem);
4702 %}
4703 
4704 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4705 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4706   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4707   effect(KILL cr);
4708 
4709   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4710             "andl    $dst, $mask" %}
4711   ins_encode %{
4712     Register Rdst = $dst$$Register;
4713     __ movzwq(Rdst, $mem$$Address);
4714     __ andl(Rdst, $mask$$constant);
4715   %}
4716   ins_pipe(ialu_reg_mem);
4717 %}
4718 
4719 // Load Integer
4720 instruct loadI(rRegI dst, memory mem)
4721 %{
4722   match(Set dst (LoadI mem));
4723 
4724   ins_cost(125);
4725   format %{ "movl    $dst, $mem\t# int" %}
4726 
4727   ins_encode %{
4728     __ movl($dst$$Register, $mem$$Address);
4729   %}
4730 
4731   ins_pipe(ialu_reg_mem);
4732 %}
4733 
4734 // Load Integer (32 bit signed) to Byte (8 bit signed)
4735 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4736   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4737 
4738   ins_cost(125);
4739   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4740   ins_encode %{
4741     __ movsbl($dst$$Register, $mem$$Address);
4742   %}
4743   ins_pipe(ialu_reg_mem);
4744 %}
4745 
4746 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4747 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4748   match(Set dst (AndI (LoadI mem) mask));
4749 
4750   ins_cost(125);
4751   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4752   ins_encode %{
4753     __ movzbl($dst$$Register, $mem$$Address);
4754   %}
4755   ins_pipe(ialu_reg_mem);
4756 %}
4757 
4758 // Load Integer (32 bit signed) to Short (16 bit signed)
4759 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4760   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4761 
4762   ins_cost(125);
4763   format %{ "movswl  $dst, $mem\t# int -> short" %}
4764   ins_encode %{
4765     __ movswl($dst$$Register, $mem$$Address);
4766   %}
4767   ins_pipe(ialu_reg_mem);
4768 %}
4769 
4770 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4771 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4772   match(Set dst (AndI (LoadI mem) mask));
4773 
4774   ins_cost(125);
4775   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4776   ins_encode %{
4777     __ movzwl($dst$$Register, $mem$$Address);
4778   %}
4779   ins_pipe(ialu_reg_mem);
4780 %}
4781 
4782 // Load Integer into Long Register
4783 instruct loadI2L(rRegL dst, memory mem)
4784 %{
4785   match(Set dst (ConvI2L (LoadI mem)));
4786 
4787   ins_cost(125);
4788   format %{ "movslq  $dst, $mem\t# int -> long" %}
4789 
4790   ins_encode %{
4791     __ movslq($dst$$Register, $mem$$Address);
4792   %}
4793 
4794   ins_pipe(ialu_reg_mem);
4795 %}
4796 
4797 // Load Integer with mask 0xFF into Long Register
4798 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4799   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4800 
4801   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4802   ins_encode %{
4803     __ movzbq($dst$$Register, $mem$$Address);
4804   %}
4805   ins_pipe(ialu_reg_mem);
4806 %}
4807 
4808 // Load Integer with mask 0xFFFF into Long Register
4809 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4810   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4811 
4812   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4813   ins_encode %{
4814     __ movzwq($dst$$Register, $mem$$Address);
4815   %}
4816   ins_pipe(ialu_reg_mem);
4817 %}
4818 
4819 // Load Integer with a 31-bit mask into Long Register
4820 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4821   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4822   effect(KILL cr);
4823 
4824   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4825             "andl    $dst, $mask" %}
4826   ins_encode %{
4827     Register Rdst = $dst$$Register;
4828     __ movl(Rdst, $mem$$Address);
4829     __ andl(Rdst, $mask$$constant);
4830   %}
4831   ins_pipe(ialu_reg_mem);
4832 %}
4833 
4834 // Load Unsigned Integer into Long Register
4835 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4836 %{
4837   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4838 
4839   ins_cost(125);
4840   format %{ "movl    $dst, $mem\t# uint -> long" %}
4841 
4842   ins_encode %{
4843     __ movl($dst$$Register, $mem$$Address);
4844   %}
4845 
4846   ins_pipe(ialu_reg_mem);
4847 %}
4848 
4849 // Load Long
4850 instruct loadL(rRegL dst, memory mem)
4851 %{
4852   match(Set dst (LoadL mem));
4853 
4854   ins_cost(125);
4855   format %{ "movq    $dst, $mem\t# long" %}
4856 
4857   ins_encode %{
4858     __ movq($dst$$Register, $mem$$Address);
4859   %}
4860 
4861   ins_pipe(ialu_reg_mem); // XXX
4862 %}
4863 
4864 // Load Range
4865 instruct loadRange(rRegI dst, memory mem)
4866 %{
4867   match(Set dst (LoadRange mem));
4868 
4869   ins_cost(125); // XXX
4870   format %{ "movl    $dst, $mem\t# range" %}
4871   opcode(0x8B);
4872   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4873   ins_pipe(ialu_reg_mem);
4874 %}
4875 
4876 // Load Pointer
4877 instruct loadP(rRegP dst, memory mem)
4878 %{
4879   match(Set dst (LoadP mem));
4880 
4881   ins_cost(125); // XXX
4882   format %{ "movq    $dst, $mem\t# ptr" %}
4883   opcode(0x8B);
4884   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4885   ins_pipe(ialu_reg_mem); // XXX
4886 %}
4887 
4888 // Load Compressed Pointer
4889 instruct loadN(rRegN dst, memory mem)
4890 %{
4891    match(Set dst (LoadN mem));
4892 
4893    ins_cost(125); // XXX
4894    format %{ "movl    $dst, $mem\t# compressed ptr" %}
4895    ins_encode %{
4896      __ movl($dst$$Register, $mem$$Address);
4897    %}
4898    ins_pipe(ialu_reg_mem); // XXX
4899 %}
4900 
4901 
4902 // Load Klass Pointer
4903 instruct loadKlass(rRegP dst, memory mem)
4904 %{
4905   match(Set dst (LoadKlass mem));
4906 
4907   ins_cost(125); // XXX
4908   format %{ "movq    $dst, $mem\t# class" %}
4909   opcode(0x8B);
4910   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4911   ins_pipe(ialu_reg_mem); // XXX
4912 %}
4913 
4914 // Load narrow Klass Pointer
4915 instruct loadNKlass(rRegN dst, memory mem)
4916 %{
4917   match(Set dst (LoadNKlass mem));
4918 
4919   ins_cost(125); // XXX
4920   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
4921   ins_encode %{
4922     __ movl($dst$$Register, $mem$$Address);
4923   %}
4924   ins_pipe(ialu_reg_mem); // XXX
4925 %}
4926 
4927 // Load Float
4928 instruct loadF(regF dst, memory mem)
4929 %{
4930   match(Set dst (LoadF mem));
4931 
4932   ins_cost(145); // XXX
4933   format %{ "movss   $dst, $mem\t# float" %}
4934   ins_encode %{
4935     __ movflt($dst$$XMMRegister, $mem$$Address);
4936   %}
4937   ins_pipe(pipe_slow); // XXX
4938 %}
4939 
4940 // Load Double
4941 instruct loadD_partial(regD dst, memory mem)
4942 %{
4943   predicate(!UseXmmLoadAndClearUpper);
4944   match(Set dst (LoadD mem));
4945 
4946   ins_cost(145); // XXX
4947   format %{ "movlpd  $dst, $mem\t# double" %}
4948   ins_encode %{
4949     __ movdbl($dst$$XMMRegister, $mem$$Address);
4950   %}
4951   ins_pipe(pipe_slow); // XXX
4952 %}
4953 
4954 instruct loadD(regD dst, memory mem)
4955 %{
4956   predicate(UseXmmLoadAndClearUpper);
4957   match(Set dst (LoadD mem));
4958 
4959   ins_cost(145); // XXX
4960   format %{ "movsd   $dst, $mem\t# double" %}
4961   ins_encode %{
4962     __ movdbl($dst$$XMMRegister, $mem$$Address);
4963   %}
4964   ins_pipe(pipe_slow); // XXX
4965 %}
4966 
4967 // Load Effective Address
4968 instruct leaP8(rRegP dst, indOffset8 mem)
4969 %{
4970   match(Set dst mem);
4971 
4972   ins_cost(110); // XXX
4973   format %{ "leaq    $dst, $mem\t# ptr 8" %}
4974   opcode(0x8D);
4975   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4976   ins_pipe(ialu_reg_reg_fat);
4977 %}
4978 
4979 instruct leaP32(rRegP dst, indOffset32 mem)
4980 %{
4981   match(Set dst mem);
4982 
4983   ins_cost(110);
4984   format %{ "leaq    $dst, $mem\t# ptr 32" %}
4985   opcode(0x8D);
4986   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4987   ins_pipe(ialu_reg_reg_fat);
4988 %}
4989 
4990 // instruct leaPIdx(rRegP dst, indIndex mem)
4991 // %{
4992 //   match(Set dst mem);
4993 
4994 //   ins_cost(110);
4995 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
4996 //   opcode(0x8D);
4997 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4998 //   ins_pipe(ialu_reg_reg_fat);
4999 // %}
5000 
5001 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5002 %{
5003   match(Set dst mem);
5004 
5005   ins_cost(110);
5006   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5007   opcode(0x8D);
5008   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5009   ins_pipe(ialu_reg_reg_fat);
5010 %}
5011 
5012 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5013 %{
5014   match(Set dst mem);
5015 
5016   ins_cost(110);
5017   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5018   opcode(0x8D);
5019   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5020   ins_pipe(ialu_reg_reg_fat);
5021 %}
5022 
5023 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5024 %{
5025   match(Set dst mem);
5026 
5027   ins_cost(110);
5028   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5029   opcode(0x8D);
5030   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5031   ins_pipe(ialu_reg_reg_fat);
5032 %}
5033 
5034 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5035 %{
5036   match(Set dst mem);
5037 
5038   ins_cost(110);
5039   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5040   opcode(0x8D);
5041   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5042   ins_pipe(ialu_reg_reg_fat);
5043 %}
5044 
5045 // Load Effective Address which uses Narrow (32-bits) oop
5046 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5047 %{
5048   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5049   match(Set dst mem);
5050 
5051   ins_cost(110);
5052   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5053   opcode(0x8D);
5054   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5055   ins_pipe(ialu_reg_reg_fat);
5056 %}
5057 
5058 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5059 %{
5060   predicate(Universe::narrow_oop_shift() == 0);
5061   match(Set dst mem);
5062 
5063   ins_cost(110); // XXX
5064   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5065   opcode(0x8D);
5066   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5067   ins_pipe(ialu_reg_reg_fat);
5068 %}
5069 
5070 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5071 %{
5072   predicate(Universe::narrow_oop_shift() == 0);
5073   match(Set dst mem);
5074 
5075   ins_cost(110);
5076   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5077   opcode(0x8D);
5078   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5079   ins_pipe(ialu_reg_reg_fat);
5080 %}
5081 
5082 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5083 %{
5084   predicate(Universe::narrow_oop_shift() == 0);
5085   match(Set dst mem);
5086 
5087   ins_cost(110);
5088   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5089   opcode(0x8D);
5090   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5091   ins_pipe(ialu_reg_reg_fat);
5092 %}
5093 
5094 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5095 %{
5096   predicate(Universe::narrow_oop_shift() == 0);
5097   match(Set dst mem);
5098 
5099   ins_cost(110);
5100   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5101   opcode(0x8D);
5102   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5103   ins_pipe(ialu_reg_reg_fat);
5104 %}
5105 
5106 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5107 %{
5108   predicate(Universe::narrow_oop_shift() == 0);
5109   match(Set dst mem);
5110 
5111   ins_cost(110);
5112   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5113   opcode(0x8D);
5114   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5115   ins_pipe(ialu_reg_reg_fat);
5116 %}
5117 
5118 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5119 %{
5120   predicate(Universe::narrow_oop_shift() == 0);
5121   match(Set dst mem);
5122 
5123   ins_cost(110);
5124   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5125   opcode(0x8D);
5126   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5127   ins_pipe(ialu_reg_reg_fat);
5128 %}
5129 
5130 instruct loadConI(rRegI dst, immI src)
5131 %{
5132   match(Set dst src);
5133 
5134   format %{ "movl    $dst, $src\t# int" %}
5135   ins_encode(load_immI(dst, src));
5136   ins_pipe(ialu_reg_fat); // XXX
5137 %}
5138 
5139 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5140 %{
5141   match(Set dst src);
5142   effect(KILL cr);
5143 
5144   ins_cost(50);
5145   format %{ "xorl    $dst, $dst\t# int" %}
5146   opcode(0x33); /* + rd */
5147   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5148   ins_pipe(ialu_reg);
5149 %}
5150 
5151 instruct loadConL(rRegL dst, immL src)
5152 %{
5153   match(Set dst src);
5154 
5155   ins_cost(150);
5156   format %{ "movq    $dst, $src\t# long" %}
5157   ins_encode(load_immL(dst, src));
5158   ins_pipe(ialu_reg);
5159 %}
5160 
5161 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5162 %{
5163   match(Set dst src);
5164   effect(KILL cr);
5165 
5166   ins_cost(50);
5167   format %{ "xorl    $dst, $dst\t# long" %}
5168   opcode(0x33); /* + rd */
5169   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5170   ins_pipe(ialu_reg); // XXX
5171 %}
5172 
5173 instruct loadConUL32(rRegL dst, immUL32 src)
5174 %{
5175   match(Set dst src);
5176 
5177   ins_cost(60);
5178   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5179   ins_encode(load_immUL32(dst, src));
5180   ins_pipe(ialu_reg);
5181 %}
5182 
5183 instruct loadConL32(rRegL dst, immL32 src)
5184 %{
5185   match(Set dst src);
5186 
5187   ins_cost(70);
5188   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5189   ins_encode(load_immL32(dst, src));
5190   ins_pipe(ialu_reg);
5191 %}
5192 
5193 instruct loadConP(rRegP dst, immP con) %{
5194   match(Set dst con);
5195 
5196   format %{ "movq    $dst, $con\t# ptr" %}
5197   ins_encode(load_immP(dst, con));
5198   ins_pipe(ialu_reg_fat); // XXX
5199 %}
5200 
5201 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5202 %{
5203   match(Set dst src);
5204   effect(KILL cr);
5205 
5206   ins_cost(50);
5207   format %{ "xorl    $dst, $dst\t# ptr" %}
5208   opcode(0x33); /* + rd */
5209   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5210   ins_pipe(ialu_reg);
5211 %}
5212 
5213 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5214 %{
5215   match(Set dst src);
5216   effect(KILL cr);
5217 
5218   ins_cost(60);
5219   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5220   ins_encode(load_immP31(dst, src));
5221   ins_pipe(ialu_reg);
5222 %}
5223 
5224 instruct loadConF(regF dst, immF con) %{
5225   match(Set dst con);
5226   ins_cost(125);
5227   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5228   ins_encode %{
5229     __ movflt($dst$$XMMRegister, $constantaddress($con));
5230   %}
5231   ins_pipe(pipe_slow);
5232 %}
5233 
5234 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5235   match(Set dst src);
5236   effect(KILL cr);
5237   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5238   ins_encode %{
5239     __ xorq($dst$$Register, $dst$$Register);
5240   %}
5241   ins_pipe(ialu_reg);
5242 %}
5243 
5244 instruct loadConN(rRegN dst, immN src) %{
5245   match(Set dst src);
5246 
5247   ins_cost(125);
5248   format %{ "movl    $dst, $src\t# compressed ptr" %}
5249   ins_encode %{
5250     address con = (address)$src$$constant;
5251     if (con == NULL) {
5252       ShouldNotReachHere();
5253     } else {
5254       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5255     }
5256   %}
5257   ins_pipe(ialu_reg_fat); // XXX
5258 %}
5259 
5260 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5261   match(Set dst src);
5262 
5263   ins_cost(125);
5264   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5265   ins_encode %{
5266     address con = (address)$src$$constant;
5267     if (con == NULL) {
5268       ShouldNotReachHere();
5269     } else {
5270       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5271     }
5272   %}
5273   ins_pipe(ialu_reg_fat); // XXX
5274 %}
5275 
5276 instruct loadConF0(regF dst, immF0 src)
5277 %{
5278   match(Set dst src);
5279   ins_cost(100);
5280 
5281   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5282   ins_encode %{
5283     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5284   %}
5285   ins_pipe(pipe_slow);
5286 %}
5287 
5288 // Use the same format since predicate() can not be used here.
5289 instruct loadConD(regD dst, immD con) %{
5290   match(Set dst con);
5291   ins_cost(125);
5292   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5293   ins_encode %{
5294     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5295   %}
5296   ins_pipe(pipe_slow);
5297 %}
5298 
5299 instruct loadConD0(regD dst, immD0 src)
5300 %{
5301   match(Set dst src);
5302   ins_cost(100);
5303 
5304   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5305   ins_encode %{
5306     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5307   %}
5308   ins_pipe(pipe_slow);
5309 %}
5310 
5311 instruct loadSSI(rRegI dst, stackSlotI src)
5312 %{
5313   match(Set dst src);
5314 
5315   ins_cost(125);
5316   format %{ "movl    $dst, $src\t# int stk" %}
5317   opcode(0x8B);
5318   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5319   ins_pipe(ialu_reg_mem);
5320 %}
5321 
5322 instruct loadSSL(rRegL dst, stackSlotL src)
5323 %{
5324   match(Set dst src);
5325 
5326   ins_cost(125);
5327   format %{ "movq    $dst, $src\t# long stk" %}
5328   opcode(0x8B);
5329   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5330   ins_pipe(ialu_reg_mem);
5331 %}
5332 
5333 instruct loadSSP(rRegP dst, stackSlotP src)
5334 %{
5335   match(Set dst src);
5336 
5337   ins_cost(125);
5338   format %{ "movq    $dst, $src\t# ptr stk" %}
5339   opcode(0x8B);
5340   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5341   ins_pipe(ialu_reg_mem);
5342 %}
5343 
5344 instruct loadSSF(regF dst, stackSlotF src)
5345 %{
5346   match(Set dst src);
5347 
5348   ins_cost(125);
5349   format %{ "movss   $dst, $src\t# float stk" %}
5350   ins_encode %{
5351     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5352   %}
5353   ins_pipe(pipe_slow); // XXX
5354 %}
5355 
5356 // Use the same format since predicate() can not be used here.
5357 instruct loadSSD(regD dst, stackSlotD src)
5358 %{
5359   match(Set dst src);
5360 
5361   ins_cost(125);
5362   format %{ "movsd   $dst, $src\t# double stk" %}
5363   ins_encode  %{
5364     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5365   %}
5366   ins_pipe(pipe_slow); // XXX
5367 %}
5368 
5369 // Prefetch instructions.
5370 // Must be safe to execute with invalid address (cannot fault).
5371 
5372 instruct prefetchr( memory mem ) %{
5373   predicate(ReadPrefetchInstr==3);
5374   match(PrefetchRead mem);
5375   ins_cost(125);
5376 
5377   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5378   ins_encode %{
5379     __ prefetchr($mem$$Address);
5380   %}
5381   ins_pipe(ialu_mem);
5382 %}
5383 
5384 instruct prefetchrNTA( memory mem ) %{
5385   predicate(ReadPrefetchInstr==0);
5386   match(PrefetchRead mem);
5387   ins_cost(125);
5388 
5389   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5390   ins_encode %{
5391     __ prefetchnta($mem$$Address);
5392   %}
5393   ins_pipe(ialu_mem);
5394 %}
5395 
5396 instruct prefetchrT0( memory mem ) %{
5397   predicate(ReadPrefetchInstr==1);
5398   match(PrefetchRead mem);
5399   ins_cost(125);
5400 
5401   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5402   ins_encode %{
5403     __ prefetcht0($mem$$Address);
5404   %}
5405   ins_pipe(ialu_mem);
5406 %}
5407 
5408 instruct prefetchrT2( memory mem ) %{
5409   predicate(ReadPrefetchInstr==2);
5410   match(PrefetchRead mem);
5411   ins_cost(125);
5412 
5413   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5414   ins_encode %{
5415     __ prefetcht2($mem$$Address);
5416   %}
5417   ins_pipe(ialu_mem);
5418 %}
5419 
5420 instruct prefetchwNTA( memory mem ) %{
5421   match(PrefetchWrite mem);
5422   ins_cost(125);
5423 
5424   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5425   ins_encode %{
5426     __ prefetchnta($mem$$Address);
5427   %}
5428   ins_pipe(ialu_mem);
5429 %}
5430 
5431 // Prefetch instructions for allocation.
5432 
5433 instruct prefetchAlloc( memory mem ) %{
5434   predicate(AllocatePrefetchInstr==3);
5435   match(PrefetchAllocation mem);
5436   ins_cost(125);
5437 
5438   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5439   ins_encode %{
5440     __ prefetchw($mem$$Address);
5441   %}
5442   ins_pipe(ialu_mem);
5443 %}
5444 
5445 instruct prefetchAllocNTA( memory mem ) %{
5446   predicate(AllocatePrefetchInstr==0);
5447   match(PrefetchAllocation mem);
5448   ins_cost(125);
5449 
5450   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5451   ins_encode %{
5452     __ prefetchnta($mem$$Address);
5453   %}
5454   ins_pipe(ialu_mem);
5455 %}
5456 
5457 instruct prefetchAllocT0( memory mem ) %{
5458   predicate(AllocatePrefetchInstr==1);
5459   match(PrefetchAllocation mem);
5460   ins_cost(125);
5461 
5462   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5463   ins_encode %{
5464     __ prefetcht0($mem$$Address);
5465   %}
5466   ins_pipe(ialu_mem);
5467 %}
5468 
5469 instruct prefetchAllocT2( memory mem ) %{
5470   predicate(AllocatePrefetchInstr==2);
5471   match(PrefetchAllocation mem);
5472   ins_cost(125);
5473 
5474   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5475   ins_encode %{
5476     __ prefetcht2($mem$$Address);
5477   %}
5478   ins_pipe(ialu_mem);
5479 %}
5480 
5481 //----------Store Instructions-------------------------------------------------
5482 
5483 // Store Byte
5484 instruct storeB(memory mem, rRegI src)
5485 %{
5486   match(Set mem (StoreB mem src));
5487 
5488   ins_cost(125); // XXX
5489   format %{ "movb    $mem, $src\t# byte" %}
5490   opcode(0x88);
5491   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5492   ins_pipe(ialu_mem_reg);
5493 %}
5494 
5495 // Store Char/Short
5496 instruct storeC(memory mem, rRegI src)
5497 %{
5498   match(Set mem (StoreC mem src));
5499 
5500   ins_cost(125); // XXX
5501   format %{ "movw    $mem, $src\t# char/short" %}
5502   opcode(0x89);
5503   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5504   ins_pipe(ialu_mem_reg);
5505 %}
5506 
5507 // Store Integer
5508 instruct storeI(memory mem, rRegI src)
5509 %{
5510   match(Set mem (StoreI mem src));
5511 
5512   ins_cost(125); // XXX
5513   format %{ "movl    $mem, $src\t# int" %}
5514   opcode(0x89);
5515   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5516   ins_pipe(ialu_mem_reg);
5517 %}
5518 
5519 // Store Long
5520 instruct storeL(memory mem, rRegL src)
5521 %{
5522   match(Set mem (StoreL mem src));
5523 
5524   ins_cost(125); // XXX
5525   format %{ "movq    $mem, $src\t# long" %}
5526   opcode(0x89);
5527   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5528   ins_pipe(ialu_mem_reg); // XXX
5529 %}
5530 
5531 // Store Pointer
5532 instruct storeP(memory mem, any_RegP src)
5533 %{
5534   match(Set mem (StoreP mem src));
5535 
5536   ins_cost(125); // XXX
5537   format %{ "movq    $mem, $src\t# ptr" %}
5538   opcode(0x89);
5539   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5540   ins_pipe(ialu_mem_reg);
5541 %}
5542 
5543 instruct storeImmP0(memory mem, immP0 zero)
5544 %{
5545   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5546   match(Set mem (StoreP mem zero));
5547 
5548   ins_cost(125); // XXX
5549   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5550   ins_encode %{
5551     __ movq($mem$$Address, r12);
5552   %}
5553   ins_pipe(ialu_mem_reg);
5554 %}
5555 
5556 // Store NULL Pointer, mark word, or other simple pointer constant.
5557 instruct storeImmP(memory mem, immP31 src)
5558 %{
5559   match(Set mem (StoreP mem src));
5560 
5561   ins_cost(150); // XXX
5562   format %{ "movq    $mem, $src\t# ptr" %}
5563   opcode(0xC7); /* C7 /0 */
5564   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5565   ins_pipe(ialu_mem_imm);
5566 %}
5567 
5568 // Store Compressed Pointer
5569 instruct storeN(memory mem, rRegN src)
5570 %{
5571   match(Set mem (StoreN mem src));
5572 
5573   ins_cost(125); // XXX
5574   format %{ "movl    $mem, $src\t# compressed ptr" %}
5575   ins_encode %{
5576     __ movl($mem$$Address, $src$$Register);
5577   %}
5578   ins_pipe(ialu_mem_reg);
5579 %}
5580 
5581 instruct storeNKlass(memory mem, rRegN src)
5582 %{
5583   match(Set mem (StoreNKlass mem src));
5584 
5585   ins_cost(125); // XXX
5586   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5587   ins_encode %{
5588     __ movl($mem$$Address, $src$$Register);
5589   %}
5590   ins_pipe(ialu_mem_reg);
5591 %}
5592 
5593 instruct storeImmN0(memory mem, immN0 zero)
5594 %{
5595   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5596   match(Set mem (StoreN mem zero));
5597 
5598   ins_cost(125); // XXX
5599   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5600   ins_encode %{
5601     __ movl($mem$$Address, r12);
5602   %}
5603   ins_pipe(ialu_mem_reg);
5604 %}
5605 
5606 instruct storeImmN(memory mem, immN src)
5607 %{
5608   match(Set mem (StoreN mem src));
5609 
5610   ins_cost(150); // XXX
5611   format %{ "movl    $mem, $src\t# compressed ptr" %}
5612   ins_encode %{
5613     address con = (address)$src$$constant;
5614     if (con == NULL) {
5615       __ movl($mem$$Address, (int32_t)0);
5616     } else {
5617       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5618     }
5619   %}
5620   ins_pipe(ialu_mem_imm);
5621 %}
5622 
5623 instruct storeImmNKlass(memory mem, immNKlass src)
5624 %{
5625   match(Set mem (StoreNKlass mem src));
5626 
5627   ins_cost(150); // XXX
5628   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5629   ins_encode %{
5630     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5631   %}
5632   ins_pipe(ialu_mem_imm);
5633 %}
5634 
5635 // Store Integer Immediate
5636 instruct storeImmI0(memory mem, immI0 zero)
5637 %{
5638   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5639   match(Set mem (StoreI mem zero));
5640 
5641   ins_cost(125); // XXX
5642   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5643   ins_encode %{
5644     __ movl($mem$$Address, r12);
5645   %}
5646   ins_pipe(ialu_mem_reg);
5647 %}
5648 
5649 instruct storeImmI(memory mem, immI src)
5650 %{
5651   match(Set mem (StoreI mem src));
5652 
5653   ins_cost(150);
5654   format %{ "movl    $mem, $src\t# int" %}
5655   opcode(0xC7); /* C7 /0 */
5656   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5657   ins_pipe(ialu_mem_imm);
5658 %}
5659 
5660 // Store Long Immediate
5661 instruct storeImmL0(memory mem, immL0 zero)
5662 %{
5663   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5664   match(Set mem (StoreL mem zero));
5665 
5666   ins_cost(125); // XXX
5667   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5668   ins_encode %{
5669     __ movq($mem$$Address, r12);
5670   %}
5671   ins_pipe(ialu_mem_reg);
5672 %}
5673 
5674 instruct storeImmL(memory mem, immL32 src)
5675 %{
5676   match(Set mem (StoreL mem src));
5677 
5678   ins_cost(150);
5679   format %{ "movq    $mem, $src\t# long" %}
5680   opcode(0xC7); /* C7 /0 */
5681   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5682   ins_pipe(ialu_mem_imm);
5683 %}
5684 
5685 // Store Short/Char Immediate
5686 instruct storeImmC0(memory mem, immI0 zero)
5687 %{
5688   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5689   match(Set mem (StoreC mem zero));
5690 
5691   ins_cost(125); // XXX
5692   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5693   ins_encode %{
5694     __ movw($mem$$Address, r12);
5695   %}
5696   ins_pipe(ialu_mem_reg);
5697 %}
5698 
5699 instruct storeImmI16(memory mem, immI16 src)
5700 %{
5701   predicate(UseStoreImmI16);
5702   match(Set mem (StoreC mem src));
5703 
5704   ins_cost(150);
5705   format %{ "movw    $mem, $src\t# short/char" %}
5706   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5707   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5708   ins_pipe(ialu_mem_imm);
5709 %}
5710 
5711 // Store Byte Immediate
5712 instruct storeImmB0(memory mem, immI0 zero)
5713 %{
5714   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5715   match(Set mem (StoreB mem zero));
5716 
5717   ins_cost(125); // XXX
5718   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5719   ins_encode %{
5720     __ movb($mem$$Address, r12);
5721   %}
5722   ins_pipe(ialu_mem_reg);
5723 %}
5724 
5725 instruct storeImmB(memory mem, immI8 src)
5726 %{
5727   match(Set mem (StoreB mem src));
5728 
5729   ins_cost(150); // XXX
5730   format %{ "movb    $mem, $src\t# byte" %}
5731   opcode(0xC6); /* C6 /0 */
5732   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5733   ins_pipe(ialu_mem_imm);
5734 %}
5735 
5736 // Store CMS card-mark Immediate
5737 instruct storeImmCM0_reg(memory mem, immI0 zero)
5738 %{
5739   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5740   match(Set mem (StoreCM mem zero));
5741 
5742   ins_cost(125); // XXX
5743   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5744   ins_encode %{
5745     __ movb($mem$$Address, r12);
5746   %}
5747   ins_pipe(ialu_mem_reg);
5748 %}
5749 
5750 instruct storeImmCM0(memory mem, immI0 src)
5751 %{
5752   match(Set mem (StoreCM mem src));
5753 
5754   ins_cost(150); // XXX
5755   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5756   opcode(0xC6); /* C6 /0 */
5757   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5758   ins_pipe(ialu_mem_imm);
5759 %}
5760 
5761 // Store Float
5762 instruct storeF(memory mem, regF src)
5763 %{
5764   match(Set mem (StoreF mem src));
5765 
5766   ins_cost(95); // XXX
5767   format %{ "movss   $mem, $src\t# float" %}
5768   ins_encode %{
5769     __ movflt($mem$$Address, $src$$XMMRegister);
5770   %}
5771   ins_pipe(pipe_slow); // XXX
5772 %}
5773 
5774 // Store immediate Float value (it is faster than store from XMM register)
5775 instruct storeF0(memory mem, immF0 zero)
5776 %{
5777   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5778   match(Set mem (StoreF mem zero));
5779 
5780   ins_cost(25); // XXX
5781   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5782   ins_encode %{
5783     __ movl($mem$$Address, r12);
5784   %}
5785   ins_pipe(ialu_mem_reg);
5786 %}
5787 
5788 instruct storeF_imm(memory mem, immF src)
5789 %{
5790   match(Set mem (StoreF mem src));
5791 
5792   ins_cost(50);
5793   format %{ "movl    $mem, $src\t# float" %}
5794   opcode(0xC7); /* C7 /0 */
5795   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5796   ins_pipe(ialu_mem_imm);
5797 %}
5798 
5799 // Store Double
5800 instruct storeD(memory mem, regD src)
5801 %{
5802   match(Set mem (StoreD mem src));
5803 
5804   ins_cost(95); // XXX
5805   format %{ "movsd   $mem, $src\t# double" %}
5806   ins_encode %{
5807     __ movdbl($mem$$Address, $src$$XMMRegister);
5808   %}
5809   ins_pipe(pipe_slow); // XXX
5810 %}
5811 
5812 // Store immediate double 0.0 (it is faster than store from XMM register)
5813 instruct storeD0_imm(memory mem, immD0 src)
5814 %{
5815   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5816   match(Set mem (StoreD mem src));
5817 
5818   ins_cost(50);
5819   format %{ "movq    $mem, $src\t# double 0." %}
5820   opcode(0xC7); /* C7 /0 */
5821   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5822   ins_pipe(ialu_mem_imm);
5823 %}
5824 
5825 instruct storeD0(memory mem, immD0 zero)
5826 %{
5827   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5828   match(Set mem (StoreD mem zero));
5829 
5830   ins_cost(25); // XXX
5831   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5832   ins_encode %{
5833     __ movq($mem$$Address, r12);
5834   %}
5835   ins_pipe(ialu_mem_reg);
5836 %}
5837 
5838 instruct storeSSI(stackSlotI dst, rRegI src)
5839 %{
5840   match(Set dst src);
5841 
5842   ins_cost(100);
5843   format %{ "movl    $dst, $src\t# int stk" %}
5844   opcode(0x89);
5845   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5846   ins_pipe( ialu_mem_reg );
5847 %}
5848 
5849 instruct storeSSL(stackSlotL dst, rRegL src)
5850 %{
5851   match(Set dst src);
5852 
5853   ins_cost(100);
5854   format %{ "movq    $dst, $src\t# long stk" %}
5855   opcode(0x89);
5856   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5857   ins_pipe(ialu_mem_reg);
5858 %}
5859 
5860 instruct storeSSP(stackSlotP dst, rRegP src)
5861 %{
5862   match(Set dst src);
5863 
5864   ins_cost(100);
5865   format %{ "movq    $dst, $src\t# ptr stk" %}
5866   opcode(0x89);
5867   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5868   ins_pipe(ialu_mem_reg);
5869 %}
5870 
5871 instruct storeSSF(stackSlotF dst, regF src)
5872 %{
5873   match(Set dst src);
5874 
5875   ins_cost(95); // XXX
5876   format %{ "movss   $dst, $src\t# float stk" %}
5877   ins_encode %{
5878     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5879   %}
5880   ins_pipe(pipe_slow); // XXX
5881 %}
5882 
5883 instruct storeSSD(stackSlotD dst, regD src)
5884 %{
5885   match(Set dst src);
5886 
5887   ins_cost(95); // XXX
5888   format %{ "movsd   $dst, $src\t# double stk" %}
5889   ins_encode %{
5890     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5891   %}
5892   ins_pipe(pipe_slow); // XXX
5893 %}
5894 
5895 //----------BSWAP Instructions-------------------------------------------------
5896 instruct bytes_reverse_int(rRegI dst) %{
5897   match(Set dst (ReverseBytesI dst));
5898 
5899   format %{ "bswapl  $dst" %}
5900   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
5901   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
5902   ins_pipe( ialu_reg );
5903 %}
5904 
5905 instruct bytes_reverse_long(rRegL dst) %{
5906   match(Set dst (ReverseBytesL dst));
5907 
5908   format %{ "bswapq  $dst" %}
5909   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
5910   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
5911   ins_pipe( ialu_reg);
5912 %}
5913 
5914 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
5915   match(Set dst (ReverseBytesUS dst));
5916   effect(KILL cr);
5917 
5918   format %{ "bswapl  $dst\n\t"
5919             "shrl    $dst,16\n\t" %}
5920   ins_encode %{
5921     __ bswapl($dst$$Register);
5922     __ shrl($dst$$Register, 16);
5923   %}
5924   ins_pipe( ialu_reg );
5925 %}
5926 
5927 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
5928   match(Set dst (ReverseBytesS dst));
5929   effect(KILL cr);
5930 
5931   format %{ "bswapl  $dst\n\t"
5932             "sar     $dst,16\n\t" %}
5933   ins_encode %{
5934     __ bswapl($dst$$Register);
5935     __ sarl($dst$$Register, 16);
5936   %}
5937   ins_pipe( ialu_reg );
5938 %}
5939 
5940 //---------- Zeros Count Instructions ------------------------------------------
5941 
5942 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5943   predicate(UseCountLeadingZerosInstruction);
5944   match(Set dst (CountLeadingZerosI src));
5945   effect(KILL cr);
5946 
5947   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
5948   ins_encode %{
5949     __ lzcntl($dst$$Register, $src$$Register);
5950   %}
5951   ins_pipe(ialu_reg);
5952 %}
5953 
5954 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
5955   predicate(!UseCountLeadingZerosInstruction);
5956   match(Set dst (CountLeadingZerosI src));
5957   effect(KILL cr);
5958 
5959   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
5960             "jnz     skip\n\t"
5961             "movl    $dst, -1\n"
5962       "skip:\n\t"
5963             "negl    $dst\n\t"
5964             "addl    $dst, 31" %}
5965   ins_encode %{
5966     Register Rdst = $dst$$Register;
5967     Register Rsrc = $src$$Register;
5968     Label skip;
5969     __ bsrl(Rdst, Rsrc);
5970     __ jccb(Assembler::notZero, skip);
5971     __ movl(Rdst, -1);
5972     __ bind(skip);
5973     __ negl(Rdst);
5974     __ addl(Rdst, BitsPerInt - 1);
5975   %}
5976   ins_pipe(ialu_reg);
5977 %}
5978 
5979 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5980   predicate(UseCountLeadingZerosInstruction);
5981   match(Set dst (CountLeadingZerosL src));
5982   effect(KILL cr);
5983 
5984   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
5985   ins_encode %{
5986     __ lzcntq($dst$$Register, $src$$Register);
5987   %}
5988   ins_pipe(ialu_reg);
5989 %}
5990 
5991 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
5992   predicate(!UseCountLeadingZerosInstruction);
5993   match(Set dst (CountLeadingZerosL src));
5994   effect(KILL cr);
5995 
5996   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
5997             "jnz     skip\n\t"
5998             "movl    $dst, -1\n"
5999       "skip:\n\t"
6000             "negl    $dst\n\t"
6001             "addl    $dst, 63" %}
6002   ins_encode %{
6003     Register Rdst = $dst$$Register;
6004     Register Rsrc = $src$$Register;
6005     Label skip;
6006     __ bsrq(Rdst, Rsrc);
6007     __ jccb(Assembler::notZero, skip);
6008     __ movl(Rdst, -1);
6009     __ bind(skip);
6010     __ negl(Rdst);
6011     __ addl(Rdst, BitsPerLong - 1);
6012   %}
6013   ins_pipe(ialu_reg);
6014 %}
6015 
6016 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6017   predicate(UseCountTrailingZerosInstruction);
6018   match(Set dst (CountTrailingZerosI src));
6019   effect(KILL cr);
6020 
6021   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6022   ins_encode %{
6023     __ tzcntl($dst$$Register, $src$$Register);
6024   %}
6025   ins_pipe(ialu_reg);
6026 %}
6027 
6028 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6029   predicate(!UseCountTrailingZerosInstruction);
6030   match(Set dst (CountTrailingZerosI src));
6031   effect(KILL cr);
6032 
6033   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6034             "jnz     done\n\t"
6035             "movl    $dst, 32\n"
6036       "done:" %}
6037   ins_encode %{
6038     Register Rdst = $dst$$Register;
6039     Label done;
6040     __ bsfl(Rdst, $src$$Register);
6041     __ jccb(Assembler::notZero, done);
6042     __ movl(Rdst, BitsPerInt);
6043     __ bind(done);
6044   %}
6045   ins_pipe(ialu_reg);
6046 %}
6047 
6048 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6049   predicate(UseCountTrailingZerosInstruction);
6050   match(Set dst (CountTrailingZerosL src));
6051   effect(KILL cr);
6052 
6053   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6054   ins_encode %{
6055     __ tzcntq($dst$$Register, $src$$Register);
6056   %}
6057   ins_pipe(ialu_reg);
6058 %}
6059 
6060 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6061   predicate(!UseCountTrailingZerosInstruction);
6062   match(Set dst (CountTrailingZerosL src));
6063   effect(KILL cr);
6064 
6065   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6066             "jnz     done\n\t"
6067             "movl    $dst, 64\n"
6068       "done:" %}
6069   ins_encode %{
6070     Register Rdst = $dst$$Register;
6071     Label done;
6072     __ bsfq(Rdst, $src$$Register);
6073     __ jccb(Assembler::notZero, done);
6074     __ movl(Rdst, BitsPerLong);
6075     __ bind(done);
6076   %}
6077   ins_pipe(ialu_reg);
6078 %}
6079 
6080 
6081 //---------- Population Count Instructions -------------------------------------
6082 
6083 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6084   predicate(UsePopCountInstruction);
6085   match(Set dst (PopCountI src));
6086   effect(KILL cr);
6087 
6088   format %{ "popcnt  $dst, $src" %}
6089   ins_encode %{
6090     __ popcntl($dst$$Register, $src$$Register);
6091   %}
6092   ins_pipe(ialu_reg);
6093 %}
6094 
6095 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6096   predicate(UsePopCountInstruction);
6097   match(Set dst (PopCountI (LoadI mem)));
6098   effect(KILL cr);
6099 
6100   format %{ "popcnt  $dst, $mem" %}
6101   ins_encode %{
6102     __ popcntl($dst$$Register, $mem$$Address);
6103   %}
6104   ins_pipe(ialu_reg);
6105 %}
6106 
6107 // Note: Long.bitCount(long) returns an int.
6108 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6109   predicate(UsePopCountInstruction);
6110   match(Set dst (PopCountL src));
6111   effect(KILL cr);
6112 
6113   format %{ "popcnt  $dst, $src" %}
6114   ins_encode %{
6115     __ popcntq($dst$$Register, $src$$Register);
6116   %}
6117   ins_pipe(ialu_reg);
6118 %}
6119 
6120 // Note: Long.bitCount(long) returns an int.
6121 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6122   predicate(UsePopCountInstruction);
6123   match(Set dst (PopCountL (LoadL mem)));
6124   effect(KILL cr);
6125 
6126   format %{ "popcnt  $dst, $mem" %}
6127   ins_encode %{
6128     __ popcntq($dst$$Register, $mem$$Address);
6129   %}
6130   ins_pipe(ialu_reg);
6131 %}
6132 
6133 
6134 //----------MemBar Instructions-----------------------------------------------
6135 // Memory barrier flavors
6136 
6137 instruct membar_acquire()
6138 %{
6139   match(MemBarAcquire);
6140   ins_cost(0);
6141 
6142   size(0);
6143   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6144   ins_encode();
6145   ins_pipe(empty);
6146 %}
6147 
6148 instruct membar_acquire_lock()
6149 %{
6150   match(MemBarAcquireLock);
6151   ins_cost(0);
6152 
6153   size(0);
6154   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6155   ins_encode();
6156   ins_pipe(empty);
6157 %}
6158 
6159 instruct membar_release()
6160 %{
6161   match(MemBarRelease);
6162   ins_cost(0);
6163 
6164   size(0);
6165   format %{ "MEMBAR-release ! (empty encoding)" %}
6166   ins_encode();
6167   ins_pipe(empty);
6168 %}
6169 
6170 instruct membar_release_lock()
6171 %{
6172   match(MemBarReleaseLock);
6173   ins_cost(0);
6174 
6175   size(0);
6176   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6177   ins_encode();
6178   ins_pipe(empty);
6179 %}
6180 
6181 instruct membar_volatile(rFlagsReg cr) %{
6182   match(MemBarVolatile);
6183   effect(KILL cr);
6184   ins_cost(400);
6185 
6186   format %{
6187     $$template
6188     if (os::is_MP()) {
6189       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6190     } else {
6191       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6192     }
6193   %}
6194   ins_encode %{
6195     __ membar(Assembler::StoreLoad);
6196   %}
6197   ins_pipe(pipe_slow);
6198 %}
6199 
6200 instruct unnecessary_membar_volatile()
6201 %{
6202   match(MemBarVolatile);
6203   predicate(Matcher::post_store_load_barrier(n));
6204   ins_cost(0);
6205 
6206   size(0);
6207   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6208   ins_encode();
6209   ins_pipe(empty);
6210 %}
6211 
6212 instruct membar_storestore() %{
6213   match(MemBarStoreStore);
6214   ins_cost(0);
6215 
6216   size(0);
6217   format %{ "MEMBAR-storestore (empty encoding)" %}
6218   ins_encode( );
6219   ins_pipe(empty);
6220 %}
6221 
6222 //----------Move Instructions--------------------------------------------------
6223 
6224 instruct castX2P(rRegP dst, rRegL src)
6225 %{
6226   match(Set dst (CastX2P src));
6227 
6228   format %{ "movq    $dst, $src\t# long->ptr" %}
6229   ins_encode %{
6230     if ($dst$$reg != $src$$reg) {
6231       __ movptr($dst$$Register, $src$$Register);
6232     }
6233   %}
6234   ins_pipe(ialu_reg_reg); // XXX
6235 %}
6236 
6237 instruct castP2X(rRegL dst, rRegP src)
6238 %{
6239   match(Set dst (CastP2X src));
6240 
6241   format %{ "movq    $dst, $src\t# ptr -> long" %}
6242   ins_encode %{
6243     if ($dst$$reg != $src$$reg) {
6244       __ movptr($dst$$Register, $src$$Register);
6245     }
6246   %}
6247   ins_pipe(ialu_reg_reg); // XXX
6248 %}
6249 
6250 // Convert oop into int for vectors alignment masking
6251 instruct convP2I(rRegI dst, rRegP src)
6252 %{
6253   match(Set dst (ConvL2I (CastP2X src)));
6254 
6255   format %{ "movl    $dst, $src\t# ptr -> int" %}
6256   ins_encode %{
6257     __ movl($dst$$Register, $src$$Register);
6258   %}
6259   ins_pipe(ialu_reg_reg); // XXX
6260 %}
6261 
6262 // Convert compressed oop into int for vectors alignment masking
6263 // in case of 32bit oops (heap < 4Gb).
6264 instruct convN2I(rRegI dst, rRegN src)
6265 %{
6266   predicate(Universe::narrow_oop_shift() == 0);
6267   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6268 
6269   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6270   ins_encode %{
6271     __ movl($dst$$Register, $src$$Register);
6272   %}
6273   ins_pipe(ialu_reg_reg); // XXX
6274 %}
6275 
6276 // Convert oop pointer into compressed form
6277 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6278   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6279   match(Set dst (EncodeP src));
6280   effect(KILL cr);
6281   format %{ "encode_heap_oop $dst,$src" %}
6282   ins_encode %{
6283     Register s = $src$$Register;
6284     Register d = $dst$$Register;
6285     if (s != d) {
6286       __ movq(d, s);
6287     }
6288     __ encode_heap_oop(d);
6289   %}
6290   ins_pipe(ialu_reg_long);
6291 %}
6292 
6293 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6294   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6295   match(Set dst (EncodeP src));
6296   effect(KILL cr);
6297   format %{ "encode_heap_oop_not_null $dst,$src" %}
6298   ins_encode %{
6299     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6300   %}
6301   ins_pipe(ialu_reg_long);
6302 %}
6303 
6304 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6305   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6306             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6307   match(Set dst (DecodeN src));
6308   effect(KILL cr);
6309   format %{ "decode_heap_oop $dst,$src" %}
6310   ins_encode %{
6311     Register s = $src$$Register;
6312     Register d = $dst$$Register;
6313     if (s != d) {
6314       __ movq(d, s);
6315     }
6316     __ decode_heap_oop(d);
6317   %}
6318   ins_pipe(ialu_reg_long);
6319 %}
6320 
6321 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6322   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6323             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6324   match(Set dst (DecodeN src));
6325   effect(KILL cr);
6326   format %{ "decode_heap_oop_not_null $dst,$src" %}
6327   ins_encode %{
6328     Register s = $src$$Register;
6329     Register d = $dst$$Register;
6330     if (s != d) {
6331       __ decode_heap_oop_not_null(d, s);
6332     } else {
6333       __ decode_heap_oop_not_null(d);
6334     }
6335   %}
6336   ins_pipe(ialu_reg_long);
6337 %}
6338 
6339 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6340   match(Set dst (EncodePKlass src));
6341   effect(KILL cr);
6342   format %{ "encode_klass_not_null $dst,$src" %}
6343   ins_encode %{
6344     __ encode_klass_not_null($dst$$Register, $src$$Register);
6345   %}
6346   ins_pipe(ialu_reg_long);
6347 %}
6348 
6349 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6350   match(Set dst (DecodeNKlass src));
6351   effect(KILL cr);
6352   format %{ "decode_klass_not_null $dst,$src" %}
6353   ins_encode %{
6354     Register s = $src$$Register;
6355     Register d = $dst$$Register;
6356     if (s != d) {
6357       __ decode_klass_not_null(d, s);
6358     } else {
6359       __ decode_klass_not_null(d);
6360     }
6361   %}
6362   ins_pipe(ialu_reg_long);
6363 %}
6364 
6365 
6366 //----------Conditional Move---------------------------------------------------
6367 // Jump
6368 // dummy instruction for generating temp registers
6369 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6370   match(Jump (LShiftL switch_val shift));
6371   ins_cost(350);
6372   predicate(false);
6373   effect(TEMP dest);
6374 
6375   format %{ "leaq    $dest, [$constantaddress]\n\t"
6376             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6377   ins_encode %{
6378     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6379     // to do that and the compiler is using that register as one it can allocate.
6380     // So we build it all by hand.
6381     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6382     // ArrayAddress dispatch(table, index);
6383     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6384     __ lea($dest$$Register, $constantaddress);
6385     __ jmp(dispatch);
6386   %}
6387   ins_pipe(pipe_jmp);
6388 %}
6389 
6390 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6391   match(Jump (AddL (LShiftL switch_val shift) offset));
6392   ins_cost(350);
6393   effect(TEMP dest);
6394 
6395   format %{ "leaq    $dest, [$constantaddress]\n\t"
6396             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6397   ins_encode %{
6398     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6399     // to do that and the compiler is using that register as one it can allocate.
6400     // So we build it all by hand.
6401     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6402     // ArrayAddress dispatch(table, index);
6403     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6404     __ lea($dest$$Register, $constantaddress);
6405     __ jmp(dispatch);
6406   %}
6407   ins_pipe(pipe_jmp);
6408 %}
6409 
6410 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6411   match(Jump switch_val);
6412   ins_cost(350);
6413   effect(TEMP dest);
6414 
6415   format %{ "leaq    $dest, [$constantaddress]\n\t"
6416             "jmp     [$dest + $switch_val]\n\t" %}
6417   ins_encode %{
6418     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6419     // to do that and the compiler is using that register as one it can allocate.
6420     // So we build it all by hand.
6421     // Address index(noreg, switch_reg, Address::times_1);
6422     // ArrayAddress dispatch(table, index);
6423     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6424     __ lea($dest$$Register, $constantaddress);
6425     __ jmp(dispatch);
6426   %}
6427   ins_pipe(pipe_jmp);
6428 %}
6429 
6430 // Conditional move
6431 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6432 %{
6433   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6434 
6435   ins_cost(200); // XXX
6436   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6437   opcode(0x0F, 0x40);
6438   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6439   ins_pipe(pipe_cmov_reg);
6440 %}
6441 
6442 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6443   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6444 
6445   ins_cost(200); // XXX
6446   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6447   opcode(0x0F, 0x40);
6448   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6449   ins_pipe(pipe_cmov_reg);
6450 %}
6451 
6452 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6453   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6454   ins_cost(200);
6455   expand %{
6456     cmovI_regU(cop, cr, dst, src);
6457   %}
6458 %}
6459 
6460 // Conditional move
6461 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6462   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6463 
6464   ins_cost(250); // XXX
6465   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6466   opcode(0x0F, 0x40);
6467   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6468   ins_pipe(pipe_cmov_mem);
6469 %}
6470 
6471 // Conditional move
6472 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6473 %{
6474   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6475 
6476   ins_cost(250); // XXX
6477   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6478   opcode(0x0F, 0x40);
6479   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6480   ins_pipe(pipe_cmov_mem);
6481 %}
6482 
6483 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6484   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6485   ins_cost(250);
6486   expand %{
6487     cmovI_memU(cop, cr, dst, src);
6488   %}
6489 %}
6490 
6491 // Conditional move
6492 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6493 %{
6494   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6495 
6496   ins_cost(200); // XXX
6497   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6498   opcode(0x0F, 0x40);
6499   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6500   ins_pipe(pipe_cmov_reg);
6501 %}
6502 
6503 // Conditional move
6504 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6505 %{
6506   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6507 
6508   ins_cost(200); // XXX
6509   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6510   opcode(0x0F, 0x40);
6511   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6512   ins_pipe(pipe_cmov_reg);
6513 %}
6514 
6515 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6516   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6517   ins_cost(200);
6518   expand %{
6519     cmovN_regU(cop, cr, dst, src);
6520   %}
6521 %}
6522 
6523 // Conditional move
6524 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6525 %{
6526   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6527 
6528   ins_cost(200); // XXX
6529   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6530   opcode(0x0F, 0x40);
6531   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6532   ins_pipe(pipe_cmov_reg);  // XXX
6533 %}
6534 
6535 // Conditional move
6536 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6537 %{
6538   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6539 
6540   ins_cost(200); // XXX
6541   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6542   opcode(0x0F, 0x40);
6543   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6544   ins_pipe(pipe_cmov_reg); // XXX
6545 %}
6546 
6547 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6548   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6549   ins_cost(200);
6550   expand %{
6551     cmovP_regU(cop, cr, dst, src);
6552   %}
6553 %}
6554 
6555 // DISABLED: Requires the ADLC to emit a bottom_type call that
6556 // correctly meets the two pointer arguments; one is an incoming
6557 // register but the other is a memory operand.  ALSO appears to
6558 // be buggy with implicit null checks.
6559 //
6560 //// Conditional move
6561 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6562 //%{
6563 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6564 //  ins_cost(250);
6565 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6566 //  opcode(0x0F,0x40);
6567 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6568 //  ins_pipe( pipe_cmov_mem );
6569 //%}
6570 //
6571 //// Conditional move
6572 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6573 //%{
6574 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6575 //  ins_cost(250);
6576 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6577 //  opcode(0x0F,0x40);
6578 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6579 //  ins_pipe( pipe_cmov_mem );
6580 //%}
6581 
6582 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6583 %{
6584   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6585 
6586   ins_cost(200); // XXX
6587   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6588   opcode(0x0F, 0x40);
6589   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6590   ins_pipe(pipe_cmov_reg);  // XXX
6591 %}
6592 
6593 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6594 %{
6595   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6596 
6597   ins_cost(200); // XXX
6598   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6599   opcode(0x0F, 0x40);
6600   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6601   ins_pipe(pipe_cmov_mem);  // XXX
6602 %}
6603 
6604 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6605 %{
6606   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6607 
6608   ins_cost(200); // XXX
6609   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6610   opcode(0x0F, 0x40);
6611   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6612   ins_pipe(pipe_cmov_reg); // XXX
6613 %}
6614 
6615 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6616   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6617   ins_cost(200);
6618   expand %{
6619     cmovL_regU(cop, cr, dst, src);
6620   %}
6621 %}
6622 
6623 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6624 %{
6625   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6626 
6627   ins_cost(200); // XXX
6628   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6629   opcode(0x0F, 0x40);
6630   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6631   ins_pipe(pipe_cmov_mem); // XXX
6632 %}
6633 
6634 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6635   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6636   ins_cost(200);
6637   expand %{
6638     cmovL_memU(cop, cr, dst, src);
6639   %}
6640 %}
6641 
6642 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6643 %{
6644   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6645 
6646   ins_cost(200); // XXX
6647   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6648             "movss     $dst, $src\n"
6649     "skip:" %}
6650   ins_encode %{
6651     Label Lskip;
6652     // Invert sense of branch from sense of CMOV
6653     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6654     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6655     __ bind(Lskip);
6656   %}
6657   ins_pipe(pipe_slow);
6658 %}
6659 
6660 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6661 // %{
6662 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6663 
6664 //   ins_cost(200); // XXX
6665 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6666 //             "movss     $dst, $src\n"
6667 //     "skip:" %}
6668 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6669 //   ins_pipe(pipe_slow);
6670 // %}
6671 
6672 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6673 %{
6674   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6675 
6676   ins_cost(200); // XXX
6677   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6678             "movss     $dst, $src\n"
6679     "skip:" %}
6680   ins_encode %{
6681     Label Lskip;
6682     // Invert sense of branch from sense of CMOV
6683     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6684     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6685     __ bind(Lskip);
6686   %}
6687   ins_pipe(pipe_slow);
6688 %}
6689 
6690 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6691   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6692   ins_cost(200);
6693   expand %{
6694     cmovF_regU(cop, cr, dst, src);
6695   %}
6696 %}
6697 
6698 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6699 %{
6700   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6701 
6702   ins_cost(200); // XXX
6703   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6704             "movsd     $dst, $src\n"
6705     "skip:" %}
6706   ins_encode %{
6707     Label Lskip;
6708     // Invert sense of branch from sense of CMOV
6709     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6710     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6711     __ bind(Lskip);
6712   %}
6713   ins_pipe(pipe_slow);
6714 %}
6715 
6716 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6717 %{
6718   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6719 
6720   ins_cost(200); // XXX
6721   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6722             "movsd     $dst, $src\n"
6723     "skip:" %}
6724   ins_encode %{
6725     Label Lskip;
6726     // Invert sense of branch from sense of CMOV
6727     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6728     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6729     __ bind(Lskip);
6730   %}
6731   ins_pipe(pipe_slow);
6732 %}
6733 
6734 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6735   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6736   ins_cost(200);
6737   expand %{
6738     cmovD_regU(cop, cr, dst, src);
6739   %}
6740 %}
6741 
6742 //----------Arithmetic Instructions--------------------------------------------
6743 //----------Addition Instructions----------------------------------------------
6744 
6745 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6746 %{
6747   match(Set dst (AddI dst src));
6748   effect(KILL cr);
6749 
6750   format %{ "addl    $dst, $src\t# int" %}
6751   opcode(0x03);
6752   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6753   ins_pipe(ialu_reg_reg);
6754 %}
6755 
6756 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6757 %{
6758   match(Set dst (AddI dst src));
6759   effect(KILL cr);
6760 
6761   format %{ "addl    $dst, $src\t# int" %}
6762   opcode(0x81, 0x00); /* /0 id */
6763   ins_encode(OpcSErm(dst, src), Con8or32(src));
6764   ins_pipe( ialu_reg );
6765 %}
6766 
6767 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6768 %{
6769   match(Set dst (AddI dst (LoadI src)));
6770   effect(KILL cr);
6771 
6772   ins_cost(125); // XXX
6773   format %{ "addl    $dst, $src\t# int" %}
6774   opcode(0x03);
6775   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6776   ins_pipe(ialu_reg_mem);
6777 %}
6778 
6779 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6780 %{
6781   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6782   effect(KILL cr);
6783 
6784   ins_cost(150); // XXX
6785   format %{ "addl    $dst, $src\t# int" %}
6786   opcode(0x01); /* Opcode 01 /r */
6787   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6788   ins_pipe(ialu_mem_reg);
6789 %}
6790 
6791 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6792 %{
6793   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6794   effect(KILL cr);
6795 
6796   ins_cost(125); // XXX
6797   format %{ "addl    $dst, $src\t# int" %}
6798   opcode(0x81); /* Opcode 81 /0 id */
6799   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6800   ins_pipe(ialu_mem_imm);
6801 %}
6802 
6803 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6804 %{
6805   predicate(UseIncDec);
6806   match(Set dst (AddI dst src));
6807   effect(KILL cr);
6808 
6809   format %{ "incl    $dst\t# int" %}
6810   opcode(0xFF, 0x00); // FF /0
6811   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6812   ins_pipe(ialu_reg);
6813 %}
6814 
6815 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6816 %{
6817   predicate(UseIncDec);
6818   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6819   effect(KILL cr);
6820 
6821   ins_cost(125); // XXX
6822   format %{ "incl    $dst\t# int" %}
6823   opcode(0xFF); /* Opcode FF /0 */
6824   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6825   ins_pipe(ialu_mem_imm);
6826 %}
6827 
6828 // XXX why does that use AddI
6829 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6830 %{
6831   predicate(UseIncDec);
6832   match(Set dst (AddI dst src));
6833   effect(KILL cr);
6834 
6835   format %{ "decl    $dst\t# int" %}
6836   opcode(0xFF, 0x01); // FF /1
6837   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6838   ins_pipe(ialu_reg);
6839 %}
6840 
6841 // XXX why does that use AddI
6842 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6843 %{
6844   predicate(UseIncDec);
6845   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6846   effect(KILL cr);
6847 
6848   ins_cost(125); // XXX
6849   format %{ "decl    $dst\t# int" %}
6850   opcode(0xFF); /* Opcode FF /1 */
6851   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
6852   ins_pipe(ialu_mem_imm);
6853 %}
6854 
6855 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
6856 %{
6857   match(Set dst (AddI src0 src1));
6858 
6859   ins_cost(110);
6860   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
6861   opcode(0x8D); /* 0x8D /r */
6862   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6863   ins_pipe(ialu_reg_reg);
6864 %}
6865 
6866 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6867 %{
6868   match(Set dst (AddL dst src));
6869   effect(KILL cr);
6870 
6871   format %{ "addq    $dst, $src\t# long" %}
6872   opcode(0x03);
6873   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6874   ins_pipe(ialu_reg_reg);
6875 %}
6876 
6877 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6878 %{
6879   match(Set dst (AddL dst src));
6880   effect(KILL cr);
6881 
6882   format %{ "addq    $dst, $src\t# long" %}
6883   opcode(0x81, 0x00); /* /0 id */
6884   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6885   ins_pipe( ialu_reg );
6886 %}
6887 
6888 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6889 %{
6890   match(Set dst (AddL dst (LoadL src)));
6891   effect(KILL cr);
6892 
6893   ins_cost(125); // XXX
6894   format %{ "addq    $dst, $src\t# long" %}
6895   opcode(0x03);
6896   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6897   ins_pipe(ialu_reg_mem);
6898 %}
6899 
6900 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6901 %{
6902   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6903   effect(KILL cr);
6904 
6905   ins_cost(150); // XXX
6906   format %{ "addq    $dst, $src\t# long" %}
6907   opcode(0x01); /* Opcode 01 /r */
6908   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6909   ins_pipe(ialu_mem_reg);
6910 %}
6911 
6912 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
6913 %{
6914   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6915   effect(KILL cr);
6916 
6917   ins_cost(125); // XXX
6918   format %{ "addq    $dst, $src\t# long" %}
6919   opcode(0x81); /* Opcode 81 /0 id */
6920   ins_encode(REX_mem_wide(dst),
6921              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6922   ins_pipe(ialu_mem_imm);
6923 %}
6924 
6925 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
6926 %{
6927   predicate(UseIncDec);
6928   match(Set dst (AddL dst src));
6929   effect(KILL cr);
6930 
6931   format %{ "incq    $dst\t# long" %}
6932   opcode(0xFF, 0x00); // FF /0
6933   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6934   ins_pipe(ialu_reg);
6935 %}
6936 
6937 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
6938 %{
6939   predicate(UseIncDec);
6940   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6941   effect(KILL cr);
6942 
6943   ins_cost(125); // XXX
6944   format %{ "incq    $dst\t# long" %}
6945   opcode(0xFF); /* Opcode FF /0 */
6946   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
6947   ins_pipe(ialu_mem_imm);
6948 %}
6949 
6950 // XXX why does that use AddL
6951 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
6952 %{
6953   predicate(UseIncDec);
6954   match(Set dst (AddL dst src));
6955   effect(KILL cr);
6956 
6957   format %{ "decq    $dst\t# long" %}
6958   opcode(0xFF, 0x01); // FF /1
6959   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6960   ins_pipe(ialu_reg);
6961 %}
6962 
6963 // XXX why does that use AddL
6964 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
6965 %{
6966   predicate(UseIncDec);
6967   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6968   effect(KILL cr);
6969 
6970   ins_cost(125); // XXX
6971   format %{ "decq    $dst\t# long" %}
6972   opcode(0xFF); /* Opcode FF /1 */
6973   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
6974   ins_pipe(ialu_mem_imm);
6975 %}
6976 
6977 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
6978 %{
6979   match(Set dst (AddL src0 src1));
6980 
6981   ins_cost(110);
6982   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
6983   opcode(0x8D); /* 0x8D /r */
6984   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6985   ins_pipe(ialu_reg_reg);
6986 %}
6987 
6988 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
6989 %{
6990   match(Set dst (AddP dst src));
6991   effect(KILL cr);
6992 
6993   format %{ "addq    $dst, $src\t# ptr" %}
6994   opcode(0x03);
6995   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6996   ins_pipe(ialu_reg_reg);
6997 %}
6998 
6999 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7000 %{
7001   match(Set dst (AddP dst src));
7002   effect(KILL cr);
7003 
7004   format %{ "addq    $dst, $src\t# ptr" %}
7005   opcode(0x81, 0x00); /* /0 id */
7006   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7007   ins_pipe( ialu_reg );
7008 %}
7009 
7010 // XXX addP mem ops ????
7011 
7012 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7013 %{
7014   match(Set dst (AddP src0 src1));
7015 
7016   ins_cost(110);
7017   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7018   opcode(0x8D); /* 0x8D /r */
7019   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7020   ins_pipe(ialu_reg_reg);
7021 %}
7022 
7023 instruct checkCastPP(rRegP dst)
7024 %{
7025   match(Set dst (CheckCastPP dst));
7026 
7027   size(0);
7028   format %{ "# checkcastPP of $dst" %}
7029   ins_encode(/* empty encoding */);
7030   ins_pipe(empty);
7031 %}
7032 
7033 instruct castPP(rRegP dst)
7034 %{
7035   match(Set dst (CastPP dst));
7036 
7037   size(0);
7038   format %{ "# castPP of $dst" %}
7039   ins_encode(/* empty encoding */);
7040   ins_pipe(empty);
7041 %}
7042 
7043 instruct castII(rRegI dst)
7044 %{
7045   match(Set dst (CastII dst));
7046 
7047   size(0);
7048   format %{ "# castII of $dst" %}
7049   ins_encode(/* empty encoding */);
7050   ins_cost(0);
7051   ins_pipe(empty);
7052 %}
7053 
7054 // LoadP-locked same as a regular LoadP when used with compare-swap
7055 instruct loadPLocked(rRegP dst, memory mem)
7056 %{
7057   match(Set dst (LoadPLocked mem));
7058 
7059   ins_cost(125); // XXX
7060   format %{ "movq    $dst, $mem\t# ptr locked" %}
7061   opcode(0x8B);
7062   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7063   ins_pipe(ialu_reg_mem); // XXX
7064 %}
7065 
7066 // Conditional-store of the updated heap-top.
7067 // Used during allocation of the shared heap.
7068 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7069 
7070 instruct storePConditional(memory heap_top_ptr,
7071                            rax_RegP oldval, rRegP newval,
7072                            rFlagsReg cr)
7073 %{
7074   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7075 
7076   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7077             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7078   opcode(0x0F, 0xB1);
7079   ins_encode(lock_prefix,
7080              REX_reg_mem_wide(newval, heap_top_ptr),
7081              OpcP, OpcS,
7082              reg_mem(newval, heap_top_ptr));
7083   ins_pipe(pipe_cmpxchg);
7084 %}
7085 
7086 // Conditional-store of an int value.
7087 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7088 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7089 %{
7090   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7091   effect(KILL oldval);
7092 
7093   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7094   opcode(0x0F, 0xB1);
7095   ins_encode(lock_prefix,
7096              REX_reg_mem(newval, mem),
7097              OpcP, OpcS,
7098              reg_mem(newval, mem));
7099   ins_pipe(pipe_cmpxchg);
7100 %}
7101 
7102 // Conditional-store of a long value.
7103 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7104 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7105 %{
7106   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7107   effect(KILL oldval);
7108 
7109   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7110   opcode(0x0F, 0xB1);
7111   ins_encode(lock_prefix,
7112              REX_reg_mem_wide(newval, mem),
7113              OpcP, OpcS,
7114              reg_mem(newval, mem));
7115   ins_pipe(pipe_cmpxchg);
7116 %}
7117 
7118 
7119 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7120 instruct compareAndSwapP(rRegI res,
7121                          memory mem_ptr,
7122                          rax_RegP oldval, rRegP newval,
7123                          rFlagsReg cr)
7124 %{
7125   predicate(VM_Version::supports_cx8());
7126   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7127   effect(KILL cr, KILL oldval);
7128 
7129   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7130             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7131             "sete    $res\n\t"
7132             "movzbl  $res, $res" %}
7133   opcode(0x0F, 0xB1);
7134   ins_encode(lock_prefix,
7135              REX_reg_mem_wide(newval, mem_ptr),
7136              OpcP, OpcS,
7137              reg_mem(newval, mem_ptr),
7138              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7139              REX_reg_breg(res, res), // movzbl
7140              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7141   ins_pipe( pipe_cmpxchg );
7142 %}
7143 
7144 instruct compareAndSwapL(rRegI res,
7145                          memory mem_ptr,
7146                          rax_RegL oldval, rRegL newval,
7147                          rFlagsReg cr)
7148 %{
7149   predicate(VM_Version::supports_cx8());
7150   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7151   effect(KILL cr, KILL oldval);
7152 
7153   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7154             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7155             "sete    $res\n\t"
7156             "movzbl  $res, $res" %}
7157   opcode(0x0F, 0xB1);
7158   ins_encode(lock_prefix,
7159              REX_reg_mem_wide(newval, mem_ptr),
7160              OpcP, OpcS,
7161              reg_mem(newval, mem_ptr),
7162              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7163              REX_reg_breg(res, res), // movzbl
7164              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7165   ins_pipe( pipe_cmpxchg );
7166 %}
7167 
7168 instruct compareAndSwapI(rRegI res,
7169                          memory mem_ptr,
7170                          rax_RegI oldval, rRegI newval,
7171                          rFlagsReg cr)
7172 %{
7173   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7174   effect(KILL cr, KILL oldval);
7175 
7176   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7177             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7178             "sete    $res\n\t"
7179             "movzbl  $res, $res" %}
7180   opcode(0x0F, 0xB1);
7181   ins_encode(lock_prefix,
7182              REX_reg_mem(newval, mem_ptr),
7183              OpcP, OpcS,
7184              reg_mem(newval, mem_ptr),
7185              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7186              REX_reg_breg(res, res), // movzbl
7187              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7188   ins_pipe( pipe_cmpxchg );
7189 %}
7190 
7191 
7192 instruct compareAndSwapN(rRegI res,
7193                           memory mem_ptr,
7194                           rax_RegN oldval, rRegN newval,
7195                           rFlagsReg cr) %{
7196   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7197   effect(KILL cr, KILL oldval);
7198 
7199   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7200             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7201             "sete    $res\n\t"
7202             "movzbl  $res, $res" %}
7203   opcode(0x0F, 0xB1);
7204   ins_encode(lock_prefix,
7205              REX_reg_mem(newval, mem_ptr),
7206              OpcP, OpcS,
7207              reg_mem(newval, mem_ptr),
7208              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7209              REX_reg_breg(res, res), // movzbl
7210              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7211   ins_pipe( pipe_cmpxchg );
7212 %}
7213 
7214 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7215   predicate(n->as_LoadStore()->result_not_used());
7216   match(Set dummy (GetAndAddI mem add));
7217   effect(KILL cr);
7218   format %{ "ADDL  [$mem],$add" %}
7219   ins_encode %{
7220     if (os::is_MP()) { __ lock(); }
7221     __ addl($mem$$Address, $add$$constant);
7222   %}
7223   ins_pipe( pipe_cmpxchg );
7224 %}
7225 
7226 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7227   match(Set newval (GetAndAddI mem newval));
7228   effect(KILL cr);
7229   format %{ "XADDL  [$mem],$newval" %}
7230   ins_encode %{
7231     if (os::is_MP()) { __ lock(); }
7232     __ xaddl($mem$$Address, $newval$$Register);
7233   %}
7234   ins_pipe( pipe_cmpxchg );
7235 %}
7236 
7237 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7238   predicate(n->as_LoadStore()->result_not_used());
7239   match(Set dummy (GetAndAddL mem add));
7240   effect(KILL cr);
7241   format %{ "ADDQ  [$mem],$add" %}
7242   ins_encode %{
7243     if (os::is_MP()) { __ lock(); }
7244     __ addq($mem$$Address, $add$$constant);
7245   %}
7246   ins_pipe( pipe_cmpxchg );
7247 %}
7248 
7249 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7250   match(Set newval (GetAndAddL mem newval));
7251   effect(KILL cr);
7252   format %{ "XADDQ  [$mem],$newval" %}
7253   ins_encode %{
7254     if (os::is_MP()) { __ lock(); }
7255     __ xaddq($mem$$Address, $newval$$Register);
7256   %}
7257   ins_pipe( pipe_cmpxchg );
7258 %}
7259 
7260 instruct xchgI( memory mem, rRegI newval) %{
7261   match(Set newval (GetAndSetI mem newval));
7262   format %{ "XCHGL  $newval,[$mem]" %}
7263   ins_encode %{
7264     __ xchgl($newval$$Register, $mem$$Address);
7265   %}
7266   ins_pipe( pipe_cmpxchg );
7267 %}
7268 
7269 instruct xchgL( memory mem, rRegL newval) %{
7270   match(Set newval (GetAndSetL mem newval));
7271   format %{ "XCHGL  $newval,[$mem]" %}
7272   ins_encode %{
7273     __ xchgq($newval$$Register, $mem$$Address);
7274   %}
7275   ins_pipe( pipe_cmpxchg );
7276 %}
7277 
7278 instruct xchgP( memory mem, rRegP newval) %{
7279   match(Set newval (GetAndSetP mem newval));
7280   format %{ "XCHGQ  $newval,[$mem]" %}
7281   ins_encode %{
7282     __ xchgq($newval$$Register, $mem$$Address);
7283   %}
7284   ins_pipe( pipe_cmpxchg );
7285 %}
7286 
7287 instruct xchgN( memory mem, rRegN newval) %{
7288   match(Set newval (GetAndSetN mem newval));
7289   format %{ "XCHGL  $newval,$mem]" %}
7290   ins_encode %{
7291     __ xchgl($newval$$Register, $mem$$Address);
7292   %}
7293   ins_pipe( pipe_cmpxchg );
7294 %}
7295 
7296 //----------Subtraction Instructions-------------------------------------------
7297 
7298 // Integer Subtraction Instructions
7299 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7300 %{
7301   match(Set dst (SubI dst src));
7302   effect(KILL cr);
7303 
7304   format %{ "subl    $dst, $src\t# int" %}
7305   opcode(0x2B);
7306   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7307   ins_pipe(ialu_reg_reg);
7308 %}
7309 
7310 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7311 %{
7312   match(Set dst (SubI dst src));
7313   effect(KILL cr);
7314 
7315   format %{ "subl    $dst, $src\t# int" %}
7316   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7317   ins_encode(OpcSErm(dst, src), Con8or32(src));
7318   ins_pipe(ialu_reg);
7319 %}
7320 
7321 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7322 %{
7323   match(Set dst (SubI dst (LoadI src)));
7324   effect(KILL cr);
7325 
7326   ins_cost(125);
7327   format %{ "subl    $dst, $src\t# int" %}
7328   opcode(0x2B);
7329   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7330   ins_pipe(ialu_reg_mem);
7331 %}
7332 
7333 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7334 %{
7335   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7336   effect(KILL cr);
7337 
7338   ins_cost(150);
7339   format %{ "subl    $dst, $src\t# int" %}
7340   opcode(0x29); /* Opcode 29 /r */
7341   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7342   ins_pipe(ialu_mem_reg);
7343 %}
7344 
7345 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7346 %{
7347   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7348   effect(KILL cr);
7349 
7350   ins_cost(125); // XXX
7351   format %{ "subl    $dst, $src\t# int" %}
7352   opcode(0x81); /* Opcode 81 /5 id */
7353   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7354   ins_pipe(ialu_mem_imm);
7355 %}
7356 
7357 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7358 %{
7359   match(Set dst (SubL dst src));
7360   effect(KILL cr);
7361 
7362   format %{ "subq    $dst, $src\t# long" %}
7363   opcode(0x2B);
7364   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7365   ins_pipe(ialu_reg_reg);
7366 %}
7367 
7368 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7369 %{
7370   match(Set dst (SubL dst src));
7371   effect(KILL cr);
7372 
7373   format %{ "subq    $dst, $src\t# long" %}
7374   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7375   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7376   ins_pipe(ialu_reg);
7377 %}
7378 
7379 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7380 %{
7381   match(Set dst (SubL dst (LoadL src)));
7382   effect(KILL cr);
7383 
7384   ins_cost(125);
7385   format %{ "subq    $dst, $src\t# long" %}
7386   opcode(0x2B);
7387   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7388   ins_pipe(ialu_reg_mem);
7389 %}
7390 
7391 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7392 %{
7393   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7394   effect(KILL cr);
7395 
7396   ins_cost(150);
7397   format %{ "subq    $dst, $src\t# long" %}
7398   opcode(0x29); /* Opcode 29 /r */
7399   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7400   ins_pipe(ialu_mem_reg);
7401 %}
7402 
7403 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7404 %{
7405   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7406   effect(KILL cr);
7407 
7408   ins_cost(125); // XXX
7409   format %{ "subq    $dst, $src\t# long" %}
7410   opcode(0x81); /* Opcode 81 /5 id */
7411   ins_encode(REX_mem_wide(dst),
7412              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7413   ins_pipe(ialu_mem_imm);
7414 %}
7415 
7416 // Subtract from a pointer
7417 // XXX hmpf???
7418 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7419 %{
7420   match(Set dst (AddP dst (SubI zero src)));
7421   effect(KILL cr);
7422 
7423   format %{ "subq    $dst, $src\t# ptr - int" %}
7424   opcode(0x2B);
7425   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7426   ins_pipe(ialu_reg_reg);
7427 %}
7428 
7429 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7430 %{
7431   match(Set dst (SubI zero dst));
7432   effect(KILL cr);
7433 
7434   format %{ "negl    $dst\t# int" %}
7435   opcode(0xF7, 0x03);  // Opcode F7 /3
7436   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7437   ins_pipe(ialu_reg);
7438 %}
7439 
7440 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7441 %{
7442   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7443   effect(KILL cr);
7444 
7445   format %{ "negl    $dst\t# int" %}
7446   opcode(0xF7, 0x03);  // Opcode F7 /3
7447   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7448   ins_pipe(ialu_reg);
7449 %}
7450 
7451 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7452 %{
7453   match(Set dst (SubL zero dst));
7454   effect(KILL cr);
7455 
7456   format %{ "negq    $dst\t# long" %}
7457   opcode(0xF7, 0x03);  // Opcode F7 /3
7458   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7459   ins_pipe(ialu_reg);
7460 %}
7461 
7462 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7463 %{
7464   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7465   effect(KILL cr);
7466 
7467   format %{ "negq    $dst\t# long" %}
7468   opcode(0xF7, 0x03);  // Opcode F7 /3
7469   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7470   ins_pipe(ialu_reg);
7471 %}
7472 
7473 //----------Multiplication/Division Instructions-------------------------------
7474 // Integer Multiplication Instructions
7475 // Multiply Register
7476 
7477 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7478 %{
7479   match(Set dst (MulI dst src));
7480   effect(KILL cr);
7481 
7482   ins_cost(300);
7483   format %{ "imull   $dst, $src\t# int" %}
7484   opcode(0x0F, 0xAF);
7485   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7486   ins_pipe(ialu_reg_reg_alu0);
7487 %}
7488 
7489 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7490 %{
7491   match(Set dst (MulI src imm));
7492   effect(KILL cr);
7493 
7494   ins_cost(300);
7495   format %{ "imull   $dst, $src, $imm\t# int" %}
7496   opcode(0x69); /* 69 /r id */
7497   ins_encode(REX_reg_reg(dst, src),
7498              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7499   ins_pipe(ialu_reg_reg_alu0);
7500 %}
7501 
7502 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7503 %{
7504   match(Set dst (MulI dst (LoadI src)));
7505   effect(KILL cr);
7506 
7507   ins_cost(350);
7508   format %{ "imull   $dst, $src\t# int" %}
7509   opcode(0x0F, 0xAF);
7510   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7511   ins_pipe(ialu_reg_mem_alu0);
7512 %}
7513 
7514 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7515 %{
7516   match(Set dst (MulI (LoadI src) imm));
7517   effect(KILL cr);
7518 
7519   ins_cost(300);
7520   format %{ "imull   $dst, $src, $imm\t# int" %}
7521   opcode(0x69); /* 69 /r id */
7522   ins_encode(REX_reg_mem(dst, src),
7523              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7524   ins_pipe(ialu_reg_mem_alu0);
7525 %}
7526 
7527 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7528 %{
7529   match(Set dst (MulL dst src));
7530   effect(KILL cr);
7531 
7532   ins_cost(300);
7533   format %{ "imulq   $dst, $src\t# long" %}
7534   opcode(0x0F, 0xAF);
7535   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7536   ins_pipe(ialu_reg_reg_alu0);
7537 %}
7538 
7539 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7540 %{
7541   match(Set dst (MulL src imm));
7542   effect(KILL cr);
7543 
7544   ins_cost(300);
7545   format %{ "imulq   $dst, $src, $imm\t# long" %}
7546   opcode(0x69); /* 69 /r id */
7547   ins_encode(REX_reg_reg_wide(dst, src),
7548              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7549   ins_pipe(ialu_reg_reg_alu0);
7550 %}
7551 
7552 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7553 %{
7554   match(Set dst (MulL dst (LoadL src)));
7555   effect(KILL cr);
7556 
7557   ins_cost(350);
7558   format %{ "imulq   $dst, $src\t# long" %}
7559   opcode(0x0F, 0xAF);
7560   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7561   ins_pipe(ialu_reg_mem_alu0);
7562 %}
7563 
7564 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7565 %{
7566   match(Set dst (MulL (LoadL src) imm));
7567   effect(KILL cr);
7568 
7569   ins_cost(300);
7570   format %{ "imulq   $dst, $src, $imm\t# long" %}
7571   opcode(0x69); /* 69 /r id */
7572   ins_encode(REX_reg_mem_wide(dst, src),
7573              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7574   ins_pipe(ialu_reg_mem_alu0);
7575 %}
7576 
7577 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7578 %{
7579   match(Set dst (MulHiL src rax));
7580   effect(USE_KILL rax, KILL cr);
7581 
7582   ins_cost(300);
7583   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7584   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7585   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7586   ins_pipe(ialu_reg_reg_alu0);
7587 %}
7588 
7589 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7590                    rFlagsReg cr)
7591 %{
7592   match(Set rax (DivI rax div));
7593   effect(KILL rdx, KILL cr);
7594 
7595   ins_cost(30*100+10*100); // XXX
7596   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7597             "jne,s   normal\n\t"
7598             "xorl    rdx, rdx\n\t"
7599             "cmpl    $div, -1\n\t"
7600             "je,s    done\n"
7601     "normal: cdql\n\t"
7602             "idivl   $div\n"
7603     "done:"        %}
7604   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7605   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7606   ins_pipe(ialu_reg_reg_alu0);
7607 %}
7608 
7609 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7610                    rFlagsReg cr)
7611 %{
7612   match(Set rax (DivL rax div));
7613   effect(KILL rdx, KILL cr);
7614 
7615   ins_cost(30*100+10*100); // XXX
7616   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7617             "cmpq    rax, rdx\n\t"
7618             "jne,s   normal\n\t"
7619             "xorl    rdx, rdx\n\t"
7620             "cmpq    $div, -1\n\t"
7621             "je,s    done\n"
7622     "normal: cdqq\n\t"
7623             "idivq   $div\n"
7624     "done:"        %}
7625   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7626   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7627   ins_pipe(ialu_reg_reg_alu0);
7628 %}
7629 
7630 // Integer DIVMOD with Register, both quotient and mod results
7631 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7632                              rFlagsReg cr)
7633 %{
7634   match(DivModI rax div);
7635   effect(KILL cr);
7636 
7637   ins_cost(30*100+10*100); // XXX
7638   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7639             "jne,s   normal\n\t"
7640             "xorl    rdx, rdx\n\t"
7641             "cmpl    $div, -1\n\t"
7642             "je,s    done\n"
7643     "normal: cdql\n\t"
7644             "idivl   $div\n"
7645     "done:"        %}
7646   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7647   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7648   ins_pipe(pipe_slow);
7649 %}
7650 
7651 // Long DIVMOD with Register, both quotient and mod results
7652 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7653                              rFlagsReg cr)
7654 %{
7655   match(DivModL rax div);
7656   effect(KILL cr);
7657 
7658   ins_cost(30*100+10*100); // XXX
7659   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7660             "cmpq    rax, rdx\n\t"
7661             "jne,s   normal\n\t"
7662             "xorl    rdx, rdx\n\t"
7663             "cmpq    $div, -1\n\t"
7664             "je,s    done\n"
7665     "normal: cdqq\n\t"
7666             "idivq   $div\n"
7667     "done:"        %}
7668   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7669   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7670   ins_pipe(pipe_slow);
7671 %}
7672 
7673 //----------- DivL-By-Constant-Expansions--------------------------------------
7674 // DivI cases are handled by the compiler
7675 
7676 // Magic constant, reciprocal of 10
7677 instruct loadConL_0x6666666666666667(rRegL dst)
7678 %{
7679   effect(DEF dst);
7680 
7681   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7682   ins_encode(load_immL(dst, 0x6666666666666667));
7683   ins_pipe(ialu_reg);
7684 %}
7685 
7686 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7687 %{
7688   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7689 
7690   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7691   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7692   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7693   ins_pipe(ialu_reg_reg_alu0);
7694 %}
7695 
7696 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7697 %{
7698   effect(USE_DEF dst, KILL cr);
7699 
7700   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7701   opcode(0xC1, 0x7); /* C1 /7 ib */
7702   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7703   ins_pipe(ialu_reg);
7704 %}
7705 
7706 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7707 %{
7708   effect(USE_DEF dst, KILL cr);
7709 
7710   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7711   opcode(0xC1, 0x7); /* C1 /7 ib */
7712   ins_encode(reg_opc_imm_wide(dst, 0x2));
7713   ins_pipe(ialu_reg);
7714 %}
7715 
7716 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7717 %{
7718   match(Set dst (DivL src div));
7719 
7720   ins_cost((5+8)*100);
7721   expand %{
7722     rax_RegL rax;                     // Killed temp
7723     rFlagsReg cr;                     // Killed
7724     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7725     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7726     sarL_rReg_63(src, cr);            // sarq  src, 63
7727     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7728     subL_rReg(dst, src, cr);          // subl  rdx, src
7729   %}
7730 %}
7731 
7732 //-----------------------------------------------------------------------------
7733 
7734 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7735                    rFlagsReg cr)
7736 %{
7737   match(Set rdx (ModI rax div));
7738   effect(KILL rax, KILL cr);
7739 
7740   ins_cost(300); // XXX
7741   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7742             "jne,s   normal\n\t"
7743             "xorl    rdx, rdx\n\t"
7744             "cmpl    $div, -1\n\t"
7745             "je,s    done\n"
7746     "normal: cdql\n\t"
7747             "idivl   $div\n"
7748     "done:"        %}
7749   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7750   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7751   ins_pipe(ialu_reg_reg_alu0);
7752 %}
7753 
7754 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7755                    rFlagsReg cr)
7756 %{
7757   match(Set rdx (ModL rax div));
7758   effect(KILL rax, KILL cr);
7759 
7760   ins_cost(300); // XXX
7761   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7762             "cmpq    rax, rdx\n\t"
7763             "jne,s   normal\n\t"
7764             "xorl    rdx, rdx\n\t"
7765             "cmpq    $div, -1\n\t"
7766             "je,s    done\n"
7767     "normal: cdqq\n\t"
7768             "idivq   $div\n"
7769     "done:"        %}
7770   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7771   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7772   ins_pipe(ialu_reg_reg_alu0);
7773 %}
7774 
7775 // Integer Shift Instructions
7776 // Shift Left by one
7777 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7778 %{
7779   match(Set dst (LShiftI dst shift));
7780   effect(KILL cr);
7781 
7782   format %{ "sall    $dst, $shift" %}
7783   opcode(0xD1, 0x4); /* D1 /4 */
7784   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7785   ins_pipe(ialu_reg);
7786 %}
7787 
7788 // Shift Left by one
7789 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7790 %{
7791   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7792   effect(KILL cr);
7793 
7794   format %{ "sall    $dst, $shift\t" %}
7795   opcode(0xD1, 0x4); /* D1 /4 */
7796   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7797   ins_pipe(ialu_mem_imm);
7798 %}
7799 
7800 // Shift Left by 8-bit immediate
7801 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7802 %{
7803   match(Set dst (LShiftI dst shift));
7804   effect(KILL cr);
7805 
7806   format %{ "sall    $dst, $shift" %}
7807   opcode(0xC1, 0x4); /* C1 /4 ib */
7808   ins_encode(reg_opc_imm(dst, shift));
7809   ins_pipe(ialu_reg);
7810 %}
7811 
7812 // Shift Left by 8-bit immediate
7813 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7814 %{
7815   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7816   effect(KILL cr);
7817 
7818   format %{ "sall    $dst, $shift" %}
7819   opcode(0xC1, 0x4); /* C1 /4 ib */
7820   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7821   ins_pipe(ialu_mem_imm);
7822 %}
7823 
7824 // Shift Left by variable
7825 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7826 %{
7827   match(Set dst (LShiftI dst shift));
7828   effect(KILL cr);
7829 
7830   format %{ "sall    $dst, $shift" %}
7831   opcode(0xD3, 0x4); /* D3 /4 */
7832   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7833   ins_pipe(ialu_reg_reg);
7834 %}
7835 
7836 // Shift Left by variable
7837 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7838 %{
7839   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7840   effect(KILL cr);
7841 
7842   format %{ "sall    $dst, $shift" %}
7843   opcode(0xD3, 0x4); /* D3 /4 */
7844   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7845   ins_pipe(ialu_mem_reg);
7846 %}
7847 
7848 // Arithmetic shift right by one
7849 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7850 %{
7851   match(Set dst (RShiftI dst shift));
7852   effect(KILL cr);
7853 
7854   format %{ "sarl    $dst, $shift" %}
7855   opcode(0xD1, 0x7); /* D1 /7 */
7856   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7857   ins_pipe(ialu_reg);
7858 %}
7859 
7860 // Arithmetic shift right by one
7861 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7862 %{
7863   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7864   effect(KILL cr);
7865 
7866   format %{ "sarl    $dst, $shift" %}
7867   opcode(0xD1, 0x7); /* D1 /7 */
7868   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7869   ins_pipe(ialu_mem_imm);
7870 %}
7871 
7872 // Arithmetic Shift Right by 8-bit immediate
7873 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7874 %{
7875   match(Set dst (RShiftI dst shift));
7876   effect(KILL cr);
7877 
7878   format %{ "sarl    $dst, $shift" %}
7879   opcode(0xC1, 0x7); /* C1 /7 ib */
7880   ins_encode(reg_opc_imm(dst, shift));
7881   ins_pipe(ialu_mem_imm);
7882 %}
7883 
7884 // Arithmetic Shift Right by 8-bit immediate
7885 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7886 %{
7887   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7888   effect(KILL cr);
7889 
7890   format %{ "sarl    $dst, $shift" %}
7891   opcode(0xC1, 0x7); /* C1 /7 ib */
7892   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7893   ins_pipe(ialu_mem_imm);
7894 %}
7895 
7896 // Arithmetic Shift Right by variable
7897 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7898 %{
7899   match(Set dst (RShiftI dst shift));
7900   effect(KILL cr);
7901 
7902   format %{ "sarl    $dst, $shift" %}
7903   opcode(0xD3, 0x7); /* D3 /7 */
7904   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7905   ins_pipe(ialu_reg_reg);
7906 %}
7907 
7908 // Arithmetic Shift Right by variable
7909 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7910 %{
7911   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7912   effect(KILL cr);
7913 
7914   format %{ "sarl    $dst, $shift" %}
7915   opcode(0xD3, 0x7); /* D3 /7 */
7916   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7917   ins_pipe(ialu_mem_reg);
7918 %}
7919 
7920 // Logical shift right by one
7921 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7922 %{
7923   match(Set dst (URShiftI dst shift));
7924   effect(KILL cr);
7925 
7926   format %{ "shrl    $dst, $shift" %}
7927   opcode(0xD1, 0x5); /* D1 /5 */
7928   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7929   ins_pipe(ialu_reg);
7930 %}
7931 
7932 // Logical shift right by one
7933 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7934 %{
7935   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7936   effect(KILL cr);
7937 
7938   format %{ "shrl    $dst, $shift" %}
7939   opcode(0xD1, 0x5); /* D1 /5 */
7940   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7941   ins_pipe(ialu_mem_imm);
7942 %}
7943 
7944 // Logical Shift Right by 8-bit immediate
7945 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7946 %{
7947   match(Set dst (URShiftI dst shift));
7948   effect(KILL cr);
7949 
7950   format %{ "shrl    $dst, $shift" %}
7951   opcode(0xC1, 0x5); /* C1 /5 ib */
7952   ins_encode(reg_opc_imm(dst, shift));
7953   ins_pipe(ialu_reg);
7954 %}
7955 
7956 // Logical Shift Right by 8-bit immediate
7957 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7958 %{
7959   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7960   effect(KILL cr);
7961 
7962   format %{ "shrl    $dst, $shift" %}
7963   opcode(0xC1, 0x5); /* C1 /5 ib */
7964   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7965   ins_pipe(ialu_mem_imm);
7966 %}
7967 
7968 // Logical Shift Right by variable
7969 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7970 %{
7971   match(Set dst (URShiftI dst shift));
7972   effect(KILL cr);
7973 
7974   format %{ "shrl    $dst, $shift" %}
7975   opcode(0xD3, 0x5); /* D3 /5 */
7976   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7977   ins_pipe(ialu_reg_reg);
7978 %}
7979 
7980 // Logical Shift Right by variable
7981 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7982 %{
7983   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7984   effect(KILL cr);
7985 
7986   format %{ "shrl    $dst, $shift" %}
7987   opcode(0xD3, 0x5); /* D3 /5 */
7988   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7989   ins_pipe(ialu_mem_reg);
7990 %}
7991 
7992 // Long Shift Instructions
7993 // Shift Left by one
7994 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
7995 %{
7996   match(Set dst (LShiftL dst shift));
7997   effect(KILL cr);
7998 
7999   format %{ "salq    $dst, $shift" %}
8000   opcode(0xD1, 0x4); /* D1 /4 */
8001   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8002   ins_pipe(ialu_reg);
8003 %}
8004 
8005 // Shift Left by one
8006 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8007 %{
8008   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8009   effect(KILL cr);
8010 
8011   format %{ "salq    $dst, $shift" %}
8012   opcode(0xD1, 0x4); /* D1 /4 */
8013   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8014   ins_pipe(ialu_mem_imm);
8015 %}
8016 
8017 // Shift Left by 8-bit immediate
8018 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8019 %{
8020   match(Set dst (LShiftL dst shift));
8021   effect(KILL cr);
8022 
8023   format %{ "salq    $dst, $shift" %}
8024   opcode(0xC1, 0x4); /* C1 /4 ib */
8025   ins_encode(reg_opc_imm_wide(dst, shift));
8026   ins_pipe(ialu_reg);
8027 %}
8028 
8029 // Shift Left by 8-bit immediate
8030 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8031 %{
8032   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8033   effect(KILL cr);
8034 
8035   format %{ "salq    $dst, $shift" %}
8036   opcode(0xC1, 0x4); /* C1 /4 ib */
8037   ins_encode(REX_mem_wide(dst), OpcP,
8038              RM_opc_mem(secondary, dst), Con8or32(shift));
8039   ins_pipe(ialu_mem_imm);
8040 %}
8041 
8042 // Shift Left by variable
8043 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8044 %{
8045   match(Set dst (LShiftL dst shift));
8046   effect(KILL cr);
8047 
8048   format %{ "salq    $dst, $shift" %}
8049   opcode(0xD3, 0x4); /* D3 /4 */
8050   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8051   ins_pipe(ialu_reg_reg);
8052 %}
8053 
8054 // Shift Left by variable
8055 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8056 %{
8057   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8058   effect(KILL cr);
8059 
8060   format %{ "salq    $dst, $shift" %}
8061   opcode(0xD3, 0x4); /* D3 /4 */
8062   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8063   ins_pipe(ialu_mem_reg);
8064 %}
8065 
8066 // Arithmetic shift right by one
8067 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8068 %{
8069   match(Set dst (RShiftL dst shift));
8070   effect(KILL cr);
8071 
8072   format %{ "sarq    $dst, $shift" %}
8073   opcode(0xD1, 0x7); /* D1 /7 */
8074   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8075   ins_pipe(ialu_reg);
8076 %}
8077 
8078 // Arithmetic shift right by one
8079 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8080 %{
8081   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8082   effect(KILL cr);
8083 
8084   format %{ "sarq    $dst, $shift" %}
8085   opcode(0xD1, 0x7); /* D1 /7 */
8086   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8087   ins_pipe(ialu_mem_imm);
8088 %}
8089 
8090 // Arithmetic Shift Right by 8-bit immediate
8091 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8092 %{
8093   match(Set dst (RShiftL dst shift));
8094   effect(KILL cr);
8095 
8096   format %{ "sarq    $dst, $shift" %}
8097   opcode(0xC1, 0x7); /* C1 /7 ib */
8098   ins_encode(reg_opc_imm_wide(dst, shift));
8099   ins_pipe(ialu_mem_imm);
8100 %}
8101 
8102 // Arithmetic Shift Right by 8-bit immediate
8103 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8104 %{
8105   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8106   effect(KILL cr);
8107 
8108   format %{ "sarq    $dst, $shift" %}
8109   opcode(0xC1, 0x7); /* C1 /7 ib */
8110   ins_encode(REX_mem_wide(dst), OpcP,
8111              RM_opc_mem(secondary, dst), Con8or32(shift));
8112   ins_pipe(ialu_mem_imm);
8113 %}
8114 
8115 // Arithmetic Shift Right by variable
8116 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8117 %{
8118   match(Set dst (RShiftL dst shift));
8119   effect(KILL cr);
8120 
8121   format %{ "sarq    $dst, $shift" %}
8122   opcode(0xD3, 0x7); /* D3 /7 */
8123   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8124   ins_pipe(ialu_reg_reg);
8125 %}
8126 
8127 // Arithmetic Shift Right by variable
8128 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8129 %{
8130   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8131   effect(KILL cr);
8132 
8133   format %{ "sarq    $dst, $shift" %}
8134   opcode(0xD3, 0x7); /* D3 /7 */
8135   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8136   ins_pipe(ialu_mem_reg);
8137 %}
8138 
8139 // Logical shift right by one
8140 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8141 %{
8142   match(Set dst (URShiftL dst shift));
8143   effect(KILL cr);
8144 
8145   format %{ "shrq    $dst, $shift" %}
8146   opcode(0xD1, 0x5); /* D1 /5 */
8147   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8148   ins_pipe(ialu_reg);
8149 %}
8150 
8151 // Logical shift right by one
8152 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8153 %{
8154   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8155   effect(KILL cr);
8156 
8157   format %{ "shrq    $dst, $shift" %}
8158   opcode(0xD1, 0x5); /* D1 /5 */
8159   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8160   ins_pipe(ialu_mem_imm);
8161 %}
8162 
8163 // Logical Shift Right by 8-bit immediate
8164 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8165 %{
8166   match(Set dst (URShiftL dst shift));
8167   effect(KILL cr);
8168 
8169   format %{ "shrq    $dst, $shift" %}
8170   opcode(0xC1, 0x5); /* C1 /5 ib */
8171   ins_encode(reg_opc_imm_wide(dst, shift));
8172   ins_pipe(ialu_reg);
8173 %}
8174 
8175 
8176 // Logical Shift Right by 8-bit immediate
8177 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8178 %{
8179   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8180   effect(KILL cr);
8181 
8182   format %{ "shrq    $dst, $shift" %}
8183   opcode(0xC1, 0x5); /* C1 /5 ib */
8184   ins_encode(REX_mem_wide(dst), OpcP,
8185              RM_opc_mem(secondary, dst), Con8or32(shift));
8186   ins_pipe(ialu_mem_imm);
8187 %}
8188 
8189 // Logical Shift Right by variable
8190 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8191 %{
8192   match(Set dst (URShiftL dst shift));
8193   effect(KILL cr);
8194 
8195   format %{ "shrq    $dst, $shift" %}
8196   opcode(0xD3, 0x5); /* D3 /5 */
8197   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8198   ins_pipe(ialu_reg_reg);
8199 %}
8200 
8201 // Logical Shift Right by variable
8202 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8203 %{
8204   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8205   effect(KILL cr);
8206 
8207   format %{ "shrq    $dst, $shift" %}
8208   opcode(0xD3, 0x5); /* D3 /5 */
8209   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8210   ins_pipe(ialu_mem_reg);
8211 %}
8212 
8213 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8214 // This idiom is used by the compiler for the i2b bytecode.
8215 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8216 %{
8217   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8218 
8219   format %{ "movsbl  $dst, $src\t# i2b" %}
8220   opcode(0x0F, 0xBE);
8221   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8222   ins_pipe(ialu_reg_reg);
8223 %}
8224 
8225 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8226 // This idiom is used by the compiler the i2s bytecode.
8227 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8228 %{
8229   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8230 
8231   format %{ "movswl  $dst, $src\t# i2s" %}
8232   opcode(0x0F, 0xBF);
8233   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8234   ins_pipe(ialu_reg_reg);
8235 %}
8236 
8237 // ROL/ROR instructions
8238 
8239 // ROL expand
8240 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8241   effect(KILL cr, USE_DEF dst);
8242 
8243   format %{ "roll    $dst" %}
8244   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8245   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8246   ins_pipe(ialu_reg);
8247 %}
8248 
8249 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8250   effect(USE_DEF dst, USE shift, KILL cr);
8251 
8252   format %{ "roll    $dst, $shift" %}
8253   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8254   ins_encode( reg_opc_imm(dst, shift) );
8255   ins_pipe(ialu_reg);
8256 %}
8257 
8258 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8259 %{
8260   effect(USE_DEF dst, USE shift, KILL cr);
8261 
8262   format %{ "roll    $dst, $shift" %}
8263   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8264   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8265   ins_pipe(ialu_reg_reg);
8266 %}
8267 // end of ROL expand
8268 
8269 // Rotate Left by one
8270 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8271 %{
8272   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8273 
8274   expand %{
8275     rolI_rReg_imm1(dst, cr);
8276   %}
8277 %}
8278 
8279 // Rotate Left by 8-bit immediate
8280 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8281 %{
8282   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8283   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8284 
8285   expand %{
8286     rolI_rReg_imm8(dst, lshift, cr);
8287   %}
8288 %}
8289 
8290 // Rotate Left by variable
8291 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8292 %{
8293   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8294 
8295   expand %{
8296     rolI_rReg_CL(dst, shift, cr);
8297   %}
8298 %}
8299 
8300 // Rotate Left by variable
8301 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8302 %{
8303   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8304 
8305   expand %{
8306     rolI_rReg_CL(dst, shift, cr);
8307   %}
8308 %}
8309 
8310 // ROR expand
8311 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8312 %{
8313   effect(USE_DEF dst, KILL cr);
8314 
8315   format %{ "rorl    $dst" %}
8316   opcode(0xD1, 0x1); /* D1 /1 */
8317   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8318   ins_pipe(ialu_reg);
8319 %}
8320 
8321 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8322 %{
8323   effect(USE_DEF dst, USE shift, KILL cr);
8324 
8325   format %{ "rorl    $dst, $shift" %}
8326   opcode(0xC1, 0x1); /* C1 /1 ib */
8327   ins_encode(reg_opc_imm(dst, shift));
8328   ins_pipe(ialu_reg);
8329 %}
8330 
8331 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8332 %{
8333   effect(USE_DEF dst, USE shift, KILL cr);
8334 
8335   format %{ "rorl    $dst, $shift" %}
8336   opcode(0xD3, 0x1); /* D3 /1 */
8337   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8338   ins_pipe(ialu_reg_reg);
8339 %}
8340 // end of ROR expand
8341 
8342 // Rotate Right by one
8343 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8344 %{
8345   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8346 
8347   expand %{
8348     rorI_rReg_imm1(dst, cr);
8349   %}
8350 %}
8351 
8352 // Rotate Right by 8-bit immediate
8353 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8354 %{
8355   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8356   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8357 
8358   expand %{
8359     rorI_rReg_imm8(dst, rshift, cr);
8360   %}
8361 %}
8362 
8363 // Rotate Right by variable
8364 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8365 %{
8366   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8367 
8368   expand %{
8369     rorI_rReg_CL(dst, shift, cr);
8370   %}
8371 %}
8372 
8373 // Rotate Right by variable
8374 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8375 %{
8376   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8377 
8378   expand %{
8379     rorI_rReg_CL(dst, shift, cr);
8380   %}
8381 %}
8382 
8383 // for long rotate
8384 // ROL expand
8385 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8386   effect(USE_DEF dst, KILL cr);
8387 
8388   format %{ "rolq    $dst" %}
8389   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8390   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8391   ins_pipe(ialu_reg);
8392 %}
8393 
8394 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8395   effect(USE_DEF dst, USE shift, KILL cr);
8396 
8397   format %{ "rolq    $dst, $shift" %}
8398   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8399   ins_encode( reg_opc_imm_wide(dst, shift) );
8400   ins_pipe(ialu_reg);
8401 %}
8402 
8403 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8404 %{
8405   effect(USE_DEF dst, USE shift, KILL cr);
8406 
8407   format %{ "rolq    $dst, $shift" %}
8408   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8409   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8410   ins_pipe(ialu_reg_reg);
8411 %}
8412 // end of ROL expand
8413 
8414 // Rotate Left by one
8415 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8416 %{
8417   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8418 
8419   expand %{
8420     rolL_rReg_imm1(dst, cr);
8421   %}
8422 %}
8423 
8424 // Rotate Left by 8-bit immediate
8425 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8426 %{
8427   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8428   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8429 
8430   expand %{
8431     rolL_rReg_imm8(dst, lshift, cr);
8432   %}
8433 %}
8434 
8435 // Rotate Left by variable
8436 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8437 %{
8438   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8439 
8440   expand %{
8441     rolL_rReg_CL(dst, shift, cr);
8442   %}
8443 %}
8444 
8445 // Rotate Left by variable
8446 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8447 %{
8448   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8449 
8450   expand %{
8451     rolL_rReg_CL(dst, shift, cr);
8452   %}
8453 %}
8454 
8455 // ROR expand
8456 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8457 %{
8458   effect(USE_DEF dst, KILL cr);
8459 
8460   format %{ "rorq    $dst" %}
8461   opcode(0xD1, 0x1); /* D1 /1 */
8462   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8463   ins_pipe(ialu_reg);
8464 %}
8465 
8466 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8467 %{
8468   effect(USE_DEF dst, USE shift, KILL cr);
8469 
8470   format %{ "rorq    $dst, $shift" %}
8471   opcode(0xC1, 0x1); /* C1 /1 ib */
8472   ins_encode(reg_opc_imm_wide(dst, shift));
8473   ins_pipe(ialu_reg);
8474 %}
8475 
8476 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8477 %{
8478   effect(USE_DEF dst, USE shift, KILL cr);
8479 
8480   format %{ "rorq    $dst, $shift" %}
8481   opcode(0xD3, 0x1); /* D3 /1 */
8482   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8483   ins_pipe(ialu_reg_reg);
8484 %}
8485 // end of ROR expand
8486 
8487 // Rotate Right by one
8488 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8489 %{
8490   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8491 
8492   expand %{
8493     rorL_rReg_imm1(dst, cr);
8494   %}
8495 %}
8496 
8497 // Rotate Right by 8-bit immediate
8498 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8499 %{
8500   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8501   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8502 
8503   expand %{
8504     rorL_rReg_imm8(dst, rshift, cr);
8505   %}
8506 %}
8507 
8508 // Rotate Right by variable
8509 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8510 %{
8511   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8512 
8513   expand %{
8514     rorL_rReg_CL(dst, shift, cr);
8515   %}
8516 %}
8517 
8518 // Rotate Right by variable
8519 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8520 %{
8521   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8522 
8523   expand %{
8524     rorL_rReg_CL(dst, shift, cr);
8525   %}
8526 %}
8527 
8528 // Logical Instructions
8529 
8530 // Integer Logical Instructions
8531 
8532 // And Instructions
8533 // And Register with Register
8534 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8535 %{
8536   match(Set dst (AndI dst src));
8537   effect(KILL cr);
8538 
8539   format %{ "andl    $dst, $src\t# int" %}
8540   opcode(0x23);
8541   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8542   ins_pipe(ialu_reg_reg);
8543 %}
8544 
8545 // And Register with Immediate 255
8546 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8547 %{
8548   match(Set dst (AndI dst src));
8549 
8550   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8551   opcode(0x0F, 0xB6);
8552   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 // And Register with Immediate 255 and promote to long
8557 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8558 %{
8559   match(Set dst (ConvI2L (AndI src mask)));
8560 
8561   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8562   opcode(0x0F, 0xB6);
8563   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8564   ins_pipe(ialu_reg);
8565 %}
8566 
8567 // And Register with Immediate 65535
8568 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8569 %{
8570   match(Set dst (AndI dst src));
8571 
8572   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8573   opcode(0x0F, 0xB7);
8574   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8575   ins_pipe(ialu_reg);
8576 %}
8577 
8578 // And Register with Immediate 65535 and promote to long
8579 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8580 %{
8581   match(Set dst (ConvI2L (AndI src mask)));
8582 
8583   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8584   opcode(0x0F, 0xB7);
8585   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8586   ins_pipe(ialu_reg);
8587 %}
8588 
8589 // And Register with Immediate
8590 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8591 %{
8592   match(Set dst (AndI dst src));
8593   effect(KILL cr);
8594 
8595   format %{ "andl    $dst, $src\t# int" %}
8596   opcode(0x81, 0x04); /* Opcode 81 /4 */
8597   ins_encode(OpcSErm(dst, src), Con8or32(src));
8598   ins_pipe(ialu_reg);
8599 %}
8600 
8601 // And Register with Memory
8602 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8603 %{
8604   match(Set dst (AndI dst (LoadI src)));
8605   effect(KILL cr);
8606 
8607   ins_cost(125);
8608   format %{ "andl    $dst, $src\t# int" %}
8609   opcode(0x23);
8610   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8611   ins_pipe(ialu_reg_mem);
8612 %}
8613 
8614 // And Memory with Register
8615 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8616 %{
8617   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8618   effect(KILL cr);
8619 
8620   ins_cost(150);
8621   format %{ "andl    $dst, $src\t# int" %}
8622   opcode(0x21); /* Opcode 21 /r */
8623   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8624   ins_pipe(ialu_mem_reg);
8625 %}
8626 
8627 // And Memory with Immediate
8628 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8629 %{
8630   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8631   effect(KILL cr);
8632 
8633   ins_cost(125);
8634   format %{ "andl    $dst, $src\t# int" %}
8635   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8636   ins_encode(REX_mem(dst), OpcSE(src),
8637              RM_opc_mem(secondary, dst), Con8or32(src));
8638   ins_pipe(ialu_mem_imm);
8639 %}
8640 
8641 // BMI1 instructions
8642 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8643   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8644   predicate(UseBMI1Instructions);
8645   effect(KILL cr);
8646 
8647   ins_cost(125);
8648   format %{ "andnl  $dst, $src1, $src2" %}
8649 
8650   ins_encode %{
8651     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8652   %}
8653   ins_pipe(ialu_reg_mem);
8654 %}
8655 
8656 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8657   match(Set dst (AndI (XorI src1 minus_1) src2));
8658   predicate(UseBMI1Instructions);
8659   effect(KILL cr);
8660 
8661   format %{ "andnl  $dst, $src1, $src2" %}
8662 
8663   ins_encode %{
8664     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8665   %}
8666   ins_pipe(ialu_reg);
8667 %}
8668 
8669 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8670   match(Set dst (AndI (SubI imm_zero src) src));
8671   predicate(UseBMI1Instructions);
8672   effect(KILL cr);
8673 
8674   format %{ "blsil  $dst, $src" %}
8675 
8676   ins_encode %{
8677     __ blsil($dst$$Register, $src$$Register);
8678   %}
8679   ins_pipe(ialu_reg);
8680 %}
8681 
8682 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8683   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8684   predicate(UseBMI1Instructions);
8685   effect(KILL cr);
8686 
8687   ins_cost(125);
8688   format %{ "blsil  $dst, $src" %}
8689 
8690   ins_encode %{
8691     __ blsil($dst$$Register, $src$$Address);
8692   %}
8693   ins_pipe(ialu_reg_mem);
8694 %}
8695 
8696 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8697 %{
8698   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8699   predicate(UseBMI1Instructions);
8700   effect(KILL cr);
8701 
8702   ins_cost(125);
8703   format %{ "blsmskl $dst, $src" %}
8704 
8705   ins_encode %{
8706     __ blsmskl($dst$$Register, $src$$Address);
8707   %}
8708   ins_pipe(ialu_reg_mem);
8709 %}
8710 
8711 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8712 %{
8713   match(Set dst (XorI (AddI src minus_1) src));
8714   predicate(UseBMI1Instructions);
8715   effect(KILL cr);
8716 
8717   format %{ "blsmskl $dst, $src" %}
8718 
8719   ins_encode %{
8720     __ blsmskl($dst$$Register, $src$$Register);
8721   %}
8722 
8723   ins_pipe(ialu_reg);
8724 %}
8725 
8726 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8727 %{
8728   match(Set dst (AndI (AddI src minus_1) src) );
8729   predicate(UseBMI1Instructions);
8730   effect(KILL cr);
8731 
8732   format %{ "blsrl  $dst, $src" %}
8733 
8734   ins_encode %{
8735     __ blsrl($dst$$Register, $src$$Register);
8736   %}
8737 
8738   ins_pipe(ialu_reg_mem);
8739 %}
8740 
8741 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8742 %{
8743   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8744   predicate(UseBMI1Instructions);
8745   effect(KILL cr);
8746 
8747   ins_cost(125);
8748   format %{ "blsrl  $dst, $src" %}
8749 
8750   ins_encode %{
8751     __ blsrl($dst$$Register, $src$$Address);
8752   %}
8753 
8754   ins_pipe(ialu_reg);
8755 %}
8756 
8757 // Or Instructions
8758 // Or Register with Register
8759 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8760 %{
8761   match(Set dst (OrI dst src));
8762   effect(KILL cr);
8763 
8764   format %{ "orl     $dst, $src\t# int" %}
8765   opcode(0x0B);
8766   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8767   ins_pipe(ialu_reg_reg);
8768 %}
8769 
8770 // Or Register with Immediate
8771 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8772 %{
8773   match(Set dst (OrI dst src));
8774   effect(KILL cr);
8775 
8776   format %{ "orl     $dst, $src\t# int" %}
8777   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8778   ins_encode(OpcSErm(dst, src), Con8or32(src));
8779   ins_pipe(ialu_reg);
8780 %}
8781 
8782 // Or Register with Memory
8783 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8784 %{
8785   match(Set dst (OrI dst (LoadI src)));
8786   effect(KILL cr);
8787 
8788   ins_cost(125);
8789   format %{ "orl     $dst, $src\t# int" %}
8790   opcode(0x0B);
8791   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8792   ins_pipe(ialu_reg_mem);
8793 %}
8794 
8795 // Or Memory with Register
8796 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8797 %{
8798   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8799   effect(KILL cr);
8800 
8801   ins_cost(150);
8802   format %{ "orl     $dst, $src\t# int" %}
8803   opcode(0x09); /* Opcode 09 /r */
8804   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8805   ins_pipe(ialu_mem_reg);
8806 %}
8807 
8808 // Or Memory with Immediate
8809 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8810 %{
8811   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8812   effect(KILL cr);
8813 
8814   ins_cost(125);
8815   format %{ "orl     $dst, $src\t# int" %}
8816   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8817   ins_encode(REX_mem(dst), OpcSE(src),
8818              RM_opc_mem(secondary, dst), Con8or32(src));
8819   ins_pipe(ialu_mem_imm);
8820 %}
8821 
8822 // Xor Instructions
8823 // Xor Register with Register
8824 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8825 %{
8826   match(Set dst (XorI dst src));
8827   effect(KILL cr);
8828 
8829   format %{ "xorl    $dst, $src\t# int" %}
8830   opcode(0x33);
8831   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8832   ins_pipe(ialu_reg_reg);
8833 %}
8834 
8835 // Xor Register with Immediate -1
8836 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8837   match(Set dst (XorI dst imm));
8838 
8839   format %{ "not    $dst" %}
8840   ins_encode %{
8841      __ notl($dst$$Register);
8842   %}
8843   ins_pipe(ialu_reg);
8844 %}
8845 
8846 // Xor Register with Immediate
8847 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8848 %{
8849   match(Set dst (XorI dst src));
8850   effect(KILL cr);
8851 
8852   format %{ "xorl    $dst, $src\t# int" %}
8853   opcode(0x81, 0x06); /* Opcode 81 /6 id */
8854   ins_encode(OpcSErm(dst, src), Con8or32(src));
8855   ins_pipe(ialu_reg);
8856 %}
8857 
8858 // Xor Register with Memory
8859 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8860 %{
8861   match(Set dst (XorI dst (LoadI src)));
8862   effect(KILL cr);
8863 
8864   ins_cost(125);
8865   format %{ "xorl    $dst, $src\t# int" %}
8866   opcode(0x33);
8867   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8868   ins_pipe(ialu_reg_mem);
8869 %}
8870 
8871 // Xor Memory with Register
8872 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8873 %{
8874   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8875   effect(KILL cr);
8876 
8877   ins_cost(150);
8878   format %{ "xorl    $dst, $src\t# int" %}
8879   opcode(0x31); /* Opcode 31 /r */
8880   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8881   ins_pipe(ialu_mem_reg);
8882 %}
8883 
8884 // Xor Memory with Immediate
8885 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
8886 %{
8887   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8888   effect(KILL cr);
8889 
8890   ins_cost(125);
8891   format %{ "xorl    $dst, $src\t# int" %}
8892   opcode(0x81, 0x6); /* Opcode 81 /6 id */
8893   ins_encode(REX_mem(dst), OpcSE(src),
8894              RM_opc_mem(secondary, dst), Con8or32(src));
8895   ins_pipe(ialu_mem_imm);
8896 %}
8897 
8898 
8899 // Long Logical Instructions
8900 
8901 // And Instructions
8902 // And Register with Register
8903 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8904 %{
8905   match(Set dst (AndL dst src));
8906   effect(KILL cr);
8907 
8908   format %{ "andq    $dst, $src\t# long" %}
8909   opcode(0x23);
8910   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8911   ins_pipe(ialu_reg_reg);
8912 %}
8913 
8914 // And Register with Immediate 255
8915 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
8916 %{
8917   match(Set dst (AndL dst src));
8918 
8919   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
8920   opcode(0x0F, 0xB6);
8921   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8922   ins_pipe(ialu_reg);
8923 %}
8924 
8925 // And Register with Immediate 65535
8926 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
8927 %{
8928   match(Set dst (AndL dst src));
8929 
8930   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
8931   opcode(0x0F, 0xB7);
8932   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8933   ins_pipe(ialu_reg);
8934 %}
8935 
8936 // And Register with Immediate
8937 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
8938 %{
8939   match(Set dst (AndL dst src));
8940   effect(KILL cr);
8941 
8942   format %{ "andq    $dst, $src\t# long" %}
8943   opcode(0x81, 0x04); /* Opcode 81 /4 */
8944   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8945   ins_pipe(ialu_reg);
8946 %}
8947 
8948 // And Register with Memory
8949 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8950 %{
8951   match(Set dst (AndL dst (LoadL src)));
8952   effect(KILL cr);
8953 
8954   ins_cost(125);
8955   format %{ "andq    $dst, $src\t# long" %}
8956   opcode(0x23);
8957   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8958   ins_pipe(ialu_reg_mem);
8959 %}
8960 
8961 // And Memory with Register
8962 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8963 %{
8964   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8965   effect(KILL cr);
8966 
8967   ins_cost(150);
8968   format %{ "andq    $dst, $src\t# long" %}
8969   opcode(0x21); /* Opcode 21 /r */
8970   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8971   ins_pipe(ialu_mem_reg);
8972 %}
8973 
8974 // And Memory with Immediate
8975 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8976 %{
8977   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8978   effect(KILL cr);
8979 
8980   ins_cost(125);
8981   format %{ "andq    $dst, $src\t# long" %}
8982   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8983   ins_encode(REX_mem_wide(dst), OpcSE(src),
8984              RM_opc_mem(secondary, dst), Con8or32(src));
8985   ins_pipe(ialu_mem_imm);
8986 %}
8987 
8988 // BMI1 instructions
8989 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
8990   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
8991   predicate(UseBMI1Instructions);
8992   effect(KILL cr);
8993 
8994   ins_cost(125);
8995   format %{ "andnq  $dst, $src1, $src2" %}
8996 
8997   ins_encode %{
8998     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
8999   %}
9000   ins_pipe(ialu_reg_mem);
9001 %}
9002 
9003 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9004   match(Set dst (AndL (XorL src1 minus_1) src2));
9005   predicate(UseBMI1Instructions);
9006   effect(KILL cr);
9007 
9008   format %{ "andnq  $dst, $src1, $src2" %}
9009 
9010   ins_encode %{
9011   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9012   %}
9013   ins_pipe(ialu_reg_mem);
9014 %}
9015 
9016 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9017   match(Set dst (AndL (SubL imm_zero src) src));
9018   predicate(UseBMI1Instructions);
9019   effect(KILL cr);
9020 
9021   format %{ "blsiq  $dst, $src" %}
9022 
9023   ins_encode %{
9024     __ blsiq($dst$$Register, $src$$Register);
9025   %}
9026   ins_pipe(ialu_reg);
9027 %}
9028 
9029 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9030   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9031   predicate(UseBMI1Instructions);
9032   effect(KILL cr);
9033 
9034   ins_cost(125);
9035   format %{ "blsiq  $dst, $src" %}
9036 
9037   ins_encode %{
9038     __ blsiq($dst$$Register, $src$$Address);
9039   %}
9040   ins_pipe(ialu_reg_mem);
9041 %}
9042 
9043 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9044 %{
9045   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9046   predicate(UseBMI1Instructions);
9047   effect(KILL cr);
9048 
9049   ins_cost(125);
9050   format %{ "blsmskq $dst, $src" %}
9051 
9052   ins_encode %{
9053     __ blsmskq($dst$$Register, $src$$Address);
9054   %}
9055   ins_pipe(ialu_reg_mem);
9056 %}
9057 
9058 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9059 %{
9060   match(Set dst (XorL (AddL src minus_1) src));
9061   predicate(UseBMI1Instructions);
9062   effect(KILL cr);
9063 
9064   format %{ "blsmskq $dst, $src" %}
9065 
9066   ins_encode %{
9067     __ blsmskq($dst$$Register, $src$$Register);
9068   %}
9069 
9070   ins_pipe(ialu_reg);
9071 %}
9072 
9073 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9074 %{
9075   match(Set dst (AndL (AddL src minus_1) src) );
9076   predicate(UseBMI1Instructions);
9077   effect(KILL cr);
9078 
9079   format %{ "blsrq  $dst, $src" %}
9080 
9081   ins_encode %{
9082     __ blsrq($dst$$Register, $src$$Register);
9083   %}
9084 
9085   ins_pipe(ialu_reg);
9086 %}
9087 
9088 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9089 %{
9090   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9091   predicate(UseBMI1Instructions);
9092   effect(KILL cr);
9093 
9094   ins_cost(125);
9095   format %{ "blsrq  $dst, $src" %}
9096 
9097   ins_encode %{
9098     __ blsrq($dst$$Register, $src$$Address);
9099   %}
9100 
9101   ins_pipe(ialu_reg);
9102 %}
9103 
9104 // Or Instructions
9105 // Or Register with Register
9106 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9107 %{
9108   match(Set dst (OrL dst src));
9109   effect(KILL cr);
9110 
9111   format %{ "orq     $dst, $src\t# long" %}
9112   opcode(0x0B);
9113   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9114   ins_pipe(ialu_reg_reg);
9115 %}
9116 
9117 // Use any_RegP to match R15 (TLS register) without spilling.
9118 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9119   match(Set dst (OrL dst (CastP2X src)));
9120   effect(KILL cr);
9121 
9122   format %{ "orq     $dst, $src\t# long" %}
9123   opcode(0x0B);
9124   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9125   ins_pipe(ialu_reg_reg);
9126 %}
9127 
9128 
9129 // Or Register with Immediate
9130 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9131 %{
9132   match(Set dst (OrL dst src));
9133   effect(KILL cr);
9134 
9135   format %{ "orq     $dst, $src\t# long" %}
9136   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9137   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9138   ins_pipe(ialu_reg);
9139 %}
9140 
9141 // Or Register with Memory
9142 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9143 %{
9144   match(Set dst (OrL dst (LoadL src)));
9145   effect(KILL cr);
9146 
9147   ins_cost(125);
9148   format %{ "orq     $dst, $src\t# long" %}
9149   opcode(0x0B);
9150   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9151   ins_pipe(ialu_reg_mem);
9152 %}
9153 
9154 // Or Memory with Register
9155 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9156 %{
9157   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9158   effect(KILL cr);
9159 
9160   ins_cost(150);
9161   format %{ "orq     $dst, $src\t# long" %}
9162   opcode(0x09); /* Opcode 09 /r */
9163   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9164   ins_pipe(ialu_mem_reg);
9165 %}
9166 
9167 // Or Memory with Immediate
9168 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9169 %{
9170   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9171   effect(KILL cr);
9172 
9173   ins_cost(125);
9174   format %{ "orq     $dst, $src\t# long" %}
9175   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9176   ins_encode(REX_mem_wide(dst), OpcSE(src),
9177              RM_opc_mem(secondary, dst), Con8or32(src));
9178   ins_pipe(ialu_mem_imm);
9179 %}
9180 
9181 // Xor Instructions
9182 // Xor Register with Register
9183 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9184 %{
9185   match(Set dst (XorL dst src));
9186   effect(KILL cr);
9187 
9188   format %{ "xorq    $dst, $src\t# long" %}
9189   opcode(0x33);
9190   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9191   ins_pipe(ialu_reg_reg);
9192 %}
9193 
9194 // Xor Register with Immediate -1
9195 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9196   match(Set dst (XorL dst imm));
9197 
9198   format %{ "notq   $dst" %}
9199   ins_encode %{
9200      __ notq($dst$$Register);
9201   %}
9202   ins_pipe(ialu_reg);
9203 %}
9204 
9205 // Xor Register with Immediate
9206 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9207 %{
9208   match(Set dst (XorL dst src));
9209   effect(KILL cr);
9210 
9211   format %{ "xorq    $dst, $src\t# long" %}
9212   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9213   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9214   ins_pipe(ialu_reg);
9215 %}
9216 
9217 // Xor Register with Memory
9218 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9219 %{
9220   match(Set dst (XorL dst (LoadL src)));
9221   effect(KILL cr);
9222 
9223   ins_cost(125);
9224   format %{ "xorq    $dst, $src\t# long" %}
9225   opcode(0x33);
9226   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9227   ins_pipe(ialu_reg_mem);
9228 %}
9229 
9230 // Xor Memory with Register
9231 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9232 %{
9233   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9234   effect(KILL cr);
9235 
9236   ins_cost(150);
9237   format %{ "xorq    $dst, $src\t# long" %}
9238   opcode(0x31); /* Opcode 31 /r */
9239   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9240   ins_pipe(ialu_mem_reg);
9241 %}
9242 
9243 // Xor Memory with Immediate
9244 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9245 %{
9246   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9247   effect(KILL cr);
9248 
9249   ins_cost(125);
9250   format %{ "xorq    $dst, $src\t# long" %}
9251   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9252   ins_encode(REX_mem_wide(dst), OpcSE(src),
9253              RM_opc_mem(secondary, dst), Con8or32(src));
9254   ins_pipe(ialu_mem_imm);
9255 %}
9256 
9257 // Convert Int to Boolean
9258 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9259 %{
9260   match(Set dst (Conv2B src));
9261   effect(KILL cr);
9262 
9263   format %{ "testl   $src, $src\t# ci2b\n\t"
9264             "setnz   $dst\n\t"
9265             "movzbl  $dst, $dst" %}
9266   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9267              setNZ_reg(dst),
9268              REX_reg_breg(dst, dst), // movzbl
9269              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9270   ins_pipe(pipe_slow); // XXX
9271 %}
9272 
9273 // Convert Pointer to Boolean
9274 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9275 %{
9276   match(Set dst (Conv2B src));
9277   effect(KILL cr);
9278 
9279   format %{ "testq   $src, $src\t# cp2b\n\t"
9280             "setnz   $dst\n\t"
9281             "movzbl  $dst, $dst" %}
9282   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9283              setNZ_reg(dst),
9284              REX_reg_breg(dst, dst), // movzbl
9285              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9286   ins_pipe(pipe_slow); // XXX
9287 %}
9288 
9289 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9290 %{
9291   match(Set dst (CmpLTMask p q));
9292   effect(KILL cr);
9293 
9294   ins_cost(400);
9295   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9296             "setlt   $dst\n\t"
9297             "movzbl  $dst, $dst\n\t"
9298             "negl    $dst" %}
9299   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9300              setLT_reg(dst),
9301              REX_reg_breg(dst, dst), // movzbl
9302              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9303              neg_reg(dst));
9304   ins_pipe(pipe_slow);
9305 %}
9306 
9307 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9308 %{
9309   match(Set dst (CmpLTMask dst zero));
9310   effect(KILL cr);
9311 
9312   ins_cost(100);
9313   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9314   ins_encode %{
9315   __ sarl($dst$$Register, 31);
9316   %}
9317   ins_pipe(ialu_reg);
9318 %}
9319 
9320 /* Better to save a register than avoid a branch */
9321 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9322 %{
9323   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9324   effect(KILL cr);
9325   ins_cost(300);
9326   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9327             "jge    done\n\t"
9328             "addl   $p,$y\n"
9329             "done:  " %}
9330   ins_encode %{
9331     Register Rp = $p$$Register;
9332     Register Rq = $q$$Register;
9333     Register Ry = $y$$Register;
9334     Label done;
9335     __ subl(Rp, Rq);
9336     __ jccb(Assembler::greaterEqual, done);
9337     __ addl(Rp, Ry);
9338     __ bind(done);
9339   %}
9340   ins_pipe(pipe_cmplt);
9341 %}
9342 
9343 /* Better to save a register than avoid a branch */
9344 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9345 %{
9346   match(Set y (AndI (CmpLTMask p q) y));
9347   effect(KILL cr);
9348 
9349   ins_cost(300);
9350 
9351   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9352             "jlt      done\n\t"
9353             "xorl     $y, $y\n"
9354             "done:  " %}
9355   ins_encode %{
9356     Register Rp = $p$$Register;
9357     Register Rq = $q$$Register;
9358     Register Ry = $y$$Register;
9359     Label done;
9360     __ cmpl(Rp, Rq);
9361     __ jccb(Assembler::less, done);
9362     __ xorl(Ry, Ry);
9363     __ bind(done);
9364   %}
9365   ins_pipe(pipe_cmplt);
9366 %}
9367 
9368 
9369 //---------- FP Instructions------------------------------------------------
9370 
9371 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9372 %{
9373   match(Set cr (CmpF src1 src2));
9374 
9375   ins_cost(145);
9376   format %{ "ucomiss $src1, $src2\n\t"
9377             "jnp,s   exit\n\t"
9378             "pushfq\t# saw NaN, set CF\n\t"
9379             "andq    [rsp], #0xffffff2b\n\t"
9380             "popfq\n"
9381     "exit:" %}
9382   ins_encode %{
9383     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9384     emit_cmpfp_fixup(_masm);
9385   %}
9386   ins_pipe(pipe_slow);
9387 %}
9388 
9389 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9390   match(Set cr (CmpF src1 src2));
9391 
9392   ins_cost(100);
9393   format %{ "ucomiss $src1, $src2" %}
9394   ins_encode %{
9395     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9396   %}
9397   ins_pipe(pipe_slow);
9398 %}
9399 
9400 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9401 %{
9402   match(Set cr (CmpF src1 (LoadF src2)));
9403 
9404   ins_cost(145);
9405   format %{ "ucomiss $src1, $src2\n\t"
9406             "jnp,s   exit\n\t"
9407             "pushfq\t# saw NaN, set CF\n\t"
9408             "andq    [rsp], #0xffffff2b\n\t"
9409             "popfq\n"
9410     "exit:" %}
9411   ins_encode %{
9412     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9413     emit_cmpfp_fixup(_masm);
9414   %}
9415   ins_pipe(pipe_slow);
9416 %}
9417 
9418 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9419   match(Set cr (CmpF src1 (LoadF src2)));
9420 
9421   ins_cost(100);
9422   format %{ "ucomiss $src1, $src2" %}
9423   ins_encode %{
9424     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9425   %}
9426   ins_pipe(pipe_slow);
9427 %}
9428 
9429 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9430   match(Set cr (CmpF src con));
9431 
9432   ins_cost(145);
9433   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9434             "jnp,s   exit\n\t"
9435             "pushfq\t# saw NaN, set CF\n\t"
9436             "andq    [rsp], #0xffffff2b\n\t"
9437             "popfq\n"
9438     "exit:" %}
9439   ins_encode %{
9440     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9441     emit_cmpfp_fixup(_masm);
9442   %}
9443   ins_pipe(pipe_slow);
9444 %}
9445 
9446 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9447   match(Set cr (CmpF src con));
9448   ins_cost(100);
9449   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9450   ins_encode %{
9451     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9452   %}
9453   ins_pipe(pipe_slow);
9454 %}
9455 
9456 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9457 %{
9458   match(Set cr (CmpD src1 src2));
9459 
9460   ins_cost(145);
9461   format %{ "ucomisd $src1, $src2\n\t"
9462             "jnp,s   exit\n\t"
9463             "pushfq\t# saw NaN, set CF\n\t"
9464             "andq    [rsp], #0xffffff2b\n\t"
9465             "popfq\n"
9466     "exit:" %}
9467   ins_encode %{
9468     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9469     emit_cmpfp_fixup(_masm);
9470   %}
9471   ins_pipe(pipe_slow);
9472 %}
9473 
9474 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9475   match(Set cr (CmpD src1 src2));
9476 
9477   ins_cost(100);
9478   format %{ "ucomisd $src1, $src2 test" %}
9479   ins_encode %{
9480     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9481   %}
9482   ins_pipe(pipe_slow);
9483 %}
9484 
9485 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9486 %{
9487   match(Set cr (CmpD src1 (LoadD src2)));
9488 
9489   ins_cost(145);
9490   format %{ "ucomisd $src1, $src2\n\t"
9491             "jnp,s   exit\n\t"
9492             "pushfq\t# saw NaN, set CF\n\t"
9493             "andq    [rsp], #0xffffff2b\n\t"
9494             "popfq\n"
9495     "exit:" %}
9496   ins_encode %{
9497     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9498     emit_cmpfp_fixup(_masm);
9499   %}
9500   ins_pipe(pipe_slow);
9501 %}
9502 
9503 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9504   match(Set cr (CmpD src1 (LoadD src2)));
9505 
9506   ins_cost(100);
9507   format %{ "ucomisd $src1, $src2" %}
9508   ins_encode %{
9509     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9510   %}
9511   ins_pipe(pipe_slow);
9512 %}
9513 
9514 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9515   match(Set cr (CmpD src con));
9516 
9517   ins_cost(145);
9518   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9519             "jnp,s   exit\n\t"
9520             "pushfq\t# saw NaN, set CF\n\t"
9521             "andq    [rsp], #0xffffff2b\n\t"
9522             "popfq\n"
9523     "exit:" %}
9524   ins_encode %{
9525     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9526     emit_cmpfp_fixup(_masm);
9527   %}
9528   ins_pipe(pipe_slow);
9529 %}
9530 
9531 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9532   match(Set cr (CmpD src con));
9533   ins_cost(100);
9534   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9535   ins_encode %{
9536     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9537   %}
9538   ins_pipe(pipe_slow);
9539 %}
9540 
9541 // Compare into -1,0,1
9542 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9543 %{
9544   match(Set dst (CmpF3 src1 src2));
9545   effect(KILL cr);
9546 
9547   ins_cost(275);
9548   format %{ "ucomiss $src1, $src2\n\t"
9549             "movl    $dst, #-1\n\t"
9550             "jp,s    done\n\t"
9551             "jb,s    done\n\t"
9552             "setne   $dst\n\t"
9553             "movzbl  $dst, $dst\n"
9554     "done:" %}
9555   ins_encode %{
9556     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9557     emit_cmpfp3(_masm, $dst$$Register);
9558   %}
9559   ins_pipe(pipe_slow);
9560 %}
9561 
9562 // Compare into -1,0,1
9563 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9564 %{
9565   match(Set dst (CmpF3 src1 (LoadF src2)));
9566   effect(KILL cr);
9567 
9568   ins_cost(275);
9569   format %{ "ucomiss $src1, $src2\n\t"
9570             "movl    $dst, #-1\n\t"
9571             "jp,s    done\n\t"
9572             "jb,s    done\n\t"
9573             "setne   $dst\n\t"
9574             "movzbl  $dst, $dst\n"
9575     "done:" %}
9576   ins_encode %{
9577     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9578     emit_cmpfp3(_masm, $dst$$Register);
9579   %}
9580   ins_pipe(pipe_slow);
9581 %}
9582 
9583 // Compare into -1,0,1
9584 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9585   match(Set dst (CmpF3 src con));
9586   effect(KILL cr);
9587 
9588   ins_cost(275);
9589   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9590             "movl    $dst, #-1\n\t"
9591             "jp,s    done\n\t"
9592             "jb,s    done\n\t"
9593             "setne   $dst\n\t"
9594             "movzbl  $dst, $dst\n"
9595     "done:" %}
9596   ins_encode %{
9597     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9598     emit_cmpfp3(_masm, $dst$$Register);
9599   %}
9600   ins_pipe(pipe_slow);
9601 %}
9602 
9603 // Compare into -1,0,1
9604 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9605 %{
9606   match(Set dst (CmpD3 src1 src2));
9607   effect(KILL cr);
9608 
9609   ins_cost(275);
9610   format %{ "ucomisd $src1, $src2\n\t"
9611             "movl    $dst, #-1\n\t"
9612             "jp,s    done\n\t"
9613             "jb,s    done\n\t"
9614             "setne   $dst\n\t"
9615             "movzbl  $dst, $dst\n"
9616     "done:" %}
9617   ins_encode %{
9618     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9619     emit_cmpfp3(_masm, $dst$$Register);
9620   %}
9621   ins_pipe(pipe_slow);
9622 %}
9623 
9624 // Compare into -1,0,1
9625 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9626 %{
9627   match(Set dst (CmpD3 src1 (LoadD src2)));
9628   effect(KILL cr);
9629 
9630   ins_cost(275);
9631   format %{ "ucomisd $src1, $src2\n\t"
9632             "movl    $dst, #-1\n\t"
9633             "jp,s    done\n\t"
9634             "jb,s    done\n\t"
9635             "setne   $dst\n\t"
9636             "movzbl  $dst, $dst\n"
9637     "done:" %}
9638   ins_encode %{
9639     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9640     emit_cmpfp3(_masm, $dst$$Register);
9641   %}
9642   ins_pipe(pipe_slow);
9643 %}
9644 
9645 // Compare into -1,0,1
9646 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9647   match(Set dst (CmpD3 src con));
9648   effect(KILL cr);
9649 
9650   ins_cost(275);
9651   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9652             "movl    $dst, #-1\n\t"
9653             "jp,s    done\n\t"
9654             "jb,s    done\n\t"
9655             "setne   $dst\n\t"
9656             "movzbl  $dst, $dst\n"
9657     "done:" %}
9658   ins_encode %{
9659     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9660     emit_cmpfp3(_masm, $dst$$Register);
9661   %}
9662   ins_pipe(pipe_slow);
9663 %}
9664 
9665 // -----------Trig and Trancendental Instructions------------------------------
9666 instruct cosD_reg(regD dst) %{
9667   match(Set dst (CosD dst));
9668 
9669   format %{ "dcos   $dst\n\t" %}
9670   opcode(0xD9, 0xFF);
9671   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9672   ins_pipe( pipe_slow );
9673 %}
9674 
9675 instruct sinD_reg(regD dst) %{
9676   match(Set dst (SinD dst));
9677 
9678   format %{ "dsin   $dst\n\t" %}
9679   opcode(0xD9, 0xFE);
9680   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9681   ins_pipe( pipe_slow );
9682 %}
9683 
9684 instruct tanD_reg(regD dst) %{
9685   match(Set dst (TanD dst));
9686 
9687   format %{ "dtan   $dst\n\t" %}
9688   ins_encode( Push_SrcXD(dst),
9689               Opcode(0xD9), Opcode(0xF2),   //fptan
9690               Opcode(0xDD), Opcode(0xD8),   //fstp st
9691               Push_ResultXD(dst) );
9692   ins_pipe( pipe_slow );
9693 %}
9694 
9695 instruct log10D_reg(regD dst) %{
9696   // The source and result Double operands in XMM registers
9697   match(Set dst (Log10D dst));
9698   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9699   // fyl2x        ; compute log_10(2) * log_2(x)
9700   format %{ "fldlg2\t\t\t#Log10\n\t"
9701             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9702          %}
9703    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9704               Push_SrcXD(dst),
9705               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9706               Push_ResultXD(dst));
9707 
9708   ins_pipe( pipe_slow );
9709 %}
9710 
9711 instruct logD_reg(regD dst) %{
9712   // The source and result Double operands in XMM registers
9713   match(Set dst (LogD dst));
9714   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9715   // fyl2x        ; compute log_e(2) * log_2(x)
9716   format %{ "fldln2\t\t\t#Log_e\n\t"
9717             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9718          %}
9719   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9720               Push_SrcXD(dst),
9721               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9722               Push_ResultXD(dst));
9723   ins_pipe( pipe_slow );
9724 %}
9725 
9726 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9727   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9728   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9729   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9730   ins_encode %{
9731     __ subptr(rsp, 8);
9732     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9733     __ fld_d(Address(rsp, 0));
9734     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9735     __ fld_d(Address(rsp, 0));
9736     __ fast_pow();
9737     __ fstp_d(Address(rsp, 0));
9738     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9739     __ addptr(rsp, 8);
9740   %}
9741   ins_pipe( pipe_slow );
9742 %}
9743 
9744 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9745   match(Set dst (ExpD src));
9746   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9747   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9748   ins_encode %{
9749     __ subptr(rsp, 8);
9750     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9751     __ fld_d(Address(rsp, 0));
9752     __ fast_exp();
9753     __ fstp_d(Address(rsp, 0));
9754     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9755     __ addptr(rsp, 8);
9756   %}
9757   ins_pipe( pipe_slow );
9758 %}
9759 
9760 //----------Arithmetic Conversion Instructions---------------------------------
9761 
9762 instruct roundFloat_nop(regF dst)
9763 %{
9764   match(Set dst (RoundFloat dst));
9765 
9766   ins_cost(0);
9767   ins_encode();
9768   ins_pipe(empty);
9769 %}
9770 
9771 instruct roundDouble_nop(regD dst)
9772 %{
9773   match(Set dst (RoundDouble dst));
9774 
9775   ins_cost(0);
9776   ins_encode();
9777   ins_pipe(empty);
9778 %}
9779 
9780 instruct convF2D_reg_reg(regD dst, regF src)
9781 %{
9782   match(Set dst (ConvF2D src));
9783 
9784   format %{ "cvtss2sd $dst, $src" %}
9785   ins_encode %{
9786     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9787   %}
9788   ins_pipe(pipe_slow); // XXX
9789 %}
9790 
9791 instruct convF2D_reg_mem(regD dst, memory src)
9792 %{
9793   match(Set dst (ConvF2D (LoadF src)));
9794 
9795   format %{ "cvtss2sd $dst, $src" %}
9796   ins_encode %{
9797     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9798   %}
9799   ins_pipe(pipe_slow); // XXX
9800 %}
9801 
9802 instruct convD2F_reg_reg(regF dst, regD src)
9803 %{
9804   match(Set dst (ConvD2F src));
9805 
9806   format %{ "cvtsd2ss $dst, $src" %}
9807   ins_encode %{
9808     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9809   %}
9810   ins_pipe(pipe_slow); // XXX
9811 %}
9812 
9813 instruct convD2F_reg_mem(regF dst, memory src)
9814 %{
9815   match(Set dst (ConvD2F (LoadD src)));
9816 
9817   format %{ "cvtsd2ss $dst, $src" %}
9818   ins_encode %{
9819     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9820   %}
9821   ins_pipe(pipe_slow); // XXX
9822 %}
9823 
9824 // XXX do mem variants
9825 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9826 %{
9827   match(Set dst (ConvF2I src));
9828   effect(KILL cr);
9829 
9830   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9831             "cmpl    $dst, #0x80000000\n\t"
9832             "jne,s   done\n\t"
9833             "subq    rsp, #8\n\t"
9834             "movss   [rsp], $src\n\t"
9835             "call    f2i_fixup\n\t"
9836             "popq    $dst\n"
9837     "done:   "%}
9838   ins_encode %{
9839     Label done;
9840     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9841     __ cmpl($dst$$Register, 0x80000000);
9842     __ jccb(Assembler::notEqual, done);
9843     __ subptr(rsp, 8);
9844     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9845     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9846     __ pop($dst$$Register);
9847     __ bind(done);
9848   %}
9849   ins_pipe(pipe_slow);
9850 %}
9851 
9852 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9853 %{
9854   match(Set dst (ConvF2L src));
9855   effect(KILL cr);
9856 
9857   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9858             "cmpq    $dst, [0x8000000000000000]\n\t"
9859             "jne,s   done\n\t"
9860             "subq    rsp, #8\n\t"
9861             "movss   [rsp], $src\n\t"
9862             "call    f2l_fixup\n\t"
9863             "popq    $dst\n"
9864     "done:   "%}
9865   ins_encode %{
9866     Label done;
9867     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9868     __ cmp64($dst$$Register,
9869              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9870     __ jccb(Assembler::notEqual, done);
9871     __ subptr(rsp, 8);
9872     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9873     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9874     __ pop($dst$$Register);
9875     __ bind(done);
9876   %}
9877   ins_pipe(pipe_slow);
9878 %}
9879 
9880 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9881 %{
9882   match(Set dst (ConvD2I src));
9883   effect(KILL cr);
9884 
9885   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9886             "cmpl    $dst, #0x80000000\n\t"
9887             "jne,s   done\n\t"
9888             "subq    rsp, #8\n\t"
9889             "movsd   [rsp], $src\n\t"
9890             "call    d2i_fixup\n\t"
9891             "popq    $dst\n"
9892     "done:   "%}
9893   ins_encode %{
9894     Label done;
9895     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
9896     __ cmpl($dst$$Register, 0x80000000);
9897     __ jccb(Assembler::notEqual, done);
9898     __ subptr(rsp, 8);
9899     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9900     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9901     __ pop($dst$$Register);
9902     __ bind(done);
9903   %}
9904   ins_pipe(pipe_slow);
9905 %}
9906 
9907 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
9908 %{
9909   match(Set dst (ConvD2L src));
9910   effect(KILL cr);
9911 
9912   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
9913             "cmpq    $dst, [0x8000000000000000]\n\t"
9914             "jne,s   done\n\t"
9915             "subq    rsp, #8\n\t"
9916             "movsd   [rsp], $src\n\t"
9917             "call    d2l_fixup\n\t"
9918             "popq    $dst\n"
9919     "done:   "%}
9920   ins_encode %{
9921     Label done;
9922     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
9923     __ cmp64($dst$$Register,
9924              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9925     __ jccb(Assembler::notEqual, done);
9926     __ subptr(rsp, 8);
9927     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9928     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9929     __ pop($dst$$Register);
9930     __ bind(done);
9931   %}
9932   ins_pipe(pipe_slow);
9933 %}
9934 
9935 instruct convI2F_reg_reg(regF dst, rRegI src)
9936 %{
9937   predicate(!UseXmmI2F);
9938   match(Set dst (ConvI2F src));
9939 
9940   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9941   ins_encode %{
9942     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
9943   %}
9944   ins_pipe(pipe_slow); // XXX
9945 %}
9946 
9947 instruct convI2F_reg_mem(regF dst, memory src)
9948 %{
9949   match(Set dst (ConvI2F (LoadI src)));
9950 
9951   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9952   ins_encode %{
9953     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
9954   %}
9955   ins_pipe(pipe_slow); // XXX
9956 %}
9957 
9958 instruct convI2D_reg_reg(regD dst, rRegI src)
9959 %{
9960   predicate(!UseXmmI2D);
9961   match(Set dst (ConvI2D src));
9962 
9963   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9964   ins_encode %{
9965     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
9966   %}
9967   ins_pipe(pipe_slow); // XXX
9968 %}
9969 
9970 instruct convI2D_reg_mem(regD dst, memory src)
9971 %{
9972   match(Set dst (ConvI2D (LoadI src)));
9973 
9974   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9975   ins_encode %{
9976     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
9977   %}
9978   ins_pipe(pipe_slow); // XXX
9979 %}
9980 
9981 instruct convXI2F_reg(regF dst, rRegI src)
9982 %{
9983   predicate(UseXmmI2F);
9984   match(Set dst (ConvI2F src));
9985 
9986   format %{ "movdl $dst, $src\n\t"
9987             "cvtdq2psl $dst, $dst\t# i2f" %}
9988   ins_encode %{
9989     __ movdl($dst$$XMMRegister, $src$$Register);
9990     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
9991   %}
9992   ins_pipe(pipe_slow); // XXX
9993 %}
9994 
9995 instruct convXI2D_reg(regD dst, rRegI src)
9996 %{
9997   predicate(UseXmmI2D);
9998   match(Set dst (ConvI2D src));
9999 
10000   format %{ "movdl $dst, $src\n\t"
10001             "cvtdq2pdl $dst, $dst\t# i2d" %}
10002   ins_encode %{
10003     __ movdl($dst$$XMMRegister, $src$$Register);
10004     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10005   %}
10006   ins_pipe(pipe_slow); // XXX
10007 %}
10008 
10009 instruct convL2F_reg_reg(regF dst, rRegL src)
10010 %{
10011   match(Set dst (ConvL2F src));
10012 
10013   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10014   ins_encode %{
10015     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10016   %}
10017   ins_pipe(pipe_slow); // XXX
10018 %}
10019 
10020 instruct convL2F_reg_mem(regF dst, memory src)
10021 %{
10022   match(Set dst (ConvL2F (LoadL src)));
10023 
10024   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10025   ins_encode %{
10026     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10027   %}
10028   ins_pipe(pipe_slow); // XXX
10029 %}
10030 
10031 instruct convL2D_reg_reg(regD dst, rRegL src)
10032 %{
10033   match(Set dst (ConvL2D src));
10034 
10035   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10036   ins_encode %{
10037     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10038   %}
10039   ins_pipe(pipe_slow); // XXX
10040 %}
10041 
10042 instruct convL2D_reg_mem(regD dst, memory src)
10043 %{
10044   match(Set dst (ConvL2D (LoadL src)));
10045 
10046   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10047   ins_encode %{
10048     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10049   %}
10050   ins_pipe(pipe_slow); // XXX
10051 %}
10052 
10053 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10054 %{
10055   match(Set dst (ConvI2L src));
10056 
10057   ins_cost(125);
10058   format %{ "movslq  $dst, $src\t# i2l" %}
10059   ins_encode %{
10060     __ movslq($dst$$Register, $src$$Register);
10061   %}
10062   ins_pipe(ialu_reg_reg);
10063 %}
10064 
10065 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10066 // %{
10067 //   match(Set dst (ConvI2L src));
10068 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10069 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10070 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10071 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10072 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10073 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10074 
10075 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10076 //   ins_encode(enc_copy(dst, src));
10077 // //   opcode(0x63); // needs REX.W
10078 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10079 //   ins_pipe(ialu_reg_reg);
10080 // %}
10081 
10082 // Zero-extend convert int to long
10083 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10084 %{
10085   match(Set dst (AndL (ConvI2L src) mask));
10086 
10087   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10088   ins_encode %{
10089     if ($dst$$reg != $src$$reg) {
10090       __ movl($dst$$Register, $src$$Register);
10091     }
10092   %}
10093   ins_pipe(ialu_reg_reg);
10094 %}
10095 
10096 // Zero-extend convert int to long
10097 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10098 %{
10099   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10100 
10101   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10102   ins_encode %{
10103     __ movl($dst$$Register, $src$$Address);
10104   %}
10105   ins_pipe(ialu_reg_mem);
10106 %}
10107 
10108 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10109 %{
10110   match(Set dst (AndL src mask));
10111 
10112   format %{ "movl    $dst, $src\t# zero-extend long" %}
10113   ins_encode %{
10114     __ movl($dst$$Register, $src$$Register);
10115   %}
10116   ins_pipe(ialu_reg_reg);
10117 %}
10118 
10119 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10120 %{
10121   match(Set dst (ConvL2I src));
10122 
10123   format %{ "movl    $dst, $src\t# l2i" %}
10124   ins_encode %{
10125     __ movl($dst$$Register, $src$$Register);
10126   %}
10127   ins_pipe(ialu_reg_reg);
10128 %}
10129 
10130 
10131 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10132   match(Set dst (MoveF2I src));
10133   effect(DEF dst, USE src);
10134 
10135   ins_cost(125);
10136   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10137   ins_encode %{
10138     __ movl($dst$$Register, Address(rsp, $src$$disp));
10139   %}
10140   ins_pipe(ialu_reg_mem);
10141 %}
10142 
10143 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10144   match(Set dst (MoveI2F src));
10145   effect(DEF dst, USE src);
10146 
10147   ins_cost(125);
10148   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10149   ins_encode %{
10150     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10151   %}
10152   ins_pipe(pipe_slow);
10153 %}
10154 
10155 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10156   match(Set dst (MoveD2L src));
10157   effect(DEF dst, USE src);
10158 
10159   ins_cost(125);
10160   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10161   ins_encode %{
10162     __ movq($dst$$Register, Address(rsp, $src$$disp));
10163   %}
10164   ins_pipe(ialu_reg_mem);
10165 %}
10166 
10167 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10168   predicate(!UseXmmLoadAndClearUpper);
10169   match(Set dst (MoveL2D src));
10170   effect(DEF dst, USE src);
10171 
10172   ins_cost(125);
10173   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10174   ins_encode %{
10175     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10176   %}
10177   ins_pipe(pipe_slow);
10178 %}
10179 
10180 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10181   predicate(UseXmmLoadAndClearUpper);
10182   match(Set dst (MoveL2D src));
10183   effect(DEF dst, USE src);
10184 
10185   ins_cost(125);
10186   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10187   ins_encode %{
10188     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10189   %}
10190   ins_pipe(pipe_slow);
10191 %}
10192 
10193 
10194 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10195   match(Set dst (MoveF2I src));
10196   effect(DEF dst, USE src);
10197 
10198   ins_cost(95); // XXX
10199   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10200   ins_encode %{
10201     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10202   %}
10203   ins_pipe(pipe_slow);
10204 %}
10205 
10206 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10207   match(Set dst (MoveI2F src));
10208   effect(DEF dst, USE src);
10209 
10210   ins_cost(100);
10211   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10212   ins_encode %{
10213     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10214   %}
10215   ins_pipe( ialu_mem_reg );
10216 %}
10217 
10218 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10219   match(Set dst (MoveD2L src));
10220   effect(DEF dst, USE src);
10221 
10222   ins_cost(95); // XXX
10223   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10224   ins_encode %{
10225     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10226   %}
10227   ins_pipe(pipe_slow);
10228 %}
10229 
10230 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10231   match(Set dst (MoveL2D src));
10232   effect(DEF dst, USE src);
10233 
10234   ins_cost(100);
10235   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10236   ins_encode %{
10237     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10238   %}
10239   ins_pipe(ialu_mem_reg);
10240 %}
10241 
10242 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10243   match(Set dst (MoveF2I src));
10244   effect(DEF dst, USE src);
10245   ins_cost(85);
10246   format %{ "movd    $dst,$src\t# MoveF2I" %}
10247   ins_encode %{
10248     __ movdl($dst$$Register, $src$$XMMRegister);
10249   %}
10250   ins_pipe( pipe_slow );
10251 %}
10252 
10253 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10254   match(Set dst (MoveD2L src));
10255   effect(DEF dst, USE src);
10256   ins_cost(85);
10257   format %{ "movd    $dst,$src\t# MoveD2L" %}
10258   ins_encode %{
10259     __ movdq($dst$$Register, $src$$XMMRegister);
10260   %}
10261   ins_pipe( pipe_slow );
10262 %}
10263 
10264 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10265   match(Set dst (MoveI2F src));
10266   effect(DEF dst, USE src);
10267   ins_cost(100);
10268   format %{ "movd    $dst,$src\t# MoveI2F" %}
10269   ins_encode %{
10270     __ movdl($dst$$XMMRegister, $src$$Register);
10271   %}
10272   ins_pipe( pipe_slow );
10273 %}
10274 
10275 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10276   match(Set dst (MoveL2D src));
10277   effect(DEF dst, USE src);
10278   ins_cost(100);
10279   format %{ "movd    $dst,$src\t# MoveL2D" %}
10280   ins_encode %{
10281      __ movdq($dst$$XMMRegister, $src$$Register);
10282   %}
10283   ins_pipe( pipe_slow );
10284 %}
10285 
10286 
10287 // =======================================================================
10288 // fast clearing of an array
10289 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10290                   rFlagsReg cr)
10291 %{
10292   predicate(!UseFastStosb);
10293   match(Set dummy (ClearArray cnt base));
10294   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10295 
10296   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10297             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10298   ins_encode %{ 
10299     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10300   %}
10301   ins_pipe(pipe_slow);
10302 %}
10303 
10304 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10305                         rFlagsReg cr)
10306 %{
10307   predicate(UseFastStosb);
10308   match(Set dummy (ClearArray cnt base));
10309   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10310   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10311             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10312             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10313   ins_encode %{ 
10314     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10315   %}
10316   ins_pipe( pipe_slow );
10317 %}
10318 
10319 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10320                         rax_RegI result, regD tmp1, rFlagsReg cr)
10321 %{
10322   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10323   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10324 
10325   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10326   ins_encode %{
10327     __ string_compare($str1$$Register, $str2$$Register,
10328                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10329                       $tmp1$$XMMRegister);
10330   %}
10331   ins_pipe( pipe_slow );
10332 %}
10333 
10334 // fast search of substring with known size.
10335 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10336                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10337 %{
10338   predicate(UseSSE42Intrinsics);
10339   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10340   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10341 
10342   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10343   ins_encode %{
10344     int icnt2 = (int)$int_cnt2$$constant;
10345     if (icnt2 >= 8) {
10346       // IndexOf for constant substrings with size >= 8 elements
10347       // which don't need to be loaded through stack.
10348       __ string_indexofC8($str1$$Register, $str2$$Register,
10349                           $cnt1$$Register, $cnt2$$Register,
10350                           icnt2, $result$$Register,
10351                           $vec$$XMMRegister, $tmp$$Register);
10352     } else {
10353       // Small strings are loaded through stack if they cross page boundary.
10354       __ string_indexof($str1$$Register, $str2$$Register,
10355                         $cnt1$$Register, $cnt2$$Register,
10356                         icnt2, $result$$Register,
10357                         $vec$$XMMRegister, $tmp$$Register);
10358     }
10359   %}
10360   ins_pipe( pipe_slow );
10361 %}
10362 
10363 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10364                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10365 %{
10366   predicate(UseSSE42Intrinsics);
10367   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10368   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10369 
10370   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10371   ins_encode %{
10372     __ string_indexof($str1$$Register, $str2$$Register,
10373                       $cnt1$$Register, $cnt2$$Register,
10374                       (-1), $result$$Register,
10375                       $vec$$XMMRegister, $tmp$$Register);
10376   %}
10377   ins_pipe( pipe_slow );
10378 %}
10379 
10380 // fast string equals
10381 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10382                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10383 %{
10384   match(Set result (StrEquals (Binary str1 str2) cnt));
10385   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10386 
10387   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10388   ins_encode %{
10389     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10390                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10391                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10392   %}
10393   ins_pipe( pipe_slow );
10394 %}
10395 
10396 // fast array equals
10397 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10398                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10399 %{
10400   match(Set result (AryEq ary1 ary2));
10401   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10402   //ins_cost(300);
10403 
10404   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10405   ins_encode %{
10406     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10407                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10408                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10409   %}
10410   ins_pipe( pipe_slow );
10411 %}
10412 
10413 // encode char[] to byte[] in ISO_8859_1
10414 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10415                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10416                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10417   match(Set result (EncodeISOArray src (Binary dst len)));
10418   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10419 
10420   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10421   ins_encode %{
10422     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10423                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10424                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10425   %}
10426   ins_pipe( pipe_slow );
10427 %}
10428 
10429 //----------Overflow Math Instructions-----------------------------------------
10430 
10431 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10432 %{
10433   match(Set cr (OverflowAddI op1 op2));
10434   effect(DEF cr, USE_KILL op1, USE op2);
10435 
10436   format %{ "addl    $op1, $op2\t# overflow check int" %}
10437 
10438   ins_encode %{
10439     __ addl($op1$$Register, $op2$$Register);
10440   %}
10441   ins_pipe(ialu_reg_reg);
10442 %}
10443 
10444 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10445 %{
10446   match(Set cr (OverflowAddI op1 op2));
10447   effect(DEF cr, USE_KILL op1, USE op2);
10448 
10449   format %{ "addl    $op1, $op2\t# overflow check int" %}
10450 
10451   ins_encode %{
10452     __ addl($op1$$Register, $op2$$constant);
10453   %}
10454   ins_pipe(ialu_reg_reg);
10455 %}
10456 
10457 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10458 %{
10459   match(Set cr (OverflowAddL op1 op2));
10460   effect(DEF cr, USE_KILL op1, USE op2);
10461 
10462   format %{ "addq    $op1, $op2\t# overflow check long" %}
10463   ins_encode %{
10464     __ addq($op1$$Register, $op2$$Register);
10465   %}
10466   ins_pipe(ialu_reg_reg);
10467 %}
10468 
10469 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10470 %{
10471   match(Set cr (OverflowAddL op1 op2));
10472   effect(DEF cr, USE_KILL op1, USE op2);
10473 
10474   format %{ "addq    $op1, $op2\t# overflow check long" %}
10475   ins_encode %{
10476     __ addq($op1$$Register, $op2$$constant);
10477   %}
10478   ins_pipe(ialu_reg_reg);
10479 %}
10480 
10481 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10482 %{
10483   match(Set cr (OverflowSubI op1 op2));
10484 
10485   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10486   ins_encode %{
10487     __ cmpl($op1$$Register, $op2$$Register);
10488   %}
10489   ins_pipe(ialu_reg_reg);
10490 %}
10491 
10492 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10493 %{
10494   match(Set cr (OverflowSubI op1 op2));
10495 
10496   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10497   ins_encode %{
10498     __ cmpl($op1$$Register, $op2$$constant);
10499   %}
10500   ins_pipe(ialu_reg_reg);
10501 %}
10502 
10503 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10504 %{
10505   match(Set cr (OverflowSubL op1 op2));
10506 
10507   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10508   ins_encode %{
10509     __ cmpq($op1$$Register, $op2$$Register);
10510   %}
10511   ins_pipe(ialu_reg_reg);
10512 %}
10513 
10514 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10515 %{
10516   match(Set cr (OverflowSubL op1 op2));
10517 
10518   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10519   ins_encode %{
10520     __ cmpq($op1$$Register, $op2$$constant);
10521   %}
10522   ins_pipe(ialu_reg_reg);
10523 %}
10524 
10525 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10526 %{
10527   match(Set cr (OverflowSubI zero op2));
10528   effect(DEF cr, USE_KILL op2);
10529 
10530   format %{ "negl    $op2\t# overflow check int" %}
10531   ins_encode %{
10532     __ negl($op2$$Register);
10533   %}
10534   ins_pipe(ialu_reg_reg);
10535 %}
10536 
10537 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10538 %{
10539   match(Set cr (OverflowSubL zero op2));
10540   effect(DEF cr, USE_KILL op2);
10541 
10542   format %{ "negq    $op2\t# overflow check long" %}
10543   ins_encode %{
10544     __ negq($op2$$Register);
10545   %}
10546   ins_pipe(ialu_reg_reg);
10547 %}
10548 
10549 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10550 %{
10551   match(Set cr (OverflowMulI op1 op2));
10552   effect(DEF cr, USE_KILL op1, USE op2);
10553 
10554   format %{ "imull    $op1, $op2\t# overflow check int" %}
10555   ins_encode %{
10556     __ imull($op1$$Register, $op2$$Register);
10557   %}
10558   ins_pipe(ialu_reg_reg_alu0);
10559 %}
10560 
10561 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10562 %{
10563   match(Set cr (OverflowMulI op1 op2));
10564   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10565 
10566   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10567   ins_encode %{
10568     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10569   %}
10570   ins_pipe(ialu_reg_reg_alu0);
10571 %}
10572 
10573 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10574 %{
10575   match(Set cr (OverflowMulL op1 op2));
10576   effect(DEF cr, USE_KILL op1, USE op2);
10577 
10578   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10579   ins_encode %{
10580     __ imulq($op1$$Register, $op2$$Register);
10581   %}
10582   ins_pipe(ialu_reg_reg_alu0);
10583 %}
10584 
10585 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10586 %{
10587   match(Set cr (OverflowMulL op1 op2));
10588   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10589 
10590   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10591   ins_encode %{
10592     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10593   %}
10594   ins_pipe(ialu_reg_reg_alu0);
10595 %}
10596 
10597 
10598 //----------Control Flow Instructions------------------------------------------
10599 // Signed compare Instructions
10600 
10601 // XXX more variants!!
10602 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10603 %{
10604   match(Set cr (CmpI op1 op2));
10605   effect(DEF cr, USE op1, USE op2);
10606 
10607   format %{ "cmpl    $op1, $op2" %}
10608   opcode(0x3B);  /* Opcode 3B /r */
10609   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10610   ins_pipe(ialu_cr_reg_reg);
10611 %}
10612 
10613 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10614 %{
10615   match(Set cr (CmpI op1 op2));
10616 
10617   format %{ "cmpl    $op1, $op2" %}
10618   opcode(0x81, 0x07); /* Opcode 81 /7 */
10619   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10620   ins_pipe(ialu_cr_reg_imm);
10621 %}
10622 
10623 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10624 %{
10625   match(Set cr (CmpI op1 (LoadI op2)));
10626 
10627   ins_cost(500); // XXX
10628   format %{ "cmpl    $op1, $op2" %}
10629   opcode(0x3B); /* Opcode 3B /r */
10630   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10631   ins_pipe(ialu_cr_reg_mem);
10632 %}
10633 
10634 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10635 %{
10636   match(Set cr (CmpI src zero));
10637 
10638   format %{ "testl   $src, $src" %}
10639   opcode(0x85);
10640   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10641   ins_pipe(ialu_cr_reg_imm);
10642 %}
10643 
10644 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10645 %{
10646   match(Set cr (CmpI (AndI src con) zero));
10647 
10648   format %{ "testl   $src, $con" %}
10649   opcode(0xF7, 0x00);
10650   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10651   ins_pipe(ialu_cr_reg_imm);
10652 %}
10653 
10654 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10655 %{
10656   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10657 
10658   format %{ "testl   $src, $mem" %}
10659   opcode(0x85);
10660   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10661   ins_pipe(ialu_cr_reg_mem);
10662 %}
10663 
10664 // Unsigned compare Instructions; really, same as signed except they
10665 // produce an rFlagsRegU instead of rFlagsReg.
10666 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10667 %{
10668   match(Set cr (CmpU op1 op2));
10669 
10670   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10671   opcode(0x3B); /* Opcode 3B /r */
10672   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10673   ins_pipe(ialu_cr_reg_reg);
10674 %}
10675 
10676 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10677 %{
10678   match(Set cr (CmpU op1 op2));
10679 
10680   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10681   opcode(0x81,0x07); /* Opcode 81 /7 */
10682   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10683   ins_pipe(ialu_cr_reg_imm);
10684 %}
10685 
10686 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10687 %{
10688   match(Set cr (CmpU op1 (LoadI op2)));
10689 
10690   ins_cost(500); // XXX
10691   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10692   opcode(0x3B); /* Opcode 3B /r */
10693   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10694   ins_pipe(ialu_cr_reg_mem);
10695 %}
10696 
10697 // // // Cisc-spilled version of cmpU_rReg
10698 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10699 // //%{
10700 // //  match(Set cr (CmpU (LoadI op1) op2));
10701 // //
10702 // //  format %{ "CMPu   $op1,$op2" %}
10703 // //  ins_cost(500);
10704 // //  opcode(0x39);  /* Opcode 39 /r */
10705 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10706 // //%}
10707 
10708 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10709 %{
10710   match(Set cr (CmpU src zero));
10711 
10712   format %{ "testl  $src, $src\t# unsigned" %}
10713   opcode(0x85);
10714   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10715   ins_pipe(ialu_cr_reg_imm);
10716 %}
10717 
10718 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10719 %{
10720   match(Set cr (CmpP op1 op2));
10721 
10722   format %{ "cmpq    $op1, $op2\t# ptr" %}
10723   opcode(0x3B); /* Opcode 3B /r */
10724   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10725   ins_pipe(ialu_cr_reg_reg);
10726 %}
10727 
10728 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10729 %{
10730   match(Set cr (CmpP op1 (LoadP op2)));
10731 
10732   ins_cost(500); // XXX
10733   format %{ "cmpq    $op1, $op2\t# ptr" %}
10734   opcode(0x3B); /* Opcode 3B /r */
10735   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10736   ins_pipe(ialu_cr_reg_mem);
10737 %}
10738 
10739 // // // Cisc-spilled version of cmpP_rReg
10740 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10741 // //%{
10742 // //  match(Set cr (CmpP (LoadP op1) op2));
10743 // //
10744 // //  format %{ "CMPu   $op1,$op2" %}
10745 // //  ins_cost(500);
10746 // //  opcode(0x39);  /* Opcode 39 /r */
10747 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10748 // //%}
10749 
10750 // XXX this is generalized by compP_rReg_mem???
10751 // Compare raw pointer (used in out-of-heap check).
10752 // Only works because non-oop pointers must be raw pointers
10753 // and raw pointers have no anti-dependencies.
10754 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10755 %{
10756   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10757   match(Set cr (CmpP op1 (LoadP op2)));
10758 
10759   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10760   opcode(0x3B); /* Opcode 3B /r */
10761   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10762   ins_pipe(ialu_cr_reg_mem);
10763 %}
10764 
10765 // This will generate a signed flags result. This should be OK since
10766 // any compare to a zero should be eq/neq.
10767 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10768 %{
10769   match(Set cr (CmpP src zero));
10770 
10771   format %{ "testq   $src, $src\t# ptr" %}
10772   opcode(0x85);
10773   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10774   ins_pipe(ialu_cr_reg_imm);
10775 %}
10776 
10777 // This will generate a signed flags result. This should be OK since
10778 // any compare to a zero should be eq/neq.
10779 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10780 %{
10781   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10782   match(Set cr (CmpP (LoadP op) zero));
10783 
10784   ins_cost(500); // XXX
10785   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10786   opcode(0xF7); /* Opcode F7 /0 */
10787   ins_encode(REX_mem_wide(op),
10788              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10789   ins_pipe(ialu_cr_reg_imm);
10790 %}
10791 
10792 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10793 %{
10794   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10795   match(Set cr (CmpP (LoadP mem) zero));
10796 
10797   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10798   ins_encode %{
10799     __ cmpq(r12, $mem$$Address);
10800   %}
10801   ins_pipe(ialu_cr_reg_mem);
10802 %}
10803 
10804 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10805 %{
10806   match(Set cr (CmpN op1 op2));
10807 
10808   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10809   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10810   ins_pipe(ialu_cr_reg_reg);
10811 %}
10812 
10813 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10814 %{
10815   match(Set cr (CmpN src (LoadN mem)));
10816 
10817   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10818   ins_encode %{
10819     __ cmpl($src$$Register, $mem$$Address);
10820   %}
10821   ins_pipe(ialu_cr_reg_mem);
10822 %}
10823 
10824 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10825   match(Set cr (CmpN op1 op2));
10826 
10827   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10828   ins_encode %{
10829     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10830   %}
10831   ins_pipe(ialu_cr_reg_imm);
10832 %}
10833 
10834 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10835 %{
10836   match(Set cr (CmpN src (LoadN mem)));
10837 
10838   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10839   ins_encode %{
10840     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10841   %}
10842   ins_pipe(ialu_cr_reg_mem);
10843 %}
10844 
10845 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10846   match(Set cr (CmpN op1 op2));
10847 
10848   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10849   ins_encode %{
10850     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10851   %}
10852   ins_pipe(ialu_cr_reg_imm);
10853 %}
10854 
10855 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10856 %{
10857   match(Set cr (CmpN src (LoadNKlass mem)));
10858 
10859   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10860   ins_encode %{
10861     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10862   %}
10863   ins_pipe(ialu_cr_reg_mem);
10864 %}
10865 
10866 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10867   match(Set cr (CmpN src zero));
10868 
10869   format %{ "testl   $src, $src\t# compressed ptr" %}
10870   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10871   ins_pipe(ialu_cr_reg_imm);
10872 %}
10873 
10874 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10875 %{
10876   predicate(Universe::narrow_oop_base() != NULL);
10877   match(Set cr (CmpN (LoadN mem) zero));
10878 
10879   ins_cost(500); // XXX
10880   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10881   ins_encode %{
10882     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10883   %}
10884   ins_pipe(ialu_cr_reg_mem);
10885 %}
10886 
10887 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10888 %{
10889   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10890   match(Set cr (CmpN (LoadN mem) zero));
10891 
10892   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10893   ins_encode %{
10894     __ cmpl(r12, $mem$$Address);
10895   %}
10896   ins_pipe(ialu_cr_reg_mem);
10897 %}
10898 
10899 // Yanked all unsigned pointer compare operations.
10900 // Pointer compares are done with CmpP which is already unsigned.
10901 
10902 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10903 %{
10904   match(Set cr (CmpL op1 op2));
10905 
10906   format %{ "cmpq    $op1, $op2" %}
10907   opcode(0x3B);  /* Opcode 3B /r */
10908   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10909   ins_pipe(ialu_cr_reg_reg);
10910 %}
10911 
10912 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10913 %{
10914   match(Set cr (CmpL op1 op2));
10915 
10916   format %{ "cmpq    $op1, $op2" %}
10917   opcode(0x81, 0x07); /* Opcode 81 /7 */
10918   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
10919   ins_pipe(ialu_cr_reg_imm);
10920 %}
10921 
10922 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
10923 %{
10924   match(Set cr (CmpL op1 (LoadL op2)));
10925 
10926   format %{ "cmpq    $op1, $op2" %}
10927   opcode(0x3B); /* Opcode 3B /r */
10928   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10929   ins_pipe(ialu_cr_reg_mem);
10930 %}
10931 
10932 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
10933 %{
10934   match(Set cr (CmpL src zero));
10935 
10936   format %{ "testq   $src, $src" %}
10937   opcode(0x85);
10938   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10939   ins_pipe(ialu_cr_reg_imm);
10940 %}
10941 
10942 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
10943 %{
10944   match(Set cr (CmpL (AndL src con) zero));
10945 
10946   format %{ "testq   $src, $con\t# long" %}
10947   opcode(0xF7, 0x00);
10948   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
10949   ins_pipe(ialu_cr_reg_imm);
10950 %}
10951 
10952 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
10953 %{
10954   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
10955 
10956   format %{ "testq   $src, $mem" %}
10957   opcode(0x85);
10958   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
10959   ins_pipe(ialu_cr_reg_mem);
10960 %}
10961 
10962 // Manifest a CmpL result in an integer register.  Very painful.
10963 // This is the test to avoid.
10964 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
10965 %{
10966   match(Set dst (CmpL3 src1 src2));
10967   effect(KILL flags);
10968 
10969   ins_cost(275); // XXX
10970   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
10971             "movl    $dst, -1\n\t"
10972             "jl,s    done\n\t"
10973             "setne   $dst\n\t"
10974             "movzbl  $dst, $dst\n\t"
10975     "done:" %}
10976   ins_encode(cmpl3_flag(src1, src2, dst));
10977   ins_pipe(pipe_slow);
10978 %}
10979 
10980 //----------Max and Min--------------------------------------------------------
10981 // Min Instructions
10982 
10983 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
10984 %{
10985   effect(USE_DEF dst, USE src, USE cr);
10986 
10987   format %{ "cmovlgt $dst, $src\t# min" %}
10988   opcode(0x0F, 0x4F);
10989   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10990   ins_pipe(pipe_cmov_reg);
10991 %}
10992 
10993 
10994 instruct minI_rReg(rRegI dst, rRegI src)
10995 %{
10996   match(Set dst (MinI dst src));
10997 
10998   ins_cost(200);
10999   expand %{
11000     rFlagsReg cr;
11001     compI_rReg(cr, dst, src);
11002     cmovI_reg_g(dst, src, cr);
11003   %}
11004 %}
11005 
11006 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11007 %{
11008   effect(USE_DEF dst, USE src, USE cr);
11009 
11010   format %{ "cmovllt $dst, $src\t# max" %}
11011   opcode(0x0F, 0x4C);
11012   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11013   ins_pipe(pipe_cmov_reg);
11014 %}
11015 
11016 
11017 instruct maxI_rReg(rRegI dst, rRegI src)
11018 %{
11019   match(Set dst (MaxI dst src));
11020 
11021   ins_cost(200);
11022   expand %{
11023     rFlagsReg cr;
11024     compI_rReg(cr, dst, src);
11025     cmovI_reg_l(dst, src, cr);
11026   %}
11027 %}
11028 
11029 // ============================================================================
11030 // Branch Instructions
11031 
11032 // Jump Direct - Label defines a relative address from JMP+1
11033 instruct jmpDir(label labl)
11034 %{
11035   match(Goto);
11036   effect(USE labl);
11037 
11038   ins_cost(300);
11039   format %{ "jmp     $labl" %}
11040   size(5);
11041   ins_encode %{
11042     Label* L = $labl$$label;
11043     __ jmp(*L, false); // Always long jump
11044   %}
11045   ins_pipe(pipe_jmp);
11046 %}
11047 
11048 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11049 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11050 %{
11051   match(If cop cr);
11052   effect(USE labl);
11053 
11054   ins_cost(300);
11055   format %{ "j$cop     $labl" %}
11056   size(6);
11057   ins_encode %{
11058     Label* L = $labl$$label;
11059     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11060   %}
11061   ins_pipe(pipe_jcc);
11062 %}
11063 
11064 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11065 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11066 %{
11067   match(CountedLoopEnd cop cr);
11068   effect(USE labl);
11069 
11070   ins_cost(300);
11071   format %{ "j$cop     $labl\t# loop end" %}
11072   size(6);
11073   ins_encode %{
11074     Label* L = $labl$$label;
11075     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11076   %}
11077   ins_pipe(pipe_jcc);
11078 %}
11079 
11080 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11081 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11082   match(CountedLoopEnd cop cmp);
11083   effect(USE labl);
11084 
11085   ins_cost(300);
11086   format %{ "j$cop,u   $labl\t# loop end" %}
11087   size(6);
11088   ins_encode %{
11089     Label* L = $labl$$label;
11090     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11091   %}
11092   ins_pipe(pipe_jcc);
11093 %}
11094 
11095 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11096   match(CountedLoopEnd cop cmp);
11097   effect(USE labl);
11098 
11099   ins_cost(200);
11100   format %{ "j$cop,u   $labl\t# loop end" %}
11101   size(6);
11102   ins_encode %{
11103     Label* L = $labl$$label;
11104     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11105   %}
11106   ins_pipe(pipe_jcc);
11107 %}
11108 
11109 // Jump Direct Conditional - using unsigned comparison
11110 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11111   match(If cop cmp);
11112   effect(USE labl);
11113 
11114   ins_cost(300);
11115   format %{ "j$cop,u  $labl" %}
11116   size(6);
11117   ins_encode %{
11118     Label* L = $labl$$label;
11119     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11120   %}
11121   ins_pipe(pipe_jcc);
11122 %}
11123 
11124 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11125   match(If cop cmp);
11126   effect(USE labl);
11127 
11128   ins_cost(200);
11129   format %{ "j$cop,u  $labl" %}
11130   size(6);
11131   ins_encode %{
11132     Label* L = $labl$$label;
11133     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11134   %}
11135   ins_pipe(pipe_jcc);
11136 %}
11137 
11138 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11139   match(If cop cmp);
11140   effect(USE labl);
11141 
11142   ins_cost(200);
11143   format %{ $$template
11144     if ($cop$$cmpcode == Assembler::notEqual) {
11145       $$emit$$"jp,u   $labl\n\t"
11146       $$emit$$"j$cop,u   $labl"
11147     } else {
11148       $$emit$$"jp,u   done\n\t"
11149       $$emit$$"j$cop,u   $labl\n\t"
11150       $$emit$$"done:"
11151     }
11152   %}
11153   ins_encode %{
11154     Label* l = $labl$$label;
11155     if ($cop$$cmpcode == Assembler::notEqual) {
11156       __ jcc(Assembler::parity, *l, false);
11157       __ jcc(Assembler::notEqual, *l, false);
11158     } else if ($cop$$cmpcode == Assembler::equal) {
11159       Label done;
11160       __ jccb(Assembler::parity, done);
11161       __ jcc(Assembler::equal, *l, false);
11162       __ bind(done);
11163     } else {
11164        ShouldNotReachHere();
11165     }
11166   %}
11167   ins_pipe(pipe_jcc);
11168 %}
11169 
11170 // ============================================================================
11171 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11172 // superklass array for an instance of the superklass.  Set a hidden
11173 // internal cache on a hit (cache is checked with exposed code in
11174 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11175 // encoding ALSO sets flags.
11176 
11177 instruct partialSubtypeCheck(rdi_RegP result,
11178                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11179                              rFlagsReg cr)
11180 %{
11181   match(Set result (PartialSubtypeCheck sub super));
11182   effect(KILL rcx, KILL cr);
11183 
11184   ins_cost(1100);  // slightly larger than the next version
11185   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11186             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11187             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11188             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11189             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11190             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11191             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11192     "miss:\t" %}
11193 
11194   opcode(0x1); // Force a XOR of RDI
11195   ins_encode(enc_PartialSubtypeCheck());
11196   ins_pipe(pipe_slow);
11197 %}
11198 
11199 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11200                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11201                                      immP0 zero,
11202                                      rdi_RegP result)
11203 %{
11204   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11205   effect(KILL rcx, KILL result);
11206 
11207   ins_cost(1000);
11208   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11209             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11210             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11211             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11212             "jne,s   miss\t\t# Missed: flags nz\n\t"
11213             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11214     "miss:\t" %}
11215 
11216   opcode(0x0); // No need to XOR RDI
11217   ins_encode(enc_PartialSubtypeCheck());
11218   ins_pipe(pipe_slow);
11219 %}
11220 
11221 // ============================================================================
11222 // Branch Instructions -- short offset versions
11223 //
11224 // These instructions are used to replace jumps of a long offset (the default
11225 // match) with jumps of a shorter offset.  These instructions are all tagged
11226 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11227 // match rules in general matching.  Instead, the ADLC generates a conversion
11228 // method in the MachNode which can be used to do in-place replacement of the
11229 // long variant with the shorter variant.  The compiler will determine if a
11230 // branch can be taken by the is_short_branch_offset() predicate in the machine
11231 // specific code section of the file.
11232 
11233 // Jump Direct - Label defines a relative address from JMP+1
11234 instruct jmpDir_short(label labl) %{
11235   match(Goto);
11236   effect(USE labl);
11237 
11238   ins_cost(300);
11239   format %{ "jmp,s   $labl" %}
11240   size(2);
11241   ins_encode %{
11242     Label* L = $labl$$label;
11243     __ jmpb(*L);
11244   %}
11245   ins_pipe(pipe_jmp);
11246   ins_short_branch(1);
11247 %}
11248 
11249 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11250 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11251   match(If cop cr);
11252   effect(USE labl);
11253 
11254   ins_cost(300);
11255   format %{ "j$cop,s   $labl" %}
11256   size(2);
11257   ins_encode %{
11258     Label* L = $labl$$label;
11259     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11260   %}
11261   ins_pipe(pipe_jcc);
11262   ins_short_branch(1);
11263 %}
11264 
11265 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11266 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11267   match(CountedLoopEnd cop cr);
11268   effect(USE labl);
11269 
11270   ins_cost(300);
11271   format %{ "j$cop,s   $labl\t# loop end" %}
11272   size(2);
11273   ins_encode %{
11274     Label* L = $labl$$label;
11275     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11276   %}
11277   ins_pipe(pipe_jcc);
11278   ins_short_branch(1);
11279 %}
11280 
11281 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11282 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11283   match(CountedLoopEnd cop cmp);
11284   effect(USE labl);
11285 
11286   ins_cost(300);
11287   format %{ "j$cop,us  $labl\t# loop end" %}
11288   size(2);
11289   ins_encode %{
11290     Label* L = $labl$$label;
11291     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11292   %}
11293   ins_pipe(pipe_jcc);
11294   ins_short_branch(1);
11295 %}
11296 
11297 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11298   match(CountedLoopEnd cop cmp);
11299   effect(USE labl);
11300 
11301   ins_cost(300);
11302   format %{ "j$cop,us  $labl\t# loop end" %}
11303   size(2);
11304   ins_encode %{
11305     Label* L = $labl$$label;
11306     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11307   %}
11308   ins_pipe(pipe_jcc);
11309   ins_short_branch(1);
11310 %}
11311 
11312 // Jump Direct Conditional - using unsigned comparison
11313 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11314   match(If cop cmp);
11315   effect(USE labl);
11316 
11317   ins_cost(300);
11318   format %{ "j$cop,us  $labl" %}
11319   size(2);
11320   ins_encode %{
11321     Label* L = $labl$$label;
11322     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11323   %}
11324   ins_pipe(pipe_jcc);
11325   ins_short_branch(1);
11326 %}
11327 
11328 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11329   match(If cop cmp);
11330   effect(USE labl);
11331 
11332   ins_cost(300);
11333   format %{ "j$cop,us  $labl" %}
11334   size(2);
11335   ins_encode %{
11336     Label* L = $labl$$label;
11337     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11338   %}
11339   ins_pipe(pipe_jcc);
11340   ins_short_branch(1);
11341 %}
11342 
11343 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11344   match(If cop cmp);
11345   effect(USE labl);
11346 
11347   ins_cost(300);
11348   format %{ $$template
11349     if ($cop$$cmpcode == Assembler::notEqual) {
11350       $$emit$$"jp,u,s   $labl\n\t"
11351       $$emit$$"j$cop,u,s   $labl"
11352     } else {
11353       $$emit$$"jp,u,s   done\n\t"
11354       $$emit$$"j$cop,u,s  $labl\n\t"
11355       $$emit$$"done:"
11356     }
11357   %}
11358   size(4);
11359   ins_encode %{
11360     Label* l = $labl$$label;
11361     if ($cop$$cmpcode == Assembler::notEqual) {
11362       __ jccb(Assembler::parity, *l);
11363       __ jccb(Assembler::notEqual, *l);
11364     } else if ($cop$$cmpcode == Assembler::equal) {
11365       Label done;
11366       __ jccb(Assembler::parity, done);
11367       __ jccb(Assembler::equal, *l);
11368       __ bind(done);
11369     } else {
11370        ShouldNotReachHere();
11371     }
11372   %}
11373   ins_pipe(pipe_jcc);
11374   ins_short_branch(1);
11375 %}
11376 
11377 // ============================================================================
11378 // inlined locking and unlocking
11379 
11380 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11381   predicate(Compile::current()->use_rtm());
11382   match(Set cr (FastLock object box));
11383   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11384   ins_cost(300);
11385   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11386   ins_encode %{
11387     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11388                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11389                  _counters, _rtm_counters, _stack_rtm_counters,
11390                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11391                  true, ra_->C->profile_rtm());
11392   %}
11393   ins_pipe(pipe_slow);
11394 %}
11395 
11396 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11397   predicate(!Compile::current()->use_rtm());
11398   match(Set cr (FastLock object box));
11399   effect(TEMP tmp, TEMP scr, USE_KILL box);
11400   ins_cost(300);
11401   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11402   ins_encode %{
11403     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11404                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11405   %}
11406   ins_pipe(pipe_slow);
11407 %}
11408 
11409 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11410   match(Set cr (FastUnlock object box));
11411   effect(TEMP tmp, USE_KILL box);
11412   ins_cost(300);
11413   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11414   ins_encode %{
11415     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11416   %}
11417   ins_pipe(pipe_slow);
11418 %}
11419 
11420 
11421 // ============================================================================
11422 // Safepoint Instructions
11423 instruct safePoint_poll(rFlagsReg cr)
11424 %{
11425   predicate(!Assembler::is_polling_page_far());
11426   match(SafePoint);
11427   effect(KILL cr);
11428 
11429   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11430             "# Safepoint: poll for GC" %}
11431   ins_cost(125);
11432   ins_encode %{
11433     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11434     __ testl(rax, addr);
11435   %}
11436   ins_pipe(ialu_reg_mem);
11437 %}
11438 
11439 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11440 %{
11441   predicate(Assembler::is_polling_page_far());
11442   match(SafePoint poll);
11443   effect(KILL cr, USE poll);
11444 
11445   format %{ "testl  rax, [$poll]\t"
11446             "# Safepoint: poll for GC" %}
11447   ins_cost(125);
11448   ins_encode %{
11449     __ relocate(relocInfo::poll_type);
11450     __ testl(rax, Address($poll$$Register, 0));
11451   %}
11452   ins_pipe(ialu_reg_mem);
11453 %}
11454 
11455 // ============================================================================
11456 // Procedure Call/Return Instructions
11457 // Call Java Static Instruction
11458 // Note: If this code changes, the corresponding ret_addr_offset() and
11459 //       compute_padding() functions will have to be adjusted.
11460 instruct CallStaticJavaDirect(method meth) %{
11461   match(CallStaticJava);
11462   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
11463   effect(USE meth);
11464 
11465   ins_cost(300);
11466   format %{ "call,static " %}
11467   opcode(0xE8); /* E8 cd */
11468   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11469   ins_pipe(pipe_slow);
11470   ins_alignment(4);
11471 %}
11472 
11473 // Call Java Static Instruction (method handle version)
11474 // Note: If this code changes, the corresponding ret_addr_offset() and
11475 //       compute_padding() functions will have to be adjusted.
11476 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
11477   match(CallStaticJava);
11478   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
11479   effect(USE meth);
11480   // RBP is saved by all callees (for interpreter stack correction).
11481   // We use it here for a similar purpose, in {preserve,restore}_SP.
11482 
11483   ins_cost(300);
11484   format %{ "call,static/MethodHandle " %}
11485   opcode(0xE8); /* E8 cd */
11486   ins_encode(clear_avx, preserve_SP,
11487              Java_Static_Call(meth),
11488              restore_SP,
11489              call_epilog);
11490   ins_pipe(pipe_slow);
11491   ins_alignment(4);
11492 %}
11493 
11494 // Call Java Dynamic Instruction
11495 // Note: If this code changes, the corresponding ret_addr_offset() and
11496 //       compute_padding() functions will have to be adjusted.
11497 instruct CallDynamicJavaDirect(method meth)
11498 %{
11499   match(CallDynamicJava);
11500   effect(USE meth);
11501 
11502   ins_cost(300);
11503   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11504             "call,dynamic " %}
11505   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11506   ins_pipe(pipe_slow);
11507   ins_alignment(4);
11508 %}
11509 
11510 // Call Runtime Instruction
11511 instruct CallRuntimeDirect(method meth)
11512 %{
11513   match(CallRuntime);
11514   effect(USE meth);
11515 
11516   ins_cost(300);
11517   format %{ "call,runtime " %}
11518   ins_encode(clear_avx, Java_To_Runtime(meth));
11519   ins_pipe(pipe_slow);
11520 %}
11521 
11522 // Call runtime without safepoint
11523 instruct CallLeafDirect(method meth)
11524 %{
11525   match(CallLeaf);
11526   effect(USE meth);
11527 
11528   ins_cost(300);
11529   format %{ "call_leaf,runtime " %}
11530   ins_encode(clear_avx, Java_To_Runtime(meth));
11531   ins_pipe(pipe_slow);
11532 %}
11533 
11534 // Call runtime without safepoint
11535 instruct CallLeafNoFPDirect(method meth)
11536 %{
11537   match(CallLeafNoFP);
11538   effect(USE meth);
11539 
11540   ins_cost(300);
11541   format %{ "call_leaf_nofp,runtime " %}
11542   ins_encode(Java_To_Runtime(meth));
11543   ins_pipe(pipe_slow);
11544 %}
11545 
11546 // Return Instruction
11547 // Remove the return address & jump to it.
11548 // Notice: We always emit a nop after a ret to make sure there is room
11549 // for safepoint patching
11550 instruct Ret()
11551 %{
11552   match(Return);
11553 
11554   format %{ "ret" %}
11555   opcode(0xC3);
11556   ins_encode(OpcP);
11557   ins_pipe(pipe_jmp);
11558 %}
11559 
11560 // Tail Call; Jump from runtime stub to Java code.
11561 // Also known as an 'interprocedural jump'.
11562 // Target of jump will eventually return to caller.
11563 // TailJump below removes the return address.
11564 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11565 %{
11566   match(TailCall jump_target method_oop);
11567 
11568   ins_cost(300);
11569   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11570   opcode(0xFF, 0x4); /* Opcode FF /4 */
11571   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11572   ins_pipe(pipe_jmp);
11573 %}
11574 
11575 // Tail Jump; remove the return address; jump to target.
11576 // TailCall above leaves the return address around.
11577 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11578 %{
11579   match(TailJump jump_target ex_oop);
11580 
11581   ins_cost(300);
11582   format %{ "popq    rdx\t# pop return address\n\t"
11583             "jmp     $jump_target" %}
11584   opcode(0xFF, 0x4); /* Opcode FF /4 */
11585   ins_encode(Opcode(0x5a), // popq rdx
11586              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11587   ins_pipe(pipe_jmp);
11588 %}
11589 
11590 // Create exception oop: created by stack-crawling runtime code.
11591 // Created exception is now available to this handler, and is setup
11592 // just prior to jumping to this handler.  No code emitted.
11593 instruct CreateException(rax_RegP ex_oop)
11594 %{
11595   match(Set ex_oop (CreateEx));
11596 
11597   size(0);
11598   // use the following format syntax
11599   format %{ "# exception oop is in rax; no code emitted" %}
11600   ins_encode();
11601   ins_pipe(empty);
11602 %}
11603 
11604 // Rethrow exception:
11605 // The exception oop will come in the first argument position.
11606 // Then JUMP (not call) to the rethrow stub code.
11607 instruct RethrowException()
11608 %{
11609   match(Rethrow);
11610 
11611   // use the following format syntax
11612   format %{ "jmp     rethrow_stub" %}
11613   ins_encode(enc_rethrow);
11614   ins_pipe(pipe_jmp);
11615 %}
11616 
11617 
11618 // ============================================================================
11619 // This name is KNOWN by the ADLC and cannot be changed.
11620 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11621 // for this guy.
11622 instruct tlsLoadP(r15_RegP dst) %{
11623   match(Set dst (ThreadLocal));
11624   effect(DEF dst);
11625 
11626   size(0);
11627   format %{ "# TLS is in R15" %}
11628   ins_encode( /*empty encoding*/ );
11629   ins_pipe(ialu_reg_reg);
11630 %}
11631 
11632 
11633 //----------PEEPHOLE RULES-----------------------------------------------------
11634 // These must follow all instruction definitions as they use the names
11635 // defined in the instructions definitions.
11636 //
11637 // peepmatch ( root_instr_name [preceding_instruction]* );
11638 //
11639 // peepconstraint %{
11640 // (instruction_number.operand_name relational_op instruction_number.operand_name
11641 //  [, ...] );
11642 // // instruction numbers are zero-based using left to right order in peepmatch
11643 //
11644 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11645 // // provide an instruction_number.operand_name for each operand that appears
11646 // // in the replacement instruction's match rule
11647 //
11648 // ---------VM FLAGS---------------------------------------------------------
11649 //
11650 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11651 //
11652 // Each peephole rule is given an identifying number starting with zero and
11653 // increasing by one in the order seen by the parser.  An individual peephole
11654 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11655 // on the command-line.
11656 //
11657 // ---------CURRENT LIMITATIONS----------------------------------------------
11658 //
11659 // Only match adjacent instructions in same basic block
11660 // Only equality constraints
11661 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11662 // Only one replacement instruction
11663 //
11664 // ---------EXAMPLE----------------------------------------------------------
11665 //
11666 // // pertinent parts of existing instructions in architecture description
11667 // instruct movI(rRegI dst, rRegI src)
11668 // %{
11669 //   match(Set dst (CopyI src));
11670 // %}
11671 //
11672 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11673 // %{
11674 //   match(Set dst (AddI dst src));
11675 //   effect(KILL cr);
11676 // %}
11677 //
11678 // // Change (inc mov) to lea
11679 // peephole %{
11680 //   // increment preceeded by register-register move
11681 //   peepmatch ( incI_rReg movI );
11682 //   // require that the destination register of the increment
11683 //   // match the destination register of the move
11684 //   peepconstraint ( 0.dst == 1.dst );
11685 //   // construct a replacement instruction that sets
11686 //   // the destination to ( move's source register + one )
11687 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11688 // %}
11689 //
11690 
11691 // Implementation no longer uses movX instructions since
11692 // machine-independent system no longer uses CopyX nodes.
11693 //
11694 // peephole
11695 // %{
11696 //   peepmatch (incI_rReg movI);
11697 //   peepconstraint (0.dst == 1.dst);
11698 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11699 // %}
11700 
11701 // peephole
11702 // %{
11703 //   peepmatch (decI_rReg movI);
11704 //   peepconstraint (0.dst == 1.dst);
11705 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11706 // %}
11707 
11708 // peephole
11709 // %{
11710 //   peepmatch (addI_rReg_imm movI);
11711 //   peepconstraint (0.dst == 1.dst);
11712 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11713 // %}
11714 
11715 // peephole
11716 // %{
11717 //   peepmatch (incL_rReg movL);
11718 //   peepconstraint (0.dst == 1.dst);
11719 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11720 // %}
11721 
11722 // peephole
11723 // %{
11724 //   peepmatch (decL_rReg movL);
11725 //   peepconstraint (0.dst == 1.dst);
11726 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11727 // %}
11728 
11729 // peephole
11730 // %{
11731 //   peepmatch (addL_rReg_imm movL);
11732 //   peepconstraint (0.dst == 1.dst);
11733 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11734 // %}
11735 
11736 // peephole
11737 // %{
11738 //   peepmatch (addP_rReg_imm movP);
11739 //   peepconstraint (0.dst == 1.dst);
11740 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11741 // %}
11742 
11743 // // Change load of spilled value to only a spill
11744 // instruct storeI(memory mem, rRegI src)
11745 // %{
11746 //   match(Set mem (StoreI mem src));
11747 // %}
11748 //
11749 // instruct loadI(rRegI dst, memory mem)
11750 // %{
11751 //   match(Set dst (LoadI mem));
11752 // %}
11753 //
11754 
11755 peephole
11756 %{
11757   peepmatch (loadI storeI);
11758   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11759   peepreplace (storeI(1.mem 1.mem 1.src));
11760 %}
11761 
11762 peephole
11763 %{
11764   peepmatch (loadL storeL);
11765   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11766   peepreplace (storeL(1.mem 1.mem 1.src));
11767 %}
11768 
11769 //----------SMARTSPILL RULES---------------------------------------------------
11770 // These must follow all instruction definitions as they use the names
11771 // defined in the instructions definitions.
--- EOF ---