1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Class for all pointer registers (including RSP)
 170 reg_class any_reg(RAX, RAX_H,
 171                   RDX, RDX_H,
 172                   RBP, RBP_H,
 173                   RDI, RDI_H,
 174                   RSI, RSI_H,
 175                   RCX, RCX_H,
 176                   RBX, RBX_H,
 177                   RSP, RSP_H,
 178                   R8,  R8_H,
 179                   R9,  R9_H,
 180                   R10, R10_H,
 181                   R11, R11_H,
 182                   R12, R12_H,
 183                   R13, R13_H,
 184                   R14, R14_H,
 185                   R15, R15_H);
 186 
 187 // Class for all pointer registers except RSP
 188 reg_class ptr_reg(RAX, RAX_H,
 189                   RDX, RDX_H,
 190                   RBP, RBP_H,
 191                   RDI, RDI_H,
 192                   RSI, RSI_H,
 193                   RCX, RCX_H,
 194                   RBX, RBX_H,
 195                   R8,  R8_H,
 196                   R9,  R9_H,
 197                   R10, R10_H,
 198                   R11, R11_H,
 199                   R13, R13_H,
 200                   R14, R14_H);
 201 
 202 // Class for all pointer registers except RAX and RSP
 203 reg_class ptr_no_rax_reg(RDX, RDX_H,
 204                          RBP, RBP_H,
 205                          RDI, RDI_H,
 206                          RSI, RSI_H,
 207                          RCX, RCX_H,
 208                          RBX, RBX_H,
 209                          R8,  R8_H,
 210                          R9,  R9_H,
 211                          R10, R10_H,
 212                          R11, R11_H,
 213                          R13, R13_H,
 214                          R14, R14_H);
 215 
 216 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 217                          RAX, RAX_H,
 218                          RDI, RDI_H,
 219                          RSI, RSI_H,
 220                          RCX, RCX_H,
 221                          RBX, RBX_H,
 222                          R8,  R8_H,
 223                          R9,  R9_H,
 224                          R10, R10_H,
 225                          R11, R11_H,
 226                          R13, R13_H,
 227                          R14, R14_H);
 228 
 229 // Class for all pointer registers except RAX, RBX and RSP
 230 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 231                              RBP, RBP_H,
 232                              RDI, RDI_H,
 233                              RSI, RSI_H,
 234                              RCX, RCX_H,
 235                              R8,  R8_H,
 236                              R9,  R9_H,
 237                              R10, R10_H,
 238                              R11, R11_H,
 239                              R13, R13_H,
 240                              R14, R14_H);
 241 
 242 // Singleton class for RAX pointer register
 243 reg_class ptr_rax_reg(RAX, RAX_H);
 244 
 245 // Singleton class for RBX pointer register
 246 reg_class ptr_rbx_reg(RBX, RBX_H);
 247 
 248 // Singleton class for RSI pointer register
 249 reg_class ptr_rsi_reg(RSI, RSI_H);
 250 
 251 // Singleton class for RDI pointer register
 252 reg_class ptr_rdi_reg(RDI, RDI_H);
 253 
 254 // Singleton class for RBP pointer register
 255 reg_class ptr_rbp_reg(RBP, RBP_H);
 256 
 257 // Singleton class for stack pointer
 258 reg_class ptr_rsp_reg(RSP, RSP_H);
 259 
 260 // Singleton class for TLS pointer
 261 reg_class ptr_r15_reg(R15, R15_H);
 262 
 263 // Class for all long registers (except RSP)
 264 reg_class long_reg(RAX, RAX_H,
 265                    RDX, RDX_H,
 266                    RBP, RBP_H,
 267                    RDI, RDI_H,
 268                    RSI, RSI_H,
 269                    RCX, RCX_H,
 270                    RBX, RBX_H,
 271                    R8,  R8_H,
 272                    R9,  R9_H,
 273                    R10, R10_H,
 274                    R11, R11_H,
 275                    R13, R13_H,
 276                    R14, R14_H);
 277 
 278 // Class for all long registers except RAX, RDX (and RSP)
 279 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 280                               RDI, RDI_H,
 281                               RSI, RSI_H,
 282                               RCX, RCX_H,
 283                               RBX, RBX_H,
 284                               R8,  R8_H,
 285                               R9,  R9_H,
 286                               R10, R10_H,
 287                               R11, R11_H,
 288                               R13, R13_H,
 289                               R14, R14_H);
 290 
 291 // Class for all long registers except RCX (and RSP)
 292 reg_class long_no_rcx_reg(RBP, RBP_H,
 293                           RDI, RDI_H,
 294                           RSI, RSI_H,
 295                           RAX, RAX_H,
 296                           RDX, RDX_H,
 297                           RBX, RBX_H,
 298                           R8,  R8_H,
 299                           R9,  R9_H,
 300                           R10, R10_H,
 301                           R11, R11_H,
 302                           R13, R13_H,
 303                           R14, R14_H);
 304 
 305 // Class for all long registers except RAX (and RSP)
 306 reg_class long_no_rax_reg(RBP, RBP_H,
 307                           RDX, RDX_H,
 308                           RDI, RDI_H,
 309                           RSI, RSI_H,
 310                           RCX, RCX_H,
 311                           RBX, RBX_H,
 312                           R8,  R8_H,
 313                           R9,  R9_H,
 314                           R10, R10_H,
 315                           R11, R11_H,
 316                           R13, R13_H,
 317                           R14, R14_H);
 318 
 319 // Singleton class for RAX long register
 320 reg_class long_rax_reg(RAX, RAX_H);
 321 
 322 // Singleton class for RCX long register
 323 reg_class long_rcx_reg(RCX, RCX_H);
 324 
 325 // Singleton class for RDX long register
 326 reg_class long_rdx_reg(RDX, RDX_H);
 327 
 328 // Class for all int registers (except RSP)
 329 reg_class int_reg(RAX,
 330                   RDX,
 331                   RBP,
 332                   RDI,
 333                   RSI,
 334                   RCX,
 335                   RBX,
 336                   R8,
 337                   R9,
 338                   R10,
 339                   R11,
 340                   R13,
 341                   R14);
 342 
 343 // Class for all int registers except RCX (and RSP)
 344 reg_class int_no_rcx_reg(RAX,
 345                          RDX,
 346                          RBP,
 347                          RDI,
 348                          RSI,
 349                          RBX,
 350                          R8,
 351                          R9,
 352                          R10,
 353                          R11,
 354                          R13,
 355                          R14);
 356 
 357 // Class for all int registers except RAX, RDX (and RSP)
 358 reg_class int_no_rax_rdx_reg(RBP,
 359                              RDI,
 360                              RSI,
 361                              RCX,
 362                              RBX,
 363                              R8,
 364                              R9,
 365                              R10,
 366                              R11,
 367                              R13,
 368                              R14);
 369 
 370 // Singleton class for RAX int register
 371 reg_class int_rax_reg(RAX);
 372 
 373 // Singleton class for RBX int register
 374 reg_class int_rbx_reg(RBX);
 375 
 376 // Singleton class for RCX int register
 377 reg_class int_rcx_reg(RCX);
 378 
 379 // Singleton class for RCX int register
 380 reg_class int_rdx_reg(RDX);
 381 
 382 // Singleton class for RCX int register
 383 reg_class int_rdi_reg(RDI);
 384 
 385 // Singleton class for instruction pointer
 386 // reg_class ip_reg(RIP);
 387 
 388 %}
 389 
 390 //----------SOURCE BLOCK-------------------------------------------------------
 391 // This is a block of C++ code which provides values, functions, and
 392 // definitions necessary in the rest of the architecture description
 393 source %{
 394 #define   RELOC_IMM64    Assembler::imm_operand
 395 #define   RELOC_DISP32   Assembler::disp32_operand
 396 
 397 #define __ _masm.
 398 
 399 static int preserve_SP_size() {
 400   return 3;  // rex.w, op, rm(reg/reg)
 401 }
 402 static int clear_avx_size() {
 403   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 404 }
 405 
 406 // !!!!! Special hack to get all types of calls to specify the byte offset
 407 //       from the start of the call to the point where the return address
 408 //       will point.
 409 int MachCallStaticJavaNode::ret_addr_offset()
 410 {
 411   int offset = 5; // 5 bytes from start of call to where return address points
 412   offset += clear_avx_size();
 413   if (_method_handle_invoke)
 414     offset += preserve_SP_size();
 415   return offset;
 416 }
 417 
 418 int MachCallDynamicJavaNode::ret_addr_offset()
 419 {
 420   int offset = 15; // 15 bytes from start of call to where return address points
 421   offset += clear_avx_size();
 422   return offset;
 423 }
 424 
 425 int MachCallRuntimeNode::ret_addr_offset() {
 426   int offset = 13; // movq r10,#addr; callq (r10)
 427   offset += clear_avx_size();
 428   return offset;
 429 }
 430 
 431 // Indicate if the safepoint node needs the polling page as an input,
 432 // it does if the polling page is more than disp32 away.
 433 bool SafePointNode::needs_polling_address_input()
 434 {
 435   return Assembler::is_polling_page_far();
 436 }
 437 
 438 //
 439 // Compute padding required for nodes which need alignment
 440 //
 441 
 442 // The address of the call instruction needs to be 4-byte aligned to
 443 // ensure that it does not span a cache line so that it can be patched.
 444 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 445 {
 446   current_offset += clear_avx_size(); // skip vzeroupper
 447   current_offset += 1; // skip call opcode byte
 448   return round_to(current_offset, alignment_required()) - current_offset;
 449 }
 450 
 451 // The address of the call instruction needs to be 4-byte aligned to
 452 // ensure that it does not span a cache line so that it can be patched.
 453 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 454 {
 455   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 456   current_offset += clear_avx_size(); // skip vzeroupper
 457   current_offset += 1; // skip call opcode byte
 458   return round_to(current_offset, alignment_required()) - current_offset;
 459 }
 460 
 461 // The address of the call instruction needs to be 4-byte aligned to
 462 // ensure that it does not span a cache line so that it can be patched.
 463 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 464 {
 465   current_offset += clear_avx_size(); // skip vzeroupper
 466   current_offset += 11; // skip movq instruction + call opcode byte
 467   return round_to(current_offset, alignment_required()) - current_offset;
 468 }
 469 
 470 // EMIT_RM()
 471 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 472   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 473   cbuf.insts()->emit_int8(c);
 474 }
 475 
 476 // EMIT_CC()
 477 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 478   unsigned char c = (unsigned char) (f1 | f2);
 479   cbuf.insts()->emit_int8(c);
 480 }
 481 
 482 // EMIT_OPCODE()
 483 void emit_opcode(CodeBuffer &cbuf, int code) {
 484   cbuf.insts()->emit_int8((unsigned char) code);
 485 }
 486 
 487 // EMIT_OPCODE() w/ relocation information
 488 void emit_opcode(CodeBuffer &cbuf,
 489                  int code, relocInfo::relocType reloc, int offset, int format)
 490 {
 491   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 492   emit_opcode(cbuf, code);
 493 }
 494 
 495 // EMIT_D8()
 496 void emit_d8(CodeBuffer &cbuf, int d8) {
 497   cbuf.insts()->emit_int8((unsigned char) d8);
 498 }
 499 
 500 // EMIT_D16()
 501 void emit_d16(CodeBuffer &cbuf, int d16) {
 502   cbuf.insts()->emit_int16(d16);
 503 }
 504 
 505 // EMIT_D32()
 506 void emit_d32(CodeBuffer &cbuf, int d32) {
 507   cbuf.insts()->emit_int32(d32);
 508 }
 509 
 510 // EMIT_D64()
 511 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 512   cbuf.insts()->emit_int64(d64);
 513 }
 514 
 515 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 516 void emit_d32_reloc(CodeBuffer& cbuf,
 517                     int d32,
 518                     relocInfo::relocType reloc,
 519                     int format)
 520 {
 521   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 522   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 523   cbuf.insts()->emit_int32(d32);
 524 }
 525 
 526 // emit 32 bit value and construct relocation entry from RelocationHolder
 527 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 528 #ifdef ASSERT
 529   if (rspec.reloc()->type() == relocInfo::oop_type &&
 530       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 531     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 532     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 533   }
 534 #endif
 535   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 536   cbuf.insts()->emit_int32(d32);
 537 }
 538 
 539 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 540   address next_ip = cbuf.insts_end() + 4;
 541   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 542                  external_word_Relocation::spec(addr),
 543                  RELOC_DISP32);
 544 }
 545 
 546 
 547 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 548 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 549   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 550   cbuf.insts()->emit_int64(d64);
 551 }
 552 
 553 // emit 64 bit value and construct relocation entry from RelocationHolder
 554 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 555 #ifdef ASSERT
 556   if (rspec.reloc()->type() == relocInfo::oop_type &&
 557       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 558     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 559     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 560            "cannot embed scavengable oops in code");
 561   }
 562 #endif
 563   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 564   cbuf.insts()->emit_int64(d64);
 565 }
 566 
 567 // Access stack slot for load or store
 568 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 569 {
 570   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 571   if (-0x80 <= disp && disp < 0x80) {
 572     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 573     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 574     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 575   } else {
 576     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 577     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 578     emit_d32(cbuf, disp);     // Displacement // R/M byte
 579   }
 580 }
 581 
 582    // rRegI ereg, memory mem) %{    // emit_reg_mem
 583 void encode_RegMem(CodeBuffer &cbuf,
 584                    int reg,
 585                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 586 {
 587   assert(disp_reloc == relocInfo::none, "cannot have disp");
 588   int regenc = reg & 7;
 589   int baseenc = base & 7;
 590   int indexenc = index & 7;
 591 
 592   // There is no index & no scale, use form without SIB byte
 593   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 594     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 595     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 596       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 597     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 598       // If 8-bit displacement, mode 0x1
 599       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 600       emit_d8(cbuf, disp);
 601     } else {
 602       // If 32-bit displacement
 603       if (base == -1) { // Special flag for absolute address
 604         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 605         if (disp_reloc != relocInfo::none) {
 606           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 607         } else {
 608           emit_d32(cbuf, disp);
 609         }
 610       } else {
 611         // Normal base + offset
 612         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 613         if (disp_reloc != relocInfo::none) {
 614           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 615         } else {
 616           emit_d32(cbuf, disp);
 617         }
 618       }
 619     }
 620   } else {
 621     // Else, encode with the SIB byte
 622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 624       // If no displacement
 625       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 626       emit_rm(cbuf, scale, indexenc, baseenc);
 627     } else {
 628       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 629         // If 8-bit displacement, mode 0x1
 630         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 631         emit_rm(cbuf, scale, indexenc, baseenc);
 632         emit_d8(cbuf, disp);
 633       } else {
 634         // If 32-bit displacement
 635         if (base == 0x04 ) {
 636           emit_rm(cbuf, 0x2, regenc, 0x4);
 637           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 638         } else {
 639           emit_rm(cbuf, 0x2, regenc, 0x4);
 640           emit_rm(cbuf, scale, indexenc, baseenc); // *
 641         }
 642         if (disp_reloc != relocInfo::none) {
 643           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 644         } else {
 645           emit_d32(cbuf, disp);
 646         }
 647       }
 648     }
 649   }
 650 }
 651 
 652 // This could be in MacroAssembler but it's fairly C2 specific
 653 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 654   Label exit;
 655   __ jccb(Assembler::noParity, exit);
 656   __ pushf();
 657   //
 658   // comiss/ucomiss instructions set ZF,PF,CF flags and
 659   // zero OF,AF,SF for NaN values.
 660   // Fixup flags by zeroing ZF,PF so that compare of NaN
 661   // values returns 'less than' result (CF is set).
 662   // Leave the rest of flags unchanged.
 663   //
 664   //    7 6 5 4 3 2 1 0
 665   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 666   //    0 0 1 0 1 0 1 1   (0x2B)
 667   //
 668   __ andq(Address(rsp, 0), 0xffffff2b);
 669   __ popf();
 670   __ bind(exit);
 671 }
 672 
 673 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 674   Label done;
 675   __ movl(dst, -1);
 676   __ jcc(Assembler::parity, done);
 677   __ jcc(Assembler::below, done);
 678   __ setb(Assembler::notEqual, dst);
 679   __ movzbl(dst, dst);
 680   __ bind(done);
 681 }
 682 
 683 
 684 //=============================================================================
 685 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 686 
 687 int Compile::ConstantTable::calculate_table_base_offset() const {
 688   return 0;  // absolute addressing, no offset
 689 }
 690 
 691 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 692 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 693   ShouldNotReachHere();
 694 }
 695 
 696 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 697   // Empty encoding
 698 }
 699 
 700 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 701   return 0;
 702 }
 703 
 704 #ifndef PRODUCT
 705 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 706   st->print("# MachConstantBaseNode (empty encoding)");
 707 }
 708 #endif
 709 
 710 
 711 //=============================================================================
 712 #ifndef PRODUCT
 713 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 714   Compile* C = ra_->C;
 715 
 716   int framesize = C->frame_slots() << LogBytesPerInt;
 717   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 718   // Remove wordSize for return addr which is already pushed.
 719   framesize -= wordSize;
 720 
 721   if (C->need_stack_bang(framesize)) {
 722     framesize -= wordSize;
 723     st->print("# stack bang");
 724     st->print("\n\t");
 725     st->print("pushq   rbp\t# Save rbp");
 726     if (framesize) {
 727       st->print("\n\t");
 728       st->print("subq    rsp, #%d\t# Create frame",framesize);
 729     }
 730   } else {
 731     st->print("subq    rsp, #%d\t# Create frame",framesize);
 732     st->print("\n\t");
 733     framesize -= wordSize;
 734     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 735   }
 736 
 737   if (VerifyStackAtCalls) {
 738     st->print("\n\t");
 739     framesize -= wordSize;
 740     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 741 #ifdef ASSERT
 742     st->print("\n\t");
 743     st->print("# stack alignment check");
 744 #endif
 745   }
 746   st->cr();
 747 }
 748 #endif
 749 
 750 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 751   Compile* C = ra_->C;
 752   MacroAssembler _masm(&cbuf);
 753 
 754   int framesize = C->frame_slots() << LogBytesPerInt;
 755 
 756   __ verified_entry(framesize, C->need_stack_bang(framesize), false);
 757 
 758   C->set_frame_complete(cbuf.insts_size());
 759 
 760   if (C->has_mach_constant_base_node()) {
 761     // NOTE: We set the table base offset here because users might be
 762     // emitted before MachConstantBaseNode.
 763     Compile::ConstantTable& constant_table = C->constant_table();
 764     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 765   }
 766 }
 767 
 768 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 769 {
 770   return MachNode::size(ra_); // too many variables; just compute it
 771                               // the hard way
 772 }
 773 
 774 int MachPrologNode::reloc() const
 775 {
 776   return 0; // a large enough number
 777 }
 778 
 779 //=============================================================================
 780 #ifndef PRODUCT
 781 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 782 {
 783   Compile* C = ra_->C;
 784   if (C->max_vector_size() > 16) {
 785     st->print("vzeroupper");
 786     st->cr(); st->print("\t");
 787   }
 788 
 789   int framesize = C->frame_slots() << LogBytesPerInt;
 790   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 791   // Remove word for return adr already pushed
 792   // and RBP
 793   framesize -= 2*wordSize;
 794 
 795   if (framesize) {
 796     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 797     st->print("\t");
 798   }
 799 
 800   st->print_cr("popq   rbp");
 801   if (do_polling() && C->is_method_compilation()) {
 802     st->print("\t");
 803     if (Assembler::is_polling_page_far()) {
 804       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 805                    "testl  rax, [rscratch1]\t"
 806                    "# Safepoint: poll for GC");
 807     } else {
 808       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 809                    "# Safepoint: poll for GC");
 810     }
 811   }
 812 }
 813 #endif
 814 
 815 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 816 {
 817   Compile* C = ra_->C;
 818   if (C->max_vector_size() > 16) {
 819     // Clear upper bits of YMM registers when current compiled code uses
 820     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 821     MacroAssembler _masm(&cbuf);
 822     __ vzeroupper();
 823   }
 824 
 825   int framesize = C->frame_slots() << LogBytesPerInt;
 826   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 827   // Remove word for return adr already pushed
 828   // and RBP
 829   framesize -= 2*wordSize;
 830 
 831   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 832 
 833   if (framesize) {
 834     emit_opcode(cbuf, Assembler::REX_W);
 835     if (framesize < 0x80) {
 836       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 837       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 838       emit_d8(cbuf, framesize);
 839     } else {
 840       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 841       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 842       emit_d32(cbuf, framesize);
 843     }
 844   }
 845 
 846   // popq rbp
 847   emit_opcode(cbuf, 0x58 | RBP_enc);
 848 
 849   if (do_polling() && C->is_method_compilation()) {
 850     MacroAssembler _masm(&cbuf);
 851     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 852     if (Assembler::is_polling_page_far()) {
 853       __ lea(rscratch1, polling_page);
 854       __ relocate(relocInfo::poll_return_type);
 855       __ testl(rax, Address(rscratch1, 0));
 856     } else {
 857       __ testl(rax, polling_page);
 858     }
 859   }
 860 }
 861 
 862 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 863 {
 864   return MachNode::size(ra_); // too many variables; just compute it
 865                               // the hard way
 866 }
 867 
 868 int MachEpilogNode::reloc() const
 869 {
 870   return 2; // a large enough number
 871 }
 872 
 873 const Pipeline* MachEpilogNode::pipeline() const
 874 {
 875   return MachNode::pipeline_class();
 876 }
 877 
 878 int MachEpilogNode::safepoint_offset() const
 879 {
 880   return 0;
 881 }
 882 
 883 //=============================================================================
 884 
 885 enum RC {
 886   rc_bad,
 887   rc_int,
 888   rc_float,
 889   rc_stack
 890 };
 891 
 892 static enum RC rc_class(OptoReg::Name reg)
 893 {
 894   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 895 
 896   if (OptoReg::is_stack(reg)) return rc_stack;
 897 
 898   VMReg r = OptoReg::as_VMReg(reg);
 899 
 900   if (r->is_Register()) return rc_int;
 901 
 902   assert(r->is_XMMRegister(), "must be");
 903   return rc_float;
 904 }
 905 
 906 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 907 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 908                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 909 
 910 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 911                             int stack_offset, int reg, uint ireg, outputStream* st);
 912 
 913 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 914                                       int dst_offset, uint ireg, outputStream* st) {
 915   if (cbuf) {
 916     MacroAssembler _masm(cbuf);
 917     switch (ireg) {
 918     case Op_VecS:
 919       __ movq(Address(rsp, -8), rax);
 920       __ movl(rax, Address(rsp, src_offset));
 921       __ movl(Address(rsp, dst_offset), rax);
 922       __ movq(rax, Address(rsp, -8));
 923       break;
 924     case Op_VecD:
 925       __ pushq(Address(rsp, src_offset));
 926       __ popq (Address(rsp, dst_offset));
 927       break;
 928     case Op_VecX:
 929       __ pushq(Address(rsp, src_offset));
 930       __ popq (Address(rsp, dst_offset));
 931       __ pushq(Address(rsp, src_offset+8));
 932       __ popq (Address(rsp, dst_offset+8));
 933       break;
 934     case Op_VecY:
 935       __ vmovdqu(Address(rsp, -32), xmm0);
 936       __ vmovdqu(xmm0, Address(rsp, src_offset));
 937       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 938       __ vmovdqu(xmm0, Address(rsp, -32));
 939       break;
 940     default:
 941       ShouldNotReachHere();
 942     }
 943 #ifndef PRODUCT
 944   } else {
 945     switch (ireg) {
 946     case Op_VecS:
 947       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 948                 "movl    rax, [rsp + #%d]\n\t"
 949                 "movl    [rsp + #%d], rax\n\t"
 950                 "movq    rax, [rsp - #8]",
 951                 src_offset, dst_offset);
 952       break;
 953     case Op_VecD:
 954       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 955                 "popq    [rsp + #%d]",
 956                 src_offset, dst_offset);
 957       break;
 958      case Op_VecX:
 959       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 960                 "popq    [rsp + #%d]\n\t"
 961                 "pushq   [rsp + #%d]\n\t"
 962                 "popq    [rsp + #%d]",
 963                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 964       break;
 965     case Op_VecY:
 966       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 967                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 968                 "vmovdqu [rsp + #%d], xmm0\n\t"
 969                 "vmovdqu xmm0, [rsp - #32]",
 970                 src_offset, dst_offset);
 971       break;
 972     default:
 973       ShouldNotReachHere();
 974     }
 975 #endif
 976   }
 977 }
 978 
 979 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 980                                        PhaseRegAlloc* ra_,
 981                                        bool do_size,
 982                                        outputStream* st) const {
 983   assert(cbuf != NULL || st  != NULL, "sanity");
 984   // Get registers to move
 985   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 986   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 987   OptoReg::Name dst_second = ra_->get_reg_second(this);
 988   OptoReg::Name dst_first = ra_->get_reg_first(this);
 989 
 990   enum RC src_second_rc = rc_class(src_second);
 991   enum RC src_first_rc = rc_class(src_first);
 992   enum RC dst_second_rc = rc_class(dst_second);
 993   enum RC dst_first_rc = rc_class(dst_first);
 994 
 995   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 996          "must move at least 1 register" );
 997 
 998   if (src_first == dst_first && src_second == dst_second) {
 999     // Self copy, no move
1000     return 0;
1001   }
1002   if (bottom_type()->isa_vect() != NULL) {
1003     uint ireg = ideal_reg();
1004     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1005     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1006     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1007       // mem -> mem
1008       int src_offset = ra_->reg2offset(src_first);
1009       int dst_offset = ra_->reg2offset(dst_first);
1010       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1011     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1012       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1013     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1014       int stack_offset = ra_->reg2offset(dst_first);
1015       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1016     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1017       int stack_offset = ra_->reg2offset(src_first);
1018       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1019     } else {
1020       ShouldNotReachHere();
1021     }
1022     return 0;
1023   }
1024   if (src_first_rc == rc_stack) {
1025     // mem ->
1026     if (dst_first_rc == rc_stack) {
1027       // mem -> mem
1028       assert(src_second != dst_first, "overlap");
1029       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1030           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1031         // 64-bit
1032         int src_offset = ra_->reg2offset(src_first);
1033         int dst_offset = ra_->reg2offset(dst_first);
1034         if (cbuf) {
1035           MacroAssembler _masm(cbuf);
1036           __ pushq(Address(rsp, src_offset));
1037           __ popq (Address(rsp, dst_offset));
1038 #ifndef PRODUCT
1039         } else {
1040           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1041                     "popq    [rsp + #%d]",
1042                      src_offset, dst_offset);
1043 #endif
1044         }
1045       } else {
1046         // 32-bit
1047         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1048         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1049         // No pushl/popl, so:
1050         int src_offset = ra_->reg2offset(src_first);
1051         int dst_offset = ra_->reg2offset(dst_first);
1052         if (cbuf) {
1053           MacroAssembler _masm(cbuf);
1054           __ movq(Address(rsp, -8), rax);
1055           __ movl(rax, Address(rsp, src_offset));
1056           __ movl(Address(rsp, dst_offset), rax);
1057           __ movq(rax, Address(rsp, -8));
1058 #ifndef PRODUCT
1059         } else {
1060           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1061                     "movl    rax, [rsp + #%d]\n\t"
1062                     "movl    [rsp + #%d], rax\n\t"
1063                     "movq    rax, [rsp - #8]",
1064                      src_offset, dst_offset);
1065 #endif
1066         }
1067       }
1068       return 0;
1069     } else if (dst_first_rc == rc_int) {
1070       // mem -> gpr
1071       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1072           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1073         // 64-bit
1074         int offset = ra_->reg2offset(src_first);
1075         if (cbuf) {
1076           MacroAssembler _masm(cbuf);
1077           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1078 #ifndef PRODUCT
1079         } else {
1080           st->print("movq    %s, [rsp + #%d]\t# spill",
1081                      Matcher::regName[dst_first],
1082                      offset);
1083 #endif
1084         }
1085       } else {
1086         // 32-bit
1087         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1088         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1089         int offset = ra_->reg2offset(src_first);
1090         if (cbuf) {
1091           MacroAssembler _masm(cbuf);
1092           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1093 #ifndef PRODUCT
1094         } else {
1095           st->print("movl    %s, [rsp + #%d]\t# spill",
1096                      Matcher::regName[dst_first],
1097                      offset);
1098 #endif
1099         }
1100       }
1101       return 0;
1102     } else if (dst_first_rc == rc_float) {
1103       // mem-> xmm
1104       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1105           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1106         // 64-bit
1107         int offset = ra_->reg2offset(src_first);
1108         if (cbuf) {
1109           MacroAssembler _masm(cbuf);
1110           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1111 #ifndef PRODUCT
1112         } else {
1113           st->print("%s  %s, [rsp + #%d]\t# spill",
1114                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1115                      Matcher::regName[dst_first],
1116                      offset);
1117 #endif
1118         }
1119       } else {
1120         // 32-bit
1121         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1122         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1123         int offset = ra_->reg2offset(src_first);
1124         if (cbuf) {
1125           MacroAssembler _masm(cbuf);
1126           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1127 #ifndef PRODUCT
1128         } else {
1129           st->print("movss   %s, [rsp + #%d]\t# spill",
1130                      Matcher::regName[dst_first],
1131                      offset);
1132 #endif
1133         }
1134       }
1135       return 0;
1136     }
1137   } else if (src_first_rc == rc_int) {
1138     // gpr ->
1139     if (dst_first_rc == rc_stack) {
1140       // gpr -> mem
1141       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1142           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1143         // 64-bit
1144         int offset = ra_->reg2offset(dst_first);
1145         if (cbuf) {
1146           MacroAssembler _masm(cbuf);
1147           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1148 #ifndef PRODUCT
1149         } else {
1150           st->print("movq    [rsp + #%d], %s\t# spill",
1151                      offset,
1152                      Matcher::regName[src_first]);
1153 #endif
1154         }
1155       } else {
1156         // 32-bit
1157         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1158         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1159         int offset = ra_->reg2offset(dst_first);
1160         if (cbuf) {
1161           MacroAssembler _masm(cbuf);
1162           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1163 #ifndef PRODUCT
1164         } else {
1165           st->print("movl    [rsp + #%d], %s\t# spill",
1166                      offset,
1167                      Matcher::regName[src_first]);
1168 #endif
1169         }
1170       }
1171       return 0;
1172     } else if (dst_first_rc == rc_int) {
1173       // gpr -> gpr
1174       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1175           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1176         // 64-bit
1177         if (cbuf) {
1178           MacroAssembler _masm(cbuf);
1179           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1180                   as_Register(Matcher::_regEncode[src_first]));
1181 #ifndef PRODUCT
1182         } else {
1183           st->print("movq    %s, %s\t# spill",
1184                      Matcher::regName[dst_first],
1185                      Matcher::regName[src_first]);
1186 #endif
1187         }
1188         return 0;
1189       } else {
1190         // 32-bit
1191         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1192         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1193         if (cbuf) {
1194           MacroAssembler _masm(cbuf);
1195           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1196                   as_Register(Matcher::_regEncode[src_first]));
1197 #ifndef PRODUCT
1198         } else {
1199           st->print("movl    %s, %s\t# spill",
1200                      Matcher::regName[dst_first],
1201                      Matcher::regName[src_first]);
1202 #endif
1203         }
1204         return 0;
1205       }
1206     } else if (dst_first_rc == rc_float) {
1207       // gpr -> xmm
1208       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1209           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1210         // 64-bit
1211         if (cbuf) {
1212           MacroAssembler _masm(cbuf);
1213           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1214 #ifndef PRODUCT
1215         } else {
1216           st->print("movdq   %s, %s\t# spill",
1217                      Matcher::regName[dst_first],
1218                      Matcher::regName[src_first]);
1219 #endif
1220         }
1221       } else {
1222         // 32-bit
1223         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1224         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1228 #ifndef PRODUCT
1229         } else {
1230           st->print("movdl   %s, %s\t# spill",
1231                      Matcher::regName[dst_first],
1232                      Matcher::regName[src_first]);
1233 #endif
1234         }
1235       }
1236       return 0;
1237     }
1238   } else if (src_first_rc == rc_float) {
1239     // xmm ->
1240     if (dst_first_rc == rc_stack) {
1241       // xmm -> mem
1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1244         // 64-bit
1245         int offset = ra_->reg2offset(dst_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("movsd   [rsp + #%d], %s\t# spill",
1252                      offset,
1253                      Matcher::regName[src_first]);
1254 #endif
1255         }
1256       } else {
1257         // 32-bit
1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1260         int offset = ra_->reg2offset(dst_first);
1261         if (cbuf) {
1262           MacroAssembler _masm(cbuf);
1263           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1264 #ifndef PRODUCT
1265         } else {
1266           st->print("movss   [rsp + #%d], %s\t# spill",
1267                      offset,
1268                      Matcher::regName[src_first]);
1269 #endif
1270         }
1271       }
1272       return 0;
1273     } else if (dst_first_rc == rc_int) {
1274       // xmm -> gpr
1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1277         // 64-bit
1278         if (cbuf) {
1279           MacroAssembler _masm(cbuf);
1280           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1281 #ifndef PRODUCT
1282         } else {
1283           st->print("movdq   %s, %s\t# spill",
1284                      Matcher::regName[dst_first],
1285                      Matcher::regName[src_first]);
1286 #endif
1287         }
1288       } else {
1289         // 32-bit
1290         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1291         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1292         if (cbuf) {
1293           MacroAssembler _masm(cbuf);
1294           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1295 #ifndef PRODUCT
1296         } else {
1297           st->print("movdl   %s, %s\t# spill",
1298                      Matcher::regName[dst_first],
1299                      Matcher::regName[src_first]);
1300 #endif
1301         }
1302       }
1303       return 0;
1304     } else if (dst_first_rc == rc_float) {
1305       // xmm -> xmm
1306       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1307           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1308         // 64-bit
1309         if (cbuf) {
1310           MacroAssembler _masm(cbuf);
1311           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1312 #ifndef PRODUCT
1313         } else {
1314           st->print("%s  %s, %s\t# spill",
1315                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1316                      Matcher::regName[dst_first],
1317                      Matcher::regName[src_first]);
1318 #endif
1319         }
1320       } else {
1321         // 32-bit
1322         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1323         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1327 #ifndef PRODUCT
1328         } else {
1329           st->print("%s  %s, %s\t# spill",
1330                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1331                      Matcher::regName[dst_first],
1332                      Matcher::regName[src_first]);
1333 #endif
1334         }
1335       }
1336       return 0;
1337     }
1338   }
1339 
1340   assert(0," foo ");
1341   Unimplemented();
1342   return 0;
1343 }
1344 
1345 #ifndef PRODUCT
1346 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1347   implementation(NULL, ra_, false, st);
1348 }
1349 #endif
1350 
1351 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1352   implementation(&cbuf, ra_, false, NULL);
1353 }
1354 
1355 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1356   return MachNode::size(ra_);
1357 }
1358 
1359 //=============================================================================
1360 #ifndef PRODUCT
1361 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1362 {
1363   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1364   int reg = ra_->get_reg_first(this);
1365   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1366             Matcher::regName[reg], offset);
1367 }
1368 #endif
1369 
1370 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1371 {
1372   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1373   int reg = ra_->get_encode(this);
1374   if (offset >= 0x80) {
1375     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1376     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1377     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1378     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1379     emit_d32(cbuf, offset);
1380   } else {
1381     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1382     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1383     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1384     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1385     emit_d8(cbuf, offset);
1386   }
1387 }
1388 
1389 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1390 {
1391   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1392   return (offset < 0x80) ? 5 : 8; // REX
1393 }
1394 
1395 //=============================================================================
1396 #ifndef PRODUCT
1397 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1398 {
1399   if (UseCompressedClassPointers) {
1400     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1401     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1402     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1403   } else {
1404     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1405                  "# Inline cache check");
1406   }
1407   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1408   st->print_cr("\tnop\t# nops to align entry point");
1409 }
1410 #endif
1411 
1412 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1413 {
1414   MacroAssembler masm(&cbuf);
1415   uint insts_size = cbuf.insts_size();
1416   if (UseCompressedClassPointers) {
1417     masm.load_klass(rscratch1, j_rarg0);
1418     masm.cmpptr(rax, rscratch1);
1419   } else {
1420     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1421   }
1422 
1423   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1424 
1425   /* WARNING these NOPs are critical so that verified entry point is properly
1426      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1427   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1428   if (OptoBreakpoint) {
1429     // Leave space for int3
1430     nops_cnt -= 1;
1431   }
1432   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1433   if (nops_cnt > 0)
1434     masm.nop(nops_cnt);
1435 }
1436 
1437 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1438 {
1439   return MachNode::size(ra_); // too many variables; just compute it
1440                               // the hard way
1441 }
1442 
1443 
1444 //=============================================================================
1445 uint size_exception_handler()
1446 {
1447   // NativeCall instruction size is the same as NativeJump.
1448   // Note that this value is also credited (in output.cpp) to
1449   // the size of the code section.
1450   return NativeJump::instruction_size;
1451 }
1452 
1453 // Emit exception handler code.
1454 int emit_exception_handler(CodeBuffer& cbuf)
1455 {
1456 
1457   // Note that the code buffer's insts_mark is always relative to insts.
1458   // That's why we must use the macroassembler to generate a handler.
1459   MacroAssembler _masm(&cbuf);
1460   address base =
1461   __ start_a_stub(size_exception_handler());
1462   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1463   int offset = __ offset();
1464   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1465   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1466   __ end_a_stub();
1467   return offset;
1468 }
1469 
1470 uint size_deopt_handler()
1471 {
1472   // three 5 byte instructions
1473   return 15;
1474 }
1475 
1476 // Emit deopt handler code.
1477 int emit_deopt_handler(CodeBuffer& cbuf)
1478 {
1479 
1480   // Note that the code buffer's insts_mark is always relative to insts.
1481   // That's why we must use the macroassembler to generate a handler.
1482   MacroAssembler _masm(&cbuf);
1483   address base =
1484   __ start_a_stub(size_deopt_handler());
1485   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1486   int offset = __ offset();
1487   address the_pc = (address) __ pc();
1488   Label next;
1489   // push a "the_pc" on the stack without destroying any registers
1490   // as they all may be live.
1491 
1492   // push address of "next"
1493   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1494   __ bind(next);
1495   // adjust it so it matches "the_pc"
1496   __ subptr(Address(rsp, 0), __ offset() - offset);
1497   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1498   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1499   __ end_a_stub();
1500   return offset;
1501 }
1502 
1503 int Matcher::regnum_to_fpu_offset(int regnum)
1504 {
1505   return regnum - 32; // The FP registers are in the second chunk
1506 }
1507 
1508 // This is UltraSparc specific, true just means we have fast l2f conversion
1509 const bool Matcher::convL2FSupported(void) {
1510   return true;
1511 }
1512 
1513 // Is this branch offset short enough that a short branch can be used?
1514 //
1515 // NOTE: If the platform does not provide any short branch variants, then
1516 //       this method should return false for offset 0.
1517 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1518   // The passed offset is relative to address of the branch.
1519   // On 86 a branch displacement is calculated relative to address
1520   // of a next instruction.
1521   offset -= br_size;
1522 
1523   // the short version of jmpConUCF2 contains multiple branches,
1524   // making the reach slightly less
1525   if (rule == jmpConUCF2_rule)
1526     return (-126 <= offset && offset <= 125);
1527   return (-128 <= offset && offset <= 127);
1528 }
1529 
1530 const bool Matcher::isSimpleConstant64(jlong value) {
1531   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1532   //return value == (int) value;  // Cf. storeImmL and immL32.
1533 
1534   // Probably always true, even if a temp register is required.
1535   return true;
1536 }
1537 
1538 // The ecx parameter to rep stosq for the ClearArray node is in words.
1539 const bool Matcher::init_array_count_is_in_bytes = false;
1540 
1541 // Threshold size for cleararray.
1542 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1543 
1544 // No additional cost for CMOVL.
1545 const int Matcher::long_cmove_cost() { return 0; }
1546 
1547 // No CMOVF/CMOVD with SSE2
1548 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1549 
1550 // Does the CPU require late expand (see block.cpp for description of late expand)?
1551 const bool Matcher::require_postalloc_expand = false;
1552 
1553 // Should the Matcher clone shifts on addressing modes, expecting them
1554 // to be subsumed into complex addressing expressions or compute them
1555 // into registers?  True for Intel but false for most RISCs
1556 const bool Matcher::clone_shift_expressions = true;
1557 
1558 // Do we need to mask the count passed to shift instructions or does
1559 // the cpu only look at the lower 5/6 bits anyway?
1560 const bool Matcher::need_masked_shift_count = false;
1561 
1562 bool Matcher::narrow_oop_use_complex_address() {
1563   assert(UseCompressedOops, "only for compressed oops code");
1564   return (LogMinObjAlignmentInBytes <= 3);
1565 }
1566 
1567 bool Matcher::narrow_klass_use_complex_address() {
1568   assert(UseCompressedClassPointers, "only for compressed klass code");
1569   return (LogKlassAlignmentInBytes <= 3);
1570 }
1571 
1572 // Is it better to copy float constants, or load them directly from
1573 // memory?  Intel can load a float constant from a direct address,
1574 // requiring no extra registers.  Most RISCs will have to materialize
1575 // an address into a register first, so they would do better to copy
1576 // the constant from stack.
1577 const bool Matcher::rematerialize_float_constants = true; // XXX
1578 
1579 // If CPU can load and store mis-aligned doubles directly then no
1580 // fixup is needed.  Else we split the double into 2 integer pieces
1581 // and move it piece-by-piece.  Only happens when passing doubles into
1582 // C code as the Java calling convention forces doubles to be aligned.
1583 const bool Matcher::misaligned_doubles_ok = true;
1584 
1585 // No-op on amd64
1586 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1587 
1588 // Advertise here if the CPU requires explicit rounding operations to
1589 // implement the UseStrictFP mode.
1590 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1591 
1592 // Are floats conerted to double when stored to stack during deoptimization?
1593 // On x64 it is stored without convertion so we can use normal access.
1594 bool Matcher::float_in_double() { return false; }
1595 
1596 // Do ints take an entire long register or just half?
1597 const bool Matcher::int_in_long = true;
1598 
1599 // Return whether or not this register is ever used as an argument.
1600 // This function is used on startup to build the trampoline stubs in
1601 // generateOptoStub.  Registers not mentioned will be killed by the VM
1602 // call in the trampoline, and arguments in those registers not be
1603 // available to the callee.
1604 bool Matcher::can_be_java_arg(int reg)
1605 {
1606   return
1607     reg ==  RDI_num || reg == RDI_H_num ||
1608     reg ==  RSI_num || reg == RSI_H_num ||
1609     reg ==  RDX_num || reg == RDX_H_num ||
1610     reg ==  RCX_num || reg == RCX_H_num ||
1611     reg ==   R8_num || reg ==  R8_H_num ||
1612     reg ==   R9_num || reg ==  R9_H_num ||
1613     reg ==  R12_num || reg == R12_H_num ||
1614     reg == XMM0_num || reg == XMM0b_num ||
1615     reg == XMM1_num || reg == XMM1b_num ||
1616     reg == XMM2_num || reg == XMM2b_num ||
1617     reg == XMM3_num || reg == XMM3b_num ||
1618     reg == XMM4_num || reg == XMM4b_num ||
1619     reg == XMM5_num || reg == XMM5b_num ||
1620     reg == XMM6_num || reg == XMM6b_num ||
1621     reg == XMM7_num || reg == XMM7b_num;
1622 }
1623 
1624 bool Matcher::is_spillable_arg(int reg)
1625 {
1626   return can_be_java_arg(reg);
1627 }
1628 
1629 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1630   // In 64 bit mode a code which use multiply when
1631   // devisor is constant is faster than hardware
1632   // DIV instruction (it uses MulHiL).
1633   return false;
1634 }
1635 
1636 // Register for DIVI projection of divmodI
1637 RegMask Matcher::divI_proj_mask() {
1638   return INT_RAX_REG_mask();
1639 }
1640 
1641 // Register for MODI projection of divmodI
1642 RegMask Matcher::modI_proj_mask() {
1643   return INT_RDX_REG_mask();
1644 }
1645 
1646 // Register for DIVL projection of divmodL
1647 RegMask Matcher::divL_proj_mask() {
1648   return LONG_RAX_REG_mask();
1649 }
1650 
1651 // Register for MODL projection of divmodL
1652 RegMask Matcher::modL_proj_mask() {
1653   return LONG_RDX_REG_mask();
1654 }
1655 
1656 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1657   return PTR_RBP_REG_mask();
1658 }
1659 
1660 const RegMask Matcher::mathExactI_result_proj_mask() {
1661   return INT_RAX_REG_mask();
1662 }
1663 
1664 const RegMask Matcher::mathExactL_result_proj_mask() {
1665   return LONG_RAX_REG_mask();
1666 }
1667 
1668 const RegMask Matcher::mathExactI_flags_proj_mask() {
1669   return INT_FLAGS_mask();
1670 }
1671 
1672 %}
1673 
1674 //----------ENCODING BLOCK-----------------------------------------------------
1675 // This block specifies the encoding classes used by the compiler to
1676 // output byte streams.  Encoding classes are parameterized macros
1677 // used by Machine Instruction Nodes in order to generate the bit
1678 // encoding of the instruction.  Operands specify their base encoding
1679 // interface with the interface keyword.  There are currently
1680 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1681 // COND_INTER.  REG_INTER causes an operand to generate a function
1682 // which returns its register number when queried.  CONST_INTER causes
1683 // an operand to generate a function which returns the value of the
1684 // constant when queried.  MEMORY_INTER causes an operand to generate
1685 // four functions which return the Base Register, the Index Register,
1686 // the Scale Value, and the Offset Value of the operand when queried.
1687 // COND_INTER causes an operand to generate six functions which return
1688 // the encoding code (ie - encoding bits for the instruction)
1689 // associated with each basic boolean condition for a conditional
1690 // instruction.
1691 //
1692 // Instructions specify two basic values for encoding.  Again, a
1693 // function is available to check if the constant displacement is an
1694 // oop. They use the ins_encode keyword to specify their encoding
1695 // classes (which must be a sequence of enc_class names, and their
1696 // parameters, specified in the encoding block), and they use the
1697 // opcode keyword to specify, in order, their primary, secondary, and
1698 // tertiary opcode.  Only the opcode sections which a particular
1699 // instruction needs for encoding need to be specified.
1700 encode %{
1701   // Build emit functions for each basic byte or larger field in the
1702   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1703   // from C++ code in the enc_class source block.  Emit functions will
1704   // live in the main source block for now.  In future, we can
1705   // generalize this by adding a syntax that specifies the sizes of
1706   // fields in an order, so that the adlc can build the emit functions
1707   // automagically
1708 
1709   // Emit primary opcode
1710   enc_class OpcP
1711   %{
1712     emit_opcode(cbuf, $primary);
1713   %}
1714 
1715   // Emit secondary opcode
1716   enc_class OpcS
1717   %{
1718     emit_opcode(cbuf, $secondary);
1719   %}
1720 
1721   // Emit tertiary opcode
1722   enc_class OpcT
1723   %{
1724     emit_opcode(cbuf, $tertiary);
1725   %}
1726 
1727   // Emit opcode directly
1728   enc_class Opcode(immI d8)
1729   %{
1730     emit_opcode(cbuf, $d8$$constant);
1731   %}
1732 
1733   // Emit size prefix
1734   enc_class SizePrefix
1735   %{
1736     emit_opcode(cbuf, 0x66);
1737   %}
1738 
1739   enc_class reg(rRegI reg)
1740   %{
1741     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1742   %}
1743 
1744   enc_class reg_reg(rRegI dst, rRegI src)
1745   %{
1746     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1747   %}
1748 
1749   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1750   %{
1751     emit_opcode(cbuf, $opcode$$constant);
1752     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1753   %}
1754 
1755   enc_class cdql_enc(no_rax_rdx_RegI div)
1756   %{
1757     // Full implementation of Java idiv and irem; checks for
1758     // special case as described in JVM spec., p.243 & p.271.
1759     //
1760     //         normal case                           special case
1761     //
1762     // input : rax: dividend                         min_int
1763     //         reg: divisor                          -1
1764     //
1765     // output: rax: quotient  (= rax idiv reg)       min_int
1766     //         rdx: remainder (= rax irem reg)       0
1767     //
1768     //  Code sequnce:
1769     //
1770     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1771     //    5:   75 07/08                jne    e <normal>
1772     //    7:   33 d2                   xor    %edx,%edx
1773     //  [div >= 8 -> offset + 1]
1774     //  [REX_B]
1775     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1776     //    c:   74 03/04                je     11 <done>
1777     // 000000000000000e <normal>:
1778     //    e:   99                      cltd
1779     //  [div >= 8 -> offset + 1]
1780     //  [REX_B]
1781     //    f:   f7 f9                   idiv   $div
1782     // 0000000000000011 <done>:
1783 
1784     // cmp    $0x80000000,%eax
1785     emit_opcode(cbuf, 0x3d);
1786     emit_d8(cbuf, 0x00);
1787     emit_d8(cbuf, 0x00);
1788     emit_d8(cbuf, 0x00);
1789     emit_d8(cbuf, 0x80);
1790 
1791     // jne    e <normal>
1792     emit_opcode(cbuf, 0x75);
1793     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1794 
1795     // xor    %edx,%edx
1796     emit_opcode(cbuf, 0x33);
1797     emit_d8(cbuf, 0xD2);
1798 
1799     // cmp    $0xffffffffffffffff,%ecx
1800     if ($div$$reg >= 8) {
1801       emit_opcode(cbuf, Assembler::REX_B);
1802     }
1803     emit_opcode(cbuf, 0x83);
1804     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1805     emit_d8(cbuf, 0xFF);
1806 
1807     // je     11 <done>
1808     emit_opcode(cbuf, 0x74);
1809     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1810 
1811     // <normal>
1812     // cltd
1813     emit_opcode(cbuf, 0x99);
1814 
1815     // idivl (note: must be emitted by the user of this rule)
1816     // <done>
1817   %}
1818 
1819   enc_class cdqq_enc(no_rax_rdx_RegL div)
1820   %{
1821     // Full implementation of Java ldiv and lrem; checks for
1822     // special case as described in JVM spec., p.243 & p.271.
1823     //
1824     //         normal case                           special case
1825     //
1826     // input : rax: dividend                         min_long
1827     //         reg: divisor                          -1
1828     //
1829     // output: rax: quotient  (= rax idiv reg)       min_long
1830     //         rdx: remainder (= rax irem reg)       0
1831     //
1832     //  Code sequnce:
1833     //
1834     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1835     //    7:   00 00 80
1836     //    a:   48 39 d0                cmp    %rdx,%rax
1837     //    d:   75 08                   jne    17 <normal>
1838     //    f:   33 d2                   xor    %edx,%edx
1839     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1840     //   15:   74 05                   je     1c <done>
1841     // 0000000000000017 <normal>:
1842     //   17:   48 99                   cqto
1843     //   19:   48 f7 f9                idiv   $div
1844     // 000000000000001c <done>:
1845 
1846     // mov    $0x8000000000000000,%rdx
1847     emit_opcode(cbuf, Assembler::REX_W);
1848     emit_opcode(cbuf, 0xBA);
1849     emit_d8(cbuf, 0x00);
1850     emit_d8(cbuf, 0x00);
1851     emit_d8(cbuf, 0x00);
1852     emit_d8(cbuf, 0x00);
1853     emit_d8(cbuf, 0x00);
1854     emit_d8(cbuf, 0x00);
1855     emit_d8(cbuf, 0x00);
1856     emit_d8(cbuf, 0x80);
1857 
1858     // cmp    %rdx,%rax
1859     emit_opcode(cbuf, Assembler::REX_W);
1860     emit_opcode(cbuf, 0x39);
1861     emit_d8(cbuf, 0xD0);
1862 
1863     // jne    17 <normal>
1864     emit_opcode(cbuf, 0x75);
1865     emit_d8(cbuf, 0x08);
1866 
1867     // xor    %edx,%edx
1868     emit_opcode(cbuf, 0x33);
1869     emit_d8(cbuf, 0xD2);
1870 
1871     // cmp    $0xffffffffffffffff,$div
1872     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1873     emit_opcode(cbuf, 0x83);
1874     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1875     emit_d8(cbuf, 0xFF);
1876 
1877     // je     1e <done>
1878     emit_opcode(cbuf, 0x74);
1879     emit_d8(cbuf, 0x05);
1880 
1881     // <normal>
1882     // cqto
1883     emit_opcode(cbuf, Assembler::REX_W);
1884     emit_opcode(cbuf, 0x99);
1885 
1886     // idivq (note: must be emitted by the user of this rule)
1887     // <done>
1888   %}
1889 
1890   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1891   enc_class OpcSE(immI imm)
1892   %{
1893     // Emit primary opcode and set sign-extend bit
1894     // Check for 8-bit immediate, and set sign extend bit in opcode
1895     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1896       emit_opcode(cbuf, $primary | 0x02);
1897     } else {
1898       // 32-bit immediate
1899       emit_opcode(cbuf, $primary);
1900     }
1901   %}
1902 
1903   enc_class OpcSErm(rRegI dst, immI imm)
1904   %{
1905     // OpcSEr/m
1906     int dstenc = $dst$$reg;
1907     if (dstenc >= 8) {
1908       emit_opcode(cbuf, Assembler::REX_B);
1909       dstenc -= 8;
1910     }
1911     // Emit primary opcode and set sign-extend bit
1912     // Check for 8-bit immediate, and set sign extend bit in opcode
1913     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1914       emit_opcode(cbuf, $primary | 0x02);
1915     } else {
1916       // 32-bit immediate
1917       emit_opcode(cbuf, $primary);
1918     }
1919     // Emit r/m byte with secondary opcode, after primary opcode.
1920     emit_rm(cbuf, 0x3, $secondary, dstenc);
1921   %}
1922 
1923   enc_class OpcSErm_wide(rRegL dst, immI imm)
1924   %{
1925     // OpcSEr/m
1926     int dstenc = $dst$$reg;
1927     if (dstenc < 8) {
1928       emit_opcode(cbuf, Assembler::REX_W);
1929     } else {
1930       emit_opcode(cbuf, Assembler::REX_WB);
1931       dstenc -= 8;
1932     }
1933     // Emit primary opcode and set sign-extend bit
1934     // Check for 8-bit immediate, and set sign extend bit in opcode
1935     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1936       emit_opcode(cbuf, $primary | 0x02);
1937     } else {
1938       // 32-bit immediate
1939       emit_opcode(cbuf, $primary);
1940     }
1941     // Emit r/m byte with secondary opcode, after primary opcode.
1942     emit_rm(cbuf, 0x3, $secondary, dstenc);
1943   %}
1944 
1945   enc_class Con8or32(immI imm)
1946   %{
1947     // Check for 8-bit immediate, and set sign extend bit in opcode
1948     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1949       $$$emit8$imm$$constant;
1950     } else {
1951       // 32-bit immediate
1952       $$$emit32$imm$$constant;
1953     }
1954   %}
1955 
1956   enc_class opc2_reg(rRegI dst)
1957   %{
1958     // BSWAP
1959     emit_cc(cbuf, $secondary, $dst$$reg);
1960   %}
1961 
1962   enc_class opc3_reg(rRegI dst)
1963   %{
1964     // BSWAP
1965     emit_cc(cbuf, $tertiary, $dst$$reg);
1966   %}
1967 
1968   enc_class reg_opc(rRegI div)
1969   %{
1970     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1971     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1972   %}
1973 
1974   enc_class enc_cmov(cmpOp cop)
1975   %{
1976     // CMOV
1977     $$$emit8$primary;
1978     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1979   %}
1980 
1981   enc_class enc_PartialSubtypeCheck()
1982   %{
1983     Register Rrdi = as_Register(RDI_enc); // result register
1984     Register Rrax = as_Register(RAX_enc); // super class
1985     Register Rrcx = as_Register(RCX_enc); // killed
1986     Register Rrsi = as_Register(RSI_enc); // sub class
1987     Label miss;
1988     const bool set_cond_codes = true;
1989 
1990     MacroAssembler _masm(&cbuf);
1991     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1992                                      NULL, &miss,
1993                                      /*set_cond_codes:*/ true);
1994     if ($primary) {
1995       __ xorptr(Rrdi, Rrdi);
1996     }
1997     __ bind(miss);
1998   %}
1999 
2000   enc_class clear_avx %{
2001     debug_only(int off0 = cbuf.insts_size());
2002     if (ra_->C->max_vector_size() > 16) {
2003       // Clear upper bits of YMM registers when current compiled code uses
2004       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2005       MacroAssembler _masm(&cbuf);
2006       __ vzeroupper();
2007     }
2008     debug_only(int off1 = cbuf.insts_size());
2009     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2010   %}
2011 
2012   enc_class Java_To_Runtime(method meth) %{
2013     // No relocation needed
2014     MacroAssembler _masm(&cbuf);
2015     __ mov64(r10, (int64_t) $meth$$method);
2016     __ call(r10);
2017   %}
2018 
2019   enc_class Java_To_Interpreter(method meth)
2020   %{
2021     // CALL Java_To_Interpreter
2022     // This is the instruction starting address for relocation info.
2023     cbuf.set_insts_mark();
2024     $$$emit8$primary;
2025     // CALL directly to the runtime
2026     emit_d32_reloc(cbuf,
2027                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2028                    runtime_call_Relocation::spec(),
2029                    RELOC_DISP32);
2030   %}
2031 
2032   enc_class Java_Static_Call(method meth)
2033   %{
2034     // JAVA STATIC CALL
2035     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2036     // determine who we intended to call.
2037     cbuf.set_insts_mark();
2038     $$$emit8$primary;
2039 
2040     if (!_method) {
2041       emit_d32_reloc(cbuf,
2042                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2043                      runtime_call_Relocation::spec(),
2044                      RELOC_DISP32);
2045     } else if (_optimized_virtual) {
2046       emit_d32_reloc(cbuf,
2047                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2048                      opt_virtual_call_Relocation::spec(),
2049                      RELOC_DISP32);
2050     } else {
2051       emit_d32_reloc(cbuf,
2052                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2053                      static_call_Relocation::spec(),
2054                      RELOC_DISP32);
2055     }
2056     if (_method) {
2057       // Emit stub for static call.
2058       CompiledStaticCall::emit_to_interp_stub(cbuf);
2059     }
2060   %}
2061 
2062   enc_class Java_Dynamic_Call(method meth) %{
2063     MacroAssembler _masm(&cbuf);
2064     __ ic_call((address)$meth$$method);
2065   %}
2066 
2067   enc_class Java_Compiled_Call(method meth)
2068   %{
2069     // JAVA COMPILED CALL
2070     int disp = in_bytes(Method:: from_compiled_offset());
2071 
2072     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2073     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2074 
2075     // callq *disp(%rax)
2076     cbuf.set_insts_mark();
2077     $$$emit8$primary;
2078     if (disp < 0x80) {
2079       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2080       emit_d8(cbuf, disp); // Displacement
2081     } else {
2082       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2083       emit_d32(cbuf, disp); // Displacement
2084     }
2085   %}
2086 
2087   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2088   %{
2089     // SAL, SAR, SHR
2090     int dstenc = $dst$$reg;
2091     if (dstenc >= 8) {
2092       emit_opcode(cbuf, Assembler::REX_B);
2093       dstenc -= 8;
2094     }
2095     $$$emit8$primary;
2096     emit_rm(cbuf, 0x3, $secondary, dstenc);
2097     $$$emit8$shift$$constant;
2098   %}
2099 
2100   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2101   %{
2102     // SAL, SAR, SHR
2103     int dstenc = $dst$$reg;
2104     if (dstenc < 8) {
2105       emit_opcode(cbuf, Assembler::REX_W);
2106     } else {
2107       emit_opcode(cbuf, Assembler::REX_WB);
2108       dstenc -= 8;
2109     }
2110     $$$emit8$primary;
2111     emit_rm(cbuf, 0x3, $secondary, dstenc);
2112     $$$emit8$shift$$constant;
2113   %}
2114 
2115   enc_class load_immI(rRegI dst, immI src)
2116   %{
2117     int dstenc = $dst$$reg;
2118     if (dstenc >= 8) {
2119       emit_opcode(cbuf, Assembler::REX_B);
2120       dstenc -= 8;
2121     }
2122     emit_opcode(cbuf, 0xB8 | dstenc);
2123     $$$emit32$src$$constant;
2124   %}
2125 
2126   enc_class load_immL(rRegL dst, immL src)
2127   %{
2128     int dstenc = $dst$$reg;
2129     if (dstenc < 8) {
2130       emit_opcode(cbuf, Assembler::REX_W);
2131     } else {
2132       emit_opcode(cbuf, Assembler::REX_WB);
2133       dstenc -= 8;
2134     }
2135     emit_opcode(cbuf, 0xB8 | dstenc);
2136     emit_d64(cbuf, $src$$constant);
2137   %}
2138 
2139   enc_class load_immUL32(rRegL dst, immUL32 src)
2140   %{
2141     // same as load_immI, but this time we care about zeroes in the high word
2142     int dstenc = $dst$$reg;
2143     if (dstenc >= 8) {
2144       emit_opcode(cbuf, Assembler::REX_B);
2145       dstenc -= 8;
2146     }
2147     emit_opcode(cbuf, 0xB8 | dstenc);
2148     $$$emit32$src$$constant;
2149   %}
2150 
2151   enc_class load_immL32(rRegL dst, immL32 src)
2152   %{
2153     int dstenc = $dst$$reg;
2154     if (dstenc < 8) {
2155       emit_opcode(cbuf, Assembler::REX_W);
2156     } else {
2157       emit_opcode(cbuf, Assembler::REX_WB);
2158       dstenc -= 8;
2159     }
2160     emit_opcode(cbuf, 0xC7);
2161     emit_rm(cbuf, 0x03, 0x00, dstenc);
2162     $$$emit32$src$$constant;
2163   %}
2164 
2165   enc_class load_immP31(rRegP dst, immP32 src)
2166   %{
2167     // same as load_immI, but this time we care about zeroes in the high word
2168     int dstenc = $dst$$reg;
2169     if (dstenc >= 8) {
2170       emit_opcode(cbuf, Assembler::REX_B);
2171       dstenc -= 8;
2172     }
2173     emit_opcode(cbuf, 0xB8 | dstenc);
2174     $$$emit32$src$$constant;
2175   %}
2176 
2177   enc_class load_immP(rRegP dst, immP src)
2178   %{
2179     int dstenc = $dst$$reg;
2180     if (dstenc < 8) {
2181       emit_opcode(cbuf, Assembler::REX_W);
2182     } else {
2183       emit_opcode(cbuf, Assembler::REX_WB);
2184       dstenc -= 8;
2185     }
2186     emit_opcode(cbuf, 0xB8 | dstenc);
2187     // This next line should be generated from ADLC
2188     if ($src->constant_reloc() != relocInfo::none) {
2189       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2190     } else {
2191       emit_d64(cbuf, $src$$constant);
2192     }
2193   %}
2194 
2195   enc_class Con32(immI src)
2196   %{
2197     // Output immediate
2198     $$$emit32$src$$constant;
2199   %}
2200 
2201   enc_class Con32F_as_bits(immF src)
2202   %{
2203     // Output Float immediate bits
2204     jfloat jf = $src$$constant;
2205     jint jf_as_bits = jint_cast(jf);
2206     emit_d32(cbuf, jf_as_bits);
2207   %}
2208 
2209   enc_class Con16(immI src)
2210   %{
2211     // Output immediate
2212     $$$emit16$src$$constant;
2213   %}
2214 
2215   // How is this different from Con32??? XXX
2216   enc_class Con_d32(immI src)
2217   %{
2218     emit_d32(cbuf,$src$$constant);
2219   %}
2220 
2221   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2222     // Output immediate memory reference
2223     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2224     emit_d32(cbuf, 0x00);
2225   %}
2226 
2227   enc_class lock_prefix()
2228   %{
2229     if (os::is_MP()) {
2230       emit_opcode(cbuf, 0xF0); // lock
2231     }
2232   %}
2233 
2234   enc_class REX_mem(memory mem)
2235   %{
2236     if ($mem$$base >= 8) {
2237       if ($mem$$index < 8) {
2238         emit_opcode(cbuf, Assembler::REX_B);
2239       } else {
2240         emit_opcode(cbuf, Assembler::REX_XB);
2241       }
2242     } else {
2243       if ($mem$$index >= 8) {
2244         emit_opcode(cbuf, Assembler::REX_X);
2245       }
2246     }
2247   %}
2248 
2249   enc_class REX_mem_wide(memory mem)
2250   %{
2251     if ($mem$$base >= 8) {
2252       if ($mem$$index < 8) {
2253         emit_opcode(cbuf, Assembler::REX_WB);
2254       } else {
2255         emit_opcode(cbuf, Assembler::REX_WXB);
2256       }
2257     } else {
2258       if ($mem$$index < 8) {
2259         emit_opcode(cbuf, Assembler::REX_W);
2260       } else {
2261         emit_opcode(cbuf, Assembler::REX_WX);
2262       }
2263     }
2264   %}
2265 
2266   // for byte regs
2267   enc_class REX_breg(rRegI reg)
2268   %{
2269     if ($reg$$reg >= 4) {
2270       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2271     }
2272   %}
2273 
2274   // for byte regs
2275   enc_class REX_reg_breg(rRegI dst, rRegI src)
2276   %{
2277     if ($dst$$reg < 8) {
2278       if ($src$$reg >= 4) {
2279         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2280       }
2281     } else {
2282       if ($src$$reg < 8) {
2283         emit_opcode(cbuf, Assembler::REX_R);
2284       } else {
2285         emit_opcode(cbuf, Assembler::REX_RB);
2286       }
2287     }
2288   %}
2289 
2290   // for byte regs
2291   enc_class REX_breg_mem(rRegI reg, memory mem)
2292   %{
2293     if ($reg$$reg < 8) {
2294       if ($mem$$base < 8) {
2295         if ($mem$$index >= 8) {
2296           emit_opcode(cbuf, Assembler::REX_X);
2297         } else if ($reg$$reg >= 4) {
2298           emit_opcode(cbuf, Assembler::REX);
2299         }
2300       } else {
2301         if ($mem$$index < 8) {
2302           emit_opcode(cbuf, Assembler::REX_B);
2303         } else {
2304           emit_opcode(cbuf, Assembler::REX_XB);
2305         }
2306       }
2307     } else {
2308       if ($mem$$base < 8) {
2309         if ($mem$$index < 8) {
2310           emit_opcode(cbuf, Assembler::REX_R);
2311         } else {
2312           emit_opcode(cbuf, Assembler::REX_RX);
2313         }
2314       } else {
2315         if ($mem$$index < 8) {
2316           emit_opcode(cbuf, Assembler::REX_RB);
2317         } else {
2318           emit_opcode(cbuf, Assembler::REX_RXB);
2319         }
2320       }
2321     }
2322   %}
2323 
2324   enc_class REX_reg(rRegI reg)
2325   %{
2326     if ($reg$$reg >= 8) {
2327       emit_opcode(cbuf, Assembler::REX_B);
2328     }
2329   %}
2330 
2331   enc_class REX_reg_wide(rRegI reg)
2332   %{
2333     if ($reg$$reg < 8) {
2334       emit_opcode(cbuf, Assembler::REX_W);
2335     } else {
2336       emit_opcode(cbuf, Assembler::REX_WB);
2337     }
2338   %}
2339 
2340   enc_class REX_reg_reg(rRegI dst, rRegI src)
2341   %{
2342     if ($dst$$reg < 8) {
2343       if ($src$$reg >= 8) {
2344         emit_opcode(cbuf, Assembler::REX_B);
2345       }
2346     } else {
2347       if ($src$$reg < 8) {
2348         emit_opcode(cbuf, Assembler::REX_R);
2349       } else {
2350         emit_opcode(cbuf, Assembler::REX_RB);
2351       }
2352     }
2353   %}
2354 
2355   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2356   %{
2357     if ($dst$$reg < 8) {
2358       if ($src$$reg < 8) {
2359         emit_opcode(cbuf, Assembler::REX_W);
2360       } else {
2361         emit_opcode(cbuf, Assembler::REX_WB);
2362       }
2363     } else {
2364       if ($src$$reg < 8) {
2365         emit_opcode(cbuf, Assembler::REX_WR);
2366       } else {
2367         emit_opcode(cbuf, Assembler::REX_WRB);
2368       }
2369     }
2370   %}
2371 
2372   enc_class REX_reg_mem(rRegI reg, memory mem)
2373   %{
2374     if ($reg$$reg < 8) {
2375       if ($mem$$base < 8) {
2376         if ($mem$$index >= 8) {
2377           emit_opcode(cbuf, Assembler::REX_X);
2378         }
2379       } else {
2380         if ($mem$$index < 8) {
2381           emit_opcode(cbuf, Assembler::REX_B);
2382         } else {
2383           emit_opcode(cbuf, Assembler::REX_XB);
2384         }
2385       }
2386     } else {
2387       if ($mem$$base < 8) {
2388         if ($mem$$index < 8) {
2389           emit_opcode(cbuf, Assembler::REX_R);
2390         } else {
2391           emit_opcode(cbuf, Assembler::REX_RX);
2392         }
2393       } else {
2394         if ($mem$$index < 8) {
2395           emit_opcode(cbuf, Assembler::REX_RB);
2396         } else {
2397           emit_opcode(cbuf, Assembler::REX_RXB);
2398         }
2399       }
2400     }
2401   %}
2402 
2403   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2404   %{
2405     if ($reg$$reg < 8) {
2406       if ($mem$$base < 8) {
2407         if ($mem$$index < 8) {
2408           emit_opcode(cbuf, Assembler::REX_W);
2409         } else {
2410           emit_opcode(cbuf, Assembler::REX_WX);
2411         }
2412       } else {
2413         if ($mem$$index < 8) {
2414           emit_opcode(cbuf, Assembler::REX_WB);
2415         } else {
2416           emit_opcode(cbuf, Assembler::REX_WXB);
2417         }
2418       }
2419     } else {
2420       if ($mem$$base < 8) {
2421         if ($mem$$index < 8) {
2422           emit_opcode(cbuf, Assembler::REX_WR);
2423         } else {
2424           emit_opcode(cbuf, Assembler::REX_WRX);
2425         }
2426       } else {
2427         if ($mem$$index < 8) {
2428           emit_opcode(cbuf, Assembler::REX_WRB);
2429         } else {
2430           emit_opcode(cbuf, Assembler::REX_WRXB);
2431         }
2432       }
2433     }
2434   %}
2435 
2436   enc_class reg_mem(rRegI ereg, memory mem)
2437   %{
2438     // High registers handle in encode_RegMem
2439     int reg = $ereg$$reg;
2440     int base = $mem$$base;
2441     int index = $mem$$index;
2442     int scale = $mem$$scale;
2443     int disp = $mem$$disp;
2444     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2445 
2446     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2447   %}
2448 
2449   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2450   %{
2451     int rm_byte_opcode = $rm_opcode$$constant;
2452 
2453     // High registers handle in encode_RegMem
2454     int base = $mem$$base;
2455     int index = $mem$$index;
2456     int scale = $mem$$scale;
2457     int displace = $mem$$disp;
2458 
2459     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2460                                             // working with static
2461                                             // globals
2462     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2463                   disp_reloc);
2464   %}
2465 
2466   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2467   %{
2468     int reg_encoding = $dst$$reg;
2469     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2470     int index        = 0x04;            // 0x04 indicates no index
2471     int scale        = 0x00;            // 0x00 indicates no scale
2472     int displace     = $src1$$constant; // 0x00 indicates no displacement
2473     relocInfo::relocType disp_reloc = relocInfo::none;
2474     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2475                   disp_reloc);
2476   %}
2477 
2478   enc_class neg_reg(rRegI dst)
2479   %{
2480     int dstenc = $dst$$reg;
2481     if (dstenc >= 8) {
2482       emit_opcode(cbuf, Assembler::REX_B);
2483       dstenc -= 8;
2484     }
2485     // NEG $dst
2486     emit_opcode(cbuf, 0xF7);
2487     emit_rm(cbuf, 0x3, 0x03, dstenc);
2488   %}
2489 
2490   enc_class neg_reg_wide(rRegI dst)
2491   %{
2492     int dstenc = $dst$$reg;
2493     if (dstenc < 8) {
2494       emit_opcode(cbuf, Assembler::REX_W);
2495     } else {
2496       emit_opcode(cbuf, Assembler::REX_WB);
2497       dstenc -= 8;
2498     }
2499     // NEG $dst
2500     emit_opcode(cbuf, 0xF7);
2501     emit_rm(cbuf, 0x3, 0x03, dstenc);
2502   %}
2503 
2504   enc_class setLT_reg(rRegI dst)
2505   %{
2506     int dstenc = $dst$$reg;
2507     if (dstenc >= 8) {
2508       emit_opcode(cbuf, Assembler::REX_B);
2509       dstenc -= 8;
2510     } else if (dstenc >= 4) {
2511       emit_opcode(cbuf, Assembler::REX);
2512     }
2513     // SETLT $dst
2514     emit_opcode(cbuf, 0x0F);
2515     emit_opcode(cbuf, 0x9C);
2516     emit_rm(cbuf, 0x3, 0x0, dstenc);
2517   %}
2518 
2519   enc_class setNZ_reg(rRegI dst)
2520   %{
2521     int dstenc = $dst$$reg;
2522     if (dstenc >= 8) {
2523       emit_opcode(cbuf, Assembler::REX_B);
2524       dstenc -= 8;
2525     } else if (dstenc >= 4) {
2526       emit_opcode(cbuf, Assembler::REX);
2527     }
2528     // SETNZ $dst
2529     emit_opcode(cbuf, 0x0F);
2530     emit_opcode(cbuf, 0x95);
2531     emit_rm(cbuf, 0x3, 0x0, dstenc);
2532   %}
2533 
2534 
2535   // Compare the lonogs and set -1, 0, or 1 into dst
2536   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2537   %{
2538     int src1enc = $src1$$reg;
2539     int src2enc = $src2$$reg;
2540     int dstenc = $dst$$reg;
2541 
2542     // cmpq $src1, $src2
2543     if (src1enc < 8) {
2544       if (src2enc < 8) {
2545         emit_opcode(cbuf, Assembler::REX_W);
2546       } else {
2547         emit_opcode(cbuf, Assembler::REX_WB);
2548       }
2549     } else {
2550       if (src2enc < 8) {
2551         emit_opcode(cbuf, Assembler::REX_WR);
2552       } else {
2553         emit_opcode(cbuf, Assembler::REX_WRB);
2554       }
2555     }
2556     emit_opcode(cbuf, 0x3B);
2557     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2558 
2559     // movl $dst, -1
2560     if (dstenc >= 8) {
2561       emit_opcode(cbuf, Assembler::REX_B);
2562     }
2563     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2564     emit_d32(cbuf, -1);
2565 
2566     // jl,s done
2567     emit_opcode(cbuf, 0x7C);
2568     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2569 
2570     // setne $dst
2571     if (dstenc >= 4) {
2572       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2573     }
2574     emit_opcode(cbuf, 0x0F);
2575     emit_opcode(cbuf, 0x95);
2576     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2577 
2578     // movzbl $dst, $dst
2579     if (dstenc >= 4) {
2580       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2581     }
2582     emit_opcode(cbuf, 0x0F);
2583     emit_opcode(cbuf, 0xB6);
2584     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2585   %}
2586 
2587   enc_class Push_ResultXD(regD dst) %{
2588     MacroAssembler _masm(&cbuf);
2589     __ fstp_d(Address(rsp, 0));
2590     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2591     __ addptr(rsp, 8);
2592   %}
2593 
2594   enc_class Push_SrcXD(regD src) %{
2595     MacroAssembler _masm(&cbuf);
2596     __ subptr(rsp, 8);
2597     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2598     __ fld_d(Address(rsp, 0));
2599   %}
2600 
2601 

































































































































































































































2602   enc_class enc_rethrow()
2603   %{
2604     cbuf.set_insts_mark();
2605     emit_opcode(cbuf, 0xE9); // jmp entry
2606     emit_d32_reloc(cbuf,
2607                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2608                    runtime_call_Relocation::spec(),
2609                    RELOC_DISP32);
2610   %}
2611 
2612 %}
2613 
2614 
2615 
2616 //----------FRAME--------------------------------------------------------------
2617 // Definition of frame structure and management information.
2618 //
2619 //  S T A C K   L A Y O U T    Allocators stack-slot number
2620 //                             |   (to get allocators register number
2621 //  G  Owned by    |        |  v    add OptoReg::stack0())
2622 //  r   CALLER     |        |
2623 //  o     |        +--------+      pad to even-align allocators stack-slot
2624 //  w     V        |  pad0  |        numbers; owned by CALLER
2625 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2626 //  h     ^        |   in   |  5
2627 //        |        |  args  |  4   Holes in incoming args owned by SELF
2628 //  |     |        |        |  3
2629 //  |     |        +--------+
2630 //  V     |        | old out|      Empty on Intel, window on Sparc
2631 //        |    old |preserve|      Must be even aligned.
2632 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2633 //        |        |   in   |  3   area for Intel ret address
2634 //     Owned by    |preserve|      Empty on Sparc.
2635 //       SELF      +--------+
2636 //        |        |  pad2  |  2   pad to align old SP
2637 //        |        +--------+  1
2638 //        |        | locks  |  0
2639 //        |        +--------+----> OptoReg::stack0(), even aligned
2640 //        |        |  pad1  | 11   pad to align new SP
2641 //        |        +--------+
2642 //        |        |        | 10
2643 //        |        | spills |  9   spills
2644 //        V        |        |  8   (pad0 slot for callee)
2645 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2646 //        ^        |  out   |  7
2647 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2648 //     Owned by    +--------+
2649 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2650 //        |    new |preserve|      Must be even-aligned.
2651 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2652 //        |        |        |
2653 //
2654 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2655 //         known from SELF's arguments and the Java calling convention.
2656 //         Region 6-7 is determined per call site.
2657 // Note 2: If the calling convention leaves holes in the incoming argument
2658 //         area, those holes are owned by SELF.  Holes in the outgoing area
2659 //         are owned by the CALLEE.  Holes should not be nessecary in the
2660 //         incoming area, as the Java calling convention is completely under
2661 //         the control of the AD file.  Doubles can be sorted and packed to
2662 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2663 //         varargs C calling conventions.
2664 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2665 //         even aligned with pad0 as needed.
2666 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2667 //         region 6-11 is even aligned; it may be padded out more so that
2668 //         the region from SP to FP meets the minimum stack alignment.
2669 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2670 //         alignment.  Region 11, pad1, may be dynamically extended so that
2671 //         SP meets the minimum alignment.
2672 
2673 frame
2674 %{
2675   // What direction does stack grow in (assumed to be same for C & Java)
2676   stack_direction(TOWARDS_LOW);
2677 
2678   // These three registers define part of the calling convention
2679   // between compiled code and the interpreter.
2680   inline_cache_reg(RAX);                // Inline Cache Register
2681   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2682                                         // calling interpreter
2683 
2684   // Optional: name the operand used by cisc-spilling to access
2685   // [stack_pointer + offset]
2686   cisc_spilling_operand_name(indOffset32);
2687 
2688   // Number of stack slots consumed by locking an object
2689   sync_stack_slots(2);
2690 
2691   // Compiled code's Frame Pointer
2692   frame_pointer(RSP);
2693 
2694   // Interpreter stores its frame pointer in a register which is
2695   // stored to the stack by I2CAdaptors.
2696   // I2CAdaptors convert from interpreted java to compiled java.
2697   interpreter_frame_pointer(RBP);
2698 
2699   // Stack alignment requirement
2700   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2701 
2702   // Number of stack slots between incoming argument block and the start of
2703   // a new frame.  The PROLOG must add this many slots to the stack.  The
2704   // EPILOG must remove this many slots.  amd64 needs two slots for
2705   // return address.
2706   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2707 
2708   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2709   // for calls to C.  Supports the var-args backing area for register parms.
2710   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2711 
2712   // The after-PROLOG location of the return address.  Location of
2713   // return address specifies a type (REG or STACK) and a number
2714   // representing the register number (i.e. - use a register name) or
2715   // stack slot.
2716   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2717   // Otherwise, it is above the locks and verification slot and alignment word
2718   return_addr(STACK - 2 +
2719               round_to((Compile::current()->in_preserve_stack_slots() +
2720                         Compile::current()->fixed_slots()),
2721                        stack_alignment_in_slots()));
2722 
2723   // Body of function which returns an integer array locating
2724   // arguments either in registers or in stack slots.  Passed an array
2725   // of ideal registers called "sig" and a "length" count.  Stack-slot
2726   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2727   // arguments for a CALLEE.  Incoming stack arguments are
2728   // automatically biased by the preserve_stack_slots field above.
2729 
2730   calling_convention
2731   %{
2732     // No difference between ingoing/outgoing just pass false
2733     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2734   %}
2735 
2736   c_calling_convention
2737   %{
2738     // This is obviously always outgoing
2739     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2740   %}
2741 
2742   // Location of compiled Java return values.  Same as C for now.
2743   return_value
2744   %{
2745     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2746            "only return normal values");
2747 
2748     static const int lo[Op_RegL + 1] = {
2749       0,
2750       0,
2751       RAX_num,  // Op_RegN
2752       RAX_num,  // Op_RegI
2753       RAX_num,  // Op_RegP
2754       XMM0_num, // Op_RegF
2755       XMM0_num, // Op_RegD
2756       RAX_num   // Op_RegL
2757     };
2758     static const int hi[Op_RegL + 1] = {
2759       0,
2760       0,
2761       OptoReg::Bad, // Op_RegN
2762       OptoReg::Bad, // Op_RegI
2763       RAX_H_num,    // Op_RegP
2764       OptoReg::Bad, // Op_RegF
2765       XMM0b_num,    // Op_RegD
2766       RAX_H_num     // Op_RegL
2767     };
2768     // Excluded flags and vector registers.
2769     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2770     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2771   %}
2772 %}
2773 
2774 //----------ATTRIBUTES---------------------------------------------------------
2775 //----------Operand Attributes-------------------------------------------------
2776 op_attrib op_cost(0);        // Required cost attribute
2777 
2778 //----------Instruction Attributes---------------------------------------------
2779 ins_attrib ins_cost(100);       // Required cost attribute
2780 ins_attrib ins_size(8);         // Required size attribute (in bits)
2781 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2782                                 // a non-matching short branch variant
2783                                 // of some long branch?
2784 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2785                                 // be a power of 2) specifies the
2786                                 // alignment that some part of the
2787                                 // instruction (not necessarily the
2788                                 // start) requires.  If > 1, a
2789                                 // compute_padding() function must be
2790                                 // provided for the instruction
2791 
2792 //----------OPERANDS-----------------------------------------------------------
2793 // Operand definitions must precede instruction definitions for correct parsing
2794 // in the ADLC because operands constitute user defined types which are used in
2795 // instruction definitions.
2796 
2797 //----------Simple Operands----------------------------------------------------
2798 // Immediate Operands
2799 // Integer Immediate
2800 operand immI()
2801 %{
2802   match(ConI);
2803 
2804   op_cost(10);
2805   format %{ %}
2806   interface(CONST_INTER);
2807 %}
2808 
2809 // Constant for test vs zero
2810 operand immI0()
2811 %{
2812   predicate(n->get_int() == 0);
2813   match(ConI);
2814 
2815   op_cost(0);
2816   format %{ %}
2817   interface(CONST_INTER);
2818 %}
2819 
2820 // Constant for increment
2821 operand immI1()
2822 %{
2823   predicate(n->get_int() == 1);
2824   match(ConI);
2825 
2826   op_cost(0);
2827   format %{ %}
2828   interface(CONST_INTER);
2829 %}
2830 
2831 // Constant for decrement
2832 operand immI_M1()
2833 %{
2834   predicate(n->get_int() == -1);
2835   match(ConI);
2836 
2837   op_cost(0);
2838   format %{ %}
2839   interface(CONST_INTER);
2840 %}
2841 
2842 // Valid scale values for addressing modes
2843 operand immI2()
2844 %{
2845   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2846   match(ConI);
2847 
2848   format %{ %}
2849   interface(CONST_INTER);
2850 %}
2851 
2852 operand immI8()
2853 %{
2854   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2855   match(ConI);
2856 
2857   op_cost(5);
2858   format %{ %}
2859   interface(CONST_INTER);
2860 %}
2861 
2862 operand immI16()
2863 %{
2864   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2865   match(ConI);
2866 
2867   op_cost(10);
2868   format %{ %}
2869   interface(CONST_INTER);
2870 %}
2871 
2872 // Int Immediate non-negative
2873 operand immU31()
2874 %{
2875   predicate(n->get_int() >= 0);
2876   match(ConI);
2877 
2878   op_cost(0);
2879   format %{ %}
2880   interface(CONST_INTER);
2881 %}
2882 
2883 // Constant for long shifts
2884 operand immI_32()
2885 %{
2886   predicate( n->get_int() == 32 );
2887   match(ConI);
2888 
2889   op_cost(0);
2890   format %{ %}
2891   interface(CONST_INTER);
2892 %}
2893 
2894 // Constant for long shifts
2895 operand immI_64()
2896 %{
2897   predicate( n->get_int() == 64 );
2898   match(ConI);
2899 
2900   op_cost(0);
2901   format %{ %}
2902   interface(CONST_INTER);
2903 %}
2904 
2905 // Pointer Immediate
2906 operand immP()
2907 %{
2908   match(ConP);
2909 
2910   op_cost(10);
2911   format %{ %}
2912   interface(CONST_INTER);
2913 %}
2914 
2915 // NULL Pointer Immediate
2916 operand immP0()
2917 %{
2918   predicate(n->get_ptr() == 0);
2919   match(ConP);
2920 
2921   op_cost(5);
2922   format %{ %}
2923   interface(CONST_INTER);
2924 %}
2925 
2926 // Pointer Immediate
2927 operand immN() %{
2928   match(ConN);
2929 
2930   op_cost(10);
2931   format %{ %}
2932   interface(CONST_INTER);
2933 %}
2934 
2935 operand immNKlass() %{
2936   match(ConNKlass);
2937 
2938   op_cost(10);
2939   format %{ %}
2940   interface(CONST_INTER);
2941 %}
2942 
2943 // NULL Pointer Immediate
2944 operand immN0() %{
2945   predicate(n->get_narrowcon() == 0);
2946   match(ConN);
2947 
2948   op_cost(5);
2949   format %{ %}
2950   interface(CONST_INTER);
2951 %}
2952 
2953 operand immP31()
2954 %{
2955   predicate(n->as_Type()->type()->reloc() == relocInfo::none
2956             && (n->get_ptr() >> 31) == 0);
2957   match(ConP);
2958 
2959   op_cost(5);
2960   format %{ %}
2961   interface(CONST_INTER);
2962 %}
2963 
2964 
2965 // Long Immediate
2966 operand immL()
2967 %{
2968   match(ConL);
2969 
2970   op_cost(20);
2971   format %{ %}
2972   interface(CONST_INTER);
2973 %}
2974 
2975 // Long Immediate 8-bit
2976 operand immL8()
2977 %{
2978   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2979   match(ConL);
2980 
2981   op_cost(5);
2982   format %{ %}
2983   interface(CONST_INTER);
2984 %}
2985 
2986 // Long Immediate 32-bit unsigned
2987 operand immUL32()
2988 %{
2989   predicate(n->get_long() == (unsigned int) (n->get_long()));
2990   match(ConL);
2991 
2992   op_cost(10);
2993   format %{ %}
2994   interface(CONST_INTER);
2995 %}
2996 
2997 // Long Immediate 32-bit signed
2998 operand immL32()
2999 %{
3000   predicate(n->get_long() == (int) (n->get_long()));
3001   match(ConL);
3002 
3003   op_cost(15);
3004   format %{ %}
3005   interface(CONST_INTER);
3006 %}
3007 
3008 // Long Immediate zero
3009 operand immL0()
3010 %{
3011   predicate(n->get_long() == 0L);
3012   match(ConL);
3013 
3014   op_cost(10);
3015   format %{ %}
3016   interface(CONST_INTER);
3017 %}
3018 
3019 // Constant for increment
3020 operand immL1()
3021 %{
3022   predicate(n->get_long() == 1);
3023   match(ConL);
3024 
3025   format %{ %}
3026   interface(CONST_INTER);
3027 %}
3028 
3029 // Constant for decrement
3030 operand immL_M1()
3031 %{
3032   predicate(n->get_long() == -1);
3033   match(ConL);
3034 
3035   format %{ %}
3036   interface(CONST_INTER);
3037 %}
3038 
3039 // Long Immediate: the value 10
3040 operand immL10()
3041 %{
3042   predicate(n->get_long() == 10);
3043   match(ConL);
3044 
3045   format %{ %}
3046   interface(CONST_INTER);
3047 %}
3048 
3049 // Long immediate from 0 to 127.
3050 // Used for a shorter form of long mul by 10.
3051 operand immL_127()
3052 %{
3053   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3054   match(ConL);
3055 
3056   op_cost(10);
3057   format %{ %}
3058   interface(CONST_INTER);
3059 %}
3060 
3061 // Long Immediate: low 32-bit mask
3062 operand immL_32bits()
3063 %{
3064   predicate(n->get_long() == 0xFFFFFFFFL);
3065   match(ConL);
3066   op_cost(20);
3067 
3068   format %{ %}
3069   interface(CONST_INTER);
3070 %}
3071 
3072 // Float Immediate zero
3073 operand immF0()
3074 %{
3075   predicate(jint_cast(n->getf()) == 0);
3076   match(ConF);
3077 
3078   op_cost(5);
3079   format %{ %}
3080   interface(CONST_INTER);
3081 %}
3082 
3083 // Float Immediate
3084 operand immF()
3085 %{
3086   match(ConF);
3087 
3088   op_cost(15);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Double Immediate zero
3094 operand immD0()
3095 %{
3096   predicate(jlong_cast(n->getd()) == 0);
3097   match(ConD);
3098 
3099   op_cost(5);
3100   format %{ %}
3101   interface(CONST_INTER);
3102 %}
3103 
3104 // Double Immediate
3105 operand immD()
3106 %{
3107   match(ConD);
3108 
3109   op_cost(15);
3110   format %{ %}
3111   interface(CONST_INTER);
3112 %}
3113 
3114 // Immediates for special shifts (sign extend)
3115 
3116 // Constants for increment
3117 operand immI_16()
3118 %{
3119   predicate(n->get_int() == 16);
3120   match(ConI);
3121 
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 operand immI_24()
3127 %{
3128   predicate(n->get_int() == 24);
3129   match(ConI);
3130 
3131   format %{ %}
3132   interface(CONST_INTER);
3133 %}
3134 
3135 // Constant for byte-wide masking
3136 operand immI_255()
3137 %{
3138   predicate(n->get_int() == 255);
3139   match(ConI);
3140 
3141   format %{ %}
3142   interface(CONST_INTER);
3143 %}
3144 
3145 // Constant for short-wide masking
3146 operand immI_65535()
3147 %{
3148   predicate(n->get_int() == 65535);
3149   match(ConI);
3150 
3151   format %{ %}
3152   interface(CONST_INTER);
3153 %}
3154 
3155 // Constant for byte-wide masking
3156 operand immL_255()
3157 %{
3158   predicate(n->get_long() == 255);
3159   match(ConL);
3160 
3161   format %{ %}
3162   interface(CONST_INTER);
3163 %}
3164 
3165 // Constant for short-wide masking
3166 operand immL_65535()
3167 %{
3168   predicate(n->get_long() == 65535);
3169   match(ConL);
3170 
3171   format %{ %}
3172   interface(CONST_INTER);
3173 %}
3174 
3175 // Register Operands
3176 // Integer Register
3177 operand rRegI()
3178 %{
3179   constraint(ALLOC_IN_RC(int_reg));
3180   match(RegI);
3181 
3182   match(rax_RegI);
3183   match(rbx_RegI);
3184   match(rcx_RegI);
3185   match(rdx_RegI);
3186   match(rdi_RegI);
3187 
3188   format %{ %}
3189   interface(REG_INTER);
3190 %}
3191 
3192 // Special Registers
3193 operand rax_RegI()
3194 %{
3195   constraint(ALLOC_IN_RC(int_rax_reg));
3196   match(RegI);
3197   match(rRegI);
3198 
3199   format %{ "RAX" %}
3200   interface(REG_INTER);
3201 %}
3202 
3203 // Special Registers
3204 operand rbx_RegI()
3205 %{
3206   constraint(ALLOC_IN_RC(int_rbx_reg));
3207   match(RegI);
3208   match(rRegI);
3209 
3210   format %{ "RBX" %}
3211   interface(REG_INTER);
3212 %}
3213 
3214 operand rcx_RegI()
3215 %{
3216   constraint(ALLOC_IN_RC(int_rcx_reg));
3217   match(RegI);
3218   match(rRegI);
3219 
3220   format %{ "RCX" %}
3221   interface(REG_INTER);
3222 %}
3223 
3224 operand rdx_RegI()
3225 %{
3226   constraint(ALLOC_IN_RC(int_rdx_reg));
3227   match(RegI);
3228   match(rRegI);
3229 
3230   format %{ "RDX" %}
3231   interface(REG_INTER);
3232 %}
3233 
3234 operand rdi_RegI()
3235 %{
3236   constraint(ALLOC_IN_RC(int_rdi_reg));
3237   match(RegI);
3238   match(rRegI);
3239 
3240   format %{ "RDI" %}
3241   interface(REG_INTER);
3242 %}
3243 
3244 operand no_rcx_RegI()
3245 %{
3246   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3247   match(RegI);
3248   match(rax_RegI);
3249   match(rbx_RegI);
3250   match(rdx_RegI);
3251   match(rdi_RegI);
3252 
3253   format %{ %}
3254   interface(REG_INTER);
3255 %}
3256 
3257 operand no_rax_rdx_RegI()
3258 %{
3259   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3260   match(RegI);
3261   match(rbx_RegI);
3262   match(rcx_RegI);
3263   match(rdi_RegI);
3264 
3265   format %{ %}
3266   interface(REG_INTER);
3267 %}
3268 
3269 // Pointer Register
3270 operand any_RegP()
3271 %{
3272   constraint(ALLOC_IN_RC(any_reg));
3273   match(RegP);
3274   match(rax_RegP);
3275   match(rbx_RegP);
3276   match(rdi_RegP);
3277   match(rsi_RegP);
3278   match(rbp_RegP);
3279   match(r15_RegP);
3280   match(rRegP);
3281 
3282   format %{ %}
3283   interface(REG_INTER);
3284 %}
3285 
3286 operand rRegP()
3287 %{
3288   constraint(ALLOC_IN_RC(ptr_reg));
3289   match(RegP);
3290   match(rax_RegP);
3291   match(rbx_RegP);
3292   match(rdi_RegP);
3293   match(rsi_RegP);
3294   match(rbp_RegP);
3295   match(r15_RegP);  // See Q&A below about r15_RegP.
3296 
3297   format %{ %}
3298   interface(REG_INTER);
3299 %}
3300 
3301 operand rRegN() %{
3302   constraint(ALLOC_IN_RC(int_reg));
3303   match(RegN);
3304 
3305   format %{ %}
3306   interface(REG_INTER);
3307 %}
3308 
3309 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3310 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3311 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
3312 // The output of an instruction is controlled by the allocator, which respects
3313 // register class masks, not match rules.  Unless an instruction mentions
3314 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3315 // by the allocator as an input.
3316 
3317 operand no_rax_RegP()
3318 %{
3319   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3320   match(RegP);
3321   match(rbx_RegP);
3322   match(rsi_RegP);
3323   match(rdi_RegP);
3324 
3325   format %{ %}
3326   interface(REG_INTER);
3327 %}
3328 
3329 operand no_rbp_RegP()
3330 %{
3331   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
3332   match(RegP);
3333   match(rbx_RegP);
3334   match(rsi_RegP);
3335   match(rdi_RegP);
3336 
3337   format %{ %}
3338   interface(REG_INTER);
3339 %}
3340 
3341 operand no_rax_rbx_RegP()
3342 %{
3343   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3344   match(RegP);
3345   match(rsi_RegP);
3346   match(rdi_RegP);
3347 
3348   format %{ %}
3349   interface(REG_INTER);
3350 %}
3351 
3352 // Special Registers
3353 // Return a pointer value
3354 operand rax_RegP()
3355 %{
3356   constraint(ALLOC_IN_RC(ptr_rax_reg));
3357   match(RegP);
3358   match(rRegP);
3359 
3360   format %{ %}
3361   interface(REG_INTER);
3362 %}
3363 
3364 // Special Registers
3365 // Return a compressed pointer value
3366 operand rax_RegN()
3367 %{
3368   constraint(ALLOC_IN_RC(int_rax_reg));
3369   match(RegN);
3370   match(rRegN);
3371 
3372   format %{ %}
3373   interface(REG_INTER);
3374 %}
3375 
3376 // Used in AtomicAdd
3377 operand rbx_RegP()
3378 %{
3379   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3380   match(RegP);
3381   match(rRegP);
3382 
3383   format %{ %}
3384   interface(REG_INTER);
3385 %}
3386 
3387 operand rsi_RegP()
3388 %{
3389   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3390   match(RegP);
3391   match(rRegP);
3392 
3393   format %{ %}
3394   interface(REG_INTER);
3395 %}
3396 
3397 // Used in rep stosq
3398 operand rdi_RegP()
3399 %{
3400   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3401   match(RegP);
3402   match(rRegP);
3403 
3404   format %{ %}
3405   interface(REG_INTER);
3406 %}
3407 
3408 operand rbp_RegP()
3409 %{
3410   constraint(ALLOC_IN_RC(ptr_rbp_reg));
3411   match(RegP);
3412   match(rRegP);
3413 
3414   format %{ %}
3415   interface(REG_INTER);
3416 %}
3417 
3418 operand r15_RegP()
3419 %{
3420   constraint(ALLOC_IN_RC(ptr_r15_reg));
3421   match(RegP);
3422   match(rRegP);
3423 
3424   format %{ %}
3425   interface(REG_INTER);
3426 %}
3427 
3428 operand rRegL()
3429 %{
3430   constraint(ALLOC_IN_RC(long_reg));
3431   match(RegL);
3432   match(rax_RegL);
3433   match(rdx_RegL);
3434 
3435   format %{ %}
3436   interface(REG_INTER);
3437 %}
3438 
3439 // Special Registers
3440 operand no_rax_rdx_RegL()
3441 %{
3442   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3443   match(RegL);
3444   match(rRegL);
3445 
3446   format %{ %}
3447   interface(REG_INTER);
3448 %}
3449 
3450 operand no_rax_RegL()
3451 %{
3452   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3453   match(RegL);
3454   match(rRegL);
3455   match(rdx_RegL);
3456 
3457   format %{ %}
3458   interface(REG_INTER);
3459 %}
3460 
3461 operand no_rcx_RegL()
3462 %{
3463   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3464   match(RegL);
3465   match(rRegL);
3466 
3467   format %{ %}
3468   interface(REG_INTER);
3469 %}
3470 
3471 operand rax_RegL()
3472 %{
3473   constraint(ALLOC_IN_RC(long_rax_reg));
3474   match(RegL);
3475   match(rRegL);
3476 
3477   format %{ "RAX" %}
3478   interface(REG_INTER);
3479 %}
3480 
3481 operand rcx_RegL()
3482 %{
3483   constraint(ALLOC_IN_RC(long_rcx_reg));
3484   match(RegL);
3485   match(rRegL);
3486 
3487   format %{ %}
3488   interface(REG_INTER);
3489 %}
3490 
3491 operand rdx_RegL()
3492 %{
3493   constraint(ALLOC_IN_RC(long_rdx_reg));
3494   match(RegL);
3495   match(rRegL);
3496 
3497   format %{ %}
3498   interface(REG_INTER);
3499 %}
3500 
3501 // Flags register, used as output of compare instructions
3502 operand rFlagsReg()
3503 %{
3504   constraint(ALLOC_IN_RC(int_flags));
3505   match(RegFlags);
3506 
3507   format %{ "RFLAGS" %}
3508   interface(REG_INTER);
3509 %}
3510 
3511 // Flags register, used as output of FLOATING POINT compare instructions
3512 operand rFlagsRegU()
3513 %{
3514   constraint(ALLOC_IN_RC(int_flags));
3515   match(RegFlags);
3516 
3517   format %{ "RFLAGS_U" %}
3518   interface(REG_INTER);
3519 %}
3520 
3521 operand rFlagsRegUCF() %{
3522   constraint(ALLOC_IN_RC(int_flags));
3523   match(RegFlags);
3524   predicate(false);
3525 
3526   format %{ "RFLAGS_U_CF" %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 // Float register operands
3531 operand regF()
3532 %{
3533   constraint(ALLOC_IN_RC(float_reg));
3534   match(RegF);
3535 
3536   format %{ %}
3537   interface(REG_INTER);
3538 %}
3539 
3540 // Double register operands
3541 operand regD()
3542 %{
3543   constraint(ALLOC_IN_RC(double_reg));
3544   match(RegD);
3545 
3546   format %{ %}
3547   interface(REG_INTER);
3548 %}
3549 
3550 //----------Memory Operands----------------------------------------------------
3551 // Direct Memory Operand
3552 // operand direct(immP addr)
3553 // %{
3554 //   match(addr);
3555 
3556 //   format %{ "[$addr]" %}
3557 //   interface(MEMORY_INTER) %{
3558 //     base(0xFFFFFFFF);
3559 //     index(0x4);
3560 //     scale(0x0);
3561 //     disp($addr);
3562 //   %}
3563 // %}
3564 
3565 // Indirect Memory Operand
3566 operand indirect(any_RegP reg)
3567 %{
3568   constraint(ALLOC_IN_RC(ptr_reg));
3569   match(reg);
3570 
3571   format %{ "[$reg]" %}
3572   interface(MEMORY_INTER) %{
3573     base($reg);
3574     index(0x4);
3575     scale(0x0);
3576     disp(0x0);
3577   %}
3578 %}
3579 
3580 // Indirect Memory Plus Short Offset Operand
3581 operand indOffset8(any_RegP reg, immL8 off)
3582 %{
3583   constraint(ALLOC_IN_RC(ptr_reg));
3584   match(AddP reg off);
3585 
3586   format %{ "[$reg + $off (8-bit)]" %}
3587   interface(MEMORY_INTER) %{
3588     base($reg);
3589     index(0x4);
3590     scale(0x0);
3591     disp($off);
3592   %}
3593 %}
3594 
3595 // Indirect Memory Plus Long Offset Operand
3596 operand indOffset32(any_RegP reg, immL32 off)
3597 %{
3598   constraint(ALLOC_IN_RC(ptr_reg));
3599   match(AddP reg off);
3600 
3601   format %{ "[$reg + $off (32-bit)]" %}
3602   interface(MEMORY_INTER) %{
3603     base($reg);
3604     index(0x4);
3605     scale(0x0);
3606     disp($off);
3607   %}
3608 %}
3609 
3610 // Indirect Memory Plus Index Register Plus Offset Operand
3611 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3612 %{
3613   constraint(ALLOC_IN_RC(ptr_reg));
3614   match(AddP (AddP reg lreg) off);
3615 
3616   op_cost(10);
3617   format %{"[$reg + $off + $lreg]" %}
3618   interface(MEMORY_INTER) %{
3619     base($reg);
3620     index($lreg);
3621     scale(0x0);
3622     disp($off);
3623   %}
3624 %}
3625 
3626 // Indirect Memory Plus Index Register Plus Offset Operand
3627 operand indIndex(any_RegP reg, rRegL lreg)
3628 %{
3629   constraint(ALLOC_IN_RC(ptr_reg));
3630   match(AddP reg lreg);
3631 
3632   op_cost(10);
3633   format %{"[$reg + $lreg]" %}
3634   interface(MEMORY_INTER) %{
3635     base($reg);
3636     index($lreg);
3637     scale(0x0);
3638     disp(0x0);
3639   %}
3640 %}
3641 
3642 // Indirect Memory Times Scale Plus Index Register
3643 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3644 %{
3645   constraint(ALLOC_IN_RC(ptr_reg));
3646   match(AddP reg (LShiftL lreg scale));
3647 
3648   op_cost(10);
3649   format %{"[$reg + $lreg << $scale]" %}
3650   interface(MEMORY_INTER) %{
3651     base($reg);
3652     index($lreg);
3653     scale($scale);
3654     disp(0x0);
3655   %}
3656 %}
3657 
3658 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3659 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3660 %{
3661   constraint(ALLOC_IN_RC(ptr_reg));
3662   match(AddP (AddP reg (LShiftL lreg scale)) off);
3663 
3664   op_cost(10);
3665   format %{"[$reg + $off + $lreg << $scale]" %}
3666   interface(MEMORY_INTER) %{
3667     base($reg);
3668     index($lreg);
3669     scale($scale);
3670     disp($off);
3671   %}
3672 %}
3673 
3674 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3675 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3676 %{
3677   constraint(ALLOC_IN_RC(ptr_reg));
3678   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3679   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3680 
3681   op_cost(10);
3682   format %{"[$reg + $off + $idx << $scale]" %}
3683   interface(MEMORY_INTER) %{
3684     base($reg);
3685     index($idx);
3686     scale($scale);
3687     disp($off);
3688   %}
3689 %}
3690 
3691 // Indirect Narrow Oop Plus Offset Operand
3692 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3693 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3694 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3695   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3696   constraint(ALLOC_IN_RC(ptr_reg));
3697   match(AddP (DecodeN reg) off);
3698 
3699   op_cost(10);
3700   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3701   interface(MEMORY_INTER) %{
3702     base(0xc); // R12
3703     index($reg);
3704     scale(0x3);
3705     disp($off);
3706   %}
3707 %}
3708 
3709 // Indirect Memory Operand
3710 operand indirectNarrow(rRegN reg)
3711 %{
3712   predicate(Universe::narrow_oop_shift() == 0);
3713   constraint(ALLOC_IN_RC(ptr_reg));
3714   match(DecodeN reg);
3715 
3716   format %{ "[$reg]" %}
3717   interface(MEMORY_INTER) %{
3718     base($reg);
3719     index(0x4);
3720     scale(0x0);
3721     disp(0x0);
3722   %}
3723 %}
3724 
3725 // Indirect Memory Plus Short Offset Operand
3726 operand indOffset8Narrow(rRegN reg, immL8 off)
3727 %{
3728   predicate(Universe::narrow_oop_shift() == 0);
3729   constraint(ALLOC_IN_RC(ptr_reg));
3730   match(AddP (DecodeN reg) off);
3731 
3732   format %{ "[$reg + $off (8-bit)]" %}
3733   interface(MEMORY_INTER) %{
3734     base($reg);
3735     index(0x4);
3736     scale(0x0);
3737     disp($off);
3738   %}
3739 %}
3740 
3741 // Indirect Memory Plus Long Offset Operand
3742 operand indOffset32Narrow(rRegN reg, immL32 off)
3743 %{
3744   predicate(Universe::narrow_oop_shift() == 0);
3745   constraint(ALLOC_IN_RC(ptr_reg));
3746   match(AddP (DecodeN reg) off);
3747 
3748   format %{ "[$reg + $off (32-bit)]" %}
3749   interface(MEMORY_INTER) %{
3750     base($reg);
3751     index(0x4);
3752     scale(0x0);
3753     disp($off);
3754   %}
3755 %}
3756 
3757 // Indirect Memory Plus Index Register Plus Offset Operand
3758 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3759 %{
3760   predicate(Universe::narrow_oop_shift() == 0);
3761   constraint(ALLOC_IN_RC(ptr_reg));
3762   match(AddP (AddP (DecodeN reg) lreg) off);
3763 
3764   op_cost(10);
3765   format %{"[$reg + $off + $lreg]" %}
3766   interface(MEMORY_INTER) %{
3767     base($reg);
3768     index($lreg);
3769     scale(0x0);
3770     disp($off);
3771   %}
3772 %}
3773 
3774 // Indirect Memory Plus Index Register Plus Offset Operand
3775 operand indIndexNarrow(rRegN reg, rRegL lreg)
3776 %{
3777   predicate(Universe::narrow_oop_shift() == 0);
3778   constraint(ALLOC_IN_RC(ptr_reg));
3779   match(AddP (DecodeN reg) lreg);
3780 
3781   op_cost(10);
3782   format %{"[$reg + $lreg]" %}
3783   interface(MEMORY_INTER) %{
3784     base($reg);
3785     index($lreg);
3786     scale(0x0);
3787     disp(0x0);
3788   %}
3789 %}
3790 
3791 // Indirect Memory Times Scale Plus Index Register
3792 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3793 %{
3794   predicate(Universe::narrow_oop_shift() == 0);
3795   constraint(ALLOC_IN_RC(ptr_reg));
3796   match(AddP (DecodeN reg) (LShiftL lreg scale));
3797 
3798   op_cost(10);
3799   format %{"[$reg + $lreg << $scale]" %}
3800   interface(MEMORY_INTER) %{
3801     base($reg);
3802     index($lreg);
3803     scale($scale);
3804     disp(0x0);
3805   %}
3806 %}
3807 
3808 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3809 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3810 %{
3811   predicate(Universe::narrow_oop_shift() == 0);
3812   constraint(ALLOC_IN_RC(ptr_reg));
3813   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3814 
3815   op_cost(10);
3816   format %{"[$reg + $off + $lreg << $scale]" %}
3817   interface(MEMORY_INTER) %{
3818     base($reg);
3819     index($lreg);
3820     scale($scale);
3821     disp($off);
3822   %}
3823 %}
3824 
3825 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3826 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3827 %{
3828   constraint(ALLOC_IN_RC(ptr_reg));
3829   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3830   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3831 
3832   op_cost(10);
3833   format %{"[$reg + $off + $idx << $scale]" %}
3834   interface(MEMORY_INTER) %{
3835     base($reg);
3836     index($idx);
3837     scale($scale);
3838     disp($off);
3839   %}
3840 %}
3841 
3842 //----------Special Memory Operands--------------------------------------------
3843 // Stack Slot Operand - This operand is used for loading and storing temporary
3844 //                      values on the stack where a match requires a value to
3845 //                      flow through memory.
3846 operand stackSlotP(sRegP reg)
3847 %{
3848   constraint(ALLOC_IN_RC(stack_slots));
3849   // No match rule because this operand is only generated in matching
3850 
3851   format %{ "[$reg]" %}
3852   interface(MEMORY_INTER) %{
3853     base(0x4);   // RSP
3854     index(0x4);  // No Index
3855     scale(0x0);  // No Scale
3856     disp($reg);  // Stack Offset
3857   %}
3858 %}
3859 
3860 operand stackSlotI(sRegI reg)
3861 %{
3862   constraint(ALLOC_IN_RC(stack_slots));
3863   // No match rule because this operand is only generated in matching
3864 
3865   format %{ "[$reg]" %}
3866   interface(MEMORY_INTER) %{
3867     base(0x4);   // RSP
3868     index(0x4);  // No Index
3869     scale(0x0);  // No Scale
3870     disp($reg);  // Stack Offset
3871   %}
3872 %}
3873 
3874 operand stackSlotF(sRegF reg)
3875 %{
3876   constraint(ALLOC_IN_RC(stack_slots));
3877   // No match rule because this operand is only generated in matching
3878 
3879   format %{ "[$reg]" %}
3880   interface(MEMORY_INTER) %{
3881     base(0x4);   // RSP
3882     index(0x4);  // No Index
3883     scale(0x0);  // No Scale
3884     disp($reg);  // Stack Offset
3885   %}
3886 %}
3887 
3888 operand stackSlotD(sRegD reg)
3889 %{
3890   constraint(ALLOC_IN_RC(stack_slots));
3891   // No match rule because this operand is only generated in matching
3892 
3893   format %{ "[$reg]" %}
3894   interface(MEMORY_INTER) %{
3895     base(0x4);   // RSP
3896     index(0x4);  // No Index
3897     scale(0x0);  // No Scale
3898     disp($reg);  // Stack Offset
3899   %}
3900 %}
3901 operand stackSlotL(sRegL reg)
3902 %{
3903   constraint(ALLOC_IN_RC(stack_slots));
3904   // No match rule because this operand is only generated in matching
3905 
3906   format %{ "[$reg]" %}
3907   interface(MEMORY_INTER) %{
3908     base(0x4);   // RSP
3909     index(0x4);  // No Index
3910     scale(0x0);  // No Scale
3911     disp($reg);  // Stack Offset
3912   %}
3913 %}
3914 
3915 //----------Conditional Branch Operands----------------------------------------
3916 // Comparison Op  - This is the operation of the comparison, and is limited to
3917 //                  the following set of codes:
3918 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3919 //
3920 // Other attributes of the comparison, such as unsignedness, are specified
3921 // by the comparison instruction that sets a condition code flags register.
3922 // That result is represented by a flags operand whose subtype is appropriate
3923 // to the unsignedness (etc.) of the comparison.
3924 //
3925 // Later, the instruction which matches both the Comparison Op (a Bool) and
3926 // the flags (produced by the Cmp) specifies the coding of the comparison op
3927 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3928 
3929 // Comparision Code
3930 operand cmpOp()
3931 %{
3932   match(Bool);
3933 
3934   format %{ "" %}
3935   interface(COND_INTER) %{
3936     equal(0x4, "e");
3937     not_equal(0x5, "ne");
3938     less(0xC, "l");
3939     greater_equal(0xD, "ge");
3940     less_equal(0xE, "le");
3941     greater(0xF, "g");
3942     overflow(0x0, "o");
3943     no_overflow(0x1, "no");
3944   %}
3945 %}
3946 
3947 // Comparison Code, unsigned compare.  Used by FP also, with
3948 // C2 (unordered) turned into GT or LT already.  The other bits
3949 // C0 and C3 are turned into Carry & Zero flags.
3950 operand cmpOpU()
3951 %{
3952   match(Bool);
3953 
3954   format %{ "" %}
3955   interface(COND_INTER) %{
3956     equal(0x4, "e");
3957     not_equal(0x5, "ne");
3958     less(0x2, "b");
3959     greater_equal(0x3, "nb");
3960     less_equal(0x6, "be");
3961     greater(0x7, "nbe");
3962     overflow(0x0, "o");
3963     no_overflow(0x1, "no");
3964   %}
3965 %}
3966 
3967 
3968 // Floating comparisons that don't require any fixup for the unordered case
3969 operand cmpOpUCF() %{
3970   match(Bool);
3971   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
3972             n->as_Bool()->_test._test == BoolTest::ge ||
3973             n->as_Bool()->_test._test == BoolTest::le ||
3974             n->as_Bool()->_test._test == BoolTest::gt);
3975   format %{ "" %}
3976   interface(COND_INTER) %{
3977     equal(0x4, "e");
3978     not_equal(0x5, "ne");
3979     less(0x2, "b");
3980     greater_equal(0x3, "nb");
3981     less_equal(0x6, "be");
3982     greater(0x7, "nbe");
3983     overflow(0x0, "o");
3984     no_overflow(0x1, "no");
3985   %}
3986 %}
3987 
3988 
3989 // Floating comparisons that can be fixed up with extra conditional jumps
3990 operand cmpOpUCF2() %{
3991   match(Bool);
3992   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
3993             n->as_Bool()->_test._test == BoolTest::eq);
3994   format %{ "" %}
3995   interface(COND_INTER) %{
3996     equal(0x4, "e");
3997     not_equal(0x5, "ne");
3998     less(0x2, "b");
3999     greater_equal(0x3, "nb");
4000     less_equal(0x6, "be");
4001     greater(0x7, "nbe");
4002     overflow(0x0, "o");
4003     no_overflow(0x1, "no");
4004   %}
4005 %}
4006 
4007 
4008 //----------OPERAND CLASSES----------------------------------------------------
4009 // Operand Classes are groups of operands that are used as to simplify
4010 // instruction definitions by not requiring the AD writer to specify separate
4011 // instructions for every form of operand when the instruction accepts
4012 // multiple operand types with the same basic encoding and format.  The classic
4013 // case of this is memory operands.
4014 
4015 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4016                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
4017                indCompressedOopOffset,
4018                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4019                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4020                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
4021 
4022 //----------PIPELINE-----------------------------------------------------------
4023 // Rules which define the behavior of the target architectures pipeline.
4024 pipeline %{
4025 
4026 //----------ATTRIBUTES---------------------------------------------------------
4027 attributes %{
4028   variable_size_instructions;        // Fixed size instructions
4029   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4030   instruction_unit_size = 1;         // An instruction is 1 bytes long
4031   instruction_fetch_unit_size = 16;  // The processor fetches one line
4032   instruction_fetch_units = 1;       // of 16 bytes
4033 
4034   // List of nop instructions
4035   nops( MachNop );
4036 %}
4037 
4038 //----------RESOURCES----------------------------------------------------------
4039 // Resources are the functional units available to the machine
4040 
4041 // Generic P2/P3 pipeline
4042 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4043 // 3 instructions decoded per cycle.
4044 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4045 // 3 ALU op, only ALU0 handles mul instructions.
4046 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4047            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4048            BR, FPU,
4049            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4050 
4051 //----------PIPELINE DESCRIPTION-----------------------------------------------
4052 // Pipeline Description specifies the stages in the machine's pipeline
4053 
4054 // Generic P2/P3 pipeline
4055 pipe_desc(S0, S1, S2, S3, S4, S5);
4056 
4057 //----------PIPELINE CLASSES---------------------------------------------------
4058 // Pipeline Classes describe the stages in which input and output are
4059 // referenced by the hardware pipeline.
4060 
4061 // Naming convention: ialu or fpu
4062 // Then: _reg
4063 // Then: _reg if there is a 2nd register
4064 // Then: _long if it's a pair of instructions implementing a long
4065 // Then: _fat if it requires the big decoder
4066 //   Or: _mem if it requires the big decoder and a memory unit.
4067 
4068 // Integer ALU reg operation
4069 pipe_class ialu_reg(rRegI dst)
4070 %{
4071     single_instruction;
4072     dst    : S4(write);
4073     dst    : S3(read);
4074     DECODE : S0;        // any decoder
4075     ALU    : S3;        // any alu
4076 %}
4077 
4078 // Long ALU reg operation
4079 pipe_class ialu_reg_long(rRegL dst)
4080 %{
4081     instruction_count(2);
4082     dst    : S4(write);
4083     dst    : S3(read);
4084     DECODE : S0(2);     // any 2 decoders
4085     ALU    : S3(2);     // both alus
4086 %}
4087 
4088 // Integer ALU reg operation using big decoder
4089 pipe_class ialu_reg_fat(rRegI dst)
4090 %{
4091     single_instruction;
4092     dst    : S4(write);
4093     dst    : S3(read);
4094     D0     : S0;        // big decoder only
4095     ALU    : S3;        // any alu
4096 %}
4097 
4098 // Long ALU reg operation using big decoder
4099 pipe_class ialu_reg_long_fat(rRegL dst)
4100 %{
4101     instruction_count(2);
4102     dst    : S4(write);
4103     dst    : S3(read);
4104     D0     : S0(2);     // big decoder only; twice
4105     ALU    : S3(2);     // any 2 alus
4106 %}
4107 
4108 // Integer ALU reg-reg operation
4109 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4110 %{
4111     single_instruction;
4112     dst    : S4(write);
4113     src    : S3(read);
4114     DECODE : S0;        // any decoder
4115     ALU    : S3;        // any alu
4116 %}
4117 
4118 // Long ALU reg-reg operation
4119 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4120 %{
4121     instruction_count(2);
4122     dst    : S4(write);
4123     src    : S3(read);
4124     DECODE : S0(2);     // any 2 decoders
4125     ALU    : S3(2);     // both alus
4126 %}
4127 
4128 // Integer ALU reg-reg operation
4129 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4130 %{
4131     single_instruction;
4132     dst    : S4(write);
4133     src    : S3(read);
4134     D0     : S0;        // big decoder only
4135     ALU    : S3;        // any alu
4136 %}
4137 
4138 // Long ALU reg-reg operation
4139 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4140 %{
4141     instruction_count(2);
4142     dst    : S4(write);
4143     src    : S3(read);
4144     D0     : S0(2);     // big decoder only; twice
4145     ALU    : S3(2);     // both alus
4146 %}
4147 
4148 // Integer ALU reg-mem operation
4149 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4150 %{
4151     single_instruction;
4152     dst    : S5(write);
4153     mem    : S3(read);
4154     D0     : S0;        // big decoder only
4155     ALU    : S4;        // any alu
4156     MEM    : S3;        // any mem
4157 %}
4158 
4159 // Integer mem operation (prefetch)
4160 pipe_class ialu_mem(memory mem)
4161 %{
4162     single_instruction;
4163     mem    : S3(read);
4164     D0     : S0;        // big decoder only
4165     MEM    : S3;        // any mem
4166 %}
4167 
4168 // Integer Store to Memory
4169 pipe_class ialu_mem_reg(memory mem, rRegI src)
4170 %{
4171     single_instruction;
4172     mem    : S3(read);
4173     src    : S5(read);
4174     D0     : S0;        // big decoder only
4175     ALU    : S4;        // any alu
4176     MEM    : S3;
4177 %}
4178 
4179 // // Long Store to Memory
4180 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4181 // %{
4182 //     instruction_count(2);
4183 //     mem    : S3(read);
4184 //     src    : S5(read);
4185 //     D0     : S0(2);          // big decoder only; twice
4186 //     ALU    : S4(2);     // any 2 alus
4187 //     MEM    : S3(2);  // Both mems
4188 // %}
4189 
4190 // Integer Store to Memory
4191 pipe_class ialu_mem_imm(memory mem)
4192 %{
4193     single_instruction;
4194     mem    : S3(read);
4195     D0     : S0;        // big decoder only
4196     ALU    : S4;        // any alu
4197     MEM    : S3;
4198 %}
4199 
4200 // Integer ALU0 reg-reg operation
4201 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4202 %{
4203     single_instruction;
4204     dst    : S4(write);
4205     src    : S3(read);
4206     D0     : S0;        // Big decoder only
4207     ALU0   : S3;        // only alu0
4208 %}
4209 
4210 // Integer ALU0 reg-mem operation
4211 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4212 %{
4213     single_instruction;
4214     dst    : S5(write);
4215     mem    : S3(read);
4216     D0     : S0;        // big decoder only
4217     ALU0   : S4;        // ALU0 only
4218     MEM    : S3;        // any mem
4219 %}
4220 
4221 // Integer ALU reg-reg operation
4222 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4223 %{
4224     single_instruction;
4225     cr     : S4(write);
4226     src1   : S3(read);
4227     src2   : S3(read);
4228     DECODE : S0;        // any decoder
4229     ALU    : S3;        // any alu
4230 %}
4231 
4232 // Integer ALU reg-imm operation
4233 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4234 %{
4235     single_instruction;
4236     cr     : S4(write);
4237     src1   : S3(read);
4238     DECODE : S0;        // any decoder
4239     ALU    : S3;        // any alu
4240 %}
4241 
4242 // Integer ALU reg-mem operation
4243 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4244 %{
4245     single_instruction;
4246     cr     : S4(write);
4247     src1   : S3(read);
4248     src2   : S3(read);
4249     D0     : S0;        // big decoder only
4250     ALU    : S4;        // any alu
4251     MEM    : S3;
4252 %}
4253 
4254 // Conditional move reg-reg
4255 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4256 %{
4257     instruction_count(4);
4258     y      : S4(read);
4259     q      : S3(read);
4260     p      : S3(read);
4261     DECODE : S0(4);     // any decoder
4262 %}
4263 
4264 // Conditional move reg-reg
4265 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4266 %{
4267     single_instruction;
4268     dst    : S4(write);
4269     src    : S3(read);
4270     cr     : S3(read);
4271     DECODE : S0;        // any decoder
4272 %}
4273 
4274 // Conditional move reg-mem
4275 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4276 %{
4277     single_instruction;
4278     dst    : S4(write);
4279     src    : S3(read);
4280     cr     : S3(read);
4281     DECODE : S0;        // any decoder
4282     MEM    : S3;
4283 %}
4284 
4285 // Conditional move reg-reg long
4286 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4287 %{
4288     single_instruction;
4289     dst    : S4(write);
4290     src    : S3(read);
4291     cr     : S3(read);
4292     DECODE : S0(2);     // any 2 decoders
4293 %}
4294 
4295 // XXX
4296 // // Conditional move double reg-reg
4297 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4298 // %{
4299 //     single_instruction;
4300 //     dst    : S4(write);
4301 //     src    : S3(read);
4302 //     cr     : S3(read);
4303 //     DECODE : S0;     // any decoder
4304 // %}
4305 
4306 // Float reg-reg operation
4307 pipe_class fpu_reg(regD dst)
4308 %{
4309     instruction_count(2);
4310     dst    : S3(read);
4311     DECODE : S0(2);     // any 2 decoders
4312     FPU    : S3;
4313 %}
4314 
4315 // Float reg-reg operation
4316 pipe_class fpu_reg_reg(regD dst, regD src)
4317 %{
4318     instruction_count(2);
4319     dst    : S4(write);
4320     src    : S3(read);
4321     DECODE : S0(2);     // any 2 decoders
4322     FPU    : S3;
4323 %}
4324 
4325 // Float reg-reg operation
4326 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4327 %{
4328     instruction_count(3);
4329     dst    : S4(write);
4330     src1   : S3(read);
4331     src2   : S3(read);
4332     DECODE : S0(3);     // any 3 decoders
4333     FPU    : S3(2);
4334 %}
4335 
4336 // Float reg-reg operation
4337 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4338 %{
4339     instruction_count(4);
4340     dst    : S4(write);
4341     src1   : S3(read);
4342     src2   : S3(read);
4343     src3   : S3(read);
4344     DECODE : S0(4);     // any 3 decoders
4345     FPU    : S3(2);
4346 %}
4347 
4348 // Float reg-reg operation
4349 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4350 %{
4351     instruction_count(4);
4352     dst    : S4(write);
4353     src1   : S3(read);
4354     src2   : S3(read);
4355     src3   : S3(read);
4356     DECODE : S1(3);     // any 3 decoders
4357     D0     : S0;        // Big decoder only
4358     FPU    : S3(2);
4359     MEM    : S3;
4360 %}
4361 
4362 // Float reg-mem operation
4363 pipe_class fpu_reg_mem(regD dst, memory mem)
4364 %{
4365     instruction_count(2);
4366     dst    : S5(write);
4367     mem    : S3(read);
4368     D0     : S0;        // big decoder only
4369     DECODE : S1;        // any decoder for FPU POP
4370     FPU    : S4;
4371     MEM    : S3;        // any mem
4372 %}
4373 
4374 // Float reg-mem operation
4375 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4376 %{
4377     instruction_count(3);
4378     dst    : S5(write);
4379     src1   : S3(read);
4380     mem    : S3(read);
4381     D0     : S0;        // big decoder only
4382     DECODE : S1(2);     // any decoder for FPU POP
4383     FPU    : S4;
4384     MEM    : S3;        // any mem
4385 %}
4386 
4387 // Float mem-reg operation
4388 pipe_class fpu_mem_reg(memory mem, regD src)
4389 %{
4390     instruction_count(2);
4391     src    : S5(read);
4392     mem    : S3(read);
4393     DECODE : S0;        // any decoder for FPU PUSH
4394     D0     : S1;        // big decoder only
4395     FPU    : S4;
4396     MEM    : S3;        // any mem
4397 %}
4398 
4399 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4400 %{
4401     instruction_count(3);
4402     src1   : S3(read);
4403     src2   : S3(read);
4404     mem    : S3(read);
4405     DECODE : S0(2);     // any decoder for FPU PUSH
4406     D0     : S1;        // big decoder only
4407     FPU    : S4;
4408     MEM    : S3;        // any mem
4409 %}
4410 
4411 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4412 %{
4413     instruction_count(3);
4414     src1   : S3(read);
4415     src2   : S3(read);
4416     mem    : S4(read);
4417     DECODE : S0;        // any decoder for FPU PUSH
4418     D0     : S0(2);     // big decoder only
4419     FPU    : S4;
4420     MEM    : S3(2);     // any mem
4421 %}
4422 
4423 pipe_class fpu_mem_mem(memory dst, memory src1)
4424 %{
4425     instruction_count(2);
4426     src1   : S3(read);
4427     dst    : S4(read);
4428     D0     : S0(2);     // big decoder only
4429     MEM    : S3(2);     // any mem
4430 %}
4431 
4432 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4433 %{
4434     instruction_count(3);
4435     src1   : S3(read);
4436     src2   : S3(read);
4437     dst    : S4(read);
4438     D0     : S0(3);     // big decoder only
4439     FPU    : S4;
4440     MEM    : S3(3);     // any mem
4441 %}
4442 
4443 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4444 %{
4445     instruction_count(3);
4446     src1   : S4(read);
4447     mem    : S4(read);
4448     DECODE : S0;        // any decoder for FPU PUSH
4449     D0     : S0(2);     // big decoder only
4450     FPU    : S4;
4451     MEM    : S3(2);     // any mem
4452 %}
4453 
4454 // Float load constant
4455 pipe_class fpu_reg_con(regD dst)
4456 %{
4457     instruction_count(2);
4458     dst    : S5(write);
4459     D0     : S0;        // big decoder only for the load
4460     DECODE : S1;        // any decoder for FPU POP
4461     FPU    : S4;
4462     MEM    : S3;        // any mem
4463 %}
4464 
4465 // Float load constant
4466 pipe_class fpu_reg_reg_con(regD dst, regD src)
4467 %{
4468     instruction_count(3);
4469     dst    : S5(write);
4470     src    : S3(read);
4471     D0     : S0;        // big decoder only for the load
4472     DECODE : S1(2);     // any decoder for FPU POP
4473     FPU    : S4;
4474     MEM    : S3;        // any mem
4475 %}
4476 
4477 // UnConditional branch
4478 pipe_class pipe_jmp(label labl)
4479 %{
4480     single_instruction;
4481     BR   : S3;
4482 %}
4483 
4484 // Conditional branch
4485 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4486 %{
4487     single_instruction;
4488     cr    : S1(read);
4489     BR    : S3;
4490 %}
4491 
4492 // Allocation idiom
4493 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4494 %{
4495     instruction_count(1); force_serialization;
4496     fixed_latency(6);
4497     heap_ptr : S3(read);
4498     DECODE   : S0(3);
4499     D0       : S2;
4500     MEM      : S3;
4501     ALU      : S3(2);
4502     dst      : S5(write);
4503     BR       : S5;
4504 %}
4505 
4506 // Generic big/slow expanded idiom
4507 pipe_class pipe_slow()
4508 %{
4509     instruction_count(10); multiple_bundles; force_serialization;
4510     fixed_latency(100);
4511     D0  : S0(2);
4512     MEM : S3(2);
4513 %}
4514 
4515 // The real do-nothing guy
4516 pipe_class empty()
4517 %{
4518     instruction_count(0);
4519 %}
4520 
4521 // Define the class for the Nop node
4522 define
4523 %{
4524    MachNop = empty;
4525 %}
4526 
4527 %}
4528 
4529 //----------INSTRUCTIONS-------------------------------------------------------
4530 //
4531 // match      -- States which machine-independent subtree may be replaced
4532 //               by this instruction.
4533 // ins_cost   -- The estimated cost of this instruction is used by instruction
4534 //               selection to identify a minimum cost tree of machine
4535 //               instructions that matches a tree of machine-independent
4536 //               instructions.
4537 // format     -- A string providing the disassembly for this instruction.
4538 //               The value of an instruction's operand may be inserted
4539 //               by referring to it with a '$' prefix.
4540 // opcode     -- Three instruction opcodes may be provided.  These are referred
4541 //               to within an encode class as $primary, $secondary, and $tertiary
4542 //               rrspectively.  The primary opcode is commonly used to
4543 //               indicate the type of machine instruction, while secondary
4544 //               and tertiary are often used for prefix options or addressing
4545 //               modes.
4546 // ins_encode -- A list of encode classes with parameters. The encode class
4547 //               name must have been defined in an 'enc_class' specification
4548 //               in the encode section of the architecture description.
4549 
4550 
4551 //----------Load/Store/Move Instructions---------------------------------------
4552 //----------Load Instructions--------------------------------------------------
4553 
4554 // Load Byte (8 bit signed)
4555 instruct loadB(rRegI dst, memory mem)
4556 %{
4557   match(Set dst (LoadB mem));
4558 
4559   ins_cost(125);
4560   format %{ "movsbl  $dst, $mem\t# byte" %}
4561 
4562   ins_encode %{
4563     __ movsbl($dst$$Register, $mem$$Address);
4564   %}
4565 
4566   ins_pipe(ialu_reg_mem);
4567 %}
4568 
4569 // Load Byte (8 bit signed) into Long Register
4570 instruct loadB2L(rRegL dst, memory mem)
4571 %{
4572   match(Set dst (ConvI2L (LoadB mem)));
4573 
4574   ins_cost(125);
4575   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4576 
4577   ins_encode %{
4578     __ movsbq($dst$$Register, $mem$$Address);
4579   %}
4580 
4581   ins_pipe(ialu_reg_mem);
4582 %}
4583 
4584 // Load Unsigned Byte (8 bit UNsigned)
4585 instruct loadUB(rRegI dst, memory mem)
4586 %{
4587   match(Set dst (LoadUB mem));
4588 
4589   ins_cost(125);
4590   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4591 
4592   ins_encode %{
4593     __ movzbl($dst$$Register, $mem$$Address);
4594   %}
4595 
4596   ins_pipe(ialu_reg_mem);
4597 %}
4598 
4599 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4600 instruct loadUB2L(rRegL dst, memory mem)
4601 %{
4602   match(Set dst (ConvI2L (LoadUB mem)));
4603 
4604   ins_cost(125);
4605   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4606 
4607   ins_encode %{
4608     __ movzbq($dst$$Register, $mem$$Address);
4609   %}
4610 
4611   ins_pipe(ialu_reg_mem);
4612 %}
4613 
4614 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4615 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4616   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4617   effect(KILL cr);
4618 
4619   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4620             "andl    $dst, $mask" %}
4621   ins_encode %{
4622     Register Rdst = $dst$$Register;
4623     __ movzbq(Rdst, $mem$$Address);
4624     __ andl(Rdst, $mask$$constant);
4625   %}
4626   ins_pipe(ialu_reg_mem);
4627 %}
4628 
4629 // Load Short (16 bit signed)
4630 instruct loadS(rRegI dst, memory mem)
4631 %{
4632   match(Set dst (LoadS mem));
4633 
4634   ins_cost(125);
4635   format %{ "movswl $dst, $mem\t# short" %}
4636 
4637   ins_encode %{
4638     __ movswl($dst$$Register, $mem$$Address);
4639   %}
4640 
4641   ins_pipe(ialu_reg_mem);
4642 %}
4643 
4644 // Load Short (16 bit signed) to Byte (8 bit signed)
4645 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4646   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4647 
4648   ins_cost(125);
4649   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4650   ins_encode %{
4651     __ movsbl($dst$$Register, $mem$$Address);
4652   %}
4653   ins_pipe(ialu_reg_mem);
4654 %}
4655 
4656 // Load Short (16 bit signed) into Long Register
4657 instruct loadS2L(rRegL dst, memory mem)
4658 %{
4659   match(Set dst (ConvI2L (LoadS mem)));
4660 
4661   ins_cost(125);
4662   format %{ "movswq $dst, $mem\t# short -> long" %}
4663 
4664   ins_encode %{
4665     __ movswq($dst$$Register, $mem$$Address);
4666   %}
4667 
4668   ins_pipe(ialu_reg_mem);
4669 %}
4670 
4671 // Load Unsigned Short/Char (16 bit UNsigned)
4672 instruct loadUS(rRegI dst, memory mem)
4673 %{
4674   match(Set dst (LoadUS mem));
4675 
4676   ins_cost(125);
4677   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4678 
4679   ins_encode %{
4680     __ movzwl($dst$$Register, $mem$$Address);
4681   %}
4682 
4683   ins_pipe(ialu_reg_mem);
4684 %}
4685 
4686 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4687 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4688   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4689 
4690   ins_cost(125);
4691   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4692   ins_encode %{
4693     __ movsbl($dst$$Register, $mem$$Address);
4694   %}
4695   ins_pipe(ialu_reg_mem);
4696 %}
4697 
4698 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4699 instruct loadUS2L(rRegL dst, memory mem)
4700 %{
4701   match(Set dst (ConvI2L (LoadUS mem)));
4702 
4703   ins_cost(125);
4704   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4705 
4706   ins_encode %{
4707     __ movzwq($dst$$Register, $mem$$Address);
4708   %}
4709 
4710   ins_pipe(ialu_reg_mem);
4711 %}
4712 
4713 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4714 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4715   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4716 
4717   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4718   ins_encode %{
4719     __ movzbq($dst$$Register, $mem$$Address);
4720   %}
4721   ins_pipe(ialu_reg_mem);
4722 %}
4723 
4724 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4725 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4726   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4727   effect(KILL cr);
4728 
4729   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4730             "andl    $dst, $mask" %}
4731   ins_encode %{
4732     Register Rdst = $dst$$Register;
4733     __ movzwq(Rdst, $mem$$Address);
4734     __ andl(Rdst, $mask$$constant);
4735   %}
4736   ins_pipe(ialu_reg_mem);
4737 %}
4738 
4739 // Load Integer
4740 instruct loadI(rRegI dst, memory mem)
4741 %{
4742   match(Set dst (LoadI mem));
4743 
4744   ins_cost(125);
4745   format %{ "movl    $dst, $mem\t# int" %}
4746 
4747   ins_encode %{
4748     __ movl($dst$$Register, $mem$$Address);
4749   %}
4750 
4751   ins_pipe(ialu_reg_mem);
4752 %}
4753 
4754 // Load Integer (32 bit signed) to Byte (8 bit signed)
4755 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4756   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4757 
4758   ins_cost(125);
4759   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4760   ins_encode %{
4761     __ movsbl($dst$$Register, $mem$$Address);
4762   %}
4763   ins_pipe(ialu_reg_mem);
4764 %}
4765 
4766 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4767 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4768   match(Set dst (AndI (LoadI mem) mask));
4769 
4770   ins_cost(125);
4771   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4772   ins_encode %{
4773     __ movzbl($dst$$Register, $mem$$Address);
4774   %}
4775   ins_pipe(ialu_reg_mem);
4776 %}
4777 
4778 // Load Integer (32 bit signed) to Short (16 bit signed)
4779 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4780   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4781 
4782   ins_cost(125);
4783   format %{ "movswl  $dst, $mem\t# int -> short" %}
4784   ins_encode %{
4785     __ movswl($dst$$Register, $mem$$Address);
4786   %}
4787   ins_pipe(ialu_reg_mem);
4788 %}
4789 
4790 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4791 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4792   match(Set dst (AndI (LoadI mem) mask));
4793 
4794   ins_cost(125);
4795   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4796   ins_encode %{
4797     __ movzwl($dst$$Register, $mem$$Address);
4798   %}
4799   ins_pipe(ialu_reg_mem);
4800 %}
4801 
4802 // Load Integer into Long Register
4803 instruct loadI2L(rRegL dst, memory mem)
4804 %{
4805   match(Set dst (ConvI2L (LoadI mem)));
4806 
4807   ins_cost(125);
4808   format %{ "movslq  $dst, $mem\t# int -> long" %}
4809 
4810   ins_encode %{
4811     __ movslq($dst$$Register, $mem$$Address);
4812   %}
4813 
4814   ins_pipe(ialu_reg_mem);
4815 %}
4816 
4817 // Load Integer with mask 0xFF into Long Register
4818 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4819   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4820 
4821   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4822   ins_encode %{
4823     __ movzbq($dst$$Register, $mem$$Address);
4824   %}
4825   ins_pipe(ialu_reg_mem);
4826 %}
4827 
4828 // Load Integer with mask 0xFFFF into Long Register
4829 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4830   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4831 
4832   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4833   ins_encode %{
4834     __ movzwq($dst$$Register, $mem$$Address);
4835   %}
4836   ins_pipe(ialu_reg_mem);
4837 %}
4838 
4839 // Load Integer with a 31-bit mask into Long Register
4840 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4841   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4842   effect(KILL cr);
4843 
4844   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4845             "andl    $dst, $mask" %}
4846   ins_encode %{
4847     Register Rdst = $dst$$Register;
4848     __ movl(Rdst, $mem$$Address);
4849     __ andl(Rdst, $mask$$constant);
4850   %}
4851   ins_pipe(ialu_reg_mem);
4852 %}
4853 
4854 // Load Unsigned Integer into Long Register
4855 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4856 %{
4857   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4858 
4859   ins_cost(125);
4860   format %{ "movl    $dst, $mem\t# uint -> long" %}
4861 
4862   ins_encode %{
4863     __ movl($dst$$Register, $mem$$Address);
4864   %}
4865 
4866   ins_pipe(ialu_reg_mem);
4867 %}
4868 
4869 // Load Long
4870 instruct loadL(rRegL dst, memory mem)
4871 %{
4872   match(Set dst (LoadL mem));
4873 
4874   ins_cost(125);
4875   format %{ "movq    $dst, $mem\t# long" %}
4876 
4877   ins_encode %{
4878     __ movq($dst$$Register, $mem$$Address);
4879   %}
4880 
4881   ins_pipe(ialu_reg_mem); // XXX
4882 %}
4883 
4884 // Load Range
4885 instruct loadRange(rRegI dst, memory mem)
4886 %{
4887   match(Set dst (LoadRange mem));
4888 
4889   ins_cost(125); // XXX
4890   format %{ "movl    $dst, $mem\t# range" %}
4891   opcode(0x8B);
4892   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4893   ins_pipe(ialu_reg_mem);
4894 %}
4895 
4896 // Load Pointer
4897 instruct loadP(rRegP dst, memory mem)
4898 %{
4899   match(Set dst (LoadP mem));
4900 
4901   ins_cost(125); // XXX
4902   format %{ "movq    $dst, $mem\t# ptr" %}
4903   opcode(0x8B);
4904   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4905   ins_pipe(ialu_reg_mem); // XXX
4906 %}
4907 
4908 // Load Compressed Pointer
4909 instruct loadN(rRegN dst, memory mem)
4910 %{
4911    match(Set dst (LoadN mem));
4912 
4913    ins_cost(125); // XXX
4914    format %{ "movl    $dst, $mem\t# compressed ptr" %}
4915    ins_encode %{
4916      __ movl($dst$$Register, $mem$$Address);
4917    %}
4918    ins_pipe(ialu_reg_mem); // XXX
4919 %}
4920 
4921 
4922 // Load Klass Pointer
4923 instruct loadKlass(rRegP dst, memory mem)
4924 %{
4925   match(Set dst (LoadKlass mem));
4926 
4927   ins_cost(125); // XXX
4928   format %{ "movq    $dst, $mem\t# class" %}
4929   opcode(0x8B);
4930   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4931   ins_pipe(ialu_reg_mem); // XXX
4932 %}
4933 
4934 // Load narrow Klass Pointer
4935 instruct loadNKlass(rRegN dst, memory mem)
4936 %{
4937   match(Set dst (LoadNKlass mem));
4938 
4939   ins_cost(125); // XXX
4940   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
4941   ins_encode %{
4942     __ movl($dst$$Register, $mem$$Address);
4943   %}
4944   ins_pipe(ialu_reg_mem); // XXX
4945 %}
4946 
4947 // Load Float
4948 instruct loadF(regF dst, memory mem)
4949 %{
4950   match(Set dst (LoadF mem));
4951 
4952   ins_cost(145); // XXX
4953   format %{ "movss   $dst, $mem\t# float" %}
4954   ins_encode %{
4955     __ movflt($dst$$XMMRegister, $mem$$Address);
4956   %}
4957   ins_pipe(pipe_slow); // XXX
4958 %}
4959 
4960 // Load Double
4961 instruct loadD_partial(regD dst, memory mem)
4962 %{
4963   predicate(!UseXmmLoadAndClearUpper);
4964   match(Set dst (LoadD mem));
4965 
4966   ins_cost(145); // XXX
4967   format %{ "movlpd  $dst, $mem\t# double" %}
4968   ins_encode %{
4969     __ movdbl($dst$$XMMRegister, $mem$$Address);
4970   %}
4971   ins_pipe(pipe_slow); // XXX
4972 %}
4973 
4974 instruct loadD(regD dst, memory mem)
4975 %{
4976   predicate(UseXmmLoadAndClearUpper);
4977   match(Set dst (LoadD mem));
4978 
4979   ins_cost(145); // XXX
4980   format %{ "movsd   $dst, $mem\t# double" %}
4981   ins_encode %{
4982     __ movdbl($dst$$XMMRegister, $mem$$Address);
4983   %}
4984   ins_pipe(pipe_slow); // XXX
4985 %}
4986 
4987 // Load Effective Address
4988 instruct leaP8(rRegP dst, indOffset8 mem)
4989 %{
4990   match(Set dst mem);
4991 
4992   ins_cost(110); // XXX
4993   format %{ "leaq    $dst, $mem\t# ptr 8" %}
4994   opcode(0x8D);
4995   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4996   ins_pipe(ialu_reg_reg_fat);
4997 %}
4998 
4999 instruct leaP32(rRegP dst, indOffset32 mem)
5000 %{
5001   match(Set dst mem);
5002 
5003   ins_cost(110);
5004   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5005   opcode(0x8D);
5006   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5007   ins_pipe(ialu_reg_reg_fat);
5008 %}
5009 
5010 // instruct leaPIdx(rRegP dst, indIndex mem)
5011 // %{
5012 //   match(Set dst mem);
5013 
5014 //   ins_cost(110);
5015 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5016 //   opcode(0x8D);
5017 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5018 //   ins_pipe(ialu_reg_reg_fat);
5019 // %}
5020 
5021 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5022 %{
5023   match(Set dst mem);
5024 
5025   ins_cost(110);
5026   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5027   opcode(0x8D);
5028   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5029   ins_pipe(ialu_reg_reg_fat);
5030 %}
5031 
5032 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5033 %{
5034   match(Set dst mem);
5035 
5036   ins_cost(110);
5037   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5038   opcode(0x8D);
5039   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5040   ins_pipe(ialu_reg_reg_fat);
5041 %}
5042 
5043 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5044 %{
5045   match(Set dst mem);
5046 
5047   ins_cost(110);
5048   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5049   opcode(0x8D);
5050   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5051   ins_pipe(ialu_reg_reg_fat);
5052 %}
5053 
5054 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5055 %{
5056   match(Set dst mem);
5057 
5058   ins_cost(110);
5059   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5060   opcode(0x8D);
5061   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5062   ins_pipe(ialu_reg_reg_fat);
5063 %}
5064 
5065 // Load Effective Address which uses Narrow (32-bits) oop
5066 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5067 %{
5068   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5069   match(Set dst mem);
5070 
5071   ins_cost(110);
5072   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5073   opcode(0x8D);
5074   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5075   ins_pipe(ialu_reg_reg_fat);
5076 %}
5077 
5078 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5079 %{
5080   predicate(Universe::narrow_oop_shift() == 0);
5081   match(Set dst mem);
5082 
5083   ins_cost(110); // XXX
5084   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5085   opcode(0x8D);
5086   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5087   ins_pipe(ialu_reg_reg_fat);
5088 %}
5089 
5090 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5091 %{
5092   predicate(Universe::narrow_oop_shift() == 0);
5093   match(Set dst mem);
5094 
5095   ins_cost(110);
5096   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5097   opcode(0x8D);
5098   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5099   ins_pipe(ialu_reg_reg_fat);
5100 %}
5101 
5102 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5103 %{
5104   predicate(Universe::narrow_oop_shift() == 0);
5105   match(Set dst mem);
5106 
5107   ins_cost(110);
5108   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5109   opcode(0x8D);
5110   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5111   ins_pipe(ialu_reg_reg_fat);
5112 %}
5113 
5114 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5115 %{
5116   predicate(Universe::narrow_oop_shift() == 0);
5117   match(Set dst mem);
5118 
5119   ins_cost(110);
5120   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5121   opcode(0x8D);
5122   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5123   ins_pipe(ialu_reg_reg_fat);
5124 %}
5125 
5126 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5127 %{
5128   predicate(Universe::narrow_oop_shift() == 0);
5129   match(Set dst mem);
5130 
5131   ins_cost(110);
5132   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5133   opcode(0x8D);
5134   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5135   ins_pipe(ialu_reg_reg_fat);
5136 %}
5137 
5138 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5139 %{
5140   predicate(Universe::narrow_oop_shift() == 0);
5141   match(Set dst mem);
5142 
5143   ins_cost(110);
5144   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5145   opcode(0x8D);
5146   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5147   ins_pipe(ialu_reg_reg_fat);
5148 %}
5149 
5150 instruct loadConI(rRegI dst, immI src)
5151 %{
5152   match(Set dst src);
5153 
5154   format %{ "movl    $dst, $src\t# int" %}
5155   ins_encode(load_immI(dst, src));
5156   ins_pipe(ialu_reg_fat); // XXX
5157 %}
5158 
5159 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5160 %{
5161   match(Set dst src);
5162   effect(KILL cr);
5163 
5164   ins_cost(50);
5165   format %{ "xorl    $dst, $dst\t# int" %}
5166   opcode(0x33); /* + rd */
5167   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5168   ins_pipe(ialu_reg);
5169 %}
5170 
5171 instruct loadConL(rRegL dst, immL src)
5172 %{
5173   match(Set dst src);
5174 
5175   ins_cost(150);
5176   format %{ "movq    $dst, $src\t# long" %}
5177   ins_encode(load_immL(dst, src));
5178   ins_pipe(ialu_reg);
5179 %}
5180 
5181 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5182 %{
5183   match(Set dst src);
5184   effect(KILL cr);
5185 
5186   ins_cost(50);
5187   format %{ "xorl    $dst, $dst\t# long" %}
5188   opcode(0x33); /* + rd */
5189   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5190   ins_pipe(ialu_reg); // XXX
5191 %}
5192 
5193 instruct loadConUL32(rRegL dst, immUL32 src)
5194 %{
5195   match(Set dst src);
5196 
5197   ins_cost(60);
5198   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5199   ins_encode(load_immUL32(dst, src));
5200   ins_pipe(ialu_reg);
5201 %}
5202 
5203 instruct loadConL32(rRegL dst, immL32 src)
5204 %{
5205   match(Set dst src);
5206 
5207   ins_cost(70);
5208   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5209   ins_encode(load_immL32(dst, src));
5210   ins_pipe(ialu_reg);
5211 %}
5212 
5213 instruct loadConP(rRegP dst, immP con) %{
5214   match(Set dst con);
5215 
5216   format %{ "movq    $dst, $con\t# ptr" %}
5217   ins_encode(load_immP(dst, con));
5218   ins_pipe(ialu_reg_fat); // XXX
5219 %}
5220 
5221 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5222 %{
5223   match(Set dst src);
5224   effect(KILL cr);
5225 
5226   ins_cost(50);
5227   format %{ "xorl    $dst, $dst\t# ptr" %}
5228   opcode(0x33); /* + rd */
5229   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5230   ins_pipe(ialu_reg);
5231 %}
5232 
5233 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5234 %{
5235   match(Set dst src);
5236   effect(KILL cr);
5237 
5238   ins_cost(60);
5239   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5240   ins_encode(load_immP31(dst, src));
5241   ins_pipe(ialu_reg);
5242 %}
5243 
5244 instruct loadConF(regF dst, immF con) %{
5245   match(Set dst con);
5246   ins_cost(125);
5247   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5248   ins_encode %{
5249     __ movflt($dst$$XMMRegister, $constantaddress($con));
5250   %}
5251   ins_pipe(pipe_slow);
5252 %}
5253 
5254 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5255   match(Set dst src);
5256   effect(KILL cr);
5257   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5258   ins_encode %{
5259     __ xorq($dst$$Register, $dst$$Register);
5260   %}
5261   ins_pipe(ialu_reg);
5262 %}
5263 
5264 instruct loadConN(rRegN dst, immN src) %{
5265   match(Set dst src);
5266 
5267   ins_cost(125);
5268   format %{ "movl    $dst, $src\t# compressed ptr" %}
5269   ins_encode %{
5270     address con = (address)$src$$constant;
5271     if (con == NULL) {
5272       ShouldNotReachHere();
5273     } else {
5274       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5275     }
5276   %}
5277   ins_pipe(ialu_reg_fat); // XXX
5278 %}
5279 
5280 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5281   match(Set dst src);
5282 
5283   ins_cost(125);
5284   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5285   ins_encode %{
5286     address con = (address)$src$$constant;
5287     if (con == NULL) {
5288       ShouldNotReachHere();
5289     } else {
5290       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5291     }
5292   %}
5293   ins_pipe(ialu_reg_fat); // XXX
5294 %}
5295 
5296 instruct loadConF0(regF dst, immF0 src)
5297 %{
5298   match(Set dst src);
5299   ins_cost(100);
5300 
5301   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5302   ins_encode %{
5303     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5304   %}
5305   ins_pipe(pipe_slow);
5306 %}
5307 
5308 // Use the same format since predicate() can not be used here.
5309 instruct loadConD(regD dst, immD con) %{
5310   match(Set dst con);
5311   ins_cost(125);
5312   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5313   ins_encode %{
5314     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5315   %}
5316   ins_pipe(pipe_slow);
5317 %}
5318 
5319 instruct loadConD0(regD dst, immD0 src)
5320 %{
5321   match(Set dst src);
5322   ins_cost(100);
5323 
5324   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5325   ins_encode %{
5326     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5327   %}
5328   ins_pipe(pipe_slow);
5329 %}
5330 
5331 instruct loadSSI(rRegI dst, stackSlotI src)
5332 %{
5333   match(Set dst src);
5334 
5335   ins_cost(125);
5336   format %{ "movl    $dst, $src\t# int stk" %}
5337   opcode(0x8B);
5338   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5339   ins_pipe(ialu_reg_mem);
5340 %}
5341 
5342 instruct loadSSL(rRegL dst, stackSlotL src)
5343 %{
5344   match(Set dst src);
5345 
5346   ins_cost(125);
5347   format %{ "movq    $dst, $src\t# long stk" %}
5348   opcode(0x8B);
5349   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5350   ins_pipe(ialu_reg_mem);
5351 %}
5352 
5353 instruct loadSSP(rRegP dst, stackSlotP src)
5354 %{
5355   match(Set dst src);
5356 
5357   ins_cost(125);
5358   format %{ "movq    $dst, $src\t# ptr stk" %}
5359   opcode(0x8B);
5360   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5361   ins_pipe(ialu_reg_mem);
5362 %}
5363 
5364 instruct loadSSF(regF dst, stackSlotF src)
5365 %{
5366   match(Set dst src);
5367 
5368   ins_cost(125);
5369   format %{ "movss   $dst, $src\t# float stk" %}
5370   ins_encode %{
5371     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5372   %}
5373   ins_pipe(pipe_slow); // XXX
5374 %}
5375 
5376 // Use the same format since predicate() can not be used here.
5377 instruct loadSSD(regD dst, stackSlotD src)
5378 %{
5379   match(Set dst src);
5380 
5381   ins_cost(125);
5382   format %{ "movsd   $dst, $src\t# double stk" %}
5383   ins_encode  %{
5384     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5385   %}
5386   ins_pipe(pipe_slow); // XXX
5387 %}
5388 
5389 // Prefetch instructions.
5390 // Must be safe to execute with invalid address (cannot fault).
5391 
5392 instruct prefetchr( memory mem ) %{
5393   predicate(ReadPrefetchInstr==3);
5394   match(PrefetchRead mem);
5395   ins_cost(125);
5396 
5397   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5398   ins_encode %{
5399     __ prefetchr($mem$$Address);
5400   %}
5401   ins_pipe(ialu_mem);
5402 %}
5403 
5404 instruct prefetchrNTA( memory mem ) %{
5405   predicate(ReadPrefetchInstr==0);
5406   match(PrefetchRead mem);
5407   ins_cost(125);
5408 
5409   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5410   ins_encode %{
5411     __ prefetchnta($mem$$Address);
5412   %}
5413   ins_pipe(ialu_mem);
5414 %}
5415 
5416 instruct prefetchrT0( memory mem ) %{
5417   predicate(ReadPrefetchInstr==1);
5418   match(PrefetchRead mem);
5419   ins_cost(125);
5420 
5421   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5422   ins_encode %{
5423     __ prefetcht0($mem$$Address);
5424   %}
5425   ins_pipe(ialu_mem);
5426 %}
5427 
5428 instruct prefetchrT2( memory mem ) %{
5429   predicate(ReadPrefetchInstr==2);
5430   match(PrefetchRead mem);
5431   ins_cost(125);
5432 
5433   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5434   ins_encode %{
5435     __ prefetcht2($mem$$Address);
5436   %}
5437   ins_pipe(ialu_mem);
5438 %}
5439 
5440 instruct prefetchwNTA( memory mem ) %{
5441   match(PrefetchWrite mem);
5442   ins_cost(125);
5443 
5444   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5445   ins_encode %{
5446     __ prefetchnta($mem$$Address);
5447   %}
5448   ins_pipe(ialu_mem);
5449 %}
5450 
5451 // Prefetch instructions for allocation.
5452 
5453 instruct prefetchAlloc( memory mem ) %{
5454   predicate(AllocatePrefetchInstr==3);
5455   match(PrefetchAllocation mem);
5456   ins_cost(125);
5457 
5458   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5459   ins_encode %{
5460     __ prefetchw($mem$$Address);
5461   %}
5462   ins_pipe(ialu_mem);
5463 %}
5464 
5465 instruct prefetchAllocNTA( memory mem ) %{
5466   predicate(AllocatePrefetchInstr==0);
5467   match(PrefetchAllocation mem);
5468   ins_cost(125);
5469 
5470   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5471   ins_encode %{
5472     __ prefetchnta($mem$$Address);
5473   %}
5474   ins_pipe(ialu_mem);
5475 %}
5476 
5477 instruct prefetchAllocT0( memory mem ) %{
5478   predicate(AllocatePrefetchInstr==1);
5479   match(PrefetchAllocation mem);
5480   ins_cost(125);
5481 
5482   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5483   ins_encode %{
5484     __ prefetcht0($mem$$Address);
5485   %}
5486   ins_pipe(ialu_mem);
5487 %}
5488 
5489 instruct prefetchAllocT2( memory mem ) %{
5490   predicate(AllocatePrefetchInstr==2);
5491   match(PrefetchAllocation mem);
5492   ins_cost(125);
5493 
5494   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5495   ins_encode %{
5496     __ prefetcht2($mem$$Address);
5497   %}
5498   ins_pipe(ialu_mem);
5499 %}
5500 
5501 //----------Store Instructions-------------------------------------------------
5502 
5503 // Store Byte
5504 instruct storeB(memory mem, rRegI src)
5505 %{
5506   match(Set mem (StoreB mem src));
5507 
5508   ins_cost(125); // XXX
5509   format %{ "movb    $mem, $src\t# byte" %}
5510   opcode(0x88);
5511   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5512   ins_pipe(ialu_mem_reg);
5513 %}
5514 
5515 // Store Char/Short
5516 instruct storeC(memory mem, rRegI src)
5517 %{
5518   match(Set mem (StoreC mem src));
5519 
5520   ins_cost(125); // XXX
5521   format %{ "movw    $mem, $src\t# char/short" %}
5522   opcode(0x89);
5523   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5524   ins_pipe(ialu_mem_reg);
5525 %}
5526 
5527 // Store Integer
5528 instruct storeI(memory mem, rRegI src)
5529 %{
5530   match(Set mem (StoreI mem src));
5531 
5532   ins_cost(125); // XXX
5533   format %{ "movl    $mem, $src\t# int" %}
5534   opcode(0x89);
5535   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5536   ins_pipe(ialu_mem_reg);
5537 %}
5538 
5539 // Store Long
5540 instruct storeL(memory mem, rRegL src)
5541 %{
5542   match(Set mem (StoreL mem src));
5543 
5544   ins_cost(125); // XXX
5545   format %{ "movq    $mem, $src\t# long" %}
5546   opcode(0x89);
5547   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5548   ins_pipe(ialu_mem_reg); // XXX
5549 %}
5550 
5551 // Store Pointer
5552 instruct storeP(memory mem, any_RegP src)
5553 %{
5554   match(Set mem (StoreP mem src));
5555 
5556   ins_cost(125); // XXX
5557   format %{ "movq    $mem, $src\t# ptr" %}
5558   opcode(0x89);
5559   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5560   ins_pipe(ialu_mem_reg);
5561 %}
5562 
5563 instruct storeImmP0(memory mem, immP0 zero)
5564 %{
5565   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5566   match(Set mem (StoreP mem zero));
5567 
5568   ins_cost(125); // XXX
5569   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5570   ins_encode %{
5571     __ movq($mem$$Address, r12);
5572   %}
5573   ins_pipe(ialu_mem_reg);
5574 %}
5575 
5576 // Store NULL Pointer, mark word, or other simple pointer constant.
5577 instruct storeImmP(memory mem, immP31 src)
5578 %{
5579   match(Set mem (StoreP mem src));
5580 
5581   ins_cost(150); // XXX
5582   format %{ "movq    $mem, $src\t# ptr" %}
5583   opcode(0xC7); /* C7 /0 */
5584   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5585   ins_pipe(ialu_mem_imm);
5586 %}
5587 
5588 // Store Compressed Pointer
5589 instruct storeN(memory mem, rRegN src)
5590 %{
5591   match(Set mem (StoreN mem src));
5592 
5593   ins_cost(125); // XXX
5594   format %{ "movl    $mem, $src\t# compressed ptr" %}
5595   ins_encode %{
5596     __ movl($mem$$Address, $src$$Register);
5597   %}
5598   ins_pipe(ialu_mem_reg);
5599 %}
5600 
5601 instruct storeNKlass(memory mem, rRegN src)
5602 %{
5603   match(Set mem (StoreNKlass mem src));
5604 
5605   ins_cost(125); // XXX
5606   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5607   ins_encode %{
5608     __ movl($mem$$Address, $src$$Register);
5609   %}
5610   ins_pipe(ialu_mem_reg);
5611 %}
5612 
5613 instruct storeImmN0(memory mem, immN0 zero)
5614 %{
5615   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5616   match(Set mem (StoreN mem zero));
5617 
5618   ins_cost(125); // XXX
5619   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5620   ins_encode %{
5621     __ movl($mem$$Address, r12);
5622   %}
5623   ins_pipe(ialu_mem_reg);
5624 %}
5625 
5626 instruct storeImmN(memory mem, immN src)
5627 %{
5628   match(Set mem (StoreN mem src));
5629 
5630   ins_cost(150); // XXX
5631   format %{ "movl    $mem, $src\t# compressed ptr" %}
5632   ins_encode %{
5633     address con = (address)$src$$constant;
5634     if (con == NULL) {
5635       __ movl($mem$$Address, (int32_t)0);
5636     } else {
5637       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5638     }
5639   %}
5640   ins_pipe(ialu_mem_imm);
5641 %}
5642 
5643 instruct storeImmNKlass(memory mem, immNKlass src)
5644 %{
5645   match(Set mem (StoreNKlass mem src));
5646 
5647   ins_cost(150); // XXX
5648   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5649   ins_encode %{
5650     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5651   %}
5652   ins_pipe(ialu_mem_imm);
5653 %}
5654 
5655 // Store Integer Immediate
5656 instruct storeImmI0(memory mem, immI0 zero)
5657 %{
5658   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5659   match(Set mem (StoreI mem zero));
5660 
5661   ins_cost(125); // XXX
5662   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5663   ins_encode %{
5664     __ movl($mem$$Address, r12);
5665   %}
5666   ins_pipe(ialu_mem_reg);
5667 %}
5668 
5669 instruct storeImmI(memory mem, immI src)
5670 %{
5671   match(Set mem (StoreI mem src));
5672 
5673   ins_cost(150);
5674   format %{ "movl    $mem, $src\t# int" %}
5675   opcode(0xC7); /* C7 /0 */
5676   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5677   ins_pipe(ialu_mem_imm);
5678 %}
5679 
5680 // Store Long Immediate
5681 instruct storeImmL0(memory mem, immL0 zero)
5682 %{
5683   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5684   match(Set mem (StoreL mem zero));
5685 
5686   ins_cost(125); // XXX
5687   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5688   ins_encode %{
5689     __ movq($mem$$Address, r12);
5690   %}
5691   ins_pipe(ialu_mem_reg);
5692 %}
5693 
5694 instruct storeImmL(memory mem, immL32 src)
5695 %{
5696   match(Set mem (StoreL mem src));
5697 
5698   ins_cost(150);
5699   format %{ "movq    $mem, $src\t# long" %}
5700   opcode(0xC7); /* C7 /0 */
5701   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5702   ins_pipe(ialu_mem_imm);
5703 %}
5704 
5705 // Store Short/Char Immediate
5706 instruct storeImmC0(memory mem, immI0 zero)
5707 %{
5708   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5709   match(Set mem (StoreC mem zero));
5710 
5711   ins_cost(125); // XXX
5712   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5713   ins_encode %{
5714     __ movw($mem$$Address, r12);
5715   %}
5716   ins_pipe(ialu_mem_reg);
5717 %}
5718 
5719 instruct storeImmI16(memory mem, immI16 src)
5720 %{
5721   predicate(UseStoreImmI16);
5722   match(Set mem (StoreC mem src));
5723 
5724   ins_cost(150);
5725   format %{ "movw    $mem, $src\t# short/char" %}
5726   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5727   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5728   ins_pipe(ialu_mem_imm);
5729 %}
5730 
5731 // Store Byte Immediate
5732 instruct storeImmB0(memory mem, immI0 zero)
5733 %{
5734   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5735   match(Set mem (StoreB mem zero));
5736 
5737   ins_cost(125); // XXX
5738   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5739   ins_encode %{
5740     __ movb($mem$$Address, r12);
5741   %}
5742   ins_pipe(ialu_mem_reg);
5743 %}
5744 
5745 instruct storeImmB(memory mem, immI8 src)
5746 %{
5747   match(Set mem (StoreB mem src));
5748 
5749   ins_cost(150); // XXX
5750   format %{ "movb    $mem, $src\t# byte" %}
5751   opcode(0xC6); /* C6 /0 */
5752   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5753   ins_pipe(ialu_mem_imm);
5754 %}
5755 
5756 // Store CMS card-mark Immediate
5757 instruct storeImmCM0_reg(memory mem, immI0 zero)
5758 %{
5759   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5760   match(Set mem (StoreCM mem zero));
5761 
5762   ins_cost(125); // XXX
5763   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5764   ins_encode %{
5765     __ movb($mem$$Address, r12);
5766   %}
5767   ins_pipe(ialu_mem_reg);
5768 %}
5769 
5770 instruct storeImmCM0(memory mem, immI0 src)
5771 %{
5772   match(Set mem (StoreCM mem src));
5773 
5774   ins_cost(150); // XXX
5775   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5776   opcode(0xC6); /* C6 /0 */
5777   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5778   ins_pipe(ialu_mem_imm);
5779 %}
5780 
5781 // Store Float
5782 instruct storeF(memory mem, regF src)
5783 %{
5784   match(Set mem (StoreF mem src));
5785 
5786   ins_cost(95); // XXX
5787   format %{ "movss   $mem, $src\t# float" %}
5788   ins_encode %{
5789     __ movflt($mem$$Address, $src$$XMMRegister);
5790   %}
5791   ins_pipe(pipe_slow); // XXX
5792 %}
5793 
5794 // Store immediate Float value (it is faster than store from XMM register)
5795 instruct storeF0(memory mem, immF0 zero)
5796 %{
5797   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5798   match(Set mem (StoreF mem zero));
5799 
5800   ins_cost(25); // XXX
5801   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5802   ins_encode %{
5803     __ movl($mem$$Address, r12);
5804   %}
5805   ins_pipe(ialu_mem_reg);
5806 %}
5807 
5808 instruct storeF_imm(memory mem, immF src)
5809 %{
5810   match(Set mem (StoreF mem src));
5811 
5812   ins_cost(50);
5813   format %{ "movl    $mem, $src\t# float" %}
5814   opcode(0xC7); /* C7 /0 */
5815   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5816   ins_pipe(ialu_mem_imm);
5817 %}
5818 
5819 // Store Double
5820 instruct storeD(memory mem, regD src)
5821 %{
5822   match(Set mem (StoreD mem src));
5823 
5824   ins_cost(95); // XXX
5825   format %{ "movsd   $mem, $src\t# double" %}
5826   ins_encode %{
5827     __ movdbl($mem$$Address, $src$$XMMRegister);
5828   %}
5829   ins_pipe(pipe_slow); // XXX
5830 %}
5831 
5832 // Store immediate double 0.0 (it is faster than store from XMM register)
5833 instruct storeD0_imm(memory mem, immD0 src)
5834 %{
5835   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5836   match(Set mem (StoreD mem src));
5837 
5838   ins_cost(50);
5839   format %{ "movq    $mem, $src\t# double 0." %}
5840   opcode(0xC7); /* C7 /0 */
5841   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5842   ins_pipe(ialu_mem_imm);
5843 %}
5844 
5845 instruct storeD0(memory mem, immD0 zero)
5846 %{
5847   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5848   match(Set mem (StoreD mem zero));
5849 
5850   ins_cost(25); // XXX
5851   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5852   ins_encode %{
5853     __ movq($mem$$Address, r12);
5854   %}
5855   ins_pipe(ialu_mem_reg);
5856 %}
5857 
5858 instruct storeSSI(stackSlotI dst, rRegI src)
5859 %{
5860   match(Set dst src);
5861 
5862   ins_cost(100);
5863   format %{ "movl    $dst, $src\t# int stk" %}
5864   opcode(0x89);
5865   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5866   ins_pipe( ialu_mem_reg );
5867 %}
5868 
5869 instruct storeSSL(stackSlotL dst, rRegL src)
5870 %{
5871   match(Set dst src);
5872 
5873   ins_cost(100);
5874   format %{ "movq    $dst, $src\t# long stk" %}
5875   opcode(0x89);
5876   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5877   ins_pipe(ialu_mem_reg);
5878 %}
5879 
5880 instruct storeSSP(stackSlotP dst, rRegP src)
5881 %{
5882   match(Set dst src);
5883 
5884   ins_cost(100);
5885   format %{ "movq    $dst, $src\t# ptr stk" %}
5886   opcode(0x89);
5887   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5888   ins_pipe(ialu_mem_reg);
5889 %}
5890 
5891 instruct storeSSF(stackSlotF dst, regF src)
5892 %{
5893   match(Set dst src);
5894 
5895   ins_cost(95); // XXX
5896   format %{ "movss   $dst, $src\t# float stk" %}
5897   ins_encode %{
5898     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5899   %}
5900   ins_pipe(pipe_slow); // XXX
5901 %}
5902 
5903 instruct storeSSD(stackSlotD dst, regD src)
5904 %{
5905   match(Set dst src);
5906 
5907   ins_cost(95); // XXX
5908   format %{ "movsd   $dst, $src\t# double stk" %}
5909   ins_encode %{
5910     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5911   %}
5912   ins_pipe(pipe_slow); // XXX
5913 %}
5914 
5915 //----------BSWAP Instructions-------------------------------------------------
5916 instruct bytes_reverse_int(rRegI dst) %{
5917   match(Set dst (ReverseBytesI dst));
5918 
5919   format %{ "bswapl  $dst" %}
5920   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
5921   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
5922   ins_pipe( ialu_reg );
5923 %}
5924 
5925 instruct bytes_reverse_long(rRegL dst) %{
5926   match(Set dst (ReverseBytesL dst));
5927 
5928   format %{ "bswapq  $dst" %}
5929   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
5930   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
5931   ins_pipe( ialu_reg);
5932 %}
5933 
5934 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
5935   match(Set dst (ReverseBytesUS dst));
5936   effect(KILL cr);
5937 
5938   format %{ "bswapl  $dst\n\t"
5939             "shrl    $dst,16\n\t" %}
5940   ins_encode %{
5941     __ bswapl($dst$$Register);
5942     __ shrl($dst$$Register, 16);
5943   %}
5944   ins_pipe( ialu_reg );
5945 %}
5946 
5947 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
5948   match(Set dst (ReverseBytesS dst));
5949   effect(KILL cr);
5950 
5951   format %{ "bswapl  $dst\n\t"
5952             "sar     $dst,16\n\t" %}
5953   ins_encode %{
5954     __ bswapl($dst$$Register);
5955     __ sarl($dst$$Register, 16);
5956   %}
5957   ins_pipe( ialu_reg );
5958 %}
5959 
5960 //---------- Zeros Count Instructions ------------------------------------------
5961 
5962 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5963   predicate(UseCountLeadingZerosInstruction);
5964   match(Set dst (CountLeadingZerosI src));
5965   effect(KILL cr);
5966 
5967   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
5968   ins_encode %{
5969     __ lzcntl($dst$$Register, $src$$Register);
5970   %}
5971   ins_pipe(ialu_reg);
5972 %}
5973 
5974 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
5975   predicate(!UseCountLeadingZerosInstruction);
5976   match(Set dst (CountLeadingZerosI src));
5977   effect(KILL cr);
5978 
5979   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
5980             "jnz     skip\n\t"
5981             "movl    $dst, -1\n"
5982       "skip:\n\t"
5983             "negl    $dst\n\t"
5984             "addl    $dst, 31" %}
5985   ins_encode %{
5986     Register Rdst = $dst$$Register;
5987     Register Rsrc = $src$$Register;
5988     Label skip;
5989     __ bsrl(Rdst, Rsrc);
5990     __ jccb(Assembler::notZero, skip);
5991     __ movl(Rdst, -1);
5992     __ bind(skip);
5993     __ negl(Rdst);
5994     __ addl(Rdst, BitsPerInt - 1);
5995   %}
5996   ins_pipe(ialu_reg);
5997 %}
5998 
5999 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6000   predicate(UseCountLeadingZerosInstruction);
6001   match(Set dst (CountLeadingZerosL src));
6002   effect(KILL cr);
6003 
6004   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6005   ins_encode %{
6006     __ lzcntq($dst$$Register, $src$$Register);
6007   %}
6008   ins_pipe(ialu_reg);
6009 %}
6010 
6011 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6012   predicate(!UseCountLeadingZerosInstruction);
6013   match(Set dst (CountLeadingZerosL src));
6014   effect(KILL cr);
6015 
6016   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6017             "jnz     skip\n\t"
6018             "movl    $dst, -1\n"
6019       "skip:\n\t"
6020             "negl    $dst\n\t"
6021             "addl    $dst, 63" %}
6022   ins_encode %{
6023     Register Rdst = $dst$$Register;
6024     Register Rsrc = $src$$Register;
6025     Label skip;
6026     __ bsrq(Rdst, Rsrc);
6027     __ jccb(Assembler::notZero, skip);
6028     __ movl(Rdst, -1);
6029     __ bind(skip);
6030     __ negl(Rdst);
6031     __ addl(Rdst, BitsPerLong - 1);
6032   %}
6033   ins_pipe(ialu_reg);
6034 %}
6035 
6036 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6037   match(Set dst (CountTrailingZerosI src));
6038   effect(KILL cr);
6039 
6040   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6041             "jnz     done\n\t"
6042             "movl    $dst, 32\n"
6043       "done:" %}
6044   ins_encode %{
6045     Register Rdst = $dst$$Register;
6046     Label done;
6047     __ bsfl(Rdst, $src$$Register);
6048     __ jccb(Assembler::notZero, done);
6049     __ movl(Rdst, BitsPerInt);
6050     __ bind(done);
6051   %}
6052   ins_pipe(ialu_reg);
6053 %}
6054 
6055 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6056   match(Set dst (CountTrailingZerosL src));
6057   effect(KILL cr);
6058 
6059   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6060             "jnz     done\n\t"
6061             "movl    $dst, 64\n"
6062       "done:" %}
6063   ins_encode %{
6064     Register Rdst = $dst$$Register;
6065     Label done;
6066     __ bsfq(Rdst, $src$$Register);
6067     __ jccb(Assembler::notZero, done);
6068     __ movl(Rdst, BitsPerLong);
6069     __ bind(done);
6070   %}
6071   ins_pipe(ialu_reg);
6072 %}
6073 
6074 
6075 //---------- Population Count Instructions -------------------------------------
6076 
6077 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6078   predicate(UsePopCountInstruction);
6079   match(Set dst (PopCountI src));
6080   effect(KILL cr);
6081 
6082   format %{ "popcnt  $dst, $src" %}
6083   ins_encode %{
6084     __ popcntl($dst$$Register, $src$$Register);
6085   %}
6086   ins_pipe(ialu_reg);
6087 %}
6088 
6089 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6090   predicate(UsePopCountInstruction);
6091   match(Set dst (PopCountI (LoadI mem)));
6092   effect(KILL cr);
6093 
6094   format %{ "popcnt  $dst, $mem" %}
6095   ins_encode %{
6096     __ popcntl($dst$$Register, $mem$$Address);
6097   %}
6098   ins_pipe(ialu_reg);
6099 %}
6100 
6101 // Note: Long.bitCount(long) returns an int.
6102 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6103   predicate(UsePopCountInstruction);
6104   match(Set dst (PopCountL src));
6105   effect(KILL cr);
6106 
6107   format %{ "popcnt  $dst, $src" %}
6108   ins_encode %{
6109     __ popcntq($dst$$Register, $src$$Register);
6110   %}
6111   ins_pipe(ialu_reg);
6112 %}
6113 
6114 // Note: Long.bitCount(long) returns an int.
6115 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6116   predicate(UsePopCountInstruction);
6117   match(Set dst (PopCountL (LoadL mem)));
6118   effect(KILL cr);
6119 
6120   format %{ "popcnt  $dst, $mem" %}
6121   ins_encode %{
6122     __ popcntq($dst$$Register, $mem$$Address);
6123   %}
6124   ins_pipe(ialu_reg);
6125 %}
6126 
6127 
6128 //----------MemBar Instructions-----------------------------------------------
6129 // Memory barrier flavors
6130 
6131 instruct membar_acquire()
6132 %{
6133   match(MemBarAcquire);
6134   match(LoadFence);
6135   ins_cost(0);
6136 
6137   size(0);
6138   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6139   ins_encode();
6140   ins_pipe(empty);
6141 %}
6142 
6143 instruct membar_acquire_lock()
6144 %{
6145   match(MemBarAcquireLock);
6146   ins_cost(0);
6147 
6148   size(0);
6149   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6150   ins_encode();
6151   ins_pipe(empty);
6152 %}
6153 
6154 instruct membar_release()
6155 %{
6156   match(MemBarRelease);
6157   match(StoreFence);
6158   ins_cost(0);
6159 
6160   size(0);
6161   format %{ "MEMBAR-release ! (empty encoding)" %}
6162   ins_encode();
6163   ins_pipe(empty);
6164 %}
6165 
6166 instruct membar_release_lock()
6167 %{
6168   match(MemBarReleaseLock);
6169   ins_cost(0);
6170 
6171   size(0);
6172   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6173   ins_encode();
6174   ins_pipe(empty);
6175 %}
6176 
6177 instruct membar_volatile(rFlagsReg cr) %{
6178   match(MemBarVolatile);
6179   effect(KILL cr);
6180   ins_cost(400);
6181 
6182   format %{
6183     $$template
6184     if (os::is_MP()) {
6185       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6186     } else {
6187       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6188     }
6189   %}
6190   ins_encode %{
6191     __ membar(Assembler::StoreLoad);
6192   %}
6193   ins_pipe(pipe_slow);
6194 %}
6195 
6196 instruct unnecessary_membar_volatile()
6197 %{
6198   match(MemBarVolatile);
6199   predicate(Matcher::post_store_load_barrier(n));
6200   ins_cost(0);
6201 
6202   size(0);
6203   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6204   ins_encode();
6205   ins_pipe(empty);
6206 %}
6207 
6208 instruct membar_storestore() %{
6209   match(MemBarStoreStore);
6210   ins_cost(0);
6211 
6212   size(0);
6213   format %{ "MEMBAR-storestore (empty encoding)" %}
6214   ins_encode( );
6215   ins_pipe(empty);
6216 %}
6217 
6218 //----------Move Instructions--------------------------------------------------
6219 
6220 instruct castX2P(rRegP dst, rRegL src)
6221 %{
6222   match(Set dst (CastX2P src));
6223 
6224   format %{ "movq    $dst, $src\t# long->ptr" %}
6225   ins_encode %{
6226     if ($dst$$reg != $src$$reg) {
6227       __ movptr($dst$$Register, $src$$Register);
6228     }
6229   %}
6230   ins_pipe(ialu_reg_reg); // XXX
6231 %}
6232 
6233 instruct castP2X(rRegL dst, rRegP src)
6234 %{
6235   match(Set dst (CastP2X src));
6236 
6237   format %{ "movq    $dst, $src\t# ptr -> long" %}
6238   ins_encode %{
6239     if ($dst$$reg != $src$$reg) {
6240       __ movptr($dst$$Register, $src$$Register);
6241     }
6242   %}
6243   ins_pipe(ialu_reg_reg); // XXX
6244 %}
6245 
6246 // Convert oop into int for vectors alignment masking
6247 instruct convP2I(rRegI dst, rRegP src)
6248 %{
6249   match(Set dst (ConvL2I (CastP2X src)));
6250 
6251   format %{ "movl    $dst, $src\t# ptr -> int" %}
6252   ins_encode %{
6253     __ movl($dst$$Register, $src$$Register);
6254   %}
6255   ins_pipe(ialu_reg_reg); // XXX
6256 %}
6257 
6258 // Convert compressed oop into int for vectors alignment masking
6259 // in case of 32bit oops (heap < 4Gb).
6260 instruct convN2I(rRegI dst, rRegN src)
6261 %{
6262   predicate(Universe::narrow_oop_shift() == 0);
6263   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6264 
6265   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6266   ins_encode %{
6267     __ movl($dst$$Register, $src$$Register);
6268   %}
6269   ins_pipe(ialu_reg_reg); // XXX
6270 %}
6271 
6272 // Convert oop pointer into compressed form
6273 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6274   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6275   match(Set dst (EncodeP src));
6276   effect(KILL cr);
6277   format %{ "encode_heap_oop $dst,$src" %}
6278   ins_encode %{
6279     Register s = $src$$Register;
6280     Register d = $dst$$Register;
6281     if (s != d) {
6282       __ movq(d, s);
6283     }
6284     __ encode_heap_oop(d);
6285   %}
6286   ins_pipe(ialu_reg_long);
6287 %}
6288 
6289 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6290   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6291   match(Set dst (EncodeP src));
6292   effect(KILL cr);
6293   format %{ "encode_heap_oop_not_null $dst,$src" %}
6294   ins_encode %{
6295     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6296   %}
6297   ins_pipe(ialu_reg_long);
6298 %}
6299 
6300 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6301   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6302             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6303   match(Set dst (DecodeN src));
6304   effect(KILL cr);
6305   format %{ "decode_heap_oop $dst,$src" %}
6306   ins_encode %{
6307     Register s = $src$$Register;
6308     Register d = $dst$$Register;
6309     if (s != d) {
6310       __ movq(d, s);
6311     }
6312     __ decode_heap_oop(d);
6313   %}
6314   ins_pipe(ialu_reg_long);
6315 %}
6316 
6317 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6318   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6319             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6320   match(Set dst (DecodeN src));
6321   effect(KILL cr);
6322   format %{ "decode_heap_oop_not_null $dst,$src" %}
6323   ins_encode %{
6324     Register s = $src$$Register;
6325     Register d = $dst$$Register;
6326     if (s != d) {
6327       __ decode_heap_oop_not_null(d, s);
6328     } else {
6329       __ decode_heap_oop_not_null(d);
6330     }
6331   %}
6332   ins_pipe(ialu_reg_long);
6333 %}
6334 
6335 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6336   match(Set dst (EncodePKlass src));
6337   effect(KILL cr);
6338   format %{ "encode_klass_not_null $dst,$src" %}
6339   ins_encode %{
6340     __ encode_klass_not_null($dst$$Register, $src$$Register);
6341   %}
6342   ins_pipe(ialu_reg_long);
6343 %}
6344 
6345 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6346   match(Set dst (DecodeNKlass src));
6347   effect(KILL cr);
6348   format %{ "decode_klass_not_null $dst,$src" %}
6349   ins_encode %{
6350     Register s = $src$$Register;
6351     Register d = $dst$$Register;
6352     if (s != d) {
6353       __ decode_klass_not_null(d, s);
6354     } else {
6355       __ decode_klass_not_null(d);
6356     }
6357   %}
6358   ins_pipe(ialu_reg_long);
6359 %}
6360 
6361 
6362 //----------Conditional Move---------------------------------------------------
6363 // Jump
6364 // dummy instruction for generating temp registers
6365 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6366   match(Jump (LShiftL switch_val shift));
6367   ins_cost(350);
6368   predicate(false);
6369   effect(TEMP dest);
6370 
6371   format %{ "leaq    $dest, [$constantaddress]\n\t"
6372             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6373   ins_encode %{
6374     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6375     // to do that and the compiler is using that register as one it can allocate.
6376     // So we build it all by hand.
6377     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6378     // ArrayAddress dispatch(table, index);
6379     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6380     __ lea($dest$$Register, $constantaddress);
6381     __ jmp(dispatch);
6382   %}
6383   ins_pipe(pipe_jmp);
6384 %}
6385 
6386 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6387   match(Jump (AddL (LShiftL switch_val shift) offset));
6388   ins_cost(350);
6389   effect(TEMP dest);
6390 
6391   format %{ "leaq    $dest, [$constantaddress]\n\t"
6392             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6393   ins_encode %{
6394     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6395     // to do that and the compiler is using that register as one it can allocate.
6396     // So we build it all by hand.
6397     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6398     // ArrayAddress dispatch(table, index);
6399     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6400     __ lea($dest$$Register, $constantaddress);
6401     __ jmp(dispatch);
6402   %}
6403   ins_pipe(pipe_jmp);
6404 %}
6405 
6406 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6407   match(Jump switch_val);
6408   ins_cost(350);
6409   effect(TEMP dest);
6410 
6411   format %{ "leaq    $dest, [$constantaddress]\n\t"
6412             "jmp     [$dest + $switch_val]\n\t" %}
6413   ins_encode %{
6414     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6415     // to do that and the compiler is using that register as one it can allocate.
6416     // So we build it all by hand.
6417     // Address index(noreg, switch_reg, Address::times_1);
6418     // ArrayAddress dispatch(table, index);
6419     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6420     __ lea($dest$$Register, $constantaddress);
6421     __ jmp(dispatch);
6422   %}
6423   ins_pipe(pipe_jmp);
6424 %}
6425 
6426 // Conditional move
6427 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6428 %{
6429   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6430 
6431   ins_cost(200); // XXX
6432   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6433   opcode(0x0F, 0x40);
6434   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6435   ins_pipe(pipe_cmov_reg);
6436 %}
6437 
6438 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6439   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6440 
6441   ins_cost(200); // XXX
6442   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6443   opcode(0x0F, 0x40);
6444   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6445   ins_pipe(pipe_cmov_reg);
6446 %}
6447 
6448 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6449   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6450   ins_cost(200);
6451   expand %{
6452     cmovI_regU(cop, cr, dst, src);
6453   %}
6454 %}
6455 
6456 // Conditional move
6457 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6458   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6459 
6460   ins_cost(250); // XXX
6461   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6462   opcode(0x0F, 0x40);
6463   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6464   ins_pipe(pipe_cmov_mem);
6465 %}
6466 
6467 // Conditional move
6468 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6469 %{
6470   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6471 
6472   ins_cost(250); // XXX
6473   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6474   opcode(0x0F, 0x40);
6475   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6476   ins_pipe(pipe_cmov_mem);
6477 %}
6478 
6479 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6480   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6481   ins_cost(250);
6482   expand %{
6483     cmovI_memU(cop, cr, dst, src);
6484   %}
6485 %}
6486 
6487 // Conditional move
6488 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6489 %{
6490   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6491 
6492   ins_cost(200); // XXX
6493   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6494   opcode(0x0F, 0x40);
6495   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6496   ins_pipe(pipe_cmov_reg);
6497 %}
6498 
6499 // Conditional move
6500 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6501 %{
6502   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6503 
6504   ins_cost(200); // XXX
6505   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6506   opcode(0x0F, 0x40);
6507   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6508   ins_pipe(pipe_cmov_reg);
6509 %}
6510 
6511 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6512   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6513   ins_cost(200);
6514   expand %{
6515     cmovN_regU(cop, cr, dst, src);
6516   %}
6517 %}
6518 
6519 // Conditional move
6520 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6521 %{
6522   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6523 
6524   ins_cost(200); // XXX
6525   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6526   opcode(0x0F, 0x40);
6527   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6528   ins_pipe(pipe_cmov_reg);  // XXX
6529 %}
6530 
6531 // Conditional move
6532 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6533 %{
6534   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6535 
6536   ins_cost(200); // XXX
6537   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6538   opcode(0x0F, 0x40);
6539   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6540   ins_pipe(pipe_cmov_reg); // XXX
6541 %}
6542 
6543 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6544   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6545   ins_cost(200);
6546   expand %{
6547     cmovP_regU(cop, cr, dst, src);
6548   %}
6549 %}
6550 
6551 // DISABLED: Requires the ADLC to emit a bottom_type call that
6552 // correctly meets the two pointer arguments; one is an incoming
6553 // register but the other is a memory operand.  ALSO appears to
6554 // be buggy with implicit null checks.
6555 //
6556 //// Conditional move
6557 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6558 //%{
6559 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6560 //  ins_cost(250);
6561 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6562 //  opcode(0x0F,0x40);
6563 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6564 //  ins_pipe( pipe_cmov_mem );
6565 //%}
6566 //
6567 //// Conditional move
6568 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6569 //%{
6570 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6571 //  ins_cost(250);
6572 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6573 //  opcode(0x0F,0x40);
6574 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6575 //  ins_pipe( pipe_cmov_mem );
6576 //%}
6577 
6578 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6579 %{
6580   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6581 
6582   ins_cost(200); // XXX
6583   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6584   opcode(0x0F, 0x40);
6585   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6586   ins_pipe(pipe_cmov_reg);  // XXX
6587 %}
6588 
6589 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6590 %{
6591   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6592 
6593   ins_cost(200); // XXX
6594   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6595   opcode(0x0F, 0x40);
6596   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6597   ins_pipe(pipe_cmov_mem);  // XXX
6598 %}
6599 
6600 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6601 %{
6602   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6603 
6604   ins_cost(200); // XXX
6605   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6606   opcode(0x0F, 0x40);
6607   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6608   ins_pipe(pipe_cmov_reg); // XXX
6609 %}
6610 
6611 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6612   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6613   ins_cost(200);
6614   expand %{
6615     cmovL_regU(cop, cr, dst, src);
6616   %}
6617 %}
6618 
6619 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6620 %{
6621   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6622 
6623   ins_cost(200); // XXX
6624   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6625   opcode(0x0F, 0x40);
6626   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6627   ins_pipe(pipe_cmov_mem); // XXX
6628 %}
6629 
6630 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6631   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6632   ins_cost(200);
6633   expand %{
6634     cmovL_memU(cop, cr, dst, src);
6635   %}
6636 %}
6637 
6638 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6639 %{
6640   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6641 
6642   ins_cost(200); // XXX
6643   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6644             "movss     $dst, $src\n"
6645     "skip:" %}
6646   ins_encode %{
6647     Label Lskip;
6648     // Invert sense of branch from sense of CMOV
6649     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6650     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6651     __ bind(Lskip);
6652   %}
6653   ins_pipe(pipe_slow);
6654 %}
6655 
6656 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6657 // %{
6658 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6659 
6660 //   ins_cost(200); // XXX
6661 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6662 //             "movss     $dst, $src\n"
6663 //     "skip:" %}
6664 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6665 //   ins_pipe(pipe_slow);
6666 // %}
6667 
6668 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6669 %{
6670   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6671 
6672   ins_cost(200); // XXX
6673   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6674             "movss     $dst, $src\n"
6675     "skip:" %}
6676   ins_encode %{
6677     Label Lskip;
6678     // Invert sense of branch from sense of CMOV
6679     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6680     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6681     __ bind(Lskip);
6682   %}
6683   ins_pipe(pipe_slow);
6684 %}
6685 
6686 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6687   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6688   ins_cost(200);
6689   expand %{
6690     cmovF_regU(cop, cr, dst, src);
6691   %}
6692 %}
6693 
6694 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6695 %{
6696   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6697 
6698   ins_cost(200); // XXX
6699   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6700             "movsd     $dst, $src\n"
6701     "skip:" %}
6702   ins_encode %{
6703     Label Lskip;
6704     // Invert sense of branch from sense of CMOV
6705     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6706     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6707     __ bind(Lskip);
6708   %}
6709   ins_pipe(pipe_slow);
6710 %}
6711 
6712 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6713 %{
6714   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6715 
6716   ins_cost(200); // XXX
6717   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6718             "movsd     $dst, $src\n"
6719     "skip:" %}
6720   ins_encode %{
6721     Label Lskip;
6722     // Invert sense of branch from sense of CMOV
6723     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6724     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6725     __ bind(Lskip);
6726   %}
6727   ins_pipe(pipe_slow);
6728 %}
6729 
6730 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6731   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6732   ins_cost(200);
6733   expand %{
6734     cmovD_regU(cop, cr, dst, src);
6735   %}
6736 %}
6737 
6738 //----------Arithmetic Instructions--------------------------------------------
6739 //----------Addition Instructions----------------------------------------------
6740 
6741 instruct addExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
6742 %{
6743   match(AddExactI dst src);
6744   effect(DEF cr);
6745 
6746   format %{ "addl    $dst, $src\t# addExact int" %}
6747   ins_encode %{
6748     __ addl($dst$$Register, $src$$Register);
6749   %}
6750   ins_pipe(ialu_reg_reg);
6751 %}
6752 
6753 instruct addExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr)
6754 %{
6755   match(AddExactI dst src);
6756   effect(DEF cr);
6757 
6758   format %{ "addl    $dst, $src\t# addExact int" %}
6759   ins_encode %{
6760     __ addl($dst$$Register, $src$$constant);
6761   %}
6762   ins_pipe(ialu_reg_reg);
6763 %}
6764 
6765 instruct addExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
6766 %{
6767   match(AddExactI dst (LoadI src));
6768   effect(DEF cr);
6769 
6770   ins_cost(125); // XXX
6771   format %{ "addl    $dst, $src\t# addExact int" %}
6772   ins_encode %{
6773     __ addl($dst$$Register, $src$$Address);
6774   %}
6775 
6776   ins_pipe(ialu_reg_mem);
6777 %}
6778 
6779 instruct addExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
6780 %{
6781   match(AddExactL dst src);
6782   effect(DEF cr);
6783 
6784   format %{ "addq    $dst, $src\t# addExact long" %}
6785   ins_encode %{
6786     __ addq($dst$$Register, $src$$Register);
6787   %}
6788   ins_pipe(ialu_reg_reg);
6789 %}
6790 
6791 instruct addExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr)
6792 %{
6793   match(AddExactL dst src);
6794   effect(DEF cr);
6795 
6796   format %{ "addq    $dst, $src\t# addExact long" %}
6797   ins_encode %{
6798     __ addq($dst$$Register, $src$$constant);
6799   %}
6800   ins_pipe(ialu_reg_reg);
6801 %}
6802 
6803 instruct addExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr)
6804 %{
6805   match(AddExactL dst (LoadL src));
6806   effect(DEF cr);
6807 
6808   ins_cost(125); // XXX
6809   format %{ "addq    $dst, $src\t# addExact long" %}
6810   ins_encode %{
6811     __ addq($dst$$Register, $src$$Address);
6812   %}
6813 
6814   ins_pipe(ialu_reg_mem);
6815 %}
6816 
6817 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6818 %{
6819   match(Set dst (AddI dst src));
6820   effect(KILL cr);
6821 
6822   format %{ "addl    $dst, $src\t# int" %}
6823   opcode(0x03);
6824   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6825   ins_pipe(ialu_reg_reg);
6826 %}
6827 
6828 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6829 %{
6830   match(Set dst (AddI dst src));
6831   effect(KILL cr);
6832 
6833   format %{ "addl    $dst, $src\t# int" %}
6834   opcode(0x81, 0x00); /* /0 id */
6835   ins_encode(OpcSErm(dst, src), Con8or32(src));
6836   ins_pipe( ialu_reg );
6837 %}
6838 
6839 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6840 %{
6841   match(Set dst (AddI dst (LoadI src)));
6842   effect(KILL cr);
6843 
6844   ins_cost(125); // XXX
6845   format %{ "addl    $dst, $src\t# int" %}
6846   opcode(0x03);
6847   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6848   ins_pipe(ialu_reg_mem);
6849 %}
6850 
6851 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6852 %{
6853   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6854   effect(KILL cr);
6855 
6856   ins_cost(150); // XXX
6857   format %{ "addl    $dst, $src\t# int" %}
6858   opcode(0x01); /* Opcode 01 /r */
6859   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6860   ins_pipe(ialu_mem_reg);
6861 %}
6862 
6863 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6864 %{
6865   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6866   effect(KILL cr);
6867 
6868   ins_cost(125); // XXX
6869   format %{ "addl    $dst, $src\t# int" %}
6870   opcode(0x81); /* Opcode 81 /0 id */
6871   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6872   ins_pipe(ialu_mem_imm);
6873 %}
6874 
6875 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6876 %{
6877   predicate(UseIncDec);
6878   match(Set dst (AddI dst src));
6879   effect(KILL cr);
6880 
6881   format %{ "incl    $dst\t# int" %}
6882   opcode(0xFF, 0x00); // FF /0
6883   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6884   ins_pipe(ialu_reg);
6885 %}
6886 
6887 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6888 %{
6889   predicate(UseIncDec);
6890   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6891   effect(KILL cr);
6892 
6893   ins_cost(125); // XXX
6894   format %{ "incl    $dst\t# int" %}
6895   opcode(0xFF); /* Opcode FF /0 */
6896   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6897   ins_pipe(ialu_mem_imm);
6898 %}
6899 
6900 // XXX why does that use AddI
6901 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6902 %{
6903   predicate(UseIncDec);
6904   match(Set dst (AddI dst src));
6905   effect(KILL cr);
6906 
6907   format %{ "decl    $dst\t# int" %}
6908   opcode(0xFF, 0x01); // FF /1
6909   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6910   ins_pipe(ialu_reg);
6911 %}
6912 
6913 // XXX why does that use AddI
6914 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6915 %{
6916   predicate(UseIncDec);
6917   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6918   effect(KILL cr);
6919 
6920   ins_cost(125); // XXX
6921   format %{ "decl    $dst\t# int" %}
6922   opcode(0xFF); /* Opcode FF /1 */
6923   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
6924   ins_pipe(ialu_mem_imm);
6925 %}
6926 
6927 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
6928 %{
6929   match(Set dst (AddI src0 src1));
6930 
6931   ins_cost(110);
6932   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
6933   opcode(0x8D); /* 0x8D /r */
6934   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6935   ins_pipe(ialu_reg_reg);
6936 %}
6937 
6938 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6939 %{
6940   match(Set dst (AddL dst src));
6941   effect(KILL cr);
6942 
6943   format %{ "addq    $dst, $src\t# long" %}
6944   opcode(0x03);
6945   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6946   ins_pipe(ialu_reg_reg);
6947 %}
6948 
6949 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6950 %{
6951   match(Set dst (AddL dst src));
6952   effect(KILL cr);
6953 
6954   format %{ "addq    $dst, $src\t# long" %}
6955   opcode(0x81, 0x00); /* /0 id */
6956   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6957   ins_pipe( ialu_reg );
6958 %}
6959 
6960 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6961 %{
6962   match(Set dst (AddL dst (LoadL src)));
6963   effect(KILL cr);
6964 
6965   ins_cost(125); // XXX
6966   format %{ "addq    $dst, $src\t# long" %}
6967   opcode(0x03);
6968   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6969   ins_pipe(ialu_reg_mem);
6970 %}
6971 
6972 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6973 %{
6974   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6975   effect(KILL cr);
6976 
6977   ins_cost(150); // XXX
6978   format %{ "addq    $dst, $src\t# long" %}
6979   opcode(0x01); /* Opcode 01 /r */
6980   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6981   ins_pipe(ialu_mem_reg);
6982 %}
6983 
6984 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
6985 %{
6986   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6987   effect(KILL cr);
6988 
6989   ins_cost(125); // XXX
6990   format %{ "addq    $dst, $src\t# long" %}
6991   opcode(0x81); /* Opcode 81 /0 id */
6992   ins_encode(REX_mem_wide(dst),
6993              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6994   ins_pipe(ialu_mem_imm);
6995 %}
6996 
6997 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
6998 %{
6999   predicate(UseIncDec);
7000   match(Set dst (AddL dst src));
7001   effect(KILL cr);
7002 
7003   format %{ "incq    $dst\t# long" %}
7004   opcode(0xFF, 0x00); // FF /0
7005   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7006   ins_pipe(ialu_reg);
7007 %}
7008 
7009 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7010 %{
7011   predicate(UseIncDec);
7012   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7013   effect(KILL cr);
7014 
7015   ins_cost(125); // XXX
7016   format %{ "incq    $dst\t# long" %}
7017   opcode(0xFF); /* Opcode FF /0 */
7018   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7019   ins_pipe(ialu_mem_imm);
7020 %}
7021 
7022 // XXX why does that use AddL
7023 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7024 %{
7025   predicate(UseIncDec);
7026   match(Set dst (AddL dst src));
7027   effect(KILL cr);
7028 
7029   format %{ "decq    $dst\t# long" %}
7030   opcode(0xFF, 0x01); // FF /1
7031   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7032   ins_pipe(ialu_reg);
7033 %}
7034 
7035 // XXX why does that use AddL
7036 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7037 %{
7038   predicate(UseIncDec);
7039   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7040   effect(KILL cr);
7041 
7042   ins_cost(125); // XXX
7043   format %{ "decq    $dst\t# long" %}
7044   opcode(0xFF); /* Opcode FF /1 */
7045   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7046   ins_pipe(ialu_mem_imm);
7047 %}
7048 
7049 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7050 %{
7051   match(Set dst (AddL src0 src1));
7052 
7053   ins_cost(110);
7054   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7055   opcode(0x8D); /* 0x8D /r */
7056   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7057   ins_pipe(ialu_reg_reg);
7058 %}
7059 
7060 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7061 %{
7062   match(Set dst (AddP dst src));
7063   effect(KILL cr);
7064 
7065   format %{ "addq    $dst, $src\t# ptr" %}
7066   opcode(0x03);
7067   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7068   ins_pipe(ialu_reg_reg);
7069 %}
7070 
7071 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7072 %{
7073   match(Set dst (AddP dst src));
7074   effect(KILL cr);
7075 
7076   format %{ "addq    $dst, $src\t# ptr" %}
7077   opcode(0x81, 0x00); /* /0 id */
7078   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7079   ins_pipe( ialu_reg );
7080 %}
7081 
7082 // XXX addP mem ops ????
7083 
7084 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7085 %{
7086   match(Set dst (AddP src0 src1));
7087 
7088   ins_cost(110);
7089   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7090   opcode(0x8D); /* 0x8D /r */
7091   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7092   ins_pipe(ialu_reg_reg);
7093 %}
7094 
7095 instruct checkCastPP(rRegP dst)
7096 %{
7097   match(Set dst (CheckCastPP dst));
7098 
7099   size(0);
7100   format %{ "# checkcastPP of $dst" %}
7101   ins_encode(/* empty encoding */);
7102   ins_pipe(empty);
7103 %}
7104 
7105 instruct castPP(rRegP dst)
7106 %{
7107   match(Set dst (CastPP dst));
7108 
7109   size(0);
7110   format %{ "# castPP of $dst" %}
7111   ins_encode(/* empty encoding */);
7112   ins_pipe(empty);
7113 %}
7114 
7115 instruct castII(rRegI dst)
7116 %{
7117   match(Set dst (CastII dst));
7118 
7119   size(0);
7120   format %{ "# castII of $dst" %}
7121   ins_encode(/* empty encoding */);
7122   ins_cost(0);
7123   ins_pipe(empty);
7124 %}
7125 
7126 // LoadP-locked same as a regular LoadP when used with compare-swap
7127 instruct loadPLocked(rRegP dst, memory mem)
7128 %{
7129   match(Set dst (LoadPLocked mem));
7130 
7131   ins_cost(125); // XXX
7132   format %{ "movq    $dst, $mem\t# ptr locked" %}
7133   opcode(0x8B);
7134   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7135   ins_pipe(ialu_reg_mem); // XXX
7136 %}
7137 
7138 // Conditional-store of the updated heap-top.
7139 // Used during allocation of the shared heap.
7140 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7141 
7142 instruct storePConditional(memory heap_top_ptr,
7143                            rax_RegP oldval, rRegP newval,
7144                            rFlagsReg cr)
7145 %{
7146   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7147 
7148   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7149             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7150   opcode(0x0F, 0xB1);
7151   ins_encode(lock_prefix,
7152              REX_reg_mem_wide(newval, heap_top_ptr),
7153              OpcP, OpcS,
7154              reg_mem(newval, heap_top_ptr));
7155   ins_pipe(pipe_cmpxchg);
7156 %}
7157 
7158 // Conditional-store of an int value.
7159 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7160 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7161 %{
7162   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7163   effect(KILL oldval);
7164 
7165   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7166   opcode(0x0F, 0xB1);
7167   ins_encode(lock_prefix,
7168              REX_reg_mem(newval, mem),
7169              OpcP, OpcS,
7170              reg_mem(newval, mem));
7171   ins_pipe(pipe_cmpxchg);
7172 %}
7173 
7174 // Conditional-store of a long value.
7175 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7176 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7177 %{
7178   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7179   effect(KILL oldval);
7180 
7181   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7182   opcode(0x0F, 0xB1);
7183   ins_encode(lock_prefix,
7184              REX_reg_mem_wide(newval, mem),
7185              OpcP, OpcS,
7186              reg_mem(newval, mem));
7187   ins_pipe(pipe_cmpxchg);
7188 %}
7189 
7190 
7191 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7192 instruct compareAndSwapP(rRegI res,
7193                          memory mem_ptr,
7194                          rax_RegP oldval, rRegP newval,
7195                          rFlagsReg cr)
7196 %{
7197   predicate(VM_Version::supports_cx8());
7198   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7199   effect(KILL cr, KILL oldval);
7200 
7201   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7202             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7203             "sete    $res\n\t"
7204             "movzbl  $res, $res" %}
7205   opcode(0x0F, 0xB1);
7206   ins_encode(lock_prefix,
7207              REX_reg_mem_wide(newval, mem_ptr),
7208              OpcP, OpcS,
7209              reg_mem(newval, mem_ptr),
7210              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7211              REX_reg_breg(res, res), // movzbl
7212              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7213   ins_pipe( pipe_cmpxchg );
7214 %}
7215 
7216 instruct compareAndSwapL(rRegI res,
7217                          memory mem_ptr,
7218                          rax_RegL oldval, rRegL newval,
7219                          rFlagsReg cr)
7220 %{
7221   predicate(VM_Version::supports_cx8());
7222   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7223   effect(KILL cr, KILL oldval);
7224 
7225   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7226             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7227             "sete    $res\n\t"
7228             "movzbl  $res, $res" %}
7229   opcode(0x0F, 0xB1);
7230   ins_encode(lock_prefix,
7231              REX_reg_mem_wide(newval, mem_ptr),
7232              OpcP, OpcS,
7233              reg_mem(newval, mem_ptr),
7234              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7235              REX_reg_breg(res, res), // movzbl
7236              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7237   ins_pipe( pipe_cmpxchg );
7238 %}
7239 
7240 instruct compareAndSwapI(rRegI res,
7241                          memory mem_ptr,
7242                          rax_RegI oldval, rRegI newval,
7243                          rFlagsReg cr)
7244 %{
7245   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7246   effect(KILL cr, KILL oldval);
7247 
7248   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7249             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7250             "sete    $res\n\t"
7251             "movzbl  $res, $res" %}
7252   opcode(0x0F, 0xB1);
7253   ins_encode(lock_prefix,
7254              REX_reg_mem(newval, mem_ptr),
7255              OpcP, OpcS,
7256              reg_mem(newval, mem_ptr),
7257              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7258              REX_reg_breg(res, res), // movzbl
7259              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7260   ins_pipe( pipe_cmpxchg );
7261 %}
7262 
7263 
7264 instruct compareAndSwapN(rRegI res,
7265                           memory mem_ptr,
7266                           rax_RegN oldval, rRegN newval,
7267                           rFlagsReg cr) %{
7268   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7269   effect(KILL cr, KILL oldval);
7270 
7271   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7272             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7273             "sete    $res\n\t"
7274             "movzbl  $res, $res" %}
7275   opcode(0x0F, 0xB1);
7276   ins_encode(lock_prefix,
7277              REX_reg_mem(newval, mem_ptr),
7278              OpcP, OpcS,
7279              reg_mem(newval, mem_ptr),
7280              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7281              REX_reg_breg(res, res), // movzbl
7282              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7283   ins_pipe( pipe_cmpxchg );
7284 %}
7285 
7286 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7287   predicate(n->as_LoadStore()->result_not_used());
7288   match(Set dummy (GetAndAddI mem add));
7289   effect(KILL cr);
7290   format %{ "ADDL  [$mem],$add" %}
7291   ins_encode %{
7292     if (os::is_MP()) { __ lock(); }
7293     __ addl($mem$$Address, $add$$constant);
7294   %}
7295   ins_pipe( pipe_cmpxchg );
7296 %}
7297 
7298 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7299   match(Set newval (GetAndAddI mem newval));
7300   effect(KILL cr);
7301   format %{ "XADDL  [$mem],$newval" %}
7302   ins_encode %{
7303     if (os::is_MP()) { __ lock(); }
7304     __ xaddl($mem$$Address, $newval$$Register);
7305   %}
7306   ins_pipe( pipe_cmpxchg );
7307 %}
7308 
7309 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7310   predicate(n->as_LoadStore()->result_not_used());
7311   match(Set dummy (GetAndAddL mem add));
7312   effect(KILL cr);
7313   format %{ "ADDQ  [$mem],$add" %}
7314   ins_encode %{
7315     if (os::is_MP()) { __ lock(); }
7316     __ addq($mem$$Address, $add$$constant);
7317   %}
7318   ins_pipe( pipe_cmpxchg );
7319 %}
7320 
7321 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7322   match(Set newval (GetAndAddL mem newval));
7323   effect(KILL cr);
7324   format %{ "XADDQ  [$mem],$newval" %}
7325   ins_encode %{
7326     if (os::is_MP()) { __ lock(); }
7327     __ xaddq($mem$$Address, $newval$$Register);
7328   %}
7329   ins_pipe( pipe_cmpxchg );
7330 %}
7331 
7332 instruct xchgI( memory mem, rRegI newval) %{
7333   match(Set newval (GetAndSetI mem newval));
7334   format %{ "XCHGL  $newval,[$mem]" %}
7335   ins_encode %{
7336     __ xchgl($newval$$Register, $mem$$Address);
7337   %}
7338   ins_pipe( pipe_cmpxchg );
7339 %}
7340 
7341 instruct xchgL( memory mem, rRegL newval) %{
7342   match(Set newval (GetAndSetL mem newval));
7343   format %{ "XCHGL  $newval,[$mem]" %}
7344   ins_encode %{
7345     __ xchgq($newval$$Register, $mem$$Address);
7346   %}
7347   ins_pipe( pipe_cmpxchg );
7348 %}
7349 
7350 instruct xchgP( memory mem, rRegP newval) %{
7351   match(Set newval (GetAndSetP mem newval));
7352   format %{ "XCHGQ  $newval,[$mem]" %}
7353   ins_encode %{
7354     __ xchgq($newval$$Register, $mem$$Address);
7355   %}
7356   ins_pipe( pipe_cmpxchg );
7357 %}
7358 
7359 instruct xchgN( memory mem, rRegN newval) %{
7360   match(Set newval (GetAndSetN mem newval));
7361   format %{ "XCHGL  $newval,$mem]" %}
7362   ins_encode %{
7363     __ xchgl($newval$$Register, $mem$$Address);
7364   %}
7365   ins_pipe( pipe_cmpxchg );
7366 %}
7367 
7368 //----------Subtraction Instructions-------------------------------------------
7369 
7370 // Integer Subtraction Instructions
7371 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7372 %{
7373   match(Set dst (SubI dst src));
7374   effect(KILL cr);
7375 
7376   format %{ "subl    $dst, $src\t# int" %}
7377   opcode(0x2B);
7378   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7379   ins_pipe(ialu_reg_reg);
7380 %}
7381 
7382 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7383 %{
7384   match(Set dst (SubI dst src));
7385   effect(KILL cr);
7386 
7387   format %{ "subl    $dst, $src\t# int" %}
7388   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7389   ins_encode(OpcSErm(dst, src), Con8or32(src));
7390   ins_pipe(ialu_reg);
7391 %}
7392 
7393 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7394 %{
7395   match(Set dst (SubI dst (LoadI src)));
7396   effect(KILL cr);
7397 
7398   ins_cost(125);
7399   format %{ "subl    $dst, $src\t# int" %}
7400   opcode(0x2B);
7401   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7402   ins_pipe(ialu_reg_mem);
7403 %}
7404 
7405 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7406 %{
7407   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7408   effect(KILL cr);
7409 
7410   ins_cost(150);
7411   format %{ "subl    $dst, $src\t# int" %}
7412   opcode(0x29); /* Opcode 29 /r */
7413   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7414   ins_pipe(ialu_mem_reg);
7415 %}
7416 
7417 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7418 %{
7419   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7420   effect(KILL cr);
7421 
7422   ins_cost(125); // XXX
7423   format %{ "subl    $dst, $src\t# int" %}
7424   opcode(0x81); /* Opcode 81 /5 id */
7425   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7426   ins_pipe(ialu_mem_imm);
7427 %}
7428 
7429 instruct subExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
7430 %{
7431   match(SubExactI dst src);
7432   effect(DEF cr);
7433 
7434   format %{ "subl    $dst, $src\t# subExact int" %}
7435   ins_encode %{
7436     __ subl($dst$$Register, $src$$Register);
7437   %}
7438   ins_pipe(ialu_reg_reg);
7439 %}
7440 
7441 instruct subExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr)
7442 %{
7443   match(SubExactI dst src);
7444   effect(DEF cr);
7445 
7446   format %{ "subl    $dst, $src\t# subExact int" %}
7447   ins_encode %{
7448     __ subl($dst$$Register, $src$$constant);
7449   %}
7450   ins_pipe(ialu_reg_reg);
7451 %}
7452 
7453 instruct subExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
7454 %{
7455   match(SubExactI dst (LoadI src));
7456   effect(DEF cr);
7457 
7458   ins_cost(125);
7459   format %{ "subl    $dst, $src\t# subExact int" %}
7460   ins_encode %{
7461     __ subl($dst$$Register, $src$$Address);
7462   %}
7463   ins_pipe(ialu_reg_mem);
7464 %}
7465 
7466 instruct subExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
7467 %{
7468   match(SubExactL dst src);
7469   effect(DEF cr);
7470 
7471   format %{ "subq    $dst, $src\t# subExact long" %}
7472   ins_encode %{
7473     __ subq($dst$$Register, $src$$Register);
7474   %}
7475   ins_pipe(ialu_reg_reg);
7476 %}
7477 
7478 instruct subExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr)
7479 %{
7480   match(SubExactL dst (LoadL src));
7481   effect(DEF cr);
7482 
7483   format %{ "subq    $dst, $src\t# subExact long" %}
7484   ins_encode %{
7485     __ subq($dst$$Register, $src$$constant);
7486   %}
7487   ins_pipe(ialu_reg_reg);
7488 %}
7489 
7490 instruct subExactL_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
7491 %{
7492   match(SubExactI dst src);
7493   effect(DEF cr);
7494 
7495   ins_cost(125);
7496   format %{ "subq    $dst, $src\t# subExact long" %}
7497   ins_encode %{
7498     __ subq($dst$$Register, $src$$Address);
7499   %}
7500   ins_pipe(ialu_reg_mem);
7501 %}
7502 
7503 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7504 %{
7505   match(Set dst (SubL dst src));
7506   effect(KILL cr);
7507 
7508   format %{ "subq    $dst, $src\t# long" %}
7509   opcode(0x2B);
7510   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7511   ins_pipe(ialu_reg_reg);
7512 %}
7513 
7514 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7515 %{
7516   match(Set dst (SubL dst src));
7517   effect(KILL cr);
7518 
7519   format %{ "subq    $dst, $src\t# long" %}
7520   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7521   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7522   ins_pipe(ialu_reg);
7523 %}
7524 
7525 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7526 %{
7527   match(Set dst (SubL dst (LoadL src)));
7528   effect(KILL cr);
7529 
7530   ins_cost(125);
7531   format %{ "subq    $dst, $src\t# long" %}
7532   opcode(0x2B);
7533   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7534   ins_pipe(ialu_reg_mem);
7535 %}
7536 
7537 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7538 %{
7539   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7540   effect(KILL cr);
7541 
7542   ins_cost(150);
7543   format %{ "subq    $dst, $src\t# long" %}
7544   opcode(0x29); /* Opcode 29 /r */
7545   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7546   ins_pipe(ialu_mem_reg);
7547 %}
7548 
7549 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7550 %{
7551   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7552   effect(KILL cr);
7553 
7554   ins_cost(125); // XXX
7555   format %{ "subq    $dst, $src\t# long" %}
7556   opcode(0x81); /* Opcode 81 /5 id */
7557   ins_encode(REX_mem_wide(dst),
7558              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7559   ins_pipe(ialu_mem_imm);
7560 %}
7561 
7562 // Subtract from a pointer
7563 // XXX hmpf???
7564 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7565 %{
7566   match(Set dst (AddP dst (SubI zero src)));
7567   effect(KILL cr);
7568 
7569   format %{ "subq    $dst, $src\t# ptr - int" %}
7570   opcode(0x2B);
7571   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7572   ins_pipe(ialu_reg_reg);
7573 %}
7574 
7575 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7576 %{
7577   match(Set dst (SubI zero dst));
7578   effect(KILL cr);
7579 
7580   format %{ "negl    $dst\t# int" %}
7581   opcode(0xF7, 0x03);  // Opcode F7 /3
7582   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7583   ins_pipe(ialu_reg);
7584 %}
7585 
7586 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7587 %{
7588   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7589   effect(KILL cr);
7590 
7591   format %{ "negl    $dst\t# int" %}
7592   opcode(0xF7, 0x03);  // Opcode F7 /3
7593   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7594   ins_pipe(ialu_reg);
7595 %}
7596 
7597 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7598 %{
7599   match(Set dst (SubL zero dst));
7600   effect(KILL cr);
7601 
7602   format %{ "negq    $dst\t# long" %}
7603   opcode(0xF7, 0x03);  // Opcode F7 /3
7604   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7605   ins_pipe(ialu_reg);
7606 %}
7607 
7608 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7609 %{
7610   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7611   effect(KILL cr);
7612 
7613   format %{ "negq    $dst\t# long" %}
7614   opcode(0xF7, 0x03);  // Opcode F7 /3
7615   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7616   ins_pipe(ialu_reg);
7617 %}
7618 
7619 instruct negExactI_rReg(rax_RegI dst, rFlagsReg cr)
7620 %{
7621   match(NegExactI dst);
7622   effect(KILL cr);
7623 
7624   format %{ "negl    $dst\t# negExact int" %}
7625   ins_encode %{
7626     __ negl($dst$$Register);
7627   %}
7628   ins_pipe(ialu_reg);
7629 %}
7630 
7631 instruct negExactL_rReg(rax_RegL dst, rFlagsReg cr)
7632 %{
7633   match(NegExactL dst);
7634   effect(KILL cr);
7635 
7636   format %{ "negq    $dst\t# negExact long" %}
7637   ins_encode %{
7638     __ negq($dst$$Register);
7639   %}
7640   ins_pipe(ialu_reg);
7641 %}
7642 
7643 
7644 //----------Multiplication/Division Instructions-------------------------------
7645 // Integer Multiplication Instructions
7646 // Multiply Register
7647 
7648 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7649 %{
7650   match(Set dst (MulI dst src));
7651   effect(KILL cr);
7652 
7653   ins_cost(300);
7654   format %{ "imull   $dst, $src\t# int" %}
7655   opcode(0x0F, 0xAF);
7656   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7657   ins_pipe(ialu_reg_reg_alu0);
7658 %}
7659 
7660 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7661 %{
7662   match(Set dst (MulI src imm));
7663   effect(KILL cr);
7664 
7665   ins_cost(300);
7666   format %{ "imull   $dst, $src, $imm\t# int" %}
7667   opcode(0x69); /* 69 /r id */
7668   ins_encode(REX_reg_reg(dst, src),
7669              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7670   ins_pipe(ialu_reg_reg_alu0);
7671 %}
7672 
7673 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7674 %{
7675   match(Set dst (MulI dst (LoadI src)));
7676   effect(KILL cr);
7677 
7678   ins_cost(350);
7679   format %{ "imull   $dst, $src\t# int" %}
7680   opcode(0x0F, 0xAF);
7681   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7682   ins_pipe(ialu_reg_mem_alu0);
7683 %}
7684 
7685 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7686 %{
7687   match(Set dst (MulI (LoadI src) imm));
7688   effect(KILL cr);
7689 
7690   ins_cost(300);
7691   format %{ "imull   $dst, $src, $imm\t# int" %}
7692   opcode(0x69); /* 69 /r id */
7693   ins_encode(REX_reg_mem(dst, src),
7694              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7695   ins_pipe(ialu_reg_mem_alu0);
7696 %}
7697 
7698 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7699 %{
7700   match(Set dst (MulL dst src));
7701   effect(KILL cr);
7702 
7703   ins_cost(300);
7704   format %{ "imulq   $dst, $src\t# long" %}
7705   opcode(0x0F, 0xAF);
7706   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7707   ins_pipe(ialu_reg_reg_alu0);
7708 %}
7709 
7710 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7711 %{
7712   match(Set dst (MulL src imm));
7713   effect(KILL cr);
7714 
7715   ins_cost(300);
7716   format %{ "imulq   $dst, $src, $imm\t# long" %}
7717   opcode(0x69); /* 69 /r id */
7718   ins_encode(REX_reg_reg_wide(dst, src),
7719              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7720   ins_pipe(ialu_reg_reg_alu0);
7721 %}
7722 
7723 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7724 %{
7725   match(Set dst (MulL dst (LoadL src)));
7726   effect(KILL cr);
7727 
7728   ins_cost(350);
7729   format %{ "imulq   $dst, $src\t# long" %}
7730   opcode(0x0F, 0xAF);
7731   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7732   ins_pipe(ialu_reg_mem_alu0);
7733 %}
7734 
7735 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7736 %{
7737   match(Set dst (MulL (LoadL src) imm));
7738   effect(KILL cr);
7739 
7740   ins_cost(300);
7741   format %{ "imulq   $dst, $src, $imm\t# long" %}
7742   opcode(0x69); /* 69 /r id */
7743   ins_encode(REX_reg_mem_wide(dst, src),
7744              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7745   ins_pipe(ialu_reg_mem_alu0);
7746 %}
7747 
7748 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7749 %{
7750   match(Set dst (MulHiL src rax));
7751   effect(USE_KILL rax, KILL cr);
7752 
7753   ins_cost(300);
7754   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7755   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7756   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7757   ins_pipe(ialu_reg_reg_alu0);
7758 %}
7759 
7760 
7761 instruct mulExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
7762 %{
7763   match(MulExactI dst src);
7764   effect(DEF cr);
7765 
7766   ins_cost(300);
7767   format %{ "imull   $dst, $src\t# mulExact int" %}
7768   ins_encode %{
7769     __ imull($dst$$Register, $src$$Register);
7770   %}
7771   ins_pipe(ialu_reg_reg_alu0);
7772 %}
7773 
7774 
7775 instruct mulExactI_rReg_imm(rax_RegI dst, rRegI src, immI imm, rFlagsReg cr)
7776 %{
7777   match(MulExactI src imm);
7778   effect(DEF cr);
7779 
7780   ins_cost(300);
7781   format %{ "imull   $dst, $src, $imm\t# mulExact int" %}
7782   ins_encode %{
7783     __ imull($dst$$Register, $src$$Register, $imm$$constant);
7784   %}
7785   ins_pipe(ialu_reg_reg_alu0);
7786 %}
7787 
7788 instruct mulExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
7789 %{
7790   match(MulExactI dst (LoadI src));
7791   effect(DEF cr);
7792 
7793   ins_cost(350);
7794   format %{ "imull   $dst, $src\t# mulExact int" %}
7795   ins_encode %{
7796     __ imull($dst$$Register, $src$$Address);
7797   %}
7798   ins_pipe(ialu_reg_mem_alu0);
7799 %}
7800 
7801 instruct mulExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
7802 %{
7803   match(MulExactL dst src);
7804   effect(DEF cr);
7805 
7806   ins_cost(300);
7807   format %{ "imulq   $dst, $src\t# mulExact long" %}
7808   ins_encode %{
7809     __ imulq($dst$$Register, $src$$Register);
7810   %}
7811   ins_pipe(ialu_reg_reg_alu0);
7812 %}
7813 
7814 instruct mulExactL_rReg_imm(rax_RegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7815 %{
7816   match(MulExactL src imm);
7817   effect(DEF cr);
7818 
7819   ins_cost(300);
7820   format %{ "imulq   $dst, $src, $imm\t# mulExact long" %}
7821   ins_encode %{
7822     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
7823   %}
7824   ins_pipe(ialu_reg_reg_alu0);
7825 %}
7826 
7827 instruct mulExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr)
7828 %{
7829   match(MulExactL dst (LoadL src));
7830   effect(DEF cr);
7831 
7832   ins_cost(350);
7833   format %{ "imulq   $dst, $src\t# mulExact long" %}
7834   ins_encode %{
7835     __ imulq($dst$$Register, $src$$Address);
7836   %}
7837   ins_pipe(ialu_reg_mem_alu0);
7838 %}
7839 
7840 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7841                    rFlagsReg cr)
7842 %{
7843   match(Set rax (DivI rax div));
7844   effect(KILL rdx, KILL cr);
7845 
7846   ins_cost(30*100+10*100); // XXX
7847   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7848             "jne,s   normal\n\t"
7849             "xorl    rdx, rdx\n\t"
7850             "cmpl    $div, -1\n\t"
7851             "je,s    done\n"
7852     "normal: cdql\n\t"
7853             "idivl   $div\n"
7854     "done:"        %}
7855   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7856   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7857   ins_pipe(ialu_reg_reg_alu0);
7858 %}
7859 
7860 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7861                    rFlagsReg cr)
7862 %{
7863   match(Set rax (DivL rax div));
7864   effect(KILL rdx, KILL cr);
7865 
7866   ins_cost(30*100+10*100); // XXX
7867   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7868             "cmpq    rax, rdx\n\t"
7869             "jne,s   normal\n\t"
7870             "xorl    rdx, rdx\n\t"
7871             "cmpq    $div, -1\n\t"
7872             "je,s    done\n"
7873     "normal: cdqq\n\t"
7874             "idivq   $div\n"
7875     "done:"        %}
7876   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7877   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7878   ins_pipe(ialu_reg_reg_alu0);
7879 %}
7880 
7881 // Integer DIVMOD with Register, both quotient and mod results
7882 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7883                              rFlagsReg cr)
7884 %{
7885   match(DivModI rax div);
7886   effect(KILL cr);
7887 
7888   ins_cost(30*100+10*100); // XXX
7889   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7890             "jne,s   normal\n\t"
7891             "xorl    rdx, rdx\n\t"
7892             "cmpl    $div, -1\n\t"
7893             "je,s    done\n"
7894     "normal: cdql\n\t"
7895             "idivl   $div\n"
7896     "done:"        %}
7897   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7898   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7899   ins_pipe(pipe_slow);
7900 %}
7901 
7902 // Long DIVMOD with Register, both quotient and mod results
7903 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7904                              rFlagsReg cr)
7905 %{
7906   match(DivModL rax div);
7907   effect(KILL cr);
7908 
7909   ins_cost(30*100+10*100); // XXX
7910   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7911             "cmpq    rax, rdx\n\t"
7912             "jne,s   normal\n\t"
7913             "xorl    rdx, rdx\n\t"
7914             "cmpq    $div, -1\n\t"
7915             "je,s    done\n"
7916     "normal: cdqq\n\t"
7917             "idivq   $div\n"
7918     "done:"        %}
7919   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7920   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7921   ins_pipe(pipe_slow);
7922 %}
7923 
7924 //----------- DivL-By-Constant-Expansions--------------------------------------
7925 // DivI cases are handled by the compiler
7926 
7927 // Magic constant, reciprocal of 10
7928 instruct loadConL_0x6666666666666667(rRegL dst)
7929 %{
7930   effect(DEF dst);
7931 
7932   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7933   ins_encode(load_immL(dst, 0x6666666666666667));
7934   ins_pipe(ialu_reg);
7935 %}
7936 
7937 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7938 %{
7939   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7940 
7941   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7942   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7943   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7944   ins_pipe(ialu_reg_reg_alu0);
7945 %}
7946 
7947 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7948 %{
7949   effect(USE_DEF dst, KILL cr);
7950 
7951   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7952   opcode(0xC1, 0x7); /* C1 /7 ib */
7953   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7954   ins_pipe(ialu_reg);
7955 %}
7956 
7957 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7958 %{
7959   effect(USE_DEF dst, KILL cr);
7960 
7961   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7962   opcode(0xC1, 0x7); /* C1 /7 ib */
7963   ins_encode(reg_opc_imm_wide(dst, 0x2));
7964   ins_pipe(ialu_reg);
7965 %}
7966 
7967 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7968 %{
7969   match(Set dst (DivL src div));
7970 
7971   ins_cost((5+8)*100);
7972   expand %{
7973     rax_RegL rax;                     // Killed temp
7974     rFlagsReg cr;                     // Killed
7975     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7976     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7977     sarL_rReg_63(src, cr);            // sarq  src, 63
7978     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7979     subL_rReg(dst, src, cr);          // subl  rdx, src
7980   %}
7981 %}
7982 
7983 //-----------------------------------------------------------------------------
7984 
7985 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7986                    rFlagsReg cr)
7987 %{
7988   match(Set rdx (ModI rax div));
7989   effect(KILL rax, KILL cr);
7990 
7991   ins_cost(300); // XXX
7992   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7993             "jne,s   normal\n\t"
7994             "xorl    rdx, rdx\n\t"
7995             "cmpl    $div, -1\n\t"
7996             "je,s    done\n"
7997     "normal: cdql\n\t"
7998             "idivl   $div\n"
7999     "done:"        %}
8000   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8001   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8002   ins_pipe(ialu_reg_reg_alu0);
8003 %}
8004 
8005 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8006                    rFlagsReg cr)
8007 %{
8008   match(Set rdx (ModL rax div));
8009   effect(KILL rax, KILL cr);
8010 
8011   ins_cost(300); // XXX
8012   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8013             "cmpq    rax, rdx\n\t"
8014             "jne,s   normal\n\t"
8015             "xorl    rdx, rdx\n\t"
8016             "cmpq    $div, -1\n\t"
8017             "je,s    done\n"
8018     "normal: cdqq\n\t"
8019             "idivq   $div\n"
8020     "done:"        %}
8021   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8022   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8023   ins_pipe(ialu_reg_reg_alu0);
8024 %}
8025 
8026 // Integer Shift Instructions
8027 // Shift Left by one
8028 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8029 %{
8030   match(Set dst (LShiftI dst shift));
8031   effect(KILL cr);
8032 
8033   format %{ "sall    $dst, $shift" %}
8034   opcode(0xD1, 0x4); /* D1 /4 */
8035   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8036   ins_pipe(ialu_reg);
8037 %}
8038 
8039 // Shift Left by one
8040 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8041 %{
8042   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8043   effect(KILL cr);
8044 
8045   format %{ "sall    $dst, $shift\t" %}
8046   opcode(0xD1, 0x4); /* D1 /4 */
8047   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8048   ins_pipe(ialu_mem_imm);
8049 %}
8050 
8051 // Shift Left by 8-bit immediate
8052 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8053 %{
8054   match(Set dst (LShiftI dst shift));
8055   effect(KILL cr);
8056 
8057   format %{ "sall    $dst, $shift" %}
8058   opcode(0xC1, 0x4); /* C1 /4 ib */
8059   ins_encode(reg_opc_imm(dst, shift));
8060   ins_pipe(ialu_reg);
8061 %}
8062 
8063 // Shift Left by 8-bit immediate
8064 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8065 %{
8066   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8067   effect(KILL cr);
8068 
8069   format %{ "sall    $dst, $shift" %}
8070   opcode(0xC1, 0x4); /* C1 /4 ib */
8071   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8072   ins_pipe(ialu_mem_imm);
8073 %}
8074 
8075 // Shift Left by variable
8076 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8077 %{
8078   match(Set dst (LShiftI dst shift));
8079   effect(KILL cr);
8080 
8081   format %{ "sall    $dst, $shift" %}
8082   opcode(0xD3, 0x4); /* D3 /4 */
8083   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8084   ins_pipe(ialu_reg_reg);
8085 %}
8086 
8087 // Shift Left by variable
8088 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8089 %{
8090   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8091   effect(KILL cr);
8092 
8093   format %{ "sall    $dst, $shift" %}
8094   opcode(0xD3, 0x4); /* D3 /4 */
8095   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8096   ins_pipe(ialu_mem_reg);
8097 %}
8098 
8099 // Arithmetic shift right by one
8100 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8101 %{
8102   match(Set dst (RShiftI dst shift));
8103   effect(KILL cr);
8104 
8105   format %{ "sarl    $dst, $shift" %}
8106   opcode(0xD1, 0x7); /* D1 /7 */
8107   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8108   ins_pipe(ialu_reg);
8109 %}
8110 
8111 // Arithmetic shift right by one
8112 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8113 %{
8114   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8115   effect(KILL cr);
8116 
8117   format %{ "sarl    $dst, $shift" %}
8118   opcode(0xD1, 0x7); /* D1 /7 */
8119   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8120   ins_pipe(ialu_mem_imm);
8121 %}
8122 
8123 // Arithmetic Shift Right by 8-bit immediate
8124 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8125 %{
8126   match(Set dst (RShiftI dst shift));
8127   effect(KILL cr);
8128 
8129   format %{ "sarl    $dst, $shift" %}
8130   opcode(0xC1, 0x7); /* C1 /7 ib */
8131   ins_encode(reg_opc_imm(dst, shift));
8132   ins_pipe(ialu_mem_imm);
8133 %}
8134 
8135 // Arithmetic Shift Right by 8-bit immediate
8136 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8137 %{
8138   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8139   effect(KILL cr);
8140 
8141   format %{ "sarl    $dst, $shift" %}
8142   opcode(0xC1, 0x7); /* C1 /7 ib */
8143   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8144   ins_pipe(ialu_mem_imm);
8145 %}
8146 
8147 // Arithmetic Shift Right by variable
8148 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8149 %{
8150   match(Set dst (RShiftI dst shift));
8151   effect(KILL cr);
8152 
8153   format %{ "sarl    $dst, $shift" %}
8154   opcode(0xD3, 0x7); /* D3 /7 */
8155   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8156   ins_pipe(ialu_reg_reg);
8157 %}
8158 
8159 // Arithmetic Shift Right by variable
8160 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8161 %{
8162   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8163   effect(KILL cr);
8164 
8165   format %{ "sarl    $dst, $shift" %}
8166   opcode(0xD3, 0x7); /* D3 /7 */
8167   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8168   ins_pipe(ialu_mem_reg);
8169 %}
8170 
8171 // Logical shift right by one
8172 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8173 %{
8174   match(Set dst (URShiftI dst shift));
8175   effect(KILL cr);
8176 
8177   format %{ "shrl    $dst, $shift" %}
8178   opcode(0xD1, 0x5); /* D1 /5 */
8179   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8180   ins_pipe(ialu_reg);
8181 %}
8182 
8183 // Logical shift right by one
8184 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8185 %{
8186   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8187   effect(KILL cr);
8188 
8189   format %{ "shrl    $dst, $shift" %}
8190   opcode(0xD1, 0x5); /* D1 /5 */
8191   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8192   ins_pipe(ialu_mem_imm);
8193 %}
8194 
8195 // Logical Shift Right by 8-bit immediate
8196 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8197 %{
8198   match(Set dst (URShiftI dst shift));
8199   effect(KILL cr);
8200 
8201   format %{ "shrl    $dst, $shift" %}
8202   opcode(0xC1, 0x5); /* C1 /5 ib */
8203   ins_encode(reg_opc_imm(dst, shift));
8204   ins_pipe(ialu_reg);
8205 %}
8206 
8207 // Logical Shift Right by 8-bit immediate
8208 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8209 %{
8210   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8211   effect(KILL cr);
8212 
8213   format %{ "shrl    $dst, $shift" %}
8214   opcode(0xC1, 0x5); /* C1 /5 ib */
8215   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8216   ins_pipe(ialu_mem_imm);
8217 %}
8218 
8219 // Logical Shift Right by variable
8220 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8221 %{
8222   match(Set dst (URShiftI dst shift));
8223   effect(KILL cr);
8224 
8225   format %{ "shrl    $dst, $shift" %}
8226   opcode(0xD3, 0x5); /* D3 /5 */
8227   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8228   ins_pipe(ialu_reg_reg);
8229 %}
8230 
8231 // Logical Shift Right by variable
8232 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8233 %{
8234   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8235   effect(KILL cr);
8236 
8237   format %{ "shrl    $dst, $shift" %}
8238   opcode(0xD3, 0x5); /* D3 /5 */
8239   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8240   ins_pipe(ialu_mem_reg);
8241 %}
8242 
8243 // Long Shift Instructions
8244 // Shift Left by one
8245 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8246 %{
8247   match(Set dst (LShiftL dst shift));
8248   effect(KILL cr);
8249 
8250   format %{ "salq    $dst, $shift" %}
8251   opcode(0xD1, 0x4); /* D1 /4 */
8252   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8253   ins_pipe(ialu_reg);
8254 %}
8255 
8256 // Shift Left by one
8257 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8258 %{
8259   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8260   effect(KILL cr);
8261 
8262   format %{ "salq    $dst, $shift" %}
8263   opcode(0xD1, 0x4); /* D1 /4 */
8264   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8265   ins_pipe(ialu_mem_imm);
8266 %}
8267 
8268 // Shift Left by 8-bit immediate
8269 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8270 %{
8271   match(Set dst (LShiftL dst shift));
8272   effect(KILL cr);
8273 
8274   format %{ "salq    $dst, $shift" %}
8275   opcode(0xC1, 0x4); /* C1 /4 ib */
8276   ins_encode(reg_opc_imm_wide(dst, shift));
8277   ins_pipe(ialu_reg);
8278 %}
8279 
8280 // Shift Left by 8-bit immediate
8281 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8282 %{
8283   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8284   effect(KILL cr);
8285 
8286   format %{ "salq    $dst, $shift" %}
8287   opcode(0xC1, 0x4); /* C1 /4 ib */
8288   ins_encode(REX_mem_wide(dst), OpcP,
8289              RM_opc_mem(secondary, dst), Con8or32(shift));
8290   ins_pipe(ialu_mem_imm);
8291 %}
8292 
8293 // Shift Left by variable
8294 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8295 %{
8296   match(Set dst (LShiftL dst shift));
8297   effect(KILL cr);
8298 
8299   format %{ "salq    $dst, $shift" %}
8300   opcode(0xD3, 0x4); /* D3 /4 */
8301   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8302   ins_pipe(ialu_reg_reg);
8303 %}
8304 
8305 // Shift Left by variable
8306 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8307 %{
8308   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8309   effect(KILL cr);
8310 
8311   format %{ "salq    $dst, $shift" %}
8312   opcode(0xD3, 0x4); /* D3 /4 */
8313   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8314   ins_pipe(ialu_mem_reg);
8315 %}
8316 
8317 // Arithmetic shift right by one
8318 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8319 %{
8320   match(Set dst (RShiftL dst shift));
8321   effect(KILL cr);
8322 
8323   format %{ "sarq    $dst, $shift" %}
8324   opcode(0xD1, 0x7); /* D1 /7 */
8325   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8326   ins_pipe(ialu_reg);
8327 %}
8328 
8329 // Arithmetic shift right by one
8330 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8331 %{
8332   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8333   effect(KILL cr);
8334 
8335   format %{ "sarq    $dst, $shift" %}
8336   opcode(0xD1, 0x7); /* D1 /7 */
8337   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8338   ins_pipe(ialu_mem_imm);
8339 %}
8340 
8341 // Arithmetic Shift Right by 8-bit immediate
8342 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8343 %{
8344   match(Set dst (RShiftL dst shift));
8345   effect(KILL cr);
8346 
8347   format %{ "sarq    $dst, $shift" %}
8348   opcode(0xC1, 0x7); /* C1 /7 ib */
8349   ins_encode(reg_opc_imm_wide(dst, shift));
8350   ins_pipe(ialu_mem_imm);
8351 %}
8352 
8353 // Arithmetic Shift Right by 8-bit immediate
8354 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8355 %{
8356   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8357   effect(KILL cr);
8358 
8359   format %{ "sarq    $dst, $shift" %}
8360   opcode(0xC1, 0x7); /* C1 /7 ib */
8361   ins_encode(REX_mem_wide(dst), OpcP,
8362              RM_opc_mem(secondary, dst), Con8or32(shift));
8363   ins_pipe(ialu_mem_imm);
8364 %}
8365 
8366 // Arithmetic Shift Right by variable
8367 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8368 %{
8369   match(Set dst (RShiftL dst shift));
8370   effect(KILL cr);
8371 
8372   format %{ "sarq    $dst, $shift" %}
8373   opcode(0xD3, 0x7); /* D3 /7 */
8374   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8375   ins_pipe(ialu_reg_reg);
8376 %}
8377 
8378 // Arithmetic Shift Right by variable
8379 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8380 %{
8381   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8382   effect(KILL cr);
8383 
8384   format %{ "sarq    $dst, $shift" %}
8385   opcode(0xD3, 0x7); /* D3 /7 */
8386   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8387   ins_pipe(ialu_mem_reg);
8388 %}
8389 
8390 // Logical shift right by one
8391 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8392 %{
8393   match(Set dst (URShiftL dst shift));
8394   effect(KILL cr);
8395 
8396   format %{ "shrq    $dst, $shift" %}
8397   opcode(0xD1, 0x5); /* D1 /5 */
8398   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8399   ins_pipe(ialu_reg);
8400 %}
8401 
8402 // Logical shift right by one
8403 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8404 %{
8405   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8406   effect(KILL cr);
8407 
8408   format %{ "shrq    $dst, $shift" %}
8409   opcode(0xD1, 0x5); /* D1 /5 */
8410   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8411   ins_pipe(ialu_mem_imm);
8412 %}
8413 
8414 // Logical Shift Right by 8-bit immediate
8415 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8416 %{
8417   match(Set dst (URShiftL dst shift));
8418   effect(KILL cr);
8419 
8420   format %{ "shrq    $dst, $shift" %}
8421   opcode(0xC1, 0x5); /* C1 /5 ib */
8422   ins_encode(reg_opc_imm_wide(dst, shift));
8423   ins_pipe(ialu_reg);
8424 %}
8425 
8426 
8427 // Logical Shift Right by 8-bit immediate
8428 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8429 %{
8430   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8431   effect(KILL cr);
8432 
8433   format %{ "shrq    $dst, $shift" %}
8434   opcode(0xC1, 0x5); /* C1 /5 ib */
8435   ins_encode(REX_mem_wide(dst), OpcP,
8436              RM_opc_mem(secondary, dst), Con8or32(shift));
8437   ins_pipe(ialu_mem_imm);
8438 %}
8439 
8440 // Logical Shift Right by variable
8441 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8442 %{
8443   match(Set dst (URShiftL dst shift));
8444   effect(KILL cr);
8445 
8446   format %{ "shrq    $dst, $shift" %}
8447   opcode(0xD3, 0x5); /* D3 /5 */
8448   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8449   ins_pipe(ialu_reg_reg);
8450 %}
8451 
8452 // Logical Shift Right by variable
8453 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8454 %{
8455   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8456   effect(KILL cr);
8457 
8458   format %{ "shrq    $dst, $shift" %}
8459   opcode(0xD3, 0x5); /* D3 /5 */
8460   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8461   ins_pipe(ialu_mem_reg);
8462 %}
8463 
8464 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8465 // This idiom is used by the compiler for the i2b bytecode.
8466 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8467 %{
8468   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8469 
8470   format %{ "movsbl  $dst, $src\t# i2b" %}
8471   opcode(0x0F, 0xBE);
8472   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8473   ins_pipe(ialu_reg_reg);
8474 %}
8475 
8476 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8477 // This idiom is used by the compiler the i2s bytecode.
8478 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8479 %{
8480   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8481 
8482   format %{ "movswl  $dst, $src\t# i2s" %}
8483   opcode(0x0F, 0xBF);
8484   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8485   ins_pipe(ialu_reg_reg);
8486 %}
8487 
8488 // ROL/ROR instructions
8489 
8490 // ROL expand
8491 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8492   effect(KILL cr, USE_DEF dst);
8493 
8494   format %{ "roll    $dst" %}
8495   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8496   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8497   ins_pipe(ialu_reg);
8498 %}
8499 
8500 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8501   effect(USE_DEF dst, USE shift, KILL cr);
8502 
8503   format %{ "roll    $dst, $shift" %}
8504   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8505   ins_encode( reg_opc_imm(dst, shift) );
8506   ins_pipe(ialu_reg);
8507 %}
8508 
8509 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8510 %{
8511   effect(USE_DEF dst, USE shift, KILL cr);
8512 
8513   format %{ "roll    $dst, $shift" %}
8514   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8515   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8516   ins_pipe(ialu_reg_reg);
8517 %}
8518 // end of ROL expand
8519 
8520 // Rotate Left by one
8521 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8522 %{
8523   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8524 
8525   expand %{
8526     rolI_rReg_imm1(dst, cr);
8527   %}
8528 %}
8529 
8530 // Rotate Left by 8-bit immediate
8531 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8532 %{
8533   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8534   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8535 
8536   expand %{
8537     rolI_rReg_imm8(dst, lshift, cr);
8538   %}
8539 %}
8540 
8541 // Rotate Left by variable
8542 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8543 %{
8544   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8545 
8546   expand %{
8547     rolI_rReg_CL(dst, shift, cr);
8548   %}
8549 %}
8550 
8551 // Rotate Left by variable
8552 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8553 %{
8554   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8555 
8556   expand %{
8557     rolI_rReg_CL(dst, shift, cr);
8558   %}
8559 %}
8560 
8561 // ROR expand
8562 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8563 %{
8564   effect(USE_DEF dst, KILL cr);
8565 
8566   format %{ "rorl    $dst" %}
8567   opcode(0xD1, 0x1); /* D1 /1 */
8568   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8569   ins_pipe(ialu_reg);
8570 %}
8571 
8572 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8573 %{
8574   effect(USE_DEF dst, USE shift, KILL cr);
8575 
8576   format %{ "rorl    $dst, $shift" %}
8577   opcode(0xC1, 0x1); /* C1 /1 ib */
8578   ins_encode(reg_opc_imm(dst, shift));
8579   ins_pipe(ialu_reg);
8580 %}
8581 
8582 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8583 %{
8584   effect(USE_DEF dst, USE shift, KILL cr);
8585 
8586   format %{ "rorl    $dst, $shift" %}
8587   opcode(0xD3, 0x1); /* D3 /1 */
8588   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8589   ins_pipe(ialu_reg_reg);
8590 %}
8591 // end of ROR expand
8592 
8593 // Rotate Right by one
8594 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8595 %{
8596   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8597 
8598   expand %{
8599     rorI_rReg_imm1(dst, cr);
8600   %}
8601 %}
8602 
8603 // Rotate Right by 8-bit immediate
8604 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8605 %{
8606   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8607   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8608 
8609   expand %{
8610     rorI_rReg_imm8(dst, rshift, cr);
8611   %}
8612 %}
8613 
8614 // Rotate Right by variable
8615 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8616 %{
8617   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8618 
8619   expand %{
8620     rorI_rReg_CL(dst, shift, cr);
8621   %}
8622 %}
8623 
8624 // Rotate Right by variable
8625 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8626 %{
8627   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8628 
8629   expand %{
8630     rorI_rReg_CL(dst, shift, cr);
8631   %}
8632 %}
8633 
8634 // for long rotate
8635 // ROL expand
8636 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8637   effect(USE_DEF dst, KILL cr);
8638 
8639   format %{ "rolq    $dst" %}
8640   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8641   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8642   ins_pipe(ialu_reg);
8643 %}
8644 
8645 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8646   effect(USE_DEF dst, USE shift, KILL cr);
8647 
8648   format %{ "rolq    $dst, $shift" %}
8649   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8650   ins_encode( reg_opc_imm_wide(dst, shift) );
8651   ins_pipe(ialu_reg);
8652 %}
8653 
8654 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8655 %{
8656   effect(USE_DEF dst, USE shift, KILL cr);
8657 
8658   format %{ "rolq    $dst, $shift" %}
8659   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8660   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8661   ins_pipe(ialu_reg_reg);
8662 %}
8663 // end of ROL expand
8664 
8665 // Rotate Left by one
8666 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8667 %{
8668   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8669 
8670   expand %{
8671     rolL_rReg_imm1(dst, cr);
8672   %}
8673 %}
8674 
8675 // Rotate Left by 8-bit immediate
8676 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8677 %{
8678   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8679   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8680 
8681   expand %{
8682     rolL_rReg_imm8(dst, lshift, cr);
8683   %}
8684 %}
8685 
8686 // Rotate Left by variable
8687 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8688 %{
8689   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8690 
8691   expand %{
8692     rolL_rReg_CL(dst, shift, cr);
8693   %}
8694 %}
8695 
8696 // Rotate Left by variable
8697 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8698 %{
8699   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8700 
8701   expand %{
8702     rolL_rReg_CL(dst, shift, cr);
8703   %}
8704 %}
8705 
8706 // ROR expand
8707 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8708 %{
8709   effect(USE_DEF dst, KILL cr);
8710 
8711   format %{ "rorq    $dst" %}
8712   opcode(0xD1, 0x1); /* D1 /1 */
8713   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8714   ins_pipe(ialu_reg);
8715 %}
8716 
8717 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8718 %{
8719   effect(USE_DEF dst, USE shift, KILL cr);
8720 
8721   format %{ "rorq    $dst, $shift" %}
8722   opcode(0xC1, 0x1); /* C1 /1 ib */
8723   ins_encode(reg_opc_imm_wide(dst, shift));
8724   ins_pipe(ialu_reg);
8725 %}
8726 
8727 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8728 %{
8729   effect(USE_DEF dst, USE shift, KILL cr);
8730 
8731   format %{ "rorq    $dst, $shift" %}
8732   opcode(0xD3, 0x1); /* D3 /1 */
8733   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8734   ins_pipe(ialu_reg_reg);
8735 %}
8736 // end of ROR expand
8737 
8738 // Rotate Right by one
8739 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8740 %{
8741   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8742 
8743   expand %{
8744     rorL_rReg_imm1(dst, cr);
8745   %}
8746 %}
8747 
8748 // Rotate Right by 8-bit immediate
8749 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8750 %{
8751   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8752   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8753 
8754   expand %{
8755     rorL_rReg_imm8(dst, rshift, cr);
8756   %}
8757 %}
8758 
8759 // Rotate Right by variable
8760 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8761 %{
8762   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8763 
8764   expand %{
8765     rorL_rReg_CL(dst, shift, cr);
8766   %}
8767 %}
8768 
8769 // Rotate Right by variable
8770 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8771 %{
8772   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8773 
8774   expand %{
8775     rorL_rReg_CL(dst, shift, cr);
8776   %}
8777 %}
8778 
8779 // Logical Instructions
8780 
8781 // Integer Logical Instructions
8782 
8783 // And Instructions
8784 // And Register with Register
8785 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8786 %{
8787   match(Set dst (AndI dst src));
8788   effect(KILL cr);
8789 
8790   format %{ "andl    $dst, $src\t# int" %}
8791   opcode(0x23);
8792   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8793   ins_pipe(ialu_reg_reg);
8794 %}
8795 
8796 // And Register with Immediate 255
8797 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8798 %{
8799   match(Set dst (AndI dst src));
8800 
8801   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8802   opcode(0x0F, 0xB6);
8803   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8804   ins_pipe(ialu_reg);
8805 %}
8806 
8807 // And Register with Immediate 255 and promote to long
8808 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8809 %{
8810   match(Set dst (ConvI2L (AndI src mask)));
8811 
8812   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8813   opcode(0x0F, 0xB6);
8814   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8815   ins_pipe(ialu_reg);
8816 %}
8817 
8818 // And Register with Immediate 65535
8819 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8820 %{
8821   match(Set dst (AndI dst src));
8822 
8823   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8824   opcode(0x0F, 0xB7);
8825   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8826   ins_pipe(ialu_reg);
8827 %}
8828 
8829 // And Register with Immediate 65535 and promote to long
8830 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8831 %{
8832   match(Set dst (ConvI2L (AndI src mask)));
8833 
8834   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8835   opcode(0x0F, 0xB7);
8836   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8837   ins_pipe(ialu_reg);
8838 %}
8839 
8840 // And Register with Immediate
8841 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8842 %{
8843   match(Set dst (AndI dst src));
8844   effect(KILL cr);
8845 
8846   format %{ "andl    $dst, $src\t# int" %}
8847   opcode(0x81, 0x04); /* Opcode 81 /4 */
8848   ins_encode(OpcSErm(dst, src), Con8or32(src));
8849   ins_pipe(ialu_reg);
8850 %}
8851 
8852 // And Register with Memory
8853 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8854 %{
8855   match(Set dst (AndI dst (LoadI src)));
8856   effect(KILL cr);
8857 
8858   ins_cost(125);
8859   format %{ "andl    $dst, $src\t# int" %}
8860   opcode(0x23);
8861   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8862   ins_pipe(ialu_reg_mem);
8863 %}
8864 
8865 // And Memory with Register
8866 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8867 %{
8868   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8869   effect(KILL cr);
8870 
8871   ins_cost(150);
8872   format %{ "andl    $dst, $src\t# int" %}
8873   opcode(0x21); /* Opcode 21 /r */
8874   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8875   ins_pipe(ialu_mem_reg);
8876 %}
8877 
8878 // And Memory with Immediate
8879 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8880 %{
8881   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8882   effect(KILL cr);
8883 
8884   ins_cost(125);
8885   format %{ "andl    $dst, $src\t# int" %}
8886   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8887   ins_encode(REX_mem(dst), OpcSE(src),
8888              RM_opc_mem(secondary, dst), Con8or32(src));
8889   ins_pipe(ialu_mem_imm);
8890 %}
8891 
8892 // Or Instructions
8893 // Or Register with Register
8894 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8895 %{
8896   match(Set dst (OrI dst src));
8897   effect(KILL cr);
8898 
8899   format %{ "orl     $dst, $src\t# int" %}
8900   opcode(0x0B);
8901   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8902   ins_pipe(ialu_reg_reg);
8903 %}
8904 
8905 // Or Register with Immediate
8906 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8907 %{
8908   match(Set dst (OrI dst src));
8909   effect(KILL cr);
8910 
8911   format %{ "orl     $dst, $src\t# int" %}
8912   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8913   ins_encode(OpcSErm(dst, src), Con8or32(src));
8914   ins_pipe(ialu_reg);
8915 %}
8916 
8917 // Or Register with Memory
8918 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8919 %{
8920   match(Set dst (OrI dst (LoadI src)));
8921   effect(KILL cr);
8922 
8923   ins_cost(125);
8924   format %{ "orl     $dst, $src\t# int" %}
8925   opcode(0x0B);
8926   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8927   ins_pipe(ialu_reg_mem);
8928 %}
8929 
8930 // Or Memory with Register
8931 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8932 %{
8933   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8934   effect(KILL cr);
8935 
8936   ins_cost(150);
8937   format %{ "orl     $dst, $src\t# int" %}
8938   opcode(0x09); /* Opcode 09 /r */
8939   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8940   ins_pipe(ialu_mem_reg);
8941 %}
8942 
8943 // Or Memory with Immediate
8944 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8945 %{
8946   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8947   effect(KILL cr);
8948 
8949   ins_cost(125);
8950   format %{ "orl     $dst, $src\t# int" %}
8951   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8952   ins_encode(REX_mem(dst), OpcSE(src),
8953              RM_opc_mem(secondary, dst), Con8or32(src));
8954   ins_pipe(ialu_mem_imm);
8955 %}
8956 
8957 // Xor Instructions
8958 // Xor Register with Register
8959 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8960 %{
8961   match(Set dst (XorI dst src));
8962   effect(KILL cr);
8963 
8964   format %{ "xorl    $dst, $src\t# int" %}
8965   opcode(0x33);
8966   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8967   ins_pipe(ialu_reg_reg);
8968 %}
8969 
8970 // Xor Register with Immediate -1
8971 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8972   match(Set dst (XorI dst imm));
8973 
8974   format %{ "not    $dst" %}
8975   ins_encode %{
8976      __ notl($dst$$Register);
8977   %}
8978   ins_pipe(ialu_reg);
8979 %}
8980 
8981 // Xor Register with Immediate
8982 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8983 %{
8984   match(Set dst (XorI dst src));
8985   effect(KILL cr);
8986 
8987   format %{ "xorl    $dst, $src\t# int" %}
8988   opcode(0x81, 0x06); /* Opcode 81 /6 id */
8989   ins_encode(OpcSErm(dst, src), Con8or32(src));
8990   ins_pipe(ialu_reg);
8991 %}
8992 
8993 // Xor Register with Memory
8994 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8995 %{
8996   match(Set dst (XorI dst (LoadI src)));
8997   effect(KILL cr);
8998 
8999   ins_cost(125);
9000   format %{ "xorl    $dst, $src\t# int" %}
9001   opcode(0x33);
9002   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9003   ins_pipe(ialu_reg_mem);
9004 %}
9005 
9006 // Xor Memory with Register
9007 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9008 %{
9009   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9010   effect(KILL cr);
9011 
9012   ins_cost(150);
9013   format %{ "xorl    $dst, $src\t# int" %}
9014   opcode(0x31); /* Opcode 31 /r */
9015   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9016   ins_pipe(ialu_mem_reg);
9017 %}
9018 
9019 // Xor Memory with Immediate
9020 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9021 %{
9022   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9023   effect(KILL cr);
9024 
9025   ins_cost(125);
9026   format %{ "xorl    $dst, $src\t# int" %}
9027   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9028   ins_encode(REX_mem(dst), OpcSE(src),
9029              RM_opc_mem(secondary, dst), Con8or32(src));
9030   ins_pipe(ialu_mem_imm);
9031 %}
9032 
9033 
9034 // Long Logical Instructions
9035 
9036 // And Instructions
9037 // And Register with Register
9038 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9039 %{
9040   match(Set dst (AndL dst src));
9041   effect(KILL cr);
9042 
9043   format %{ "andq    $dst, $src\t# long" %}
9044   opcode(0x23);
9045   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9046   ins_pipe(ialu_reg_reg);
9047 %}
9048 
9049 // And Register with Immediate 255
9050 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9051 %{
9052   match(Set dst (AndL dst src));
9053 
9054   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9055   opcode(0x0F, 0xB6);
9056   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9057   ins_pipe(ialu_reg);
9058 %}
9059 
9060 // And Register with Immediate 65535
9061 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9062 %{
9063   match(Set dst (AndL dst src));
9064 
9065   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9066   opcode(0x0F, 0xB7);
9067   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9068   ins_pipe(ialu_reg);
9069 %}
9070 
9071 // And Register with Immediate
9072 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9073 %{
9074   match(Set dst (AndL dst src));
9075   effect(KILL cr);
9076 
9077   format %{ "andq    $dst, $src\t# long" %}
9078   opcode(0x81, 0x04); /* Opcode 81 /4 */
9079   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9080   ins_pipe(ialu_reg);
9081 %}
9082 
9083 // And Register with Memory
9084 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9085 %{
9086   match(Set dst (AndL dst (LoadL src)));
9087   effect(KILL cr);
9088 
9089   ins_cost(125);
9090   format %{ "andq    $dst, $src\t# long" %}
9091   opcode(0x23);
9092   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9093   ins_pipe(ialu_reg_mem);
9094 %}
9095 
9096 // And Memory with Register
9097 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9098 %{
9099   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9100   effect(KILL cr);
9101 
9102   ins_cost(150);
9103   format %{ "andq    $dst, $src\t# long" %}
9104   opcode(0x21); /* Opcode 21 /r */
9105   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9106   ins_pipe(ialu_mem_reg);
9107 %}
9108 
9109 // And Memory with Immediate
9110 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9111 %{
9112   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9113   effect(KILL cr);
9114 
9115   ins_cost(125);
9116   format %{ "andq    $dst, $src\t# long" %}
9117   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9118   ins_encode(REX_mem_wide(dst), OpcSE(src),
9119              RM_opc_mem(secondary, dst), Con8or32(src));
9120   ins_pipe(ialu_mem_imm);
9121 %}
9122 
9123 // Or Instructions
9124 // Or Register with Register
9125 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9126 %{
9127   match(Set dst (OrL dst src));
9128   effect(KILL cr);
9129 
9130   format %{ "orq     $dst, $src\t# long" %}
9131   opcode(0x0B);
9132   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9133   ins_pipe(ialu_reg_reg);
9134 %}
9135 
9136 // Use any_RegP to match R15 (TLS register) without spilling.
9137 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9138   match(Set dst (OrL dst (CastP2X src)));
9139   effect(KILL cr);
9140 
9141   format %{ "orq     $dst, $src\t# long" %}
9142   opcode(0x0B);
9143   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9144   ins_pipe(ialu_reg_reg);
9145 %}
9146 
9147 
9148 // Or Register with Immediate
9149 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9150 %{
9151   match(Set dst (OrL dst src));
9152   effect(KILL cr);
9153 
9154   format %{ "orq     $dst, $src\t# long" %}
9155   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9156   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9157   ins_pipe(ialu_reg);
9158 %}
9159 
9160 // Or Register with Memory
9161 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9162 %{
9163   match(Set dst (OrL dst (LoadL src)));
9164   effect(KILL cr);
9165 
9166   ins_cost(125);
9167   format %{ "orq     $dst, $src\t# long" %}
9168   opcode(0x0B);
9169   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9170   ins_pipe(ialu_reg_mem);
9171 %}
9172 
9173 // Or Memory with Register
9174 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9175 %{
9176   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9177   effect(KILL cr);
9178 
9179   ins_cost(150);
9180   format %{ "orq     $dst, $src\t# long" %}
9181   opcode(0x09); /* Opcode 09 /r */
9182   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9183   ins_pipe(ialu_mem_reg);
9184 %}
9185 
9186 // Or Memory with Immediate
9187 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9188 %{
9189   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9190   effect(KILL cr);
9191 
9192   ins_cost(125);
9193   format %{ "orq     $dst, $src\t# long" %}
9194   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9195   ins_encode(REX_mem_wide(dst), OpcSE(src),
9196              RM_opc_mem(secondary, dst), Con8or32(src));
9197   ins_pipe(ialu_mem_imm);
9198 %}
9199 
9200 // Xor Instructions
9201 // Xor Register with Register
9202 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9203 %{
9204   match(Set dst (XorL dst src));
9205   effect(KILL cr);
9206 
9207   format %{ "xorq    $dst, $src\t# long" %}
9208   opcode(0x33);
9209   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9210   ins_pipe(ialu_reg_reg);
9211 %}
9212 
9213 // Xor Register with Immediate -1
9214 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9215   match(Set dst (XorL dst imm));
9216 
9217   format %{ "notq   $dst" %}
9218   ins_encode %{
9219      __ notq($dst$$Register);
9220   %}
9221   ins_pipe(ialu_reg);
9222 %}
9223 
9224 // Xor Register with Immediate
9225 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9226 %{
9227   match(Set dst (XorL dst src));
9228   effect(KILL cr);
9229 
9230   format %{ "xorq    $dst, $src\t# long" %}
9231   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9232   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9233   ins_pipe(ialu_reg);
9234 %}
9235 
9236 // Xor Register with Memory
9237 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9238 %{
9239   match(Set dst (XorL dst (LoadL src)));
9240   effect(KILL cr);
9241 
9242   ins_cost(125);
9243   format %{ "xorq    $dst, $src\t# long" %}
9244   opcode(0x33);
9245   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9246   ins_pipe(ialu_reg_mem);
9247 %}
9248 
9249 // Xor Memory with Register
9250 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9251 %{
9252   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9253   effect(KILL cr);
9254 
9255   ins_cost(150);
9256   format %{ "xorq    $dst, $src\t# long" %}
9257   opcode(0x31); /* Opcode 31 /r */
9258   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9259   ins_pipe(ialu_mem_reg);
9260 %}
9261 
9262 // Xor Memory with Immediate
9263 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9264 %{
9265   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9266   effect(KILL cr);
9267 
9268   ins_cost(125);
9269   format %{ "xorq    $dst, $src\t# long" %}
9270   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9271   ins_encode(REX_mem_wide(dst), OpcSE(src),
9272              RM_opc_mem(secondary, dst), Con8or32(src));
9273   ins_pipe(ialu_mem_imm);
9274 %}
9275 
9276 // Convert Int to Boolean
9277 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9278 %{
9279   match(Set dst (Conv2B src));
9280   effect(KILL cr);
9281 
9282   format %{ "testl   $src, $src\t# ci2b\n\t"
9283             "setnz   $dst\n\t"
9284             "movzbl  $dst, $dst" %}
9285   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9286              setNZ_reg(dst),
9287              REX_reg_breg(dst, dst), // movzbl
9288              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9289   ins_pipe(pipe_slow); // XXX
9290 %}
9291 
9292 // Convert Pointer to Boolean
9293 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9294 %{
9295   match(Set dst (Conv2B src));
9296   effect(KILL cr);
9297 
9298   format %{ "testq   $src, $src\t# cp2b\n\t"
9299             "setnz   $dst\n\t"
9300             "movzbl  $dst, $dst" %}
9301   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9302              setNZ_reg(dst),
9303              REX_reg_breg(dst, dst), // movzbl
9304              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9305   ins_pipe(pipe_slow); // XXX
9306 %}
9307 
9308 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9309 %{
9310   match(Set dst (CmpLTMask p q));
9311   effect(KILL cr);
9312 
9313   ins_cost(400);
9314   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9315             "setlt   $dst\n\t"
9316             "movzbl  $dst, $dst\n\t"
9317             "negl    $dst" %}
9318   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9319              setLT_reg(dst),
9320              REX_reg_breg(dst, dst), // movzbl
9321              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9322              neg_reg(dst));
9323   ins_pipe(pipe_slow);
9324 %}
9325 
9326 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9327 %{
9328   match(Set dst (CmpLTMask dst zero));
9329   effect(KILL cr);
9330 
9331   ins_cost(100);
9332   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9333   ins_encode %{
9334   __ sarl($dst$$Register, 31);
9335   %}
9336   ins_pipe(ialu_reg);
9337 %}
9338 
9339 /* Better to save a register than avoid a branch */
9340 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9341 %{
9342   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9343   effect(KILL cr);
9344   ins_cost(300);
9345   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9346             "jge    done\n\t"
9347             "addl   $p,$y\n"
9348             "done:  " %}
9349   ins_encode %{
9350     Register Rp = $p$$Register;
9351     Register Rq = $q$$Register;
9352     Register Ry = $y$$Register;
9353     Label done;
9354     __ subl(Rp, Rq);
9355     __ jccb(Assembler::greaterEqual, done);
9356     __ addl(Rp, Ry);
9357     __ bind(done);
9358   %}
9359   ins_pipe(pipe_cmplt);
9360 %}
9361 
9362 /* Better to save a register than avoid a branch */
9363 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9364 %{
9365   match(Set y (AndI (CmpLTMask p q) y));
9366   effect(KILL cr);
9367 
9368   ins_cost(300);
9369 
9370   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9371             "jlt      done\n\t"
9372             "xorl     $y, $y\n"
9373             "done:  " %}
9374   ins_encode %{
9375     Register Rp = $p$$Register;
9376     Register Rq = $q$$Register;
9377     Register Ry = $y$$Register;
9378     Label done;
9379     __ cmpl(Rp, Rq);
9380     __ jccb(Assembler::less, done);
9381     __ xorl(Ry, Ry);
9382     __ bind(done);
9383   %}
9384   ins_pipe(pipe_cmplt);
9385 %}
9386 
9387 
9388 //---------- FP Instructions------------------------------------------------
9389 
9390 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9391 %{
9392   match(Set cr (CmpF src1 src2));
9393 
9394   ins_cost(145);
9395   format %{ "ucomiss $src1, $src2\n\t"
9396             "jnp,s   exit\n\t"
9397             "pushfq\t# saw NaN, set CF\n\t"
9398             "andq    [rsp], #0xffffff2b\n\t"
9399             "popfq\n"
9400     "exit:" %}
9401   ins_encode %{
9402     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9403     emit_cmpfp_fixup(_masm);
9404   %}
9405   ins_pipe(pipe_slow);
9406 %}
9407 
9408 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9409   match(Set cr (CmpF src1 src2));
9410 
9411   ins_cost(100);
9412   format %{ "ucomiss $src1, $src2" %}
9413   ins_encode %{
9414     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9415   %}
9416   ins_pipe(pipe_slow);
9417 %}
9418 
9419 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9420 %{
9421   match(Set cr (CmpF src1 (LoadF src2)));
9422 
9423   ins_cost(145);
9424   format %{ "ucomiss $src1, $src2\n\t"
9425             "jnp,s   exit\n\t"
9426             "pushfq\t# saw NaN, set CF\n\t"
9427             "andq    [rsp], #0xffffff2b\n\t"
9428             "popfq\n"
9429     "exit:" %}
9430   ins_encode %{
9431     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9432     emit_cmpfp_fixup(_masm);
9433   %}
9434   ins_pipe(pipe_slow);
9435 %}
9436 
9437 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9438   match(Set cr (CmpF src1 (LoadF src2)));
9439 
9440   ins_cost(100);
9441   format %{ "ucomiss $src1, $src2" %}
9442   ins_encode %{
9443     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9444   %}
9445   ins_pipe(pipe_slow);
9446 %}
9447 
9448 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9449   match(Set cr (CmpF src con));
9450 
9451   ins_cost(145);
9452   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9453             "jnp,s   exit\n\t"
9454             "pushfq\t# saw NaN, set CF\n\t"
9455             "andq    [rsp], #0xffffff2b\n\t"
9456             "popfq\n"
9457     "exit:" %}
9458   ins_encode %{
9459     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9460     emit_cmpfp_fixup(_masm);
9461   %}
9462   ins_pipe(pipe_slow);
9463 %}
9464 
9465 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9466   match(Set cr (CmpF src con));
9467   ins_cost(100);
9468   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9469   ins_encode %{
9470     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9471   %}
9472   ins_pipe(pipe_slow);
9473 %}
9474 
9475 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9476 %{
9477   match(Set cr (CmpD src1 src2));
9478 
9479   ins_cost(145);
9480   format %{ "ucomisd $src1, $src2\n\t"
9481             "jnp,s   exit\n\t"
9482             "pushfq\t# saw NaN, set CF\n\t"
9483             "andq    [rsp], #0xffffff2b\n\t"
9484             "popfq\n"
9485     "exit:" %}
9486   ins_encode %{
9487     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9488     emit_cmpfp_fixup(_masm);
9489   %}
9490   ins_pipe(pipe_slow);
9491 %}
9492 
9493 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9494   match(Set cr (CmpD src1 src2));
9495 
9496   ins_cost(100);
9497   format %{ "ucomisd $src1, $src2 test" %}
9498   ins_encode %{
9499     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9500   %}
9501   ins_pipe(pipe_slow);
9502 %}
9503 
9504 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9505 %{
9506   match(Set cr (CmpD src1 (LoadD src2)));
9507 
9508   ins_cost(145);
9509   format %{ "ucomisd $src1, $src2\n\t"
9510             "jnp,s   exit\n\t"
9511             "pushfq\t# saw NaN, set CF\n\t"
9512             "andq    [rsp], #0xffffff2b\n\t"
9513             "popfq\n"
9514     "exit:" %}
9515   ins_encode %{
9516     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9517     emit_cmpfp_fixup(_masm);
9518   %}
9519   ins_pipe(pipe_slow);
9520 %}
9521 
9522 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9523   match(Set cr (CmpD src1 (LoadD src2)));
9524 
9525   ins_cost(100);
9526   format %{ "ucomisd $src1, $src2" %}
9527   ins_encode %{
9528     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9529   %}
9530   ins_pipe(pipe_slow);
9531 %}
9532 
9533 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9534   match(Set cr (CmpD src con));
9535 
9536   ins_cost(145);
9537   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9538             "jnp,s   exit\n\t"
9539             "pushfq\t# saw NaN, set CF\n\t"
9540             "andq    [rsp], #0xffffff2b\n\t"
9541             "popfq\n"
9542     "exit:" %}
9543   ins_encode %{
9544     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9545     emit_cmpfp_fixup(_masm);
9546   %}
9547   ins_pipe(pipe_slow);
9548 %}
9549 
9550 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9551   match(Set cr (CmpD src con));
9552   ins_cost(100);
9553   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9554   ins_encode %{
9555     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9556   %}
9557   ins_pipe(pipe_slow);
9558 %}
9559 
9560 // Compare into -1,0,1
9561 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9562 %{
9563   match(Set dst (CmpF3 src1 src2));
9564   effect(KILL cr);
9565 
9566   ins_cost(275);
9567   format %{ "ucomiss $src1, $src2\n\t"
9568             "movl    $dst, #-1\n\t"
9569             "jp,s    done\n\t"
9570             "jb,s    done\n\t"
9571             "setne   $dst\n\t"
9572             "movzbl  $dst, $dst\n"
9573     "done:" %}
9574   ins_encode %{
9575     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9576     emit_cmpfp3(_masm, $dst$$Register);
9577   %}
9578   ins_pipe(pipe_slow);
9579 %}
9580 
9581 // Compare into -1,0,1
9582 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9583 %{
9584   match(Set dst (CmpF3 src1 (LoadF src2)));
9585   effect(KILL cr);
9586 
9587   ins_cost(275);
9588   format %{ "ucomiss $src1, $src2\n\t"
9589             "movl    $dst, #-1\n\t"
9590             "jp,s    done\n\t"
9591             "jb,s    done\n\t"
9592             "setne   $dst\n\t"
9593             "movzbl  $dst, $dst\n"
9594     "done:" %}
9595   ins_encode %{
9596     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9597     emit_cmpfp3(_masm, $dst$$Register);
9598   %}
9599   ins_pipe(pipe_slow);
9600 %}
9601 
9602 // Compare into -1,0,1
9603 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9604   match(Set dst (CmpF3 src con));
9605   effect(KILL cr);
9606 
9607   ins_cost(275);
9608   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9609             "movl    $dst, #-1\n\t"
9610             "jp,s    done\n\t"
9611             "jb,s    done\n\t"
9612             "setne   $dst\n\t"
9613             "movzbl  $dst, $dst\n"
9614     "done:" %}
9615   ins_encode %{
9616     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9617     emit_cmpfp3(_masm, $dst$$Register);
9618   %}
9619   ins_pipe(pipe_slow);
9620 %}
9621 
9622 // Compare into -1,0,1
9623 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9624 %{
9625   match(Set dst (CmpD3 src1 src2));
9626   effect(KILL cr);
9627 
9628   ins_cost(275);
9629   format %{ "ucomisd $src1, $src2\n\t"
9630             "movl    $dst, #-1\n\t"
9631             "jp,s    done\n\t"
9632             "jb,s    done\n\t"
9633             "setne   $dst\n\t"
9634             "movzbl  $dst, $dst\n"
9635     "done:" %}
9636   ins_encode %{
9637     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9638     emit_cmpfp3(_masm, $dst$$Register);
9639   %}
9640   ins_pipe(pipe_slow);
9641 %}
9642 
9643 // Compare into -1,0,1
9644 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9645 %{
9646   match(Set dst (CmpD3 src1 (LoadD src2)));
9647   effect(KILL cr);
9648 
9649   ins_cost(275);
9650   format %{ "ucomisd $src1, $src2\n\t"
9651             "movl    $dst, #-1\n\t"
9652             "jp,s    done\n\t"
9653             "jb,s    done\n\t"
9654             "setne   $dst\n\t"
9655             "movzbl  $dst, $dst\n"
9656     "done:" %}
9657   ins_encode %{
9658     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9659     emit_cmpfp3(_masm, $dst$$Register);
9660   %}
9661   ins_pipe(pipe_slow);
9662 %}
9663 
9664 // Compare into -1,0,1
9665 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9666   match(Set dst (CmpD3 src con));
9667   effect(KILL cr);
9668 
9669   ins_cost(275);
9670   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9671             "movl    $dst, #-1\n\t"
9672             "jp,s    done\n\t"
9673             "jb,s    done\n\t"
9674             "setne   $dst\n\t"
9675             "movzbl  $dst, $dst\n"
9676     "done:" %}
9677   ins_encode %{
9678     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9679     emit_cmpfp3(_masm, $dst$$Register);
9680   %}
9681   ins_pipe(pipe_slow);
9682 %}
9683 
9684 // -----------Trig and Trancendental Instructions------------------------------
9685 instruct cosD_reg(regD dst) %{
9686   match(Set dst (CosD dst));
9687 
9688   format %{ "dcos   $dst\n\t" %}
9689   opcode(0xD9, 0xFF);
9690   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9691   ins_pipe( pipe_slow );
9692 %}
9693 
9694 instruct sinD_reg(regD dst) %{
9695   match(Set dst (SinD dst));
9696 
9697   format %{ "dsin   $dst\n\t" %}
9698   opcode(0xD9, 0xFE);
9699   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9700   ins_pipe( pipe_slow );
9701 %}
9702 
9703 instruct tanD_reg(regD dst) %{
9704   match(Set dst (TanD dst));
9705 
9706   format %{ "dtan   $dst\n\t" %}
9707   ins_encode( Push_SrcXD(dst),
9708               Opcode(0xD9), Opcode(0xF2),   //fptan
9709               Opcode(0xDD), Opcode(0xD8),   //fstp st
9710               Push_ResultXD(dst) );
9711   ins_pipe( pipe_slow );
9712 %}
9713 
9714 instruct log10D_reg(regD dst) %{
9715   // The source and result Double operands in XMM registers
9716   match(Set dst (Log10D dst));
9717   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9718   // fyl2x        ; compute log_10(2) * log_2(x)
9719   format %{ "fldlg2\t\t\t#Log10\n\t"
9720             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9721          %}
9722    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9723               Push_SrcXD(dst),
9724               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9725               Push_ResultXD(dst));
9726 
9727   ins_pipe( pipe_slow );
9728 %}
9729 
9730 instruct logD_reg(regD dst) %{
9731   // The source and result Double operands in XMM registers
9732   match(Set dst (LogD dst));
9733   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9734   // fyl2x        ; compute log_e(2) * log_2(x)
9735   format %{ "fldln2\t\t\t#Log_e\n\t"
9736             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9737          %}
9738   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9739               Push_SrcXD(dst),
9740               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9741               Push_ResultXD(dst));
9742   ins_pipe( pipe_slow );
9743 %}
9744 
9745 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9746   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9747   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9748   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9749   ins_encode %{
9750     __ subptr(rsp, 8);
9751     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9752     __ fld_d(Address(rsp, 0));
9753     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9754     __ fld_d(Address(rsp, 0));
9755     __ fast_pow();
9756     __ fstp_d(Address(rsp, 0));
9757     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9758     __ addptr(rsp, 8);
9759   %}
9760   ins_pipe( pipe_slow );
9761 %}
9762 
9763 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9764   match(Set dst (ExpD src));
9765   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9766   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9767   ins_encode %{
9768     __ subptr(rsp, 8);
9769     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9770     __ fld_d(Address(rsp, 0));
9771     __ fast_exp();
9772     __ fstp_d(Address(rsp, 0));
9773     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9774     __ addptr(rsp, 8);
9775   %}
9776   ins_pipe( pipe_slow );
9777 %}
9778 
9779 //----------Arithmetic Conversion Instructions---------------------------------
9780 
9781 instruct roundFloat_nop(regF dst)
9782 %{
9783   match(Set dst (RoundFloat dst));
9784 
9785   ins_cost(0);
9786   ins_encode();
9787   ins_pipe(empty);
9788 %}
9789 
9790 instruct roundDouble_nop(regD dst)
9791 %{
9792   match(Set dst (RoundDouble dst));
9793 
9794   ins_cost(0);
9795   ins_encode();
9796   ins_pipe(empty);
9797 %}
9798 
9799 instruct convF2D_reg_reg(regD dst, regF src)
9800 %{
9801   match(Set dst (ConvF2D src));
9802 
9803   format %{ "cvtss2sd $dst, $src" %}
9804   ins_encode %{
9805     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9806   %}
9807   ins_pipe(pipe_slow); // XXX
9808 %}
9809 
9810 instruct convF2D_reg_mem(regD dst, memory src)
9811 %{
9812   match(Set dst (ConvF2D (LoadF src)));
9813 
9814   format %{ "cvtss2sd $dst, $src" %}
9815   ins_encode %{
9816     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9817   %}
9818   ins_pipe(pipe_slow); // XXX
9819 %}
9820 
9821 instruct convD2F_reg_reg(regF dst, regD src)
9822 %{
9823   match(Set dst (ConvD2F src));
9824 
9825   format %{ "cvtsd2ss $dst, $src" %}
9826   ins_encode %{
9827     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9828   %}
9829   ins_pipe(pipe_slow); // XXX
9830 %}
9831 
9832 instruct convD2F_reg_mem(regF dst, memory src)
9833 %{
9834   match(Set dst (ConvD2F (LoadD src)));
9835 
9836   format %{ "cvtsd2ss $dst, $src" %}
9837   ins_encode %{
9838     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9839   %}
9840   ins_pipe(pipe_slow); // XXX
9841 %}
9842 
9843 // XXX do mem variants
9844 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9845 %{
9846   match(Set dst (ConvF2I src));
9847   effect(KILL cr);
9848 
9849   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9850             "cmpl    $dst, #0x80000000\n\t"
9851             "jne,s   done\n\t"
9852             "subq    rsp, #8\n\t"
9853             "movss   [rsp], $src\n\t"
9854             "call    f2i_fixup\n\t"
9855             "popq    $dst\n"
9856     "done:   "%}
9857   ins_encode %{
9858     Label done;
9859     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9860     __ cmpl($dst$$Register, 0x80000000);
9861     __ jccb(Assembler::notEqual, done);
9862     __ subptr(rsp, 8);
9863     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9864     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9865     __ pop($dst$$Register);
9866     __ bind(done);
9867   %}
9868   ins_pipe(pipe_slow);
9869 %}
9870 
9871 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9872 %{
9873   match(Set dst (ConvF2L src));
9874   effect(KILL cr);
9875 
9876   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9877             "cmpq    $dst, [0x8000000000000000]\n\t"
9878             "jne,s   done\n\t"
9879             "subq    rsp, #8\n\t"
9880             "movss   [rsp], $src\n\t"
9881             "call    f2l_fixup\n\t"
9882             "popq    $dst\n"
9883     "done:   "%}
9884   ins_encode %{
9885     Label done;
9886     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9887     __ cmp64($dst$$Register,
9888              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9889     __ jccb(Assembler::notEqual, done);
9890     __ subptr(rsp, 8);
9891     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9892     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9893     __ pop($dst$$Register);
9894     __ bind(done);
9895   %}
9896   ins_pipe(pipe_slow);
9897 %}
9898 
9899 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9900 %{
9901   match(Set dst (ConvD2I src));
9902   effect(KILL cr);
9903 
9904   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9905             "cmpl    $dst, #0x80000000\n\t"
9906             "jne,s   done\n\t"
9907             "subq    rsp, #8\n\t"
9908             "movsd   [rsp], $src\n\t"
9909             "call    d2i_fixup\n\t"
9910             "popq    $dst\n"
9911     "done:   "%}
9912   ins_encode %{
9913     Label done;
9914     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
9915     __ cmpl($dst$$Register, 0x80000000);
9916     __ jccb(Assembler::notEqual, done);
9917     __ subptr(rsp, 8);
9918     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9919     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9920     __ pop($dst$$Register);
9921     __ bind(done);
9922   %}
9923   ins_pipe(pipe_slow);
9924 %}
9925 
9926 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
9927 %{
9928   match(Set dst (ConvD2L src));
9929   effect(KILL cr);
9930 
9931   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
9932             "cmpq    $dst, [0x8000000000000000]\n\t"
9933             "jne,s   done\n\t"
9934             "subq    rsp, #8\n\t"
9935             "movsd   [rsp], $src\n\t"
9936             "call    d2l_fixup\n\t"
9937             "popq    $dst\n"
9938     "done:   "%}
9939   ins_encode %{
9940     Label done;
9941     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
9942     __ cmp64($dst$$Register,
9943              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9944     __ jccb(Assembler::notEqual, done);
9945     __ subptr(rsp, 8);
9946     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9947     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9948     __ pop($dst$$Register);
9949     __ bind(done);
9950   %}
9951   ins_pipe(pipe_slow);
9952 %}
9953 
9954 instruct convI2F_reg_reg(regF dst, rRegI src)
9955 %{
9956   predicate(!UseXmmI2F);
9957   match(Set dst (ConvI2F src));
9958 
9959   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9960   ins_encode %{
9961     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
9962   %}
9963   ins_pipe(pipe_slow); // XXX
9964 %}
9965 
9966 instruct convI2F_reg_mem(regF dst, memory src)
9967 %{
9968   match(Set dst (ConvI2F (LoadI src)));
9969 
9970   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9971   ins_encode %{
9972     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
9973   %}
9974   ins_pipe(pipe_slow); // XXX
9975 %}
9976 
9977 instruct convI2D_reg_reg(regD dst, rRegI src)
9978 %{
9979   predicate(!UseXmmI2D);
9980   match(Set dst (ConvI2D src));
9981 
9982   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9983   ins_encode %{
9984     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
9985   %}
9986   ins_pipe(pipe_slow); // XXX
9987 %}
9988 
9989 instruct convI2D_reg_mem(regD dst, memory src)
9990 %{
9991   match(Set dst (ConvI2D (LoadI src)));
9992 
9993   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9994   ins_encode %{
9995     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
9996   %}
9997   ins_pipe(pipe_slow); // XXX
9998 %}
9999 
10000 instruct convXI2F_reg(regF dst, rRegI src)
10001 %{
10002   predicate(UseXmmI2F);
10003   match(Set dst (ConvI2F src));
10004 
10005   format %{ "movdl $dst, $src\n\t"
10006             "cvtdq2psl $dst, $dst\t# i2f" %}
10007   ins_encode %{
10008     __ movdl($dst$$XMMRegister, $src$$Register);
10009     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10010   %}
10011   ins_pipe(pipe_slow); // XXX
10012 %}
10013 
10014 instruct convXI2D_reg(regD dst, rRegI src)
10015 %{
10016   predicate(UseXmmI2D);
10017   match(Set dst (ConvI2D src));
10018 
10019   format %{ "movdl $dst, $src\n\t"
10020             "cvtdq2pdl $dst, $dst\t# i2d" %}
10021   ins_encode %{
10022     __ movdl($dst$$XMMRegister, $src$$Register);
10023     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10024   %}
10025   ins_pipe(pipe_slow); // XXX
10026 %}
10027 
10028 instruct convL2F_reg_reg(regF dst, rRegL src)
10029 %{
10030   match(Set dst (ConvL2F src));
10031 
10032   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10033   ins_encode %{
10034     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10035   %}
10036   ins_pipe(pipe_slow); // XXX
10037 %}
10038 
10039 instruct convL2F_reg_mem(regF dst, memory src)
10040 %{
10041   match(Set dst (ConvL2F (LoadL src)));
10042 
10043   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10044   ins_encode %{
10045     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10046   %}
10047   ins_pipe(pipe_slow); // XXX
10048 %}
10049 
10050 instruct convL2D_reg_reg(regD dst, rRegL src)
10051 %{
10052   match(Set dst (ConvL2D src));
10053 
10054   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10055   ins_encode %{
10056     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10057   %}
10058   ins_pipe(pipe_slow); // XXX
10059 %}
10060 
10061 instruct convL2D_reg_mem(regD dst, memory src)
10062 %{
10063   match(Set dst (ConvL2D (LoadL src)));
10064 
10065   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10066   ins_encode %{
10067     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10068   %}
10069   ins_pipe(pipe_slow); // XXX
10070 %}
10071 
10072 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10073 %{
10074   match(Set dst (ConvI2L src));
10075 
10076   ins_cost(125);
10077   format %{ "movslq  $dst, $src\t# i2l" %}
10078   ins_encode %{
10079     __ movslq($dst$$Register, $src$$Register);
10080   %}
10081   ins_pipe(ialu_reg_reg);
10082 %}
10083 
10084 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10085 // %{
10086 //   match(Set dst (ConvI2L src));
10087 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10088 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10089 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10090 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10091 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10092 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10093 
10094 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10095 //   ins_encode(enc_copy(dst, src));
10096 // //   opcode(0x63); // needs REX.W
10097 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10098 //   ins_pipe(ialu_reg_reg);
10099 // %}
10100 
10101 // Zero-extend convert int to long
10102 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10103 %{
10104   match(Set dst (AndL (ConvI2L src) mask));
10105 
10106   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10107   ins_encode %{
10108     if ($dst$$reg != $src$$reg) {
10109       __ movl($dst$$Register, $src$$Register);
10110     }
10111   %}
10112   ins_pipe(ialu_reg_reg);
10113 %}
10114 
10115 // Zero-extend convert int to long
10116 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10117 %{
10118   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10119 
10120   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10121   ins_encode %{
10122     __ movl($dst$$Register, $src$$Address);
10123   %}
10124   ins_pipe(ialu_reg_mem);
10125 %}
10126 
10127 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10128 %{
10129   match(Set dst (AndL src mask));
10130 
10131   format %{ "movl    $dst, $src\t# zero-extend long" %}
10132   ins_encode %{
10133     __ movl($dst$$Register, $src$$Register);
10134   %}
10135   ins_pipe(ialu_reg_reg);
10136 %}
10137 
10138 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10139 %{
10140   match(Set dst (ConvL2I src));
10141 
10142   format %{ "movl    $dst, $src\t# l2i" %}
10143   ins_encode %{
10144     __ movl($dst$$Register, $src$$Register);
10145   %}
10146   ins_pipe(ialu_reg_reg);
10147 %}
10148 
10149 
10150 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10151   match(Set dst (MoveF2I src));
10152   effect(DEF dst, USE src);
10153 
10154   ins_cost(125);
10155   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10156   ins_encode %{
10157     __ movl($dst$$Register, Address(rsp, $src$$disp));
10158   %}
10159   ins_pipe(ialu_reg_mem);
10160 %}
10161 
10162 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10163   match(Set dst (MoveI2F src));
10164   effect(DEF dst, USE src);
10165 
10166   ins_cost(125);
10167   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10168   ins_encode %{
10169     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10170   %}
10171   ins_pipe(pipe_slow);
10172 %}
10173 
10174 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10175   match(Set dst (MoveD2L src));
10176   effect(DEF dst, USE src);
10177 
10178   ins_cost(125);
10179   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10180   ins_encode %{
10181     __ movq($dst$$Register, Address(rsp, $src$$disp));
10182   %}
10183   ins_pipe(ialu_reg_mem);
10184 %}
10185 
10186 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10187   predicate(!UseXmmLoadAndClearUpper);
10188   match(Set dst (MoveL2D src));
10189   effect(DEF dst, USE src);
10190 
10191   ins_cost(125);
10192   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10193   ins_encode %{
10194     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10195   %}
10196   ins_pipe(pipe_slow);
10197 %}
10198 
10199 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10200   predicate(UseXmmLoadAndClearUpper);
10201   match(Set dst (MoveL2D src));
10202   effect(DEF dst, USE src);
10203 
10204   ins_cost(125);
10205   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10206   ins_encode %{
10207     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10208   %}
10209   ins_pipe(pipe_slow);
10210 %}
10211 
10212 
10213 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10214   match(Set dst (MoveF2I src));
10215   effect(DEF dst, USE src);
10216 
10217   ins_cost(95); // XXX
10218   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10219   ins_encode %{
10220     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10221   %}
10222   ins_pipe(pipe_slow);
10223 %}
10224 
10225 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10226   match(Set dst (MoveI2F src));
10227   effect(DEF dst, USE src);
10228 
10229   ins_cost(100);
10230   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10231   ins_encode %{
10232     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10233   %}
10234   ins_pipe( ialu_mem_reg );
10235 %}
10236 
10237 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10238   match(Set dst (MoveD2L src));
10239   effect(DEF dst, USE src);
10240 
10241   ins_cost(95); // XXX
10242   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10243   ins_encode %{
10244     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10245   %}
10246   ins_pipe(pipe_slow);
10247 %}
10248 
10249 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10250   match(Set dst (MoveL2D src));
10251   effect(DEF dst, USE src);
10252 
10253   ins_cost(100);
10254   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10255   ins_encode %{
10256     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10257   %}
10258   ins_pipe(ialu_mem_reg);
10259 %}
10260 
10261 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10262   match(Set dst (MoveF2I src));
10263   effect(DEF dst, USE src);
10264   ins_cost(85);
10265   format %{ "movd    $dst,$src\t# MoveF2I" %}
10266   ins_encode %{
10267     __ movdl($dst$$Register, $src$$XMMRegister);
10268   %}
10269   ins_pipe( pipe_slow );
10270 %}
10271 
10272 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10273   match(Set dst (MoveD2L src));
10274   effect(DEF dst, USE src);
10275   ins_cost(85);
10276   format %{ "movd    $dst,$src\t# MoveD2L" %}
10277   ins_encode %{
10278     __ movdq($dst$$Register, $src$$XMMRegister);
10279   %}
10280   ins_pipe( pipe_slow );
10281 %}
10282 
10283 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10284   match(Set dst (MoveI2F src));
10285   effect(DEF dst, USE src);
10286   ins_cost(100);
10287   format %{ "movd    $dst,$src\t# MoveI2F" %}
10288   ins_encode %{
10289     __ movdl($dst$$XMMRegister, $src$$Register);
10290   %}
10291   ins_pipe( pipe_slow );
10292 %}
10293 
10294 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10295   match(Set dst (MoveL2D src));
10296   effect(DEF dst, USE src);
10297   ins_cost(100);
10298   format %{ "movd    $dst,$src\t# MoveL2D" %}
10299   ins_encode %{
10300      __ movdq($dst$$XMMRegister, $src$$Register);
10301   %}
10302   ins_pipe( pipe_slow );
10303 %}
10304 
10305 
10306 // =======================================================================
10307 // fast clearing of an array
10308 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10309                   rFlagsReg cr)
10310 %{
10311   predicate(!UseFastStosb);
10312   match(Set dummy (ClearArray cnt base));
10313   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10314 
10315   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10316             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10317   ins_encode %{ 
10318     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10319   %}
10320   ins_pipe(pipe_slow);
10321 %}
10322 
10323 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10324                         rFlagsReg cr)
10325 %{
10326   predicate(UseFastStosb);
10327   match(Set dummy (ClearArray cnt base));
10328   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10329   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10330             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10331             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10332   ins_encode %{ 
10333     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10334   %}
10335   ins_pipe( pipe_slow );
10336 %}
10337 
10338 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10339                         rax_RegI result, regD tmp1, rFlagsReg cr)
10340 %{
10341   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10342   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10343 
10344   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10345   ins_encode %{
10346     __ string_compare($str1$$Register, $str2$$Register,
10347                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10348                       $tmp1$$XMMRegister);
10349   %}
10350   ins_pipe( pipe_slow );
10351 %}
10352 
10353 // fast search of substring with known size.
10354 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10355                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10356 %{
10357   predicate(UseSSE42Intrinsics);
10358   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10359   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10360 
10361   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10362   ins_encode %{
10363     int icnt2 = (int)$int_cnt2$$constant;
10364     if (icnt2 >= 8) {
10365       // IndexOf for constant substrings with size >= 8 elements
10366       // which don't need to be loaded through stack.
10367       __ string_indexofC8($str1$$Register, $str2$$Register,
10368                           $cnt1$$Register, $cnt2$$Register,
10369                           icnt2, $result$$Register,
10370                           $vec$$XMMRegister, $tmp$$Register);
10371     } else {
10372       // Small strings are loaded through stack if they cross page boundary.
10373       __ string_indexof($str1$$Register, $str2$$Register,
10374                         $cnt1$$Register, $cnt2$$Register,
10375                         icnt2, $result$$Register,
10376                         $vec$$XMMRegister, $tmp$$Register);
10377     }
10378   %}
10379   ins_pipe( pipe_slow );
10380 %}
10381 
10382 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10383                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10384 %{
10385   predicate(UseSSE42Intrinsics);
10386   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10387   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10388 
10389   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10390   ins_encode %{
10391     __ string_indexof($str1$$Register, $str2$$Register,
10392                       $cnt1$$Register, $cnt2$$Register,
10393                       (-1), $result$$Register,
10394                       $vec$$XMMRegister, $tmp$$Register);
10395   %}
10396   ins_pipe( pipe_slow );
10397 %}
10398 
10399 // fast string equals
10400 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10401                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10402 %{
10403   match(Set result (StrEquals (Binary str1 str2) cnt));
10404   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10405 
10406   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10407   ins_encode %{
10408     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10409                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10410                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10411   %}
10412   ins_pipe( pipe_slow );
10413 %}
10414 
10415 // fast array equals
10416 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10417                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10418 %{
10419   match(Set result (AryEq ary1 ary2));
10420   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10421   //ins_cost(300);
10422 
10423   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10424   ins_encode %{
10425     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10426                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10427                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10428   %}
10429   ins_pipe( pipe_slow );
10430 %}
10431 
10432 // encode char[] to byte[] in ISO_8859_1
10433 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10434                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10435                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10436   match(Set result (EncodeISOArray src (Binary dst len)));
10437   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10438 
10439   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10440   ins_encode %{
10441     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10442                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10443                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10444   %}
10445   ins_pipe( pipe_slow );
10446 %}
10447 
10448 
10449 //----------Control Flow Instructions------------------------------------------
10450 // Signed compare Instructions
10451 
10452 // XXX more variants!!
10453 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10454 %{
10455   match(Set cr (CmpI op1 op2));
10456   effect(DEF cr, USE op1, USE op2);
10457 
10458   format %{ "cmpl    $op1, $op2" %}
10459   opcode(0x3B);  /* Opcode 3B /r */
10460   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10461   ins_pipe(ialu_cr_reg_reg);
10462 %}
10463 
10464 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10465 %{
10466   match(Set cr (CmpI op1 op2));
10467 
10468   format %{ "cmpl    $op1, $op2" %}
10469   opcode(0x81, 0x07); /* Opcode 81 /7 */
10470   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10471   ins_pipe(ialu_cr_reg_imm);
10472 %}
10473 
10474 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10475 %{
10476   match(Set cr (CmpI op1 (LoadI op2)));
10477 
10478   ins_cost(500); // XXX
10479   format %{ "cmpl    $op1, $op2" %}
10480   opcode(0x3B); /* Opcode 3B /r */
10481   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10482   ins_pipe(ialu_cr_reg_mem);
10483 %}
10484 
10485 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10486 %{
10487   match(Set cr (CmpI src zero));
10488 
10489   format %{ "testl   $src, $src" %}
10490   opcode(0x85);
10491   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10492   ins_pipe(ialu_cr_reg_imm);
10493 %}
10494 
10495 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10496 %{
10497   match(Set cr (CmpI (AndI src con) zero));
10498 
10499   format %{ "testl   $src, $con" %}
10500   opcode(0xF7, 0x00);
10501   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10502   ins_pipe(ialu_cr_reg_imm);
10503 %}
10504 
10505 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10506 %{
10507   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10508 
10509   format %{ "testl   $src, $mem" %}
10510   opcode(0x85);
10511   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10512   ins_pipe(ialu_cr_reg_mem);
10513 %}
10514 
10515 // Unsigned compare Instructions; really, same as signed except they
10516 // produce an rFlagsRegU instead of rFlagsReg.
10517 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10518 %{
10519   match(Set cr (CmpU op1 op2));
10520 
10521   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10522   opcode(0x3B); /* Opcode 3B /r */
10523   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10524   ins_pipe(ialu_cr_reg_reg);
10525 %}
10526 
10527 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10528 %{
10529   match(Set cr (CmpU op1 op2));
10530 
10531   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10532   opcode(0x81,0x07); /* Opcode 81 /7 */
10533   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10534   ins_pipe(ialu_cr_reg_imm);
10535 %}
10536 
10537 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10538 %{
10539   match(Set cr (CmpU op1 (LoadI op2)));
10540 
10541   ins_cost(500); // XXX
10542   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10543   opcode(0x3B); /* Opcode 3B /r */
10544   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10545   ins_pipe(ialu_cr_reg_mem);
10546 %}
10547 
10548 // // // Cisc-spilled version of cmpU_rReg
10549 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10550 // //%{
10551 // //  match(Set cr (CmpU (LoadI op1) op2));
10552 // //
10553 // //  format %{ "CMPu   $op1,$op2" %}
10554 // //  ins_cost(500);
10555 // //  opcode(0x39);  /* Opcode 39 /r */
10556 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10557 // //%}
10558 
10559 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10560 %{
10561   match(Set cr (CmpU src zero));
10562 
10563   format %{ "testl  $src, $src\t# unsigned" %}
10564   opcode(0x85);
10565   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10566   ins_pipe(ialu_cr_reg_imm);
10567 %}
10568 
10569 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10570 %{
10571   match(Set cr (CmpP op1 op2));
10572 
10573   format %{ "cmpq    $op1, $op2\t# ptr" %}
10574   opcode(0x3B); /* Opcode 3B /r */
10575   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10576   ins_pipe(ialu_cr_reg_reg);
10577 %}
10578 
10579 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10580 %{
10581   match(Set cr (CmpP op1 (LoadP op2)));
10582 
10583   ins_cost(500); // XXX
10584   format %{ "cmpq    $op1, $op2\t# ptr" %}
10585   opcode(0x3B); /* Opcode 3B /r */
10586   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10587   ins_pipe(ialu_cr_reg_mem);
10588 %}
10589 
10590 // // // Cisc-spilled version of cmpP_rReg
10591 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10592 // //%{
10593 // //  match(Set cr (CmpP (LoadP op1) op2));
10594 // //
10595 // //  format %{ "CMPu   $op1,$op2" %}
10596 // //  ins_cost(500);
10597 // //  opcode(0x39);  /* Opcode 39 /r */
10598 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10599 // //%}
10600 
10601 // XXX this is generalized by compP_rReg_mem???
10602 // Compare raw pointer (used in out-of-heap check).
10603 // Only works because non-oop pointers must be raw pointers
10604 // and raw pointers have no anti-dependencies.
10605 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10606 %{
10607   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10608   match(Set cr (CmpP op1 (LoadP op2)));
10609 
10610   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10611   opcode(0x3B); /* Opcode 3B /r */
10612   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10613   ins_pipe(ialu_cr_reg_mem);
10614 %}
10615 
10616 // This will generate a signed flags result. This should be OK since
10617 // any compare to a zero should be eq/neq.
10618 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10619 %{
10620   match(Set cr (CmpP src zero));
10621 
10622   format %{ "testq   $src, $src\t# ptr" %}
10623   opcode(0x85);
10624   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10625   ins_pipe(ialu_cr_reg_imm);
10626 %}
10627 
10628 // This will generate a signed flags result. This should be OK since
10629 // any compare to a zero should be eq/neq.
10630 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10631 %{
10632   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10633   match(Set cr (CmpP (LoadP op) zero));
10634 
10635   ins_cost(500); // XXX
10636   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10637   opcode(0xF7); /* Opcode F7 /0 */
10638   ins_encode(REX_mem_wide(op),
10639              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10640   ins_pipe(ialu_cr_reg_imm);
10641 %}
10642 
10643 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10644 %{
10645   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10646   match(Set cr (CmpP (LoadP mem) zero));
10647 
10648   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10649   ins_encode %{
10650     __ cmpq(r12, $mem$$Address);
10651   %}
10652   ins_pipe(ialu_cr_reg_mem);
10653 %}
10654 
10655 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10656 %{
10657   match(Set cr (CmpN op1 op2));
10658 
10659   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10660   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10661   ins_pipe(ialu_cr_reg_reg);
10662 %}
10663 
10664 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10665 %{
10666   match(Set cr (CmpN src (LoadN mem)));
10667 
10668   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10669   ins_encode %{
10670     __ cmpl($src$$Register, $mem$$Address);
10671   %}
10672   ins_pipe(ialu_cr_reg_mem);
10673 %}
10674 
10675 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10676   match(Set cr (CmpN op1 op2));
10677 
10678   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10679   ins_encode %{
10680     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10681   %}
10682   ins_pipe(ialu_cr_reg_imm);
10683 %}
10684 
10685 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10686 %{
10687   match(Set cr (CmpN src (LoadN mem)));
10688 
10689   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10690   ins_encode %{
10691     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10692   %}
10693   ins_pipe(ialu_cr_reg_mem);
10694 %}
10695 
10696 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10697   match(Set cr (CmpN op1 op2));
10698 
10699   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10700   ins_encode %{
10701     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10702   %}
10703   ins_pipe(ialu_cr_reg_imm);
10704 %}
10705 
10706 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10707 %{
10708   match(Set cr (CmpN src (LoadNKlass mem)));
10709 
10710   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10711   ins_encode %{
10712     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10713   %}
10714   ins_pipe(ialu_cr_reg_mem);
10715 %}
10716 
10717 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10718   match(Set cr (CmpN src zero));
10719 
10720   format %{ "testl   $src, $src\t# compressed ptr" %}
10721   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10722   ins_pipe(ialu_cr_reg_imm);
10723 %}
10724 
10725 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10726 %{
10727   predicate(Universe::narrow_oop_base() != NULL);
10728   match(Set cr (CmpN (LoadN mem) zero));
10729 
10730   ins_cost(500); // XXX
10731   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10732   ins_encode %{
10733     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10734   %}
10735   ins_pipe(ialu_cr_reg_mem);
10736 %}
10737 
10738 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10739 %{
10740   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10741   match(Set cr (CmpN (LoadN mem) zero));
10742 
10743   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10744   ins_encode %{
10745     __ cmpl(r12, $mem$$Address);
10746   %}
10747   ins_pipe(ialu_cr_reg_mem);
10748 %}
10749 
10750 // Yanked all unsigned pointer compare operations.
10751 // Pointer compares are done with CmpP which is already unsigned.
10752 
10753 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10754 %{
10755   match(Set cr (CmpL op1 op2));
10756 
10757   format %{ "cmpq    $op1, $op2" %}
10758   opcode(0x3B);  /* Opcode 3B /r */
10759   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10760   ins_pipe(ialu_cr_reg_reg);
10761 %}
10762 
10763 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10764 %{
10765   match(Set cr (CmpL op1 op2));
10766 
10767   format %{ "cmpq    $op1, $op2" %}
10768   opcode(0x81, 0x07); /* Opcode 81 /7 */
10769   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
10770   ins_pipe(ialu_cr_reg_imm);
10771 %}
10772 
10773 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
10774 %{
10775   match(Set cr (CmpL op1 (LoadL op2)));
10776 
10777   format %{ "cmpq    $op1, $op2" %}
10778   opcode(0x3B); /* Opcode 3B /r */
10779   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10780   ins_pipe(ialu_cr_reg_mem);
10781 %}
10782 
10783 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
10784 %{
10785   match(Set cr (CmpL src zero));
10786 
10787   format %{ "testq   $src, $src" %}
10788   opcode(0x85);
10789   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10790   ins_pipe(ialu_cr_reg_imm);
10791 %}
10792 
10793 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
10794 %{
10795   match(Set cr (CmpL (AndL src con) zero));
10796 
10797   format %{ "testq   $src, $con\t# long" %}
10798   opcode(0xF7, 0x00);
10799   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
10800   ins_pipe(ialu_cr_reg_imm);
10801 %}
10802 
10803 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
10804 %{
10805   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
10806 
10807   format %{ "testq   $src, $mem" %}
10808   opcode(0x85);
10809   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
10810   ins_pipe(ialu_cr_reg_mem);
10811 %}
10812 
10813 // Manifest a CmpL result in an integer register.  Very painful.
10814 // This is the test to avoid.
10815 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
10816 %{
10817   match(Set dst (CmpL3 src1 src2));
10818   effect(KILL flags);
10819 
10820   ins_cost(275); // XXX
10821   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
10822             "movl    $dst, -1\n\t"
10823             "jl,s    done\n\t"
10824             "setne   $dst\n\t"
10825             "movzbl  $dst, $dst\n\t"
10826     "done:" %}
10827   ins_encode(cmpl3_flag(src1, src2, dst));
10828   ins_pipe(pipe_slow);
10829 %}
10830 
10831 //----------Max and Min--------------------------------------------------------
10832 // Min Instructions
10833 
10834 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
10835 %{
10836   effect(USE_DEF dst, USE src, USE cr);
10837 
10838   format %{ "cmovlgt $dst, $src\t# min" %}
10839   opcode(0x0F, 0x4F);
10840   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10841   ins_pipe(pipe_cmov_reg);
10842 %}
10843 
10844 
10845 instruct minI_rReg(rRegI dst, rRegI src)
10846 %{
10847   match(Set dst (MinI dst src));
10848 
10849   ins_cost(200);
10850   expand %{
10851     rFlagsReg cr;
10852     compI_rReg(cr, dst, src);
10853     cmovI_reg_g(dst, src, cr);
10854   %}
10855 %}
10856 
10857 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
10858 %{
10859   effect(USE_DEF dst, USE src, USE cr);
10860 
10861   format %{ "cmovllt $dst, $src\t# max" %}
10862   opcode(0x0F, 0x4C);
10863   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10864   ins_pipe(pipe_cmov_reg);
10865 %}
10866 
10867 
10868 instruct maxI_rReg(rRegI dst, rRegI src)
10869 %{
10870   match(Set dst (MaxI dst src));
10871 
10872   ins_cost(200);
10873   expand %{
10874     rFlagsReg cr;
10875     compI_rReg(cr, dst, src);
10876     cmovI_reg_l(dst, src, cr);
10877   %}
10878 %}
10879 
10880 // ============================================================================
10881 // Branch Instructions
10882 
10883 // Jump Direct - Label defines a relative address from JMP+1
10884 instruct jmpDir(label labl)
10885 %{
10886   match(Goto);
10887   effect(USE labl);
10888 
10889   ins_cost(300);
10890   format %{ "jmp     $labl" %}
10891   size(5);
10892   ins_encode %{
10893     Label* L = $labl$$label;
10894     __ jmp(*L, false); // Always long jump
10895   %}
10896   ins_pipe(pipe_jmp);
10897 %}
10898 
10899 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10900 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
10901 %{
10902   match(If cop cr);
10903   effect(USE labl);
10904 
10905   ins_cost(300);
10906   format %{ "j$cop     $labl" %}
10907   size(6);
10908   ins_encode %{
10909     Label* L = $labl$$label;
10910     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10911   %}
10912   ins_pipe(pipe_jcc);
10913 %}
10914 
10915 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10916 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
10917 %{
10918   match(CountedLoopEnd cop cr);
10919   effect(USE labl);
10920 
10921   ins_cost(300);
10922   format %{ "j$cop     $labl\t# loop end" %}
10923   size(6);
10924   ins_encode %{
10925     Label* L = $labl$$label;
10926     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10927   %}
10928   ins_pipe(pipe_jcc);
10929 %}
10930 
10931 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10932 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
10933   match(CountedLoopEnd cop cmp);
10934   effect(USE labl);
10935 
10936   ins_cost(300);
10937   format %{ "j$cop,u   $labl\t# loop end" %}
10938   size(6);
10939   ins_encode %{
10940     Label* L = $labl$$label;
10941     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10942   %}
10943   ins_pipe(pipe_jcc);
10944 %}
10945 
10946 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
10947   match(CountedLoopEnd cop cmp);
10948   effect(USE labl);
10949 
10950   ins_cost(200);
10951   format %{ "j$cop,u   $labl\t# loop end" %}
10952   size(6);
10953   ins_encode %{
10954     Label* L = $labl$$label;
10955     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10956   %}
10957   ins_pipe(pipe_jcc);
10958 %}
10959 
10960 // Jump Direct Conditional - using unsigned comparison
10961 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
10962   match(If cop cmp);
10963   effect(USE labl);
10964 
10965   ins_cost(300);
10966   format %{ "j$cop,u  $labl" %}
10967   size(6);
10968   ins_encode %{
10969     Label* L = $labl$$label;
10970     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10971   %}
10972   ins_pipe(pipe_jcc);
10973 %}
10974 
10975 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
10976   match(If cop cmp);
10977   effect(USE labl);
10978 
10979   ins_cost(200);
10980   format %{ "j$cop,u  $labl" %}
10981   size(6);
10982   ins_encode %{
10983     Label* L = $labl$$label;
10984     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10985   %}
10986   ins_pipe(pipe_jcc);
10987 %}
10988 
10989 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
10990   match(If cop cmp);
10991   effect(USE labl);
10992 
10993   ins_cost(200);
10994   format %{ $$template
10995     if ($cop$$cmpcode == Assembler::notEqual) {
10996       $$emit$$"jp,u   $labl\n\t"
10997       $$emit$$"j$cop,u   $labl"
10998     } else {
10999       $$emit$$"jp,u   done\n\t"
11000       $$emit$$"j$cop,u   $labl\n\t"
11001       $$emit$$"done:"
11002     }
11003   %}
11004   ins_encode %{
11005     Label* l = $labl$$label;
11006     if ($cop$$cmpcode == Assembler::notEqual) {
11007       __ jcc(Assembler::parity, *l, false);
11008       __ jcc(Assembler::notEqual, *l, false);
11009     } else if ($cop$$cmpcode == Assembler::equal) {
11010       Label done;
11011       __ jccb(Assembler::parity, done);
11012       __ jcc(Assembler::equal, *l, false);
11013       __ bind(done);
11014     } else {
11015        ShouldNotReachHere();
11016     }
11017   %}
11018   ins_pipe(pipe_jcc);
11019 %}
11020 
11021 // ============================================================================
11022 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11023 // superklass array for an instance of the superklass.  Set a hidden
11024 // internal cache on a hit (cache is checked with exposed code in
11025 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11026 // encoding ALSO sets flags.
11027 
11028 instruct partialSubtypeCheck(rdi_RegP result,
11029                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11030                              rFlagsReg cr)
11031 %{
11032   match(Set result (PartialSubtypeCheck sub super));
11033   effect(KILL rcx, KILL cr);
11034 
11035   ins_cost(1100);  // slightly larger than the next version
11036   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11037             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11038             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11039             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11040             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11041             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11042             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11043     "miss:\t" %}
11044 
11045   opcode(0x1); // Force a XOR of RDI
11046   ins_encode(enc_PartialSubtypeCheck());
11047   ins_pipe(pipe_slow);
11048 %}
11049 
11050 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11051                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11052                                      immP0 zero,
11053                                      rdi_RegP result)
11054 %{
11055   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11056   effect(KILL rcx, KILL result);
11057 
11058   ins_cost(1000);
11059   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11060             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11061             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11062             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11063             "jne,s   miss\t\t# Missed: flags nz\n\t"
11064             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11065     "miss:\t" %}
11066 
11067   opcode(0x0); // No need to XOR RDI
11068   ins_encode(enc_PartialSubtypeCheck());
11069   ins_pipe(pipe_slow);
11070 %}
11071 
11072 // ============================================================================
11073 // Branch Instructions -- short offset versions
11074 //
11075 // These instructions are used to replace jumps of a long offset (the default
11076 // match) with jumps of a shorter offset.  These instructions are all tagged
11077 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11078 // match rules in general matching.  Instead, the ADLC generates a conversion
11079 // method in the MachNode which can be used to do in-place replacement of the
11080 // long variant with the shorter variant.  The compiler will determine if a
11081 // branch can be taken by the is_short_branch_offset() predicate in the machine
11082 // specific code section of the file.
11083 
11084 // Jump Direct - Label defines a relative address from JMP+1
11085 instruct jmpDir_short(label labl) %{
11086   match(Goto);
11087   effect(USE labl);
11088 
11089   ins_cost(300);
11090   format %{ "jmp,s   $labl" %}
11091   size(2);
11092   ins_encode %{
11093     Label* L = $labl$$label;
11094     __ jmpb(*L);
11095   %}
11096   ins_pipe(pipe_jmp);
11097   ins_short_branch(1);
11098 %}
11099 
11100 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11101 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11102   match(If cop cr);
11103   effect(USE labl);
11104 
11105   ins_cost(300);
11106   format %{ "j$cop,s   $labl" %}
11107   size(2);
11108   ins_encode %{
11109     Label* L = $labl$$label;
11110     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11111   %}
11112   ins_pipe(pipe_jcc);
11113   ins_short_branch(1);
11114 %}
11115 
11116 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11117 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11118   match(CountedLoopEnd cop cr);
11119   effect(USE labl);
11120 
11121   ins_cost(300);
11122   format %{ "j$cop,s   $labl\t# loop end" %}
11123   size(2);
11124   ins_encode %{
11125     Label* L = $labl$$label;
11126     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11127   %}
11128   ins_pipe(pipe_jcc);
11129   ins_short_branch(1);
11130 %}
11131 
11132 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11133 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11134   match(CountedLoopEnd cop cmp);
11135   effect(USE labl);
11136 
11137   ins_cost(300);
11138   format %{ "j$cop,us  $labl\t# loop end" %}
11139   size(2);
11140   ins_encode %{
11141     Label* L = $labl$$label;
11142     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11143   %}
11144   ins_pipe(pipe_jcc);
11145   ins_short_branch(1);
11146 %}
11147 
11148 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11149   match(CountedLoopEnd cop cmp);
11150   effect(USE labl);
11151 
11152   ins_cost(300);
11153   format %{ "j$cop,us  $labl\t# loop end" %}
11154   size(2);
11155   ins_encode %{
11156     Label* L = $labl$$label;
11157     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11158   %}
11159   ins_pipe(pipe_jcc);
11160   ins_short_branch(1);
11161 %}
11162 
11163 // Jump Direct Conditional - using unsigned comparison
11164 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11165   match(If cop cmp);
11166   effect(USE labl);
11167 
11168   ins_cost(300);
11169   format %{ "j$cop,us  $labl" %}
11170   size(2);
11171   ins_encode %{
11172     Label* L = $labl$$label;
11173     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11174   %}
11175   ins_pipe(pipe_jcc);
11176   ins_short_branch(1);
11177 %}
11178 
11179 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11180   match(If cop cmp);
11181   effect(USE labl);
11182 
11183   ins_cost(300);
11184   format %{ "j$cop,us  $labl" %}
11185   size(2);
11186   ins_encode %{
11187     Label* L = $labl$$label;
11188     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11189   %}
11190   ins_pipe(pipe_jcc);
11191   ins_short_branch(1);
11192 %}
11193 
11194 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11195   match(If cop cmp);
11196   effect(USE labl);
11197 
11198   ins_cost(300);
11199   format %{ $$template
11200     if ($cop$$cmpcode == Assembler::notEqual) {
11201       $$emit$$"jp,u,s   $labl\n\t"
11202       $$emit$$"j$cop,u,s   $labl"
11203     } else {
11204       $$emit$$"jp,u,s   done\n\t"
11205       $$emit$$"j$cop,u,s  $labl\n\t"
11206       $$emit$$"done:"
11207     }
11208   %}
11209   size(4);
11210   ins_encode %{
11211     Label* l = $labl$$label;
11212     if ($cop$$cmpcode == Assembler::notEqual) {
11213       __ jccb(Assembler::parity, *l);
11214       __ jccb(Assembler::notEqual, *l);
11215     } else if ($cop$$cmpcode == Assembler::equal) {
11216       Label done;
11217       __ jccb(Assembler::parity, done);
11218       __ jccb(Assembler::equal, *l);
11219       __ bind(done);
11220     } else {
11221        ShouldNotReachHere();
11222     }
11223   %}
11224   ins_pipe(pipe_jcc);
11225   ins_short_branch(1);
11226 %}
11227 
11228 // ============================================================================
11229 // inlined locking and unlocking
11230 
11231 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{


11232   match(Set cr (FastLock object box));
11233   effect(TEMP tmp, TEMP scr, USE_KILL box);

11234   ins_cost(300);
11235   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11236   ins_encode %{
11237     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
11238   %}
11239   ins_pipe(pipe_slow);
11240 %}
11241 
11242 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{


11243   match(Set cr (FastUnlock object box));
11244   effect(TEMP tmp, USE_KILL box);

11245   ins_cost(300);
11246   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11247   ins_encode %{
11248     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
11249   %}
11250   ins_pipe(pipe_slow);
11251 %}
11252 
11253 
11254 // ============================================================================
11255 // Safepoint Instructions
11256 instruct safePoint_poll(rFlagsReg cr)
11257 %{
11258   predicate(!Assembler::is_polling_page_far());
11259   match(SafePoint);
11260   effect(KILL cr);
11261 
11262   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11263             "# Safepoint: poll for GC" %}
11264   ins_cost(125);
11265   ins_encode %{
11266     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11267     __ testl(rax, addr);
11268   %}
11269   ins_pipe(ialu_reg_mem);
11270 %}
11271 
11272 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11273 %{
11274   predicate(Assembler::is_polling_page_far());
11275   match(SafePoint poll);
11276   effect(KILL cr, USE poll);
11277 
11278   format %{ "testl  rax, [$poll]\t"
11279             "# Safepoint: poll for GC" %}
11280   ins_cost(125);
11281   ins_encode %{
11282     __ relocate(relocInfo::poll_type);
11283     __ testl(rax, Address($poll$$Register, 0));
11284   %}
11285   ins_pipe(ialu_reg_mem);
11286 %}
11287 
11288 // ============================================================================
11289 // Procedure Call/Return Instructions
11290 // Call Java Static Instruction
11291 // Note: If this code changes, the corresponding ret_addr_offset() and
11292 //       compute_padding() functions will have to be adjusted.
11293 instruct CallStaticJavaDirect(method meth) %{
11294   match(CallStaticJava);
11295   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
11296   effect(USE meth);
11297 
11298   ins_cost(300);
11299   format %{ "call,static " %}
11300   opcode(0xE8); /* E8 cd */
11301   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11302   ins_pipe(pipe_slow);
11303   ins_alignment(4);
11304 %}
11305 
11306 // Call Java Static Instruction (method handle version)
11307 // Note: If this code changes, the corresponding ret_addr_offset() and
11308 //       compute_padding() functions will have to be adjusted.
11309 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
11310   match(CallStaticJava);
11311   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
11312   effect(USE meth);
11313   // RBP is saved by all callees (for interpreter stack correction).
11314   // We use it here for a similar purpose, in {preserve,restore}_SP.
11315 
11316   ins_cost(300);
11317   format %{ "call,static/MethodHandle " %}
11318   opcode(0xE8); /* E8 cd */
11319   ins_encode(clear_avx, preserve_SP,
11320              Java_Static_Call(meth),
11321              restore_SP,
11322              call_epilog);
11323   ins_pipe(pipe_slow);
11324   ins_alignment(4);
11325 %}
11326 
11327 // Call Java Dynamic Instruction
11328 // Note: If this code changes, the corresponding ret_addr_offset() and
11329 //       compute_padding() functions will have to be adjusted.
11330 instruct CallDynamicJavaDirect(method meth)
11331 %{
11332   match(CallDynamicJava);
11333   effect(USE meth);
11334 
11335   ins_cost(300);
11336   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11337             "call,dynamic " %}
11338   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11339   ins_pipe(pipe_slow);
11340   ins_alignment(4);
11341 %}
11342 
11343 // Call Runtime Instruction
11344 instruct CallRuntimeDirect(method meth)
11345 %{
11346   match(CallRuntime);
11347   effect(USE meth);
11348 
11349   ins_cost(300);
11350   format %{ "call,runtime " %}
11351   ins_encode(clear_avx, Java_To_Runtime(meth));
11352   ins_pipe(pipe_slow);
11353 %}
11354 
11355 // Call runtime without safepoint
11356 instruct CallLeafDirect(method meth)
11357 %{
11358   match(CallLeaf);
11359   effect(USE meth);
11360 
11361   ins_cost(300);
11362   format %{ "call_leaf,runtime " %}
11363   ins_encode(clear_avx, Java_To_Runtime(meth));
11364   ins_pipe(pipe_slow);
11365 %}
11366 
11367 // Call runtime without safepoint
11368 instruct CallLeafNoFPDirect(method meth)
11369 %{
11370   match(CallLeafNoFP);
11371   effect(USE meth);
11372 
11373   ins_cost(300);
11374   format %{ "call_leaf_nofp,runtime " %}
11375   ins_encode(Java_To_Runtime(meth));
11376   ins_pipe(pipe_slow);
11377 %}
11378 
11379 // Return Instruction
11380 // Remove the return address & jump to it.
11381 // Notice: We always emit a nop after a ret to make sure there is room
11382 // for safepoint patching
11383 instruct Ret()
11384 %{
11385   match(Return);
11386 
11387   format %{ "ret" %}
11388   opcode(0xC3);
11389   ins_encode(OpcP);
11390   ins_pipe(pipe_jmp);
11391 %}
11392 
11393 // Tail Call; Jump from runtime stub to Java code.
11394 // Also known as an 'interprocedural jump'.
11395 // Target of jump will eventually return to caller.
11396 // TailJump below removes the return address.
11397 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11398 %{
11399   match(TailCall jump_target method_oop);
11400 
11401   ins_cost(300);
11402   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11403   opcode(0xFF, 0x4); /* Opcode FF /4 */
11404   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11405   ins_pipe(pipe_jmp);
11406 %}
11407 
11408 // Tail Jump; remove the return address; jump to target.
11409 // TailCall above leaves the return address around.
11410 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11411 %{
11412   match(TailJump jump_target ex_oop);
11413 
11414   ins_cost(300);
11415   format %{ "popq    rdx\t# pop return address\n\t"
11416             "jmp     $jump_target" %}
11417   opcode(0xFF, 0x4); /* Opcode FF /4 */
11418   ins_encode(Opcode(0x5a), // popq rdx
11419              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11420   ins_pipe(pipe_jmp);
11421 %}
11422 
11423 // Create exception oop: created by stack-crawling runtime code.
11424 // Created exception is now available to this handler, and is setup
11425 // just prior to jumping to this handler.  No code emitted.
11426 instruct CreateException(rax_RegP ex_oop)
11427 %{
11428   match(Set ex_oop (CreateEx));
11429 
11430   size(0);
11431   // use the following format syntax
11432   format %{ "# exception oop is in rax; no code emitted" %}
11433   ins_encode();
11434   ins_pipe(empty);
11435 %}
11436 
11437 // Rethrow exception:
11438 // The exception oop will come in the first argument position.
11439 // Then JUMP (not call) to the rethrow stub code.
11440 instruct RethrowException()
11441 %{
11442   match(Rethrow);
11443 
11444   // use the following format syntax
11445   format %{ "jmp     rethrow_stub" %}
11446   ins_encode(enc_rethrow);
11447   ins_pipe(pipe_jmp);
11448 %}
11449 
11450 
11451 // ============================================================================
11452 // This name is KNOWN by the ADLC and cannot be changed.
11453 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11454 // for this guy.
11455 instruct tlsLoadP(r15_RegP dst) %{
11456   match(Set dst (ThreadLocal));
11457   effect(DEF dst);
11458 
11459   size(0);
11460   format %{ "# TLS is in R15" %}
11461   ins_encode( /*empty encoding*/ );
11462   ins_pipe(ialu_reg_reg);
11463 %}
11464 
11465 
11466 //----------PEEPHOLE RULES-----------------------------------------------------
11467 // These must follow all instruction definitions as they use the names
11468 // defined in the instructions definitions.
11469 //
11470 // peepmatch ( root_instr_name [preceding_instruction]* );
11471 //
11472 // peepconstraint %{
11473 // (instruction_number.operand_name relational_op instruction_number.operand_name
11474 //  [, ...] );
11475 // // instruction numbers are zero-based using left to right order in peepmatch
11476 //
11477 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11478 // // provide an instruction_number.operand_name for each operand that appears
11479 // // in the replacement instruction's match rule
11480 //
11481 // ---------VM FLAGS---------------------------------------------------------
11482 //
11483 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11484 //
11485 // Each peephole rule is given an identifying number starting with zero and
11486 // increasing by one in the order seen by the parser.  An individual peephole
11487 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11488 // on the command-line.
11489 //
11490 // ---------CURRENT LIMITATIONS----------------------------------------------
11491 //
11492 // Only match adjacent instructions in same basic block
11493 // Only equality constraints
11494 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11495 // Only one replacement instruction
11496 //
11497 // ---------EXAMPLE----------------------------------------------------------
11498 //
11499 // // pertinent parts of existing instructions in architecture description
11500 // instruct movI(rRegI dst, rRegI src)
11501 // %{
11502 //   match(Set dst (CopyI src));
11503 // %}
11504 //
11505 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11506 // %{
11507 //   match(Set dst (AddI dst src));
11508 //   effect(KILL cr);
11509 // %}
11510 //
11511 // // Change (inc mov) to lea
11512 // peephole %{
11513 //   // increment preceeded by register-register move
11514 //   peepmatch ( incI_rReg movI );
11515 //   // require that the destination register of the increment
11516 //   // match the destination register of the move
11517 //   peepconstraint ( 0.dst == 1.dst );
11518 //   // construct a replacement instruction that sets
11519 //   // the destination to ( move's source register + one )
11520 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11521 // %}
11522 //
11523 
11524 // Implementation no longer uses movX instructions since
11525 // machine-independent system no longer uses CopyX nodes.
11526 //
11527 // peephole
11528 // %{
11529 //   peepmatch (incI_rReg movI);
11530 //   peepconstraint (0.dst == 1.dst);
11531 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11532 // %}
11533 
11534 // peephole
11535 // %{
11536 //   peepmatch (decI_rReg movI);
11537 //   peepconstraint (0.dst == 1.dst);
11538 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11539 // %}
11540 
11541 // peephole
11542 // %{
11543 //   peepmatch (addI_rReg_imm movI);
11544 //   peepconstraint (0.dst == 1.dst);
11545 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11546 // %}
11547 
11548 // peephole
11549 // %{
11550 //   peepmatch (incL_rReg movL);
11551 //   peepconstraint (0.dst == 1.dst);
11552 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11553 // %}
11554 
11555 // peephole
11556 // %{
11557 //   peepmatch (decL_rReg movL);
11558 //   peepconstraint (0.dst == 1.dst);
11559 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11560 // %}
11561 
11562 // peephole
11563 // %{
11564 //   peepmatch (addL_rReg_imm movL);
11565 //   peepconstraint (0.dst == 1.dst);
11566 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11567 // %}
11568 
11569 // peephole
11570 // %{
11571 //   peepmatch (addP_rReg_imm movP);
11572 //   peepconstraint (0.dst == 1.dst);
11573 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11574 // %}
11575 
11576 // // Change load of spilled value to only a spill
11577 // instruct storeI(memory mem, rRegI src)
11578 // %{
11579 //   match(Set mem (StoreI mem src));
11580 // %}
11581 //
11582 // instruct loadI(rRegI dst, memory mem)
11583 // %{
11584 //   match(Set dst (LoadI mem));
11585 // %}
11586 //
11587 
11588 peephole
11589 %{
11590   peepmatch (loadI storeI);
11591   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11592   peepreplace (storeI(1.mem 1.mem 1.src));
11593 %}
11594 
11595 peephole
11596 %{
11597   peepmatch (loadL storeL);
11598   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11599   peepreplace (storeL(1.mem 1.mem 1.src));
11600 %}
11601 
11602 //----------SMARTSPILL RULES---------------------------------------------------
11603 // These must follow all instruction definitions as they use the names
11604 // defined in the instructions definitions.
--- EOF ---