1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Class for all pointer registers (including RSP)
 170 reg_class any_reg(RAX, RAX_H,
 171                   RDX, RDX_H,
 172                   RBP, RBP_H,
 173                   RDI, RDI_H,
 174                   RSI, RSI_H,
 175                   RCX, RCX_H,
 176                   RBX, RBX_H,
 177                   RSP, RSP_H,
 178                   R8,  R8_H,
 179                   R9,  R9_H,
 180                   R10, R10_H,
 181                   R11, R11_H,
 182                   R12, R12_H,
 183                   R13, R13_H,
 184                   R14, R14_H,
 185                   R15, R15_H);
 186 
 187 // Class for all pointer registers except RSP
 188 reg_class ptr_reg(RAX, RAX_H,
 189                   RDX, RDX_H,
 190                   RBP, RBP_H,
 191                   RDI, RDI_H,
 192                   RSI, RSI_H,
 193                   RCX, RCX_H,
 194                   RBX, RBX_H,
 195                   R8,  R8_H,
 196                   R9,  R9_H,
 197                   R10, R10_H,
 198                   R11, R11_H,
 199                   R13, R13_H,
 200                   R14, R14_H);
 201 
 202 // Class for all pointer registers except RAX and RSP
 203 reg_class ptr_no_rax_reg(RDX, RDX_H,
 204                          RBP, RBP_H,
 205                          RDI, RDI_H,
 206                          RSI, RSI_H,
 207                          RCX, RCX_H,
 208                          RBX, RBX_H,
 209                          R8,  R8_H,
 210                          R9,  R9_H,
 211                          R10, R10_H,
 212                          R11, R11_H,
 213                          R13, R13_H,
 214                          R14, R14_H);
 215 
 216 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 217                          RAX, RAX_H,
 218                          RDI, RDI_H,
 219                          RSI, RSI_H,
 220                          RCX, RCX_H,
 221                          RBX, RBX_H,
 222                          R8,  R8_H,
 223                          R9,  R9_H,
 224                          R10, R10_H,
 225                          R11, R11_H,
 226                          R13, R13_H,
 227                          R14, R14_H);
 228 
 229 // Class for all pointer registers except RAX, RBX and RSP
 230 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 231                              RBP, RBP_H,
 232                              RDI, RDI_H,
 233                              RSI, RSI_H,
 234                              RCX, RCX_H,
 235                              R8,  R8_H,
 236                              R9,  R9_H,
 237                              R10, R10_H,
 238                              R11, R11_H,
 239                              R13, R13_H,
 240                              R14, R14_H);
 241 
 242 // Singleton class for RAX pointer register
 243 reg_class ptr_rax_reg(RAX, RAX_H);
 244 
 245 // Singleton class for RBX pointer register
 246 reg_class ptr_rbx_reg(RBX, RBX_H);
 247 
 248 // Singleton class for RSI pointer register
 249 reg_class ptr_rsi_reg(RSI, RSI_H);
 250 
 251 // Singleton class for RDI pointer register
 252 reg_class ptr_rdi_reg(RDI, RDI_H);
 253 
 254 // Singleton class for RBP pointer register
 255 reg_class ptr_rbp_reg(RBP, RBP_H);
 256 
 257 // Singleton class for stack pointer
 258 reg_class ptr_rsp_reg(RSP, RSP_H);
 259 
 260 // Singleton class for TLS pointer
 261 reg_class ptr_r15_reg(R15, R15_H);
 262 
 263 // Class for all long registers (except RSP)
 264 reg_class long_reg(RAX, RAX_H,
 265                    RDX, RDX_H,
 266                    RBP, RBP_H,
 267                    RDI, RDI_H,
 268                    RSI, RSI_H,
 269                    RCX, RCX_H,
 270                    RBX, RBX_H,
 271                    R8,  R8_H,
 272                    R9,  R9_H,
 273                    R10, R10_H,
 274                    R11, R11_H,
 275                    R13, R13_H,
 276                    R14, R14_H);
 277 
 278 // Class for all long registers except RAX, RDX (and RSP)
 279 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 280                               RDI, RDI_H,
 281                               RSI, RSI_H,
 282                               RCX, RCX_H,
 283                               RBX, RBX_H,
 284                               R8,  R8_H,
 285                               R9,  R9_H,
 286                               R10, R10_H,
 287                               R11, R11_H,
 288                               R13, R13_H,
 289                               R14, R14_H);
 290 
 291 // Class for all long registers except RCX (and RSP)
 292 reg_class long_no_rcx_reg(RBP, RBP_H,
 293                           RDI, RDI_H,
 294                           RSI, RSI_H,
 295                           RAX, RAX_H,
 296                           RDX, RDX_H,
 297                           RBX, RBX_H,
 298                           R8,  R8_H,
 299                           R9,  R9_H,
 300                           R10, R10_H,
 301                           R11, R11_H,
 302                           R13, R13_H,
 303                           R14, R14_H);
 304 
 305 // Class for all long registers except RAX (and RSP)
 306 reg_class long_no_rax_reg(RBP, RBP_H,
 307                           RDX, RDX_H,
 308                           RDI, RDI_H,
 309                           RSI, RSI_H,
 310                           RCX, RCX_H,
 311                           RBX, RBX_H,
 312                           R8,  R8_H,
 313                           R9,  R9_H,
 314                           R10, R10_H,
 315                           R11, R11_H,
 316                           R13, R13_H,
 317                           R14, R14_H);
 318 
 319 // Singleton class for RAX long register
 320 reg_class long_rax_reg(RAX, RAX_H);
 321 
 322 // Singleton class for RCX long register
 323 reg_class long_rcx_reg(RCX, RCX_H);
 324 
 325 // Singleton class for RDX long register
 326 reg_class long_rdx_reg(RDX, RDX_H);
 327 
 328 // Class for all int registers (except RSP)
 329 reg_class int_reg(RAX,
 330                   RDX,
 331                   RBP,
 332                   RDI,
 333                   RSI,
 334                   RCX,
 335                   RBX,
 336                   R8,
 337                   R9,
 338                   R10,
 339                   R11,
 340                   R13,
 341                   R14);
 342 
 343 // Class for all int registers except RCX (and RSP)
 344 reg_class int_no_rcx_reg(RAX,
 345                          RDX,
 346                          RBP,
 347                          RDI,
 348                          RSI,
 349                          RBX,
 350                          R8,
 351                          R9,
 352                          R10,
 353                          R11,
 354                          R13,
 355                          R14);
 356 
 357 // Class for all int registers except RAX, RDX (and RSP)
 358 reg_class int_no_rax_rdx_reg(RBP,
 359                              RDI,
 360                              RSI,
 361                              RCX,
 362                              RBX,
 363                              R8,
 364                              R9,
 365                              R10,
 366                              R11,
 367                              R13,
 368                              R14);
 369 
 370 // Singleton class for RAX int register
 371 reg_class int_rax_reg(RAX);
 372 
 373 // Singleton class for RBX int register
 374 reg_class int_rbx_reg(RBX);
 375 
 376 // Singleton class for RCX int register
 377 reg_class int_rcx_reg(RCX);
 378 
 379 // Singleton class for RCX int register
 380 reg_class int_rdx_reg(RDX);
 381 
 382 // Singleton class for RCX int register
 383 reg_class int_rdi_reg(RDI);
 384 
 385 // Singleton class for instruction pointer
 386 // reg_class ip_reg(RIP);
 387 
 388 %}
 389 
 390 //----------SOURCE BLOCK-------------------------------------------------------
 391 // This is a block of C++ code which provides values, functions, and
 392 // definitions necessary in the rest of the architecture description
 393 source %{
 394 #define   RELOC_IMM64    Assembler::imm_operand
 395 #define   RELOC_DISP32   Assembler::disp32_operand
 396 
 397 #define __ _masm.
 398 
 399 static int preserve_SP_size() {
 400   return 3;  // rex.w, op, rm(reg/reg)
 401 }
 402 static int clear_avx_size() {
 403   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 404 }
 405 
 406 // !!!!! Special hack to get all types of calls to specify the byte offset
 407 //       from the start of the call to the point where the return address
 408 //       will point.
 409 int MachCallStaticJavaNode::ret_addr_offset()
 410 {
 411   int offset = 5; // 5 bytes from start of call to where return address points
 412   offset += clear_avx_size();
 413   if (_method_handle_invoke)
 414     offset += preserve_SP_size();
 415   return offset;
 416 }
 417 
 418 int MachCallDynamicJavaNode::ret_addr_offset()
 419 {
 420   int offset = 15; // 15 bytes from start of call to where return address points
 421   offset += clear_avx_size();
 422   return offset;
 423 }
 424 
 425 int MachCallRuntimeNode::ret_addr_offset() {
 426   int offset = 13; // movq r10,#addr; callq (r10)
 427   offset += clear_avx_size();
 428   return offset;
 429 }
 430 
 431 // Indicate if the safepoint node needs the polling page as an input,
 432 // it does if the polling page is more than disp32 away.
 433 bool SafePointNode::needs_polling_address_input()
 434 {
 435   return Assembler::is_polling_page_far();
 436 }
 437 
 438 //
 439 // Compute padding required for nodes which need alignment
 440 //
 441 
 442 // The address of the call instruction needs to be 4-byte aligned to
 443 // ensure that it does not span a cache line so that it can be patched.
 444 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 445 {
 446   current_offset += clear_avx_size(); // skip vzeroupper
 447   current_offset += 1; // skip call opcode byte
 448   return round_to(current_offset, alignment_required()) - current_offset;
 449 }
 450 
 451 // The address of the call instruction needs to be 4-byte aligned to
 452 // ensure that it does not span a cache line so that it can be patched.
 453 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 454 {
 455   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 456   current_offset += clear_avx_size(); // skip vzeroupper
 457   current_offset += 1; // skip call opcode byte
 458   return round_to(current_offset, alignment_required()) - current_offset;
 459 }
 460 
 461 // The address of the call instruction needs to be 4-byte aligned to
 462 // ensure that it does not span a cache line so that it can be patched.
 463 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 464 {
 465   current_offset += clear_avx_size(); // skip vzeroupper
 466   current_offset += 11; // skip movq instruction + call opcode byte
 467   return round_to(current_offset, alignment_required()) - current_offset;
 468 }
 469 
 470 // EMIT_RM()
 471 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 472   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 473   cbuf.insts()->emit_int8(c);
 474 }
 475 
 476 // EMIT_CC()
 477 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 478   unsigned char c = (unsigned char) (f1 | f2);
 479   cbuf.insts()->emit_int8(c);
 480 }
 481 
 482 // EMIT_OPCODE()
 483 void emit_opcode(CodeBuffer &cbuf, int code) {
 484   cbuf.insts()->emit_int8((unsigned char) code);
 485 }
 486 
 487 // EMIT_OPCODE() w/ relocation information
 488 void emit_opcode(CodeBuffer &cbuf,
 489                  int code, relocInfo::relocType reloc, int offset, int format)
 490 {
 491   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 492   emit_opcode(cbuf, code);
 493 }
 494 
 495 // EMIT_D8()
 496 void emit_d8(CodeBuffer &cbuf, int d8) {
 497   cbuf.insts()->emit_int8((unsigned char) d8);
 498 }
 499 
 500 // EMIT_D16()
 501 void emit_d16(CodeBuffer &cbuf, int d16) {
 502   cbuf.insts()->emit_int16(d16);
 503 }
 504 
 505 // EMIT_D32()
 506 void emit_d32(CodeBuffer &cbuf, int d32) {
 507   cbuf.insts()->emit_int32(d32);
 508 }
 509 
 510 // EMIT_D64()
 511 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 512   cbuf.insts()->emit_int64(d64);
 513 }
 514 
 515 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 516 void emit_d32_reloc(CodeBuffer& cbuf,
 517                     int d32,
 518                     relocInfo::relocType reloc,
 519                     int format)
 520 {
 521   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 522   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 523   cbuf.insts()->emit_int32(d32);
 524 }
 525 
 526 // emit 32 bit value and construct relocation entry from RelocationHolder
 527 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 528 #ifdef ASSERT
 529   if (rspec.reloc()->type() == relocInfo::oop_type &&
 530       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 531     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 532     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 533   }
 534 #endif
 535   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 536   cbuf.insts()->emit_int32(d32);
 537 }
 538 
 539 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 540   address next_ip = cbuf.insts_end() + 4;
 541   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 542                  external_word_Relocation::spec(addr),
 543                  RELOC_DISP32);
 544 }
 545 
 546 
 547 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 548 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 549   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 550   cbuf.insts()->emit_int64(d64);
 551 }
 552 
 553 // emit 64 bit value and construct relocation entry from RelocationHolder
 554 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 555 #ifdef ASSERT
 556   if (rspec.reloc()->type() == relocInfo::oop_type &&
 557       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 558     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 559     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 560            "cannot embed scavengable oops in code");
 561   }
 562 #endif
 563   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 564   cbuf.insts()->emit_int64(d64);
 565 }
 566 
 567 // Access stack slot for load or store
 568 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 569 {
 570   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 571   if (-0x80 <= disp && disp < 0x80) {
 572     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 573     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 574     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 575   } else {
 576     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 577     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 578     emit_d32(cbuf, disp);     // Displacement // R/M byte
 579   }
 580 }
 581 
 582    // rRegI ereg, memory mem) %{    // emit_reg_mem
 583 void encode_RegMem(CodeBuffer &cbuf,
 584                    int reg,
 585                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 586 {
 587   assert(disp_reloc == relocInfo::none, "cannot have disp");
 588   int regenc = reg & 7;
 589   int baseenc = base & 7;
 590   int indexenc = index & 7;
 591 
 592   // There is no index & no scale, use form without SIB byte
 593   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 594     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 595     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 596       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 597     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 598       // If 8-bit displacement, mode 0x1
 599       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 600       emit_d8(cbuf, disp);
 601     } else {
 602       // If 32-bit displacement
 603       if (base == -1) { // Special flag for absolute address
 604         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 605         if (disp_reloc != relocInfo::none) {
 606           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 607         } else {
 608           emit_d32(cbuf, disp);
 609         }
 610       } else {
 611         // Normal base + offset
 612         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 613         if (disp_reloc != relocInfo::none) {
 614           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 615         } else {
 616           emit_d32(cbuf, disp);
 617         }
 618       }
 619     }
 620   } else {
 621     // Else, encode with the SIB byte
 622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 624       // If no displacement
 625       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 626       emit_rm(cbuf, scale, indexenc, baseenc);
 627     } else {
 628       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 629         // If 8-bit displacement, mode 0x1
 630         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 631         emit_rm(cbuf, scale, indexenc, baseenc);
 632         emit_d8(cbuf, disp);
 633       } else {
 634         // If 32-bit displacement
 635         if (base == 0x04 ) {
 636           emit_rm(cbuf, 0x2, regenc, 0x4);
 637           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 638         } else {
 639           emit_rm(cbuf, 0x2, regenc, 0x4);
 640           emit_rm(cbuf, scale, indexenc, baseenc); // *
 641         }
 642         if (disp_reloc != relocInfo::none) {
 643           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 644         } else {
 645           emit_d32(cbuf, disp);
 646         }
 647       }
 648     }
 649   }
 650 }
 651 
 652 // This could be in MacroAssembler but it's fairly C2 specific
 653 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 654   Label exit;
 655   __ jccb(Assembler::noParity, exit);
 656   __ pushf();
 657   //
 658   // comiss/ucomiss instructions set ZF,PF,CF flags and
 659   // zero OF,AF,SF for NaN values.
 660   // Fixup flags by zeroing ZF,PF so that compare of NaN
 661   // values returns 'less than' result (CF is set).
 662   // Leave the rest of flags unchanged.
 663   //
 664   //    7 6 5 4 3 2 1 0
 665   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 666   //    0 0 1 0 1 0 1 1   (0x2B)
 667   //
 668   __ andq(Address(rsp, 0), 0xffffff2b);
 669   __ popf();
 670   __ bind(exit);
 671 }
 672 
 673 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 674   Label done;
 675   __ movl(dst, -1);
 676   __ jcc(Assembler::parity, done);
 677   __ jcc(Assembler::below, done);
 678   __ setb(Assembler::notEqual, dst);
 679   __ movzbl(dst, dst);
 680   __ bind(done);
 681 }
 682 
 683 
 684 //=============================================================================
 685 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 686 
 687 int Compile::ConstantTable::calculate_table_base_offset() const {
 688   return 0;  // absolute addressing, no offset
 689 }
 690 
 691 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 692 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 693   ShouldNotReachHere();
 694 }
 695 
 696 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 697   // Empty encoding
 698 }
 699 
 700 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 701   return 0;
 702 }
 703 
 704 #ifndef PRODUCT
 705 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 706   st->print("# MachConstantBaseNode (empty encoding)");
 707 }
 708 #endif
 709 
 710 
 711 //=============================================================================
 712 #ifndef PRODUCT
 713 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 714   Compile* C = ra_->C;
 715 
 716   int framesize = C->frame_size_in_bytes();
 717   int bangsize = C->bang_size_in_bytes();
 718   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 719   // Remove wordSize for return addr which is already pushed.
 720   framesize -= wordSize;
 721 
 722   if (C->need_stack_bang(bangsize)) {
 723     framesize -= wordSize;
 724     st->print("# stack bang (%d bytes)", bangsize);
 725     st->print("\n\t");
 726     st->print("pushq   rbp\t# Save rbp");
 727     if (framesize) {
 728       st->print("\n\t");
 729       st->print("subq    rsp, #%d\t# Create frame",framesize);
 730     }
 731   } else {
 732     st->print("subq    rsp, #%d\t# Create frame",framesize);
 733     st->print("\n\t");
 734     framesize -= wordSize;
 735     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 736   }
 737 
 738   if (VerifyStackAtCalls) {
 739     st->print("\n\t");
 740     framesize -= wordSize;
 741     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 742 #ifdef ASSERT
 743     st->print("\n\t");
 744     st->print("# stack alignment check");
 745 #endif
 746   }
 747   st->cr();
 748 }
 749 #endif
 750 
 751 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 752   Compile* C = ra_->C;
 753   MacroAssembler _masm(&cbuf);
 754 
 755   int framesize = C->frame_size_in_bytes();
 756   int bangsize = C->bang_size_in_bytes();
 757 
 758   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 759 
 760   C->set_frame_complete(cbuf.insts_size());
 761 
 762   if (C->has_mach_constant_base_node()) {
 763     // NOTE: We set the table base offset here because users might be
 764     // emitted before MachConstantBaseNode.
 765     Compile::ConstantTable& constant_table = C->constant_table();
 766     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 767   }
 768 }
 769 
 770 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 771 {
 772   return MachNode::size(ra_); // too many variables; just compute it
 773                               // the hard way
 774 }
 775 
 776 int MachPrologNode::reloc() const
 777 {
 778   return 0; // a large enough number
 779 }
 780 
 781 //=============================================================================
 782 #ifndef PRODUCT
 783 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 784 {
 785   Compile* C = ra_->C;
 786   if (C->max_vector_size() > 16) {
 787     st->print("vzeroupper");
 788     st->cr(); st->print("\t");
 789   }
 790 
 791   int framesize = C->frame_size_in_bytes();
 792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 793   // Remove word for return adr already pushed
 794   // and RBP
 795   framesize -= 2*wordSize;
 796 
 797   if (framesize) {
 798     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 799     st->print("\t");
 800   }
 801 
 802   st->print_cr("popq   rbp");
 803   if (do_polling() && C->is_method_compilation()) {
 804     st->print("\t");
 805     if (Assembler::is_polling_page_far()) {
 806       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 807                    "testl  rax, [rscratch1]\t"
 808                    "# Safepoint: poll for GC");
 809     } else {
 810       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 811                    "# Safepoint: poll for GC");
 812     }
 813   }
 814 }
 815 #endif
 816 
 817 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 818 {
 819   Compile* C = ra_->C;
 820   if (C->max_vector_size() > 16) {
 821     // Clear upper bits of YMM registers when current compiled code uses
 822     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 823     MacroAssembler _masm(&cbuf);
 824     __ vzeroupper();
 825   }
 826 
 827   int framesize = C->frame_size_in_bytes();
 828   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 829   // Remove word for return adr already pushed
 830   // and RBP
 831   framesize -= 2*wordSize;
 832 
 833   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 834 
 835   if (framesize) {
 836     emit_opcode(cbuf, Assembler::REX_W);
 837     if (framesize < 0x80) {
 838       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 839       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 840       emit_d8(cbuf, framesize);
 841     } else {
 842       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 843       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 844       emit_d32(cbuf, framesize);
 845     }
 846   }
 847 
 848   // popq rbp
 849   emit_opcode(cbuf, 0x58 | RBP_enc);
 850 
 851   if (do_polling() && C->is_method_compilation()) {
 852     MacroAssembler _masm(&cbuf);
 853     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 854     if (Assembler::is_polling_page_far()) {
 855       __ lea(rscratch1, polling_page);
 856       __ relocate(relocInfo::poll_return_type);
 857       __ testl(rax, Address(rscratch1, 0));
 858     } else {
 859       __ testl(rax, polling_page);
 860     }
 861   }
 862 }
 863 
 864 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 865 {
 866   return MachNode::size(ra_); // too many variables; just compute it
 867                               // the hard way
 868 }
 869 
 870 int MachEpilogNode::reloc() const
 871 {
 872   return 2; // a large enough number
 873 }
 874 
 875 const Pipeline* MachEpilogNode::pipeline() const
 876 {
 877   return MachNode::pipeline_class();
 878 }
 879 
 880 int MachEpilogNode::safepoint_offset() const
 881 {
 882   return 0;
 883 }
 884 
 885 //=============================================================================
 886 
 887 enum RC {
 888   rc_bad,
 889   rc_int,
 890   rc_float,
 891   rc_stack
 892 };
 893 
 894 static enum RC rc_class(OptoReg::Name reg)
 895 {
 896   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 897 
 898   if (OptoReg::is_stack(reg)) return rc_stack;
 899 
 900   VMReg r = OptoReg::as_VMReg(reg);
 901 
 902   if (r->is_Register()) return rc_int;
 903 
 904   assert(r->is_XMMRegister(), "must be");
 905   return rc_float;
 906 }
 907 
 908 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 909 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 910                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 911 
 912 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 913                             int stack_offset, int reg, uint ireg, outputStream* st);
 914 
 915 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 916                                       int dst_offset, uint ireg, outputStream* st) {
 917   if (cbuf) {
 918     MacroAssembler _masm(cbuf);
 919     switch (ireg) {
 920     case Op_VecS:
 921       __ movq(Address(rsp, -8), rax);
 922       __ movl(rax, Address(rsp, src_offset));
 923       __ movl(Address(rsp, dst_offset), rax);
 924       __ movq(rax, Address(rsp, -8));
 925       break;
 926     case Op_VecD:
 927       __ pushq(Address(rsp, src_offset));
 928       __ popq (Address(rsp, dst_offset));
 929       break;
 930     case Op_VecX:
 931       __ pushq(Address(rsp, src_offset));
 932       __ popq (Address(rsp, dst_offset));
 933       __ pushq(Address(rsp, src_offset+8));
 934       __ popq (Address(rsp, dst_offset+8));
 935       break;
 936     case Op_VecY:
 937       __ vmovdqu(Address(rsp, -32), xmm0);
 938       __ vmovdqu(xmm0, Address(rsp, src_offset));
 939       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 940       __ vmovdqu(xmm0, Address(rsp, -32));
 941       break;
 942     default:
 943       ShouldNotReachHere();
 944     }
 945 #ifndef PRODUCT
 946   } else {
 947     switch (ireg) {
 948     case Op_VecS:
 949       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 950                 "movl    rax, [rsp + #%d]\n\t"
 951                 "movl    [rsp + #%d], rax\n\t"
 952                 "movq    rax, [rsp - #8]",
 953                 src_offset, dst_offset);
 954       break;
 955     case Op_VecD:
 956       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 957                 "popq    [rsp + #%d]",
 958                 src_offset, dst_offset);
 959       break;
 960      case Op_VecX:
 961       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 962                 "popq    [rsp + #%d]\n\t"
 963                 "pushq   [rsp + #%d]\n\t"
 964                 "popq    [rsp + #%d]",
 965                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 966       break;
 967     case Op_VecY:
 968       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 969                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 970                 "vmovdqu [rsp + #%d], xmm0\n\t"
 971                 "vmovdqu xmm0, [rsp - #32]",
 972                 src_offset, dst_offset);
 973       break;
 974     default:
 975       ShouldNotReachHere();
 976     }
 977 #endif
 978   }
 979 }
 980 
 981 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 982                                        PhaseRegAlloc* ra_,
 983                                        bool do_size,
 984                                        outputStream* st) const {
 985   assert(cbuf != NULL || st  != NULL, "sanity");
 986   // Get registers to move
 987   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 988   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 989   OptoReg::Name dst_second = ra_->get_reg_second(this);
 990   OptoReg::Name dst_first = ra_->get_reg_first(this);
 991 
 992   enum RC src_second_rc = rc_class(src_second);
 993   enum RC src_first_rc = rc_class(src_first);
 994   enum RC dst_second_rc = rc_class(dst_second);
 995   enum RC dst_first_rc = rc_class(dst_first);
 996 
 997   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 998          "must move at least 1 register" );
 999 
1000   if (src_first == dst_first && src_second == dst_second) {
1001     // Self copy, no move
1002     return 0;
1003   }
1004   if (bottom_type()->isa_vect() != NULL) {
1005     uint ireg = ideal_reg();
1006     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1007     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1008     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1009       // mem -> mem
1010       int src_offset = ra_->reg2offset(src_first);
1011       int dst_offset = ra_->reg2offset(dst_first);
1012       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1013     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1014       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1015     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1016       int stack_offset = ra_->reg2offset(dst_first);
1017       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1018     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1019       int stack_offset = ra_->reg2offset(src_first);
1020       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1021     } else {
1022       ShouldNotReachHere();
1023     }
1024     return 0;
1025   }
1026   if (src_first_rc == rc_stack) {
1027     // mem ->
1028     if (dst_first_rc == rc_stack) {
1029       // mem -> mem
1030       assert(src_second != dst_first, "overlap");
1031       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1032           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1033         // 64-bit
1034         int src_offset = ra_->reg2offset(src_first);
1035         int dst_offset = ra_->reg2offset(dst_first);
1036         if (cbuf) {
1037           MacroAssembler _masm(cbuf);
1038           __ pushq(Address(rsp, src_offset));
1039           __ popq (Address(rsp, dst_offset));
1040 #ifndef PRODUCT
1041         } else {
1042           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1043                     "popq    [rsp + #%d]",
1044                      src_offset, dst_offset);
1045 #endif
1046         }
1047       } else {
1048         // 32-bit
1049         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1050         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1051         // No pushl/popl, so:
1052         int src_offset = ra_->reg2offset(src_first);
1053         int dst_offset = ra_->reg2offset(dst_first);
1054         if (cbuf) {
1055           MacroAssembler _masm(cbuf);
1056           __ movq(Address(rsp, -8), rax);
1057           __ movl(rax, Address(rsp, src_offset));
1058           __ movl(Address(rsp, dst_offset), rax);
1059           __ movq(rax, Address(rsp, -8));
1060 #ifndef PRODUCT
1061         } else {
1062           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1063                     "movl    rax, [rsp + #%d]\n\t"
1064                     "movl    [rsp + #%d], rax\n\t"
1065                     "movq    rax, [rsp - #8]",
1066                      src_offset, dst_offset);
1067 #endif
1068         }
1069       }
1070       return 0;
1071     } else if (dst_first_rc == rc_int) {
1072       // mem -> gpr
1073       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1074           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1075         // 64-bit
1076         int offset = ra_->reg2offset(src_first);
1077         if (cbuf) {
1078           MacroAssembler _masm(cbuf);
1079           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1080 #ifndef PRODUCT
1081         } else {
1082           st->print("movq    %s, [rsp + #%d]\t# spill",
1083                      Matcher::regName[dst_first],
1084                      offset);
1085 #endif
1086         }
1087       } else {
1088         // 32-bit
1089         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1090         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1091         int offset = ra_->reg2offset(src_first);
1092         if (cbuf) {
1093           MacroAssembler _masm(cbuf);
1094           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1095 #ifndef PRODUCT
1096         } else {
1097           st->print("movl    %s, [rsp + #%d]\t# spill",
1098                      Matcher::regName[dst_first],
1099                      offset);
1100 #endif
1101         }
1102       }
1103       return 0;
1104     } else if (dst_first_rc == rc_float) {
1105       // mem-> xmm
1106       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1107           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1108         // 64-bit
1109         int offset = ra_->reg2offset(src_first);
1110         if (cbuf) {
1111           MacroAssembler _masm(cbuf);
1112           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1113 #ifndef PRODUCT
1114         } else {
1115           st->print("%s  %s, [rsp + #%d]\t# spill",
1116                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1117                      Matcher::regName[dst_first],
1118                      offset);
1119 #endif
1120         }
1121       } else {
1122         // 32-bit
1123         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1124         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1125         int offset = ra_->reg2offset(src_first);
1126         if (cbuf) {
1127           MacroAssembler _masm(cbuf);
1128           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1129 #ifndef PRODUCT
1130         } else {
1131           st->print("movss   %s, [rsp + #%d]\t# spill",
1132                      Matcher::regName[dst_first],
1133                      offset);
1134 #endif
1135         }
1136       }
1137       return 0;
1138     }
1139   } else if (src_first_rc == rc_int) {
1140     // gpr ->
1141     if (dst_first_rc == rc_stack) {
1142       // gpr -> mem
1143       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1144           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1145         // 64-bit
1146         int offset = ra_->reg2offset(dst_first);
1147         if (cbuf) {
1148           MacroAssembler _masm(cbuf);
1149           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1150 #ifndef PRODUCT
1151         } else {
1152           st->print("movq    [rsp + #%d], %s\t# spill",
1153                      offset,
1154                      Matcher::regName[src_first]);
1155 #endif
1156         }
1157       } else {
1158         // 32-bit
1159         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1160         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1161         int offset = ra_->reg2offset(dst_first);
1162         if (cbuf) {
1163           MacroAssembler _masm(cbuf);
1164           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1165 #ifndef PRODUCT
1166         } else {
1167           st->print("movl    [rsp + #%d], %s\t# spill",
1168                      offset,
1169                      Matcher::regName[src_first]);
1170 #endif
1171         }
1172       }
1173       return 0;
1174     } else if (dst_first_rc == rc_int) {
1175       // gpr -> gpr
1176       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1177           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1178         // 64-bit
1179         if (cbuf) {
1180           MacroAssembler _masm(cbuf);
1181           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1182                   as_Register(Matcher::_regEncode[src_first]));
1183 #ifndef PRODUCT
1184         } else {
1185           st->print("movq    %s, %s\t# spill",
1186                      Matcher::regName[dst_first],
1187                      Matcher::regName[src_first]);
1188 #endif
1189         }
1190         return 0;
1191       } else {
1192         // 32-bit
1193         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1194         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1195         if (cbuf) {
1196           MacroAssembler _masm(cbuf);
1197           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1198                   as_Register(Matcher::_regEncode[src_first]));
1199 #ifndef PRODUCT
1200         } else {
1201           st->print("movl    %s, %s\t# spill",
1202                      Matcher::regName[dst_first],
1203                      Matcher::regName[src_first]);
1204 #endif
1205         }
1206         return 0;
1207       }
1208     } else if (dst_first_rc == rc_float) {
1209       // gpr -> xmm
1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1212         // 64-bit
1213         if (cbuf) {
1214           MacroAssembler _masm(cbuf);
1215           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1216 #ifndef PRODUCT
1217         } else {
1218           st->print("movdq   %s, %s\t# spill",
1219                      Matcher::regName[dst_first],
1220                      Matcher::regName[src_first]);
1221 #endif
1222         }
1223       } else {
1224         // 32-bit
1225         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1226         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1227         if (cbuf) {
1228           MacroAssembler _masm(cbuf);
1229           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1230 #ifndef PRODUCT
1231         } else {
1232           st->print("movdl   %s, %s\t# spill",
1233                      Matcher::regName[dst_first],
1234                      Matcher::regName[src_first]);
1235 #endif
1236         }
1237       }
1238       return 0;
1239     }
1240   } else if (src_first_rc == rc_float) {
1241     // xmm ->
1242     if (dst_first_rc == rc_stack) {
1243       // xmm -> mem
1244       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1245           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1246         // 64-bit
1247         int offset = ra_->reg2offset(dst_first);
1248         if (cbuf) {
1249           MacroAssembler _masm(cbuf);
1250           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1251 #ifndef PRODUCT
1252         } else {
1253           st->print("movsd   [rsp + #%d], %s\t# spill",
1254                      offset,
1255                      Matcher::regName[src_first]);
1256 #endif
1257         }
1258       } else {
1259         // 32-bit
1260         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1261         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1262         int offset = ra_->reg2offset(dst_first);
1263         if (cbuf) {
1264           MacroAssembler _masm(cbuf);
1265           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("movss   [rsp + #%d], %s\t# spill",
1269                      offset,
1270                      Matcher::regName[src_first]);
1271 #endif
1272         }
1273       }
1274       return 0;
1275     } else if (dst_first_rc == rc_int) {
1276       // xmm -> gpr
1277       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1278           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1279         // 64-bit
1280         if (cbuf) {
1281           MacroAssembler _masm(cbuf);
1282           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1283 #ifndef PRODUCT
1284         } else {
1285           st->print("movdq   %s, %s\t# spill",
1286                      Matcher::regName[dst_first],
1287                      Matcher::regName[src_first]);
1288 #endif
1289         }
1290       } else {
1291         // 32-bit
1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1294         if (cbuf) {
1295           MacroAssembler _masm(cbuf);
1296           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1297 #ifndef PRODUCT
1298         } else {
1299           st->print("movdl   %s, %s\t# spill",
1300                      Matcher::regName[dst_first],
1301                      Matcher::regName[src_first]);
1302 #endif
1303         }
1304       }
1305       return 0;
1306     } else if (dst_first_rc == rc_float) {
1307       // xmm -> xmm
1308       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1309           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1310         // 64-bit
1311         if (cbuf) {
1312           MacroAssembler _masm(cbuf);
1313           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1314 #ifndef PRODUCT
1315         } else {
1316           st->print("%s  %s, %s\t# spill",
1317                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1318                      Matcher::regName[dst_first],
1319                      Matcher::regName[src_first]);
1320 #endif
1321         }
1322       } else {
1323         // 32-bit
1324         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1325         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1326         if (cbuf) {
1327           MacroAssembler _masm(cbuf);
1328           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1329 #ifndef PRODUCT
1330         } else {
1331           st->print("%s  %s, %s\t# spill",
1332                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1333                      Matcher::regName[dst_first],
1334                      Matcher::regName[src_first]);
1335 #endif
1336         }
1337       }
1338       return 0;
1339     }
1340   }
1341 
1342   assert(0," foo ");
1343   Unimplemented();
1344   return 0;
1345 }
1346 
1347 #ifndef PRODUCT
1348 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1349   implementation(NULL, ra_, false, st);
1350 }
1351 #endif
1352 
1353 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1354   implementation(&cbuf, ra_, false, NULL);
1355 }
1356 
1357 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1358   return MachNode::size(ra_);
1359 }
1360 
1361 //=============================================================================
1362 #ifndef PRODUCT
1363 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1364 {
1365   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1366   int reg = ra_->get_reg_first(this);
1367   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1368             Matcher::regName[reg], offset);
1369 }
1370 #endif
1371 
1372 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1373 {
1374   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1375   int reg = ra_->get_encode(this);
1376   if (offset >= 0x80) {
1377     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1378     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1379     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1380     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1381     emit_d32(cbuf, offset);
1382   } else {
1383     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1384     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1385     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1386     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1387     emit_d8(cbuf, offset);
1388   }
1389 }
1390 
1391 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1392 {
1393   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1394   return (offset < 0x80) ? 5 : 8; // REX
1395 }
1396 
1397 //=============================================================================
1398 #ifndef PRODUCT
1399 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1400 {
1401   if (UseCompressedClassPointers) {
1402     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1403     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1404     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1405   } else {
1406     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1407                  "# Inline cache check");
1408   }
1409   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1410   st->print_cr("\tnop\t# nops to align entry point");
1411 }
1412 #endif
1413 
1414 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1415 {
1416   MacroAssembler masm(&cbuf);
1417   uint insts_size = cbuf.insts_size();
1418   if (UseCompressedClassPointers) {
1419     masm.load_klass(rscratch1, j_rarg0);
1420     masm.cmpptr(rax, rscratch1);
1421   } else {
1422     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1423   }
1424 
1425   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1426 
1427   /* WARNING these NOPs are critical so that verified entry point is properly
1428      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1429   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1430   if (OptoBreakpoint) {
1431     // Leave space for int3
1432     nops_cnt -= 1;
1433   }
1434   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1435   if (nops_cnt > 0)
1436     masm.nop(nops_cnt);
1437 }
1438 
1439 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1440 {
1441   return MachNode::size(ra_); // too many variables; just compute it
1442                               // the hard way
1443 }
1444  
1445 
1446 //=============================================================================
1447 
1448 int Matcher::regnum_to_fpu_offset(int regnum)
1449 {
1450   return regnum - 32; // The FP registers are in the second chunk
1451 }
1452 
1453 // This is UltraSparc specific, true just means we have fast l2f conversion
1454 const bool Matcher::convL2FSupported(void) {
1455   return true;
1456 }
1457 
1458 // Is this branch offset short enough that a short branch can be used?
1459 //
1460 // NOTE: If the platform does not provide any short branch variants, then
1461 //       this method should return false for offset 0.
1462 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1463   // The passed offset is relative to address of the branch.
1464   // On 86 a branch displacement is calculated relative to address
1465   // of a next instruction.
1466   offset -= br_size;
1467 
1468   // the short version of jmpConUCF2 contains multiple branches,
1469   // making the reach slightly less
1470   if (rule == jmpConUCF2_rule)
1471     return (-126 <= offset && offset <= 125);
1472   return (-128 <= offset && offset <= 127);
1473 }
1474 
1475 const bool Matcher::isSimpleConstant64(jlong value) {
1476   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1477   //return value == (int) value;  // Cf. storeImmL and immL32.
1478 
1479   // Probably always true, even if a temp register is required.
1480   return true;
1481 }
1482 
1483 // The ecx parameter to rep stosq for the ClearArray node is in words.
1484 const bool Matcher::init_array_count_is_in_bytes = false;
1485 
1486 // Threshold size for cleararray.
1487 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1488 
1489 // No additional cost for CMOVL.
1490 const int Matcher::long_cmove_cost() { return 0; }
1491 
1492 // No CMOVF/CMOVD with SSE2
1493 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1494 
1495 // Does the CPU require late expand (see block.cpp for description of late expand)?
1496 const bool Matcher::require_postalloc_expand = false;
1497 
1498 // Should the Matcher clone shifts on addressing modes, expecting them
1499 // to be subsumed into complex addressing expressions or compute them
1500 // into registers?  True for Intel but false for most RISCs
1501 const bool Matcher::clone_shift_expressions = true;
1502 
1503 // Do we need to mask the count passed to shift instructions or does
1504 // the cpu only look at the lower 5/6 bits anyway?
1505 const bool Matcher::need_masked_shift_count = false;
1506 
1507 bool Matcher::narrow_oop_use_complex_address() {
1508   assert(UseCompressedOops, "only for compressed oops code");
1509   return (LogMinObjAlignmentInBytes <= 3);
1510 }
1511 
1512 bool Matcher::narrow_klass_use_complex_address() {
1513   assert(UseCompressedClassPointers, "only for compressed klass code");
1514   return (LogKlassAlignmentInBytes <= 3);
1515 }
1516 
1517 // Is it better to copy float constants, or load them directly from
1518 // memory?  Intel can load a float constant from a direct address,
1519 // requiring no extra registers.  Most RISCs will have to materialize
1520 // an address into a register first, so they would do better to copy
1521 // the constant from stack.
1522 const bool Matcher::rematerialize_float_constants = true; // XXX
1523 
1524 // If CPU can load and store mis-aligned doubles directly then no
1525 // fixup is needed.  Else we split the double into 2 integer pieces
1526 // and move it piece-by-piece.  Only happens when passing doubles into
1527 // C code as the Java calling convention forces doubles to be aligned.
1528 const bool Matcher::misaligned_doubles_ok = true;
1529 
1530 // No-op on amd64
1531 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1532 
1533 // Advertise here if the CPU requires explicit rounding operations to
1534 // implement the UseStrictFP mode.
1535 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1536 
1537 // Are floats conerted to double when stored to stack during deoptimization?
1538 // On x64 it is stored without convertion so we can use normal access.
1539 bool Matcher::float_in_double() { return false; }
1540 
1541 // Do ints take an entire long register or just half?
1542 const bool Matcher::int_in_long = true;
1543 
1544 // Return whether or not this register is ever used as an argument.
1545 // This function is used on startup to build the trampoline stubs in
1546 // generateOptoStub.  Registers not mentioned will be killed by the VM
1547 // call in the trampoline, and arguments in those registers not be
1548 // available to the callee.
1549 bool Matcher::can_be_java_arg(int reg)
1550 {
1551   return
1552     reg ==  RDI_num || reg == RDI_H_num ||
1553     reg ==  RSI_num || reg == RSI_H_num ||
1554     reg ==  RDX_num || reg == RDX_H_num ||
1555     reg ==  RCX_num || reg == RCX_H_num ||
1556     reg ==   R8_num || reg ==  R8_H_num ||
1557     reg ==   R9_num || reg ==  R9_H_num ||
1558     reg ==  R12_num || reg == R12_H_num ||
1559     reg == XMM0_num || reg == XMM0b_num ||
1560     reg == XMM1_num || reg == XMM1b_num ||
1561     reg == XMM2_num || reg == XMM2b_num ||
1562     reg == XMM3_num || reg == XMM3b_num ||
1563     reg == XMM4_num || reg == XMM4b_num ||
1564     reg == XMM5_num || reg == XMM5b_num ||
1565     reg == XMM6_num || reg == XMM6b_num ||
1566     reg == XMM7_num || reg == XMM7b_num;
1567 }
1568 
1569 bool Matcher::is_spillable_arg(int reg)
1570 {
1571   return can_be_java_arg(reg);
1572 }
1573 
1574 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1575   // In 64 bit mode a code which use multiply when
1576   // devisor is constant is faster than hardware
1577   // DIV instruction (it uses MulHiL).
1578   return false;
1579 }
1580 
1581 // Register for DIVI projection of divmodI
1582 RegMask Matcher::divI_proj_mask() {
1583   return INT_RAX_REG_mask();
1584 }
1585 
1586 // Register for MODI projection of divmodI
1587 RegMask Matcher::modI_proj_mask() {
1588   return INT_RDX_REG_mask();
1589 }
1590 
1591 // Register for DIVL projection of divmodL
1592 RegMask Matcher::divL_proj_mask() {
1593   return LONG_RAX_REG_mask();
1594 }
1595 
1596 // Register for MODL projection of divmodL
1597 RegMask Matcher::modL_proj_mask() {
1598   return LONG_RDX_REG_mask();
1599 }
1600 
1601 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1602   return PTR_RBP_REG_mask();
1603 }
1604 
1605 %}
1606 
1607 //----------ENCODING BLOCK-----------------------------------------------------
1608 // This block specifies the encoding classes used by the compiler to
1609 // output byte streams.  Encoding classes are parameterized macros
1610 // used by Machine Instruction Nodes in order to generate the bit
1611 // encoding of the instruction.  Operands specify their base encoding
1612 // interface with the interface keyword.  There are currently
1613 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1614 // COND_INTER.  REG_INTER causes an operand to generate a function
1615 // which returns its register number when queried.  CONST_INTER causes
1616 // an operand to generate a function which returns the value of the
1617 // constant when queried.  MEMORY_INTER causes an operand to generate
1618 // four functions which return the Base Register, the Index Register,
1619 // the Scale Value, and the Offset Value of the operand when queried.
1620 // COND_INTER causes an operand to generate six functions which return
1621 // the encoding code (ie - encoding bits for the instruction)
1622 // associated with each basic boolean condition for a conditional
1623 // instruction.
1624 //
1625 // Instructions specify two basic values for encoding.  Again, a
1626 // function is available to check if the constant displacement is an
1627 // oop. They use the ins_encode keyword to specify their encoding
1628 // classes (which must be a sequence of enc_class names, and their
1629 // parameters, specified in the encoding block), and they use the
1630 // opcode keyword to specify, in order, their primary, secondary, and
1631 // tertiary opcode.  Only the opcode sections which a particular
1632 // instruction needs for encoding need to be specified.
1633 encode %{
1634   // Build emit functions for each basic byte or larger field in the
1635   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1636   // from C++ code in the enc_class source block.  Emit functions will
1637   // live in the main source block for now.  In future, we can
1638   // generalize this by adding a syntax that specifies the sizes of
1639   // fields in an order, so that the adlc can build the emit functions
1640   // automagically
1641 
1642   // Emit primary opcode
1643   enc_class OpcP
1644   %{
1645     emit_opcode(cbuf, $primary);
1646   %}
1647 
1648   // Emit secondary opcode
1649   enc_class OpcS
1650   %{
1651     emit_opcode(cbuf, $secondary);
1652   %}
1653 
1654   // Emit tertiary opcode
1655   enc_class OpcT
1656   %{
1657     emit_opcode(cbuf, $tertiary);
1658   %}
1659 
1660   // Emit opcode directly
1661   enc_class Opcode(immI d8)
1662   %{
1663     emit_opcode(cbuf, $d8$$constant);
1664   %}
1665 
1666   // Emit size prefix
1667   enc_class SizePrefix
1668   %{
1669     emit_opcode(cbuf, 0x66);
1670   %}
1671 
1672   enc_class reg(rRegI reg)
1673   %{
1674     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1675   %}
1676 
1677   enc_class reg_reg(rRegI dst, rRegI src)
1678   %{
1679     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1680   %}
1681 
1682   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1683   %{
1684     emit_opcode(cbuf, $opcode$$constant);
1685     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1686   %}
1687 
1688   enc_class cdql_enc(no_rax_rdx_RegI div)
1689   %{
1690     // Full implementation of Java idiv and irem; checks for
1691     // special case as described in JVM spec., p.243 & p.271.
1692     //
1693     //         normal case                           special case
1694     //
1695     // input : rax: dividend                         min_int
1696     //         reg: divisor                          -1
1697     //
1698     // output: rax: quotient  (= rax idiv reg)       min_int
1699     //         rdx: remainder (= rax irem reg)       0
1700     //
1701     //  Code sequnce:
1702     //
1703     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1704     //    5:   75 07/08                jne    e <normal>
1705     //    7:   33 d2                   xor    %edx,%edx
1706     //  [div >= 8 -> offset + 1]
1707     //  [REX_B]
1708     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1709     //    c:   74 03/04                je     11 <done>
1710     // 000000000000000e <normal>:
1711     //    e:   99                      cltd
1712     //  [div >= 8 -> offset + 1]
1713     //  [REX_B]
1714     //    f:   f7 f9                   idiv   $div
1715     // 0000000000000011 <done>:
1716 
1717     // cmp    $0x80000000,%eax
1718     emit_opcode(cbuf, 0x3d);
1719     emit_d8(cbuf, 0x00);
1720     emit_d8(cbuf, 0x00);
1721     emit_d8(cbuf, 0x00);
1722     emit_d8(cbuf, 0x80);
1723 
1724     // jne    e <normal>
1725     emit_opcode(cbuf, 0x75);
1726     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1727 
1728     // xor    %edx,%edx
1729     emit_opcode(cbuf, 0x33);
1730     emit_d8(cbuf, 0xD2);
1731 
1732     // cmp    $0xffffffffffffffff,%ecx
1733     if ($div$$reg >= 8) {
1734       emit_opcode(cbuf, Assembler::REX_B);
1735     }
1736     emit_opcode(cbuf, 0x83);
1737     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1738     emit_d8(cbuf, 0xFF);
1739 
1740     // je     11 <done>
1741     emit_opcode(cbuf, 0x74);
1742     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1743 
1744     // <normal>
1745     // cltd
1746     emit_opcode(cbuf, 0x99);
1747 
1748     // idivl (note: must be emitted by the user of this rule)
1749     // <done>
1750   %}
1751 
1752   enc_class cdqq_enc(no_rax_rdx_RegL div)
1753   %{
1754     // Full implementation of Java ldiv and lrem; checks for
1755     // special case as described in JVM spec., p.243 & p.271.
1756     //
1757     //         normal case                           special case
1758     //
1759     // input : rax: dividend                         min_long
1760     //         reg: divisor                          -1
1761     //
1762     // output: rax: quotient  (= rax idiv reg)       min_long
1763     //         rdx: remainder (= rax irem reg)       0
1764     //
1765     //  Code sequnce:
1766     //
1767     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1768     //    7:   00 00 80
1769     //    a:   48 39 d0                cmp    %rdx,%rax
1770     //    d:   75 08                   jne    17 <normal>
1771     //    f:   33 d2                   xor    %edx,%edx
1772     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1773     //   15:   74 05                   je     1c <done>
1774     // 0000000000000017 <normal>:
1775     //   17:   48 99                   cqto
1776     //   19:   48 f7 f9                idiv   $div
1777     // 000000000000001c <done>:
1778 
1779     // mov    $0x8000000000000000,%rdx
1780     emit_opcode(cbuf, Assembler::REX_W);
1781     emit_opcode(cbuf, 0xBA);
1782     emit_d8(cbuf, 0x00);
1783     emit_d8(cbuf, 0x00);
1784     emit_d8(cbuf, 0x00);
1785     emit_d8(cbuf, 0x00);
1786     emit_d8(cbuf, 0x00);
1787     emit_d8(cbuf, 0x00);
1788     emit_d8(cbuf, 0x00);
1789     emit_d8(cbuf, 0x80);
1790 
1791     // cmp    %rdx,%rax
1792     emit_opcode(cbuf, Assembler::REX_W);
1793     emit_opcode(cbuf, 0x39);
1794     emit_d8(cbuf, 0xD0);
1795 
1796     // jne    17 <normal>
1797     emit_opcode(cbuf, 0x75);
1798     emit_d8(cbuf, 0x08);
1799 
1800     // xor    %edx,%edx
1801     emit_opcode(cbuf, 0x33);
1802     emit_d8(cbuf, 0xD2);
1803 
1804     // cmp    $0xffffffffffffffff,$div
1805     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1806     emit_opcode(cbuf, 0x83);
1807     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1808     emit_d8(cbuf, 0xFF);
1809 
1810     // je     1e <done>
1811     emit_opcode(cbuf, 0x74);
1812     emit_d8(cbuf, 0x05);
1813 
1814     // <normal>
1815     // cqto
1816     emit_opcode(cbuf, Assembler::REX_W);
1817     emit_opcode(cbuf, 0x99);
1818 
1819     // idivq (note: must be emitted by the user of this rule)
1820     // <done>
1821   %}
1822 
1823   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1824   enc_class OpcSE(immI imm)
1825   %{
1826     // Emit primary opcode and set sign-extend bit
1827     // Check for 8-bit immediate, and set sign extend bit in opcode
1828     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1829       emit_opcode(cbuf, $primary | 0x02);
1830     } else {
1831       // 32-bit immediate
1832       emit_opcode(cbuf, $primary);
1833     }
1834   %}
1835 
1836   enc_class OpcSErm(rRegI dst, immI imm)
1837   %{
1838     // OpcSEr/m
1839     int dstenc = $dst$$reg;
1840     if (dstenc >= 8) {
1841       emit_opcode(cbuf, Assembler::REX_B);
1842       dstenc -= 8;
1843     }
1844     // Emit primary opcode and set sign-extend bit
1845     // Check for 8-bit immediate, and set sign extend bit in opcode
1846     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1847       emit_opcode(cbuf, $primary | 0x02);
1848     } else {
1849       // 32-bit immediate
1850       emit_opcode(cbuf, $primary);
1851     }
1852     // Emit r/m byte with secondary opcode, after primary opcode.
1853     emit_rm(cbuf, 0x3, $secondary, dstenc);
1854   %}
1855 
1856   enc_class OpcSErm_wide(rRegL dst, immI imm)
1857   %{
1858     // OpcSEr/m
1859     int dstenc = $dst$$reg;
1860     if (dstenc < 8) {
1861       emit_opcode(cbuf, Assembler::REX_W);
1862     } else {
1863       emit_opcode(cbuf, Assembler::REX_WB);
1864       dstenc -= 8;
1865     }
1866     // Emit primary opcode and set sign-extend bit
1867     // Check for 8-bit immediate, and set sign extend bit in opcode
1868     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1869       emit_opcode(cbuf, $primary | 0x02);
1870     } else {
1871       // 32-bit immediate
1872       emit_opcode(cbuf, $primary);
1873     }
1874     // Emit r/m byte with secondary opcode, after primary opcode.
1875     emit_rm(cbuf, 0x3, $secondary, dstenc);
1876   %}
1877 
1878   enc_class Con8or32(immI imm)
1879   %{
1880     // Check for 8-bit immediate, and set sign extend bit in opcode
1881     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1882       $$$emit8$imm$$constant;
1883     } else {
1884       // 32-bit immediate
1885       $$$emit32$imm$$constant;
1886     }
1887   %}
1888 
1889   enc_class opc2_reg(rRegI dst)
1890   %{
1891     // BSWAP
1892     emit_cc(cbuf, $secondary, $dst$$reg);
1893   %}
1894 
1895   enc_class opc3_reg(rRegI dst)
1896   %{
1897     // BSWAP
1898     emit_cc(cbuf, $tertiary, $dst$$reg);
1899   %}
1900 
1901   enc_class reg_opc(rRegI div)
1902   %{
1903     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1904     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1905   %}
1906 
1907   enc_class enc_cmov(cmpOp cop)
1908   %{
1909     // CMOV
1910     $$$emit8$primary;
1911     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1912   %}
1913 
1914   enc_class enc_PartialSubtypeCheck()
1915   %{
1916     Register Rrdi = as_Register(RDI_enc); // result register
1917     Register Rrax = as_Register(RAX_enc); // super class
1918     Register Rrcx = as_Register(RCX_enc); // killed
1919     Register Rrsi = as_Register(RSI_enc); // sub class
1920     Label miss;
1921     const bool set_cond_codes = true;
1922 
1923     MacroAssembler _masm(&cbuf);
1924     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1925                                      NULL, &miss,
1926                                      /*set_cond_codes:*/ true);
1927     if ($primary) {
1928       __ xorptr(Rrdi, Rrdi);
1929     }
1930     __ bind(miss);
1931   %}
1932 
1933   enc_class clear_avx %{
1934     debug_only(int off0 = cbuf.insts_size());
1935     if (ra_->C->max_vector_size() > 16) {
1936       // Clear upper bits of YMM registers when current compiled code uses
1937       // wide vectors to avoid AVX <-> SSE transition penalty during call.
1938       MacroAssembler _masm(&cbuf);
1939       __ vzeroupper();
1940     }
1941     debug_only(int off1 = cbuf.insts_size());
1942     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1943   %}
1944 
1945   enc_class Java_To_Runtime(method meth) %{
1946     // No relocation needed
1947     MacroAssembler _masm(&cbuf);
1948     __ mov64(r10, (int64_t) $meth$$method);
1949     __ call(r10);
1950   %}
1951 
1952   enc_class Java_To_Interpreter(method meth)
1953   %{
1954     // CALL Java_To_Interpreter
1955     // This is the instruction starting address for relocation info.
1956     cbuf.set_insts_mark();
1957     $$$emit8$primary;
1958     // CALL directly to the runtime
1959     emit_d32_reloc(cbuf,
1960                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1961                    runtime_call_Relocation::spec(),
1962                    RELOC_DISP32);
1963   %}
1964 
1965   enc_class Java_Static_Call(method meth)
1966   %{
1967     // JAVA STATIC CALL
1968     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
1969     // determine who we intended to call.
1970     cbuf.set_insts_mark();
1971     $$$emit8$primary;
1972 
1973     if (!_method) {
1974       emit_d32_reloc(cbuf,
1975                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1976                      runtime_call_Relocation::spec(),
1977                      RELOC_DISP32);
1978     } else if (_optimized_virtual) {
1979       emit_d32_reloc(cbuf,
1980                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1981                      opt_virtual_call_Relocation::spec(),
1982                      RELOC_DISP32);
1983     } else {
1984       emit_d32_reloc(cbuf,
1985                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1986                      static_call_Relocation::spec(),
1987                      RELOC_DISP32);
1988     }
1989     if (_method) {
1990       // Emit stub for static call.
1991       CompiledStaticCall::emit_to_interp_stub(cbuf);
1992     }
1993   %}
1994 
1995   enc_class Java_Dynamic_Call(method meth) %{
1996     MacroAssembler _masm(&cbuf);
1997     __ ic_call((address)$meth$$method);
1998   %}
1999 
2000   enc_class Java_Compiled_Call(method meth)
2001   %{
2002     // JAVA COMPILED CALL
2003     int disp = in_bytes(Method:: from_compiled_offset());
2004 
2005     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2006     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2007 
2008     // callq *disp(%rax)
2009     cbuf.set_insts_mark();
2010     $$$emit8$primary;
2011     if (disp < 0x80) {
2012       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2013       emit_d8(cbuf, disp); // Displacement
2014     } else {
2015       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2016       emit_d32(cbuf, disp); // Displacement
2017     }
2018   %}
2019 
2020   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2021   %{
2022     // SAL, SAR, SHR
2023     int dstenc = $dst$$reg;
2024     if (dstenc >= 8) {
2025       emit_opcode(cbuf, Assembler::REX_B);
2026       dstenc -= 8;
2027     }
2028     $$$emit8$primary;
2029     emit_rm(cbuf, 0x3, $secondary, dstenc);
2030     $$$emit8$shift$$constant;
2031   %}
2032 
2033   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2034   %{
2035     // SAL, SAR, SHR
2036     int dstenc = $dst$$reg;
2037     if (dstenc < 8) {
2038       emit_opcode(cbuf, Assembler::REX_W);
2039     } else {
2040       emit_opcode(cbuf, Assembler::REX_WB);
2041       dstenc -= 8;
2042     }
2043     $$$emit8$primary;
2044     emit_rm(cbuf, 0x3, $secondary, dstenc);
2045     $$$emit8$shift$$constant;
2046   %}
2047 
2048   enc_class load_immI(rRegI dst, immI src)
2049   %{
2050     int dstenc = $dst$$reg;
2051     if (dstenc >= 8) {
2052       emit_opcode(cbuf, Assembler::REX_B);
2053       dstenc -= 8;
2054     }
2055     emit_opcode(cbuf, 0xB8 | dstenc);
2056     $$$emit32$src$$constant;
2057   %}
2058 
2059   enc_class load_immL(rRegL dst, immL src)
2060   %{
2061     int dstenc = $dst$$reg;
2062     if (dstenc < 8) {
2063       emit_opcode(cbuf, Assembler::REX_W);
2064     } else {
2065       emit_opcode(cbuf, Assembler::REX_WB);
2066       dstenc -= 8;
2067     }
2068     emit_opcode(cbuf, 0xB8 | dstenc);
2069     emit_d64(cbuf, $src$$constant);
2070   %}
2071 
2072   enc_class load_immUL32(rRegL dst, immUL32 src)
2073   %{
2074     // same as load_immI, but this time we care about zeroes in the high word
2075     int dstenc = $dst$$reg;
2076     if (dstenc >= 8) {
2077       emit_opcode(cbuf, Assembler::REX_B);
2078       dstenc -= 8;
2079     }
2080     emit_opcode(cbuf, 0xB8 | dstenc);
2081     $$$emit32$src$$constant;
2082   %}
2083 
2084   enc_class load_immL32(rRegL dst, immL32 src)
2085   %{
2086     int dstenc = $dst$$reg;
2087     if (dstenc < 8) {
2088       emit_opcode(cbuf, Assembler::REX_W);
2089     } else {
2090       emit_opcode(cbuf, Assembler::REX_WB);
2091       dstenc -= 8;
2092     }
2093     emit_opcode(cbuf, 0xC7);
2094     emit_rm(cbuf, 0x03, 0x00, dstenc);
2095     $$$emit32$src$$constant;
2096   %}
2097 
2098   enc_class load_immP31(rRegP dst, immP32 src)
2099   %{
2100     // same as load_immI, but this time we care about zeroes in the high word
2101     int dstenc = $dst$$reg;
2102     if (dstenc >= 8) {
2103       emit_opcode(cbuf, Assembler::REX_B);
2104       dstenc -= 8;
2105     }
2106     emit_opcode(cbuf, 0xB8 | dstenc);
2107     $$$emit32$src$$constant;
2108   %}
2109 
2110   enc_class load_immP(rRegP dst, immP src)
2111   %{
2112     int dstenc = $dst$$reg;
2113     if (dstenc < 8) {
2114       emit_opcode(cbuf, Assembler::REX_W);
2115     } else {
2116       emit_opcode(cbuf, Assembler::REX_WB);
2117       dstenc -= 8;
2118     }
2119     emit_opcode(cbuf, 0xB8 | dstenc);
2120     // This next line should be generated from ADLC
2121     if ($src->constant_reloc() != relocInfo::none) {
2122       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2123     } else {
2124       emit_d64(cbuf, $src$$constant);
2125     }
2126   %}
2127 
2128   enc_class Con32(immI src)
2129   %{
2130     // Output immediate
2131     $$$emit32$src$$constant;
2132   %}
2133 
2134   enc_class Con32F_as_bits(immF src)
2135   %{
2136     // Output Float immediate bits
2137     jfloat jf = $src$$constant;
2138     jint jf_as_bits = jint_cast(jf);
2139     emit_d32(cbuf, jf_as_bits);
2140   %}
2141 
2142   enc_class Con16(immI src)
2143   %{
2144     // Output immediate
2145     $$$emit16$src$$constant;
2146   %}
2147 
2148   // How is this different from Con32??? XXX
2149   enc_class Con_d32(immI src)
2150   %{
2151     emit_d32(cbuf,$src$$constant);
2152   %}
2153 
2154   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2155     // Output immediate memory reference
2156     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2157     emit_d32(cbuf, 0x00);
2158   %}
2159 
2160   enc_class lock_prefix()
2161   %{
2162     if (os::is_MP()) {
2163       emit_opcode(cbuf, 0xF0); // lock
2164     }
2165   %}
2166 
2167   enc_class REX_mem(memory mem)
2168   %{
2169     if ($mem$$base >= 8) {
2170       if ($mem$$index < 8) {
2171         emit_opcode(cbuf, Assembler::REX_B);
2172       } else {
2173         emit_opcode(cbuf, Assembler::REX_XB);
2174       }
2175     } else {
2176       if ($mem$$index >= 8) {
2177         emit_opcode(cbuf, Assembler::REX_X);
2178       }
2179     }
2180   %}
2181 
2182   enc_class REX_mem_wide(memory mem)
2183   %{
2184     if ($mem$$base >= 8) {
2185       if ($mem$$index < 8) {
2186         emit_opcode(cbuf, Assembler::REX_WB);
2187       } else {
2188         emit_opcode(cbuf, Assembler::REX_WXB);
2189       }
2190     } else {
2191       if ($mem$$index < 8) {
2192         emit_opcode(cbuf, Assembler::REX_W);
2193       } else {
2194         emit_opcode(cbuf, Assembler::REX_WX);
2195       }
2196     }
2197   %}
2198 
2199   // for byte regs
2200   enc_class REX_breg(rRegI reg)
2201   %{
2202     if ($reg$$reg >= 4) {
2203       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2204     }
2205   %}
2206 
2207   // for byte regs
2208   enc_class REX_reg_breg(rRegI dst, rRegI src)
2209   %{
2210     if ($dst$$reg < 8) {
2211       if ($src$$reg >= 4) {
2212         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2213       }
2214     } else {
2215       if ($src$$reg < 8) {
2216         emit_opcode(cbuf, Assembler::REX_R);
2217       } else {
2218         emit_opcode(cbuf, Assembler::REX_RB);
2219       }
2220     }
2221   %}
2222 
2223   // for byte regs
2224   enc_class REX_breg_mem(rRegI reg, memory mem)
2225   %{
2226     if ($reg$$reg < 8) {
2227       if ($mem$$base < 8) {
2228         if ($mem$$index >= 8) {
2229           emit_opcode(cbuf, Assembler::REX_X);
2230         } else if ($reg$$reg >= 4) {
2231           emit_opcode(cbuf, Assembler::REX);
2232         }
2233       } else {
2234         if ($mem$$index < 8) {
2235           emit_opcode(cbuf, Assembler::REX_B);
2236         } else {
2237           emit_opcode(cbuf, Assembler::REX_XB);
2238         }
2239       }
2240     } else {
2241       if ($mem$$base < 8) {
2242         if ($mem$$index < 8) {
2243           emit_opcode(cbuf, Assembler::REX_R);
2244         } else {
2245           emit_opcode(cbuf, Assembler::REX_RX);
2246         }
2247       } else {
2248         if ($mem$$index < 8) {
2249           emit_opcode(cbuf, Assembler::REX_RB);
2250         } else {
2251           emit_opcode(cbuf, Assembler::REX_RXB);
2252         }
2253       }
2254     }
2255   %}
2256 
2257   enc_class REX_reg(rRegI reg)
2258   %{
2259     if ($reg$$reg >= 8) {
2260       emit_opcode(cbuf, Assembler::REX_B);
2261     }
2262   %}
2263 
2264   enc_class REX_reg_wide(rRegI reg)
2265   %{
2266     if ($reg$$reg < 8) {
2267       emit_opcode(cbuf, Assembler::REX_W);
2268     } else {
2269       emit_opcode(cbuf, Assembler::REX_WB);
2270     }
2271   %}
2272 
2273   enc_class REX_reg_reg(rRegI dst, rRegI src)
2274   %{
2275     if ($dst$$reg < 8) {
2276       if ($src$$reg >= 8) {
2277         emit_opcode(cbuf, Assembler::REX_B);
2278       }
2279     } else {
2280       if ($src$$reg < 8) {
2281         emit_opcode(cbuf, Assembler::REX_R);
2282       } else {
2283         emit_opcode(cbuf, Assembler::REX_RB);
2284       }
2285     }
2286   %}
2287 
2288   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2289   %{
2290     if ($dst$$reg < 8) {
2291       if ($src$$reg < 8) {
2292         emit_opcode(cbuf, Assembler::REX_W);
2293       } else {
2294         emit_opcode(cbuf, Assembler::REX_WB);
2295       }
2296     } else {
2297       if ($src$$reg < 8) {
2298         emit_opcode(cbuf, Assembler::REX_WR);
2299       } else {
2300         emit_opcode(cbuf, Assembler::REX_WRB);
2301       }
2302     }
2303   %}
2304 
2305   enc_class REX_reg_mem(rRegI reg, memory mem)
2306   %{
2307     if ($reg$$reg < 8) {
2308       if ($mem$$base < 8) {
2309         if ($mem$$index >= 8) {
2310           emit_opcode(cbuf, Assembler::REX_X);
2311         }
2312       } else {
2313         if ($mem$$index < 8) {
2314           emit_opcode(cbuf, Assembler::REX_B);
2315         } else {
2316           emit_opcode(cbuf, Assembler::REX_XB);
2317         }
2318       }
2319     } else {
2320       if ($mem$$base < 8) {
2321         if ($mem$$index < 8) {
2322           emit_opcode(cbuf, Assembler::REX_R);
2323         } else {
2324           emit_opcode(cbuf, Assembler::REX_RX);
2325         }
2326       } else {
2327         if ($mem$$index < 8) {
2328           emit_opcode(cbuf, Assembler::REX_RB);
2329         } else {
2330           emit_opcode(cbuf, Assembler::REX_RXB);
2331         }
2332       }
2333     }
2334   %}
2335 
2336   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2337   %{
2338     if ($reg$$reg < 8) {
2339       if ($mem$$base < 8) {
2340         if ($mem$$index < 8) {
2341           emit_opcode(cbuf, Assembler::REX_W);
2342         } else {
2343           emit_opcode(cbuf, Assembler::REX_WX);
2344         }
2345       } else {
2346         if ($mem$$index < 8) {
2347           emit_opcode(cbuf, Assembler::REX_WB);
2348         } else {
2349           emit_opcode(cbuf, Assembler::REX_WXB);
2350         }
2351       }
2352     } else {
2353       if ($mem$$base < 8) {
2354         if ($mem$$index < 8) {
2355           emit_opcode(cbuf, Assembler::REX_WR);
2356         } else {
2357           emit_opcode(cbuf, Assembler::REX_WRX);
2358         }
2359       } else {
2360         if ($mem$$index < 8) {
2361           emit_opcode(cbuf, Assembler::REX_WRB);
2362         } else {
2363           emit_opcode(cbuf, Assembler::REX_WRXB);
2364         }
2365       }
2366     }
2367   %}
2368 
2369   enc_class reg_mem(rRegI ereg, memory mem)
2370   %{
2371     // High registers handle in encode_RegMem
2372     int reg = $ereg$$reg;
2373     int base = $mem$$base;
2374     int index = $mem$$index;
2375     int scale = $mem$$scale;
2376     int disp = $mem$$disp;
2377     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2378 
2379     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2380   %}
2381 
2382   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2383   %{
2384     int rm_byte_opcode = $rm_opcode$$constant;
2385 
2386     // High registers handle in encode_RegMem
2387     int base = $mem$$base;
2388     int index = $mem$$index;
2389     int scale = $mem$$scale;
2390     int displace = $mem$$disp;
2391 
2392     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2393                                             // working with static
2394                                             // globals
2395     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2396                   disp_reloc);
2397   %}
2398 
2399   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2400   %{
2401     int reg_encoding = $dst$$reg;
2402     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2403     int index        = 0x04;            // 0x04 indicates no index
2404     int scale        = 0x00;            // 0x00 indicates no scale
2405     int displace     = $src1$$constant; // 0x00 indicates no displacement
2406     relocInfo::relocType disp_reloc = relocInfo::none;
2407     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2408                   disp_reloc);
2409   %}
2410 
2411   enc_class neg_reg(rRegI dst)
2412   %{
2413     int dstenc = $dst$$reg;
2414     if (dstenc >= 8) {
2415       emit_opcode(cbuf, Assembler::REX_B);
2416       dstenc -= 8;
2417     }
2418     // NEG $dst
2419     emit_opcode(cbuf, 0xF7);
2420     emit_rm(cbuf, 0x3, 0x03, dstenc);
2421   %}
2422 
2423   enc_class neg_reg_wide(rRegI dst)
2424   %{
2425     int dstenc = $dst$$reg;
2426     if (dstenc < 8) {
2427       emit_opcode(cbuf, Assembler::REX_W);
2428     } else {
2429       emit_opcode(cbuf, Assembler::REX_WB);
2430       dstenc -= 8;
2431     }
2432     // NEG $dst
2433     emit_opcode(cbuf, 0xF7);
2434     emit_rm(cbuf, 0x3, 0x03, dstenc);
2435   %}
2436 
2437   enc_class setLT_reg(rRegI dst)
2438   %{
2439     int dstenc = $dst$$reg;
2440     if (dstenc >= 8) {
2441       emit_opcode(cbuf, Assembler::REX_B);
2442       dstenc -= 8;
2443     } else if (dstenc >= 4) {
2444       emit_opcode(cbuf, Assembler::REX);
2445     }
2446     // SETLT $dst
2447     emit_opcode(cbuf, 0x0F);
2448     emit_opcode(cbuf, 0x9C);
2449     emit_rm(cbuf, 0x3, 0x0, dstenc);
2450   %}
2451 
2452   enc_class setNZ_reg(rRegI dst)
2453   %{
2454     int dstenc = $dst$$reg;
2455     if (dstenc >= 8) {
2456       emit_opcode(cbuf, Assembler::REX_B);
2457       dstenc -= 8;
2458     } else if (dstenc >= 4) {
2459       emit_opcode(cbuf, Assembler::REX);
2460     }
2461     // SETNZ $dst
2462     emit_opcode(cbuf, 0x0F);
2463     emit_opcode(cbuf, 0x95);
2464     emit_rm(cbuf, 0x3, 0x0, dstenc);
2465   %}
2466 
2467 
2468   // Compare the lonogs and set -1, 0, or 1 into dst
2469   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2470   %{
2471     int src1enc = $src1$$reg;
2472     int src2enc = $src2$$reg;
2473     int dstenc = $dst$$reg;
2474 
2475     // cmpq $src1, $src2
2476     if (src1enc < 8) {
2477       if (src2enc < 8) {
2478         emit_opcode(cbuf, Assembler::REX_W);
2479       } else {
2480         emit_opcode(cbuf, Assembler::REX_WB);
2481       }
2482     } else {
2483       if (src2enc < 8) {
2484         emit_opcode(cbuf, Assembler::REX_WR);
2485       } else {
2486         emit_opcode(cbuf, Assembler::REX_WRB);
2487       }
2488     }
2489     emit_opcode(cbuf, 0x3B);
2490     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2491 
2492     // movl $dst, -1
2493     if (dstenc >= 8) {
2494       emit_opcode(cbuf, Assembler::REX_B);
2495     }
2496     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2497     emit_d32(cbuf, -1);
2498 
2499     // jl,s done
2500     emit_opcode(cbuf, 0x7C);
2501     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2502 
2503     // setne $dst
2504     if (dstenc >= 4) {
2505       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2506     }
2507     emit_opcode(cbuf, 0x0F);
2508     emit_opcode(cbuf, 0x95);
2509     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2510 
2511     // movzbl $dst, $dst
2512     if (dstenc >= 4) {
2513       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2514     }
2515     emit_opcode(cbuf, 0x0F);
2516     emit_opcode(cbuf, 0xB6);
2517     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2518   %}
2519 
2520   enc_class Push_ResultXD(regD dst) %{
2521     MacroAssembler _masm(&cbuf);
2522     __ fstp_d(Address(rsp, 0));
2523     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2524     __ addptr(rsp, 8);
2525   %}
2526 
2527   enc_class Push_SrcXD(regD src) %{
2528     MacroAssembler _masm(&cbuf);
2529     __ subptr(rsp, 8);
2530     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2531     __ fld_d(Address(rsp, 0));
2532   %}
2533 
2534 
2535   enc_class enc_rethrow()
2536   %{
2537     cbuf.set_insts_mark();
2538     emit_opcode(cbuf, 0xE9); // jmp entry
2539     emit_d32_reloc(cbuf,
2540                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2541                    runtime_call_Relocation::spec(),
2542                    RELOC_DISP32);
2543   %}
2544 
2545 %}
2546 
2547 
2548 
2549 //----------FRAME--------------------------------------------------------------
2550 // Definition of frame structure and management information.
2551 //
2552 //  S T A C K   L A Y O U T    Allocators stack-slot number
2553 //                             |   (to get allocators register number
2554 //  G  Owned by    |        |  v    add OptoReg::stack0())
2555 //  r   CALLER     |        |
2556 //  o     |        +--------+      pad to even-align allocators stack-slot
2557 //  w     V        |  pad0  |        numbers; owned by CALLER
2558 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2559 //  h     ^        |   in   |  5
2560 //        |        |  args  |  4   Holes in incoming args owned by SELF
2561 //  |     |        |        |  3
2562 //  |     |        +--------+
2563 //  V     |        | old out|      Empty on Intel, window on Sparc
2564 //        |    old |preserve|      Must be even aligned.
2565 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2566 //        |        |   in   |  3   area for Intel ret address
2567 //     Owned by    |preserve|      Empty on Sparc.
2568 //       SELF      +--------+
2569 //        |        |  pad2  |  2   pad to align old SP
2570 //        |        +--------+  1
2571 //        |        | locks  |  0
2572 //        |        +--------+----> OptoReg::stack0(), even aligned
2573 //        |        |  pad1  | 11   pad to align new SP
2574 //        |        +--------+
2575 //        |        |        | 10
2576 //        |        | spills |  9   spills
2577 //        V        |        |  8   (pad0 slot for callee)
2578 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2579 //        ^        |  out   |  7
2580 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2581 //     Owned by    +--------+
2582 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2583 //        |    new |preserve|      Must be even-aligned.
2584 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2585 //        |        |        |
2586 //
2587 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2588 //         known from SELF's arguments and the Java calling convention.
2589 //         Region 6-7 is determined per call site.
2590 // Note 2: If the calling convention leaves holes in the incoming argument
2591 //         area, those holes are owned by SELF.  Holes in the outgoing area
2592 //         are owned by the CALLEE.  Holes should not be nessecary in the
2593 //         incoming area, as the Java calling convention is completely under
2594 //         the control of the AD file.  Doubles can be sorted and packed to
2595 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2596 //         varargs C calling conventions.
2597 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2598 //         even aligned with pad0 as needed.
2599 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2600 //         region 6-11 is even aligned; it may be padded out more so that
2601 //         the region from SP to FP meets the minimum stack alignment.
2602 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2603 //         alignment.  Region 11, pad1, may be dynamically extended so that
2604 //         SP meets the minimum alignment.
2605 
2606 frame
2607 %{
2608   // What direction does stack grow in (assumed to be same for C & Java)
2609   stack_direction(TOWARDS_LOW);
2610 
2611   // These three registers define part of the calling convention
2612   // between compiled code and the interpreter.
2613   inline_cache_reg(RAX);                // Inline Cache Register
2614   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2615                                         // calling interpreter
2616 
2617   // Optional: name the operand used by cisc-spilling to access
2618   // [stack_pointer + offset]
2619   cisc_spilling_operand_name(indOffset32);
2620 
2621   // Number of stack slots consumed by locking an object
2622   sync_stack_slots(2);
2623 
2624   // Compiled code's Frame Pointer
2625   frame_pointer(RSP);
2626 
2627   // Interpreter stores its frame pointer in a register which is
2628   // stored to the stack by I2CAdaptors.
2629   // I2CAdaptors convert from interpreted java to compiled java.
2630   interpreter_frame_pointer(RBP);
2631 
2632   // Stack alignment requirement
2633   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2634 
2635   // Number of stack slots between incoming argument block and the start of
2636   // a new frame.  The PROLOG must add this many slots to the stack.  The
2637   // EPILOG must remove this many slots.  amd64 needs two slots for
2638   // return address.
2639   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2640 
2641   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2642   // for calls to C.  Supports the var-args backing area for register parms.
2643   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2644 
2645   // The after-PROLOG location of the return address.  Location of
2646   // return address specifies a type (REG or STACK) and a number
2647   // representing the register number (i.e. - use a register name) or
2648   // stack slot.
2649   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2650   // Otherwise, it is above the locks and verification slot and alignment word
2651   return_addr(STACK - 2 +
2652               round_to((Compile::current()->in_preserve_stack_slots() +
2653                         Compile::current()->fixed_slots()),
2654                        stack_alignment_in_slots()));
2655 
2656   // Body of function which returns an integer array locating
2657   // arguments either in registers or in stack slots.  Passed an array
2658   // of ideal registers called "sig" and a "length" count.  Stack-slot
2659   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2660   // arguments for a CALLEE.  Incoming stack arguments are
2661   // automatically biased by the preserve_stack_slots field above.
2662 
2663   calling_convention
2664   %{
2665     // No difference between ingoing/outgoing just pass false
2666     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2667   %}
2668 
2669   c_calling_convention
2670   %{
2671     // This is obviously always outgoing
2672     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2673   %}
2674 
2675   // Location of compiled Java return values.  Same as C for now.
2676   return_value
2677   %{
2678     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2679            "only return normal values");
2680 
2681     static const int lo[Op_RegL + 1] = {
2682       0,
2683       0,
2684       RAX_num,  // Op_RegN
2685       RAX_num,  // Op_RegI
2686       RAX_num,  // Op_RegP
2687       XMM0_num, // Op_RegF
2688       XMM0_num, // Op_RegD
2689       RAX_num   // Op_RegL
2690     };
2691     static const int hi[Op_RegL + 1] = {
2692       0,
2693       0,
2694       OptoReg::Bad, // Op_RegN
2695       OptoReg::Bad, // Op_RegI
2696       RAX_H_num,    // Op_RegP
2697       OptoReg::Bad, // Op_RegF
2698       XMM0b_num,    // Op_RegD
2699       RAX_H_num     // Op_RegL
2700     };
2701     // Excluded flags and vector registers.
2702     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2703     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2704   %}
2705 %}
2706 
2707 //----------ATTRIBUTES---------------------------------------------------------
2708 //----------Operand Attributes-------------------------------------------------
2709 op_attrib op_cost(0);        // Required cost attribute
2710 
2711 //----------Instruction Attributes---------------------------------------------
2712 ins_attrib ins_cost(100);       // Required cost attribute
2713 ins_attrib ins_size(8);         // Required size attribute (in bits)
2714 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2715                                 // a non-matching short branch variant
2716                                 // of some long branch?
2717 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2718                                 // be a power of 2) specifies the
2719                                 // alignment that some part of the
2720                                 // instruction (not necessarily the
2721                                 // start) requires.  If > 1, a
2722                                 // compute_padding() function must be
2723                                 // provided for the instruction
2724 
2725 //----------OPERANDS-----------------------------------------------------------
2726 // Operand definitions must precede instruction definitions for correct parsing
2727 // in the ADLC because operands constitute user defined types which are used in
2728 // instruction definitions.
2729 
2730 //----------Simple Operands----------------------------------------------------
2731 // Immediate Operands
2732 // Integer Immediate
2733 operand immI()
2734 %{
2735   match(ConI);
2736 
2737   op_cost(10);
2738   format %{ %}
2739   interface(CONST_INTER);
2740 %}
2741 
2742 // Constant for test vs zero
2743 operand immI0()
2744 %{
2745   predicate(n->get_int() == 0);
2746   match(ConI);
2747 
2748   op_cost(0);
2749   format %{ %}
2750   interface(CONST_INTER);
2751 %}
2752 
2753 // Constant for increment
2754 operand immI1()
2755 %{
2756   predicate(n->get_int() == 1);
2757   match(ConI);
2758 
2759   op_cost(0);
2760   format %{ %}
2761   interface(CONST_INTER);
2762 %}
2763 
2764 // Constant for decrement
2765 operand immI_M1()
2766 %{
2767   predicate(n->get_int() == -1);
2768   match(ConI);
2769 
2770   op_cost(0);
2771   format %{ %}
2772   interface(CONST_INTER);
2773 %}
2774 
2775 // Valid scale values for addressing modes
2776 operand immI2()
2777 %{
2778   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2779   match(ConI);
2780 
2781   format %{ %}
2782   interface(CONST_INTER);
2783 %}
2784 
2785 operand immI8()
2786 %{
2787   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2788   match(ConI);
2789 
2790   op_cost(5);
2791   format %{ %}
2792   interface(CONST_INTER);
2793 %}
2794 
2795 operand immI16()
2796 %{
2797   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2798   match(ConI);
2799 
2800   op_cost(10);
2801   format %{ %}
2802   interface(CONST_INTER);
2803 %}
2804 
2805 // Int Immediate non-negative
2806 operand immU31()
2807 %{
2808   predicate(n->get_int() >= 0);
2809   match(ConI);
2810 
2811   op_cost(0);
2812   format %{ %}
2813   interface(CONST_INTER);
2814 %}
2815 
2816 // Constant for long shifts
2817 operand immI_32()
2818 %{
2819   predicate( n->get_int() == 32 );
2820   match(ConI);
2821 
2822   op_cost(0);
2823   format %{ %}
2824   interface(CONST_INTER);
2825 %}
2826 
2827 // Constant for long shifts
2828 operand immI_64()
2829 %{
2830   predicate( n->get_int() == 64 );
2831   match(ConI);
2832 
2833   op_cost(0);
2834   format %{ %}
2835   interface(CONST_INTER);
2836 %}
2837 
2838 // Pointer Immediate
2839 operand immP()
2840 %{
2841   match(ConP);
2842 
2843   op_cost(10);
2844   format %{ %}
2845   interface(CONST_INTER);
2846 %}
2847 
2848 // NULL Pointer Immediate
2849 operand immP0()
2850 %{
2851   predicate(n->get_ptr() == 0);
2852   match(ConP);
2853 
2854   op_cost(5);
2855   format %{ %}
2856   interface(CONST_INTER);
2857 %}
2858 
2859 // Pointer Immediate
2860 operand immN() %{
2861   match(ConN);
2862 
2863   op_cost(10);
2864   format %{ %}
2865   interface(CONST_INTER);
2866 %}
2867 
2868 operand immNKlass() %{
2869   match(ConNKlass);
2870 
2871   op_cost(10);
2872   format %{ %}
2873   interface(CONST_INTER);
2874 %}
2875 
2876 // NULL Pointer Immediate
2877 operand immN0() %{
2878   predicate(n->get_narrowcon() == 0);
2879   match(ConN);
2880 
2881   op_cost(5);
2882   format %{ %}
2883   interface(CONST_INTER);
2884 %}
2885 
2886 operand immP31()
2887 %{
2888   predicate(n->as_Type()->type()->reloc() == relocInfo::none
2889             && (n->get_ptr() >> 31) == 0);
2890   match(ConP);
2891 
2892   op_cost(5);
2893   format %{ %}
2894   interface(CONST_INTER);
2895 %}
2896 
2897 
2898 // Long Immediate
2899 operand immL()
2900 %{
2901   match(ConL);
2902 
2903   op_cost(20);
2904   format %{ %}
2905   interface(CONST_INTER);
2906 %}
2907 
2908 // Long Immediate 8-bit
2909 operand immL8()
2910 %{
2911   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2912   match(ConL);
2913 
2914   op_cost(5);
2915   format %{ %}
2916   interface(CONST_INTER);
2917 %}
2918 
2919 // Long Immediate 32-bit unsigned
2920 operand immUL32()
2921 %{
2922   predicate(n->get_long() == (unsigned int) (n->get_long()));
2923   match(ConL);
2924 
2925   op_cost(10);
2926   format %{ %}
2927   interface(CONST_INTER);
2928 %}
2929 
2930 // Long Immediate 32-bit signed
2931 operand immL32()
2932 %{
2933   predicate(n->get_long() == (int) (n->get_long()));
2934   match(ConL);
2935 
2936   op_cost(15);
2937   format %{ %}
2938   interface(CONST_INTER);
2939 %}
2940 
2941 // Long Immediate zero
2942 operand immL0()
2943 %{
2944   predicate(n->get_long() == 0L);
2945   match(ConL);
2946 
2947   op_cost(10);
2948   format %{ %}
2949   interface(CONST_INTER);
2950 %}
2951 
2952 // Constant for increment
2953 operand immL1()
2954 %{
2955   predicate(n->get_long() == 1);
2956   match(ConL);
2957 
2958   format %{ %}
2959   interface(CONST_INTER);
2960 %}
2961 
2962 // Constant for decrement
2963 operand immL_M1()
2964 %{
2965   predicate(n->get_long() == -1);
2966   match(ConL);
2967 
2968   format %{ %}
2969   interface(CONST_INTER);
2970 %}
2971 
2972 // Long Immediate: the value 10
2973 operand immL10()
2974 %{
2975   predicate(n->get_long() == 10);
2976   match(ConL);
2977 
2978   format %{ %}
2979   interface(CONST_INTER);
2980 %}
2981 
2982 // Long immediate from 0 to 127.
2983 // Used for a shorter form of long mul by 10.
2984 operand immL_127()
2985 %{
2986   predicate(0 <= n->get_long() && n->get_long() < 0x80);
2987   match(ConL);
2988 
2989   op_cost(10);
2990   format %{ %}
2991   interface(CONST_INTER);
2992 %}
2993 
2994 // Long Immediate: low 32-bit mask
2995 operand immL_32bits()
2996 %{
2997   predicate(n->get_long() == 0xFFFFFFFFL);
2998   match(ConL);
2999   op_cost(20);
3000 
3001   format %{ %}
3002   interface(CONST_INTER);
3003 %}
3004 
3005 // Float Immediate zero
3006 operand immF0()
3007 %{
3008   predicate(jint_cast(n->getf()) == 0);
3009   match(ConF);
3010 
3011   op_cost(5);
3012   format %{ %}
3013   interface(CONST_INTER);
3014 %}
3015 
3016 // Float Immediate
3017 operand immF()
3018 %{
3019   match(ConF);
3020 
3021   op_cost(15);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // Double Immediate zero
3027 operand immD0()
3028 %{
3029   predicate(jlong_cast(n->getd()) == 0);
3030   match(ConD);
3031 
3032   op_cost(5);
3033   format %{ %}
3034   interface(CONST_INTER);
3035 %}
3036 
3037 // Double Immediate
3038 operand immD()
3039 %{
3040   match(ConD);
3041 
3042   op_cost(15);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 // Immediates for special shifts (sign extend)
3048 
3049 // Constants for increment
3050 operand immI_16()
3051 %{
3052   predicate(n->get_int() == 16);
3053   match(ConI);
3054 
3055   format %{ %}
3056   interface(CONST_INTER);
3057 %}
3058 
3059 operand immI_24()
3060 %{
3061   predicate(n->get_int() == 24);
3062   match(ConI);
3063 
3064   format %{ %}
3065   interface(CONST_INTER);
3066 %}
3067 
3068 // Constant for byte-wide masking
3069 operand immI_255()
3070 %{
3071   predicate(n->get_int() == 255);
3072   match(ConI);
3073 
3074   format %{ %}
3075   interface(CONST_INTER);
3076 %}
3077 
3078 // Constant for short-wide masking
3079 operand immI_65535()
3080 %{
3081   predicate(n->get_int() == 65535);
3082   match(ConI);
3083 
3084   format %{ %}
3085   interface(CONST_INTER);
3086 %}
3087 
3088 // Constant for byte-wide masking
3089 operand immL_255()
3090 %{
3091   predicate(n->get_long() == 255);
3092   match(ConL);
3093 
3094   format %{ %}
3095   interface(CONST_INTER);
3096 %}
3097 
3098 // Constant for short-wide masking
3099 operand immL_65535()
3100 %{
3101   predicate(n->get_long() == 65535);
3102   match(ConL);
3103 
3104   format %{ %}
3105   interface(CONST_INTER);
3106 %}
3107 
3108 // Register Operands
3109 // Integer Register
3110 operand rRegI()
3111 %{
3112   constraint(ALLOC_IN_RC(int_reg));
3113   match(RegI);
3114 
3115   match(rax_RegI);
3116   match(rbx_RegI);
3117   match(rcx_RegI);
3118   match(rdx_RegI);
3119   match(rdi_RegI);
3120 
3121   format %{ %}
3122   interface(REG_INTER);
3123 %}
3124 
3125 // Special Registers
3126 operand rax_RegI()
3127 %{
3128   constraint(ALLOC_IN_RC(int_rax_reg));
3129   match(RegI);
3130   match(rRegI);
3131 
3132   format %{ "RAX" %}
3133   interface(REG_INTER);
3134 %}
3135 
3136 // Special Registers
3137 operand rbx_RegI()
3138 %{
3139   constraint(ALLOC_IN_RC(int_rbx_reg));
3140   match(RegI);
3141   match(rRegI);
3142 
3143   format %{ "RBX" %}
3144   interface(REG_INTER);
3145 %}
3146 
3147 operand rcx_RegI()
3148 %{
3149   constraint(ALLOC_IN_RC(int_rcx_reg));
3150   match(RegI);
3151   match(rRegI);
3152 
3153   format %{ "RCX" %}
3154   interface(REG_INTER);
3155 %}
3156 
3157 operand rdx_RegI()
3158 %{
3159   constraint(ALLOC_IN_RC(int_rdx_reg));
3160   match(RegI);
3161   match(rRegI);
3162 
3163   format %{ "RDX" %}
3164   interface(REG_INTER);
3165 %}
3166 
3167 operand rdi_RegI()
3168 %{
3169   constraint(ALLOC_IN_RC(int_rdi_reg));
3170   match(RegI);
3171   match(rRegI);
3172 
3173   format %{ "RDI" %}
3174   interface(REG_INTER);
3175 %}
3176 
3177 operand no_rcx_RegI()
3178 %{
3179   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3180   match(RegI);
3181   match(rax_RegI);
3182   match(rbx_RegI);
3183   match(rdx_RegI);
3184   match(rdi_RegI);
3185 
3186   format %{ %}
3187   interface(REG_INTER);
3188 %}
3189 
3190 operand no_rax_rdx_RegI()
3191 %{
3192   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3193   match(RegI);
3194   match(rbx_RegI);
3195   match(rcx_RegI);
3196   match(rdi_RegI);
3197 
3198   format %{ %}
3199   interface(REG_INTER);
3200 %}
3201 
3202 // Pointer Register
3203 operand any_RegP()
3204 %{
3205   constraint(ALLOC_IN_RC(any_reg));
3206   match(RegP);
3207   match(rax_RegP);
3208   match(rbx_RegP);
3209   match(rdi_RegP);
3210   match(rsi_RegP);
3211   match(rbp_RegP);
3212   match(r15_RegP);
3213   match(rRegP);
3214 
3215   format %{ %}
3216   interface(REG_INTER);
3217 %}
3218 
3219 operand rRegP()
3220 %{
3221   constraint(ALLOC_IN_RC(ptr_reg));
3222   match(RegP);
3223   match(rax_RegP);
3224   match(rbx_RegP);
3225   match(rdi_RegP);
3226   match(rsi_RegP);
3227   match(rbp_RegP);
3228   match(r15_RegP);  // See Q&A below about r15_RegP.
3229 
3230   format %{ %}
3231   interface(REG_INTER);
3232 %}
3233 
3234 operand rRegN() %{
3235   constraint(ALLOC_IN_RC(int_reg));
3236   match(RegN);
3237 
3238   format %{ %}
3239   interface(REG_INTER);
3240 %}
3241 
3242 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3243 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3244 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
3245 // The output of an instruction is controlled by the allocator, which respects
3246 // register class masks, not match rules.  Unless an instruction mentions
3247 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3248 // by the allocator as an input.
3249 
3250 operand no_rax_RegP()
3251 %{
3252   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3253   match(RegP);
3254   match(rbx_RegP);
3255   match(rsi_RegP);
3256   match(rdi_RegP);
3257 
3258   format %{ %}
3259   interface(REG_INTER);
3260 %}
3261 
3262 operand no_rbp_RegP()
3263 %{
3264   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
3265   match(RegP);
3266   match(rbx_RegP);
3267   match(rsi_RegP);
3268   match(rdi_RegP);
3269 
3270   format %{ %}
3271   interface(REG_INTER);
3272 %}
3273 
3274 operand no_rax_rbx_RegP()
3275 %{
3276   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3277   match(RegP);
3278   match(rsi_RegP);
3279   match(rdi_RegP);
3280 
3281   format %{ %}
3282   interface(REG_INTER);
3283 %}
3284 
3285 // Special Registers
3286 // Return a pointer value
3287 operand rax_RegP()
3288 %{
3289   constraint(ALLOC_IN_RC(ptr_rax_reg));
3290   match(RegP);
3291   match(rRegP);
3292 
3293   format %{ %}
3294   interface(REG_INTER);
3295 %}
3296 
3297 // Special Registers
3298 // Return a compressed pointer value
3299 operand rax_RegN()
3300 %{
3301   constraint(ALLOC_IN_RC(int_rax_reg));
3302   match(RegN);
3303   match(rRegN);
3304 
3305   format %{ %}
3306   interface(REG_INTER);
3307 %}
3308 
3309 // Used in AtomicAdd
3310 operand rbx_RegP()
3311 %{
3312   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3313   match(RegP);
3314   match(rRegP);
3315 
3316   format %{ %}
3317   interface(REG_INTER);
3318 %}
3319 
3320 operand rsi_RegP()
3321 %{
3322   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3323   match(RegP);
3324   match(rRegP);
3325 
3326   format %{ %}
3327   interface(REG_INTER);
3328 %}
3329 
3330 // Used in rep stosq
3331 operand rdi_RegP()
3332 %{
3333   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3334   match(RegP);
3335   match(rRegP);
3336 
3337   format %{ %}
3338   interface(REG_INTER);
3339 %}
3340 
3341 operand rbp_RegP()
3342 %{
3343   constraint(ALLOC_IN_RC(ptr_rbp_reg));
3344   match(RegP);
3345   match(rRegP);
3346 
3347   format %{ %}
3348   interface(REG_INTER);
3349 %}
3350 
3351 operand r15_RegP()
3352 %{
3353   constraint(ALLOC_IN_RC(ptr_r15_reg));
3354   match(RegP);
3355   match(rRegP);
3356 
3357   format %{ %}
3358   interface(REG_INTER);
3359 %}
3360 
3361 operand rRegL()
3362 %{
3363   constraint(ALLOC_IN_RC(long_reg));
3364   match(RegL);
3365   match(rax_RegL);
3366   match(rdx_RegL);
3367 
3368   format %{ %}
3369   interface(REG_INTER);
3370 %}
3371 
3372 // Special Registers
3373 operand no_rax_rdx_RegL()
3374 %{
3375   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3376   match(RegL);
3377   match(rRegL);
3378 
3379   format %{ %}
3380   interface(REG_INTER);
3381 %}
3382 
3383 operand no_rax_RegL()
3384 %{
3385   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3386   match(RegL);
3387   match(rRegL);
3388   match(rdx_RegL);
3389 
3390   format %{ %}
3391   interface(REG_INTER);
3392 %}
3393 
3394 operand no_rcx_RegL()
3395 %{
3396   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3397   match(RegL);
3398   match(rRegL);
3399 
3400   format %{ %}
3401   interface(REG_INTER);
3402 %}
3403 
3404 operand rax_RegL()
3405 %{
3406   constraint(ALLOC_IN_RC(long_rax_reg));
3407   match(RegL);
3408   match(rRegL);
3409 
3410   format %{ "RAX" %}
3411   interface(REG_INTER);
3412 %}
3413 
3414 operand rcx_RegL()
3415 %{
3416   constraint(ALLOC_IN_RC(long_rcx_reg));
3417   match(RegL);
3418   match(rRegL);
3419 
3420   format %{ %}
3421   interface(REG_INTER);
3422 %}
3423 
3424 operand rdx_RegL()
3425 %{
3426   constraint(ALLOC_IN_RC(long_rdx_reg));
3427   match(RegL);
3428   match(rRegL);
3429 
3430   format %{ %}
3431   interface(REG_INTER);
3432 %}
3433 
3434 // Flags register, used as output of compare instructions
3435 operand rFlagsReg()
3436 %{
3437   constraint(ALLOC_IN_RC(int_flags));
3438   match(RegFlags);
3439 
3440   format %{ "RFLAGS" %}
3441   interface(REG_INTER);
3442 %}
3443 
3444 // Flags register, used as output of FLOATING POINT compare instructions
3445 operand rFlagsRegU()
3446 %{
3447   constraint(ALLOC_IN_RC(int_flags));
3448   match(RegFlags);
3449 
3450   format %{ "RFLAGS_U" %}
3451   interface(REG_INTER);
3452 %}
3453 
3454 operand rFlagsRegUCF() %{
3455   constraint(ALLOC_IN_RC(int_flags));
3456   match(RegFlags);
3457   predicate(false);
3458 
3459   format %{ "RFLAGS_U_CF" %}
3460   interface(REG_INTER);
3461 %}
3462 
3463 // Float register operands
3464 operand regF()
3465 %{
3466   constraint(ALLOC_IN_RC(float_reg));
3467   match(RegF);
3468 
3469   format %{ %}
3470   interface(REG_INTER);
3471 %}
3472 
3473 // Double register operands
3474 operand regD()
3475 %{
3476   constraint(ALLOC_IN_RC(double_reg));
3477   match(RegD);
3478 
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 //----------Memory Operands----------------------------------------------------
3484 // Direct Memory Operand
3485 // operand direct(immP addr)
3486 // %{
3487 //   match(addr);
3488 
3489 //   format %{ "[$addr]" %}
3490 //   interface(MEMORY_INTER) %{
3491 //     base(0xFFFFFFFF);
3492 //     index(0x4);
3493 //     scale(0x0);
3494 //     disp($addr);
3495 //   %}
3496 // %}
3497 
3498 // Indirect Memory Operand
3499 operand indirect(any_RegP reg)
3500 %{
3501   constraint(ALLOC_IN_RC(ptr_reg));
3502   match(reg);
3503 
3504   format %{ "[$reg]" %}
3505   interface(MEMORY_INTER) %{
3506     base($reg);
3507     index(0x4);
3508     scale(0x0);
3509     disp(0x0);
3510   %}
3511 %}
3512 
3513 // Indirect Memory Plus Short Offset Operand
3514 operand indOffset8(any_RegP reg, immL8 off)
3515 %{
3516   constraint(ALLOC_IN_RC(ptr_reg));
3517   match(AddP reg off);
3518 
3519   format %{ "[$reg + $off (8-bit)]" %}
3520   interface(MEMORY_INTER) %{
3521     base($reg);
3522     index(0x4);
3523     scale(0x0);
3524     disp($off);
3525   %}
3526 %}
3527 
3528 // Indirect Memory Plus Long Offset Operand
3529 operand indOffset32(any_RegP reg, immL32 off)
3530 %{
3531   constraint(ALLOC_IN_RC(ptr_reg));
3532   match(AddP reg off);
3533 
3534   format %{ "[$reg + $off (32-bit)]" %}
3535   interface(MEMORY_INTER) %{
3536     base($reg);
3537     index(0x4);
3538     scale(0x0);
3539     disp($off);
3540   %}
3541 %}
3542 
3543 // Indirect Memory Plus Index Register Plus Offset Operand
3544 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3545 %{
3546   constraint(ALLOC_IN_RC(ptr_reg));
3547   match(AddP (AddP reg lreg) off);
3548 
3549   op_cost(10);
3550   format %{"[$reg + $off + $lreg]" %}
3551   interface(MEMORY_INTER) %{
3552     base($reg);
3553     index($lreg);
3554     scale(0x0);
3555     disp($off);
3556   %}
3557 %}
3558 
3559 // Indirect Memory Plus Index Register Plus Offset Operand
3560 operand indIndex(any_RegP reg, rRegL lreg)
3561 %{
3562   constraint(ALLOC_IN_RC(ptr_reg));
3563   match(AddP reg lreg);
3564 
3565   op_cost(10);
3566   format %{"[$reg + $lreg]" %}
3567   interface(MEMORY_INTER) %{
3568     base($reg);
3569     index($lreg);
3570     scale(0x0);
3571     disp(0x0);
3572   %}
3573 %}
3574 
3575 // Indirect Memory Times Scale Plus Index Register
3576 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3577 %{
3578   constraint(ALLOC_IN_RC(ptr_reg));
3579   match(AddP reg (LShiftL lreg scale));
3580 
3581   op_cost(10);
3582   format %{"[$reg + $lreg << $scale]" %}
3583   interface(MEMORY_INTER) %{
3584     base($reg);
3585     index($lreg);
3586     scale($scale);
3587     disp(0x0);
3588   %}
3589 %}
3590 
3591 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3592 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3593 %{
3594   constraint(ALLOC_IN_RC(ptr_reg));
3595   match(AddP (AddP reg (LShiftL lreg scale)) off);
3596 
3597   op_cost(10);
3598   format %{"[$reg + $off + $lreg << $scale]" %}
3599   interface(MEMORY_INTER) %{
3600     base($reg);
3601     index($lreg);
3602     scale($scale);
3603     disp($off);
3604   %}
3605 %}
3606 
3607 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3608 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3609 %{
3610   constraint(ALLOC_IN_RC(ptr_reg));
3611   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3612   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3613 
3614   op_cost(10);
3615   format %{"[$reg + $off + $idx << $scale]" %}
3616   interface(MEMORY_INTER) %{
3617     base($reg);
3618     index($idx);
3619     scale($scale);
3620     disp($off);
3621   %}
3622 %}
3623 
3624 // Indirect Narrow Oop Plus Offset Operand
3625 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3626 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3627 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3628   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3629   constraint(ALLOC_IN_RC(ptr_reg));
3630   match(AddP (DecodeN reg) off);
3631 
3632   op_cost(10);
3633   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3634   interface(MEMORY_INTER) %{
3635     base(0xc); // R12
3636     index($reg);
3637     scale(0x3);
3638     disp($off);
3639   %}
3640 %}
3641 
3642 // Indirect Memory Operand
3643 operand indirectNarrow(rRegN reg)
3644 %{
3645   predicate(Universe::narrow_oop_shift() == 0);
3646   constraint(ALLOC_IN_RC(ptr_reg));
3647   match(DecodeN reg);
3648 
3649   format %{ "[$reg]" %}
3650   interface(MEMORY_INTER) %{
3651     base($reg);
3652     index(0x4);
3653     scale(0x0);
3654     disp(0x0);
3655   %}
3656 %}
3657 
3658 // Indirect Memory Plus Short Offset Operand
3659 operand indOffset8Narrow(rRegN reg, immL8 off)
3660 %{
3661   predicate(Universe::narrow_oop_shift() == 0);
3662   constraint(ALLOC_IN_RC(ptr_reg));
3663   match(AddP (DecodeN reg) off);
3664 
3665   format %{ "[$reg + $off (8-bit)]" %}
3666   interface(MEMORY_INTER) %{
3667     base($reg);
3668     index(0x4);
3669     scale(0x0);
3670     disp($off);
3671   %}
3672 %}
3673 
3674 // Indirect Memory Plus Long Offset Operand
3675 operand indOffset32Narrow(rRegN reg, immL32 off)
3676 %{
3677   predicate(Universe::narrow_oop_shift() == 0);
3678   constraint(ALLOC_IN_RC(ptr_reg));
3679   match(AddP (DecodeN reg) off);
3680 
3681   format %{ "[$reg + $off (32-bit)]" %}
3682   interface(MEMORY_INTER) %{
3683     base($reg);
3684     index(0x4);
3685     scale(0x0);
3686     disp($off);
3687   %}
3688 %}
3689 
3690 // Indirect Memory Plus Index Register Plus Offset Operand
3691 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3692 %{
3693   predicate(Universe::narrow_oop_shift() == 0);
3694   constraint(ALLOC_IN_RC(ptr_reg));
3695   match(AddP (AddP (DecodeN reg) lreg) off);
3696 
3697   op_cost(10);
3698   format %{"[$reg + $off + $lreg]" %}
3699   interface(MEMORY_INTER) %{
3700     base($reg);
3701     index($lreg);
3702     scale(0x0);
3703     disp($off);
3704   %}
3705 %}
3706 
3707 // Indirect Memory Plus Index Register Plus Offset Operand
3708 operand indIndexNarrow(rRegN reg, rRegL lreg)
3709 %{
3710   predicate(Universe::narrow_oop_shift() == 0);
3711   constraint(ALLOC_IN_RC(ptr_reg));
3712   match(AddP (DecodeN reg) lreg);
3713 
3714   op_cost(10);
3715   format %{"[$reg + $lreg]" %}
3716   interface(MEMORY_INTER) %{
3717     base($reg);
3718     index($lreg);
3719     scale(0x0);
3720     disp(0x0);
3721   %}
3722 %}
3723 
3724 // Indirect Memory Times Scale Plus Index Register
3725 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3726 %{
3727   predicate(Universe::narrow_oop_shift() == 0);
3728   constraint(ALLOC_IN_RC(ptr_reg));
3729   match(AddP (DecodeN reg) (LShiftL lreg scale));
3730 
3731   op_cost(10);
3732   format %{"[$reg + $lreg << $scale]" %}
3733   interface(MEMORY_INTER) %{
3734     base($reg);
3735     index($lreg);
3736     scale($scale);
3737     disp(0x0);
3738   %}
3739 %}
3740 
3741 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3742 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3743 %{
3744   predicate(Universe::narrow_oop_shift() == 0);
3745   constraint(ALLOC_IN_RC(ptr_reg));
3746   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3747 
3748   op_cost(10);
3749   format %{"[$reg + $off + $lreg << $scale]" %}
3750   interface(MEMORY_INTER) %{
3751     base($reg);
3752     index($lreg);
3753     scale($scale);
3754     disp($off);
3755   %}
3756 %}
3757 
3758 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3759 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3760 %{
3761   constraint(ALLOC_IN_RC(ptr_reg));
3762   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3763   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3764 
3765   op_cost(10);
3766   format %{"[$reg + $off + $idx << $scale]" %}
3767   interface(MEMORY_INTER) %{
3768     base($reg);
3769     index($idx);
3770     scale($scale);
3771     disp($off);
3772   %}
3773 %}
3774 
3775 //----------Special Memory Operands--------------------------------------------
3776 // Stack Slot Operand - This operand is used for loading and storing temporary
3777 //                      values on the stack where a match requires a value to
3778 //                      flow through memory.
3779 operand stackSlotP(sRegP reg)
3780 %{
3781   constraint(ALLOC_IN_RC(stack_slots));
3782   // No match rule because this operand is only generated in matching
3783 
3784   format %{ "[$reg]" %}
3785   interface(MEMORY_INTER) %{
3786     base(0x4);   // RSP
3787     index(0x4);  // No Index
3788     scale(0x0);  // No Scale
3789     disp($reg);  // Stack Offset
3790   %}
3791 %}
3792 
3793 operand stackSlotI(sRegI reg)
3794 %{
3795   constraint(ALLOC_IN_RC(stack_slots));
3796   // No match rule because this operand is only generated in matching
3797 
3798   format %{ "[$reg]" %}
3799   interface(MEMORY_INTER) %{
3800     base(0x4);   // RSP
3801     index(0x4);  // No Index
3802     scale(0x0);  // No Scale
3803     disp($reg);  // Stack Offset
3804   %}
3805 %}
3806 
3807 operand stackSlotF(sRegF reg)
3808 %{
3809   constraint(ALLOC_IN_RC(stack_slots));
3810   // No match rule because this operand is only generated in matching
3811 
3812   format %{ "[$reg]" %}
3813   interface(MEMORY_INTER) %{
3814     base(0x4);   // RSP
3815     index(0x4);  // No Index
3816     scale(0x0);  // No Scale
3817     disp($reg);  // Stack Offset
3818   %}
3819 %}
3820 
3821 operand stackSlotD(sRegD reg)
3822 %{
3823   constraint(ALLOC_IN_RC(stack_slots));
3824   // No match rule because this operand is only generated in matching
3825 
3826   format %{ "[$reg]" %}
3827   interface(MEMORY_INTER) %{
3828     base(0x4);   // RSP
3829     index(0x4);  // No Index
3830     scale(0x0);  // No Scale
3831     disp($reg);  // Stack Offset
3832   %}
3833 %}
3834 operand stackSlotL(sRegL reg)
3835 %{
3836   constraint(ALLOC_IN_RC(stack_slots));
3837   // No match rule because this operand is only generated in matching
3838 
3839   format %{ "[$reg]" %}
3840   interface(MEMORY_INTER) %{
3841     base(0x4);   // RSP
3842     index(0x4);  // No Index
3843     scale(0x0);  // No Scale
3844     disp($reg);  // Stack Offset
3845   %}
3846 %}
3847 
3848 //----------Conditional Branch Operands----------------------------------------
3849 // Comparison Op  - This is the operation of the comparison, and is limited to
3850 //                  the following set of codes:
3851 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3852 //
3853 // Other attributes of the comparison, such as unsignedness, are specified
3854 // by the comparison instruction that sets a condition code flags register.
3855 // That result is represented by a flags operand whose subtype is appropriate
3856 // to the unsignedness (etc.) of the comparison.
3857 //
3858 // Later, the instruction which matches both the Comparison Op (a Bool) and
3859 // the flags (produced by the Cmp) specifies the coding of the comparison op
3860 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3861 
3862 // Comparision Code
3863 operand cmpOp()
3864 %{
3865   match(Bool);
3866 
3867   format %{ "" %}
3868   interface(COND_INTER) %{
3869     equal(0x4, "e");
3870     not_equal(0x5, "ne");
3871     less(0xC, "l");
3872     greater_equal(0xD, "ge");
3873     less_equal(0xE, "le");
3874     greater(0xF, "g");
3875     overflow(0x0, "o");
3876     no_overflow(0x1, "no");
3877   %}
3878 %}
3879 
3880 // Comparison Code, unsigned compare.  Used by FP also, with
3881 // C2 (unordered) turned into GT or LT already.  The other bits
3882 // C0 and C3 are turned into Carry & Zero flags.
3883 operand cmpOpU()
3884 %{
3885   match(Bool);
3886 
3887   format %{ "" %}
3888   interface(COND_INTER) %{
3889     equal(0x4, "e");
3890     not_equal(0x5, "ne");
3891     less(0x2, "b");
3892     greater_equal(0x3, "nb");
3893     less_equal(0x6, "be");
3894     greater(0x7, "nbe");
3895     overflow(0x0, "o");
3896     no_overflow(0x1, "no");
3897   %}
3898 %}
3899 
3900 
3901 // Floating comparisons that don't require any fixup for the unordered case
3902 operand cmpOpUCF() %{
3903   match(Bool);
3904   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
3905             n->as_Bool()->_test._test == BoolTest::ge ||
3906             n->as_Bool()->_test._test == BoolTest::le ||
3907             n->as_Bool()->_test._test == BoolTest::gt);
3908   format %{ "" %}
3909   interface(COND_INTER) %{
3910     equal(0x4, "e");
3911     not_equal(0x5, "ne");
3912     less(0x2, "b");
3913     greater_equal(0x3, "nb");
3914     less_equal(0x6, "be");
3915     greater(0x7, "nbe");
3916     overflow(0x0, "o");
3917     no_overflow(0x1, "no");
3918   %}
3919 %}
3920 
3921 
3922 // Floating comparisons that can be fixed up with extra conditional jumps
3923 operand cmpOpUCF2() %{
3924   match(Bool);
3925   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
3926             n->as_Bool()->_test._test == BoolTest::eq);
3927   format %{ "" %}
3928   interface(COND_INTER) %{
3929     equal(0x4, "e");
3930     not_equal(0x5, "ne");
3931     less(0x2, "b");
3932     greater_equal(0x3, "nb");
3933     less_equal(0x6, "be");
3934     greater(0x7, "nbe");
3935     overflow(0x0, "o");
3936     no_overflow(0x1, "no");
3937   %}
3938 %}
3939 
3940 
3941 //----------OPERAND CLASSES----------------------------------------------------
3942 // Operand Classes are groups of operands that are used as to simplify
3943 // instruction definitions by not requiring the AD writer to specify separate
3944 // instructions for every form of operand when the instruction accepts
3945 // multiple operand types with the same basic encoding and format.  The classic
3946 // case of this is memory operands.
3947 
3948 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
3949                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
3950                indCompressedOopOffset,
3951                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
3952                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
3953                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
3954 
3955 //----------PIPELINE-----------------------------------------------------------
3956 // Rules which define the behavior of the target architectures pipeline.
3957 pipeline %{
3958 
3959 //----------ATTRIBUTES---------------------------------------------------------
3960 attributes %{
3961   variable_size_instructions;        // Fixed size instructions
3962   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
3963   instruction_unit_size = 1;         // An instruction is 1 bytes long
3964   instruction_fetch_unit_size = 16;  // The processor fetches one line
3965   instruction_fetch_units = 1;       // of 16 bytes
3966 
3967   // List of nop instructions
3968   nops( MachNop );
3969 %}
3970 
3971 //----------RESOURCES----------------------------------------------------------
3972 // Resources are the functional units available to the machine
3973 
3974 // Generic P2/P3 pipeline
3975 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
3976 // 3 instructions decoded per cycle.
3977 // 2 load/store ops per cycle, 1 branch, 1 FPU,
3978 // 3 ALU op, only ALU0 handles mul instructions.
3979 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
3980            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
3981            BR, FPU,
3982            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
3983 
3984 //----------PIPELINE DESCRIPTION-----------------------------------------------
3985 // Pipeline Description specifies the stages in the machine's pipeline
3986 
3987 // Generic P2/P3 pipeline
3988 pipe_desc(S0, S1, S2, S3, S4, S5);
3989 
3990 //----------PIPELINE CLASSES---------------------------------------------------
3991 // Pipeline Classes describe the stages in which input and output are
3992 // referenced by the hardware pipeline.
3993 
3994 // Naming convention: ialu or fpu
3995 // Then: _reg
3996 // Then: _reg if there is a 2nd register
3997 // Then: _long if it's a pair of instructions implementing a long
3998 // Then: _fat if it requires the big decoder
3999 //   Or: _mem if it requires the big decoder and a memory unit.
4000 
4001 // Integer ALU reg operation
4002 pipe_class ialu_reg(rRegI dst)
4003 %{
4004     single_instruction;
4005     dst    : S4(write);
4006     dst    : S3(read);
4007     DECODE : S0;        // any decoder
4008     ALU    : S3;        // any alu
4009 %}
4010 
4011 // Long ALU reg operation
4012 pipe_class ialu_reg_long(rRegL dst)
4013 %{
4014     instruction_count(2);
4015     dst    : S4(write);
4016     dst    : S3(read);
4017     DECODE : S0(2);     // any 2 decoders
4018     ALU    : S3(2);     // both alus
4019 %}
4020 
4021 // Integer ALU reg operation using big decoder
4022 pipe_class ialu_reg_fat(rRegI dst)
4023 %{
4024     single_instruction;
4025     dst    : S4(write);
4026     dst    : S3(read);
4027     D0     : S0;        // big decoder only
4028     ALU    : S3;        // any alu
4029 %}
4030 
4031 // Long ALU reg operation using big decoder
4032 pipe_class ialu_reg_long_fat(rRegL dst)
4033 %{
4034     instruction_count(2);
4035     dst    : S4(write);
4036     dst    : S3(read);
4037     D0     : S0(2);     // big decoder only; twice
4038     ALU    : S3(2);     // any 2 alus
4039 %}
4040 
4041 // Integer ALU reg-reg operation
4042 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4043 %{
4044     single_instruction;
4045     dst    : S4(write);
4046     src    : S3(read);
4047     DECODE : S0;        // any decoder
4048     ALU    : S3;        // any alu
4049 %}
4050 
4051 // Long ALU reg-reg operation
4052 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4053 %{
4054     instruction_count(2);
4055     dst    : S4(write);
4056     src    : S3(read);
4057     DECODE : S0(2);     // any 2 decoders
4058     ALU    : S3(2);     // both alus
4059 %}
4060 
4061 // Integer ALU reg-reg operation
4062 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4063 %{
4064     single_instruction;
4065     dst    : S4(write);
4066     src    : S3(read);
4067     D0     : S0;        // big decoder only
4068     ALU    : S3;        // any alu
4069 %}
4070 
4071 // Long ALU reg-reg operation
4072 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4073 %{
4074     instruction_count(2);
4075     dst    : S4(write);
4076     src    : S3(read);
4077     D0     : S0(2);     // big decoder only; twice
4078     ALU    : S3(2);     // both alus
4079 %}
4080 
4081 // Integer ALU reg-mem operation
4082 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4083 %{
4084     single_instruction;
4085     dst    : S5(write);
4086     mem    : S3(read);
4087     D0     : S0;        // big decoder only
4088     ALU    : S4;        // any alu
4089     MEM    : S3;        // any mem
4090 %}
4091 
4092 // Integer mem operation (prefetch)
4093 pipe_class ialu_mem(memory mem)
4094 %{
4095     single_instruction;
4096     mem    : S3(read);
4097     D0     : S0;        // big decoder only
4098     MEM    : S3;        // any mem
4099 %}
4100 
4101 // Integer Store to Memory
4102 pipe_class ialu_mem_reg(memory mem, rRegI src)
4103 %{
4104     single_instruction;
4105     mem    : S3(read);
4106     src    : S5(read);
4107     D0     : S0;        // big decoder only
4108     ALU    : S4;        // any alu
4109     MEM    : S3;
4110 %}
4111 
4112 // // Long Store to Memory
4113 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4114 // %{
4115 //     instruction_count(2);
4116 //     mem    : S3(read);
4117 //     src    : S5(read);
4118 //     D0     : S0(2);          // big decoder only; twice
4119 //     ALU    : S4(2);     // any 2 alus
4120 //     MEM    : S3(2);  // Both mems
4121 // %}
4122 
4123 // Integer Store to Memory
4124 pipe_class ialu_mem_imm(memory mem)
4125 %{
4126     single_instruction;
4127     mem    : S3(read);
4128     D0     : S0;        // big decoder only
4129     ALU    : S4;        // any alu
4130     MEM    : S3;
4131 %}
4132 
4133 // Integer ALU0 reg-reg operation
4134 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4135 %{
4136     single_instruction;
4137     dst    : S4(write);
4138     src    : S3(read);
4139     D0     : S0;        // Big decoder only
4140     ALU0   : S3;        // only alu0
4141 %}
4142 
4143 // Integer ALU0 reg-mem operation
4144 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4145 %{
4146     single_instruction;
4147     dst    : S5(write);
4148     mem    : S3(read);
4149     D0     : S0;        // big decoder only
4150     ALU0   : S4;        // ALU0 only
4151     MEM    : S3;        // any mem
4152 %}
4153 
4154 // Integer ALU reg-reg operation
4155 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4156 %{
4157     single_instruction;
4158     cr     : S4(write);
4159     src1   : S3(read);
4160     src2   : S3(read);
4161     DECODE : S0;        // any decoder
4162     ALU    : S3;        // any alu
4163 %}
4164 
4165 // Integer ALU reg-imm operation
4166 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4167 %{
4168     single_instruction;
4169     cr     : S4(write);
4170     src1   : S3(read);
4171     DECODE : S0;        // any decoder
4172     ALU    : S3;        // any alu
4173 %}
4174 
4175 // Integer ALU reg-mem operation
4176 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4177 %{
4178     single_instruction;
4179     cr     : S4(write);
4180     src1   : S3(read);
4181     src2   : S3(read);
4182     D0     : S0;        // big decoder only
4183     ALU    : S4;        // any alu
4184     MEM    : S3;
4185 %}
4186 
4187 // Conditional move reg-reg
4188 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4189 %{
4190     instruction_count(4);
4191     y      : S4(read);
4192     q      : S3(read);
4193     p      : S3(read);
4194     DECODE : S0(4);     // any decoder
4195 %}
4196 
4197 // Conditional move reg-reg
4198 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4199 %{
4200     single_instruction;
4201     dst    : S4(write);
4202     src    : S3(read);
4203     cr     : S3(read);
4204     DECODE : S0;        // any decoder
4205 %}
4206 
4207 // Conditional move reg-mem
4208 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4209 %{
4210     single_instruction;
4211     dst    : S4(write);
4212     src    : S3(read);
4213     cr     : S3(read);
4214     DECODE : S0;        // any decoder
4215     MEM    : S3;
4216 %}
4217 
4218 // Conditional move reg-reg long
4219 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4220 %{
4221     single_instruction;
4222     dst    : S4(write);
4223     src    : S3(read);
4224     cr     : S3(read);
4225     DECODE : S0(2);     // any 2 decoders
4226 %}
4227 
4228 // XXX
4229 // // Conditional move double reg-reg
4230 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4231 // %{
4232 //     single_instruction;
4233 //     dst    : S4(write);
4234 //     src    : S3(read);
4235 //     cr     : S3(read);
4236 //     DECODE : S0;     // any decoder
4237 // %}
4238 
4239 // Float reg-reg operation
4240 pipe_class fpu_reg(regD dst)
4241 %{
4242     instruction_count(2);
4243     dst    : S3(read);
4244     DECODE : S0(2);     // any 2 decoders
4245     FPU    : S3;
4246 %}
4247 
4248 // Float reg-reg operation
4249 pipe_class fpu_reg_reg(regD dst, regD src)
4250 %{
4251     instruction_count(2);
4252     dst    : S4(write);
4253     src    : S3(read);
4254     DECODE : S0(2);     // any 2 decoders
4255     FPU    : S3;
4256 %}
4257 
4258 // Float reg-reg operation
4259 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4260 %{
4261     instruction_count(3);
4262     dst    : S4(write);
4263     src1   : S3(read);
4264     src2   : S3(read);
4265     DECODE : S0(3);     // any 3 decoders
4266     FPU    : S3(2);
4267 %}
4268 
4269 // Float reg-reg operation
4270 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4271 %{
4272     instruction_count(4);
4273     dst    : S4(write);
4274     src1   : S3(read);
4275     src2   : S3(read);
4276     src3   : S3(read);
4277     DECODE : S0(4);     // any 3 decoders
4278     FPU    : S3(2);
4279 %}
4280 
4281 // Float reg-reg operation
4282 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4283 %{
4284     instruction_count(4);
4285     dst    : S4(write);
4286     src1   : S3(read);
4287     src2   : S3(read);
4288     src3   : S3(read);
4289     DECODE : S1(3);     // any 3 decoders
4290     D0     : S0;        // Big decoder only
4291     FPU    : S3(2);
4292     MEM    : S3;
4293 %}
4294 
4295 // Float reg-mem operation
4296 pipe_class fpu_reg_mem(regD dst, memory mem)
4297 %{
4298     instruction_count(2);
4299     dst    : S5(write);
4300     mem    : S3(read);
4301     D0     : S0;        // big decoder only
4302     DECODE : S1;        // any decoder for FPU POP
4303     FPU    : S4;
4304     MEM    : S3;        // any mem
4305 %}
4306 
4307 // Float reg-mem operation
4308 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4309 %{
4310     instruction_count(3);
4311     dst    : S5(write);
4312     src1   : S3(read);
4313     mem    : S3(read);
4314     D0     : S0;        // big decoder only
4315     DECODE : S1(2);     // any decoder for FPU POP
4316     FPU    : S4;
4317     MEM    : S3;        // any mem
4318 %}
4319 
4320 // Float mem-reg operation
4321 pipe_class fpu_mem_reg(memory mem, regD src)
4322 %{
4323     instruction_count(2);
4324     src    : S5(read);
4325     mem    : S3(read);
4326     DECODE : S0;        // any decoder for FPU PUSH
4327     D0     : S1;        // big decoder only
4328     FPU    : S4;
4329     MEM    : S3;        // any mem
4330 %}
4331 
4332 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4333 %{
4334     instruction_count(3);
4335     src1   : S3(read);
4336     src2   : S3(read);
4337     mem    : S3(read);
4338     DECODE : S0(2);     // any decoder for FPU PUSH
4339     D0     : S1;        // big decoder only
4340     FPU    : S4;
4341     MEM    : S3;        // any mem
4342 %}
4343 
4344 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4345 %{
4346     instruction_count(3);
4347     src1   : S3(read);
4348     src2   : S3(read);
4349     mem    : S4(read);
4350     DECODE : S0;        // any decoder for FPU PUSH
4351     D0     : S0(2);     // big decoder only
4352     FPU    : S4;
4353     MEM    : S3(2);     // any mem
4354 %}
4355 
4356 pipe_class fpu_mem_mem(memory dst, memory src1)
4357 %{
4358     instruction_count(2);
4359     src1   : S3(read);
4360     dst    : S4(read);
4361     D0     : S0(2);     // big decoder only
4362     MEM    : S3(2);     // any mem
4363 %}
4364 
4365 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4366 %{
4367     instruction_count(3);
4368     src1   : S3(read);
4369     src2   : S3(read);
4370     dst    : S4(read);
4371     D0     : S0(3);     // big decoder only
4372     FPU    : S4;
4373     MEM    : S3(3);     // any mem
4374 %}
4375 
4376 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4377 %{
4378     instruction_count(3);
4379     src1   : S4(read);
4380     mem    : S4(read);
4381     DECODE : S0;        // any decoder for FPU PUSH
4382     D0     : S0(2);     // big decoder only
4383     FPU    : S4;
4384     MEM    : S3(2);     // any mem
4385 %}
4386 
4387 // Float load constant
4388 pipe_class fpu_reg_con(regD dst)
4389 %{
4390     instruction_count(2);
4391     dst    : S5(write);
4392     D0     : S0;        // big decoder only for the load
4393     DECODE : S1;        // any decoder for FPU POP
4394     FPU    : S4;
4395     MEM    : S3;        // any mem
4396 %}
4397 
4398 // Float load constant
4399 pipe_class fpu_reg_reg_con(regD dst, regD src)
4400 %{
4401     instruction_count(3);
4402     dst    : S5(write);
4403     src    : S3(read);
4404     D0     : S0;        // big decoder only for the load
4405     DECODE : S1(2);     // any decoder for FPU POP
4406     FPU    : S4;
4407     MEM    : S3;        // any mem
4408 %}
4409 
4410 // UnConditional branch
4411 pipe_class pipe_jmp(label labl)
4412 %{
4413     single_instruction;
4414     BR   : S3;
4415 %}
4416 
4417 // Conditional branch
4418 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4419 %{
4420     single_instruction;
4421     cr    : S1(read);
4422     BR    : S3;
4423 %}
4424 
4425 // Allocation idiom
4426 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4427 %{
4428     instruction_count(1); force_serialization;
4429     fixed_latency(6);
4430     heap_ptr : S3(read);
4431     DECODE   : S0(3);
4432     D0       : S2;
4433     MEM      : S3;
4434     ALU      : S3(2);
4435     dst      : S5(write);
4436     BR       : S5;
4437 %}
4438 
4439 // Generic big/slow expanded idiom
4440 pipe_class pipe_slow()
4441 %{
4442     instruction_count(10); multiple_bundles; force_serialization;
4443     fixed_latency(100);
4444     D0  : S0(2);
4445     MEM : S3(2);
4446 %}
4447 
4448 // The real do-nothing guy
4449 pipe_class empty()
4450 %{
4451     instruction_count(0);
4452 %}
4453 
4454 // Define the class for the Nop node
4455 define
4456 %{
4457    MachNop = empty;
4458 %}
4459 
4460 %}
4461 
4462 //----------INSTRUCTIONS-------------------------------------------------------
4463 //
4464 // match      -- States which machine-independent subtree may be replaced
4465 //               by this instruction.
4466 // ins_cost   -- The estimated cost of this instruction is used by instruction
4467 //               selection to identify a minimum cost tree of machine
4468 //               instructions that matches a tree of machine-independent
4469 //               instructions.
4470 // format     -- A string providing the disassembly for this instruction.
4471 //               The value of an instruction's operand may be inserted
4472 //               by referring to it with a '$' prefix.
4473 // opcode     -- Three instruction opcodes may be provided.  These are referred
4474 //               to within an encode class as $primary, $secondary, and $tertiary
4475 //               rrspectively.  The primary opcode is commonly used to
4476 //               indicate the type of machine instruction, while secondary
4477 //               and tertiary are often used for prefix options or addressing
4478 //               modes.
4479 // ins_encode -- A list of encode classes with parameters. The encode class
4480 //               name must have been defined in an 'enc_class' specification
4481 //               in the encode section of the architecture description.
4482 
4483 
4484 //----------Load/Store/Move Instructions---------------------------------------
4485 //----------Load Instructions--------------------------------------------------
4486 
4487 // Load Byte (8 bit signed)
4488 instruct loadB(rRegI dst, memory mem)
4489 %{
4490   match(Set dst (LoadB mem));
4491 
4492   ins_cost(125);
4493   format %{ "movsbl  $dst, $mem\t# byte" %}
4494 
4495   ins_encode %{
4496     __ movsbl($dst$$Register, $mem$$Address);
4497   %}
4498 
4499   ins_pipe(ialu_reg_mem);
4500 %}
4501 
4502 // Load Byte (8 bit signed) into Long Register
4503 instruct loadB2L(rRegL dst, memory mem)
4504 %{
4505   match(Set dst (ConvI2L (LoadB mem)));
4506 
4507   ins_cost(125);
4508   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4509 
4510   ins_encode %{
4511     __ movsbq($dst$$Register, $mem$$Address);
4512   %}
4513 
4514   ins_pipe(ialu_reg_mem);
4515 %}
4516 
4517 // Load Unsigned Byte (8 bit UNsigned)
4518 instruct loadUB(rRegI dst, memory mem)
4519 %{
4520   match(Set dst (LoadUB mem));
4521 
4522   ins_cost(125);
4523   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4524 
4525   ins_encode %{
4526     __ movzbl($dst$$Register, $mem$$Address);
4527   %}
4528 
4529   ins_pipe(ialu_reg_mem);
4530 %}
4531 
4532 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4533 instruct loadUB2L(rRegL dst, memory mem)
4534 %{
4535   match(Set dst (ConvI2L (LoadUB mem)));
4536 
4537   ins_cost(125);
4538   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4539 
4540   ins_encode %{
4541     __ movzbq($dst$$Register, $mem$$Address);
4542   %}
4543 
4544   ins_pipe(ialu_reg_mem);
4545 %}
4546 
4547 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4548 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4549   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4550   effect(KILL cr);
4551 
4552   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4553             "andl    $dst, $mask" %}
4554   ins_encode %{
4555     Register Rdst = $dst$$Register;
4556     __ movzbq(Rdst, $mem$$Address);
4557     __ andl(Rdst, $mask$$constant);
4558   %}
4559   ins_pipe(ialu_reg_mem);
4560 %}
4561 
4562 // Load Short (16 bit signed)
4563 instruct loadS(rRegI dst, memory mem)
4564 %{
4565   match(Set dst (LoadS mem));
4566 
4567   ins_cost(125);
4568   format %{ "movswl $dst, $mem\t# short" %}
4569 
4570   ins_encode %{
4571     __ movswl($dst$$Register, $mem$$Address);
4572   %}
4573 
4574   ins_pipe(ialu_reg_mem);
4575 %}
4576 
4577 // Load Short (16 bit signed) to Byte (8 bit signed)
4578 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4579   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4580 
4581   ins_cost(125);
4582   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4583   ins_encode %{
4584     __ movsbl($dst$$Register, $mem$$Address);
4585   %}
4586   ins_pipe(ialu_reg_mem);
4587 %}
4588 
4589 // Load Short (16 bit signed) into Long Register
4590 instruct loadS2L(rRegL dst, memory mem)
4591 %{
4592   match(Set dst (ConvI2L (LoadS mem)));
4593 
4594   ins_cost(125);
4595   format %{ "movswq $dst, $mem\t# short -> long" %}
4596 
4597   ins_encode %{
4598     __ movswq($dst$$Register, $mem$$Address);
4599   %}
4600 
4601   ins_pipe(ialu_reg_mem);
4602 %}
4603 
4604 // Load Unsigned Short/Char (16 bit UNsigned)
4605 instruct loadUS(rRegI dst, memory mem)
4606 %{
4607   match(Set dst (LoadUS mem));
4608 
4609   ins_cost(125);
4610   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4611 
4612   ins_encode %{
4613     __ movzwl($dst$$Register, $mem$$Address);
4614   %}
4615 
4616   ins_pipe(ialu_reg_mem);
4617 %}
4618 
4619 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4620 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4621   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4622 
4623   ins_cost(125);
4624   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4625   ins_encode %{
4626     __ movsbl($dst$$Register, $mem$$Address);
4627   %}
4628   ins_pipe(ialu_reg_mem);
4629 %}
4630 
4631 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4632 instruct loadUS2L(rRegL dst, memory mem)
4633 %{
4634   match(Set dst (ConvI2L (LoadUS mem)));
4635 
4636   ins_cost(125);
4637   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4638 
4639   ins_encode %{
4640     __ movzwq($dst$$Register, $mem$$Address);
4641   %}
4642 
4643   ins_pipe(ialu_reg_mem);
4644 %}
4645 
4646 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4647 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4648   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4649 
4650   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4651   ins_encode %{
4652     __ movzbq($dst$$Register, $mem$$Address);
4653   %}
4654   ins_pipe(ialu_reg_mem);
4655 %}
4656 
4657 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4658 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4659   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4660   effect(KILL cr);
4661 
4662   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4663             "andl    $dst, $mask" %}
4664   ins_encode %{
4665     Register Rdst = $dst$$Register;
4666     __ movzwq(Rdst, $mem$$Address);
4667     __ andl(Rdst, $mask$$constant);
4668   %}
4669   ins_pipe(ialu_reg_mem);
4670 %}
4671 
4672 // Load Integer
4673 instruct loadI(rRegI dst, memory mem)
4674 %{
4675   match(Set dst (LoadI mem));
4676 
4677   ins_cost(125);
4678   format %{ "movl    $dst, $mem\t# int" %}
4679 
4680   ins_encode %{
4681     __ movl($dst$$Register, $mem$$Address);
4682   %}
4683 
4684   ins_pipe(ialu_reg_mem);
4685 %}
4686 
4687 // Load Integer (32 bit signed) to Byte (8 bit signed)
4688 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4689   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4690 
4691   ins_cost(125);
4692   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4693   ins_encode %{
4694     __ movsbl($dst$$Register, $mem$$Address);
4695   %}
4696   ins_pipe(ialu_reg_mem);
4697 %}
4698 
4699 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4700 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4701   match(Set dst (AndI (LoadI mem) mask));
4702 
4703   ins_cost(125);
4704   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4705   ins_encode %{
4706     __ movzbl($dst$$Register, $mem$$Address);
4707   %}
4708   ins_pipe(ialu_reg_mem);
4709 %}
4710 
4711 // Load Integer (32 bit signed) to Short (16 bit signed)
4712 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4713   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4714 
4715   ins_cost(125);
4716   format %{ "movswl  $dst, $mem\t# int -> short" %}
4717   ins_encode %{
4718     __ movswl($dst$$Register, $mem$$Address);
4719   %}
4720   ins_pipe(ialu_reg_mem);
4721 %}
4722 
4723 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4724 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4725   match(Set dst (AndI (LoadI mem) mask));
4726 
4727   ins_cost(125);
4728   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4729   ins_encode %{
4730     __ movzwl($dst$$Register, $mem$$Address);
4731   %}
4732   ins_pipe(ialu_reg_mem);
4733 %}
4734 
4735 // Load Integer into Long Register
4736 instruct loadI2L(rRegL dst, memory mem)
4737 %{
4738   match(Set dst (ConvI2L (LoadI mem)));
4739 
4740   ins_cost(125);
4741   format %{ "movslq  $dst, $mem\t# int -> long" %}
4742 
4743   ins_encode %{
4744     __ movslq($dst$$Register, $mem$$Address);
4745   %}
4746 
4747   ins_pipe(ialu_reg_mem);
4748 %}
4749 
4750 // Load Integer with mask 0xFF into Long Register
4751 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4752   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4753 
4754   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4755   ins_encode %{
4756     __ movzbq($dst$$Register, $mem$$Address);
4757   %}
4758   ins_pipe(ialu_reg_mem);
4759 %}
4760 
4761 // Load Integer with mask 0xFFFF into Long Register
4762 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4763   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4764 
4765   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4766   ins_encode %{
4767     __ movzwq($dst$$Register, $mem$$Address);
4768   %}
4769   ins_pipe(ialu_reg_mem);
4770 %}
4771 
4772 // Load Integer with a 31-bit mask into Long Register
4773 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4774   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4775   effect(KILL cr);
4776 
4777   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4778             "andl    $dst, $mask" %}
4779   ins_encode %{
4780     Register Rdst = $dst$$Register;
4781     __ movl(Rdst, $mem$$Address);
4782     __ andl(Rdst, $mask$$constant);
4783   %}
4784   ins_pipe(ialu_reg_mem);
4785 %}
4786 
4787 // Load Unsigned Integer into Long Register
4788 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4789 %{
4790   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4791 
4792   ins_cost(125);
4793   format %{ "movl    $dst, $mem\t# uint -> long" %}
4794 
4795   ins_encode %{
4796     __ movl($dst$$Register, $mem$$Address);
4797   %}
4798 
4799   ins_pipe(ialu_reg_mem);
4800 %}
4801 
4802 // Load Long
4803 instruct loadL(rRegL dst, memory mem)
4804 %{
4805   match(Set dst (LoadL mem));
4806 
4807   ins_cost(125);
4808   format %{ "movq    $dst, $mem\t# long" %}
4809 
4810   ins_encode %{
4811     __ movq($dst$$Register, $mem$$Address);
4812   %}
4813 
4814   ins_pipe(ialu_reg_mem); // XXX
4815 %}
4816 
4817 // Load Range
4818 instruct loadRange(rRegI dst, memory mem)
4819 %{
4820   match(Set dst (LoadRange mem));
4821 
4822   ins_cost(125); // XXX
4823   format %{ "movl    $dst, $mem\t# range" %}
4824   opcode(0x8B);
4825   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4826   ins_pipe(ialu_reg_mem);
4827 %}
4828 
4829 // Load Pointer
4830 instruct loadP(rRegP dst, memory mem)
4831 %{
4832   match(Set dst (LoadP mem));
4833 
4834   ins_cost(125); // XXX
4835   format %{ "movq    $dst, $mem\t# ptr" %}
4836   opcode(0x8B);
4837   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4838   ins_pipe(ialu_reg_mem); // XXX
4839 %}
4840 
4841 // Load Compressed Pointer
4842 instruct loadN(rRegN dst, memory mem)
4843 %{
4844    match(Set dst (LoadN mem));
4845 
4846    ins_cost(125); // XXX
4847    format %{ "movl    $dst, $mem\t# compressed ptr" %}
4848    ins_encode %{
4849      __ movl($dst$$Register, $mem$$Address);
4850    %}
4851    ins_pipe(ialu_reg_mem); // XXX
4852 %}
4853 
4854 
4855 // Load Klass Pointer
4856 instruct loadKlass(rRegP dst, memory mem)
4857 %{
4858   match(Set dst (LoadKlass mem));
4859 
4860   ins_cost(125); // XXX
4861   format %{ "movq    $dst, $mem\t# class" %}
4862   opcode(0x8B);
4863   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4864   ins_pipe(ialu_reg_mem); // XXX
4865 %}
4866 
4867 // Load narrow Klass Pointer
4868 instruct loadNKlass(rRegN dst, memory mem)
4869 %{
4870   match(Set dst (LoadNKlass mem));
4871 
4872   ins_cost(125); // XXX
4873   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
4874   ins_encode %{
4875     __ movl($dst$$Register, $mem$$Address);
4876   %}
4877   ins_pipe(ialu_reg_mem); // XXX
4878 %}
4879 
4880 // Load Float
4881 instruct loadF(regF dst, memory mem)
4882 %{
4883   match(Set dst (LoadF mem));
4884 
4885   ins_cost(145); // XXX
4886   format %{ "movss   $dst, $mem\t# float" %}
4887   ins_encode %{
4888     __ movflt($dst$$XMMRegister, $mem$$Address);
4889   %}
4890   ins_pipe(pipe_slow); // XXX
4891 %}
4892 
4893 // Load Double
4894 instruct loadD_partial(regD dst, memory mem)
4895 %{
4896   predicate(!UseXmmLoadAndClearUpper);
4897   match(Set dst (LoadD mem));
4898 
4899   ins_cost(145); // XXX
4900   format %{ "movlpd  $dst, $mem\t# double" %}
4901   ins_encode %{
4902     __ movdbl($dst$$XMMRegister, $mem$$Address);
4903   %}
4904   ins_pipe(pipe_slow); // XXX
4905 %}
4906 
4907 instruct loadD(regD dst, memory mem)
4908 %{
4909   predicate(UseXmmLoadAndClearUpper);
4910   match(Set dst (LoadD mem));
4911 
4912   ins_cost(145); // XXX
4913   format %{ "movsd   $dst, $mem\t# double" %}
4914   ins_encode %{
4915     __ movdbl($dst$$XMMRegister, $mem$$Address);
4916   %}
4917   ins_pipe(pipe_slow); // XXX
4918 %}
4919 
4920 // Load Effective Address
4921 instruct leaP8(rRegP dst, indOffset8 mem)
4922 %{
4923   match(Set dst mem);
4924 
4925   ins_cost(110); // XXX
4926   format %{ "leaq    $dst, $mem\t# ptr 8" %}
4927   opcode(0x8D);
4928   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4929   ins_pipe(ialu_reg_reg_fat);
4930 %}
4931 
4932 instruct leaP32(rRegP dst, indOffset32 mem)
4933 %{
4934   match(Set dst mem);
4935 
4936   ins_cost(110);
4937   format %{ "leaq    $dst, $mem\t# ptr 32" %}
4938   opcode(0x8D);
4939   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4940   ins_pipe(ialu_reg_reg_fat);
4941 %}
4942 
4943 // instruct leaPIdx(rRegP dst, indIndex mem)
4944 // %{
4945 //   match(Set dst mem);
4946 
4947 //   ins_cost(110);
4948 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
4949 //   opcode(0x8D);
4950 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4951 //   ins_pipe(ialu_reg_reg_fat);
4952 // %}
4953 
4954 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
4955 %{
4956   match(Set dst mem);
4957 
4958   ins_cost(110);
4959   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
4960   opcode(0x8D);
4961   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4962   ins_pipe(ialu_reg_reg_fat);
4963 %}
4964 
4965 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
4966 %{
4967   match(Set dst mem);
4968 
4969   ins_cost(110);
4970   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
4971   opcode(0x8D);
4972   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4973   ins_pipe(ialu_reg_reg_fat);
4974 %}
4975 
4976 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
4977 %{
4978   match(Set dst mem);
4979 
4980   ins_cost(110);
4981   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
4982   opcode(0x8D);
4983   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4984   ins_pipe(ialu_reg_reg_fat);
4985 %}
4986 
4987 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
4988 %{
4989   match(Set dst mem);
4990 
4991   ins_cost(110);
4992   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
4993   opcode(0x8D);
4994   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4995   ins_pipe(ialu_reg_reg_fat);
4996 %}
4997 
4998 // Load Effective Address which uses Narrow (32-bits) oop
4999 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5000 %{
5001   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5002   match(Set dst mem);
5003 
5004   ins_cost(110);
5005   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5006   opcode(0x8D);
5007   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5008   ins_pipe(ialu_reg_reg_fat);
5009 %}
5010 
5011 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5012 %{
5013   predicate(Universe::narrow_oop_shift() == 0);
5014   match(Set dst mem);
5015 
5016   ins_cost(110); // XXX
5017   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5018   opcode(0x8D);
5019   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5020   ins_pipe(ialu_reg_reg_fat);
5021 %}
5022 
5023 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5024 %{
5025   predicate(Universe::narrow_oop_shift() == 0);
5026   match(Set dst mem);
5027 
5028   ins_cost(110);
5029   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5030   opcode(0x8D);
5031   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5032   ins_pipe(ialu_reg_reg_fat);
5033 %}
5034 
5035 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5036 %{
5037   predicate(Universe::narrow_oop_shift() == 0);
5038   match(Set dst mem);
5039 
5040   ins_cost(110);
5041   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5042   opcode(0x8D);
5043   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5044   ins_pipe(ialu_reg_reg_fat);
5045 %}
5046 
5047 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5048 %{
5049   predicate(Universe::narrow_oop_shift() == 0);
5050   match(Set dst mem);
5051 
5052   ins_cost(110);
5053   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5054   opcode(0x8D);
5055   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5056   ins_pipe(ialu_reg_reg_fat);
5057 %}
5058 
5059 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5060 %{
5061   predicate(Universe::narrow_oop_shift() == 0);
5062   match(Set dst mem);
5063 
5064   ins_cost(110);
5065   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5066   opcode(0x8D);
5067   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5068   ins_pipe(ialu_reg_reg_fat);
5069 %}
5070 
5071 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5072 %{
5073   predicate(Universe::narrow_oop_shift() == 0);
5074   match(Set dst mem);
5075 
5076   ins_cost(110);
5077   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5078   opcode(0x8D);
5079   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5080   ins_pipe(ialu_reg_reg_fat);
5081 %}
5082 
5083 instruct loadConI(rRegI dst, immI src)
5084 %{
5085   match(Set dst src);
5086 
5087   format %{ "movl    $dst, $src\t# int" %}
5088   ins_encode(load_immI(dst, src));
5089   ins_pipe(ialu_reg_fat); // XXX
5090 %}
5091 
5092 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5093 %{
5094   match(Set dst src);
5095   effect(KILL cr);
5096 
5097   ins_cost(50);
5098   format %{ "xorl    $dst, $dst\t# int" %}
5099   opcode(0x33); /* + rd */
5100   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5101   ins_pipe(ialu_reg);
5102 %}
5103 
5104 instruct loadConL(rRegL dst, immL src)
5105 %{
5106   match(Set dst src);
5107 
5108   ins_cost(150);
5109   format %{ "movq    $dst, $src\t# long" %}
5110   ins_encode(load_immL(dst, src));
5111   ins_pipe(ialu_reg);
5112 %}
5113 
5114 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5115 %{
5116   match(Set dst src);
5117   effect(KILL cr);
5118 
5119   ins_cost(50);
5120   format %{ "xorl    $dst, $dst\t# long" %}
5121   opcode(0x33); /* + rd */
5122   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5123   ins_pipe(ialu_reg); // XXX
5124 %}
5125 
5126 instruct loadConUL32(rRegL dst, immUL32 src)
5127 %{
5128   match(Set dst src);
5129 
5130   ins_cost(60);
5131   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5132   ins_encode(load_immUL32(dst, src));
5133   ins_pipe(ialu_reg);
5134 %}
5135 
5136 instruct loadConL32(rRegL dst, immL32 src)
5137 %{
5138   match(Set dst src);
5139 
5140   ins_cost(70);
5141   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5142   ins_encode(load_immL32(dst, src));
5143   ins_pipe(ialu_reg);
5144 %}
5145 
5146 instruct loadConP(rRegP dst, immP con) %{
5147   match(Set dst con);
5148 
5149   format %{ "movq    $dst, $con\t# ptr" %}
5150   ins_encode(load_immP(dst, con));
5151   ins_pipe(ialu_reg_fat); // XXX
5152 %}
5153 
5154 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5155 %{
5156   match(Set dst src);
5157   effect(KILL cr);
5158 
5159   ins_cost(50);
5160   format %{ "xorl    $dst, $dst\t# ptr" %}
5161   opcode(0x33); /* + rd */
5162   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5163   ins_pipe(ialu_reg);
5164 %}
5165 
5166 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5167 %{
5168   match(Set dst src);
5169   effect(KILL cr);
5170 
5171   ins_cost(60);
5172   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5173   ins_encode(load_immP31(dst, src));
5174   ins_pipe(ialu_reg);
5175 %}
5176 
5177 instruct loadConF(regF dst, immF con) %{
5178   match(Set dst con);
5179   ins_cost(125);
5180   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5181   ins_encode %{
5182     __ movflt($dst$$XMMRegister, $constantaddress($con));
5183   %}
5184   ins_pipe(pipe_slow);
5185 %}
5186 
5187 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5188   match(Set dst src);
5189   effect(KILL cr);
5190   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5191   ins_encode %{
5192     __ xorq($dst$$Register, $dst$$Register);
5193   %}
5194   ins_pipe(ialu_reg);
5195 %}
5196 
5197 instruct loadConN(rRegN dst, immN src) %{
5198   match(Set dst src);
5199 
5200   ins_cost(125);
5201   format %{ "movl    $dst, $src\t# compressed ptr" %}
5202   ins_encode %{
5203     address con = (address)$src$$constant;
5204     if (con == NULL) {
5205       ShouldNotReachHere();
5206     } else {
5207       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5208     }
5209   %}
5210   ins_pipe(ialu_reg_fat); // XXX
5211 %}
5212 
5213 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5214   match(Set dst src);
5215 
5216   ins_cost(125);
5217   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5218   ins_encode %{
5219     address con = (address)$src$$constant;
5220     if (con == NULL) {
5221       ShouldNotReachHere();
5222     } else {
5223       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5224     }
5225   %}
5226   ins_pipe(ialu_reg_fat); // XXX
5227 %}
5228 
5229 instruct loadConF0(regF dst, immF0 src)
5230 %{
5231   match(Set dst src);
5232   ins_cost(100);
5233 
5234   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5235   ins_encode %{
5236     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5237   %}
5238   ins_pipe(pipe_slow);
5239 %}
5240 
5241 // Use the same format since predicate() can not be used here.
5242 instruct loadConD(regD dst, immD con) %{
5243   match(Set dst con);
5244   ins_cost(125);
5245   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5246   ins_encode %{
5247     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5248   %}
5249   ins_pipe(pipe_slow);
5250 %}
5251 
5252 instruct loadConD0(regD dst, immD0 src)
5253 %{
5254   match(Set dst src);
5255   ins_cost(100);
5256 
5257   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5258   ins_encode %{
5259     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5260   %}
5261   ins_pipe(pipe_slow);
5262 %}
5263 
5264 instruct loadSSI(rRegI dst, stackSlotI src)
5265 %{
5266   match(Set dst src);
5267 
5268   ins_cost(125);
5269   format %{ "movl    $dst, $src\t# int stk" %}
5270   opcode(0x8B);
5271   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5272   ins_pipe(ialu_reg_mem);
5273 %}
5274 
5275 instruct loadSSL(rRegL dst, stackSlotL src)
5276 %{
5277   match(Set dst src);
5278 
5279   ins_cost(125);
5280   format %{ "movq    $dst, $src\t# long stk" %}
5281   opcode(0x8B);
5282   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5283   ins_pipe(ialu_reg_mem);
5284 %}
5285 
5286 instruct loadSSP(rRegP dst, stackSlotP src)
5287 %{
5288   match(Set dst src);
5289 
5290   ins_cost(125);
5291   format %{ "movq    $dst, $src\t# ptr stk" %}
5292   opcode(0x8B);
5293   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5294   ins_pipe(ialu_reg_mem);
5295 %}
5296 
5297 instruct loadSSF(regF dst, stackSlotF src)
5298 %{
5299   match(Set dst src);
5300 
5301   ins_cost(125);
5302   format %{ "movss   $dst, $src\t# float stk" %}
5303   ins_encode %{
5304     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5305   %}
5306   ins_pipe(pipe_slow); // XXX
5307 %}
5308 
5309 // Use the same format since predicate() can not be used here.
5310 instruct loadSSD(regD dst, stackSlotD src)
5311 %{
5312   match(Set dst src);
5313 
5314   ins_cost(125);
5315   format %{ "movsd   $dst, $src\t# double stk" %}
5316   ins_encode  %{
5317     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5318   %}
5319   ins_pipe(pipe_slow); // XXX
5320 %}
5321 
5322 // Prefetch instructions.
5323 // Must be safe to execute with invalid address (cannot fault).
5324 
5325 instruct prefetchr( memory mem ) %{
5326   predicate(ReadPrefetchInstr==3);
5327   match(PrefetchRead mem);
5328   ins_cost(125);
5329 
5330   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5331   ins_encode %{
5332     __ prefetchr($mem$$Address);
5333   %}
5334   ins_pipe(ialu_mem);
5335 %}
5336 
5337 instruct prefetchrNTA( memory mem ) %{
5338   predicate(ReadPrefetchInstr==0);
5339   match(PrefetchRead mem);
5340   ins_cost(125);
5341 
5342   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5343   ins_encode %{
5344     __ prefetchnta($mem$$Address);
5345   %}
5346   ins_pipe(ialu_mem);
5347 %}
5348 
5349 instruct prefetchrT0( memory mem ) %{
5350   predicate(ReadPrefetchInstr==1);
5351   match(PrefetchRead mem);
5352   ins_cost(125);
5353 
5354   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5355   ins_encode %{
5356     __ prefetcht0($mem$$Address);
5357   %}
5358   ins_pipe(ialu_mem);
5359 %}
5360 
5361 instruct prefetchrT2( memory mem ) %{
5362   predicate(ReadPrefetchInstr==2);
5363   match(PrefetchRead mem);
5364   ins_cost(125);
5365 
5366   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5367   ins_encode %{
5368     __ prefetcht2($mem$$Address);
5369   %}
5370   ins_pipe(ialu_mem);
5371 %}
5372 
5373 instruct prefetchwNTA( memory mem ) %{
5374   match(PrefetchWrite mem);
5375   ins_cost(125);
5376 
5377   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5378   ins_encode %{
5379     __ prefetchnta($mem$$Address);
5380   %}
5381   ins_pipe(ialu_mem);
5382 %}
5383 
5384 // Prefetch instructions for allocation.
5385 
5386 instruct prefetchAlloc( memory mem ) %{
5387   predicate(AllocatePrefetchInstr==3);
5388   match(PrefetchAllocation mem);
5389   ins_cost(125);
5390 
5391   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5392   ins_encode %{
5393     __ prefetchw($mem$$Address);
5394   %}
5395   ins_pipe(ialu_mem);
5396 %}
5397 
5398 instruct prefetchAllocNTA( memory mem ) %{
5399   predicate(AllocatePrefetchInstr==0);
5400   match(PrefetchAllocation mem);
5401   ins_cost(125);
5402 
5403   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5404   ins_encode %{
5405     __ prefetchnta($mem$$Address);
5406   %}
5407   ins_pipe(ialu_mem);
5408 %}
5409 
5410 instruct prefetchAllocT0( memory mem ) %{
5411   predicate(AllocatePrefetchInstr==1);
5412   match(PrefetchAllocation mem);
5413   ins_cost(125);
5414 
5415   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5416   ins_encode %{
5417     __ prefetcht0($mem$$Address);
5418   %}
5419   ins_pipe(ialu_mem);
5420 %}
5421 
5422 instruct prefetchAllocT2( memory mem ) %{
5423   predicate(AllocatePrefetchInstr==2);
5424   match(PrefetchAllocation mem);
5425   ins_cost(125);
5426 
5427   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5428   ins_encode %{
5429     __ prefetcht2($mem$$Address);
5430   %}
5431   ins_pipe(ialu_mem);
5432 %}
5433 
5434 //----------Store Instructions-------------------------------------------------
5435 
5436 // Store Byte
5437 instruct storeB(memory mem, rRegI src)
5438 %{
5439   match(Set mem (StoreB mem src));
5440 
5441   ins_cost(125); // XXX
5442   format %{ "movb    $mem, $src\t# byte" %}
5443   opcode(0x88);
5444   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5445   ins_pipe(ialu_mem_reg);
5446 %}
5447 
5448 // Store Char/Short
5449 instruct storeC(memory mem, rRegI src)
5450 %{
5451   match(Set mem (StoreC mem src));
5452 
5453   ins_cost(125); // XXX
5454   format %{ "movw    $mem, $src\t# char/short" %}
5455   opcode(0x89);
5456   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5457   ins_pipe(ialu_mem_reg);
5458 %}
5459 
5460 // Store Integer
5461 instruct storeI(memory mem, rRegI src)
5462 %{
5463   match(Set mem (StoreI mem src));
5464 
5465   ins_cost(125); // XXX
5466   format %{ "movl    $mem, $src\t# int" %}
5467   opcode(0x89);
5468   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5469   ins_pipe(ialu_mem_reg);
5470 %}
5471 
5472 // Store Long
5473 instruct storeL(memory mem, rRegL src)
5474 %{
5475   match(Set mem (StoreL mem src));
5476 
5477   ins_cost(125); // XXX
5478   format %{ "movq    $mem, $src\t# long" %}
5479   opcode(0x89);
5480   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5481   ins_pipe(ialu_mem_reg); // XXX
5482 %}
5483 
5484 // Store Pointer
5485 instruct storeP(memory mem, any_RegP src)
5486 %{
5487   match(Set mem (StoreP mem src));
5488 
5489   ins_cost(125); // XXX
5490   format %{ "movq    $mem, $src\t# ptr" %}
5491   opcode(0x89);
5492   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5493   ins_pipe(ialu_mem_reg);
5494 %}
5495 
5496 instruct storeImmP0(memory mem, immP0 zero)
5497 %{
5498   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5499   match(Set mem (StoreP mem zero));
5500 
5501   ins_cost(125); // XXX
5502   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5503   ins_encode %{
5504     __ movq($mem$$Address, r12);
5505   %}
5506   ins_pipe(ialu_mem_reg);
5507 %}
5508 
5509 // Store NULL Pointer, mark word, or other simple pointer constant.
5510 instruct storeImmP(memory mem, immP31 src)
5511 %{
5512   match(Set mem (StoreP mem src));
5513 
5514   ins_cost(150); // XXX
5515   format %{ "movq    $mem, $src\t# ptr" %}
5516   opcode(0xC7); /* C7 /0 */
5517   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5518   ins_pipe(ialu_mem_imm);
5519 %}
5520 
5521 // Store Compressed Pointer
5522 instruct storeN(memory mem, rRegN src)
5523 %{
5524   match(Set mem (StoreN mem src));
5525 
5526   ins_cost(125); // XXX
5527   format %{ "movl    $mem, $src\t# compressed ptr" %}
5528   ins_encode %{
5529     __ movl($mem$$Address, $src$$Register);
5530   %}
5531   ins_pipe(ialu_mem_reg);
5532 %}
5533 
5534 instruct storeNKlass(memory mem, rRegN src)
5535 %{
5536   match(Set mem (StoreNKlass mem src));
5537 
5538   ins_cost(125); // XXX
5539   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5540   ins_encode %{
5541     __ movl($mem$$Address, $src$$Register);
5542   %}
5543   ins_pipe(ialu_mem_reg);
5544 %}
5545 
5546 instruct storeImmN0(memory mem, immN0 zero)
5547 %{
5548   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5549   match(Set mem (StoreN mem zero));
5550 
5551   ins_cost(125); // XXX
5552   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5553   ins_encode %{
5554     __ movl($mem$$Address, r12);
5555   %}
5556   ins_pipe(ialu_mem_reg);
5557 %}
5558 
5559 instruct storeImmN(memory mem, immN src)
5560 %{
5561   match(Set mem (StoreN mem src));
5562 
5563   ins_cost(150); // XXX
5564   format %{ "movl    $mem, $src\t# compressed ptr" %}
5565   ins_encode %{
5566     address con = (address)$src$$constant;
5567     if (con == NULL) {
5568       __ movl($mem$$Address, (int32_t)0);
5569     } else {
5570       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5571     }
5572   %}
5573   ins_pipe(ialu_mem_imm);
5574 %}
5575 
5576 instruct storeImmNKlass(memory mem, immNKlass src)
5577 %{
5578   match(Set mem (StoreNKlass mem src));
5579 
5580   ins_cost(150); // XXX
5581   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5582   ins_encode %{
5583     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5584   %}
5585   ins_pipe(ialu_mem_imm);
5586 %}
5587 
5588 // Store Integer Immediate
5589 instruct storeImmI0(memory mem, immI0 zero)
5590 %{
5591   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5592   match(Set mem (StoreI mem zero));
5593 
5594   ins_cost(125); // XXX
5595   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5596   ins_encode %{
5597     __ movl($mem$$Address, r12);
5598   %}
5599   ins_pipe(ialu_mem_reg);
5600 %}
5601 
5602 instruct storeImmI(memory mem, immI src)
5603 %{
5604   match(Set mem (StoreI mem src));
5605 
5606   ins_cost(150);
5607   format %{ "movl    $mem, $src\t# int" %}
5608   opcode(0xC7); /* C7 /0 */
5609   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5610   ins_pipe(ialu_mem_imm);
5611 %}
5612 
5613 // Store Long Immediate
5614 instruct storeImmL0(memory mem, immL0 zero)
5615 %{
5616   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5617   match(Set mem (StoreL mem zero));
5618 
5619   ins_cost(125); // XXX
5620   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5621   ins_encode %{
5622     __ movq($mem$$Address, r12);
5623   %}
5624   ins_pipe(ialu_mem_reg);
5625 %}
5626 
5627 instruct storeImmL(memory mem, immL32 src)
5628 %{
5629   match(Set mem (StoreL mem src));
5630 
5631   ins_cost(150);
5632   format %{ "movq    $mem, $src\t# long" %}
5633   opcode(0xC7); /* C7 /0 */
5634   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5635   ins_pipe(ialu_mem_imm);
5636 %}
5637 
5638 // Store Short/Char Immediate
5639 instruct storeImmC0(memory mem, immI0 zero)
5640 %{
5641   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5642   match(Set mem (StoreC mem zero));
5643 
5644   ins_cost(125); // XXX
5645   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5646   ins_encode %{
5647     __ movw($mem$$Address, r12);
5648   %}
5649   ins_pipe(ialu_mem_reg);
5650 %}
5651 
5652 instruct storeImmI16(memory mem, immI16 src)
5653 %{
5654   predicate(UseStoreImmI16);
5655   match(Set mem (StoreC mem src));
5656 
5657   ins_cost(150);
5658   format %{ "movw    $mem, $src\t# short/char" %}
5659   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5660   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5661   ins_pipe(ialu_mem_imm);
5662 %}
5663 
5664 // Store Byte Immediate
5665 instruct storeImmB0(memory mem, immI0 zero)
5666 %{
5667   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5668   match(Set mem (StoreB mem zero));
5669 
5670   ins_cost(125); // XXX
5671   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5672   ins_encode %{
5673     __ movb($mem$$Address, r12);
5674   %}
5675   ins_pipe(ialu_mem_reg);
5676 %}
5677 
5678 instruct storeImmB(memory mem, immI8 src)
5679 %{
5680   match(Set mem (StoreB mem src));
5681 
5682   ins_cost(150); // XXX
5683   format %{ "movb    $mem, $src\t# byte" %}
5684   opcode(0xC6); /* C6 /0 */
5685   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5686   ins_pipe(ialu_mem_imm);
5687 %}
5688 
5689 // Store CMS card-mark Immediate
5690 instruct storeImmCM0_reg(memory mem, immI0 zero)
5691 %{
5692   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5693   match(Set mem (StoreCM mem zero));
5694 
5695   ins_cost(125); // XXX
5696   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5697   ins_encode %{
5698     __ movb($mem$$Address, r12);
5699   %}
5700   ins_pipe(ialu_mem_reg);
5701 %}
5702 
5703 instruct storeImmCM0(memory mem, immI0 src)
5704 %{
5705   match(Set mem (StoreCM mem src));
5706 
5707   ins_cost(150); // XXX
5708   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5709   opcode(0xC6); /* C6 /0 */
5710   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5711   ins_pipe(ialu_mem_imm);
5712 %}
5713 
5714 // Store Float
5715 instruct storeF(memory mem, regF src)
5716 %{
5717   match(Set mem (StoreF mem src));
5718 
5719   ins_cost(95); // XXX
5720   format %{ "movss   $mem, $src\t# float" %}
5721   ins_encode %{
5722     __ movflt($mem$$Address, $src$$XMMRegister);
5723   %}
5724   ins_pipe(pipe_slow); // XXX
5725 %}
5726 
5727 // Store immediate Float value (it is faster than store from XMM register)
5728 instruct storeF0(memory mem, immF0 zero)
5729 %{
5730   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5731   match(Set mem (StoreF mem zero));
5732 
5733   ins_cost(25); // XXX
5734   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5735   ins_encode %{
5736     __ movl($mem$$Address, r12);
5737   %}
5738   ins_pipe(ialu_mem_reg);
5739 %}
5740 
5741 instruct storeF_imm(memory mem, immF src)
5742 %{
5743   match(Set mem (StoreF mem src));
5744 
5745   ins_cost(50);
5746   format %{ "movl    $mem, $src\t# float" %}
5747   opcode(0xC7); /* C7 /0 */
5748   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5749   ins_pipe(ialu_mem_imm);
5750 %}
5751 
5752 // Store Double
5753 instruct storeD(memory mem, regD src)
5754 %{
5755   match(Set mem (StoreD mem src));
5756 
5757   ins_cost(95); // XXX
5758   format %{ "movsd   $mem, $src\t# double" %}
5759   ins_encode %{
5760     __ movdbl($mem$$Address, $src$$XMMRegister);
5761   %}
5762   ins_pipe(pipe_slow); // XXX
5763 %}
5764 
5765 // Store immediate double 0.0 (it is faster than store from XMM register)
5766 instruct storeD0_imm(memory mem, immD0 src)
5767 %{
5768   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5769   match(Set mem (StoreD mem src));
5770 
5771   ins_cost(50);
5772   format %{ "movq    $mem, $src\t# double 0." %}
5773   opcode(0xC7); /* C7 /0 */
5774   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5775   ins_pipe(ialu_mem_imm);
5776 %}
5777 
5778 instruct storeD0(memory mem, immD0 zero)
5779 %{
5780   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5781   match(Set mem (StoreD mem zero));
5782 
5783   ins_cost(25); // XXX
5784   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5785   ins_encode %{
5786     __ movq($mem$$Address, r12);
5787   %}
5788   ins_pipe(ialu_mem_reg);
5789 %}
5790 
5791 instruct storeSSI(stackSlotI dst, rRegI src)
5792 %{
5793   match(Set dst src);
5794 
5795   ins_cost(100);
5796   format %{ "movl    $dst, $src\t# int stk" %}
5797   opcode(0x89);
5798   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5799   ins_pipe( ialu_mem_reg );
5800 %}
5801 
5802 instruct storeSSL(stackSlotL dst, rRegL src)
5803 %{
5804   match(Set dst src);
5805 
5806   ins_cost(100);
5807   format %{ "movq    $dst, $src\t# long stk" %}
5808   opcode(0x89);
5809   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5810   ins_pipe(ialu_mem_reg);
5811 %}
5812 
5813 instruct storeSSP(stackSlotP dst, rRegP src)
5814 %{
5815   match(Set dst src);
5816 
5817   ins_cost(100);
5818   format %{ "movq    $dst, $src\t# ptr stk" %}
5819   opcode(0x89);
5820   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5821   ins_pipe(ialu_mem_reg);
5822 %}
5823 
5824 instruct storeSSF(stackSlotF dst, regF src)
5825 %{
5826   match(Set dst src);
5827 
5828   ins_cost(95); // XXX
5829   format %{ "movss   $dst, $src\t# float stk" %}
5830   ins_encode %{
5831     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5832   %}
5833   ins_pipe(pipe_slow); // XXX
5834 %}
5835 
5836 instruct storeSSD(stackSlotD dst, regD src)
5837 %{
5838   match(Set dst src);
5839 
5840   ins_cost(95); // XXX
5841   format %{ "movsd   $dst, $src\t# double stk" %}
5842   ins_encode %{
5843     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5844   %}
5845   ins_pipe(pipe_slow); // XXX
5846 %}
5847 
5848 //----------BSWAP Instructions-------------------------------------------------
5849 instruct bytes_reverse_int(rRegI dst) %{
5850   match(Set dst (ReverseBytesI dst));
5851 
5852   format %{ "bswapl  $dst" %}
5853   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
5854   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
5855   ins_pipe( ialu_reg );
5856 %}
5857 
5858 instruct bytes_reverse_long(rRegL dst) %{
5859   match(Set dst (ReverseBytesL dst));
5860 
5861   format %{ "bswapq  $dst" %}
5862   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
5863   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
5864   ins_pipe( ialu_reg);
5865 %}
5866 
5867 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
5868   match(Set dst (ReverseBytesUS dst));
5869   effect(KILL cr);
5870 
5871   format %{ "bswapl  $dst\n\t"
5872             "shrl    $dst,16\n\t" %}
5873   ins_encode %{
5874     __ bswapl($dst$$Register);
5875     __ shrl($dst$$Register, 16);
5876   %}
5877   ins_pipe( ialu_reg );
5878 %}
5879 
5880 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
5881   match(Set dst (ReverseBytesS dst));
5882   effect(KILL cr);
5883 
5884   format %{ "bswapl  $dst\n\t"
5885             "sar     $dst,16\n\t" %}
5886   ins_encode %{
5887     __ bswapl($dst$$Register);
5888     __ sarl($dst$$Register, 16);
5889   %}
5890   ins_pipe( ialu_reg );
5891 %}
5892 
5893 //---------- Zeros Count Instructions ------------------------------------------
5894 
5895 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5896   predicate(UseCountLeadingZerosInstruction);
5897   match(Set dst (CountLeadingZerosI src));
5898   effect(KILL cr);
5899 
5900   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
5901   ins_encode %{
5902     __ lzcntl($dst$$Register, $src$$Register);
5903   %}
5904   ins_pipe(ialu_reg);
5905 %}
5906 
5907 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
5908   predicate(!UseCountLeadingZerosInstruction);
5909   match(Set dst (CountLeadingZerosI src));
5910   effect(KILL cr);
5911 
5912   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
5913             "jnz     skip\n\t"
5914             "movl    $dst, -1\n"
5915       "skip:\n\t"
5916             "negl    $dst\n\t"
5917             "addl    $dst, 31" %}
5918   ins_encode %{
5919     Register Rdst = $dst$$Register;
5920     Register Rsrc = $src$$Register;
5921     Label skip;
5922     __ bsrl(Rdst, Rsrc);
5923     __ jccb(Assembler::notZero, skip);
5924     __ movl(Rdst, -1);
5925     __ bind(skip);
5926     __ negl(Rdst);
5927     __ addl(Rdst, BitsPerInt - 1);
5928   %}
5929   ins_pipe(ialu_reg);
5930 %}
5931 
5932 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5933   predicate(UseCountLeadingZerosInstruction);
5934   match(Set dst (CountLeadingZerosL src));
5935   effect(KILL cr);
5936 
5937   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
5938   ins_encode %{
5939     __ lzcntq($dst$$Register, $src$$Register);
5940   %}
5941   ins_pipe(ialu_reg);
5942 %}
5943 
5944 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
5945   predicate(!UseCountLeadingZerosInstruction);
5946   match(Set dst (CountLeadingZerosL src));
5947   effect(KILL cr);
5948 
5949   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
5950             "jnz     skip\n\t"
5951             "movl    $dst, -1\n"
5952       "skip:\n\t"
5953             "negl    $dst\n\t"
5954             "addl    $dst, 63" %}
5955   ins_encode %{
5956     Register Rdst = $dst$$Register;
5957     Register Rsrc = $src$$Register;
5958     Label skip;
5959     __ bsrq(Rdst, Rsrc);
5960     __ jccb(Assembler::notZero, skip);
5961     __ movl(Rdst, -1);
5962     __ bind(skip);
5963     __ negl(Rdst);
5964     __ addl(Rdst, BitsPerLong - 1);
5965   %}
5966   ins_pipe(ialu_reg);
5967 %}
5968 
5969 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5970   predicate(UseCountTrailingZerosInstruction);
5971   match(Set dst (CountTrailingZerosI src));
5972   effect(KILL cr);
5973 
5974   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
5975   ins_encode %{
5976     __ tzcntl($dst$$Register, $src$$Register);
5977   %}
5978   ins_pipe(ialu_reg);
5979 %}
5980 
5981 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
5982   predicate(!UseCountTrailingZerosInstruction);
5983   match(Set dst (CountTrailingZerosI src));
5984   effect(KILL cr);
5985 
5986   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
5987             "jnz     done\n\t"
5988             "movl    $dst, 32\n"
5989       "done:" %}
5990   ins_encode %{
5991     Register Rdst = $dst$$Register;
5992     Label done;
5993     __ bsfl(Rdst, $src$$Register);
5994     __ jccb(Assembler::notZero, done);
5995     __ movl(Rdst, BitsPerInt);
5996     __ bind(done);
5997   %}
5998   ins_pipe(ialu_reg);
5999 %}
6000 
6001 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6002   predicate(UseCountTrailingZerosInstruction);
6003   match(Set dst (CountTrailingZerosL src));
6004   effect(KILL cr);
6005 
6006   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6007   ins_encode %{
6008     __ tzcntq($dst$$Register, $src$$Register);
6009   %}
6010   ins_pipe(ialu_reg);
6011 %}
6012 
6013 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6014   predicate(!UseCountTrailingZerosInstruction);
6015   match(Set dst (CountTrailingZerosL src));
6016   effect(KILL cr);
6017 
6018   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6019             "jnz     done\n\t"
6020             "movl    $dst, 64\n"
6021       "done:" %}
6022   ins_encode %{
6023     Register Rdst = $dst$$Register;
6024     Label done;
6025     __ bsfq(Rdst, $src$$Register);
6026     __ jccb(Assembler::notZero, done);
6027     __ movl(Rdst, BitsPerLong);
6028     __ bind(done);
6029   %}
6030   ins_pipe(ialu_reg);
6031 %}
6032 
6033 
6034 //---------- Population Count Instructions -------------------------------------
6035 
6036 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6037   predicate(UsePopCountInstruction);
6038   match(Set dst (PopCountI src));
6039   effect(KILL cr);
6040 
6041   format %{ "popcnt  $dst, $src" %}
6042   ins_encode %{
6043     __ popcntl($dst$$Register, $src$$Register);
6044   %}
6045   ins_pipe(ialu_reg);
6046 %}
6047 
6048 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6049   predicate(UsePopCountInstruction);
6050   match(Set dst (PopCountI (LoadI mem)));
6051   effect(KILL cr);
6052 
6053   format %{ "popcnt  $dst, $mem" %}
6054   ins_encode %{
6055     __ popcntl($dst$$Register, $mem$$Address);
6056   %}
6057   ins_pipe(ialu_reg);
6058 %}
6059 
6060 // Note: Long.bitCount(long) returns an int.
6061 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6062   predicate(UsePopCountInstruction);
6063   match(Set dst (PopCountL src));
6064   effect(KILL cr);
6065 
6066   format %{ "popcnt  $dst, $src" %}
6067   ins_encode %{
6068     __ popcntq($dst$$Register, $src$$Register);
6069   %}
6070   ins_pipe(ialu_reg);
6071 %}
6072 
6073 // Note: Long.bitCount(long) returns an int.
6074 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6075   predicate(UsePopCountInstruction);
6076   match(Set dst (PopCountL (LoadL mem)));
6077   effect(KILL cr);
6078 
6079   format %{ "popcnt  $dst, $mem" %}
6080   ins_encode %{
6081     __ popcntq($dst$$Register, $mem$$Address);
6082   %}
6083   ins_pipe(ialu_reg);
6084 %}
6085 
6086 
6087 //----------MemBar Instructions-----------------------------------------------
6088 // Memory barrier flavors
6089 
6090 instruct membar_acquire()
6091 %{
6092   match(MemBarAcquire);
6093   match(LoadFence);
6094   ins_cost(0);
6095 
6096   size(0);
6097   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6098   ins_encode();
6099   ins_pipe(empty);
6100 %}
6101 
6102 instruct membar_acquire_lock()
6103 %{
6104   match(MemBarAcquireLock);
6105   ins_cost(0);
6106 
6107   size(0);
6108   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6109   ins_encode();
6110   ins_pipe(empty);
6111 %}
6112 
6113 instruct membar_release()
6114 %{
6115   match(MemBarRelease);
6116   match(StoreFence);
6117   ins_cost(0);
6118 
6119   size(0);
6120   format %{ "MEMBAR-release ! (empty encoding)" %}
6121   ins_encode();
6122   ins_pipe(empty);
6123 %}
6124 
6125 instruct membar_release_lock()
6126 %{
6127   match(MemBarReleaseLock);
6128   ins_cost(0);
6129 
6130   size(0);
6131   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6132   ins_encode();
6133   ins_pipe(empty);
6134 %}
6135 
6136 instruct membar_volatile(rFlagsReg cr) %{
6137   match(MemBarVolatile);
6138   effect(KILL cr);
6139   ins_cost(400);
6140 
6141   format %{
6142     $$template
6143     if (os::is_MP()) {
6144       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6145     } else {
6146       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6147     }
6148   %}
6149   ins_encode %{
6150     __ membar(Assembler::StoreLoad);
6151   %}
6152   ins_pipe(pipe_slow);
6153 %}
6154 
6155 instruct unnecessary_membar_volatile()
6156 %{
6157   match(MemBarVolatile);
6158   predicate(Matcher::post_store_load_barrier(n));
6159   ins_cost(0);
6160 
6161   size(0);
6162   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6163   ins_encode();
6164   ins_pipe(empty);
6165 %}
6166 
6167 instruct membar_storestore() %{
6168   match(MemBarStoreStore);
6169   ins_cost(0);
6170 
6171   size(0);
6172   format %{ "MEMBAR-storestore (empty encoding)" %}
6173   ins_encode( );
6174   ins_pipe(empty);
6175 %}
6176 
6177 //----------Move Instructions--------------------------------------------------
6178 
6179 instruct castX2P(rRegP dst, rRegL src)
6180 %{
6181   match(Set dst (CastX2P src));
6182 
6183   format %{ "movq    $dst, $src\t# long->ptr" %}
6184   ins_encode %{
6185     if ($dst$$reg != $src$$reg) {
6186       __ movptr($dst$$Register, $src$$Register);
6187     }
6188   %}
6189   ins_pipe(ialu_reg_reg); // XXX
6190 %}
6191 
6192 instruct castP2X(rRegL dst, rRegP src)
6193 %{
6194   match(Set dst (CastP2X src));
6195 
6196   format %{ "movq    $dst, $src\t# ptr -> long" %}
6197   ins_encode %{
6198     if ($dst$$reg != $src$$reg) {
6199       __ movptr($dst$$Register, $src$$Register);
6200     }
6201   %}
6202   ins_pipe(ialu_reg_reg); // XXX
6203 %}
6204 
6205 // Convert oop into int for vectors alignment masking
6206 instruct convP2I(rRegI dst, rRegP src)
6207 %{
6208   match(Set dst (ConvL2I (CastP2X src)));
6209 
6210   format %{ "movl    $dst, $src\t# ptr -> int" %}
6211   ins_encode %{
6212     __ movl($dst$$Register, $src$$Register);
6213   %}
6214   ins_pipe(ialu_reg_reg); // XXX
6215 %}
6216 
6217 // Convert compressed oop into int for vectors alignment masking
6218 // in case of 32bit oops (heap < 4Gb).
6219 instruct convN2I(rRegI dst, rRegN src)
6220 %{
6221   predicate(Universe::narrow_oop_shift() == 0);
6222   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6223 
6224   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6225   ins_encode %{
6226     __ movl($dst$$Register, $src$$Register);
6227   %}
6228   ins_pipe(ialu_reg_reg); // XXX
6229 %}
6230 
6231 // Convert oop pointer into compressed form
6232 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6233   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6234   match(Set dst (EncodeP src));
6235   effect(KILL cr);
6236   format %{ "encode_heap_oop $dst,$src" %}
6237   ins_encode %{
6238     Register s = $src$$Register;
6239     Register d = $dst$$Register;
6240     if (s != d) {
6241       __ movq(d, s);
6242     }
6243     __ encode_heap_oop(d);
6244   %}
6245   ins_pipe(ialu_reg_long);
6246 %}
6247 
6248 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6249   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6250   match(Set dst (EncodeP src));
6251   effect(KILL cr);
6252   format %{ "encode_heap_oop_not_null $dst,$src" %}
6253   ins_encode %{
6254     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6255   %}
6256   ins_pipe(ialu_reg_long);
6257 %}
6258 
6259 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6260   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6261             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6262   match(Set dst (DecodeN src));
6263   effect(KILL cr);
6264   format %{ "decode_heap_oop $dst,$src" %}
6265   ins_encode %{
6266     Register s = $src$$Register;
6267     Register d = $dst$$Register;
6268     if (s != d) {
6269       __ movq(d, s);
6270     }
6271     __ decode_heap_oop(d);
6272   %}
6273   ins_pipe(ialu_reg_long);
6274 %}
6275 
6276 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6277   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6278             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6279   match(Set dst (DecodeN src));
6280   effect(KILL cr);
6281   format %{ "decode_heap_oop_not_null $dst,$src" %}
6282   ins_encode %{
6283     Register s = $src$$Register;
6284     Register d = $dst$$Register;
6285     if (s != d) {
6286       __ decode_heap_oop_not_null(d, s);
6287     } else {
6288       __ decode_heap_oop_not_null(d);
6289     }
6290   %}
6291   ins_pipe(ialu_reg_long);
6292 %}
6293 
6294 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6295   match(Set dst (EncodePKlass src));
6296   effect(KILL cr);
6297   format %{ "encode_klass_not_null $dst,$src" %}
6298   ins_encode %{
6299     __ encode_klass_not_null($dst$$Register, $src$$Register);
6300   %}
6301   ins_pipe(ialu_reg_long);
6302 %}
6303 
6304 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6305   match(Set dst (DecodeNKlass src));
6306   effect(KILL cr);
6307   format %{ "decode_klass_not_null $dst,$src" %}
6308   ins_encode %{
6309     Register s = $src$$Register;
6310     Register d = $dst$$Register;
6311     if (s != d) {
6312       __ decode_klass_not_null(d, s);
6313     } else {
6314       __ decode_klass_not_null(d);
6315     }
6316   %}
6317   ins_pipe(ialu_reg_long);
6318 %}
6319 
6320 
6321 //----------Conditional Move---------------------------------------------------
6322 // Jump
6323 // dummy instruction for generating temp registers
6324 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6325   match(Jump (LShiftL switch_val shift));
6326   ins_cost(350);
6327   predicate(false);
6328   effect(TEMP dest);
6329 
6330   format %{ "leaq    $dest, [$constantaddress]\n\t"
6331             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6332   ins_encode %{
6333     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6334     // to do that and the compiler is using that register as one it can allocate.
6335     // So we build it all by hand.
6336     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6337     // ArrayAddress dispatch(table, index);
6338     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6339     __ lea($dest$$Register, $constantaddress);
6340     __ jmp(dispatch);
6341   %}
6342   ins_pipe(pipe_jmp);
6343 %}
6344 
6345 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6346   match(Jump (AddL (LShiftL switch_val shift) offset));
6347   ins_cost(350);
6348   effect(TEMP dest);
6349 
6350   format %{ "leaq    $dest, [$constantaddress]\n\t"
6351             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6352   ins_encode %{
6353     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6354     // to do that and the compiler is using that register as one it can allocate.
6355     // So we build it all by hand.
6356     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6357     // ArrayAddress dispatch(table, index);
6358     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6359     __ lea($dest$$Register, $constantaddress);
6360     __ jmp(dispatch);
6361   %}
6362   ins_pipe(pipe_jmp);
6363 %}
6364 
6365 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6366   match(Jump switch_val);
6367   ins_cost(350);
6368   effect(TEMP dest);
6369 
6370   format %{ "leaq    $dest, [$constantaddress]\n\t"
6371             "jmp     [$dest + $switch_val]\n\t" %}
6372   ins_encode %{
6373     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6374     // to do that and the compiler is using that register as one it can allocate.
6375     // So we build it all by hand.
6376     // Address index(noreg, switch_reg, Address::times_1);
6377     // ArrayAddress dispatch(table, index);
6378     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6379     __ lea($dest$$Register, $constantaddress);
6380     __ jmp(dispatch);
6381   %}
6382   ins_pipe(pipe_jmp);
6383 %}
6384 
6385 // Conditional move
6386 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6387 %{
6388   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6389 
6390   ins_cost(200); // XXX
6391   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6392   opcode(0x0F, 0x40);
6393   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6394   ins_pipe(pipe_cmov_reg);
6395 %}
6396 
6397 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6398   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6399 
6400   ins_cost(200); // XXX
6401   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6402   opcode(0x0F, 0x40);
6403   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6404   ins_pipe(pipe_cmov_reg);
6405 %}
6406 
6407 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6408   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6409   ins_cost(200);
6410   expand %{
6411     cmovI_regU(cop, cr, dst, src);
6412   %}
6413 %}
6414 
6415 // Conditional move
6416 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6417   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6418 
6419   ins_cost(250); // XXX
6420   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6421   opcode(0x0F, 0x40);
6422   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6423   ins_pipe(pipe_cmov_mem);
6424 %}
6425 
6426 // Conditional move
6427 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6428 %{
6429   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6430 
6431   ins_cost(250); // XXX
6432   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6433   opcode(0x0F, 0x40);
6434   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6435   ins_pipe(pipe_cmov_mem);
6436 %}
6437 
6438 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6439   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6440   ins_cost(250);
6441   expand %{
6442     cmovI_memU(cop, cr, dst, src);
6443   %}
6444 %}
6445 
6446 // Conditional move
6447 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6448 %{
6449   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6450 
6451   ins_cost(200); // XXX
6452   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6453   opcode(0x0F, 0x40);
6454   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6455   ins_pipe(pipe_cmov_reg);
6456 %}
6457 
6458 // Conditional move
6459 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6460 %{
6461   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6462 
6463   ins_cost(200); // XXX
6464   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6465   opcode(0x0F, 0x40);
6466   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6467   ins_pipe(pipe_cmov_reg);
6468 %}
6469 
6470 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6471   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6472   ins_cost(200);
6473   expand %{
6474     cmovN_regU(cop, cr, dst, src);
6475   %}
6476 %}
6477 
6478 // Conditional move
6479 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6480 %{
6481   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6482 
6483   ins_cost(200); // XXX
6484   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6485   opcode(0x0F, 0x40);
6486   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6487   ins_pipe(pipe_cmov_reg);  // XXX
6488 %}
6489 
6490 // Conditional move
6491 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6492 %{
6493   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6494 
6495   ins_cost(200); // XXX
6496   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6497   opcode(0x0F, 0x40);
6498   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6499   ins_pipe(pipe_cmov_reg); // XXX
6500 %}
6501 
6502 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6503   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6504   ins_cost(200);
6505   expand %{
6506     cmovP_regU(cop, cr, dst, src);
6507   %}
6508 %}
6509 
6510 // DISABLED: Requires the ADLC to emit a bottom_type call that
6511 // correctly meets the two pointer arguments; one is an incoming
6512 // register but the other is a memory operand.  ALSO appears to
6513 // be buggy with implicit null checks.
6514 //
6515 //// Conditional move
6516 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6517 //%{
6518 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6519 //  ins_cost(250);
6520 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6521 //  opcode(0x0F,0x40);
6522 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6523 //  ins_pipe( pipe_cmov_mem );
6524 //%}
6525 //
6526 //// Conditional move
6527 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6528 //%{
6529 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6530 //  ins_cost(250);
6531 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6532 //  opcode(0x0F,0x40);
6533 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6534 //  ins_pipe( pipe_cmov_mem );
6535 //%}
6536 
6537 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6538 %{
6539   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6540 
6541   ins_cost(200); // XXX
6542   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6543   opcode(0x0F, 0x40);
6544   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6545   ins_pipe(pipe_cmov_reg);  // XXX
6546 %}
6547 
6548 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6549 %{
6550   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6551 
6552   ins_cost(200); // XXX
6553   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6554   opcode(0x0F, 0x40);
6555   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6556   ins_pipe(pipe_cmov_mem);  // XXX
6557 %}
6558 
6559 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6560 %{
6561   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6562 
6563   ins_cost(200); // XXX
6564   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6565   opcode(0x0F, 0x40);
6566   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6567   ins_pipe(pipe_cmov_reg); // XXX
6568 %}
6569 
6570 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6571   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6572   ins_cost(200);
6573   expand %{
6574     cmovL_regU(cop, cr, dst, src);
6575   %}
6576 %}
6577 
6578 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6579 %{
6580   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6581 
6582   ins_cost(200); // XXX
6583   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6584   opcode(0x0F, 0x40);
6585   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6586   ins_pipe(pipe_cmov_mem); // XXX
6587 %}
6588 
6589 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6590   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6591   ins_cost(200);
6592   expand %{
6593     cmovL_memU(cop, cr, dst, src);
6594   %}
6595 %}
6596 
6597 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6598 %{
6599   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6600 
6601   ins_cost(200); // XXX
6602   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6603             "movss     $dst, $src\n"
6604     "skip:" %}
6605   ins_encode %{
6606     Label Lskip;
6607     // Invert sense of branch from sense of CMOV
6608     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6609     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6610     __ bind(Lskip);
6611   %}
6612   ins_pipe(pipe_slow);
6613 %}
6614 
6615 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6616 // %{
6617 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6618 
6619 //   ins_cost(200); // XXX
6620 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6621 //             "movss     $dst, $src\n"
6622 //     "skip:" %}
6623 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6624 //   ins_pipe(pipe_slow);
6625 // %}
6626 
6627 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6628 %{
6629   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6630 
6631   ins_cost(200); // XXX
6632   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6633             "movss     $dst, $src\n"
6634     "skip:" %}
6635   ins_encode %{
6636     Label Lskip;
6637     // Invert sense of branch from sense of CMOV
6638     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6639     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6640     __ bind(Lskip);
6641   %}
6642   ins_pipe(pipe_slow);
6643 %}
6644 
6645 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6646   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6647   ins_cost(200);
6648   expand %{
6649     cmovF_regU(cop, cr, dst, src);
6650   %}
6651 %}
6652 
6653 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6654 %{
6655   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6656 
6657   ins_cost(200); // XXX
6658   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6659             "movsd     $dst, $src\n"
6660     "skip:" %}
6661   ins_encode %{
6662     Label Lskip;
6663     // Invert sense of branch from sense of CMOV
6664     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6665     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6666     __ bind(Lskip);
6667   %}
6668   ins_pipe(pipe_slow);
6669 %}
6670 
6671 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6672 %{
6673   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6674 
6675   ins_cost(200); // XXX
6676   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6677             "movsd     $dst, $src\n"
6678     "skip:" %}
6679   ins_encode %{
6680     Label Lskip;
6681     // Invert sense of branch from sense of CMOV
6682     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6683     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6684     __ bind(Lskip);
6685   %}
6686   ins_pipe(pipe_slow);
6687 %}
6688 
6689 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6690   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6691   ins_cost(200);
6692   expand %{
6693     cmovD_regU(cop, cr, dst, src);
6694   %}
6695 %}
6696 
6697 //----------Arithmetic Instructions--------------------------------------------
6698 //----------Addition Instructions----------------------------------------------
6699 
6700 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6701 %{
6702   match(Set dst (AddI dst src));
6703   effect(KILL cr);
6704 
6705   format %{ "addl    $dst, $src\t# int" %}
6706   opcode(0x03);
6707   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6708   ins_pipe(ialu_reg_reg);
6709 %}
6710 
6711 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6712 %{
6713   match(Set dst (AddI dst src));
6714   effect(KILL cr);
6715 
6716   format %{ "addl    $dst, $src\t# int" %}
6717   opcode(0x81, 0x00); /* /0 id */
6718   ins_encode(OpcSErm(dst, src), Con8or32(src));
6719   ins_pipe( ialu_reg );
6720 %}
6721 
6722 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6723 %{
6724   match(Set dst (AddI dst (LoadI src)));
6725   effect(KILL cr);
6726 
6727   ins_cost(125); // XXX
6728   format %{ "addl    $dst, $src\t# int" %}
6729   opcode(0x03);
6730   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6731   ins_pipe(ialu_reg_mem);
6732 %}
6733 
6734 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6735 %{
6736   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6737   effect(KILL cr);
6738 
6739   ins_cost(150); // XXX
6740   format %{ "addl    $dst, $src\t# int" %}
6741   opcode(0x01); /* Opcode 01 /r */
6742   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6743   ins_pipe(ialu_mem_reg);
6744 %}
6745 
6746 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6747 %{
6748   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6749   effect(KILL cr);
6750 
6751   ins_cost(125); // XXX
6752   format %{ "addl    $dst, $src\t# int" %}
6753   opcode(0x81); /* Opcode 81 /0 id */
6754   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6755   ins_pipe(ialu_mem_imm);
6756 %}
6757 
6758 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6759 %{
6760   predicate(UseIncDec);
6761   match(Set dst (AddI dst src));
6762   effect(KILL cr);
6763 
6764   format %{ "incl    $dst\t# int" %}
6765   opcode(0xFF, 0x00); // FF /0
6766   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6767   ins_pipe(ialu_reg);
6768 %}
6769 
6770 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6771 %{
6772   predicate(UseIncDec);
6773   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6774   effect(KILL cr);
6775 
6776   ins_cost(125); // XXX
6777   format %{ "incl    $dst\t# int" %}
6778   opcode(0xFF); /* Opcode FF /0 */
6779   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6780   ins_pipe(ialu_mem_imm);
6781 %}
6782 
6783 // XXX why does that use AddI
6784 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6785 %{
6786   predicate(UseIncDec);
6787   match(Set dst (AddI dst src));
6788   effect(KILL cr);
6789 
6790   format %{ "decl    $dst\t# int" %}
6791   opcode(0xFF, 0x01); // FF /1
6792   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6793   ins_pipe(ialu_reg);
6794 %}
6795 
6796 // XXX why does that use AddI
6797 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6798 %{
6799   predicate(UseIncDec);
6800   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6801   effect(KILL cr);
6802 
6803   ins_cost(125); // XXX
6804   format %{ "decl    $dst\t# int" %}
6805   opcode(0xFF); /* Opcode FF /1 */
6806   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
6807   ins_pipe(ialu_mem_imm);
6808 %}
6809 
6810 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
6811 %{
6812   match(Set dst (AddI src0 src1));
6813 
6814   ins_cost(110);
6815   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
6816   opcode(0x8D); /* 0x8D /r */
6817   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6818   ins_pipe(ialu_reg_reg);
6819 %}
6820 
6821 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6822 %{
6823   match(Set dst (AddL dst src));
6824   effect(KILL cr);
6825 
6826   format %{ "addq    $dst, $src\t# long" %}
6827   opcode(0x03);
6828   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6829   ins_pipe(ialu_reg_reg);
6830 %}
6831 
6832 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6833 %{
6834   match(Set dst (AddL dst src));
6835   effect(KILL cr);
6836 
6837   format %{ "addq    $dst, $src\t# long" %}
6838   opcode(0x81, 0x00); /* /0 id */
6839   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6840   ins_pipe( ialu_reg );
6841 %}
6842 
6843 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6844 %{
6845   match(Set dst (AddL dst (LoadL src)));
6846   effect(KILL cr);
6847 
6848   ins_cost(125); // XXX
6849   format %{ "addq    $dst, $src\t# long" %}
6850   opcode(0x03);
6851   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6852   ins_pipe(ialu_reg_mem);
6853 %}
6854 
6855 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6856 %{
6857   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6858   effect(KILL cr);
6859 
6860   ins_cost(150); // XXX
6861   format %{ "addq    $dst, $src\t# long" %}
6862   opcode(0x01); /* Opcode 01 /r */
6863   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6864   ins_pipe(ialu_mem_reg);
6865 %}
6866 
6867 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
6868 %{
6869   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6870   effect(KILL cr);
6871 
6872   ins_cost(125); // XXX
6873   format %{ "addq    $dst, $src\t# long" %}
6874   opcode(0x81); /* Opcode 81 /0 id */
6875   ins_encode(REX_mem_wide(dst),
6876              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6877   ins_pipe(ialu_mem_imm);
6878 %}
6879 
6880 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
6881 %{
6882   predicate(UseIncDec);
6883   match(Set dst (AddL dst src));
6884   effect(KILL cr);
6885 
6886   format %{ "incq    $dst\t# long" %}
6887   opcode(0xFF, 0x00); // FF /0
6888   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6889   ins_pipe(ialu_reg);
6890 %}
6891 
6892 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
6893 %{
6894   predicate(UseIncDec);
6895   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6896   effect(KILL cr);
6897 
6898   ins_cost(125); // XXX
6899   format %{ "incq    $dst\t# long" %}
6900   opcode(0xFF); /* Opcode FF /0 */
6901   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
6902   ins_pipe(ialu_mem_imm);
6903 %}
6904 
6905 // XXX why does that use AddL
6906 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
6907 %{
6908   predicate(UseIncDec);
6909   match(Set dst (AddL dst src));
6910   effect(KILL cr);
6911 
6912   format %{ "decq    $dst\t# long" %}
6913   opcode(0xFF, 0x01); // FF /1
6914   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6915   ins_pipe(ialu_reg);
6916 %}
6917 
6918 // XXX why does that use AddL
6919 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
6920 %{
6921   predicate(UseIncDec);
6922   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6923   effect(KILL cr);
6924 
6925   ins_cost(125); // XXX
6926   format %{ "decq    $dst\t# long" %}
6927   opcode(0xFF); /* Opcode FF /1 */
6928   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
6929   ins_pipe(ialu_mem_imm);
6930 %}
6931 
6932 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
6933 %{
6934   match(Set dst (AddL src0 src1));
6935 
6936   ins_cost(110);
6937   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
6938   opcode(0x8D); /* 0x8D /r */
6939   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6940   ins_pipe(ialu_reg_reg);
6941 %}
6942 
6943 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
6944 %{
6945   match(Set dst (AddP dst src));
6946   effect(KILL cr);
6947 
6948   format %{ "addq    $dst, $src\t# ptr" %}
6949   opcode(0x03);
6950   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6951   ins_pipe(ialu_reg_reg);
6952 %}
6953 
6954 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
6955 %{
6956   match(Set dst (AddP dst src));
6957   effect(KILL cr);
6958 
6959   format %{ "addq    $dst, $src\t# ptr" %}
6960   opcode(0x81, 0x00); /* /0 id */
6961   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6962   ins_pipe( ialu_reg );
6963 %}
6964 
6965 // XXX addP mem ops ????
6966 
6967 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
6968 %{
6969   match(Set dst (AddP src0 src1));
6970 
6971   ins_cost(110);
6972   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
6973   opcode(0x8D); /* 0x8D /r */
6974   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
6975   ins_pipe(ialu_reg_reg);
6976 %}
6977 
6978 instruct checkCastPP(rRegP dst)
6979 %{
6980   match(Set dst (CheckCastPP dst));
6981 
6982   size(0);
6983   format %{ "# checkcastPP of $dst" %}
6984   ins_encode(/* empty encoding */);
6985   ins_pipe(empty);
6986 %}
6987 
6988 instruct castPP(rRegP dst)
6989 %{
6990   match(Set dst (CastPP dst));
6991 
6992   size(0);
6993   format %{ "# castPP of $dst" %}
6994   ins_encode(/* empty encoding */);
6995   ins_pipe(empty);
6996 %}
6997 
6998 instruct castII(rRegI dst)
6999 %{
7000   match(Set dst (CastII dst));
7001 
7002   size(0);
7003   format %{ "# castII of $dst" %}
7004   ins_encode(/* empty encoding */);
7005   ins_cost(0);
7006   ins_pipe(empty);
7007 %}
7008 
7009 // LoadP-locked same as a regular LoadP when used with compare-swap
7010 instruct loadPLocked(rRegP dst, memory mem)
7011 %{
7012   match(Set dst (LoadPLocked mem));
7013 
7014   ins_cost(125); // XXX
7015   format %{ "movq    $dst, $mem\t# ptr locked" %}
7016   opcode(0x8B);
7017   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7018   ins_pipe(ialu_reg_mem); // XXX
7019 %}
7020 
7021 // Conditional-store of the updated heap-top.
7022 // Used during allocation of the shared heap.
7023 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7024 
7025 instruct storePConditional(memory heap_top_ptr,
7026                            rax_RegP oldval, rRegP newval,
7027                            rFlagsReg cr)
7028 %{
7029   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7030 
7031   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7032             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7033   opcode(0x0F, 0xB1);
7034   ins_encode(lock_prefix,
7035              REX_reg_mem_wide(newval, heap_top_ptr),
7036              OpcP, OpcS,
7037              reg_mem(newval, heap_top_ptr));
7038   ins_pipe(pipe_cmpxchg);
7039 %}
7040 
7041 // Conditional-store of an int value.
7042 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7043 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7044 %{
7045   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7046   effect(KILL oldval);
7047 
7048   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7049   opcode(0x0F, 0xB1);
7050   ins_encode(lock_prefix,
7051              REX_reg_mem(newval, mem),
7052              OpcP, OpcS,
7053              reg_mem(newval, mem));
7054   ins_pipe(pipe_cmpxchg);
7055 %}
7056 
7057 // Conditional-store of a long value.
7058 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7059 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7060 %{
7061   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7062   effect(KILL oldval);
7063 
7064   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7065   opcode(0x0F, 0xB1);
7066   ins_encode(lock_prefix,
7067              REX_reg_mem_wide(newval, mem),
7068              OpcP, OpcS,
7069              reg_mem(newval, mem));
7070   ins_pipe(pipe_cmpxchg);
7071 %}
7072 
7073 
7074 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7075 instruct compareAndSwapP(rRegI res,
7076                          memory mem_ptr,
7077                          rax_RegP oldval, rRegP newval,
7078                          rFlagsReg cr)
7079 %{
7080   predicate(VM_Version::supports_cx8());
7081   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7082   effect(KILL cr, KILL oldval);
7083 
7084   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7085             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7086             "sete    $res\n\t"
7087             "movzbl  $res, $res" %}
7088   opcode(0x0F, 0xB1);
7089   ins_encode(lock_prefix,
7090              REX_reg_mem_wide(newval, mem_ptr),
7091              OpcP, OpcS,
7092              reg_mem(newval, mem_ptr),
7093              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7094              REX_reg_breg(res, res), // movzbl
7095              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7096   ins_pipe( pipe_cmpxchg );
7097 %}
7098 
7099 instruct compareAndSwapL(rRegI res,
7100                          memory mem_ptr,
7101                          rax_RegL oldval, rRegL newval,
7102                          rFlagsReg cr)
7103 %{
7104   predicate(VM_Version::supports_cx8());
7105   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7106   effect(KILL cr, KILL oldval);
7107 
7108   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7109             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7110             "sete    $res\n\t"
7111             "movzbl  $res, $res" %}
7112   opcode(0x0F, 0xB1);
7113   ins_encode(lock_prefix,
7114              REX_reg_mem_wide(newval, mem_ptr),
7115              OpcP, OpcS,
7116              reg_mem(newval, mem_ptr),
7117              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7118              REX_reg_breg(res, res), // movzbl
7119              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7120   ins_pipe( pipe_cmpxchg );
7121 %}
7122 
7123 instruct compareAndSwapI(rRegI res,
7124                          memory mem_ptr,
7125                          rax_RegI oldval, rRegI newval,
7126                          rFlagsReg cr)
7127 %{
7128   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7129   effect(KILL cr, KILL oldval);
7130 
7131   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7132             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7133             "sete    $res\n\t"
7134             "movzbl  $res, $res" %}
7135   opcode(0x0F, 0xB1);
7136   ins_encode(lock_prefix,
7137              REX_reg_mem(newval, mem_ptr),
7138              OpcP, OpcS,
7139              reg_mem(newval, mem_ptr),
7140              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7141              REX_reg_breg(res, res), // movzbl
7142              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7143   ins_pipe( pipe_cmpxchg );
7144 %}
7145 
7146 
7147 instruct compareAndSwapN(rRegI res,
7148                           memory mem_ptr,
7149                           rax_RegN oldval, rRegN newval,
7150                           rFlagsReg cr) %{
7151   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7152   effect(KILL cr, KILL oldval);
7153 
7154   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7155             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7156             "sete    $res\n\t"
7157             "movzbl  $res, $res" %}
7158   opcode(0x0F, 0xB1);
7159   ins_encode(lock_prefix,
7160              REX_reg_mem(newval, mem_ptr),
7161              OpcP, OpcS,
7162              reg_mem(newval, mem_ptr),
7163              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7164              REX_reg_breg(res, res), // movzbl
7165              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7166   ins_pipe( pipe_cmpxchg );
7167 %}
7168 
7169 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7170   predicate(n->as_LoadStore()->result_not_used());
7171   match(Set dummy (GetAndAddI mem add));
7172   effect(KILL cr);
7173   format %{ "ADDL  [$mem],$add" %}
7174   ins_encode %{
7175     if (os::is_MP()) { __ lock(); }
7176     __ addl($mem$$Address, $add$$constant);
7177   %}
7178   ins_pipe( pipe_cmpxchg );
7179 %}
7180 
7181 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7182   match(Set newval (GetAndAddI mem newval));
7183   effect(KILL cr);
7184   format %{ "XADDL  [$mem],$newval" %}
7185   ins_encode %{
7186     if (os::is_MP()) { __ lock(); }
7187     __ xaddl($mem$$Address, $newval$$Register);
7188   %}
7189   ins_pipe( pipe_cmpxchg );
7190 %}
7191 
7192 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7193   predicate(n->as_LoadStore()->result_not_used());
7194   match(Set dummy (GetAndAddL mem add));
7195   effect(KILL cr);
7196   format %{ "ADDQ  [$mem],$add" %}
7197   ins_encode %{
7198     if (os::is_MP()) { __ lock(); }
7199     __ addq($mem$$Address, $add$$constant);
7200   %}
7201   ins_pipe( pipe_cmpxchg );
7202 %}
7203 
7204 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7205   match(Set newval (GetAndAddL mem newval));
7206   effect(KILL cr);
7207   format %{ "XADDQ  [$mem],$newval" %}
7208   ins_encode %{
7209     if (os::is_MP()) { __ lock(); }
7210     __ xaddq($mem$$Address, $newval$$Register);
7211   %}
7212   ins_pipe( pipe_cmpxchg );
7213 %}
7214 
7215 instruct xchgI( memory mem, rRegI newval) %{
7216   match(Set newval (GetAndSetI mem newval));
7217   format %{ "XCHGL  $newval,[$mem]" %}
7218   ins_encode %{
7219     __ xchgl($newval$$Register, $mem$$Address);
7220   %}
7221   ins_pipe( pipe_cmpxchg );
7222 %}
7223 
7224 instruct xchgL( memory mem, rRegL newval) %{
7225   match(Set newval (GetAndSetL mem newval));
7226   format %{ "XCHGL  $newval,[$mem]" %}
7227   ins_encode %{
7228     __ xchgq($newval$$Register, $mem$$Address);
7229   %}
7230   ins_pipe( pipe_cmpxchg );
7231 %}
7232 
7233 instruct xchgP( memory mem, rRegP newval) %{
7234   match(Set newval (GetAndSetP mem newval));
7235   format %{ "XCHGQ  $newval,[$mem]" %}
7236   ins_encode %{
7237     __ xchgq($newval$$Register, $mem$$Address);
7238   %}
7239   ins_pipe( pipe_cmpxchg );
7240 %}
7241 
7242 instruct xchgN( memory mem, rRegN newval) %{
7243   match(Set newval (GetAndSetN mem newval));
7244   format %{ "XCHGL  $newval,$mem]" %}
7245   ins_encode %{
7246     __ xchgl($newval$$Register, $mem$$Address);
7247   %}
7248   ins_pipe( pipe_cmpxchg );
7249 %}
7250 
7251 //----------Subtraction Instructions-------------------------------------------
7252 
7253 // Integer Subtraction Instructions
7254 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7255 %{
7256   match(Set dst (SubI dst src));
7257   effect(KILL cr);
7258 
7259   format %{ "subl    $dst, $src\t# int" %}
7260   opcode(0x2B);
7261   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7262   ins_pipe(ialu_reg_reg);
7263 %}
7264 
7265 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7266 %{
7267   match(Set dst (SubI dst src));
7268   effect(KILL cr);
7269 
7270   format %{ "subl    $dst, $src\t# int" %}
7271   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7272   ins_encode(OpcSErm(dst, src), Con8or32(src));
7273   ins_pipe(ialu_reg);
7274 %}
7275 
7276 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7277 %{
7278   match(Set dst (SubI dst (LoadI src)));
7279   effect(KILL cr);
7280 
7281   ins_cost(125);
7282   format %{ "subl    $dst, $src\t# int" %}
7283   opcode(0x2B);
7284   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7285   ins_pipe(ialu_reg_mem);
7286 %}
7287 
7288 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7289 %{
7290   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7291   effect(KILL cr);
7292 
7293   ins_cost(150);
7294   format %{ "subl    $dst, $src\t# int" %}
7295   opcode(0x29); /* Opcode 29 /r */
7296   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7297   ins_pipe(ialu_mem_reg);
7298 %}
7299 
7300 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7301 %{
7302   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7303   effect(KILL cr);
7304 
7305   ins_cost(125); // XXX
7306   format %{ "subl    $dst, $src\t# int" %}
7307   opcode(0x81); /* Opcode 81 /5 id */
7308   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7309   ins_pipe(ialu_mem_imm);
7310 %}
7311 
7312 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7313 %{
7314   match(Set dst (SubL dst src));
7315   effect(KILL cr);
7316 
7317   format %{ "subq    $dst, $src\t# long" %}
7318   opcode(0x2B);
7319   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7320   ins_pipe(ialu_reg_reg);
7321 %}
7322 
7323 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7324 %{
7325   match(Set dst (SubL dst src));
7326   effect(KILL cr);
7327 
7328   format %{ "subq    $dst, $src\t# long" %}
7329   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7330   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7331   ins_pipe(ialu_reg);
7332 %}
7333 
7334 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7335 %{
7336   match(Set dst (SubL dst (LoadL src)));
7337   effect(KILL cr);
7338 
7339   ins_cost(125);
7340   format %{ "subq    $dst, $src\t# long" %}
7341   opcode(0x2B);
7342   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7343   ins_pipe(ialu_reg_mem);
7344 %}
7345 
7346 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7347 %{
7348   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7349   effect(KILL cr);
7350 
7351   ins_cost(150);
7352   format %{ "subq    $dst, $src\t# long" %}
7353   opcode(0x29); /* Opcode 29 /r */
7354   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7355   ins_pipe(ialu_mem_reg);
7356 %}
7357 
7358 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7359 %{
7360   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7361   effect(KILL cr);
7362 
7363   ins_cost(125); // XXX
7364   format %{ "subq    $dst, $src\t# long" %}
7365   opcode(0x81); /* Opcode 81 /5 id */
7366   ins_encode(REX_mem_wide(dst),
7367              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7368   ins_pipe(ialu_mem_imm);
7369 %}
7370 
7371 // Subtract from a pointer
7372 // XXX hmpf???
7373 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7374 %{
7375   match(Set dst (AddP dst (SubI zero src)));
7376   effect(KILL cr);
7377 
7378   format %{ "subq    $dst, $src\t# ptr - int" %}
7379   opcode(0x2B);
7380   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7381   ins_pipe(ialu_reg_reg);
7382 %}
7383 
7384 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7385 %{
7386   match(Set dst (SubI zero dst));
7387   effect(KILL cr);
7388 
7389   format %{ "negl    $dst\t# int" %}
7390   opcode(0xF7, 0x03);  // Opcode F7 /3
7391   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7392   ins_pipe(ialu_reg);
7393 %}
7394 
7395 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7396 %{
7397   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7398   effect(KILL cr);
7399 
7400   format %{ "negl    $dst\t# int" %}
7401   opcode(0xF7, 0x03);  // Opcode F7 /3
7402   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7403   ins_pipe(ialu_reg);
7404 %}
7405 
7406 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7407 %{
7408   match(Set dst (SubL zero dst));
7409   effect(KILL cr);
7410 
7411   format %{ "negq    $dst\t# long" %}
7412   opcode(0xF7, 0x03);  // Opcode F7 /3
7413   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7414   ins_pipe(ialu_reg);
7415 %}
7416 
7417 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7418 %{
7419   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7420   effect(KILL cr);
7421 
7422   format %{ "negq    $dst\t# long" %}
7423   opcode(0xF7, 0x03);  // Opcode F7 /3
7424   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7425   ins_pipe(ialu_reg);
7426 %}
7427 
7428 //----------Multiplication/Division Instructions-------------------------------
7429 // Integer Multiplication Instructions
7430 // Multiply Register
7431 
7432 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7433 %{
7434   match(Set dst (MulI dst src));
7435   effect(KILL cr);
7436 
7437   ins_cost(300);
7438   format %{ "imull   $dst, $src\t# int" %}
7439   opcode(0x0F, 0xAF);
7440   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7441   ins_pipe(ialu_reg_reg_alu0);
7442 %}
7443 
7444 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7445 %{
7446   match(Set dst (MulI src imm));
7447   effect(KILL cr);
7448 
7449   ins_cost(300);
7450   format %{ "imull   $dst, $src, $imm\t# int" %}
7451   opcode(0x69); /* 69 /r id */
7452   ins_encode(REX_reg_reg(dst, src),
7453              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7454   ins_pipe(ialu_reg_reg_alu0);
7455 %}
7456 
7457 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7458 %{
7459   match(Set dst (MulI dst (LoadI src)));
7460   effect(KILL cr);
7461 
7462   ins_cost(350);
7463   format %{ "imull   $dst, $src\t# int" %}
7464   opcode(0x0F, 0xAF);
7465   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7466   ins_pipe(ialu_reg_mem_alu0);
7467 %}
7468 
7469 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7470 %{
7471   match(Set dst (MulI (LoadI src) imm));
7472   effect(KILL cr);
7473 
7474   ins_cost(300);
7475   format %{ "imull   $dst, $src, $imm\t# int" %}
7476   opcode(0x69); /* 69 /r id */
7477   ins_encode(REX_reg_mem(dst, src),
7478              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7479   ins_pipe(ialu_reg_mem_alu0);
7480 %}
7481 
7482 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7483 %{
7484   match(Set dst (MulL dst src));
7485   effect(KILL cr);
7486 
7487   ins_cost(300);
7488   format %{ "imulq   $dst, $src\t# long" %}
7489   opcode(0x0F, 0xAF);
7490   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7491   ins_pipe(ialu_reg_reg_alu0);
7492 %}
7493 
7494 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7495 %{
7496   match(Set dst (MulL src imm));
7497   effect(KILL cr);
7498 
7499   ins_cost(300);
7500   format %{ "imulq   $dst, $src, $imm\t# long" %}
7501   opcode(0x69); /* 69 /r id */
7502   ins_encode(REX_reg_reg_wide(dst, src),
7503              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7504   ins_pipe(ialu_reg_reg_alu0);
7505 %}
7506 
7507 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7508 %{
7509   match(Set dst (MulL dst (LoadL src)));
7510   effect(KILL cr);
7511 
7512   ins_cost(350);
7513   format %{ "imulq   $dst, $src\t# long" %}
7514   opcode(0x0F, 0xAF);
7515   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7516   ins_pipe(ialu_reg_mem_alu0);
7517 %}
7518 
7519 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7520 %{
7521   match(Set dst (MulL (LoadL src) imm));
7522   effect(KILL cr);
7523 
7524   ins_cost(300);
7525   format %{ "imulq   $dst, $src, $imm\t# long" %}
7526   opcode(0x69); /* 69 /r id */
7527   ins_encode(REX_reg_mem_wide(dst, src),
7528              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7529   ins_pipe(ialu_reg_mem_alu0);
7530 %}
7531 
7532 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7533 %{
7534   match(Set dst (MulHiL src rax));
7535   effect(USE_KILL rax, KILL cr);
7536 
7537   ins_cost(300);
7538   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7539   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7540   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7541   ins_pipe(ialu_reg_reg_alu0);
7542 %}
7543 
7544 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7545                    rFlagsReg cr)
7546 %{
7547   match(Set rax (DivI rax div));
7548   effect(KILL rdx, KILL cr);
7549 
7550   ins_cost(30*100+10*100); // XXX
7551   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7552             "jne,s   normal\n\t"
7553             "xorl    rdx, rdx\n\t"
7554             "cmpl    $div, -1\n\t"
7555             "je,s    done\n"
7556     "normal: cdql\n\t"
7557             "idivl   $div\n"
7558     "done:"        %}
7559   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7560   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7561   ins_pipe(ialu_reg_reg_alu0);
7562 %}
7563 
7564 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7565                    rFlagsReg cr)
7566 %{
7567   match(Set rax (DivL rax div));
7568   effect(KILL rdx, KILL cr);
7569 
7570   ins_cost(30*100+10*100); // XXX
7571   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7572             "cmpq    rax, rdx\n\t"
7573             "jne,s   normal\n\t"
7574             "xorl    rdx, rdx\n\t"
7575             "cmpq    $div, -1\n\t"
7576             "je,s    done\n"
7577     "normal: cdqq\n\t"
7578             "idivq   $div\n"
7579     "done:"        %}
7580   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7581   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7582   ins_pipe(ialu_reg_reg_alu0);
7583 %}
7584 
7585 // Integer DIVMOD with Register, both quotient and mod results
7586 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7587                              rFlagsReg cr)
7588 %{
7589   match(DivModI rax div);
7590   effect(KILL cr);
7591 
7592   ins_cost(30*100+10*100); // XXX
7593   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7594             "jne,s   normal\n\t"
7595             "xorl    rdx, rdx\n\t"
7596             "cmpl    $div, -1\n\t"
7597             "je,s    done\n"
7598     "normal: cdql\n\t"
7599             "idivl   $div\n"
7600     "done:"        %}
7601   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7602   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7603   ins_pipe(pipe_slow);
7604 %}
7605 
7606 // Long DIVMOD with Register, both quotient and mod results
7607 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7608                              rFlagsReg cr)
7609 %{
7610   match(DivModL rax div);
7611   effect(KILL cr);
7612 
7613   ins_cost(30*100+10*100); // XXX
7614   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7615             "cmpq    rax, rdx\n\t"
7616             "jne,s   normal\n\t"
7617             "xorl    rdx, rdx\n\t"
7618             "cmpq    $div, -1\n\t"
7619             "je,s    done\n"
7620     "normal: cdqq\n\t"
7621             "idivq   $div\n"
7622     "done:"        %}
7623   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7624   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7625   ins_pipe(pipe_slow);
7626 %}
7627 
7628 //----------- DivL-By-Constant-Expansions--------------------------------------
7629 // DivI cases are handled by the compiler
7630 
7631 // Magic constant, reciprocal of 10
7632 instruct loadConL_0x6666666666666667(rRegL dst)
7633 %{
7634   effect(DEF dst);
7635 
7636   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7637   ins_encode(load_immL(dst, 0x6666666666666667));
7638   ins_pipe(ialu_reg);
7639 %}
7640 
7641 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7642 %{
7643   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7644 
7645   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7646   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7647   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7648   ins_pipe(ialu_reg_reg_alu0);
7649 %}
7650 
7651 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7652 %{
7653   effect(USE_DEF dst, KILL cr);
7654 
7655   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7656   opcode(0xC1, 0x7); /* C1 /7 ib */
7657   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7658   ins_pipe(ialu_reg);
7659 %}
7660 
7661 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7662 %{
7663   effect(USE_DEF dst, KILL cr);
7664 
7665   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7666   opcode(0xC1, 0x7); /* C1 /7 ib */
7667   ins_encode(reg_opc_imm_wide(dst, 0x2));
7668   ins_pipe(ialu_reg);
7669 %}
7670 
7671 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7672 %{
7673   match(Set dst (DivL src div));
7674 
7675   ins_cost((5+8)*100);
7676   expand %{
7677     rax_RegL rax;                     // Killed temp
7678     rFlagsReg cr;                     // Killed
7679     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7680     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7681     sarL_rReg_63(src, cr);            // sarq  src, 63
7682     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7683     subL_rReg(dst, src, cr);          // subl  rdx, src
7684   %}
7685 %}
7686 
7687 //-----------------------------------------------------------------------------
7688 
7689 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7690                    rFlagsReg cr)
7691 %{
7692   match(Set rdx (ModI rax div));
7693   effect(KILL rax, KILL cr);
7694 
7695   ins_cost(300); // XXX
7696   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7697             "jne,s   normal\n\t"
7698             "xorl    rdx, rdx\n\t"
7699             "cmpl    $div, -1\n\t"
7700             "je,s    done\n"
7701     "normal: cdql\n\t"
7702             "idivl   $div\n"
7703     "done:"        %}
7704   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7705   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7706   ins_pipe(ialu_reg_reg_alu0);
7707 %}
7708 
7709 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7710                    rFlagsReg cr)
7711 %{
7712   match(Set rdx (ModL rax div));
7713   effect(KILL rax, KILL cr);
7714 
7715   ins_cost(300); // XXX
7716   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7717             "cmpq    rax, rdx\n\t"
7718             "jne,s   normal\n\t"
7719             "xorl    rdx, rdx\n\t"
7720             "cmpq    $div, -1\n\t"
7721             "je,s    done\n"
7722     "normal: cdqq\n\t"
7723             "idivq   $div\n"
7724     "done:"        %}
7725   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7726   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7727   ins_pipe(ialu_reg_reg_alu0);
7728 %}
7729 
7730 // Integer Shift Instructions
7731 // Shift Left by one
7732 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7733 %{
7734   match(Set dst (LShiftI dst shift));
7735   effect(KILL cr);
7736 
7737   format %{ "sall    $dst, $shift" %}
7738   opcode(0xD1, 0x4); /* D1 /4 */
7739   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7740   ins_pipe(ialu_reg);
7741 %}
7742 
7743 // Shift Left by one
7744 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7745 %{
7746   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7747   effect(KILL cr);
7748 
7749   format %{ "sall    $dst, $shift\t" %}
7750   opcode(0xD1, 0x4); /* D1 /4 */
7751   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7752   ins_pipe(ialu_mem_imm);
7753 %}
7754 
7755 // Shift Left by 8-bit immediate
7756 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7757 %{
7758   match(Set dst (LShiftI dst shift));
7759   effect(KILL cr);
7760 
7761   format %{ "sall    $dst, $shift" %}
7762   opcode(0xC1, 0x4); /* C1 /4 ib */
7763   ins_encode(reg_opc_imm(dst, shift));
7764   ins_pipe(ialu_reg);
7765 %}
7766 
7767 // Shift Left by 8-bit immediate
7768 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7769 %{
7770   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7771   effect(KILL cr);
7772 
7773   format %{ "sall    $dst, $shift" %}
7774   opcode(0xC1, 0x4); /* C1 /4 ib */
7775   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7776   ins_pipe(ialu_mem_imm);
7777 %}
7778 
7779 // Shift Left by variable
7780 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7781 %{
7782   match(Set dst (LShiftI dst shift));
7783   effect(KILL cr);
7784 
7785   format %{ "sall    $dst, $shift" %}
7786   opcode(0xD3, 0x4); /* D3 /4 */
7787   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7788   ins_pipe(ialu_reg_reg);
7789 %}
7790 
7791 // Shift Left by variable
7792 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7793 %{
7794   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7795   effect(KILL cr);
7796 
7797   format %{ "sall    $dst, $shift" %}
7798   opcode(0xD3, 0x4); /* D3 /4 */
7799   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7800   ins_pipe(ialu_mem_reg);
7801 %}
7802 
7803 // Arithmetic shift right by one
7804 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7805 %{
7806   match(Set dst (RShiftI dst shift));
7807   effect(KILL cr);
7808 
7809   format %{ "sarl    $dst, $shift" %}
7810   opcode(0xD1, 0x7); /* D1 /7 */
7811   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7812   ins_pipe(ialu_reg);
7813 %}
7814 
7815 // Arithmetic shift right by one
7816 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7817 %{
7818   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7819   effect(KILL cr);
7820 
7821   format %{ "sarl    $dst, $shift" %}
7822   opcode(0xD1, 0x7); /* D1 /7 */
7823   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7824   ins_pipe(ialu_mem_imm);
7825 %}
7826 
7827 // Arithmetic Shift Right by 8-bit immediate
7828 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7829 %{
7830   match(Set dst (RShiftI dst shift));
7831   effect(KILL cr);
7832 
7833   format %{ "sarl    $dst, $shift" %}
7834   opcode(0xC1, 0x7); /* C1 /7 ib */
7835   ins_encode(reg_opc_imm(dst, shift));
7836   ins_pipe(ialu_mem_imm);
7837 %}
7838 
7839 // Arithmetic Shift Right by 8-bit immediate
7840 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7841 %{
7842   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7843   effect(KILL cr);
7844 
7845   format %{ "sarl    $dst, $shift" %}
7846   opcode(0xC1, 0x7); /* C1 /7 ib */
7847   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7848   ins_pipe(ialu_mem_imm);
7849 %}
7850 
7851 // Arithmetic Shift Right by variable
7852 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7853 %{
7854   match(Set dst (RShiftI dst shift));
7855   effect(KILL cr);
7856 
7857   format %{ "sarl    $dst, $shift" %}
7858   opcode(0xD3, 0x7); /* D3 /7 */
7859   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7860   ins_pipe(ialu_reg_reg);
7861 %}
7862 
7863 // Arithmetic Shift Right by variable
7864 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7865 %{
7866   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7867   effect(KILL cr);
7868 
7869   format %{ "sarl    $dst, $shift" %}
7870   opcode(0xD3, 0x7); /* D3 /7 */
7871   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7872   ins_pipe(ialu_mem_reg);
7873 %}
7874 
7875 // Logical shift right by one
7876 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7877 %{
7878   match(Set dst (URShiftI dst shift));
7879   effect(KILL cr);
7880 
7881   format %{ "shrl    $dst, $shift" %}
7882   opcode(0xD1, 0x5); /* D1 /5 */
7883   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7884   ins_pipe(ialu_reg);
7885 %}
7886 
7887 // Logical shift right by one
7888 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7889 %{
7890   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7891   effect(KILL cr);
7892 
7893   format %{ "shrl    $dst, $shift" %}
7894   opcode(0xD1, 0x5); /* D1 /5 */
7895   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7896   ins_pipe(ialu_mem_imm);
7897 %}
7898 
7899 // Logical Shift Right by 8-bit immediate
7900 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7901 %{
7902   match(Set dst (URShiftI dst shift));
7903   effect(KILL cr);
7904 
7905   format %{ "shrl    $dst, $shift" %}
7906   opcode(0xC1, 0x5); /* C1 /5 ib */
7907   ins_encode(reg_opc_imm(dst, shift));
7908   ins_pipe(ialu_reg);
7909 %}
7910 
7911 // Logical Shift Right by 8-bit immediate
7912 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7913 %{
7914   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7915   effect(KILL cr);
7916 
7917   format %{ "shrl    $dst, $shift" %}
7918   opcode(0xC1, 0x5); /* C1 /5 ib */
7919   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7920   ins_pipe(ialu_mem_imm);
7921 %}
7922 
7923 // Logical Shift Right by variable
7924 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7925 %{
7926   match(Set dst (URShiftI dst shift));
7927   effect(KILL cr);
7928 
7929   format %{ "shrl    $dst, $shift" %}
7930   opcode(0xD3, 0x5); /* D3 /5 */
7931   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7932   ins_pipe(ialu_reg_reg);
7933 %}
7934 
7935 // Logical Shift Right by variable
7936 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7937 %{
7938   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7939   effect(KILL cr);
7940 
7941   format %{ "shrl    $dst, $shift" %}
7942   opcode(0xD3, 0x5); /* D3 /5 */
7943   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7944   ins_pipe(ialu_mem_reg);
7945 %}
7946 
7947 // Long Shift Instructions
7948 // Shift Left by one
7949 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
7950 %{
7951   match(Set dst (LShiftL dst shift));
7952   effect(KILL cr);
7953 
7954   format %{ "salq    $dst, $shift" %}
7955   opcode(0xD1, 0x4); /* D1 /4 */
7956   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7957   ins_pipe(ialu_reg);
7958 %}
7959 
7960 // Shift Left by one
7961 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7962 %{
7963   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
7964   effect(KILL cr);
7965 
7966   format %{ "salq    $dst, $shift" %}
7967   opcode(0xD1, 0x4); /* D1 /4 */
7968   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7969   ins_pipe(ialu_mem_imm);
7970 %}
7971 
7972 // Shift Left by 8-bit immediate
7973 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
7974 %{
7975   match(Set dst (LShiftL dst shift));
7976   effect(KILL cr);
7977 
7978   format %{ "salq    $dst, $shift" %}
7979   opcode(0xC1, 0x4); /* C1 /4 ib */
7980   ins_encode(reg_opc_imm_wide(dst, shift));
7981   ins_pipe(ialu_reg);
7982 %}
7983 
7984 // Shift Left by 8-bit immediate
7985 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7986 %{
7987   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
7988   effect(KILL cr);
7989 
7990   format %{ "salq    $dst, $shift" %}
7991   opcode(0xC1, 0x4); /* C1 /4 ib */
7992   ins_encode(REX_mem_wide(dst), OpcP,
7993              RM_opc_mem(secondary, dst), Con8or32(shift));
7994   ins_pipe(ialu_mem_imm);
7995 %}
7996 
7997 // Shift Left by variable
7998 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
7999 %{
8000   match(Set dst (LShiftL dst shift));
8001   effect(KILL cr);
8002 
8003   format %{ "salq    $dst, $shift" %}
8004   opcode(0xD3, 0x4); /* D3 /4 */
8005   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8006   ins_pipe(ialu_reg_reg);
8007 %}
8008 
8009 // Shift Left by variable
8010 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8011 %{
8012   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8013   effect(KILL cr);
8014 
8015   format %{ "salq    $dst, $shift" %}
8016   opcode(0xD3, 0x4); /* D3 /4 */
8017   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8018   ins_pipe(ialu_mem_reg);
8019 %}
8020 
8021 // Arithmetic shift right by one
8022 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8023 %{
8024   match(Set dst (RShiftL dst shift));
8025   effect(KILL cr);
8026 
8027   format %{ "sarq    $dst, $shift" %}
8028   opcode(0xD1, 0x7); /* D1 /7 */
8029   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8030   ins_pipe(ialu_reg);
8031 %}
8032 
8033 // Arithmetic shift right by one
8034 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8035 %{
8036   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8037   effect(KILL cr);
8038 
8039   format %{ "sarq    $dst, $shift" %}
8040   opcode(0xD1, 0x7); /* D1 /7 */
8041   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8042   ins_pipe(ialu_mem_imm);
8043 %}
8044 
8045 // Arithmetic Shift Right by 8-bit immediate
8046 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8047 %{
8048   match(Set dst (RShiftL dst shift));
8049   effect(KILL cr);
8050 
8051   format %{ "sarq    $dst, $shift" %}
8052   opcode(0xC1, 0x7); /* C1 /7 ib */
8053   ins_encode(reg_opc_imm_wide(dst, shift));
8054   ins_pipe(ialu_mem_imm);
8055 %}
8056 
8057 // Arithmetic Shift Right by 8-bit immediate
8058 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8059 %{
8060   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8061   effect(KILL cr);
8062 
8063   format %{ "sarq    $dst, $shift" %}
8064   opcode(0xC1, 0x7); /* C1 /7 ib */
8065   ins_encode(REX_mem_wide(dst), OpcP,
8066              RM_opc_mem(secondary, dst), Con8or32(shift));
8067   ins_pipe(ialu_mem_imm);
8068 %}
8069 
8070 // Arithmetic Shift Right by variable
8071 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8072 %{
8073   match(Set dst (RShiftL dst shift));
8074   effect(KILL cr);
8075 
8076   format %{ "sarq    $dst, $shift" %}
8077   opcode(0xD3, 0x7); /* D3 /7 */
8078   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8079   ins_pipe(ialu_reg_reg);
8080 %}
8081 
8082 // Arithmetic Shift Right by variable
8083 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8084 %{
8085   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8086   effect(KILL cr);
8087 
8088   format %{ "sarq    $dst, $shift" %}
8089   opcode(0xD3, 0x7); /* D3 /7 */
8090   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8091   ins_pipe(ialu_mem_reg);
8092 %}
8093 
8094 // Logical shift right by one
8095 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8096 %{
8097   match(Set dst (URShiftL dst shift));
8098   effect(KILL cr);
8099 
8100   format %{ "shrq    $dst, $shift" %}
8101   opcode(0xD1, 0x5); /* D1 /5 */
8102   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8103   ins_pipe(ialu_reg);
8104 %}
8105 
8106 // Logical shift right by one
8107 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8108 %{
8109   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8110   effect(KILL cr);
8111 
8112   format %{ "shrq    $dst, $shift" %}
8113   opcode(0xD1, 0x5); /* D1 /5 */
8114   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8115   ins_pipe(ialu_mem_imm);
8116 %}
8117 
8118 // Logical Shift Right by 8-bit immediate
8119 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8120 %{
8121   match(Set dst (URShiftL dst shift));
8122   effect(KILL cr);
8123 
8124   format %{ "shrq    $dst, $shift" %}
8125   opcode(0xC1, 0x5); /* C1 /5 ib */
8126   ins_encode(reg_opc_imm_wide(dst, shift));
8127   ins_pipe(ialu_reg);
8128 %}
8129 
8130 
8131 // Logical Shift Right by 8-bit immediate
8132 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8133 %{
8134   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8135   effect(KILL cr);
8136 
8137   format %{ "shrq    $dst, $shift" %}
8138   opcode(0xC1, 0x5); /* C1 /5 ib */
8139   ins_encode(REX_mem_wide(dst), OpcP,
8140              RM_opc_mem(secondary, dst), Con8or32(shift));
8141   ins_pipe(ialu_mem_imm);
8142 %}
8143 
8144 // Logical Shift Right by variable
8145 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8146 %{
8147   match(Set dst (URShiftL dst shift));
8148   effect(KILL cr);
8149 
8150   format %{ "shrq    $dst, $shift" %}
8151   opcode(0xD3, 0x5); /* D3 /5 */
8152   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8153   ins_pipe(ialu_reg_reg);
8154 %}
8155 
8156 // Logical Shift Right by variable
8157 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8158 %{
8159   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8160   effect(KILL cr);
8161 
8162   format %{ "shrq    $dst, $shift" %}
8163   opcode(0xD3, 0x5); /* D3 /5 */
8164   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8165   ins_pipe(ialu_mem_reg);
8166 %}
8167 
8168 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8169 // This idiom is used by the compiler for the i2b bytecode.
8170 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8171 %{
8172   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8173 
8174   format %{ "movsbl  $dst, $src\t# i2b" %}
8175   opcode(0x0F, 0xBE);
8176   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8177   ins_pipe(ialu_reg_reg);
8178 %}
8179 
8180 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8181 // This idiom is used by the compiler the i2s bytecode.
8182 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8183 %{
8184   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8185 
8186   format %{ "movswl  $dst, $src\t# i2s" %}
8187   opcode(0x0F, 0xBF);
8188   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8189   ins_pipe(ialu_reg_reg);
8190 %}
8191 
8192 // ROL/ROR instructions
8193 
8194 // ROL expand
8195 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8196   effect(KILL cr, USE_DEF dst);
8197 
8198   format %{ "roll    $dst" %}
8199   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8200   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8201   ins_pipe(ialu_reg);
8202 %}
8203 
8204 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8205   effect(USE_DEF dst, USE shift, KILL cr);
8206 
8207   format %{ "roll    $dst, $shift" %}
8208   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8209   ins_encode( reg_opc_imm(dst, shift) );
8210   ins_pipe(ialu_reg);
8211 %}
8212 
8213 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8214 %{
8215   effect(USE_DEF dst, USE shift, KILL cr);
8216 
8217   format %{ "roll    $dst, $shift" %}
8218   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8219   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8220   ins_pipe(ialu_reg_reg);
8221 %}
8222 // end of ROL expand
8223 
8224 // Rotate Left by one
8225 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8226 %{
8227   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8228 
8229   expand %{
8230     rolI_rReg_imm1(dst, cr);
8231   %}
8232 %}
8233 
8234 // Rotate Left by 8-bit immediate
8235 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8236 %{
8237   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8238   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8239 
8240   expand %{
8241     rolI_rReg_imm8(dst, lshift, cr);
8242   %}
8243 %}
8244 
8245 // Rotate Left by variable
8246 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8247 %{
8248   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8249 
8250   expand %{
8251     rolI_rReg_CL(dst, shift, cr);
8252   %}
8253 %}
8254 
8255 // Rotate Left by variable
8256 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8257 %{
8258   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8259 
8260   expand %{
8261     rolI_rReg_CL(dst, shift, cr);
8262   %}
8263 %}
8264 
8265 // ROR expand
8266 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8267 %{
8268   effect(USE_DEF dst, KILL cr);
8269 
8270   format %{ "rorl    $dst" %}
8271   opcode(0xD1, 0x1); /* D1 /1 */
8272   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8273   ins_pipe(ialu_reg);
8274 %}
8275 
8276 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8277 %{
8278   effect(USE_DEF dst, USE shift, KILL cr);
8279 
8280   format %{ "rorl    $dst, $shift" %}
8281   opcode(0xC1, 0x1); /* C1 /1 ib */
8282   ins_encode(reg_opc_imm(dst, shift));
8283   ins_pipe(ialu_reg);
8284 %}
8285 
8286 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8287 %{
8288   effect(USE_DEF dst, USE shift, KILL cr);
8289 
8290   format %{ "rorl    $dst, $shift" %}
8291   opcode(0xD3, 0x1); /* D3 /1 */
8292   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8293   ins_pipe(ialu_reg_reg);
8294 %}
8295 // end of ROR expand
8296 
8297 // Rotate Right by one
8298 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8299 %{
8300   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8301 
8302   expand %{
8303     rorI_rReg_imm1(dst, cr);
8304   %}
8305 %}
8306 
8307 // Rotate Right by 8-bit immediate
8308 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8309 %{
8310   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8311   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8312 
8313   expand %{
8314     rorI_rReg_imm8(dst, rshift, cr);
8315   %}
8316 %}
8317 
8318 // Rotate Right by variable
8319 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8320 %{
8321   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8322 
8323   expand %{
8324     rorI_rReg_CL(dst, shift, cr);
8325   %}
8326 %}
8327 
8328 // Rotate Right by variable
8329 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8330 %{
8331   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8332 
8333   expand %{
8334     rorI_rReg_CL(dst, shift, cr);
8335   %}
8336 %}
8337 
8338 // for long rotate
8339 // ROL expand
8340 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8341   effect(USE_DEF dst, KILL cr);
8342 
8343   format %{ "rolq    $dst" %}
8344   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8345   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8346   ins_pipe(ialu_reg);
8347 %}
8348 
8349 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8350   effect(USE_DEF dst, USE shift, KILL cr);
8351 
8352   format %{ "rolq    $dst, $shift" %}
8353   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8354   ins_encode( reg_opc_imm_wide(dst, shift) );
8355   ins_pipe(ialu_reg);
8356 %}
8357 
8358 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8359 %{
8360   effect(USE_DEF dst, USE shift, KILL cr);
8361 
8362   format %{ "rolq    $dst, $shift" %}
8363   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8364   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8365   ins_pipe(ialu_reg_reg);
8366 %}
8367 // end of ROL expand
8368 
8369 // Rotate Left by one
8370 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8371 %{
8372   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8373 
8374   expand %{
8375     rolL_rReg_imm1(dst, cr);
8376   %}
8377 %}
8378 
8379 // Rotate Left by 8-bit immediate
8380 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8381 %{
8382   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8383   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8384 
8385   expand %{
8386     rolL_rReg_imm8(dst, lshift, cr);
8387   %}
8388 %}
8389 
8390 // Rotate Left by variable
8391 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8392 %{
8393   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8394 
8395   expand %{
8396     rolL_rReg_CL(dst, shift, cr);
8397   %}
8398 %}
8399 
8400 // Rotate Left by variable
8401 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8402 %{
8403   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8404 
8405   expand %{
8406     rolL_rReg_CL(dst, shift, cr);
8407   %}
8408 %}
8409 
8410 // ROR expand
8411 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8412 %{
8413   effect(USE_DEF dst, KILL cr);
8414 
8415   format %{ "rorq    $dst" %}
8416   opcode(0xD1, 0x1); /* D1 /1 */
8417   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8418   ins_pipe(ialu_reg);
8419 %}
8420 
8421 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8422 %{
8423   effect(USE_DEF dst, USE shift, KILL cr);
8424 
8425   format %{ "rorq    $dst, $shift" %}
8426   opcode(0xC1, 0x1); /* C1 /1 ib */
8427   ins_encode(reg_opc_imm_wide(dst, shift));
8428   ins_pipe(ialu_reg);
8429 %}
8430 
8431 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8432 %{
8433   effect(USE_DEF dst, USE shift, KILL cr);
8434 
8435   format %{ "rorq    $dst, $shift" %}
8436   opcode(0xD3, 0x1); /* D3 /1 */
8437   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8438   ins_pipe(ialu_reg_reg);
8439 %}
8440 // end of ROR expand
8441 
8442 // Rotate Right by one
8443 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8444 %{
8445   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8446 
8447   expand %{
8448     rorL_rReg_imm1(dst, cr);
8449   %}
8450 %}
8451 
8452 // Rotate Right by 8-bit immediate
8453 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8454 %{
8455   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8456   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8457 
8458   expand %{
8459     rorL_rReg_imm8(dst, rshift, cr);
8460   %}
8461 %}
8462 
8463 // Rotate Right by variable
8464 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8465 %{
8466   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8467 
8468   expand %{
8469     rorL_rReg_CL(dst, shift, cr);
8470   %}
8471 %}
8472 
8473 // Rotate Right by variable
8474 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8475 %{
8476   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8477 
8478   expand %{
8479     rorL_rReg_CL(dst, shift, cr);
8480   %}
8481 %}
8482 
8483 // Logical Instructions
8484 
8485 // Integer Logical Instructions
8486 
8487 // And Instructions
8488 // And Register with Register
8489 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8490 %{
8491   match(Set dst (AndI dst src));
8492   effect(KILL cr);
8493 
8494   format %{ "andl    $dst, $src\t# int" %}
8495   opcode(0x23);
8496   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8497   ins_pipe(ialu_reg_reg);
8498 %}
8499 
8500 // And Register with Immediate 255
8501 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8502 %{
8503   match(Set dst (AndI dst src));
8504 
8505   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8506   opcode(0x0F, 0xB6);
8507   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8508   ins_pipe(ialu_reg);
8509 %}
8510 
8511 // And Register with Immediate 255 and promote to long
8512 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8513 %{
8514   match(Set dst (ConvI2L (AndI src mask)));
8515 
8516   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8517   opcode(0x0F, 0xB6);
8518   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8519   ins_pipe(ialu_reg);
8520 %}
8521 
8522 // And Register with Immediate 65535
8523 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8524 %{
8525   match(Set dst (AndI dst src));
8526 
8527   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8528   opcode(0x0F, 0xB7);
8529   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8530   ins_pipe(ialu_reg);
8531 %}
8532 
8533 // And Register with Immediate 65535 and promote to long
8534 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8535 %{
8536   match(Set dst (ConvI2L (AndI src mask)));
8537 
8538   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8539   opcode(0x0F, 0xB7);
8540   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8541   ins_pipe(ialu_reg);
8542 %}
8543 
8544 // And Register with Immediate
8545 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8546 %{
8547   match(Set dst (AndI dst src));
8548   effect(KILL cr);
8549 
8550   format %{ "andl    $dst, $src\t# int" %}
8551   opcode(0x81, 0x04); /* Opcode 81 /4 */
8552   ins_encode(OpcSErm(dst, src), Con8or32(src));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 // And Register with Memory
8557 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8558 %{
8559   match(Set dst (AndI dst (LoadI src)));
8560   effect(KILL cr);
8561 
8562   ins_cost(125);
8563   format %{ "andl    $dst, $src\t# int" %}
8564   opcode(0x23);
8565   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8566   ins_pipe(ialu_reg_mem);
8567 %}
8568 
8569 // And Memory with Register
8570 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8571 %{
8572   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8573   effect(KILL cr);
8574 
8575   ins_cost(150);
8576   format %{ "andl    $dst, $src\t# int" %}
8577   opcode(0x21); /* Opcode 21 /r */
8578   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8579   ins_pipe(ialu_mem_reg);
8580 %}
8581 
8582 // And Memory with Immediate
8583 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8584 %{
8585   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8586   effect(KILL cr);
8587 
8588   ins_cost(125);
8589   format %{ "andl    $dst, $src\t# int" %}
8590   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8591   ins_encode(REX_mem(dst), OpcSE(src),
8592              RM_opc_mem(secondary, dst), Con8or32(src));
8593   ins_pipe(ialu_mem_imm);
8594 %}
8595 
8596 // BMI1 instructions
8597 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8598   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8599   predicate(UseBMI1Instructions);
8600   effect(KILL cr);
8601 
8602   ins_cost(125);
8603   format %{ "andnl  $dst, $src1, $src2" %}
8604 
8605   ins_encode %{
8606     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8607   %}
8608   ins_pipe(ialu_reg_mem);
8609 %}
8610 
8611 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8612   match(Set dst (AndI (XorI src1 minus_1) src2));
8613   predicate(UseBMI1Instructions);
8614   effect(KILL cr);
8615 
8616   format %{ "andnl  $dst, $src1, $src2" %}
8617 
8618   ins_encode %{
8619     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8620   %}
8621   ins_pipe(ialu_reg);
8622 %}
8623 
8624 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8625   match(Set dst (AndI (SubI imm_zero src) src));
8626   predicate(UseBMI1Instructions);
8627   effect(KILL cr);
8628 
8629   format %{ "blsil  $dst, $src" %}
8630 
8631   ins_encode %{
8632     __ blsil($dst$$Register, $src$$Register);
8633   %}
8634   ins_pipe(ialu_reg);
8635 %}
8636 
8637 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8638   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8639   predicate(UseBMI1Instructions);
8640   effect(KILL cr);
8641 
8642   ins_cost(125);
8643   format %{ "blsil  $dst, $src" %}
8644 
8645   ins_encode %{
8646     __ blsil($dst$$Register, $src$$Address);
8647   %}
8648   ins_pipe(ialu_reg_mem);
8649 %}
8650 
8651 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8652 %{
8653   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8654   predicate(UseBMI1Instructions);
8655   effect(KILL cr);
8656 
8657   ins_cost(125);
8658   format %{ "blsmskl $dst, $src" %}
8659 
8660   ins_encode %{
8661     __ blsmskl($dst$$Register, $src$$Address);
8662   %}
8663   ins_pipe(ialu_reg_mem);
8664 %}
8665 
8666 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8667 %{
8668   match(Set dst (XorI (AddI src minus_1) src));
8669   predicate(UseBMI1Instructions);
8670   effect(KILL cr);
8671 
8672   format %{ "blsmskl $dst, $src" %}
8673 
8674   ins_encode %{
8675     __ blsmskl($dst$$Register, $src$$Register);
8676   %}
8677 
8678   ins_pipe(ialu_reg);
8679 %}
8680 
8681 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8682 %{
8683   match(Set dst (AndI (AddI src minus_1) src) );
8684   predicate(UseBMI1Instructions);
8685   effect(KILL cr);
8686 
8687   format %{ "blsrl  $dst, $src" %}
8688 
8689   ins_encode %{
8690     __ blsrl($dst$$Register, $src$$Register);
8691   %}
8692 
8693   ins_pipe(ialu_reg_mem);
8694 %}
8695 
8696 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8697 %{
8698   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8699   predicate(UseBMI1Instructions);
8700   effect(KILL cr);
8701 
8702   ins_cost(125);
8703   format %{ "blsrl  $dst, $src" %}
8704 
8705   ins_encode %{
8706     __ blsrl($dst$$Register, $src$$Address);
8707   %}
8708 
8709   ins_pipe(ialu_reg);
8710 %}
8711 
8712 // Or Instructions
8713 // Or Register with Register
8714 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8715 %{
8716   match(Set dst (OrI dst src));
8717   effect(KILL cr);
8718 
8719   format %{ "orl     $dst, $src\t# int" %}
8720   opcode(0x0B);
8721   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8722   ins_pipe(ialu_reg_reg);
8723 %}
8724 
8725 // Or Register with Immediate
8726 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8727 %{
8728   match(Set dst (OrI dst src));
8729   effect(KILL cr);
8730 
8731   format %{ "orl     $dst, $src\t# int" %}
8732   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8733   ins_encode(OpcSErm(dst, src), Con8or32(src));
8734   ins_pipe(ialu_reg);
8735 %}
8736 
8737 // Or Register with Memory
8738 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8739 %{
8740   match(Set dst (OrI dst (LoadI src)));
8741   effect(KILL cr);
8742 
8743   ins_cost(125);
8744   format %{ "orl     $dst, $src\t# int" %}
8745   opcode(0x0B);
8746   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8747   ins_pipe(ialu_reg_mem);
8748 %}
8749 
8750 // Or Memory with Register
8751 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8752 %{
8753   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8754   effect(KILL cr);
8755 
8756   ins_cost(150);
8757   format %{ "orl     $dst, $src\t# int" %}
8758   opcode(0x09); /* Opcode 09 /r */
8759   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8760   ins_pipe(ialu_mem_reg);
8761 %}
8762 
8763 // Or Memory with Immediate
8764 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8765 %{
8766   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8767   effect(KILL cr);
8768 
8769   ins_cost(125);
8770   format %{ "orl     $dst, $src\t# int" %}
8771   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8772   ins_encode(REX_mem(dst), OpcSE(src),
8773              RM_opc_mem(secondary, dst), Con8or32(src));
8774   ins_pipe(ialu_mem_imm);
8775 %}
8776 
8777 // Xor Instructions
8778 // Xor Register with Register
8779 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8780 %{
8781   match(Set dst (XorI dst src));
8782   effect(KILL cr);
8783 
8784   format %{ "xorl    $dst, $src\t# int" %}
8785   opcode(0x33);
8786   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8787   ins_pipe(ialu_reg_reg);
8788 %}
8789 
8790 // Xor Register with Immediate -1
8791 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8792   match(Set dst (XorI dst imm));
8793 
8794   format %{ "not    $dst" %}
8795   ins_encode %{
8796      __ notl($dst$$Register);
8797   %}
8798   ins_pipe(ialu_reg);
8799 %}
8800 
8801 // Xor Register with Immediate
8802 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8803 %{
8804   match(Set dst (XorI dst src));
8805   effect(KILL cr);
8806 
8807   format %{ "xorl    $dst, $src\t# int" %}
8808   opcode(0x81, 0x06); /* Opcode 81 /6 id */
8809   ins_encode(OpcSErm(dst, src), Con8or32(src));
8810   ins_pipe(ialu_reg);
8811 %}
8812 
8813 // Xor Register with Memory
8814 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8815 %{
8816   match(Set dst (XorI dst (LoadI src)));
8817   effect(KILL cr);
8818 
8819   ins_cost(125);
8820   format %{ "xorl    $dst, $src\t# int" %}
8821   opcode(0x33);
8822   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8823   ins_pipe(ialu_reg_mem);
8824 %}
8825 
8826 // Xor Memory with Register
8827 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8828 %{
8829   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8830   effect(KILL cr);
8831 
8832   ins_cost(150);
8833   format %{ "xorl    $dst, $src\t# int" %}
8834   opcode(0x31); /* Opcode 31 /r */
8835   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8836   ins_pipe(ialu_mem_reg);
8837 %}
8838 
8839 // Xor Memory with Immediate
8840 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
8841 %{
8842   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8843   effect(KILL cr);
8844 
8845   ins_cost(125);
8846   format %{ "xorl    $dst, $src\t# int" %}
8847   opcode(0x81, 0x6); /* Opcode 81 /6 id */
8848   ins_encode(REX_mem(dst), OpcSE(src),
8849              RM_opc_mem(secondary, dst), Con8or32(src));
8850   ins_pipe(ialu_mem_imm);
8851 %}
8852 
8853 
8854 // Long Logical Instructions
8855 
8856 // And Instructions
8857 // And Register with Register
8858 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8859 %{
8860   match(Set dst (AndL dst src));
8861   effect(KILL cr);
8862 
8863   format %{ "andq    $dst, $src\t# long" %}
8864   opcode(0x23);
8865   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8866   ins_pipe(ialu_reg_reg);
8867 %}
8868 
8869 // And Register with Immediate 255
8870 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
8871 %{
8872   match(Set dst (AndL dst src));
8873 
8874   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
8875   opcode(0x0F, 0xB6);
8876   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8877   ins_pipe(ialu_reg);
8878 %}
8879 
8880 // And Register with Immediate 65535
8881 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
8882 %{
8883   match(Set dst (AndL dst src));
8884 
8885   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
8886   opcode(0x0F, 0xB7);
8887   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8888   ins_pipe(ialu_reg);
8889 %}
8890 
8891 // And Register with Immediate
8892 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
8893 %{
8894   match(Set dst (AndL dst src));
8895   effect(KILL cr);
8896 
8897   format %{ "andq    $dst, $src\t# long" %}
8898   opcode(0x81, 0x04); /* Opcode 81 /4 */
8899   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8900   ins_pipe(ialu_reg);
8901 %}
8902 
8903 // And Register with Memory
8904 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8905 %{
8906   match(Set dst (AndL dst (LoadL src)));
8907   effect(KILL cr);
8908 
8909   ins_cost(125);
8910   format %{ "andq    $dst, $src\t# long" %}
8911   opcode(0x23);
8912   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8913   ins_pipe(ialu_reg_mem);
8914 %}
8915 
8916 // And Memory with Register
8917 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8918 %{
8919   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8920   effect(KILL cr);
8921 
8922   ins_cost(150);
8923   format %{ "andq    $dst, $src\t# long" %}
8924   opcode(0x21); /* Opcode 21 /r */
8925   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8926   ins_pipe(ialu_mem_reg);
8927 %}
8928 
8929 // And Memory with Immediate
8930 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8931 %{
8932   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8933   effect(KILL cr);
8934 
8935   ins_cost(125);
8936   format %{ "andq    $dst, $src\t# long" %}
8937   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8938   ins_encode(REX_mem_wide(dst), OpcSE(src),
8939              RM_opc_mem(secondary, dst), Con8or32(src));
8940   ins_pipe(ialu_mem_imm);
8941 %}
8942 
8943 // BMI1 instructions
8944 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
8945   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
8946   predicate(UseBMI1Instructions);
8947   effect(KILL cr);
8948 
8949   ins_cost(125);
8950   format %{ "andnq  $dst, $src1, $src2" %}
8951 
8952   ins_encode %{
8953     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
8954   %}
8955   ins_pipe(ialu_reg_mem);
8956 %}
8957 
8958 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
8959   match(Set dst (AndL (XorL src1 minus_1) src2));
8960   predicate(UseBMI1Instructions);
8961   effect(KILL cr);
8962 
8963   format %{ "andnq  $dst, $src1, $src2" %}
8964 
8965   ins_encode %{
8966   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
8967   %}
8968   ins_pipe(ialu_reg_mem);
8969 %}
8970 
8971 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
8972   match(Set dst (AndL (SubL imm_zero src) src));
8973   predicate(UseBMI1Instructions);
8974   effect(KILL cr);
8975 
8976   format %{ "blsiq  $dst, $src" %}
8977 
8978   ins_encode %{
8979     __ blsiq($dst$$Register, $src$$Register);
8980   %}
8981   ins_pipe(ialu_reg);
8982 %}
8983 
8984 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
8985   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
8986   predicate(UseBMI1Instructions);
8987   effect(KILL cr);
8988 
8989   ins_cost(125);
8990   format %{ "blsiq  $dst, $src" %}
8991 
8992   ins_encode %{
8993     __ blsiq($dst$$Register, $src$$Address);
8994   %}
8995   ins_pipe(ialu_reg_mem);
8996 %}
8997 
8998 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
8999 %{
9000   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9001   predicate(UseBMI1Instructions);
9002   effect(KILL cr);
9003 
9004   ins_cost(125);
9005   format %{ "blsmskq $dst, $src" %}
9006 
9007   ins_encode %{
9008     __ blsmskq($dst$$Register, $src$$Address);
9009   %}
9010   ins_pipe(ialu_reg_mem);
9011 %}
9012 
9013 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9014 %{
9015   match(Set dst (XorL (AddL src minus_1) src));
9016   predicate(UseBMI1Instructions);
9017   effect(KILL cr);
9018 
9019   format %{ "blsmskq $dst, $src" %}
9020 
9021   ins_encode %{
9022     __ blsmskq($dst$$Register, $src$$Register);
9023   %}
9024 
9025   ins_pipe(ialu_reg);
9026 %}
9027 
9028 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9029 %{
9030   match(Set dst (AndL (AddL src minus_1) src) );
9031   predicate(UseBMI1Instructions);
9032   effect(KILL cr);
9033 
9034   format %{ "blsrq  $dst, $src" %}
9035 
9036   ins_encode %{
9037     __ blsrq($dst$$Register, $src$$Register);
9038   %}
9039 
9040   ins_pipe(ialu_reg);
9041 %}
9042 
9043 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9044 %{
9045   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9046   predicate(UseBMI1Instructions);
9047   effect(KILL cr);
9048 
9049   ins_cost(125);
9050   format %{ "blsrq  $dst, $src" %}
9051 
9052   ins_encode %{
9053     __ blsrq($dst$$Register, $src$$Address);
9054   %}
9055 
9056   ins_pipe(ialu_reg);
9057 %}
9058 
9059 // Or Instructions
9060 // Or Register with Register
9061 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9062 %{
9063   match(Set dst (OrL dst src));
9064   effect(KILL cr);
9065 
9066   format %{ "orq     $dst, $src\t# long" %}
9067   opcode(0x0B);
9068   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9069   ins_pipe(ialu_reg_reg);
9070 %}
9071 
9072 // Use any_RegP to match R15 (TLS register) without spilling.
9073 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9074   match(Set dst (OrL dst (CastP2X src)));
9075   effect(KILL cr);
9076 
9077   format %{ "orq     $dst, $src\t# long" %}
9078   opcode(0x0B);
9079   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9080   ins_pipe(ialu_reg_reg);
9081 %}
9082 
9083 
9084 // Or Register with Immediate
9085 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9086 %{
9087   match(Set dst (OrL dst src));
9088   effect(KILL cr);
9089 
9090   format %{ "orq     $dst, $src\t# long" %}
9091   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9092   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9093   ins_pipe(ialu_reg);
9094 %}
9095 
9096 // Or Register with Memory
9097 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9098 %{
9099   match(Set dst (OrL dst (LoadL src)));
9100   effect(KILL cr);
9101 
9102   ins_cost(125);
9103   format %{ "orq     $dst, $src\t# long" %}
9104   opcode(0x0B);
9105   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9106   ins_pipe(ialu_reg_mem);
9107 %}
9108 
9109 // Or Memory with Register
9110 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9111 %{
9112   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9113   effect(KILL cr);
9114 
9115   ins_cost(150);
9116   format %{ "orq     $dst, $src\t# long" %}
9117   opcode(0x09); /* Opcode 09 /r */
9118   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9119   ins_pipe(ialu_mem_reg);
9120 %}
9121 
9122 // Or Memory with Immediate
9123 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9124 %{
9125   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9126   effect(KILL cr);
9127 
9128   ins_cost(125);
9129   format %{ "orq     $dst, $src\t# long" %}
9130   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9131   ins_encode(REX_mem_wide(dst), OpcSE(src),
9132              RM_opc_mem(secondary, dst), Con8or32(src));
9133   ins_pipe(ialu_mem_imm);
9134 %}
9135 
9136 // Xor Instructions
9137 // Xor Register with Register
9138 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9139 %{
9140   match(Set dst (XorL dst src));
9141   effect(KILL cr);
9142 
9143   format %{ "xorq    $dst, $src\t# long" %}
9144   opcode(0x33);
9145   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9146   ins_pipe(ialu_reg_reg);
9147 %}
9148 
9149 // Xor Register with Immediate -1
9150 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9151   match(Set dst (XorL dst imm));
9152 
9153   format %{ "notq   $dst" %}
9154   ins_encode %{
9155      __ notq($dst$$Register);
9156   %}
9157   ins_pipe(ialu_reg);
9158 %}
9159 
9160 // Xor Register with Immediate
9161 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9162 %{
9163   match(Set dst (XorL dst src));
9164   effect(KILL cr);
9165 
9166   format %{ "xorq    $dst, $src\t# long" %}
9167   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9168   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9169   ins_pipe(ialu_reg);
9170 %}
9171 
9172 // Xor Register with Memory
9173 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9174 %{
9175   match(Set dst (XorL dst (LoadL src)));
9176   effect(KILL cr);
9177 
9178   ins_cost(125);
9179   format %{ "xorq    $dst, $src\t# long" %}
9180   opcode(0x33);
9181   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9182   ins_pipe(ialu_reg_mem);
9183 %}
9184 
9185 // Xor Memory with Register
9186 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9187 %{
9188   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9189   effect(KILL cr);
9190 
9191   ins_cost(150);
9192   format %{ "xorq    $dst, $src\t# long" %}
9193   opcode(0x31); /* Opcode 31 /r */
9194   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9195   ins_pipe(ialu_mem_reg);
9196 %}
9197 
9198 // Xor Memory with Immediate
9199 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9200 %{
9201   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9202   effect(KILL cr);
9203 
9204   ins_cost(125);
9205   format %{ "xorq    $dst, $src\t# long" %}
9206   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9207   ins_encode(REX_mem_wide(dst), OpcSE(src),
9208              RM_opc_mem(secondary, dst), Con8or32(src));
9209   ins_pipe(ialu_mem_imm);
9210 %}
9211 
9212 // Convert Int to Boolean
9213 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9214 %{
9215   match(Set dst (Conv2B src));
9216   effect(KILL cr);
9217 
9218   format %{ "testl   $src, $src\t# ci2b\n\t"
9219             "setnz   $dst\n\t"
9220             "movzbl  $dst, $dst" %}
9221   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9222              setNZ_reg(dst),
9223              REX_reg_breg(dst, dst), // movzbl
9224              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9225   ins_pipe(pipe_slow); // XXX
9226 %}
9227 
9228 // Convert Pointer to Boolean
9229 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9230 %{
9231   match(Set dst (Conv2B src));
9232   effect(KILL cr);
9233 
9234   format %{ "testq   $src, $src\t# cp2b\n\t"
9235             "setnz   $dst\n\t"
9236             "movzbl  $dst, $dst" %}
9237   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9238              setNZ_reg(dst),
9239              REX_reg_breg(dst, dst), // movzbl
9240              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9241   ins_pipe(pipe_slow); // XXX
9242 %}
9243 
9244 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9245 %{
9246   match(Set dst (CmpLTMask p q));
9247   effect(KILL cr);
9248 
9249   ins_cost(400);
9250   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9251             "setlt   $dst\n\t"
9252             "movzbl  $dst, $dst\n\t"
9253             "negl    $dst" %}
9254   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9255              setLT_reg(dst),
9256              REX_reg_breg(dst, dst), // movzbl
9257              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9258              neg_reg(dst));
9259   ins_pipe(pipe_slow);
9260 %}
9261 
9262 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9263 %{
9264   match(Set dst (CmpLTMask dst zero));
9265   effect(KILL cr);
9266 
9267   ins_cost(100);
9268   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9269   ins_encode %{
9270   __ sarl($dst$$Register, 31);
9271   %}
9272   ins_pipe(ialu_reg);
9273 %}
9274 
9275 /* Better to save a register than avoid a branch */
9276 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9277 %{
9278   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9279   effect(KILL cr);
9280   ins_cost(300);
9281   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9282             "jge    done\n\t"
9283             "addl   $p,$y\n"
9284             "done:  " %}
9285   ins_encode %{
9286     Register Rp = $p$$Register;
9287     Register Rq = $q$$Register;
9288     Register Ry = $y$$Register;
9289     Label done;
9290     __ subl(Rp, Rq);
9291     __ jccb(Assembler::greaterEqual, done);
9292     __ addl(Rp, Ry);
9293     __ bind(done);
9294   %}
9295   ins_pipe(pipe_cmplt);
9296 %}
9297 
9298 /* Better to save a register than avoid a branch */
9299 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9300 %{
9301   match(Set y (AndI (CmpLTMask p q) y));
9302   effect(KILL cr);
9303 
9304   ins_cost(300);
9305 
9306   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9307             "jlt      done\n\t"
9308             "xorl     $y, $y\n"
9309             "done:  " %}
9310   ins_encode %{
9311     Register Rp = $p$$Register;
9312     Register Rq = $q$$Register;
9313     Register Ry = $y$$Register;
9314     Label done;
9315     __ cmpl(Rp, Rq);
9316     __ jccb(Assembler::less, done);
9317     __ xorl(Ry, Ry);
9318     __ bind(done);
9319   %}
9320   ins_pipe(pipe_cmplt);
9321 %}
9322 
9323 
9324 //---------- FP Instructions------------------------------------------------
9325 
9326 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9327 %{
9328   match(Set cr (CmpF src1 src2));
9329 
9330   ins_cost(145);
9331   format %{ "ucomiss $src1, $src2\n\t"
9332             "jnp,s   exit\n\t"
9333             "pushfq\t# saw NaN, set CF\n\t"
9334             "andq    [rsp], #0xffffff2b\n\t"
9335             "popfq\n"
9336     "exit:" %}
9337   ins_encode %{
9338     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9339     emit_cmpfp_fixup(_masm);
9340   %}
9341   ins_pipe(pipe_slow);
9342 %}
9343 
9344 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9345   match(Set cr (CmpF src1 src2));
9346 
9347   ins_cost(100);
9348   format %{ "ucomiss $src1, $src2" %}
9349   ins_encode %{
9350     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9351   %}
9352   ins_pipe(pipe_slow);
9353 %}
9354 
9355 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9356 %{
9357   match(Set cr (CmpF src1 (LoadF src2)));
9358 
9359   ins_cost(145);
9360   format %{ "ucomiss $src1, $src2\n\t"
9361             "jnp,s   exit\n\t"
9362             "pushfq\t# saw NaN, set CF\n\t"
9363             "andq    [rsp], #0xffffff2b\n\t"
9364             "popfq\n"
9365     "exit:" %}
9366   ins_encode %{
9367     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9368     emit_cmpfp_fixup(_masm);
9369   %}
9370   ins_pipe(pipe_slow);
9371 %}
9372 
9373 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9374   match(Set cr (CmpF src1 (LoadF src2)));
9375 
9376   ins_cost(100);
9377   format %{ "ucomiss $src1, $src2" %}
9378   ins_encode %{
9379     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9380   %}
9381   ins_pipe(pipe_slow);
9382 %}
9383 
9384 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9385   match(Set cr (CmpF src con));
9386 
9387   ins_cost(145);
9388   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9389             "jnp,s   exit\n\t"
9390             "pushfq\t# saw NaN, set CF\n\t"
9391             "andq    [rsp], #0xffffff2b\n\t"
9392             "popfq\n"
9393     "exit:" %}
9394   ins_encode %{
9395     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9396     emit_cmpfp_fixup(_masm);
9397   %}
9398   ins_pipe(pipe_slow);
9399 %}
9400 
9401 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9402   match(Set cr (CmpF src con));
9403   ins_cost(100);
9404   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9405   ins_encode %{
9406     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9407   %}
9408   ins_pipe(pipe_slow);
9409 %}
9410 
9411 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9412 %{
9413   match(Set cr (CmpD src1 src2));
9414 
9415   ins_cost(145);
9416   format %{ "ucomisd $src1, $src2\n\t"
9417             "jnp,s   exit\n\t"
9418             "pushfq\t# saw NaN, set CF\n\t"
9419             "andq    [rsp], #0xffffff2b\n\t"
9420             "popfq\n"
9421     "exit:" %}
9422   ins_encode %{
9423     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9424     emit_cmpfp_fixup(_masm);
9425   %}
9426   ins_pipe(pipe_slow);
9427 %}
9428 
9429 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9430   match(Set cr (CmpD src1 src2));
9431 
9432   ins_cost(100);
9433   format %{ "ucomisd $src1, $src2 test" %}
9434   ins_encode %{
9435     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9436   %}
9437   ins_pipe(pipe_slow);
9438 %}
9439 
9440 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9441 %{
9442   match(Set cr (CmpD src1 (LoadD src2)));
9443 
9444   ins_cost(145);
9445   format %{ "ucomisd $src1, $src2\n\t"
9446             "jnp,s   exit\n\t"
9447             "pushfq\t# saw NaN, set CF\n\t"
9448             "andq    [rsp], #0xffffff2b\n\t"
9449             "popfq\n"
9450     "exit:" %}
9451   ins_encode %{
9452     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9453     emit_cmpfp_fixup(_masm);
9454   %}
9455   ins_pipe(pipe_slow);
9456 %}
9457 
9458 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9459   match(Set cr (CmpD src1 (LoadD src2)));
9460 
9461   ins_cost(100);
9462   format %{ "ucomisd $src1, $src2" %}
9463   ins_encode %{
9464     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9465   %}
9466   ins_pipe(pipe_slow);
9467 %}
9468 
9469 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9470   match(Set cr (CmpD src con));
9471 
9472   ins_cost(145);
9473   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9474             "jnp,s   exit\n\t"
9475             "pushfq\t# saw NaN, set CF\n\t"
9476             "andq    [rsp], #0xffffff2b\n\t"
9477             "popfq\n"
9478     "exit:" %}
9479   ins_encode %{
9480     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9481     emit_cmpfp_fixup(_masm);
9482   %}
9483   ins_pipe(pipe_slow);
9484 %}
9485 
9486 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9487   match(Set cr (CmpD src con));
9488   ins_cost(100);
9489   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9490   ins_encode %{
9491     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9492   %}
9493   ins_pipe(pipe_slow);
9494 %}
9495 
9496 // Compare into -1,0,1
9497 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9498 %{
9499   match(Set dst (CmpF3 src1 src2));
9500   effect(KILL cr);
9501 
9502   ins_cost(275);
9503   format %{ "ucomiss $src1, $src2\n\t"
9504             "movl    $dst, #-1\n\t"
9505             "jp,s    done\n\t"
9506             "jb,s    done\n\t"
9507             "setne   $dst\n\t"
9508             "movzbl  $dst, $dst\n"
9509     "done:" %}
9510   ins_encode %{
9511     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9512     emit_cmpfp3(_masm, $dst$$Register);
9513   %}
9514   ins_pipe(pipe_slow);
9515 %}
9516 
9517 // Compare into -1,0,1
9518 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9519 %{
9520   match(Set dst (CmpF3 src1 (LoadF src2)));
9521   effect(KILL cr);
9522 
9523   ins_cost(275);
9524   format %{ "ucomiss $src1, $src2\n\t"
9525             "movl    $dst, #-1\n\t"
9526             "jp,s    done\n\t"
9527             "jb,s    done\n\t"
9528             "setne   $dst\n\t"
9529             "movzbl  $dst, $dst\n"
9530     "done:" %}
9531   ins_encode %{
9532     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9533     emit_cmpfp3(_masm, $dst$$Register);
9534   %}
9535   ins_pipe(pipe_slow);
9536 %}
9537 
9538 // Compare into -1,0,1
9539 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9540   match(Set dst (CmpF3 src con));
9541   effect(KILL cr);
9542 
9543   ins_cost(275);
9544   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9545             "movl    $dst, #-1\n\t"
9546             "jp,s    done\n\t"
9547             "jb,s    done\n\t"
9548             "setne   $dst\n\t"
9549             "movzbl  $dst, $dst\n"
9550     "done:" %}
9551   ins_encode %{
9552     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9553     emit_cmpfp3(_masm, $dst$$Register);
9554   %}
9555   ins_pipe(pipe_slow);
9556 %}
9557 
9558 // Compare into -1,0,1
9559 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9560 %{
9561   match(Set dst (CmpD3 src1 src2));
9562   effect(KILL cr);
9563 
9564   ins_cost(275);
9565   format %{ "ucomisd $src1, $src2\n\t"
9566             "movl    $dst, #-1\n\t"
9567             "jp,s    done\n\t"
9568             "jb,s    done\n\t"
9569             "setne   $dst\n\t"
9570             "movzbl  $dst, $dst\n"
9571     "done:" %}
9572   ins_encode %{
9573     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9574     emit_cmpfp3(_masm, $dst$$Register);
9575   %}
9576   ins_pipe(pipe_slow);
9577 %}
9578 
9579 // Compare into -1,0,1
9580 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9581 %{
9582   match(Set dst (CmpD3 src1 (LoadD src2)));
9583   effect(KILL cr);
9584 
9585   ins_cost(275);
9586   format %{ "ucomisd $src1, $src2\n\t"
9587             "movl    $dst, #-1\n\t"
9588             "jp,s    done\n\t"
9589             "jb,s    done\n\t"
9590             "setne   $dst\n\t"
9591             "movzbl  $dst, $dst\n"
9592     "done:" %}
9593   ins_encode %{
9594     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9595     emit_cmpfp3(_masm, $dst$$Register);
9596   %}
9597   ins_pipe(pipe_slow);
9598 %}
9599 
9600 // Compare into -1,0,1
9601 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9602   match(Set dst (CmpD3 src con));
9603   effect(KILL cr);
9604 
9605   ins_cost(275);
9606   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9607             "movl    $dst, #-1\n\t"
9608             "jp,s    done\n\t"
9609             "jb,s    done\n\t"
9610             "setne   $dst\n\t"
9611             "movzbl  $dst, $dst\n"
9612     "done:" %}
9613   ins_encode %{
9614     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9615     emit_cmpfp3(_masm, $dst$$Register);
9616   %}
9617   ins_pipe(pipe_slow);
9618 %}
9619 
9620 // -----------Trig and Trancendental Instructions------------------------------
9621 instruct cosD_reg(regD dst) %{
9622   match(Set dst (CosD dst));
9623 
9624   format %{ "dcos   $dst\n\t" %}
9625   opcode(0xD9, 0xFF);
9626   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9627   ins_pipe( pipe_slow );
9628 %}
9629 
9630 instruct sinD_reg(regD dst) %{
9631   match(Set dst (SinD dst));
9632 
9633   format %{ "dsin   $dst\n\t" %}
9634   opcode(0xD9, 0xFE);
9635   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9636   ins_pipe( pipe_slow );
9637 %}
9638 
9639 instruct tanD_reg(regD dst) %{
9640   match(Set dst (TanD dst));
9641 
9642   format %{ "dtan   $dst\n\t" %}
9643   ins_encode( Push_SrcXD(dst),
9644               Opcode(0xD9), Opcode(0xF2),   //fptan
9645               Opcode(0xDD), Opcode(0xD8),   //fstp st
9646               Push_ResultXD(dst) );
9647   ins_pipe( pipe_slow );
9648 %}
9649 
9650 instruct log10D_reg(regD dst) %{
9651   // The source and result Double operands in XMM registers
9652   match(Set dst (Log10D dst));
9653   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9654   // fyl2x        ; compute log_10(2) * log_2(x)
9655   format %{ "fldlg2\t\t\t#Log10\n\t"
9656             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9657          %}
9658    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9659               Push_SrcXD(dst),
9660               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9661               Push_ResultXD(dst));
9662 
9663   ins_pipe( pipe_slow );
9664 %}
9665 
9666 instruct logD_reg(regD dst) %{
9667   // The source and result Double operands in XMM registers
9668   match(Set dst (LogD dst));
9669   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9670   // fyl2x        ; compute log_e(2) * log_2(x)
9671   format %{ "fldln2\t\t\t#Log_e\n\t"
9672             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9673          %}
9674   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9675               Push_SrcXD(dst),
9676               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9677               Push_ResultXD(dst));
9678   ins_pipe( pipe_slow );
9679 %}
9680 
9681 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9682   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9683   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9684   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9685   ins_encode %{
9686     __ subptr(rsp, 8);
9687     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9688     __ fld_d(Address(rsp, 0));
9689     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9690     __ fld_d(Address(rsp, 0));
9691     __ fast_pow();
9692     __ fstp_d(Address(rsp, 0));
9693     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9694     __ addptr(rsp, 8);
9695   %}
9696   ins_pipe( pipe_slow );
9697 %}
9698 
9699 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9700   match(Set dst (ExpD src));
9701   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9702   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9703   ins_encode %{
9704     __ subptr(rsp, 8);
9705     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9706     __ fld_d(Address(rsp, 0));
9707     __ fast_exp();
9708     __ fstp_d(Address(rsp, 0));
9709     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9710     __ addptr(rsp, 8);
9711   %}
9712   ins_pipe( pipe_slow );
9713 %}
9714 
9715 //----------Arithmetic Conversion Instructions---------------------------------
9716 
9717 instruct roundFloat_nop(regF dst)
9718 %{
9719   match(Set dst (RoundFloat dst));
9720 
9721   ins_cost(0);
9722   ins_encode();
9723   ins_pipe(empty);
9724 %}
9725 
9726 instruct roundDouble_nop(regD dst)
9727 %{
9728   match(Set dst (RoundDouble dst));
9729 
9730   ins_cost(0);
9731   ins_encode();
9732   ins_pipe(empty);
9733 %}
9734 
9735 instruct convF2D_reg_reg(regD dst, regF src)
9736 %{
9737   match(Set dst (ConvF2D src));
9738 
9739   format %{ "cvtss2sd $dst, $src" %}
9740   ins_encode %{
9741     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9742   %}
9743   ins_pipe(pipe_slow); // XXX
9744 %}
9745 
9746 instruct convF2D_reg_mem(regD dst, memory src)
9747 %{
9748   match(Set dst (ConvF2D (LoadF src)));
9749 
9750   format %{ "cvtss2sd $dst, $src" %}
9751   ins_encode %{
9752     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9753   %}
9754   ins_pipe(pipe_slow); // XXX
9755 %}
9756 
9757 instruct convD2F_reg_reg(regF dst, regD src)
9758 %{
9759   match(Set dst (ConvD2F src));
9760 
9761   format %{ "cvtsd2ss $dst, $src" %}
9762   ins_encode %{
9763     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9764   %}
9765   ins_pipe(pipe_slow); // XXX
9766 %}
9767 
9768 instruct convD2F_reg_mem(regF dst, memory src)
9769 %{
9770   match(Set dst (ConvD2F (LoadD src)));
9771 
9772   format %{ "cvtsd2ss $dst, $src" %}
9773   ins_encode %{
9774     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9775   %}
9776   ins_pipe(pipe_slow); // XXX
9777 %}
9778 
9779 // XXX do mem variants
9780 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9781 %{
9782   match(Set dst (ConvF2I src));
9783   effect(KILL cr);
9784 
9785   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9786             "cmpl    $dst, #0x80000000\n\t"
9787             "jne,s   done\n\t"
9788             "subq    rsp, #8\n\t"
9789             "movss   [rsp], $src\n\t"
9790             "call    f2i_fixup\n\t"
9791             "popq    $dst\n"
9792     "done:   "%}
9793   ins_encode %{
9794     Label done;
9795     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9796     __ cmpl($dst$$Register, 0x80000000);
9797     __ jccb(Assembler::notEqual, done);
9798     __ subptr(rsp, 8);
9799     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9800     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9801     __ pop($dst$$Register);
9802     __ bind(done);
9803   %}
9804   ins_pipe(pipe_slow);
9805 %}
9806 
9807 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9808 %{
9809   match(Set dst (ConvF2L src));
9810   effect(KILL cr);
9811 
9812   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9813             "cmpq    $dst, [0x8000000000000000]\n\t"
9814             "jne,s   done\n\t"
9815             "subq    rsp, #8\n\t"
9816             "movss   [rsp], $src\n\t"
9817             "call    f2l_fixup\n\t"
9818             "popq    $dst\n"
9819     "done:   "%}
9820   ins_encode %{
9821     Label done;
9822     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9823     __ cmp64($dst$$Register,
9824              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9825     __ jccb(Assembler::notEqual, done);
9826     __ subptr(rsp, 8);
9827     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9828     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9829     __ pop($dst$$Register);
9830     __ bind(done);
9831   %}
9832   ins_pipe(pipe_slow);
9833 %}
9834 
9835 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9836 %{
9837   match(Set dst (ConvD2I src));
9838   effect(KILL cr);
9839 
9840   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9841             "cmpl    $dst, #0x80000000\n\t"
9842             "jne,s   done\n\t"
9843             "subq    rsp, #8\n\t"
9844             "movsd   [rsp], $src\n\t"
9845             "call    d2i_fixup\n\t"
9846             "popq    $dst\n"
9847     "done:   "%}
9848   ins_encode %{
9849     Label done;
9850     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
9851     __ cmpl($dst$$Register, 0x80000000);
9852     __ jccb(Assembler::notEqual, done);
9853     __ subptr(rsp, 8);
9854     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9855     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9856     __ pop($dst$$Register);
9857     __ bind(done);
9858   %}
9859   ins_pipe(pipe_slow);
9860 %}
9861 
9862 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
9863 %{
9864   match(Set dst (ConvD2L src));
9865   effect(KILL cr);
9866 
9867   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
9868             "cmpq    $dst, [0x8000000000000000]\n\t"
9869             "jne,s   done\n\t"
9870             "subq    rsp, #8\n\t"
9871             "movsd   [rsp], $src\n\t"
9872             "call    d2l_fixup\n\t"
9873             "popq    $dst\n"
9874     "done:   "%}
9875   ins_encode %{
9876     Label done;
9877     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
9878     __ cmp64($dst$$Register,
9879              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9880     __ jccb(Assembler::notEqual, done);
9881     __ subptr(rsp, 8);
9882     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9883     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9884     __ pop($dst$$Register);
9885     __ bind(done);
9886   %}
9887   ins_pipe(pipe_slow);
9888 %}
9889 
9890 instruct convI2F_reg_reg(regF dst, rRegI src)
9891 %{
9892   predicate(!UseXmmI2F);
9893   match(Set dst (ConvI2F src));
9894 
9895   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9896   ins_encode %{
9897     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
9898   %}
9899   ins_pipe(pipe_slow); // XXX
9900 %}
9901 
9902 instruct convI2F_reg_mem(regF dst, memory src)
9903 %{
9904   match(Set dst (ConvI2F (LoadI src)));
9905 
9906   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9907   ins_encode %{
9908     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
9909   %}
9910   ins_pipe(pipe_slow); // XXX
9911 %}
9912 
9913 instruct convI2D_reg_reg(regD dst, rRegI src)
9914 %{
9915   predicate(!UseXmmI2D);
9916   match(Set dst (ConvI2D src));
9917 
9918   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9919   ins_encode %{
9920     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
9921   %}
9922   ins_pipe(pipe_slow); // XXX
9923 %}
9924 
9925 instruct convI2D_reg_mem(regD dst, memory src)
9926 %{
9927   match(Set dst (ConvI2D (LoadI src)));
9928 
9929   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9930   ins_encode %{
9931     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
9932   %}
9933   ins_pipe(pipe_slow); // XXX
9934 %}
9935 
9936 instruct convXI2F_reg(regF dst, rRegI src)
9937 %{
9938   predicate(UseXmmI2F);
9939   match(Set dst (ConvI2F src));
9940 
9941   format %{ "movdl $dst, $src\n\t"
9942             "cvtdq2psl $dst, $dst\t# i2f" %}
9943   ins_encode %{
9944     __ movdl($dst$$XMMRegister, $src$$Register);
9945     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
9946   %}
9947   ins_pipe(pipe_slow); // XXX
9948 %}
9949 
9950 instruct convXI2D_reg(regD dst, rRegI src)
9951 %{
9952   predicate(UseXmmI2D);
9953   match(Set dst (ConvI2D src));
9954 
9955   format %{ "movdl $dst, $src\n\t"
9956             "cvtdq2pdl $dst, $dst\t# i2d" %}
9957   ins_encode %{
9958     __ movdl($dst$$XMMRegister, $src$$Register);
9959     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
9960   %}
9961   ins_pipe(pipe_slow); // XXX
9962 %}
9963 
9964 instruct convL2F_reg_reg(regF dst, rRegL src)
9965 %{
9966   match(Set dst (ConvL2F src));
9967 
9968   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
9969   ins_encode %{
9970     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
9971   %}
9972   ins_pipe(pipe_slow); // XXX
9973 %}
9974 
9975 instruct convL2F_reg_mem(regF dst, memory src)
9976 %{
9977   match(Set dst (ConvL2F (LoadL src)));
9978 
9979   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
9980   ins_encode %{
9981     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
9982   %}
9983   ins_pipe(pipe_slow); // XXX
9984 %}
9985 
9986 instruct convL2D_reg_reg(regD dst, rRegL src)
9987 %{
9988   match(Set dst (ConvL2D src));
9989 
9990   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
9991   ins_encode %{
9992     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
9993   %}
9994   ins_pipe(pipe_slow); // XXX
9995 %}
9996 
9997 instruct convL2D_reg_mem(regD dst, memory src)
9998 %{
9999   match(Set dst (ConvL2D (LoadL src)));
10000 
10001   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10002   ins_encode %{
10003     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10004   %}
10005   ins_pipe(pipe_slow); // XXX
10006 %}
10007 
10008 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10009 %{
10010   match(Set dst (ConvI2L src));
10011 
10012   ins_cost(125);
10013   format %{ "movslq  $dst, $src\t# i2l" %}
10014   ins_encode %{
10015     __ movslq($dst$$Register, $src$$Register);
10016   %}
10017   ins_pipe(ialu_reg_reg);
10018 %}
10019 
10020 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10021 // %{
10022 //   match(Set dst (ConvI2L src));
10023 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10024 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10025 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10026 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10027 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10028 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10029 
10030 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10031 //   ins_encode(enc_copy(dst, src));
10032 // //   opcode(0x63); // needs REX.W
10033 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10034 //   ins_pipe(ialu_reg_reg);
10035 // %}
10036 
10037 // Zero-extend convert int to long
10038 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10039 %{
10040   match(Set dst (AndL (ConvI2L src) mask));
10041 
10042   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10043   ins_encode %{
10044     if ($dst$$reg != $src$$reg) {
10045       __ movl($dst$$Register, $src$$Register);
10046     }
10047   %}
10048   ins_pipe(ialu_reg_reg);
10049 %}
10050 
10051 // Zero-extend convert int to long
10052 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10053 %{
10054   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10055 
10056   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10057   ins_encode %{
10058     __ movl($dst$$Register, $src$$Address);
10059   %}
10060   ins_pipe(ialu_reg_mem);
10061 %}
10062 
10063 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10064 %{
10065   match(Set dst (AndL src mask));
10066 
10067   format %{ "movl    $dst, $src\t# zero-extend long" %}
10068   ins_encode %{
10069     __ movl($dst$$Register, $src$$Register);
10070   %}
10071   ins_pipe(ialu_reg_reg);
10072 %}
10073 
10074 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10075 %{
10076   match(Set dst (ConvL2I src));
10077 
10078   format %{ "movl    $dst, $src\t# l2i" %}
10079   ins_encode %{
10080     __ movl($dst$$Register, $src$$Register);
10081   %}
10082   ins_pipe(ialu_reg_reg);
10083 %}
10084 
10085 
10086 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10087   match(Set dst (MoveF2I src));
10088   effect(DEF dst, USE src);
10089 
10090   ins_cost(125);
10091   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10092   ins_encode %{
10093     __ movl($dst$$Register, Address(rsp, $src$$disp));
10094   %}
10095   ins_pipe(ialu_reg_mem);
10096 %}
10097 
10098 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10099   match(Set dst (MoveI2F src));
10100   effect(DEF dst, USE src);
10101 
10102   ins_cost(125);
10103   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10104   ins_encode %{
10105     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10106   %}
10107   ins_pipe(pipe_slow);
10108 %}
10109 
10110 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10111   match(Set dst (MoveD2L src));
10112   effect(DEF dst, USE src);
10113 
10114   ins_cost(125);
10115   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10116   ins_encode %{
10117     __ movq($dst$$Register, Address(rsp, $src$$disp));
10118   %}
10119   ins_pipe(ialu_reg_mem);
10120 %}
10121 
10122 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10123   predicate(!UseXmmLoadAndClearUpper);
10124   match(Set dst (MoveL2D src));
10125   effect(DEF dst, USE src);
10126 
10127   ins_cost(125);
10128   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10129   ins_encode %{
10130     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10131   %}
10132   ins_pipe(pipe_slow);
10133 %}
10134 
10135 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10136   predicate(UseXmmLoadAndClearUpper);
10137   match(Set dst (MoveL2D src));
10138   effect(DEF dst, USE src);
10139 
10140   ins_cost(125);
10141   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10142   ins_encode %{
10143     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10144   %}
10145   ins_pipe(pipe_slow);
10146 %}
10147 
10148 
10149 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10150   match(Set dst (MoveF2I src));
10151   effect(DEF dst, USE src);
10152 
10153   ins_cost(95); // XXX
10154   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10155   ins_encode %{
10156     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10157   %}
10158   ins_pipe(pipe_slow);
10159 %}
10160 
10161 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10162   match(Set dst (MoveI2F src));
10163   effect(DEF dst, USE src);
10164 
10165   ins_cost(100);
10166   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10167   ins_encode %{
10168     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10169   %}
10170   ins_pipe( ialu_mem_reg );
10171 %}
10172 
10173 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10174   match(Set dst (MoveD2L src));
10175   effect(DEF dst, USE src);
10176 
10177   ins_cost(95); // XXX
10178   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10179   ins_encode %{
10180     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10181   %}
10182   ins_pipe(pipe_slow);
10183 %}
10184 
10185 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10186   match(Set dst (MoveL2D src));
10187   effect(DEF dst, USE src);
10188 
10189   ins_cost(100);
10190   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10191   ins_encode %{
10192     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10193   %}
10194   ins_pipe(ialu_mem_reg);
10195 %}
10196 
10197 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10198   match(Set dst (MoveF2I src));
10199   effect(DEF dst, USE src);
10200   ins_cost(85);
10201   format %{ "movd    $dst,$src\t# MoveF2I" %}
10202   ins_encode %{
10203     __ movdl($dst$$Register, $src$$XMMRegister);
10204   %}
10205   ins_pipe( pipe_slow );
10206 %}
10207 
10208 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10209   match(Set dst (MoveD2L src));
10210   effect(DEF dst, USE src);
10211   ins_cost(85);
10212   format %{ "movd    $dst,$src\t# MoveD2L" %}
10213   ins_encode %{
10214     __ movdq($dst$$Register, $src$$XMMRegister);
10215   %}
10216   ins_pipe( pipe_slow );
10217 %}
10218 
10219 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10220   match(Set dst (MoveI2F src));
10221   effect(DEF dst, USE src);
10222   ins_cost(100);
10223   format %{ "movd    $dst,$src\t# MoveI2F" %}
10224   ins_encode %{
10225     __ movdl($dst$$XMMRegister, $src$$Register);
10226   %}
10227   ins_pipe( pipe_slow );
10228 %}
10229 
10230 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10231   match(Set dst (MoveL2D src));
10232   effect(DEF dst, USE src);
10233   ins_cost(100);
10234   format %{ "movd    $dst,$src\t# MoveL2D" %}
10235   ins_encode %{
10236      __ movdq($dst$$XMMRegister, $src$$Register);
10237   %}
10238   ins_pipe( pipe_slow );
10239 %}
10240 
10241 
10242 // =======================================================================
10243 // fast clearing of an array
10244 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10245                   rFlagsReg cr)
10246 %{
10247   predicate(!UseFastStosb);
10248   match(Set dummy (ClearArray cnt base));
10249   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10250 
10251   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10252             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10253   ins_encode %{ 
10254     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10255   %}
10256   ins_pipe(pipe_slow);
10257 %}
10258 
10259 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10260                         rFlagsReg cr)
10261 %{
10262   predicate(UseFastStosb);
10263   match(Set dummy (ClearArray cnt base));
10264   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10265   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10266             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10267             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10268   ins_encode %{ 
10269     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10270   %}
10271   ins_pipe( pipe_slow );
10272 %}
10273 
10274 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10275                         rax_RegI result, regD tmp1, rFlagsReg cr)
10276 %{
10277   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10278   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10279 
10280   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10281   ins_encode %{
10282     __ string_compare($str1$$Register, $str2$$Register,
10283                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10284                       $tmp1$$XMMRegister);
10285   %}
10286   ins_pipe( pipe_slow );
10287 %}
10288 
10289 // fast search of substring with known size.
10290 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10291                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10292 %{
10293   predicate(UseSSE42Intrinsics);
10294   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10295   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10296 
10297   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10298   ins_encode %{
10299     int icnt2 = (int)$int_cnt2$$constant;
10300     if (icnt2 >= 8) {
10301       // IndexOf for constant substrings with size >= 8 elements
10302       // which don't need to be loaded through stack.
10303       __ string_indexofC8($str1$$Register, $str2$$Register,
10304                           $cnt1$$Register, $cnt2$$Register,
10305                           icnt2, $result$$Register,
10306                           $vec$$XMMRegister, $tmp$$Register);
10307     } else {
10308       // Small strings are loaded through stack if they cross page boundary.
10309       __ string_indexof($str1$$Register, $str2$$Register,
10310                         $cnt1$$Register, $cnt2$$Register,
10311                         icnt2, $result$$Register,
10312                         $vec$$XMMRegister, $tmp$$Register);
10313     }
10314   %}
10315   ins_pipe( pipe_slow );
10316 %}
10317 
10318 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10319                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10320 %{
10321   predicate(UseSSE42Intrinsics);
10322   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10323   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10324 
10325   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10326   ins_encode %{
10327     __ string_indexof($str1$$Register, $str2$$Register,
10328                       $cnt1$$Register, $cnt2$$Register,
10329                       (-1), $result$$Register,
10330                       $vec$$XMMRegister, $tmp$$Register);
10331   %}
10332   ins_pipe( pipe_slow );
10333 %}
10334 
10335 // fast string equals
10336 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10337                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10338 %{
10339   match(Set result (StrEquals (Binary str1 str2) cnt));
10340   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10341 
10342   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10343   ins_encode %{
10344     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10345                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10346                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10347   %}
10348   ins_pipe( pipe_slow );
10349 %}
10350 
10351 // fast array equals
10352 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10353                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10354 %{
10355   match(Set result (AryEq ary1 ary2));
10356   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10357   //ins_cost(300);
10358 
10359   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10360   ins_encode %{
10361     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10362                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10363                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10364   %}
10365   ins_pipe( pipe_slow );
10366 %}
10367 
10368 // encode char[] to byte[] in ISO_8859_1
10369 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10370                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10371                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10372   match(Set result (EncodeISOArray src (Binary dst len)));
10373   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10374 
10375   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10376   ins_encode %{
10377     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10378                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10379                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10380   %}
10381   ins_pipe( pipe_slow );
10382 %}
10383 
10384 //----------Overflow Math Instructions-----------------------------------------
10385 
10386 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10387 %{
10388   match(Set cr (OverflowAddI op1 op2));
10389   effect(DEF cr, USE_KILL op1, USE op2);
10390 
10391   format %{ "addl    $op1, $op2\t# overflow check int" %}
10392 
10393   ins_encode %{
10394     __ addl($op1$$Register, $op2$$Register);
10395   %}
10396   ins_pipe(ialu_reg_reg);
10397 %}
10398 
10399 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10400 %{
10401   match(Set cr (OverflowAddI op1 op2));
10402   effect(DEF cr, USE_KILL op1, USE op2);
10403 
10404   format %{ "addl    $op1, $op2\t# overflow check int" %}
10405 
10406   ins_encode %{
10407     __ addl($op1$$Register, $op2$$constant);
10408   %}
10409   ins_pipe(ialu_reg_reg);
10410 %}
10411 
10412 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10413 %{
10414   match(Set cr (OverflowAddL op1 op2));
10415   effect(DEF cr, USE_KILL op1, USE op2);
10416 
10417   format %{ "addq    $op1, $op2\t# overflow check long" %}
10418   ins_encode %{
10419     __ addq($op1$$Register, $op2$$Register);
10420   %}
10421   ins_pipe(ialu_reg_reg);
10422 %}
10423 
10424 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10425 %{
10426   match(Set cr (OverflowAddL op1 op2));
10427   effect(DEF cr, USE_KILL op1, USE op2);
10428 
10429   format %{ "addq    $op1, $op2\t# overflow check long" %}
10430   ins_encode %{
10431     __ addq($op1$$Register, $op2$$constant);
10432   %}
10433   ins_pipe(ialu_reg_reg);
10434 %}
10435 
10436 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10437 %{
10438   match(Set cr (OverflowSubI op1 op2));
10439 
10440   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10441   ins_encode %{
10442     __ cmpl($op1$$Register, $op2$$Register);
10443   %}
10444   ins_pipe(ialu_reg_reg);
10445 %}
10446 
10447 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10448 %{
10449   match(Set cr (OverflowSubI op1 op2));
10450 
10451   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10452   ins_encode %{
10453     __ cmpl($op1$$Register, $op2$$constant);
10454   %}
10455   ins_pipe(ialu_reg_reg);
10456 %}
10457 
10458 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10459 %{
10460   match(Set cr (OverflowSubL op1 op2));
10461 
10462   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10463   ins_encode %{
10464     __ cmpq($op1$$Register, $op2$$Register);
10465   %}
10466   ins_pipe(ialu_reg_reg);
10467 %}
10468 
10469 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10470 %{
10471   match(Set cr (OverflowSubL op1 op2));
10472 
10473   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10474   ins_encode %{
10475     __ cmpq($op1$$Register, $op2$$constant);
10476   %}
10477   ins_pipe(ialu_reg_reg);
10478 %}
10479 
10480 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10481 %{
10482   match(Set cr (OverflowSubI zero op2));
10483   effect(DEF cr, USE_KILL op2);
10484 
10485   format %{ "negl    $op2\t# overflow check int" %}
10486   ins_encode %{
10487     __ negl($op2$$Register);
10488   %}
10489   ins_pipe(ialu_reg_reg);
10490 %}
10491 
10492 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10493 %{
10494   match(Set cr (OverflowSubL zero op2));
10495   effect(DEF cr, USE_KILL op2);
10496 
10497   format %{ "negq    $op2\t# overflow check long" %}
10498   ins_encode %{
10499     __ negq($op2$$Register);
10500   %}
10501   ins_pipe(ialu_reg_reg);
10502 %}
10503 
10504 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10505 %{
10506   match(Set cr (OverflowMulI op1 op2));
10507   effect(DEF cr, USE_KILL op1, USE op2);
10508 
10509   format %{ "imull    $op1, $op2\t# overflow check int" %}
10510   ins_encode %{
10511     __ imull($op1$$Register, $op2$$Register);
10512   %}
10513   ins_pipe(ialu_reg_reg_alu0);
10514 %}
10515 
10516 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10517 %{
10518   match(Set cr (OverflowMulI op1 op2));
10519   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10520 
10521   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10522   ins_encode %{
10523     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10524   %}
10525   ins_pipe(ialu_reg_reg_alu0);
10526 %}
10527 
10528 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10529 %{
10530   match(Set cr (OverflowMulL op1 op2));
10531   effect(DEF cr, USE_KILL op1, USE op2);
10532 
10533   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10534   ins_encode %{
10535     __ imulq($op1$$Register, $op2$$Register);
10536   %}
10537   ins_pipe(ialu_reg_reg_alu0);
10538 %}
10539 
10540 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10541 %{
10542   match(Set cr (OverflowMulL op1 op2));
10543   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10544 
10545   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10546   ins_encode %{
10547     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10548   %}
10549   ins_pipe(ialu_reg_reg_alu0);
10550 %}
10551 
10552 
10553 //----------Control Flow Instructions------------------------------------------
10554 // Signed compare Instructions
10555 
10556 // XXX more variants!!
10557 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10558 %{
10559   match(Set cr (CmpI op1 op2));
10560   effect(DEF cr, USE op1, USE op2);
10561 
10562   format %{ "cmpl    $op1, $op2" %}
10563   opcode(0x3B);  /* Opcode 3B /r */
10564   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10565   ins_pipe(ialu_cr_reg_reg);
10566 %}
10567 
10568 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10569 %{
10570   match(Set cr (CmpI op1 op2));
10571 
10572   format %{ "cmpl    $op1, $op2" %}
10573   opcode(0x81, 0x07); /* Opcode 81 /7 */
10574   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10575   ins_pipe(ialu_cr_reg_imm);
10576 %}
10577 
10578 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10579 %{
10580   match(Set cr (CmpI op1 (LoadI op2)));
10581 
10582   ins_cost(500); // XXX
10583   format %{ "cmpl    $op1, $op2" %}
10584   opcode(0x3B); /* Opcode 3B /r */
10585   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10586   ins_pipe(ialu_cr_reg_mem);
10587 %}
10588 
10589 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10590 %{
10591   match(Set cr (CmpI src zero));
10592 
10593   format %{ "testl   $src, $src" %}
10594   opcode(0x85);
10595   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10596   ins_pipe(ialu_cr_reg_imm);
10597 %}
10598 
10599 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10600 %{
10601   match(Set cr (CmpI (AndI src con) zero));
10602 
10603   format %{ "testl   $src, $con" %}
10604   opcode(0xF7, 0x00);
10605   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10606   ins_pipe(ialu_cr_reg_imm);
10607 %}
10608 
10609 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10610 %{
10611   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10612 
10613   format %{ "testl   $src, $mem" %}
10614   opcode(0x85);
10615   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10616   ins_pipe(ialu_cr_reg_mem);
10617 %}
10618 
10619 // Unsigned compare Instructions; really, same as signed except they
10620 // produce an rFlagsRegU instead of rFlagsReg.
10621 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10622 %{
10623   match(Set cr (CmpU op1 op2));
10624 
10625   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10626   opcode(0x3B); /* Opcode 3B /r */
10627   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10628   ins_pipe(ialu_cr_reg_reg);
10629 %}
10630 
10631 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10632 %{
10633   match(Set cr (CmpU op1 op2));
10634 
10635   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10636   opcode(0x81,0x07); /* Opcode 81 /7 */
10637   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10638   ins_pipe(ialu_cr_reg_imm);
10639 %}
10640 
10641 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10642 %{
10643   match(Set cr (CmpU op1 (LoadI op2)));
10644 
10645   ins_cost(500); // XXX
10646   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10647   opcode(0x3B); /* Opcode 3B /r */
10648   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10649   ins_pipe(ialu_cr_reg_mem);
10650 %}
10651 
10652 // // // Cisc-spilled version of cmpU_rReg
10653 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10654 // //%{
10655 // //  match(Set cr (CmpU (LoadI op1) op2));
10656 // //
10657 // //  format %{ "CMPu   $op1,$op2" %}
10658 // //  ins_cost(500);
10659 // //  opcode(0x39);  /* Opcode 39 /r */
10660 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10661 // //%}
10662 
10663 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10664 %{
10665   match(Set cr (CmpU src zero));
10666 
10667   format %{ "testl  $src, $src\t# unsigned" %}
10668   opcode(0x85);
10669   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10670   ins_pipe(ialu_cr_reg_imm);
10671 %}
10672 
10673 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10674 %{
10675   match(Set cr (CmpP op1 op2));
10676 
10677   format %{ "cmpq    $op1, $op2\t# ptr" %}
10678   opcode(0x3B); /* Opcode 3B /r */
10679   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10680   ins_pipe(ialu_cr_reg_reg);
10681 %}
10682 
10683 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10684 %{
10685   match(Set cr (CmpP op1 (LoadP op2)));
10686 
10687   ins_cost(500); // XXX
10688   format %{ "cmpq    $op1, $op2\t# ptr" %}
10689   opcode(0x3B); /* Opcode 3B /r */
10690   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10691   ins_pipe(ialu_cr_reg_mem);
10692 %}
10693 
10694 // // // Cisc-spilled version of cmpP_rReg
10695 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10696 // //%{
10697 // //  match(Set cr (CmpP (LoadP op1) op2));
10698 // //
10699 // //  format %{ "CMPu   $op1,$op2" %}
10700 // //  ins_cost(500);
10701 // //  opcode(0x39);  /* Opcode 39 /r */
10702 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10703 // //%}
10704 
10705 // XXX this is generalized by compP_rReg_mem???
10706 // Compare raw pointer (used in out-of-heap check).
10707 // Only works because non-oop pointers must be raw pointers
10708 // and raw pointers have no anti-dependencies.
10709 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10710 %{
10711   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10712   match(Set cr (CmpP op1 (LoadP op2)));
10713 
10714   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10715   opcode(0x3B); /* Opcode 3B /r */
10716   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10717   ins_pipe(ialu_cr_reg_mem);
10718 %}
10719 
10720 // This will generate a signed flags result. This should be OK since
10721 // any compare to a zero should be eq/neq.
10722 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10723 %{
10724   match(Set cr (CmpP src zero));
10725 
10726   format %{ "testq   $src, $src\t# ptr" %}
10727   opcode(0x85);
10728   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10729   ins_pipe(ialu_cr_reg_imm);
10730 %}
10731 
10732 // This will generate a signed flags result. This should be OK since
10733 // any compare to a zero should be eq/neq.
10734 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10735 %{
10736   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10737   match(Set cr (CmpP (LoadP op) zero));
10738 
10739   ins_cost(500); // XXX
10740   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10741   opcode(0xF7); /* Opcode F7 /0 */
10742   ins_encode(REX_mem_wide(op),
10743              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10744   ins_pipe(ialu_cr_reg_imm);
10745 %}
10746 
10747 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10748 %{
10749   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10750   match(Set cr (CmpP (LoadP mem) zero));
10751 
10752   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10753   ins_encode %{
10754     __ cmpq(r12, $mem$$Address);
10755   %}
10756   ins_pipe(ialu_cr_reg_mem);
10757 %}
10758 
10759 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10760 %{
10761   match(Set cr (CmpN op1 op2));
10762 
10763   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10764   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10765   ins_pipe(ialu_cr_reg_reg);
10766 %}
10767 
10768 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10769 %{
10770   match(Set cr (CmpN src (LoadN mem)));
10771 
10772   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10773   ins_encode %{
10774     __ cmpl($src$$Register, $mem$$Address);
10775   %}
10776   ins_pipe(ialu_cr_reg_mem);
10777 %}
10778 
10779 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10780   match(Set cr (CmpN op1 op2));
10781 
10782   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10783   ins_encode %{
10784     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10785   %}
10786   ins_pipe(ialu_cr_reg_imm);
10787 %}
10788 
10789 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10790 %{
10791   match(Set cr (CmpN src (LoadN mem)));
10792 
10793   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10794   ins_encode %{
10795     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10796   %}
10797   ins_pipe(ialu_cr_reg_mem);
10798 %}
10799 
10800 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10801   match(Set cr (CmpN op1 op2));
10802 
10803   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10804   ins_encode %{
10805     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10806   %}
10807   ins_pipe(ialu_cr_reg_imm);
10808 %}
10809 
10810 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10811 %{
10812   match(Set cr (CmpN src (LoadNKlass mem)));
10813 
10814   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10815   ins_encode %{
10816     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10817   %}
10818   ins_pipe(ialu_cr_reg_mem);
10819 %}
10820 
10821 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10822   match(Set cr (CmpN src zero));
10823 
10824   format %{ "testl   $src, $src\t# compressed ptr" %}
10825   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10826   ins_pipe(ialu_cr_reg_imm);
10827 %}
10828 
10829 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10830 %{
10831   predicate(Universe::narrow_oop_base() != NULL);
10832   match(Set cr (CmpN (LoadN mem) zero));
10833 
10834   ins_cost(500); // XXX
10835   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10836   ins_encode %{
10837     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10838   %}
10839   ins_pipe(ialu_cr_reg_mem);
10840 %}
10841 
10842 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10843 %{
10844   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10845   match(Set cr (CmpN (LoadN mem) zero));
10846 
10847   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10848   ins_encode %{
10849     __ cmpl(r12, $mem$$Address);
10850   %}
10851   ins_pipe(ialu_cr_reg_mem);
10852 %}
10853 
10854 // Yanked all unsigned pointer compare operations.
10855 // Pointer compares are done with CmpP which is already unsigned.
10856 
10857 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10858 %{
10859   match(Set cr (CmpL op1 op2));
10860 
10861   format %{ "cmpq    $op1, $op2" %}
10862   opcode(0x3B);  /* Opcode 3B /r */
10863   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10864   ins_pipe(ialu_cr_reg_reg);
10865 %}
10866 
10867 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10868 %{
10869   match(Set cr (CmpL op1 op2));
10870 
10871   format %{ "cmpq    $op1, $op2" %}
10872   opcode(0x81, 0x07); /* Opcode 81 /7 */
10873   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
10874   ins_pipe(ialu_cr_reg_imm);
10875 %}
10876 
10877 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
10878 %{
10879   match(Set cr (CmpL op1 (LoadL op2)));
10880 
10881   format %{ "cmpq    $op1, $op2" %}
10882   opcode(0x3B); /* Opcode 3B /r */
10883   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10884   ins_pipe(ialu_cr_reg_mem);
10885 %}
10886 
10887 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
10888 %{
10889   match(Set cr (CmpL src zero));
10890 
10891   format %{ "testq   $src, $src" %}
10892   opcode(0x85);
10893   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10894   ins_pipe(ialu_cr_reg_imm);
10895 %}
10896 
10897 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
10898 %{
10899   match(Set cr (CmpL (AndL src con) zero));
10900 
10901   format %{ "testq   $src, $con\t# long" %}
10902   opcode(0xF7, 0x00);
10903   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
10904   ins_pipe(ialu_cr_reg_imm);
10905 %}
10906 
10907 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
10908 %{
10909   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
10910 
10911   format %{ "testq   $src, $mem" %}
10912   opcode(0x85);
10913   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
10914   ins_pipe(ialu_cr_reg_mem);
10915 %}
10916 
10917 // Manifest a CmpL result in an integer register.  Very painful.
10918 // This is the test to avoid.
10919 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
10920 %{
10921   match(Set dst (CmpL3 src1 src2));
10922   effect(KILL flags);
10923 
10924   ins_cost(275); // XXX
10925   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
10926             "movl    $dst, -1\n\t"
10927             "jl,s    done\n\t"
10928             "setne   $dst\n\t"
10929             "movzbl  $dst, $dst\n\t"
10930     "done:" %}
10931   ins_encode(cmpl3_flag(src1, src2, dst));
10932   ins_pipe(pipe_slow);
10933 %}
10934 
10935 //----------Max and Min--------------------------------------------------------
10936 // Min Instructions
10937 
10938 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
10939 %{
10940   effect(USE_DEF dst, USE src, USE cr);
10941 
10942   format %{ "cmovlgt $dst, $src\t# min" %}
10943   opcode(0x0F, 0x4F);
10944   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10945   ins_pipe(pipe_cmov_reg);
10946 %}
10947 
10948 
10949 instruct minI_rReg(rRegI dst, rRegI src)
10950 %{
10951   match(Set dst (MinI dst src));
10952 
10953   ins_cost(200);
10954   expand %{
10955     rFlagsReg cr;
10956     compI_rReg(cr, dst, src);
10957     cmovI_reg_g(dst, src, cr);
10958   %}
10959 %}
10960 
10961 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
10962 %{
10963   effect(USE_DEF dst, USE src, USE cr);
10964 
10965   format %{ "cmovllt $dst, $src\t# max" %}
10966   opcode(0x0F, 0x4C);
10967   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10968   ins_pipe(pipe_cmov_reg);
10969 %}
10970 
10971 
10972 instruct maxI_rReg(rRegI dst, rRegI src)
10973 %{
10974   match(Set dst (MaxI dst src));
10975 
10976   ins_cost(200);
10977   expand %{
10978     rFlagsReg cr;
10979     compI_rReg(cr, dst, src);
10980     cmovI_reg_l(dst, src, cr);
10981   %}
10982 %}
10983 
10984 // ============================================================================
10985 // Branch Instructions
10986 
10987 // Jump Direct - Label defines a relative address from JMP+1
10988 instruct jmpDir(label labl)
10989 %{
10990   match(Goto);
10991   effect(USE labl);
10992 
10993   ins_cost(300);
10994   format %{ "jmp     $labl" %}
10995   size(5);
10996   ins_encode %{
10997     Label* L = $labl$$label;
10998     __ jmp(*L, false); // Always long jump
10999   %}
11000   ins_pipe(pipe_jmp);
11001 %}
11002 
11003 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11004 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11005 %{
11006   match(If cop cr);
11007   effect(USE labl);
11008 
11009   ins_cost(300);
11010   format %{ "j$cop     $labl" %}
11011   size(6);
11012   ins_encode %{
11013     Label* L = $labl$$label;
11014     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11015   %}
11016   ins_pipe(pipe_jcc);
11017 %}
11018 
11019 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11020 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11021 %{
11022   match(CountedLoopEnd cop cr);
11023   effect(USE labl);
11024 
11025   ins_cost(300);
11026   format %{ "j$cop     $labl\t# loop end" %}
11027   size(6);
11028   ins_encode %{
11029     Label* L = $labl$$label;
11030     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11031   %}
11032   ins_pipe(pipe_jcc);
11033 %}
11034 
11035 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11036 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11037   match(CountedLoopEnd cop cmp);
11038   effect(USE labl);
11039 
11040   ins_cost(300);
11041   format %{ "j$cop,u   $labl\t# loop end" %}
11042   size(6);
11043   ins_encode %{
11044     Label* L = $labl$$label;
11045     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11046   %}
11047   ins_pipe(pipe_jcc);
11048 %}
11049 
11050 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11051   match(CountedLoopEnd cop cmp);
11052   effect(USE labl);
11053 
11054   ins_cost(200);
11055   format %{ "j$cop,u   $labl\t# loop end" %}
11056   size(6);
11057   ins_encode %{
11058     Label* L = $labl$$label;
11059     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11060   %}
11061   ins_pipe(pipe_jcc);
11062 %}
11063 
11064 // Jump Direct Conditional - using unsigned comparison
11065 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11066   match(If cop cmp);
11067   effect(USE labl);
11068 
11069   ins_cost(300);
11070   format %{ "j$cop,u  $labl" %}
11071   size(6);
11072   ins_encode %{
11073     Label* L = $labl$$label;
11074     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11075   %}
11076   ins_pipe(pipe_jcc);
11077 %}
11078 
11079 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11080   match(If cop cmp);
11081   effect(USE labl);
11082 
11083   ins_cost(200);
11084   format %{ "j$cop,u  $labl" %}
11085   size(6);
11086   ins_encode %{
11087     Label* L = $labl$$label;
11088     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11089   %}
11090   ins_pipe(pipe_jcc);
11091 %}
11092 
11093 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11094   match(If cop cmp);
11095   effect(USE labl);
11096 
11097   ins_cost(200);
11098   format %{ $$template
11099     if ($cop$$cmpcode == Assembler::notEqual) {
11100       $$emit$$"jp,u   $labl\n\t"
11101       $$emit$$"j$cop,u   $labl"
11102     } else {
11103       $$emit$$"jp,u   done\n\t"
11104       $$emit$$"j$cop,u   $labl\n\t"
11105       $$emit$$"done:"
11106     }
11107   %}
11108   ins_encode %{
11109     Label* l = $labl$$label;
11110     if ($cop$$cmpcode == Assembler::notEqual) {
11111       __ jcc(Assembler::parity, *l, false);
11112       __ jcc(Assembler::notEqual, *l, false);
11113     } else if ($cop$$cmpcode == Assembler::equal) {
11114       Label done;
11115       __ jccb(Assembler::parity, done);
11116       __ jcc(Assembler::equal, *l, false);
11117       __ bind(done);
11118     } else {
11119        ShouldNotReachHere();
11120     }
11121   %}
11122   ins_pipe(pipe_jcc);
11123 %}
11124 
11125 // ============================================================================
11126 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11127 // superklass array for an instance of the superklass.  Set a hidden
11128 // internal cache on a hit (cache is checked with exposed code in
11129 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11130 // encoding ALSO sets flags.
11131 
11132 instruct partialSubtypeCheck(rdi_RegP result,
11133                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11134                              rFlagsReg cr)
11135 %{
11136   match(Set result (PartialSubtypeCheck sub super));
11137   effect(KILL rcx, KILL cr);
11138 
11139   ins_cost(1100);  // slightly larger than the next version
11140   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11141             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11142             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11143             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11144             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11145             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11146             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11147     "miss:\t" %}
11148 
11149   opcode(0x1); // Force a XOR of RDI
11150   ins_encode(enc_PartialSubtypeCheck());
11151   ins_pipe(pipe_slow);
11152 %}
11153 
11154 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11155                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11156                                      immP0 zero,
11157                                      rdi_RegP result)
11158 %{
11159   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11160   effect(KILL rcx, KILL result);
11161 
11162   ins_cost(1000);
11163   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11164             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11165             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11166             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11167             "jne,s   miss\t\t# Missed: flags nz\n\t"
11168             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11169     "miss:\t" %}
11170 
11171   opcode(0x0); // No need to XOR RDI
11172   ins_encode(enc_PartialSubtypeCheck());
11173   ins_pipe(pipe_slow);
11174 %}
11175 
11176 // ============================================================================
11177 // Branch Instructions -- short offset versions
11178 //
11179 // These instructions are used to replace jumps of a long offset (the default
11180 // match) with jumps of a shorter offset.  These instructions are all tagged
11181 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11182 // match rules in general matching.  Instead, the ADLC generates a conversion
11183 // method in the MachNode which can be used to do in-place replacement of the
11184 // long variant with the shorter variant.  The compiler will determine if a
11185 // branch can be taken by the is_short_branch_offset() predicate in the machine
11186 // specific code section of the file.
11187 
11188 // Jump Direct - Label defines a relative address from JMP+1
11189 instruct jmpDir_short(label labl) %{
11190   match(Goto);
11191   effect(USE labl);
11192 
11193   ins_cost(300);
11194   format %{ "jmp,s   $labl" %}
11195   size(2);
11196   ins_encode %{
11197     Label* L = $labl$$label;
11198     __ jmpb(*L);
11199   %}
11200   ins_pipe(pipe_jmp);
11201   ins_short_branch(1);
11202 %}
11203 
11204 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11205 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11206   match(If cop cr);
11207   effect(USE labl);
11208 
11209   ins_cost(300);
11210   format %{ "j$cop,s   $labl" %}
11211   size(2);
11212   ins_encode %{
11213     Label* L = $labl$$label;
11214     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11215   %}
11216   ins_pipe(pipe_jcc);
11217   ins_short_branch(1);
11218 %}
11219 
11220 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11221 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11222   match(CountedLoopEnd cop cr);
11223   effect(USE labl);
11224 
11225   ins_cost(300);
11226   format %{ "j$cop,s   $labl\t# loop end" %}
11227   size(2);
11228   ins_encode %{
11229     Label* L = $labl$$label;
11230     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11231   %}
11232   ins_pipe(pipe_jcc);
11233   ins_short_branch(1);
11234 %}
11235 
11236 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11237 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11238   match(CountedLoopEnd cop cmp);
11239   effect(USE labl);
11240 
11241   ins_cost(300);
11242   format %{ "j$cop,us  $labl\t# loop end" %}
11243   size(2);
11244   ins_encode %{
11245     Label* L = $labl$$label;
11246     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11247   %}
11248   ins_pipe(pipe_jcc);
11249   ins_short_branch(1);
11250 %}
11251 
11252 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11253   match(CountedLoopEnd cop cmp);
11254   effect(USE labl);
11255 
11256   ins_cost(300);
11257   format %{ "j$cop,us  $labl\t# loop end" %}
11258   size(2);
11259   ins_encode %{
11260     Label* L = $labl$$label;
11261     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11262   %}
11263   ins_pipe(pipe_jcc);
11264   ins_short_branch(1);
11265 %}
11266 
11267 // Jump Direct Conditional - using unsigned comparison
11268 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11269   match(If cop cmp);
11270   effect(USE labl);
11271 
11272   ins_cost(300);
11273   format %{ "j$cop,us  $labl" %}
11274   size(2);
11275   ins_encode %{
11276     Label* L = $labl$$label;
11277     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11278   %}
11279   ins_pipe(pipe_jcc);
11280   ins_short_branch(1);
11281 %}
11282 
11283 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11284   match(If cop cmp);
11285   effect(USE labl);
11286 
11287   ins_cost(300);
11288   format %{ "j$cop,us  $labl" %}
11289   size(2);
11290   ins_encode %{
11291     Label* L = $labl$$label;
11292     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11293   %}
11294   ins_pipe(pipe_jcc);
11295   ins_short_branch(1);
11296 %}
11297 
11298 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11299   match(If cop cmp);
11300   effect(USE labl);
11301 
11302   ins_cost(300);
11303   format %{ $$template
11304     if ($cop$$cmpcode == Assembler::notEqual) {
11305       $$emit$$"jp,u,s   $labl\n\t"
11306       $$emit$$"j$cop,u,s   $labl"
11307     } else {
11308       $$emit$$"jp,u,s   done\n\t"
11309       $$emit$$"j$cop,u,s  $labl\n\t"
11310       $$emit$$"done:"
11311     }
11312   %}
11313   size(4);
11314   ins_encode %{
11315     Label* l = $labl$$label;
11316     if ($cop$$cmpcode == Assembler::notEqual) {
11317       __ jccb(Assembler::parity, *l);
11318       __ jccb(Assembler::notEqual, *l);
11319     } else if ($cop$$cmpcode == Assembler::equal) {
11320       Label done;
11321       __ jccb(Assembler::parity, done);
11322       __ jccb(Assembler::equal, *l);
11323       __ bind(done);
11324     } else {
11325        ShouldNotReachHere();
11326     }
11327   %}
11328   ins_pipe(pipe_jcc);
11329   ins_short_branch(1);
11330 %}
11331 
11332 // ============================================================================
11333 // inlined locking and unlocking
11334 
11335 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11336   predicate(Compile::current()->use_rtm());
11337   match(Set cr (FastLock object box));
11338   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11339   ins_cost(300);
11340   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11341   ins_encode %{
11342     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11343                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11344                  _counters, _rtm_counters, _stack_rtm_counters,
11345                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11346                  true, ra_->C->profile_rtm());
11347   %}
11348   ins_pipe(pipe_slow);
11349 %}
11350 
11351 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11352   predicate(!Compile::current()->use_rtm());
11353   match(Set cr (FastLock object box));
11354   effect(TEMP tmp, TEMP scr, USE_KILL box);
11355   ins_cost(300);
11356   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11357   ins_encode %{
11358     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11359                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11360   %}
11361   ins_pipe(pipe_slow);
11362 %}
11363 
11364 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11365   match(Set cr (FastUnlock object box));
11366   effect(TEMP tmp, USE_KILL box);
11367   ins_cost(300);
11368   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11369   ins_encode %{
11370     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11371   %}
11372   ins_pipe(pipe_slow);
11373 %}
11374 
11375 
11376 // ============================================================================
11377 // Safepoint Instructions
11378 instruct safePoint_poll(rFlagsReg cr)
11379 %{
11380   predicate(!Assembler::is_polling_page_far());
11381   match(SafePoint);
11382   effect(KILL cr);
11383 
11384   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11385             "# Safepoint: poll for GC" %}
11386   ins_cost(125);
11387   ins_encode %{
11388     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11389     __ testl(rax, addr);
11390   %}
11391   ins_pipe(ialu_reg_mem);
11392 %}
11393 
11394 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11395 %{
11396   predicate(Assembler::is_polling_page_far());
11397   match(SafePoint poll);
11398   effect(KILL cr, USE poll);
11399 
11400   format %{ "testl  rax, [$poll]\t"
11401             "# Safepoint: poll for GC" %}
11402   ins_cost(125);
11403   ins_encode %{
11404     __ relocate(relocInfo::poll_type);
11405     __ testl(rax, Address($poll$$Register, 0));
11406   %}
11407   ins_pipe(ialu_reg_mem);
11408 %}
11409 
11410 // ============================================================================
11411 // Procedure Call/Return Instructions
11412 // Call Java Static Instruction
11413 // Note: If this code changes, the corresponding ret_addr_offset() and
11414 //       compute_padding() functions will have to be adjusted.
11415 instruct CallStaticJavaDirect(method meth) %{
11416   match(CallStaticJava);
11417   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
11418   effect(USE meth);
11419 
11420   ins_cost(300);
11421   format %{ "call,static " %}
11422   opcode(0xE8); /* E8 cd */
11423   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11424   ins_pipe(pipe_slow);
11425   ins_alignment(4);
11426 %}
11427 
11428 // Call Java Static Instruction (method handle version)
11429 // Note: If this code changes, the corresponding ret_addr_offset() and
11430 //       compute_padding() functions will have to be adjusted.
11431 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
11432   match(CallStaticJava);
11433   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
11434   effect(USE meth);
11435   // RBP is saved by all callees (for interpreter stack correction).
11436   // We use it here for a similar purpose, in {preserve,restore}_SP.
11437 
11438   ins_cost(300);
11439   format %{ "call,static/MethodHandle " %}
11440   opcode(0xE8); /* E8 cd */
11441   ins_encode(clear_avx, preserve_SP,
11442              Java_Static_Call(meth),
11443              restore_SP,
11444              call_epilog);
11445   ins_pipe(pipe_slow);
11446   ins_alignment(4);
11447 %}
11448 
11449 // Call Java Dynamic Instruction
11450 // Note: If this code changes, the corresponding ret_addr_offset() and
11451 //       compute_padding() functions will have to be adjusted.
11452 instruct CallDynamicJavaDirect(method meth)
11453 %{
11454   match(CallDynamicJava);
11455   effect(USE meth);
11456 
11457   ins_cost(300);
11458   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11459             "call,dynamic " %}
11460   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11461   ins_pipe(pipe_slow);
11462   ins_alignment(4);
11463 %}
11464 
11465 // Call Runtime Instruction
11466 instruct CallRuntimeDirect(method meth)
11467 %{
11468   match(CallRuntime);
11469   effect(USE meth);
11470 
11471   ins_cost(300);
11472   format %{ "call,runtime " %}
11473   ins_encode(clear_avx, Java_To_Runtime(meth));
11474   ins_pipe(pipe_slow);
11475 %}
11476 
11477 // Call runtime without safepoint
11478 instruct CallLeafDirect(method meth)
11479 %{
11480   match(CallLeaf);
11481   effect(USE meth);
11482 
11483   ins_cost(300);
11484   format %{ "call_leaf,runtime " %}
11485   ins_encode(clear_avx, Java_To_Runtime(meth));
11486   ins_pipe(pipe_slow);
11487 %}
11488 
11489 // Call runtime without safepoint
11490 instruct CallLeafNoFPDirect(method meth)
11491 %{
11492   match(CallLeafNoFP);
11493   effect(USE meth);
11494 
11495   ins_cost(300);
11496   format %{ "call_leaf_nofp,runtime " %}
11497   ins_encode(Java_To_Runtime(meth));
11498   ins_pipe(pipe_slow);
11499 %}
11500 
11501 // Return Instruction
11502 // Remove the return address & jump to it.
11503 // Notice: We always emit a nop after a ret to make sure there is room
11504 // for safepoint patching
11505 instruct Ret()
11506 %{
11507   match(Return);
11508 
11509   format %{ "ret" %}
11510   opcode(0xC3);
11511   ins_encode(OpcP);
11512   ins_pipe(pipe_jmp);
11513 %}
11514 
11515 // Tail Call; Jump from runtime stub to Java code.
11516 // Also known as an 'interprocedural jump'.
11517 // Target of jump will eventually return to caller.
11518 // TailJump below removes the return address.
11519 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11520 %{
11521   match(TailCall jump_target method_oop);
11522 
11523   ins_cost(300);
11524   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11525   opcode(0xFF, 0x4); /* Opcode FF /4 */
11526   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11527   ins_pipe(pipe_jmp);
11528 %}
11529 
11530 // Tail Jump; remove the return address; jump to target.
11531 // TailCall above leaves the return address around.
11532 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11533 %{
11534   match(TailJump jump_target ex_oop);
11535 
11536   ins_cost(300);
11537   format %{ "popq    rdx\t# pop return address\n\t"
11538             "jmp     $jump_target" %}
11539   opcode(0xFF, 0x4); /* Opcode FF /4 */
11540   ins_encode(Opcode(0x5a), // popq rdx
11541              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11542   ins_pipe(pipe_jmp);
11543 %}
11544 
11545 // Create exception oop: created by stack-crawling runtime code.
11546 // Created exception is now available to this handler, and is setup
11547 // just prior to jumping to this handler.  No code emitted.
11548 instruct CreateException(rax_RegP ex_oop)
11549 %{
11550   match(Set ex_oop (CreateEx));
11551 
11552   size(0);
11553   // use the following format syntax
11554   format %{ "# exception oop is in rax; no code emitted" %}
11555   ins_encode();
11556   ins_pipe(empty);
11557 %}
11558 
11559 // Rethrow exception:
11560 // The exception oop will come in the first argument position.
11561 // Then JUMP (not call) to the rethrow stub code.
11562 instruct RethrowException()
11563 %{
11564   match(Rethrow);
11565 
11566   // use the following format syntax
11567   format %{ "jmp     rethrow_stub" %}
11568   ins_encode(enc_rethrow);
11569   ins_pipe(pipe_jmp);
11570 %}
11571 
11572 
11573 // ============================================================================
11574 // This name is KNOWN by the ADLC and cannot be changed.
11575 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11576 // for this guy.
11577 instruct tlsLoadP(r15_RegP dst) %{
11578   match(Set dst (ThreadLocal));
11579   effect(DEF dst);
11580 
11581   size(0);
11582   format %{ "# TLS is in R15" %}
11583   ins_encode( /*empty encoding*/ );
11584   ins_pipe(ialu_reg_reg);
11585 %}
11586 
11587 
11588 //----------PEEPHOLE RULES-----------------------------------------------------
11589 // These must follow all instruction definitions as they use the names
11590 // defined in the instructions definitions.
11591 //
11592 // peepmatch ( root_instr_name [preceding_instruction]* );
11593 //
11594 // peepconstraint %{
11595 // (instruction_number.operand_name relational_op instruction_number.operand_name
11596 //  [, ...] );
11597 // // instruction numbers are zero-based using left to right order in peepmatch
11598 //
11599 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11600 // // provide an instruction_number.operand_name for each operand that appears
11601 // // in the replacement instruction's match rule
11602 //
11603 // ---------VM FLAGS---------------------------------------------------------
11604 //
11605 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11606 //
11607 // Each peephole rule is given an identifying number starting with zero and
11608 // increasing by one in the order seen by the parser.  An individual peephole
11609 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11610 // on the command-line.
11611 //
11612 // ---------CURRENT LIMITATIONS----------------------------------------------
11613 //
11614 // Only match adjacent instructions in same basic block
11615 // Only equality constraints
11616 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11617 // Only one replacement instruction
11618 //
11619 // ---------EXAMPLE----------------------------------------------------------
11620 //
11621 // // pertinent parts of existing instructions in architecture description
11622 // instruct movI(rRegI dst, rRegI src)
11623 // %{
11624 //   match(Set dst (CopyI src));
11625 // %}
11626 //
11627 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11628 // %{
11629 //   match(Set dst (AddI dst src));
11630 //   effect(KILL cr);
11631 // %}
11632 //
11633 // // Change (inc mov) to lea
11634 // peephole %{
11635 //   // increment preceeded by register-register move
11636 //   peepmatch ( incI_rReg movI );
11637 //   // require that the destination register of the increment
11638 //   // match the destination register of the move
11639 //   peepconstraint ( 0.dst == 1.dst );
11640 //   // construct a replacement instruction that sets
11641 //   // the destination to ( move's source register + one )
11642 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11643 // %}
11644 //
11645 
11646 // Implementation no longer uses movX instructions since
11647 // machine-independent system no longer uses CopyX nodes.
11648 //
11649 // peephole
11650 // %{
11651 //   peepmatch (incI_rReg movI);
11652 //   peepconstraint (0.dst == 1.dst);
11653 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11654 // %}
11655 
11656 // peephole
11657 // %{
11658 //   peepmatch (decI_rReg movI);
11659 //   peepconstraint (0.dst == 1.dst);
11660 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11661 // %}
11662 
11663 // peephole
11664 // %{
11665 //   peepmatch (addI_rReg_imm movI);
11666 //   peepconstraint (0.dst == 1.dst);
11667 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11668 // %}
11669 
11670 // peephole
11671 // %{
11672 //   peepmatch (incL_rReg movL);
11673 //   peepconstraint (0.dst == 1.dst);
11674 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11675 // %}
11676 
11677 // peephole
11678 // %{
11679 //   peepmatch (decL_rReg movL);
11680 //   peepconstraint (0.dst == 1.dst);
11681 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11682 // %}
11683 
11684 // peephole
11685 // %{
11686 //   peepmatch (addL_rReg_imm movL);
11687 //   peepconstraint (0.dst == 1.dst);
11688 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11689 // %}
11690 
11691 // peephole
11692 // %{
11693 //   peepmatch (addP_rReg_imm movP);
11694 //   peepconstraint (0.dst == 1.dst);
11695 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11696 // %}
11697 
11698 // // Change load of spilled value to only a spill
11699 // instruct storeI(memory mem, rRegI src)
11700 // %{
11701 //   match(Set mem (StoreI mem src));
11702 // %}
11703 //
11704 // instruct loadI(rRegI dst, memory mem)
11705 // %{
11706 //   match(Set dst (LoadI mem));
11707 // %}
11708 //
11709 
11710 peephole
11711 %{
11712   peepmatch (loadI storeI);
11713   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11714   peepreplace (storeI(1.mem 1.mem 1.src));
11715 %}
11716 
11717 peephole
11718 %{
11719   peepmatch (loadL storeL);
11720   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11721   peepreplace (storeL(1.mem 1.mem 1.src));
11722 %}
11723 
11724 //----------SMARTSPILL RULES---------------------------------------------------
11725 // These must follow all instruction definitions as they use the names
11726 // defined in the instructions definitions.