1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Class for all pointer registers (including RSP)
 170 reg_class any_reg(RAX, RAX_H,
 171                   RDX, RDX_H,
 172                   RBP, RBP_H,
 173                   RDI, RDI_H,
 174                   RSI, RSI_H,
 175                   RCX, RCX_H,
 176                   RBX, RBX_H,
 177                   RSP, RSP_H,
 178                   R8,  R8_H,
 179                   R9,  R9_H,
 180                   R10, R10_H,
 181                   R11, R11_H,
 182                   R12, R12_H,
 183                   R13, R13_H,
 184                   R14, R14_H,
 185                   R15, R15_H);
 186 
 187 // Class for all pointer registers except RSP
 188 reg_class ptr_reg(RAX, RAX_H,
 189                   RDX, RDX_H,
 190                   RBP, RBP_H,
 191                   RDI, RDI_H,
 192                   RSI, RSI_H,
 193                   RCX, RCX_H,
 194                   RBX, RBX_H,
 195                   R8,  R8_H,
 196                   R9,  R9_H,
 197                   R10, R10_H,
 198                   R11, R11_H,
 199                   R13, R13_H,
 200                   R14, R14_H);
 201 
 202 // Class for all pointer registers except RAX and RSP
 203 reg_class ptr_no_rax_reg(RDX, RDX_H,
 204                          RBP, RBP_H,
 205                          RDI, RDI_H,
 206                          RSI, RSI_H,
 207                          RCX, RCX_H,
 208                          RBX, RBX_H,
 209                          R8,  R8_H,
 210                          R9,  R9_H,
 211                          R10, R10_H,
 212                          R11, R11_H,
 213                          R13, R13_H,
 214                          R14, R14_H);
 215 
 216 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 217                          RAX, RAX_H,
 218                          RDI, RDI_H,
 219                          RSI, RSI_H,
 220                          RCX, RCX_H,
 221                          RBX, RBX_H,
 222                          R8,  R8_H,
 223                          R9,  R9_H,
 224                          R10, R10_H,
 225                          R11, R11_H,
 226                          R13, R13_H,
 227                          R14, R14_H);
 228 
 229 // Class for all pointer registers except RAX, RBX and RSP
 230 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 231                              RBP, RBP_H,
 232                              RDI, RDI_H,
 233                              RSI, RSI_H,
 234                              RCX, RCX_H,
 235                              R8,  R8_H,
 236                              R9,  R9_H,
 237                              R10, R10_H,
 238                              R11, R11_H,
 239                              R13, R13_H,
 240                              R14, R14_H);
 241 
 242 // Singleton class for RAX pointer register
 243 reg_class ptr_rax_reg(RAX, RAX_H);
 244 
 245 // Singleton class for RBX pointer register
 246 reg_class ptr_rbx_reg(RBX, RBX_H);
 247 
 248 // Singleton class for RSI pointer register
 249 reg_class ptr_rsi_reg(RSI, RSI_H);
 250 
 251 // Singleton class for RDI pointer register
 252 reg_class ptr_rdi_reg(RDI, RDI_H);
 253 
 254 // Singleton class for RBP pointer register
 255 reg_class ptr_rbp_reg(RBP, RBP_H);
 256 
 257 // Singleton class for stack pointer
 258 reg_class ptr_rsp_reg(RSP, RSP_H);
 259 
 260 // Singleton class for TLS pointer
 261 reg_class ptr_r15_reg(R15, R15_H);
 262 
 263 // Class for all long registers (except RSP)
 264 reg_class long_reg(RAX, RAX_H,
 265                    RDX, RDX_H,
 266                    RBP, RBP_H,
 267                    RDI, RDI_H,
 268                    RSI, RSI_H,
 269                    RCX, RCX_H,
 270                    RBX, RBX_H,
 271                    R8,  R8_H,
 272                    R9,  R9_H,
 273                    R10, R10_H,
 274                    R11, R11_H,
 275                    R13, R13_H,
 276                    R14, R14_H);
 277 
 278 // Class for all long registers except RAX, RDX (and RSP)
 279 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 280                               RDI, RDI_H,
 281                               RSI, RSI_H,
 282                               RCX, RCX_H,
 283                               RBX, RBX_H,
 284                               R8,  R8_H,
 285                               R9,  R9_H,
 286                               R10, R10_H,
 287                               R11, R11_H,
 288                               R13, R13_H,
 289                               R14, R14_H);
 290 
 291 // Class for all long registers except RCX (and RSP)
 292 reg_class long_no_rcx_reg(RBP, RBP_H,
 293                           RDI, RDI_H,
 294                           RSI, RSI_H,
 295                           RAX, RAX_H,
 296                           RDX, RDX_H,
 297                           RBX, RBX_H,
 298                           R8,  R8_H,
 299                           R9,  R9_H,
 300                           R10, R10_H,
 301                           R11, R11_H,
 302                           R13, R13_H,
 303                           R14, R14_H);
 304 
 305 // Class for all long registers except RAX (and RSP)
 306 reg_class long_no_rax_reg(RBP, RBP_H,
 307                           RDX, RDX_H,
 308                           RDI, RDI_H,
 309                           RSI, RSI_H,
 310                           RCX, RCX_H,
 311                           RBX, RBX_H,
 312                           R8,  R8_H,
 313                           R9,  R9_H,
 314                           R10, R10_H,
 315                           R11, R11_H,
 316                           R13, R13_H,
 317                           R14, R14_H);
 318 
 319 // Singleton class for RAX long register
 320 reg_class long_rax_reg(RAX, RAX_H);
 321 
 322 // Singleton class for RCX long register
 323 reg_class long_rcx_reg(RCX, RCX_H);
 324 
 325 // Singleton class for RDX long register
 326 reg_class long_rdx_reg(RDX, RDX_H);
 327 
 328 // Class for all int registers (except RSP)
 329 reg_class int_reg(RAX,
 330                   RDX,
 331                   RBP,
 332                   RDI,
 333                   RSI,
 334                   RCX,
 335                   RBX,
 336                   R8,
 337                   R9,
 338                   R10,
 339                   R11,
 340                   R13,
 341                   R14);
 342 
 343 // Class for all int registers except RCX (and RSP)
 344 reg_class int_no_rcx_reg(RAX,
 345                          RDX,
 346                          RBP,
 347                          RDI,
 348                          RSI,
 349                          RBX,
 350                          R8,
 351                          R9,
 352                          R10,
 353                          R11,
 354                          R13,
 355                          R14);
 356 
 357 // Class for all int registers except RAX, RDX (and RSP)
 358 reg_class int_no_rax_rdx_reg(RBP,
 359                              RDI,
 360                              RSI,
 361                              RCX,
 362                              RBX,
 363                              R8,
 364                              R9,
 365                              R10,
 366                              R11,
 367                              R13,
 368                              R14);
 369 
 370 // Singleton class for RAX int register
 371 reg_class int_rax_reg(RAX);
 372 
 373 // Singleton class for RBX int register
 374 reg_class int_rbx_reg(RBX);
 375 
 376 // Singleton class for RCX int register
 377 reg_class int_rcx_reg(RCX);
 378 
 379 // Singleton class for RCX int register
 380 reg_class int_rdx_reg(RDX);
 381 
 382 // Singleton class for RCX int register
 383 reg_class int_rdi_reg(RDI);
 384 
 385 // Singleton class for instruction pointer
 386 // reg_class ip_reg(RIP);
 387 
 388 %}
 389 
 390 //----------SOURCE BLOCK-------------------------------------------------------
 391 // This is a block of C++ code which provides values, functions, and
 392 // definitions necessary in the rest of the architecture description
 393 source %{
 394 #define   RELOC_IMM64    Assembler::imm_operand
 395 #define   RELOC_DISP32   Assembler::disp32_operand
 396 
 397 #define __ _masm.
 398 
 399 static int preserve_SP_size() {
 400   return 3;  // rex.w, op, rm(reg/reg)
 401 }
 402 static int clear_avx_size() {
 403   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 404 }
 405 
 406 // !!!!! Special hack to get all types of calls to specify the byte offset
 407 //       from the start of the call to the point where the return address
 408 //       will point.
 409 int MachCallStaticJavaNode::ret_addr_offset()
 410 {
 411   int offset = 5; // 5 bytes from start of call to where return address points
 412   offset += clear_avx_size();
 413   if (_method_handle_invoke)
 414     offset += preserve_SP_size();
 415   return offset;
 416 }
 417 
 418 int MachCallDynamicJavaNode::ret_addr_offset()
 419 {
 420   int offset = 15; // 15 bytes from start of call to where return address points
 421   offset += clear_avx_size();
 422   return offset;
 423 }
 424 
 425 int MachCallRuntimeNode::ret_addr_offset() {
 426   int offset = 13; // movq r10,#addr; callq (r10)
 427   offset += clear_avx_size();
 428   return offset;
 429 }
 430 
 431 // Indicate if the safepoint node needs the polling page as an input,
 432 // it does if the polling page is more than disp32 away.
 433 bool SafePointNode::needs_polling_address_input()
 434 {
 435   return Assembler::is_polling_page_far();
 436 }
 437 
 438 //
 439 // Compute padding required for nodes which need alignment
 440 //
 441 
 442 // The address of the call instruction needs to be 4-byte aligned to
 443 // ensure that it does not span a cache line so that it can be patched.
 444 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 445 {
 446   current_offset += clear_avx_size(); // skip vzeroupper
 447   current_offset += 1; // skip call opcode byte
 448   return round_to(current_offset, alignment_required()) - current_offset;
 449 }
 450 
 451 // The address of the call instruction needs to be 4-byte aligned to
 452 // ensure that it does not span a cache line so that it can be patched.
 453 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 454 {
 455   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 456   current_offset += clear_avx_size(); // skip vzeroupper
 457   current_offset += 1; // skip call opcode byte
 458   return round_to(current_offset, alignment_required()) - current_offset;
 459 }
 460 
 461 // The address of the call instruction needs to be 4-byte aligned to
 462 // ensure that it does not span a cache line so that it can be patched.
 463 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 464 {
 465   current_offset += clear_avx_size(); // skip vzeroupper
 466   current_offset += 11; // skip movq instruction + call opcode byte
 467   return round_to(current_offset, alignment_required()) - current_offset;
 468 }
 469 
 470 // EMIT_RM()
 471 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 472   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 473   cbuf.insts()->emit_int8(c);
 474 }
 475 
 476 // EMIT_CC()
 477 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 478   unsigned char c = (unsigned char) (f1 | f2);
 479   cbuf.insts()->emit_int8(c);
 480 }
 481 
 482 // EMIT_OPCODE()
 483 void emit_opcode(CodeBuffer &cbuf, int code) {
 484   cbuf.insts()->emit_int8((unsigned char) code);
 485 }
 486 
 487 // EMIT_OPCODE() w/ relocation information
 488 void emit_opcode(CodeBuffer &cbuf,
 489                  int code, relocInfo::relocType reloc, int offset, int format)
 490 {
 491   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 492   emit_opcode(cbuf, code);
 493 }
 494 
 495 // EMIT_D8()
 496 void emit_d8(CodeBuffer &cbuf, int d8) {
 497   cbuf.insts()->emit_int8((unsigned char) d8);
 498 }
 499 
 500 // EMIT_D16()
 501 void emit_d16(CodeBuffer &cbuf, int d16) {
 502   cbuf.insts()->emit_int16(d16);
 503 }
 504 
 505 // EMIT_D32()
 506 void emit_d32(CodeBuffer &cbuf, int d32) {
 507   cbuf.insts()->emit_int32(d32);
 508 }
 509 
 510 // EMIT_D64()
 511 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 512   cbuf.insts()->emit_int64(d64);
 513 }
 514 
 515 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 516 void emit_d32_reloc(CodeBuffer& cbuf,
 517                     int d32,
 518                     relocInfo::relocType reloc,
 519                     int format)
 520 {
 521   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 522   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 523   cbuf.insts()->emit_int32(d32);
 524 }
 525 
 526 // emit 32 bit value and construct relocation entry from RelocationHolder
 527 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 528 #ifdef ASSERT
 529   if (rspec.reloc()->type() == relocInfo::oop_type &&
 530       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 531     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 532     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 533   }
 534 #endif
 535   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 536   cbuf.insts()->emit_int32(d32);
 537 }
 538 
 539 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 540   address next_ip = cbuf.insts_end() + 4;
 541   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 542                  external_word_Relocation::spec(addr),
 543                  RELOC_DISP32);
 544 }
 545 
 546 
 547 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 548 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 549   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 550   cbuf.insts()->emit_int64(d64);
 551 }
 552 
 553 // emit 64 bit value and construct relocation entry from RelocationHolder
 554 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 555 #ifdef ASSERT
 556   if (rspec.reloc()->type() == relocInfo::oop_type &&
 557       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 558     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 559     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 560            "cannot embed scavengable oops in code");
 561   }
 562 #endif
 563   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 564   cbuf.insts()->emit_int64(d64);
 565 }
 566 
 567 // Access stack slot for load or store
 568 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 569 {
 570   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 571   if (-0x80 <= disp && disp < 0x80) {
 572     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 573     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 574     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 575   } else {
 576     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 577     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 578     emit_d32(cbuf, disp);     // Displacement // R/M byte
 579   }
 580 }
 581 
 582    // rRegI ereg, memory mem) %{    // emit_reg_mem
 583 void encode_RegMem(CodeBuffer &cbuf,
 584                    int reg,
 585                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 586 {
 587   assert(disp_reloc == relocInfo::none, "cannot have disp");
 588   int regenc = reg & 7;
 589   int baseenc = base & 7;
 590   int indexenc = index & 7;
 591 
 592   // There is no index & no scale, use form without SIB byte
 593   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 594     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 595     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 596       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 597     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 598       // If 8-bit displacement, mode 0x1
 599       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 600       emit_d8(cbuf, disp);
 601     } else {
 602       // If 32-bit displacement
 603       if (base == -1) { // Special flag for absolute address
 604         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 605         if (disp_reloc != relocInfo::none) {
 606           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 607         } else {
 608           emit_d32(cbuf, disp);
 609         }
 610       } else {
 611         // Normal base + offset
 612         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 613         if (disp_reloc != relocInfo::none) {
 614           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 615         } else {
 616           emit_d32(cbuf, disp);
 617         }
 618       }
 619     }
 620   } else {
 621     // Else, encode with the SIB byte
 622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 624       // If no displacement
 625       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 626       emit_rm(cbuf, scale, indexenc, baseenc);
 627     } else {
 628       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 629         // If 8-bit displacement, mode 0x1
 630         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 631         emit_rm(cbuf, scale, indexenc, baseenc);
 632         emit_d8(cbuf, disp);
 633       } else {
 634         // If 32-bit displacement
 635         if (base == 0x04 ) {
 636           emit_rm(cbuf, 0x2, regenc, 0x4);
 637           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 638         } else {
 639           emit_rm(cbuf, 0x2, regenc, 0x4);
 640           emit_rm(cbuf, scale, indexenc, baseenc); // *
 641         }
 642         if (disp_reloc != relocInfo::none) {
 643           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 644         } else {
 645           emit_d32(cbuf, disp);
 646         }
 647       }
 648     }
 649   }
 650 }
 651 
 652 // This could be in MacroAssembler but it's fairly C2 specific
 653 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 654   Label exit;
 655   __ jccb(Assembler::noParity, exit);
 656   __ pushf();
 657   //
 658   // comiss/ucomiss instructions set ZF,PF,CF flags and
 659   // zero OF,AF,SF for NaN values.
 660   // Fixup flags by zeroing ZF,PF so that compare of NaN
 661   // values returns 'less than' result (CF is set).
 662   // Leave the rest of flags unchanged.
 663   //
 664   //    7 6 5 4 3 2 1 0
 665   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 666   //    0 0 1 0 1 0 1 1   (0x2B)
 667   //
 668   __ andq(Address(rsp, 0), 0xffffff2b);
 669   __ popf();
 670   __ bind(exit);
 671 }
 672 
 673 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 674   Label done;
 675   __ movl(dst, -1);
 676   __ jcc(Assembler::parity, done);
 677   __ jcc(Assembler::below, done);
 678   __ setb(Assembler::notEqual, dst);
 679   __ movzbl(dst, dst);
 680   __ bind(done);
 681 }
 682 
 683 
 684 //=============================================================================
 685 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 686 
 687 int Compile::ConstantTable::calculate_table_base_offset() const {
 688   return 0;  // absolute addressing, no offset
 689 }
 690 
 691 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 692 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 693   ShouldNotReachHere();
 694 }
 695 
 696 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 697   // Empty encoding
 698 }
 699 
 700 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 701   return 0;
 702 }
 703 
 704 #ifndef PRODUCT
 705 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 706   st->print("# MachConstantBaseNode (empty encoding)");
 707 }
 708 #endif
 709 
 710 
 711 //=============================================================================
 712 #ifndef PRODUCT
 713 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 714   Compile* C = ra_->C;
 715 
 716   int framesize = C->frame_size_in_bytes();
 717   int bangsize = C->bang_size_in_bytes();
 718   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 719   // Remove wordSize for return addr which is already pushed.
 720   framesize -= wordSize;
 721 
 722   if (C->need_stack_bang(bangsize)) {
 723     framesize -= wordSize;
 724     st->print("# stack bang (%d bytes)", bangsize);
 725     st->print("\n\t");
 726     st->print("pushq   rbp\t# Save rbp");
 727     if (framesize) {
 728       st->print("\n\t");
 729       st->print("subq    rsp, #%d\t# Create frame",framesize);
 730     }
 731   } else {
 732     st->print("subq    rsp, #%d\t# Create frame",framesize);
 733     st->print("\n\t");
 734     framesize -= wordSize;
 735     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 736   }
 737 
 738   if (VerifyStackAtCalls) {
 739     st->print("\n\t");
 740     framesize -= wordSize;
 741     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 742 #ifdef ASSERT
 743     st->print("\n\t");
 744     st->print("# stack alignment check");
 745 #endif
 746   }
 747   st->cr();
 748 }
 749 #endif
 750 
 751 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 752   Compile* C = ra_->C;
 753   MacroAssembler _masm(&cbuf);
 754 
 755   int framesize = C->frame_size_in_bytes();
 756   int bangsize = C->bang_size_in_bytes();
 757 
 758   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 759 
 760   C->set_frame_complete(cbuf.insts_size());
 761 
 762   if (C->has_mach_constant_base_node()) {
 763     // NOTE: We set the table base offset here because users might be
 764     // emitted before MachConstantBaseNode.
 765     Compile::ConstantTable& constant_table = C->constant_table();
 766     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 767   }
 768 }
 769 
 770 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 771 {
 772   return MachNode::size(ra_); // too many variables; just compute it
 773                               // the hard way
 774 }
 775 
 776 int MachPrologNode::reloc() const
 777 {
 778   return 0; // a large enough number
 779 }
 780 
 781 //=============================================================================
 782 #ifndef PRODUCT
 783 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 784 {
 785   Compile* C = ra_->C;
 786   if (C->max_vector_size() > 16) {
 787     st->print("vzeroupper");
 788     st->cr(); st->print("\t");
 789   }
 790 
 791   int framesize = C->frame_size_in_bytes();
 792   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 793   // Remove word for return adr already pushed
 794   // and RBP
 795   framesize -= 2*wordSize;
 796 
 797   if (framesize) {
 798     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 799     st->print("\t");
 800   }
 801 
 802   st->print_cr("popq   rbp");
 803   if (do_polling() && C->is_method_compilation()) {
 804     st->print("\t");
 805     if (Assembler::is_polling_page_far()) {
 806       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 807                    "testl  rax, [rscratch1]\t"
 808                    "# Safepoint: poll for GC");
 809     } else {
 810       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 811                    "# Safepoint: poll for GC");
 812     }
 813   }
 814 }
 815 #endif
 816 
 817 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 818 {
 819   Compile* C = ra_->C;
 820   if (C->max_vector_size() > 16) {
 821     // Clear upper bits of YMM registers when current compiled code uses
 822     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 823     MacroAssembler _masm(&cbuf);
 824     __ vzeroupper();
 825   }
 826 
 827   int framesize = C->frame_size_in_bytes();
 828   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 829   // Remove word for return adr already pushed
 830   // and RBP
 831   framesize -= 2*wordSize;
 832 
 833   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 834 
 835   if (framesize) {
 836     emit_opcode(cbuf, Assembler::REX_W);
 837     if (framesize < 0x80) {
 838       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 839       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 840       emit_d8(cbuf, framesize);
 841     } else {
 842       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 843       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 844       emit_d32(cbuf, framesize);
 845     }
 846   }
 847 
 848   // popq rbp
 849   emit_opcode(cbuf, 0x58 | RBP_enc);
 850 
 851   if (do_polling() && C->is_method_compilation()) {
 852     MacroAssembler _masm(&cbuf);
 853     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 854     if (Assembler::is_polling_page_far()) {
 855       __ lea(rscratch1, polling_page);
 856       __ relocate(relocInfo::poll_return_type);
 857       __ testl(rax, Address(rscratch1, 0));
 858     } else {
 859       __ testl(rax, polling_page);
 860     }
 861   }
 862 }
 863 
 864 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 865 {
 866   return MachNode::size(ra_); // too many variables; just compute it
 867                               // the hard way
 868 }
 869 
 870 int MachEpilogNode::reloc() const
 871 {
 872   return 2; // a large enough number
 873 }
 874 
 875 const Pipeline* MachEpilogNode::pipeline() const
 876 {
 877   return MachNode::pipeline_class();
 878 }
 879 
 880 int MachEpilogNode::safepoint_offset() const
 881 {
 882   return 0;
 883 }
 884 
 885 //=============================================================================
 886 
 887 enum RC {
 888   rc_bad,
 889   rc_int,
 890   rc_float,
 891   rc_stack
 892 };
 893 
 894 static enum RC rc_class(OptoReg::Name reg)
 895 {
 896   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 897 
 898   if (OptoReg::is_stack(reg)) return rc_stack;
 899 
 900   VMReg r = OptoReg::as_VMReg(reg);
 901 
 902   if (r->is_Register()) return rc_int;
 903 
 904   assert(r->is_XMMRegister(), "must be");
 905   return rc_float;
 906 }
 907 
 908 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 909 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 910                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 911 
 912 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 913                             int stack_offset, int reg, uint ireg, outputStream* st);
 914 
 915 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 916                                       int dst_offset, uint ireg, outputStream* st) {
 917   if (cbuf) {
 918     MacroAssembler _masm(cbuf);
 919     switch (ireg) {
 920     case Op_VecS:
 921       __ movq(Address(rsp, -8), rax);
 922       __ movl(rax, Address(rsp, src_offset));
 923       __ movl(Address(rsp, dst_offset), rax);
 924       __ movq(rax, Address(rsp, -8));
 925       break;
 926     case Op_VecD:
 927       __ pushq(Address(rsp, src_offset));
 928       __ popq (Address(rsp, dst_offset));
 929       break;
 930     case Op_VecX:
 931       __ pushq(Address(rsp, src_offset));
 932       __ popq (Address(rsp, dst_offset));
 933       __ pushq(Address(rsp, src_offset+8));
 934       __ popq (Address(rsp, dst_offset+8));
 935       break;
 936     case Op_VecY:
 937       __ vmovdqu(Address(rsp, -32), xmm0);
 938       __ vmovdqu(xmm0, Address(rsp, src_offset));
 939       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 940       __ vmovdqu(xmm0, Address(rsp, -32));
 941       break;
 942     default:
 943       ShouldNotReachHere();
 944     }
 945 #ifndef PRODUCT
 946   } else {
 947     switch (ireg) {
 948     case Op_VecS:
 949       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 950                 "movl    rax, [rsp + #%d]\n\t"
 951                 "movl    [rsp + #%d], rax\n\t"
 952                 "movq    rax, [rsp - #8]",
 953                 src_offset, dst_offset);
 954       break;
 955     case Op_VecD:
 956       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 957                 "popq    [rsp + #%d]",
 958                 src_offset, dst_offset);
 959       break;
 960      case Op_VecX:
 961       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 962                 "popq    [rsp + #%d]\n\t"
 963                 "pushq   [rsp + #%d]\n\t"
 964                 "popq    [rsp + #%d]",
 965                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 966       break;
 967     case Op_VecY:
 968       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 969                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 970                 "vmovdqu [rsp + #%d], xmm0\n\t"
 971                 "vmovdqu xmm0, [rsp - #32]",
 972                 src_offset, dst_offset);
 973       break;
 974     default:
 975       ShouldNotReachHere();
 976     }
 977 #endif
 978   }
 979 }
 980 
 981 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 982                                        PhaseRegAlloc* ra_,
 983                                        bool do_size,
 984                                        outputStream* st) const {
 985   assert(cbuf != NULL || st  != NULL, "sanity");
 986   // Get registers to move
 987   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 988   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 989   OptoReg::Name dst_second = ra_->get_reg_second(this);
 990   OptoReg::Name dst_first = ra_->get_reg_first(this);
 991 
 992   enum RC src_second_rc = rc_class(src_second);
 993   enum RC src_first_rc = rc_class(src_first);
 994   enum RC dst_second_rc = rc_class(dst_second);
 995   enum RC dst_first_rc = rc_class(dst_first);
 996 
 997   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 998          "must move at least 1 register" );
 999 
1000   if (src_first == dst_first && src_second == dst_second) {
1001     // Self copy, no move
1002     return 0;
1003   }
1004   if (bottom_type()->isa_vect() != NULL) {
1005     uint ireg = ideal_reg();
1006     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1007     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1008     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1009       // mem -> mem
1010       int src_offset = ra_->reg2offset(src_first);
1011       int dst_offset = ra_->reg2offset(dst_first);
1012       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1013     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1014       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1015     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1016       int stack_offset = ra_->reg2offset(dst_first);
1017       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1018     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1019       int stack_offset = ra_->reg2offset(src_first);
1020       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1021     } else {
1022       ShouldNotReachHere();
1023     }
1024     return 0;
1025   }
1026   if (src_first_rc == rc_stack) {
1027     // mem ->
1028     if (dst_first_rc == rc_stack) {
1029       // mem -> mem
1030       assert(src_second != dst_first, "overlap");
1031       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1032           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1033         // 64-bit
1034         int src_offset = ra_->reg2offset(src_first);
1035         int dst_offset = ra_->reg2offset(dst_first);
1036         if (cbuf) {
1037           MacroAssembler _masm(cbuf);
1038           __ pushq(Address(rsp, src_offset));
1039           __ popq (Address(rsp, dst_offset));
1040 #ifndef PRODUCT
1041         } else {
1042           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1043                     "popq    [rsp + #%d]",
1044                      src_offset, dst_offset);
1045 #endif
1046         }
1047       } else {
1048         // 32-bit
1049         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1050         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1051         // No pushl/popl, so:
1052         int src_offset = ra_->reg2offset(src_first);
1053         int dst_offset = ra_->reg2offset(dst_first);
1054         if (cbuf) {
1055           MacroAssembler _masm(cbuf);
1056           __ movq(Address(rsp, -8), rax);
1057           __ movl(rax, Address(rsp, src_offset));
1058           __ movl(Address(rsp, dst_offset), rax);
1059           __ movq(rax, Address(rsp, -8));
1060 #ifndef PRODUCT
1061         } else {
1062           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1063                     "movl    rax, [rsp + #%d]\n\t"
1064                     "movl    [rsp + #%d], rax\n\t"
1065                     "movq    rax, [rsp - #8]",
1066                      src_offset, dst_offset);
1067 #endif
1068         }
1069       }
1070       return 0;
1071     } else if (dst_first_rc == rc_int) {
1072       // mem -> gpr
1073       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1074           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1075         // 64-bit
1076         int offset = ra_->reg2offset(src_first);
1077         if (cbuf) {
1078           MacroAssembler _masm(cbuf);
1079           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1080 #ifndef PRODUCT
1081         } else {
1082           st->print("movq    %s, [rsp + #%d]\t# spill",
1083                      Matcher::regName[dst_first],
1084                      offset);
1085 #endif
1086         }
1087       } else {
1088         // 32-bit
1089         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1090         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1091         int offset = ra_->reg2offset(src_first);
1092         if (cbuf) {
1093           MacroAssembler _masm(cbuf);
1094           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1095 #ifndef PRODUCT
1096         } else {
1097           st->print("movl    %s, [rsp + #%d]\t# spill",
1098                      Matcher::regName[dst_first],
1099                      offset);
1100 #endif
1101         }
1102       }
1103       return 0;
1104     } else if (dst_first_rc == rc_float) {
1105       // mem-> xmm
1106       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1107           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1108         // 64-bit
1109         int offset = ra_->reg2offset(src_first);
1110         if (cbuf) {
1111           MacroAssembler _masm(cbuf);
1112           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1113 #ifndef PRODUCT
1114         } else {
1115           st->print("%s  %s, [rsp + #%d]\t# spill",
1116                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1117                      Matcher::regName[dst_first],
1118                      offset);
1119 #endif
1120         }
1121       } else {
1122         // 32-bit
1123         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1124         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1125         int offset = ra_->reg2offset(src_first);
1126         if (cbuf) {
1127           MacroAssembler _masm(cbuf);
1128           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1129 #ifndef PRODUCT
1130         } else {
1131           st->print("movss   %s, [rsp + #%d]\t# spill",
1132                      Matcher::regName[dst_first],
1133                      offset);
1134 #endif
1135         }
1136       }
1137       return 0;
1138     }
1139   } else if (src_first_rc == rc_int) {
1140     // gpr ->
1141     if (dst_first_rc == rc_stack) {
1142       // gpr -> mem
1143       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1144           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1145         // 64-bit
1146         int offset = ra_->reg2offset(dst_first);
1147         if (cbuf) {
1148           MacroAssembler _masm(cbuf);
1149           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1150 #ifndef PRODUCT
1151         } else {
1152           st->print("movq    [rsp + #%d], %s\t# spill",
1153                      offset,
1154                      Matcher::regName[src_first]);
1155 #endif
1156         }
1157       } else {
1158         // 32-bit
1159         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1160         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1161         int offset = ra_->reg2offset(dst_first);
1162         if (cbuf) {
1163           MacroAssembler _masm(cbuf);
1164           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1165 #ifndef PRODUCT
1166         } else {
1167           st->print("movl    [rsp + #%d], %s\t# spill",
1168                      offset,
1169                      Matcher::regName[src_first]);
1170 #endif
1171         }
1172       }
1173       return 0;
1174     } else if (dst_first_rc == rc_int) {
1175       // gpr -> gpr
1176       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1177           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1178         // 64-bit
1179         if (cbuf) {
1180           MacroAssembler _masm(cbuf);
1181           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1182                   as_Register(Matcher::_regEncode[src_first]));
1183 #ifndef PRODUCT
1184         } else {
1185           st->print("movq    %s, %s\t# spill",
1186                      Matcher::regName[dst_first],
1187                      Matcher::regName[src_first]);
1188 #endif
1189         }
1190         return 0;
1191       } else {
1192         // 32-bit
1193         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1194         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1195         if (cbuf) {
1196           MacroAssembler _masm(cbuf);
1197           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1198                   as_Register(Matcher::_regEncode[src_first]));
1199 #ifndef PRODUCT
1200         } else {
1201           st->print("movl    %s, %s\t# spill",
1202                      Matcher::regName[dst_first],
1203                      Matcher::regName[src_first]);
1204 #endif
1205         }
1206         return 0;
1207       }
1208     } else if (dst_first_rc == rc_float) {
1209       // gpr -> xmm
1210       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1211           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1212         // 64-bit
1213         if (cbuf) {
1214           MacroAssembler _masm(cbuf);
1215           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1216 #ifndef PRODUCT
1217         } else {
1218           st->print("movdq   %s, %s\t# spill",
1219                      Matcher::regName[dst_first],
1220                      Matcher::regName[src_first]);
1221 #endif
1222         }
1223       } else {
1224         // 32-bit
1225         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1226         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1227         if (cbuf) {
1228           MacroAssembler _masm(cbuf);
1229           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1230 #ifndef PRODUCT
1231         } else {
1232           st->print("movdl   %s, %s\t# spill",
1233                      Matcher::regName[dst_first],
1234                      Matcher::regName[src_first]);
1235 #endif
1236         }
1237       }
1238       return 0;
1239     }
1240   } else if (src_first_rc == rc_float) {
1241     // xmm ->
1242     if (dst_first_rc == rc_stack) {
1243       // xmm -> mem
1244       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1245           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1246         // 64-bit
1247         int offset = ra_->reg2offset(dst_first);
1248         if (cbuf) {
1249           MacroAssembler _masm(cbuf);
1250           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1251 #ifndef PRODUCT
1252         } else {
1253           st->print("movsd   [rsp + #%d], %s\t# spill",
1254                      offset,
1255                      Matcher::regName[src_first]);
1256 #endif
1257         }
1258       } else {
1259         // 32-bit
1260         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1261         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1262         int offset = ra_->reg2offset(dst_first);
1263         if (cbuf) {
1264           MacroAssembler _masm(cbuf);
1265           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("movss   [rsp + #%d], %s\t# spill",
1269                      offset,
1270                      Matcher::regName[src_first]);
1271 #endif
1272         }
1273       }
1274       return 0;
1275     } else if (dst_first_rc == rc_int) {
1276       // xmm -> gpr
1277       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1278           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1279         // 64-bit
1280         if (cbuf) {
1281           MacroAssembler _masm(cbuf);
1282           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1283 #ifndef PRODUCT
1284         } else {
1285           st->print("movdq   %s, %s\t# spill",
1286                      Matcher::regName[dst_first],
1287                      Matcher::regName[src_first]);
1288 #endif
1289         }
1290       } else {
1291         // 32-bit
1292         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1293         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1294         if (cbuf) {
1295           MacroAssembler _masm(cbuf);
1296           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1297 #ifndef PRODUCT
1298         } else {
1299           st->print("movdl   %s, %s\t# spill",
1300                      Matcher::regName[dst_first],
1301                      Matcher::regName[src_first]);
1302 #endif
1303         }
1304       }
1305       return 0;
1306     } else if (dst_first_rc == rc_float) {
1307       // xmm -> xmm
1308       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1309           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1310         // 64-bit
1311         if (cbuf) {
1312           MacroAssembler _masm(cbuf);
1313           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1314 #ifndef PRODUCT
1315         } else {
1316           st->print("%s  %s, %s\t# spill",
1317                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1318                      Matcher::regName[dst_first],
1319                      Matcher::regName[src_first]);
1320 #endif
1321         }
1322       } else {
1323         // 32-bit
1324         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1325         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1326         if (cbuf) {
1327           MacroAssembler _masm(cbuf);
1328           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1329 #ifndef PRODUCT
1330         } else {
1331           st->print("%s  %s, %s\t# spill",
1332                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1333                      Matcher::regName[dst_first],
1334                      Matcher::regName[src_first]);
1335 #endif
1336         }
1337       }
1338       return 0;
1339     }
1340   }
1341 
1342   assert(0," foo ");
1343   Unimplemented();
1344   return 0;
1345 }
1346 
1347 #ifndef PRODUCT
1348 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1349   implementation(NULL, ra_, false, st);
1350 }
1351 #endif
1352 
1353 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1354   implementation(&cbuf, ra_, false, NULL);
1355 }
1356 
1357 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1358   return MachNode::size(ra_);
1359 }
1360 
1361 //=============================================================================
1362 #ifndef PRODUCT
1363 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1364 {
1365   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1366   int reg = ra_->get_reg_first(this);
1367   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1368             Matcher::regName[reg], offset);
1369 }
1370 #endif
1371 
1372 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1373 {
1374   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1375   int reg = ra_->get_encode(this);
1376   if (offset >= 0x80) {
1377     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1378     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1379     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1380     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1381     emit_d32(cbuf, offset);
1382   } else {
1383     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1384     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1385     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1386     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1387     emit_d8(cbuf, offset);
1388   }
1389 }
1390 
1391 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1392 {
1393   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1394   return (offset < 0x80) ? 5 : 8; // REX
1395 }
1396 
1397 //=============================================================================
1398 #ifndef PRODUCT
1399 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1400 {
1401   if (UseCompressedClassPointers) {
1402     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1403     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1404     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1405   } else {
1406     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1407                  "# Inline cache check");
1408   }
1409   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1410   st->print_cr("\tnop\t# nops to align entry point");
1411 }
1412 #endif
1413 
1414 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1415 {
1416   MacroAssembler masm(&cbuf);
1417   uint insts_size = cbuf.insts_size();
1418   if (UseCompressedClassPointers) {
1419     masm.load_klass(rscratch1, j_rarg0);
1420     masm.cmpptr(rax, rscratch1);
1421   } else {
1422     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1423   }
1424 
1425   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1426 
1427   /* WARNING these NOPs are critical so that verified entry point is properly
1428      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1429   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1430   if (OptoBreakpoint) {
1431     // Leave space for int3
1432     nops_cnt -= 1;
1433   }
1434   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1435   if (nops_cnt > 0)
1436     masm.nop(nops_cnt);
1437 }
1438 
1439 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1440 {
1441   return MachNode::size(ra_); // too many variables; just compute it
1442                               // the hard way
1443 }
1444  
1445 
1446 //=============================================================================
1447 
1448 int Matcher::regnum_to_fpu_offset(int regnum)
1449 {
1450   return regnum - 32; // The FP registers are in the second chunk
1451 }
1452 
1453 // This is UltraSparc specific, true just means we have fast l2f conversion
1454 const bool Matcher::convL2FSupported(void) {
1455   return true;
1456 }
1457 
1458 // Is this branch offset short enough that a short branch can be used?
1459 //
1460 // NOTE: If the platform does not provide any short branch variants, then
1461 //       this method should return false for offset 0.
1462 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1463   // The passed offset is relative to address of the branch.
1464   // On 86 a branch displacement is calculated relative to address
1465   // of a next instruction.
1466   offset -= br_size;
1467 
1468   // the short version of jmpConUCF2 contains multiple branches,
1469   // making the reach slightly less
1470   if (rule == jmpConUCF2_rule)
1471     return (-126 <= offset && offset <= 125);
1472   return (-128 <= offset && offset <= 127);
1473 }
1474 
1475 const bool Matcher::isSimpleConstant64(jlong value) {
1476   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1477   //return value == (int) value;  // Cf. storeImmL and immL32.
1478 
1479   // Probably always true, even if a temp register is required.
1480   return true;
1481 }
1482 
1483 // The ecx parameter to rep stosq for the ClearArray node is in words.
1484 const bool Matcher::init_array_count_is_in_bytes = false;
1485 
1486 // Threshold size for cleararray.
1487 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1488 
1489 // No additional cost for CMOVL.
1490 const int Matcher::long_cmove_cost() { return 0; }
1491 
1492 // No CMOVF/CMOVD with SSE2
1493 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1494 
1495 // Does the CPU require late expand (see block.cpp for description of late expand)?
1496 const bool Matcher::require_postalloc_expand = false;
1497 
1498 // Should the Matcher clone shifts on addressing modes, expecting them
1499 // to be subsumed into complex addressing expressions or compute them
1500 // into registers?  True for Intel but false for most RISCs
1501 const bool Matcher::clone_shift_expressions = true;
1502 
1503 // Do we need to mask the count passed to shift instructions or does
1504 // the cpu only look at the lower 5/6 bits anyway?
1505 const bool Matcher::need_masked_shift_count = false;
1506 
1507 bool Matcher::narrow_oop_use_complex_address() {
1508   assert(UseCompressedOops, "only for compressed oops code");
1509   return (LogMinObjAlignmentInBytes <= 3);
1510 }
1511 
1512 bool Matcher::narrow_klass_use_complex_address() {
1513   assert(UseCompressedClassPointers, "only for compressed klass code");
1514   return (LogKlassAlignmentInBytes <= 3);
1515 }
1516 
1517 // Is it better to copy float constants, or load them directly from
1518 // memory?  Intel can load a float constant from a direct address,
1519 // requiring no extra registers.  Most RISCs will have to materialize
1520 // an address into a register first, so they would do better to copy
1521 // the constant from stack.
1522 const bool Matcher::rematerialize_float_constants = true; // XXX
1523 
1524 // If CPU can load and store mis-aligned doubles directly then no
1525 // fixup is needed.  Else we split the double into 2 integer pieces
1526 // and move it piece-by-piece.  Only happens when passing doubles into
1527 // C code as the Java calling convention forces doubles to be aligned.
1528 const bool Matcher::misaligned_doubles_ok = true;
1529 
1530 // No-op on amd64
1531 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1532 
1533 // Advertise here if the CPU requires explicit rounding operations to
1534 // implement the UseStrictFP mode.
1535 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1536 
1537 // Are floats conerted to double when stored to stack during deoptimization?
1538 // On x64 it is stored without convertion so we can use normal access.
1539 bool Matcher::float_in_double() { return false; }
1540 
1541 // Do ints take an entire long register or just half?
1542 const bool Matcher::int_in_long = true;
1543 
1544 // Return whether or not this register is ever used as an argument.
1545 // This function is used on startup to build the trampoline stubs in
1546 // generateOptoStub.  Registers not mentioned will be killed by the VM
1547 // call in the trampoline, and arguments in those registers not be
1548 // available to the callee.
1549 bool Matcher::can_be_java_arg(int reg)
1550 {
1551   return
1552     reg ==  RDI_num || reg == RDI_H_num ||
1553     reg ==  RSI_num || reg == RSI_H_num ||
1554     reg ==  RDX_num || reg == RDX_H_num ||
1555     reg ==  RCX_num || reg == RCX_H_num ||
1556     reg ==   R8_num || reg ==  R8_H_num ||
1557     reg ==   R9_num || reg ==  R9_H_num ||
1558     reg ==  R12_num || reg == R12_H_num ||
1559     reg == XMM0_num || reg == XMM0b_num ||
1560     reg == XMM1_num || reg == XMM1b_num ||
1561     reg == XMM2_num || reg == XMM2b_num ||
1562     reg == XMM3_num || reg == XMM3b_num ||
1563     reg == XMM4_num || reg == XMM4b_num ||
1564     reg == XMM5_num || reg == XMM5b_num ||
1565     reg == XMM6_num || reg == XMM6b_num ||
1566     reg == XMM7_num || reg == XMM7b_num;
1567 }
1568 
1569 bool Matcher::is_spillable_arg(int reg)
1570 {
1571   return can_be_java_arg(reg);
1572 }
1573 
1574 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1575   // In 64 bit mode a code which use multiply when
1576   // devisor is constant is faster than hardware
1577   // DIV instruction (it uses MulHiL).
1578   return false;
1579 }
1580 
1581 // Register for DIVI projection of divmodI
1582 RegMask Matcher::divI_proj_mask() {
1583   return INT_RAX_REG_mask();
1584 }
1585 
1586 // Register for MODI projection of divmodI
1587 RegMask Matcher::modI_proj_mask() {
1588   return INT_RDX_REG_mask();
1589 }
1590 
1591 // Register for DIVL projection of divmodL
1592 RegMask Matcher::divL_proj_mask() {
1593   return LONG_RAX_REG_mask();
1594 }
1595 
1596 // Register for MODL projection of divmodL
1597 RegMask Matcher::modL_proj_mask() {
1598   return LONG_RDX_REG_mask();
1599 }
1600 
1601 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1602   return PTR_RBP_REG_mask();
1603 }
1604 
1605 %}
1606 
1607 //----------ENCODING BLOCK-----------------------------------------------------
1608 // This block specifies the encoding classes used by the compiler to
1609 // output byte streams.  Encoding classes are parameterized macros
1610 // used by Machine Instruction Nodes in order to generate the bit
1611 // encoding of the instruction.  Operands specify their base encoding
1612 // interface with the interface keyword.  There are currently
1613 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1614 // COND_INTER.  REG_INTER causes an operand to generate a function
1615 // which returns its register number when queried.  CONST_INTER causes
1616 // an operand to generate a function which returns the value of the
1617 // constant when queried.  MEMORY_INTER causes an operand to generate
1618 // four functions which return the Base Register, the Index Register,
1619 // the Scale Value, and the Offset Value of the operand when queried.
1620 // COND_INTER causes an operand to generate six functions which return
1621 // the encoding code (ie - encoding bits for the instruction)
1622 // associated with each basic boolean condition for a conditional
1623 // instruction.
1624 //
1625 // Instructions specify two basic values for encoding.  Again, a
1626 // function is available to check if the constant displacement is an
1627 // oop. They use the ins_encode keyword to specify their encoding
1628 // classes (which must be a sequence of enc_class names, and their
1629 // parameters, specified in the encoding block), and they use the
1630 // opcode keyword to specify, in order, their primary, secondary, and
1631 // tertiary opcode.  Only the opcode sections which a particular
1632 // instruction needs for encoding need to be specified.
1633 encode %{
1634   // Build emit functions for each basic byte or larger field in the
1635   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1636   // from C++ code in the enc_class source block.  Emit functions will
1637   // live in the main source block for now.  In future, we can
1638   // generalize this by adding a syntax that specifies the sizes of
1639   // fields in an order, so that the adlc can build the emit functions
1640   // automagically
1641 
1642   // Emit primary opcode
1643   enc_class OpcP
1644   %{
1645     emit_opcode(cbuf, $primary);
1646   %}
1647 
1648   // Emit secondary opcode
1649   enc_class OpcS
1650   %{
1651     emit_opcode(cbuf, $secondary);
1652   %}
1653 
1654   // Emit tertiary opcode
1655   enc_class OpcT
1656   %{
1657     emit_opcode(cbuf, $tertiary);
1658   %}
1659 
1660   // Emit opcode directly
1661   enc_class Opcode(immI d8)
1662   %{
1663     emit_opcode(cbuf, $d8$$constant);
1664   %}
1665 
1666   // Emit size prefix
1667   enc_class SizePrefix
1668   %{
1669     emit_opcode(cbuf, 0x66);
1670   %}
1671 
1672   enc_class reg(rRegI reg)
1673   %{
1674     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1675   %}
1676 
1677   enc_class reg_reg(rRegI dst, rRegI src)
1678   %{
1679     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1680   %}
1681 
1682   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1683   %{
1684     emit_opcode(cbuf, $opcode$$constant);
1685     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1686   %}
1687 
1688   enc_class cdql_enc(no_rax_rdx_RegI div)
1689   %{
1690     // Full implementation of Java idiv and irem; checks for
1691     // special case as described in JVM spec., p.243 & p.271.
1692     //
1693     //         normal case                           special case
1694     //
1695     // input : rax: dividend                         min_int
1696     //         reg: divisor                          -1
1697     //
1698     // output: rax: quotient  (= rax idiv reg)       min_int
1699     //         rdx: remainder (= rax irem reg)       0
1700     //
1701     //  Code sequnce:
1702     //
1703     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1704     //    5:   75 07/08                jne    e <normal>
1705     //    7:   33 d2                   xor    %edx,%edx
1706     //  [div >= 8 -> offset + 1]
1707     //  [REX_B]
1708     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1709     //    c:   74 03/04                je     11 <done>
1710     // 000000000000000e <normal>:
1711     //    e:   99                      cltd
1712     //  [div >= 8 -> offset + 1]
1713     //  [REX_B]
1714     //    f:   f7 f9                   idiv   $div
1715     // 0000000000000011 <done>:
1716 
1717     // cmp    $0x80000000,%eax
1718     emit_opcode(cbuf, 0x3d);
1719     emit_d8(cbuf, 0x00);
1720     emit_d8(cbuf, 0x00);
1721     emit_d8(cbuf, 0x00);
1722     emit_d8(cbuf, 0x80);
1723 
1724     // jne    e <normal>
1725     emit_opcode(cbuf, 0x75);
1726     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1727 
1728     // xor    %edx,%edx
1729     emit_opcode(cbuf, 0x33);
1730     emit_d8(cbuf, 0xD2);
1731 
1732     // cmp    $0xffffffffffffffff,%ecx
1733     if ($div$$reg >= 8) {
1734       emit_opcode(cbuf, Assembler::REX_B);
1735     }
1736     emit_opcode(cbuf, 0x83);
1737     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1738     emit_d8(cbuf, 0xFF);
1739 
1740     // je     11 <done>
1741     emit_opcode(cbuf, 0x74);
1742     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1743 
1744     // <normal>
1745     // cltd
1746     emit_opcode(cbuf, 0x99);
1747 
1748     // idivl (note: must be emitted by the user of this rule)
1749     // <done>
1750   %}
1751 
1752   enc_class cdqq_enc(no_rax_rdx_RegL div)
1753   %{
1754     // Full implementation of Java ldiv and lrem; checks for
1755     // special case as described in JVM spec., p.243 & p.271.
1756     //
1757     //         normal case                           special case
1758     //
1759     // input : rax: dividend                         min_long
1760     //         reg: divisor                          -1
1761     //
1762     // output: rax: quotient  (= rax idiv reg)       min_long
1763     //         rdx: remainder (= rax irem reg)       0
1764     //
1765     //  Code sequnce:
1766     //
1767     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1768     //    7:   00 00 80
1769     //    a:   48 39 d0                cmp    %rdx,%rax
1770     //    d:   75 08                   jne    17 <normal>
1771     //    f:   33 d2                   xor    %edx,%edx
1772     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1773     //   15:   74 05                   je     1c <done>
1774     // 0000000000000017 <normal>:
1775     //   17:   48 99                   cqto
1776     //   19:   48 f7 f9                idiv   $div
1777     // 000000000000001c <done>:
1778 
1779     // mov    $0x8000000000000000,%rdx
1780     emit_opcode(cbuf, Assembler::REX_W);
1781     emit_opcode(cbuf, 0xBA);
1782     emit_d8(cbuf, 0x00);
1783     emit_d8(cbuf, 0x00);
1784     emit_d8(cbuf, 0x00);
1785     emit_d8(cbuf, 0x00);
1786     emit_d8(cbuf, 0x00);
1787     emit_d8(cbuf, 0x00);
1788     emit_d8(cbuf, 0x00);
1789     emit_d8(cbuf, 0x80);
1790 
1791     // cmp    %rdx,%rax
1792     emit_opcode(cbuf, Assembler::REX_W);
1793     emit_opcode(cbuf, 0x39);
1794     emit_d8(cbuf, 0xD0);
1795 
1796     // jne    17 <normal>
1797     emit_opcode(cbuf, 0x75);
1798     emit_d8(cbuf, 0x08);
1799 
1800     // xor    %edx,%edx
1801     emit_opcode(cbuf, 0x33);
1802     emit_d8(cbuf, 0xD2);
1803 
1804     // cmp    $0xffffffffffffffff,$div
1805     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1806     emit_opcode(cbuf, 0x83);
1807     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1808     emit_d8(cbuf, 0xFF);
1809 
1810     // je     1e <done>
1811     emit_opcode(cbuf, 0x74);
1812     emit_d8(cbuf, 0x05);
1813 
1814     // <normal>
1815     // cqto
1816     emit_opcode(cbuf, Assembler::REX_W);
1817     emit_opcode(cbuf, 0x99);
1818 
1819     // idivq (note: must be emitted by the user of this rule)
1820     // <done>
1821   %}
1822 
1823   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1824   enc_class OpcSE(immI imm)
1825   %{
1826     // Emit primary opcode and set sign-extend bit
1827     // Check for 8-bit immediate, and set sign extend bit in opcode
1828     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1829       emit_opcode(cbuf, $primary | 0x02);
1830     } else {
1831       // 32-bit immediate
1832       emit_opcode(cbuf, $primary);
1833     }
1834   %}
1835 
1836   enc_class OpcSErm(rRegI dst, immI imm)
1837   %{
1838     // OpcSEr/m
1839     int dstenc = $dst$$reg;
1840     if (dstenc >= 8) {
1841       emit_opcode(cbuf, Assembler::REX_B);
1842       dstenc -= 8;
1843     }
1844     // Emit primary opcode and set sign-extend bit
1845     // Check for 8-bit immediate, and set sign extend bit in opcode
1846     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1847       emit_opcode(cbuf, $primary | 0x02);
1848     } else {
1849       // 32-bit immediate
1850       emit_opcode(cbuf, $primary);
1851     }
1852     // Emit r/m byte with secondary opcode, after primary opcode.
1853     emit_rm(cbuf, 0x3, $secondary, dstenc);
1854   %}
1855 
1856   enc_class OpcSErm_wide(rRegL dst, immI imm)
1857   %{
1858     // OpcSEr/m
1859     int dstenc = $dst$$reg;
1860     if (dstenc < 8) {
1861       emit_opcode(cbuf, Assembler::REX_W);
1862     } else {
1863       emit_opcode(cbuf, Assembler::REX_WB);
1864       dstenc -= 8;
1865     }
1866     // Emit primary opcode and set sign-extend bit
1867     // Check for 8-bit immediate, and set sign extend bit in opcode
1868     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1869       emit_opcode(cbuf, $primary | 0x02);
1870     } else {
1871       // 32-bit immediate
1872       emit_opcode(cbuf, $primary);
1873     }
1874     // Emit r/m byte with secondary opcode, after primary opcode.
1875     emit_rm(cbuf, 0x3, $secondary, dstenc);
1876   %}
1877 
1878   enc_class Con8or32(immI imm)
1879   %{
1880     // Check for 8-bit immediate, and set sign extend bit in opcode
1881     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1882       $$$emit8$imm$$constant;
1883     } else {
1884       // 32-bit immediate
1885       $$$emit32$imm$$constant;
1886     }
1887   %}
1888 
1889   enc_class opc2_reg(rRegI dst)
1890   %{
1891     // BSWAP
1892     emit_cc(cbuf, $secondary, $dst$$reg);
1893   %}
1894 
1895   enc_class opc3_reg(rRegI dst)
1896   %{
1897     // BSWAP
1898     emit_cc(cbuf, $tertiary, $dst$$reg);
1899   %}
1900 
1901   enc_class reg_opc(rRegI div)
1902   %{
1903     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1904     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1905   %}
1906 
1907   enc_class enc_cmov(cmpOp cop)
1908   %{
1909     // CMOV
1910     $$$emit8$primary;
1911     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1912   %}
1913 
1914   enc_class enc_PartialSubtypeCheck()
1915   %{
1916     Register Rrdi = as_Register(RDI_enc); // result register
1917     Register Rrax = as_Register(RAX_enc); // super class
1918     Register Rrcx = as_Register(RCX_enc); // killed
1919     Register Rrsi = as_Register(RSI_enc); // sub class
1920     Label miss;
1921     const bool set_cond_codes = true;
1922 
1923     MacroAssembler _masm(&cbuf);
1924     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1925                                      NULL, &miss,
1926                                      /*set_cond_codes:*/ true);
1927     if ($primary) {
1928       __ xorptr(Rrdi, Rrdi);
1929     }
1930     __ bind(miss);
1931   %}
1932 
1933   enc_class clear_avx %{
1934     debug_only(int off0 = cbuf.insts_size());
1935     if (ra_->C->max_vector_size() > 16) {
1936       // Clear upper bits of YMM registers when current compiled code uses
1937       // wide vectors to avoid AVX <-> SSE transition penalty during call.
1938       MacroAssembler _masm(&cbuf);
1939       __ vzeroupper();
1940     }
1941     debug_only(int off1 = cbuf.insts_size());
1942     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
1943   %}
1944 
1945   enc_class Java_To_Runtime(method meth) %{
1946     // No relocation needed
1947     MacroAssembler _masm(&cbuf);
1948     __ mov64(r10, (int64_t) $meth$$method);
1949     __ call(r10);
1950   %}
1951 
1952   enc_class Java_To_Interpreter(method meth)
1953   %{
1954     // CALL Java_To_Interpreter
1955     // This is the instruction starting address for relocation info.
1956     cbuf.set_insts_mark();
1957     $$$emit8$primary;
1958     // CALL directly to the runtime
1959     emit_d32_reloc(cbuf,
1960                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1961                    runtime_call_Relocation::spec(),
1962                    RELOC_DISP32);
1963   %}
1964 
1965   enc_class Java_Static_Call(method meth)
1966   %{
1967     // JAVA STATIC CALL
1968     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
1969     // determine who we intended to call.
1970     cbuf.set_insts_mark();
1971     $$$emit8$primary;
1972 
1973     if (!_method) {
1974       emit_d32_reloc(cbuf,
1975                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1976                      runtime_call_Relocation::spec(),
1977                      RELOC_DISP32);
1978     } else if (_optimized_virtual) {
1979       emit_d32_reloc(cbuf,
1980                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1981                      opt_virtual_call_Relocation::spec(),
1982                      RELOC_DISP32);
1983     } else {
1984       emit_d32_reloc(cbuf,
1985                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
1986                      static_call_Relocation::spec(),
1987                      RELOC_DISP32);
1988     }
1989     if (_method) {
1990       // Emit stub for static call.
1991       CompiledStaticCall::emit_to_interp_stub(cbuf);
1992     }
1993   %}
1994 
1995   enc_class Java_Dynamic_Call(method meth) %{
1996     MacroAssembler _masm(&cbuf);
1997     __ ic_call((address)$meth$$method);
1998   %}
1999 
2000   enc_class Java_Compiled_Call(method meth)
2001   %{
2002     // JAVA COMPILED CALL
2003     int disp = in_bytes(Method:: from_compiled_offset());
2004 
2005     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2006     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2007 
2008     // callq *disp(%rax)
2009     cbuf.set_insts_mark();
2010     $$$emit8$primary;
2011     if (disp < 0x80) {
2012       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2013       emit_d8(cbuf, disp); // Displacement
2014     } else {
2015       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2016       emit_d32(cbuf, disp); // Displacement
2017     }
2018   %}
2019 
2020   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2021   %{
2022     // SAL, SAR, SHR
2023     int dstenc = $dst$$reg;
2024     if (dstenc >= 8) {
2025       emit_opcode(cbuf, Assembler::REX_B);
2026       dstenc -= 8;
2027     }
2028     $$$emit8$primary;
2029     emit_rm(cbuf, 0x3, $secondary, dstenc);
2030     $$$emit8$shift$$constant;
2031   %}
2032 
2033   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2034   %{
2035     // SAL, SAR, SHR
2036     int dstenc = $dst$$reg;
2037     if (dstenc < 8) {
2038       emit_opcode(cbuf, Assembler::REX_W);
2039     } else {
2040       emit_opcode(cbuf, Assembler::REX_WB);
2041       dstenc -= 8;
2042     }
2043     $$$emit8$primary;
2044     emit_rm(cbuf, 0x3, $secondary, dstenc);
2045     $$$emit8$shift$$constant;
2046   %}
2047 
2048   enc_class load_immI(rRegI dst, immI src)
2049   %{
2050     int dstenc = $dst$$reg;
2051     if (dstenc >= 8) {
2052       emit_opcode(cbuf, Assembler::REX_B);
2053       dstenc -= 8;
2054     }
2055     emit_opcode(cbuf, 0xB8 | dstenc);
2056     $$$emit32$src$$constant;
2057   %}
2058 
2059   enc_class load_immL(rRegL dst, immL src)
2060   %{
2061     int dstenc = $dst$$reg;
2062     if (dstenc < 8) {
2063       emit_opcode(cbuf, Assembler::REX_W);
2064     } else {
2065       emit_opcode(cbuf, Assembler::REX_WB);
2066       dstenc -= 8;
2067     }
2068     emit_opcode(cbuf, 0xB8 | dstenc);
2069     emit_d64(cbuf, $src$$constant);
2070   %}
2071 
2072   enc_class load_immUL32(rRegL dst, immUL32 src)
2073   %{
2074     // same as load_immI, but this time we care about zeroes in the high word
2075     int dstenc = $dst$$reg;
2076     if (dstenc >= 8) {
2077       emit_opcode(cbuf, Assembler::REX_B);
2078       dstenc -= 8;
2079     }
2080     emit_opcode(cbuf, 0xB8 | dstenc);
2081     $$$emit32$src$$constant;
2082   %}
2083 
2084   enc_class load_immL32(rRegL dst, immL32 src)
2085   %{
2086     int dstenc = $dst$$reg;
2087     if (dstenc < 8) {
2088       emit_opcode(cbuf, Assembler::REX_W);
2089     } else {
2090       emit_opcode(cbuf, Assembler::REX_WB);
2091       dstenc -= 8;
2092     }
2093     emit_opcode(cbuf, 0xC7);
2094     emit_rm(cbuf, 0x03, 0x00, dstenc);
2095     $$$emit32$src$$constant;
2096   %}
2097 
2098   enc_class load_immP31(rRegP dst, immP32 src)
2099   %{
2100     // same as load_immI, but this time we care about zeroes in the high word
2101     int dstenc = $dst$$reg;
2102     if (dstenc >= 8) {
2103       emit_opcode(cbuf, Assembler::REX_B);
2104       dstenc -= 8;
2105     }
2106     emit_opcode(cbuf, 0xB8 | dstenc);
2107     $$$emit32$src$$constant;
2108   %}
2109 
2110   enc_class load_immP(rRegP dst, immP src)
2111   %{
2112     int dstenc = $dst$$reg;
2113     if (dstenc < 8) {
2114       emit_opcode(cbuf, Assembler::REX_W);
2115     } else {
2116       emit_opcode(cbuf, Assembler::REX_WB);
2117       dstenc -= 8;
2118     }
2119     emit_opcode(cbuf, 0xB8 | dstenc);
2120     // This next line should be generated from ADLC
2121     if ($src->constant_reloc() != relocInfo::none) {
2122       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2123     } else {
2124       emit_d64(cbuf, $src$$constant);
2125     }
2126   %}
2127 
2128   enc_class Con32(immI src)
2129   %{
2130     // Output immediate
2131     $$$emit32$src$$constant;
2132   %}
2133 
2134   enc_class Con32F_as_bits(immF src)
2135   %{
2136     // Output Float immediate bits
2137     jfloat jf = $src$$constant;
2138     jint jf_as_bits = jint_cast(jf);
2139     emit_d32(cbuf, jf_as_bits);
2140   %}
2141 
2142   enc_class Con16(immI src)
2143   %{
2144     // Output immediate
2145     $$$emit16$src$$constant;
2146   %}
2147 
2148   // How is this different from Con32??? XXX
2149   enc_class Con_d32(immI src)
2150   %{
2151     emit_d32(cbuf,$src$$constant);
2152   %}
2153 
2154   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2155     // Output immediate memory reference
2156     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2157     emit_d32(cbuf, 0x00);
2158   %}
2159 
2160   enc_class lock_prefix()
2161   %{
2162     if (os::is_MP()) {
2163       emit_opcode(cbuf, 0xF0); // lock
2164     }
2165   %}
2166 
2167   enc_class REX_mem(memory mem)
2168   %{
2169     if ($mem$$base >= 8) {
2170       if ($mem$$index < 8) {
2171         emit_opcode(cbuf, Assembler::REX_B);
2172       } else {
2173         emit_opcode(cbuf, Assembler::REX_XB);
2174       }
2175     } else {
2176       if ($mem$$index >= 8) {
2177         emit_opcode(cbuf, Assembler::REX_X);
2178       }
2179     }
2180   %}
2181 
2182   enc_class REX_mem_wide(memory mem)
2183   %{
2184     if ($mem$$base >= 8) {
2185       if ($mem$$index < 8) {
2186         emit_opcode(cbuf, Assembler::REX_WB);
2187       } else {
2188         emit_opcode(cbuf, Assembler::REX_WXB);
2189       }
2190     } else {
2191       if ($mem$$index < 8) {
2192         emit_opcode(cbuf, Assembler::REX_W);
2193       } else {
2194         emit_opcode(cbuf, Assembler::REX_WX);
2195       }
2196     }
2197   %}
2198 
2199   // for byte regs
2200   enc_class REX_breg(rRegI reg)
2201   %{
2202     if ($reg$$reg >= 4) {
2203       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2204     }
2205   %}
2206 
2207   // for byte regs
2208   enc_class REX_reg_breg(rRegI dst, rRegI src)
2209   %{
2210     if ($dst$$reg < 8) {
2211       if ($src$$reg >= 4) {
2212         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2213       }
2214     } else {
2215       if ($src$$reg < 8) {
2216         emit_opcode(cbuf, Assembler::REX_R);
2217       } else {
2218         emit_opcode(cbuf, Assembler::REX_RB);
2219       }
2220     }
2221   %}
2222 
2223   // for byte regs
2224   enc_class REX_breg_mem(rRegI reg, memory mem)
2225   %{
2226     if ($reg$$reg < 8) {
2227       if ($mem$$base < 8) {
2228         if ($mem$$index >= 8) {
2229           emit_opcode(cbuf, Assembler::REX_X);
2230         } else if ($reg$$reg >= 4) {
2231           emit_opcode(cbuf, Assembler::REX);
2232         }
2233       } else {
2234         if ($mem$$index < 8) {
2235           emit_opcode(cbuf, Assembler::REX_B);
2236         } else {
2237           emit_opcode(cbuf, Assembler::REX_XB);
2238         }
2239       }
2240     } else {
2241       if ($mem$$base < 8) {
2242         if ($mem$$index < 8) {
2243           emit_opcode(cbuf, Assembler::REX_R);
2244         } else {
2245           emit_opcode(cbuf, Assembler::REX_RX);
2246         }
2247       } else {
2248         if ($mem$$index < 8) {
2249           emit_opcode(cbuf, Assembler::REX_RB);
2250         } else {
2251           emit_opcode(cbuf, Assembler::REX_RXB);
2252         }
2253       }
2254     }
2255   %}
2256 
2257   enc_class REX_reg(rRegI reg)
2258   %{
2259     if ($reg$$reg >= 8) {
2260       emit_opcode(cbuf, Assembler::REX_B);
2261     }
2262   %}
2263 
2264   enc_class REX_reg_wide(rRegI reg)
2265   %{
2266     if ($reg$$reg < 8) {
2267       emit_opcode(cbuf, Assembler::REX_W);
2268     } else {
2269       emit_opcode(cbuf, Assembler::REX_WB);
2270     }
2271   %}
2272 
2273   enc_class REX_reg_reg(rRegI dst, rRegI src)
2274   %{
2275     if ($dst$$reg < 8) {
2276       if ($src$$reg >= 8) {
2277         emit_opcode(cbuf, Assembler::REX_B);
2278       }
2279     } else {
2280       if ($src$$reg < 8) {
2281         emit_opcode(cbuf, Assembler::REX_R);
2282       } else {
2283         emit_opcode(cbuf, Assembler::REX_RB);
2284       }
2285     }
2286   %}
2287 
2288   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2289   %{
2290     if ($dst$$reg < 8) {
2291       if ($src$$reg < 8) {
2292         emit_opcode(cbuf, Assembler::REX_W);
2293       } else {
2294         emit_opcode(cbuf, Assembler::REX_WB);
2295       }
2296     } else {
2297       if ($src$$reg < 8) {
2298         emit_opcode(cbuf, Assembler::REX_WR);
2299       } else {
2300         emit_opcode(cbuf, Assembler::REX_WRB);
2301       }
2302     }
2303   %}
2304 
2305   enc_class REX_reg_mem(rRegI reg, memory mem)
2306   %{
2307     if ($reg$$reg < 8) {
2308       if ($mem$$base < 8) {
2309         if ($mem$$index >= 8) {
2310           emit_opcode(cbuf, Assembler::REX_X);
2311         }
2312       } else {
2313         if ($mem$$index < 8) {
2314           emit_opcode(cbuf, Assembler::REX_B);
2315         } else {
2316           emit_opcode(cbuf, Assembler::REX_XB);
2317         }
2318       }
2319     } else {
2320       if ($mem$$base < 8) {
2321         if ($mem$$index < 8) {
2322           emit_opcode(cbuf, Assembler::REX_R);
2323         } else {
2324           emit_opcode(cbuf, Assembler::REX_RX);
2325         }
2326       } else {
2327         if ($mem$$index < 8) {
2328           emit_opcode(cbuf, Assembler::REX_RB);
2329         } else {
2330           emit_opcode(cbuf, Assembler::REX_RXB);
2331         }
2332       }
2333     }
2334   %}
2335 
2336   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2337   %{
2338     if ($reg$$reg < 8) {
2339       if ($mem$$base < 8) {
2340         if ($mem$$index < 8) {
2341           emit_opcode(cbuf, Assembler::REX_W);
2342         } else {
2343           emit_opcode(cbuf, Assembler::REX_WX);
2344         }
2345       } else {
2346         if ($mem$$index < 8) {
2347           emit_opcode(cbuf, Assembler::REX_WB);
2348         } else {
2349           emit_opcode(cbuf, Assembler::REX_WXB);
2350         }
2351       }
2352     } else {
2353       if ($mem$$base < 8) {
2354         if ($mem$$index < 8) {
2355           emit_opcode(cbuf, Assembler::REX_WR);
2356         } else {
2357           emit_opcode(cbuf, Assembler::REX_WRX);
2358         }
2359       } else {
2360         if ($mem$$index < 8) {
2361           emit_opcode(cbuf, Assembler::REX_WRB);
2362         } else {
2363           emit_opcode(cbuf, Assembler::REX_WRXB);
2364         }
2365       }
2366     }
2367   %}
2368 
2369   enc_class reg_mem(rRegI ereg, memory mem)
2370   %{
2371     // High registers handle in encode_RegMem
2372     int reg = $ereg$$reg;
2373     int base = $mem$$base;
2374     int index = $mem$$index;
2375     int scale = $mem$$scale;
2376     int disp = $mem$$disp;
2377     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2378 
2379     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2380   %}
2381 
2382   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2383   %{
2384     int rm_byte_opcode = $rm_opcode$$constant;
2385 
2386     // High registers handle in encode_RegMem
2387     int base = $mem$$base;
2388     int index = $mem$$index;
2389     int scale = $mem$$scale;
2390     int displace = $mem$$disp;
2391 
2392     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2393                                             // working with static
2394                                             // globals
2395     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2396                   disp_reloc);
2397   %}
2398 
2399   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2400   %{
2401     int reg_encoding = $dst$$reg;
2402     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2403     int index        = 0x04;            // 0x04 indicates no index
2404     int scale        = 0x00;            // 0x00 indicates no scale
2405     int displace     = $src1$$constant; // 0x00 indicates no displacement
2406     relocInfo::relocType disp_reloc = relocInfo::none;
2407     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2408                   disp_reloc);
2409   %}
2410 
2411   enc_class neg_reg(rRegI dst)
2412   %{
2413     int dstenc = $dst$$reg;
2414     if (dstenc >= 8) {
2415       emit_opcode(cbuf, Assembler::REX_B);
2416       dstenc -= 8;
2417     }
2418     // NEG $dst
2419     emit_opcode(cbuf, 0xF7);
2420     emit_rm(cbuf, 0x3, 0x03, dstenc);
2421   %}
2422 
2423   enc_class neg_reg_wide(rRegI dst)
2424   %{
2425     int dstenc = $dst$$reg;
2426     if (dstenc < 8) {
2427       emit_opcode(cbuf, Assembler::REX_W);
2428     } else {
2429       emit_opcode(cbuf, Assembler::REX_WB);
2430       dstenc -= 8;
2431     }
2432     // NEG $dst
2433     emit_opcode(cbuf, 0xF7);
2434     emit_rm(cbuf, 0x3, 0x03, dstenc);
2435   %}
2436 
2437   enc_class setLT_reg(rRegI dst)
2438   %{
2439     int dstenc = $dst$$reg;
2440     if (dstenc >= 8) {
2441       emit_opcode(cbuf, Assembler::REX_B);
2442       dstenc -= 8;
2443     } else if (dstenc >= 4) {
2444       emit_opcode(cbuf, Assembler::REX);
2445     }
2446     // SETLT $dst
2447     emit_opcode(cbuf, 0x0F);
2448     emit_opcode(cbuf, 0x9C);
2449     emit_rm(cbuf, 0x3, 0x0, dstenc);
2450   %}
2451 
2452   enc_class setNZ_reg(rRegI dst)
2453   %{
2454     int dstenc = $dst$$reg;
2455     if (dstenc >= 8) {
2456       emit_opcode(cbuf, Assembler::REX_B);
2457       dstenc -= 8;
2458     } else if (dstenc >= 4) {
2459       emit_opcode(cbuf, Assembler::REX);
2460     }
2461     // SETNZ $dst
2462     emit_opcode(cbuf, 0x0F);
2463     emit_opcode(cbuf, 0x95);
2464     emit_rm(cbuf, 0x3, 0x0, dstenc);
2465   %}
2466 
2467 
2468   // Compare the lonogs and set -1, 0, or 1 into dst
2469   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2470   %{
2471     int src1enc = $src1$$reg;
2472     int src2enc = $src2$$reg;
2473     int dstenc = $dst$$reg;
2474 
2475     // cmpq $src1, $src2
2476     if (src1enc < 8) {
2477       if (src2enc < 8) {
2478         emit_opcode(cbuf, Assembler::REX_W);
2479       } else {
2480         emit_opcode(cbuf, Assembler::REX_WB);
2481       }
2482     } else {
2483       if (src2enc < 8) {
2484         emit_opcode(cbuf, Assembler::REX_WR);
2485       } else {
2486         emit_opcode(cbuf, Assembler::REX_WRB);
2487       }
2488     }
2489     emit_opcode(cbuf, 0x3B);
2490     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2491 
2492     // movl $dst, -1
2493     if (dstenc >= 8) {
2494       emit_opcode(cbuf, Assembler::REX_B);
2495     }
2496     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2497     emit_d32(cbuf, -1);
2498 
2499     // jl,s done
2500     emit_opcode(cbuf, 0x7C);
2501     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2502 
2503     // setne $dst
2504     if (dstenc >= 4) {
2505       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2506     }
2507     emit_opcode(cbuf, 0x0F);
2508     emit_opcode(cbuf, 0x95);
2509     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2510 
2511     // movzbl $dst, $dst
2512     if (dstenc >= 4) {
2513       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2514     }
2515     emit_opcode(cbuf, 0x0F);
2516     emit_opcode(cbuf, 0xB6);
2517     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2518   %}
2519 
2520   enc_class Push_ResultXD(regD dst) %{
2521     MacroAssembler _masm(&cbuf);
2522     __ fstp_d(Address(rsp, 0));
2523     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2524     __ addptr(rsp, 8);
2525   %}
2526 
2527   enc_class Push_SrcXD(regD src) %{
2528     MacroAssembler _masm(&cbuf);
2529     __ subptr(rsp, 8);
2530     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2531     __ fld_d(Address(rsp, 0));
2532   %}
2533 
2534 
2535   enc_class enc_rethrow()
2536   %{
2537     cbuf.set_insts_mark();
2538     emit_opcode(cbuf, 0xE9); // jmp entry
2539     emit_d32_reloc(cbuf,
2540                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2541                    runtime_call_Relocation::spec(),
2542                    RELOC_DISP32);
2543   %}
2544 
2545 %}
2546 
2547 
2548 
2549 //----------FRAME--------------------------------------------------------------
2550 // Definition of frame structure and management information.
2551 //
2552 //  S T A C K   L A Y O U T    Allocators stack-slot number
2553 //                             |   (to get allocators register number
2554 //  G  Owned by    |        |  v    add OptoReg::stack0())
2555 //  r   CALLER     |        |
2556 //  o     |        +--------+      pad to even-align allocators stack-slot
2557 //  w     V        |  pad0  |        numbers; owned by CALLER
2558 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2559 //  h     ^        |   in   |  5
2560 //        |        |  args  |  4   Holes in incoming args owned by SELF
2561 //  |     |        |        |  3
2562 //  |     |        +--------+
2563 //  V     |        | old out|      Empty on Intel, window on Sparc
2564 //        |    old |preserve|      Must be even aligned.
2565 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2566 //        |        |   in   |  3   area for Intel ret address
2567 //     Owned by    |preserve|      Empty on Sparc.
2568 //       SELF      +--------+
2569 //        |        |  pad2  |  2   pad to align old SP
2570 //        |        +--------+  1
2571 //        |        | locks  |  0
2572 //        |        +--------+----> OptoReg::stack0(), even aligned
2573 //        |        |  pad1  | 11   pad to align new SP
2574 //        |        +--------+
2575 //        |        |        | 10
2576 //        |        | spills |  9   spills
2577 //        V        |        |  8   (pad0 slot for callee)
2578 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2579 //        ^        |  out   |  7
2580 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2581 //     Owned by    +--------+
2582 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2583 //        |    new |preserve|      Must be even-aligned.
2584 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2585 //        |        |        |
2586 //
2587 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2588 //         known from SELF's arguments and the Java calling convention.
2589 //         Region 6-7 is determined per call site.
2590 // Note 2: If the calling convention leaves holes in the incoming argument
2591 //         area, those holes are owned by SELF.  Holes in the outgoing area
2592 //         are owned by the CALLEE.  Holes should not be nessecary in the
2593 //         incoming area, as the Java calling convention is completely under
2594 //         the control of the AD file.  Doubles can be sorted and packed to
2595 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2596 //         varargs C calling conventions.
2597 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2598 //         even aligned with pad0 as needed.
2599 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2600 //         region 6-11 is even aligned; it may be padded out more so that
2601 //         the region from SP to FP meets the minimum stack alignment.
2602 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2603 //         alignment.  Region 11, pad1, may be dynamically extended so that
2604 //         SP meets the minimum alignment.
2605 
2606 frame
2607 %{
2608   // What direction does stack grow in (assumed to be same for C & Java)
2609   stack_direction(TOWARDS_LOW);
2610 
2611   // These three registers define part of the calling convention
2612   // between compiled code and the interpreter.
2613   inline_cache_reg(RAX);                // Inline Cache Register
2614   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2615                                         // calling interpreter
2616 
2617   // Optional: name the operand used by cisc-spilling to access
2618   // [stack_pointer + offset]
2619   cisc_spilling_operand_name(indOffset32);
2620 
2621   // Number of stack slots consumed by locking an object
2622   sync_stack_slots(2);
2623 
2624   // Compiled code's Frame Pointer
2625   frame_pointer(RSP);
2626 
2627   // Interpreter stores its frame pointer in a register which is
2628   // stored to the stack by I2CAdaptors.
2629   // I2CAdaptors convert from interpreted java to compiled java.
2630   interpreter_frame_pointer(RBP);
2631 
2632   // Stack alignment requirement
2633   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2634 
2635   // Number of stack slots between incoming argument block and the start of
2636   // a new frame.  The PROLOG must add this many slots to the stack.  The
2637   // EPILOG must remove this many slots.  amd64 needs two slots for
2638   // return address.
2639   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2640 
2641   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2642   // for calls to C.  Supports the var-args backing area for register parms.
2643   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2644 
2645   // The after-PROLOG location of the return address.  Location of
2646   // return address specifies a type (REG or STACK) and a number
2647   // representing the register number (i.e. - use a register name) or
2648   // stack slot.
2649   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2650   // Otherwise, it is above the locks and verification slot and alignment word
2651   return_addr(STACK - 2 +
2652               round_to((Compile::current()->in_preserve_stack_slots() +
2653                         Compile::current()->fixed_slots()),
2654                        stack_alignment_in_slots()));
2655 
2656   // Body of function which returns an integer array locating
2657   // arguments either in registers or in stack slots.  Passed an array
2658   // of ideal registers called "sig" and a "length" count.  Stack-slot
2659   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2660   // arguments for a CALLEE.  Incoming stack arguments are
2661   // automatically biased by the preserve_stack_slots field above.
2662 
2663   calling_convention
2664   %{
2665     // No difference between ingoing/outgoing just pass false
2666     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2667   %}
2668 
2669   c_calling_convention
2670   %{
2671     // This is obviously always outgoing
2672     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2673   %}
2674 
2675   // Location of compiled Java return values.  Same as C for now.
2676   return_value
2677   %{
2678     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2679            "only return normal values");
2680 
2681     static const int lo[Op_RegL + 1] = {
2682       0,
2683       0,
2684       RAX_num,  // Op_RegN
2685       RAX_num,  // Op_RegI
2686       RAX_num,  // Op_RegP
2687       XMM0_num, // Op_RegF
2688       XMM0_num, // Op_RegD
2689       RAX_num   // Op_RegL
2690     };
2691     static const int hi[Op_RegL + 1] = {
2692       0,
2693       0,
2694       OptoReg::Bad, // Op_RegN
2695       OptoReg::Bad, // Op_RegI
2696       RAX_H_num,    // Op_RegP
2697       OptoReg::Bad, // Op_RegF
2698       XMM0b_num,    // Op_RegD
2699       RAX_H_num     // Op_RegL
2700     };
2701     // Excluded flags and vector registers.
2702     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2703     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2704   %}
2705 %}
2706 
2707 //----------ATTRIBUTES---------------------------------------------------------
2708 //----------Operand Attributes-------------------------------------------------
2709 op_attrib op_cost(0);        // Required cost attribute
2710 
2711 //----------Instruction Attributes---------------------------------------------
2712 ins_attrib ins_cost(100);       // Required cost attribute
2713 ins_attrib ins_size(8);         // Required size attribute (in bits)
2714 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2715                                 // a non-matching short branch variant
2716                                 // of some long branch?
2717 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2718                                 // be a power of 2) specifies the
2719                                 // alignment that some part of the
2720                                 // instruction (not necessarily the
2721                                 // start) requires.  If > 1, a
2722                                 // compute_padding() function must be
2723                                 // provided for the instruction
2724 
2725 //----------OPERANDS-----------------------------------------------------------
2726 // Operand definitions must precede instruction definitions for correct parsing
2727 // in the ADLC because operands constitute user defined types which are used in
2728 // instruction definitions.
2729 
2730 //----------Simple Operands----------------------------------------------------
2731 // Immediate Operands
2732 // Integer Immediate
2733 operand immI()
2734 %{
2735   match(ConI);
2736 
2737   op_cost(10);
2738   format %{ %}
2739   interface(CONST_INTER);
2740 %}
2741 
2742 // Constant for test vs zero
2743 operand immI0()
2744 %{
2745   predicate(n->get_int() == 0);
2746   match(ConI);
2747 
2748   op_cost(0);
2749   format %{ %}
2750   interface(CONST_INTER);
2751 %}
2752 
2753 // Constant for increment
2754 operand immI1()
2755 %{
2756   predicate(n->get_int() == 1);
2757   match(ConI);
2758 
2759   op_cost(0);
2760   format %{ %}
2761   interface(CONST_INTER);
2762 %}
2763 
2764 // Constant for decrement
2765 operand immI_M1()
2766 %{
2767   predicate(n->get_int() == -1);
2768   match(ConI);
2769 
2770   op_cost(0);
2771   format %{ %}
2772   interface(CONST_INTER);
2773 %}
2774 
2775 // Valid scale values for addressing modes
2776 operand immI2()
2777 %{
2778   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2779   match(ConI);
2780 
2781   format %{ %}
2782   interface(CONST_INTER);
2783 %}
2784 
2785 operand immI8()
2786 %{
2787   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2788   match(ConI);
2789 
2790   op_cost(5);
2791   format %{ %}
2792   interface(CONST_INTER);
2793 %}
2794 
2795 operand immI16()
2796 %{
2797   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2798   match(ConI);
2799 
2800   op_cost(10);
2801   format %{ %}
2802   interface(CONST_INTER);
2803 %}
2804 
2805 // Int Immediate non-negative
2806 operand immU31()
2807 %{
2808   predicate(n->get_int() >= 0);
2809   match(ConI);
2810 
2811   op_cost(0);
2812   format %{ %}
2813   interface(CONST_INTER);
2814 %}
2815 
2816 // Constant for long shifts
2817 operand immI_32()
2818 %{
2819   predicate( n->get_int() == 32 );
2820   match(ConI);
2821 
2822   op_cost(0);
2823   format %{ %}
2824   interface(CONST_INTER);
2825 %}
2826 
2827 // Constant for long shifts
2828 operand immI_64()
2829 %{
2830   predicate( n->get_int() == 64 );
2831   match(ConI);
2832 
2833   op_cost(0);
2834   format %{ %}
2835   interface(CONST_INTER);
2836 %}
2837 
2838 // Pointer Immediate
2839 operand immP()
2840 %{
2841   match(ConP);
2842 
2843   op_cost(10);
2844   format %{ %}
2845   interface(CONST_INTER);
2846 %}
2847 
2848 // NULL Pointer Immediate
2849 operand immP0()
2850 %{
2851   predicate(n->get_ptr() == 0);
2852   match(ConP);
2853 
2854   op_cost(5);
2855   format %{ %}
2856   interface(CONST_INTER);
2857 %}
2858 
2859 // Pointer Immediate
2860 operand immN() %{
2861   match(ConN);
2862 
2863   op_cost(10);
2864   format %{ %}
2865   interface(CONST_INTER);
2866 %}
2867 
2868 operand immNKlass() %{
2869   match(ConNKlass);
2870 
2871   op_cost(10);
2872   format %{ %}
2873   interface(CONST_INTER);
2874 %}
2875 
2876 // NULL Pointer Immediate
2877 operand immN0() %{
2878   predicate(n->get_narrowcon() == 0);
2879   match(ConN);
2880 
2881   op_cost(5);
2882   format %{ %}
2883   interface(CONST_INTER);
2884 %}
2885 
2886 operand immP31()
2887 %{
2888   predicate(n->as_Type()->type()->reloc() == relocInfo::none
2889             && (n->get_ptr() >> 31) == 0);
2890   match(ConP);
2891 
2892   op_cost(5);
2893   format %{ %}
2894   interface(CONST_INTER);
2895 %}
2896 
2897 
2898 // Long Immediate
2899 operand immL()
2900 %{
2901   match(ConL);
2902 
2903   op_cost(20);
2904   format %{ %}
2905   interface(CONST_INTER);
2906 %}
2907 
2908 // Long Immediate 8-bit
2909 operand immL8()
2910 %{
2911   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
2912   match(ConL);
2913 
2914   op_cost(5);
2915   format %{ %}
2916   interface(CONST_INTER);
2917 %}
2918 
2919 // Long Immediate 32-bit unsigned
2920 operand immUL32()
2921 %{
2922   predicate(n->get_long() == (unsigned int) (n->get_long()));
2923   match(ConL);
2924 
2925   op_cost(10);
2926   format %{ %}
2927   interface(CONST_INTER);
2928 %}
2929 
2930 // Long Immediate 32-bit signed
2931 operand immL32()
2932 %{
2933   predicate(n->get_long() == (int) (n->get_long()));
2934   match(ConL);
2935 
2936   op_cost(15);
2937   format %{ %}
2938   interface(CONST_INTER);
2939 %}
2940 
2941 // Long Immediate zero
2942 operand immL0()
2943 %{
2944   predicate(n->get_long() == 0L);
2945   match(ConL);
2946 
2947   op_cost(10);
2948   format %{ %}
2949   interface(CONST_INTER);
2950 %}
2951 
2952 // Constant for increment
2953 operand immL1()
2954 %{
2955   predicate(n->get_long() == 1);
2956   match(ConL);
2957 
2958   format %{ %}
2959   interface(CONST_INTER);
2960 %}
2961 
2962 // Constant for decrement
2963 operand immL_M1()
2964 %{
2965   predicate(n->get_long() == -1);
2966   match(ConL);
2967 
2968   format %{ %}
2969   interface(CONST_INTER);
2970 %}
2971 
2972 // Long Immediate: the value 10
2973 operand immL10()
2974 %{
2975   predicate(n->get_long() == 10);
2976   match(ConL);
2977 
2978   format %{ %}
2979   interface(CONST_INTER);
2980 %}
2981 
2982 // Long immediate from 0 to 127.
2983 // Used for a shorter form of long mul by 10.
2984 operand immL_127()
2985 %{
2986   predicate(0 <= n->get_long() && n->get_long() < 0x80);
2987   match(ConL);
2988 
2989   op_cost(10);
2990   format %{ %}
2991   interface(CONST_INTER);
2992 %}
2993 
2994 // Long Immediate: low 32-bit mask
2995 operand immL_32bits()
2996 %{
2997   predicate(n->get_long() == 0xFFFFFFFFL);
2998   match(ConL);
2999   op_cost(20);
3000 
3001   format %{ %}
3002   interface(CONST_INTER);
3003 %}
3004 
3005 // Float Immediate zero
3006 operand immF0()
3007 %{
3008   predicate(jint_cast(n->getf()) == 0);
3009   match(ConF);
3010 
3011   op_cost(5);
3012   format %{ %}
3013   interface(CONST_INTER);
3014 %}
3015 
3016 // Float Immediate
3017 operand immF()
3018 %{
3019   match(ConF);
3020 
3021   op_cost(15);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // Double Immediate zero
3027 operand immD0()
3028 %{
3029   predicate(jlong_cast(n->getd()) == 0);
3030   match(ConD);
3031 
3032   op_cost(5);
3033   format %{ %}
3034   interface(CONST_INTER);
3035 %}
3036 
3037 // Double Immediate
3038 operand immD()
3039 %{
3040   match(ConD);
3041 
3042   op_cost(15);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 // Immediates for special shifts (sign extend)
3048 
3049 // Constants for increment
3050 operand immI_16()
3051 %{
3052   predicate(n->get_int() == 16);
3053   match(ConI);
3054 
3055   format %{ %}
3056   interface(CONST_INTER);
3057 %}
3058 
3059 operand immI_24()
3060 %{
3061   predicate(n->get_int() == 24);
3062   match(ConI);
3063 
3064   format %{ %}
3065   interface(CONST_INTER);
3066 %}
3067 
3068 // Constant for byte-wide masking
3069 operand immI_255()
3070 %{
3071   predicate(n->get_int() == 255);
3072   match(ConI);
3073 
3074   format %{ %}
3075   interface(CONST_INTER);
3076 %}
3077 
3078 // Constant for short-wide masking
3079 operand immI_65535()
3080 %{
3081   predicate(n->get_int() == 65535);
3082   match(ConI);
3083 
3084   format %{ %}
3085   interface(CONST_INTER);
3086 %}
3087 
3088 // Constant for byte-wide masking
3089 operand immL_255()
3090 %{
3091   predicate(n->get_long() == 255);
3092   match(ConL);
3093 
3094   format %{ %}
3095   interface(CONST_INTER);
3096 %}
3097 
3098 // Constant for short-wide masking
3099 operand immL_65535()
3100 %{
3101   predicate(n->get_long() == 65535);
3102   match(ConL);
3103 
3104   format %{ %}
3105   interface(CONST_INTER);
3106 %}
3107 
3108 // Register Operands
3109 // Integer Register
3110 operand rRegI()
3111 %{
3112   constraint(ALLOC_IN_RC(int_reg));
3113   match(RegI);
3114 
3115   match(rax_RegI);
3116   match(rbx_RegI);
3117   match(rcx_RegI);
3118   match(rdx_RegI);
3119   match(rdi_RegI);
3120 
3121   format %{ %}
3122   interface(REG_INTER);
3123 %}
3124 
3125 // Special Registers
3126 operand rax_RegI()
3127 %{
3128   constraint(ALLOC_IN_RC(int_rax_reg));
3129   match(RegI);
3130   match(rRegI);
3131 
3132   format %{ "RAX" %}
3133   interface(REG_INTER);
3134 %}
3135 
3136 // Special Registers
3137 operand rbx_RegI()
3138 %{
3139   constraint(ALLOC_IN_RC(int_rbx_reg));
3140   match(RegI);
3141   match(rRegI);
3142 
3143   format %{ "RBX" %}
3144   interface(REG_INTER);
3145 %}
3146 
3147 operand rcx_RegI()
3148 %{
3149   constraint(ALLOC_IN_RC(int_rcx_reg));
3150   match(RegI);
3151   match(rRegI);
3152 
3153   format %{ "RCX" %}
3154   interface(REG_INTER);
3155 %}
3156 
3157 operand rdx_RegI()
3158 %{
3159   constraint(ALLOC_IN_RC(int_rdx_reg));
3160   match(RegI);
3161   match(rRegI);
3162 
3163   format %{ "RDX" %}
3164   interface(REG_INTER);
3165 %}
3166 
3167 operand rdi_RegI()
3168 %{
3169   constraint(ALLOC_IN_RC(int_rdi_reg));
3170   match(RegI);
3171   match(rRegI);
3172 
3173   format %{ "RDI" %}
3174   interface(REG_INTER);
3175 %}
3176 
3177 operand no_rcx_RegI()
3178 %{
3179   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3180   match(RegI);
3181   match(rax_RegI);
3182   match(rbx_RegI);
3183   match(rdx_RegI);
3184   match(rdi_RegI);
3185 
3186   format %{ %}
3187   interface(REG_INTER);
3188 %}
3189 
3190 operand no_rax_rdx_RegI()
3191 %{
3192   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3193   match(RegI);
3194   match(rbx_RegI);
3195   match(rcx_RegI);
3196   match(rdi_RegI);
3197 
3198   format %{ %}
3199   interface(REG_INTER);
3200 %}
3201 
3202 // Pointer Register
3203 operand any_RegP()
3204 %{
3205   constraint(ALLOC_IN_RC(any_reg));
3206   match(RegP);
3207   match(rax_RegP);
3208   match(rbx_RegP);
3209   match(rdi_RegP);
3210   match(rsi_RegP);
3211   match(rbp_RegP);
3212   match(r15_RegP);
3213   match(rRegP);
3214 
3215   format %{ %}
3216   interface(REG_INTER);
3217 %}
3218 
3219 operand rRegP()
3220 %{
3221   constraint(ALLOC_IN_RC(ptr_reg));
3222   match(RegP);
3223   match(rax_RegP);
3224   match(rbx_RegP);
3225   match(rdi_RegP);
3226   match(rsi_RegP);
3227   match(rbp_RegP);
3228   match(r15_RegP);  // See Q&A below about r15_RegP.
3229 
3230   format %{ %}
3231   interface(REG_INTER);
3232 %}
3233 
3234 operand rRegN() %{
3235   constraint(ALLOC_IN_RC(int_reg));
3236   match(RegN);
3237 
3238   format %{ %}
3239   interface(REG_INTER);
3240 %}
3241 
3242 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3243 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3244 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
3245 // The output of an instruction is controlled by the allocator, which respects
3246 // register class masks, not match rules.  Unless an instruction mentions
3247 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3248 // by the allocator as an input.
3249 
3250 operand no_rax_RegP()
3251 %{
3252   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3253   match(RegP);
3254   match(rbx_RegP);
3255   match(rsi_RegP);
3256   match(rdi_RegP);
3257 
3258   format %{ %}
3259   interface(REG_INTER);
3260 %}
3261 
3262 operand no_rbp_RegP()
3263 %{
3264   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
3265   match(RegP);
3266   match(rbx_RegP);
3267   match(rsi_RegP);
3268   match(rdi_RegP);
3269 
3270   format %{ %}
3271   interface(REG_INTER);
3272 %}
3273 
3274 operand no_rax_rbx_RegP()
3275 %{
3276   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3277   match(RegP);
3278   match(rsi_RegP);
3279   match(rdi_RegP);
3280 
3281   format %{ %}
3282   interface(REG_INTER);
3283 %}
3284 
3285 // Special Registers
3286 // Return a pointer value
3287 operand rax_RegP()
3288 %{
3289   constraint(ALLOC_IN_RC(ptr_rax_reg));
3290   match(RegP);
3291   match(rRegP);
3292 
3293   format %{ %}
3294   interface(REG_INTER);
3295 %}
3296 
3297 // Special Registers
3298 // Return a compressed pointer value
3299 operand rax_RegN()
3300 %{
3301   constraint(ALLOC_IN_RC(int_rax_reg));
3302   match(RegN);
3303   match(rRegN);
3304 
3305   format %{ %}
3306   interface(REG_INTER);
3307 %}
3308 
3309 // Used in AtomicAdd
3310 operand rbx_RegP()
3311 %{
3312   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3313   match(RegP);
3314   match(rRegP);
3315 
3316   format %{ %}
3317   interface(REG_INTER);
3318 %}
3319 
3320 operand rsi_RegP()
3321 %{
3322   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3323   match(RegP);
3324   match(rRegP);
3325 
3326   format %{ %}
3327   interface(REG_INTER);
3328 %}
3329 
3330 // Used in rep stosq
3331 operand rdi_RegP()
3332 %{
3333   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3334   match(RegP);
3335   match(rRegP);
3336 
3337   format %{ %}
3338   interface(REG_INTER);
3339 %}
3340 
3341 operand rbp_RegP()
3342 %{
3343   constraint(ALLOC_IN_RC(ptr_rbp_reg));
3344   match(RegP);
3345   match(rRegP);
3346 
3347   format %{ %}
3348   interface(REG_INTER);
3349 %}
3350 
3351 operand r15_RegP()
3352 %{
3353   constraint(ALLOC_IN_RC(ptr_r15_reg));
3354   match(RegP);
3355   match(rRegP);
3356 
3357   format %{ %}
3358   interface(REG_INTER);
3359 %}
3360 
3361 operand rRegL()
3362 %{
3363   constraint(ALLOC_IN_RC(long_reg));
3364   match(RegL);
3365   match(rax_RegL);
3366   match(rdx_RegL);
3367 
3368   format %{ %}
3369   interface(REG_INTER);
3370 %}
3371 
3372 // Special Registers
3373 operand no_rax_rdx_RegL()
3374 %{
3375   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3376   match(RegL);
3377   match(rRegL);
3378 
3379   format %{ %}
3380   interface(REG_INTER);
3381 %}
3382 
3383 operand no_rax_RegL()
3384 %{
3385   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3386   match(RegL);
3387   match(rRegL);
3388   match(rdx_RegL);
3389 
3390   format %{ %}
3391   interface(REG_INTER);
3392 %}
3393 
3394 operand no_rcx_RegL()
3395 %{
3396   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3397   match(RegL);
3398   match(rRegL);
3399 
3400   format %{ %}
3401   interface(REG_INTER);
3402 %}
3403 
3404 operand rax_RegL()
3405 %{
3406   constraint(ALLOC_IN_RC(long_rax_reg));
3407   match(RegL);
3408   match(rRegL);
3409 
3410   format %{ "RAX" %}
3411   interface(REG_INTER);
3412 %}
3413 
3414 operand rcx_RegL()
3415 %{
3416   constraint(ALLOC_IN_RC(long_rcx_reg));
3417   match(RegL);
3418   match(rRegL);
3419 
3420   format %{ %}
3421   interface(REG_INTER);
3422 %}
3423 
3424 operand rdx_RegL()
3425 %{
3426   constraint(ALLOC_IN_RC(long_rdx_reg));
3427   match(RegL);
3428   match(rRegL);
3429 
3430   format %{ %}
3431   interface(REG_INTER);
3432 %}
3433 
3434 // Flags register, used as output of compare instructions
3435 operand rFlagsReg()
3436 %{
3437   constraint(ALLOC_IN_RC(int_flags));
3438   match(RegFlags);
3439 
3440   format %{ "RFLAGS" %}
3441   interface(REG_INTER);
3442 %}
3443 
3444 // Flags register, used as output of FLOATING POINT compare instructions
3445 operand rFlagsRegU()
3446 %{
3447   constraint(ALLOC_IN_RC(int_flags));
3448   match(RegFlags);
3449 
3450   format %{ "RFLAGS_U" %}
3451   interface(REG_INTER);
3452 %}
3453 
3454 operand rFlagsRegUCF() %{
3455   constraint(ALLOC_IN_RC(int_flags));
3456   match(RegFlags);
3457   predicate(false);
3458 
3459   format %{ "RFLAGS_U_CF" %}
3460   interface(REG_INTER);
3461 %}
3462 
3463 // Float register operands
3464 operand regF()
3465 %{
3466   constraint(ALLOC_IN_RC(float_reg));
3467   match(RegF);
3468 
3469   format %{ %}
3470   interface(REG_INTER);
3471 %}
3472 
3473 // Double register operands
3474 operand regD()
3475 %{
3476   constraint(ALLOC_IN_RC(double_reg));
3477   match(RegD);
3478 
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 //----------Memory Operands----------------------------------------------------
3484 // Direct Memory Operand
3485 // operand direct(immP addr)
3486 // %{
3487 //   match(addr);
3488 
3489 //   format %{ "[$addr]" %}
3490 //   interface(MEMORY_INTER) %{
3491 //     base(0xFFFFFFFF);
3492 //     index(0x4);
3493 //     scale(0x0);
3494 //     disp($addr);
3495 //   %}
3496 // %}
3497 
3498 // Indirect Memory Operand
3499 operand indirect(any_RegP reg)
3500 %{
3501   constraint(ALLOC_IN_RC(ptr_reg));
3502   match(reg);
3503 
3504   format %{ "[$reg]" %}
3505   interface(MEMORY_INTER) %{
3506     base($reg);
3507     index(0x4);
3508     scale(0x0);
3509     disp(0x0);
3510   %}
3511 %}
3512 
3513 // Indirect Memory Plus Short Offset Operand
3514 operand indOffset8(any_RegP reg, immL8 off)
3515 %{
3516   constraint(ALLOC_IN_RC(ptr_reg));
3517   match(AddP reg off);
3518 
3519   format %{ "[$reg + $off (8-bit)]" %}
3520   interface(MEMORY_INTER) %{
3521     base($reg);
3522     index(0x4);
3523     scale(0x0);
3524     disp($off);
3525   %}
3526 %}
3527 
3528 // Indirect Memory Plus Long Offset Operand
3529 operand indOffset32(any_RegP reg, immL32 off)
3530 %{
3531   constraint(ALLOC_IN_RC(ptr_reg));
3532   match(AddP reg off);
3533 
3534   format %{ "[$reg + $off (32-bit)]" %}
3535   interface(MEMORY_INTER) %{
3536     base($reg);
3537     index(0x4);
3538     scale(0x0);
3539     disp($off);
3540   %}
3541 %}
3542 
3543 // Indirect Memory Plus Index Register Plus Offset Operand
3544 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3545 %{
3546   constraint(ALLOC_IN_RC(ptr_reg));
3547   match(AddP (AddP reg lreg) off);
3548 
3549   op_cost(10);
3550   format %{"[$reg + $off + $lreg]" %}
3551   interface(MEMORY_INTER) %{
3552     base($reg);
3553     index($lreg);
3554     scale(0x0);
3555     disp($off);
3556   %}
3557 %}
3558 
3559 // Indirect Memory Plus Index Register Plus Offset Operand
3560 operand indIndex(any_RegP reg, rRegL lreg)
3561 %{
3562   constraint(ALLOC_IN_RC(ptr_reg));
3563   match(AddP reg lreg);
3564 
3565   op_cost(10);
3566   format %{"[$reg + $lreg]" %}
3567   interface(MEMORY_INTER) %{
3568     base($reg);
3569     index($lreg);
3570     scale(0x0);
3571     disp(0x0);
3572   %}
3573 %}
3574 
3575 // Indirect Memory Times Scale Plus Index Register
3576 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3577 %{
3578   constraint(ALLOC_IN_RC(ptr_reg));
3579   match(AddP reg (LShiftL lreg scale));
3580 
3581   op_cost(10);
3582   format %{"[$reg + $lreg << $scale]" %}
3583   interface(MEMORY_INTER) %{
3584     base($reg);
3585     index($lreg);
3586     scale($scale);
3587     disp(0x0);
3588   %}
3589 %}
3590 
3591 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3592 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3593 %{
3594   constraint(ALLOC_IN_RC(ptr_reg));
3595   match(AddP (AddP reg (LShiftL lreg scale)) off);
3596 
3597   op_cost(10);
3598   format %{"[$reg + $off + $lreg << $scale]" %}
3599   interface(MEMORY_INTER) %{
3600     base($reg);
3601     index($lreg);
3602     scale($scale);
3603     disp($off);
3604   %}
3605 %}
3606 
3607 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3608 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3609 %{
3610   constraint(ALLOC_IN_RC(ptr_reg));
3611   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3612   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3613 
3614   op_cost(10);
3615   format %{"[$reg + $off + $idx << $scale]" %}
3616   interface(MEMORY_INTER) %{
3617     base($reg);
3618     index($idx);
3619     scale($scale);
3620     disp($off);
3621   %}
3622 %}
3623 
3624 // Indirect Narrow Oop Plus Offset Operand
3625 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3626 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3627 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3628   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3629   constraint(ALLOC_IN_RC(ptr_reg));
3630   match(AddP (DecodeN reg) off);
3631 
3632   op_cost(10);
3633   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3634   interface(MEMORY_INTER) %{
3635     base(0xc); // R12
3636     index($reg);
3637     scale(0x3);
3638     disp($off);
3639   %}
3640 %}
3641 
3642 // Indirect Memory Operand
3643 operand indirectNarrow(rRegN reg)
3644 %{
3645   predicate(Universe::narrow_oop_shift() == 0);
3646   constraint(ALLOC_IN_RC(ptr_reg));
3647   match(DecodeN reg);
3648 
3649   format %{ "[$reg]" %}
3650   interface(MEMORY_INTER) %{
3651     base($reg);
3652     index(0x4);
3653     scale(0x0);
3654     disp(0x0);
3655   %}
3656 %}
3657 
3658 // Indirect Memory Plus Short Offset Operand
3659 operand indOffset8Narrow(rRegN reg, immL8 off)
3660 %{
3661   predicate(Universe::narrow_oop_shift() == 0);
3662   constraint(ALLOC_IN_RC(ptr_reg));
3663   match(AddP (DecodeN reg) off);
3664 
3665   format %{ "[$reg + $off (8-bit)]" %}
3666   interface(MEMORY_INTER) %{
3667     base($reg);
3668     index(0x4);
3669     scale(0x0);
3670     disp($off);
3671   %}
3672 %}
3673 
3674 // Indirect Memory Plus Long Offset Operand
3675 operand indOffset32Narrow(rRegN reg, immL32 off)
3676 %{
3677   predicate(Universe::narrow_oop_shift() == 0);
3678   constraint(ALLOC_IN_RC(ptr_reg));
3679   match(AddP (DecodeN reg) off);
3680 
3681   format %{ "[$reg + $off (32-bit)]" %}
3682   interface(MEMORY_INTER) %{
3683     base($reg);
3684     index(0x4);
3685     scale(0x0);
3686     disp($off);
3687   %}
3688 %}
3689 
3690 // Indirect Memory Plus Index Register Plus Offset Operand
3691 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3692 %{
3693   predicate(Universe::narrow_oop_shift() == 0);
3694   constraint(ALLOC_IN_RC(ptr_reg));
3695   match(AddP (AddP (DecodeN reg) lreg) off);
3696 
3697   op_cost(10);
3698   format %{"[$reg + $off + $lreg]" %}
3699   interface(MEMORY_INTER) %{
3700     base($reg);
3701     index($lreg);
3702     scale(0x0);
3703     disp($off);
3704   %}
3705 %}
3706 
3707 // Indirect Memory Plus Index Register Plus Offset Operand
3708 operand indIndexNarrow(rRegN reg, rRegL lreg)
3709 %{
3710   predicate(Universe::narrow_oop_shift() == 0);
3711   constraint(ALLOC_IN_RC(ptr_reg));
3712   match(AddP (DecodeN reg) lreg);
3713 
3714   op_cost(10);
3715   format %{"[$reg + $lreg]" %}
3716   interface(MEMORY_INTER) %{
3717     base($reg);
3718     index($lreg);
3719     scale(0x0);
3720     disp(0x0);
3721   %}
3722 %}
3723 
3724 // Indirect Memory Times Scale Plus Index Register
3725 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3726 %{
3727   predicate(Universe::narrow_oop_shift() == 0);
3728   constraint(ALLOC_IN_RC(ptr_reg));
3729   match(AddP (DecodeN reg) (LShiftL lreg scale));
3730 
3731   op_cost(10);
3732   format %{"[$reg + $lreg << $scale]" %}
3733   interface(MEMORY_INTER) %{
3734     base($reg);
3735     index($lreg);
3736     scale($scale);
3737     disp(0x0);
3738   %}
3739 %}
3740 
3741 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3742 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3743 %{
3744   predicate(Universe::narrow_oop_shift() == 0);
3745   constraint(ALLOC_IN_RC(ptr_reg));
3746   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3747 
3748   op_cost(10);
3749   format %{"[$reg + $off + $lreg << $scale]" %}
3750   interface(MEMORY_INTER) %{
3751     base($reg);
3752     index($lreg);
3753     scale($scale);
3754     disp($off);
3755   %}
3756 %}
3757 
3758 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3759 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3760 %{
3761   constraint(ALLOC_IN_RC(ptr_reg));
3762   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3763   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3764 
3765   op_cost(10);
3766   format %{"[$reg + $off + $idx << $scale]" %}
3767   interface(MEMORY_INTER) %{
3768     base($reg);
3769     index($idx);
3770     scale($scale);
3771     disp($off);
3772   %}
3773 %}
3774 
3775 //----------Special Memory Operands--------------------------------------------
3776 // Stack Slot Operand - This operand is used for loading and storing temporary
3777 //                      values on the stack where a match requires a value to
3778 //                      flow through memory.
3779 operand stackSlotP(sRegP reg)
3780 %{
3781   constraint(ALLOC_IN_RC(stack_slots));
3782   // No match rule because this operand is only generated in matching
3783 
3784   format %{ "[$reg]" %}
3785   interface(MEMORY_INTER) %{
3786     base(0x4);   // RSP
3787     index(0x4);  // No Index
3788     scale(0x0);  // No Scale
3789     disp($reg);  // Stack Offset
3790   %}
3791 %}
3792 
3793 operand stackSlotI(sRegI reg)
3794 %{
3795   constraint(ALLOC_IN_RC(stack_slots));
3796   // No match rule because this operand is only generated in matching
3797 
3798   format %{ "[$reg]" %}
3799   interface(MEMORY_INTER) %{
3800     base(0x4);   // RSP
3801     index(0x4);  // No Index
3802     scale(0x0);  // No Scale
3803     disp($reg);  // Stack Offset
3804   %}
3805 %}
3806 
3807 operand stackSlotF(sRegF reg)
3808 %{
3809   constraint(ALLOC_IN_RC(stack_slots));
3810   // No match rule because this operand is only generated in matching
3811 
3812   format %{ "[$reg]" %}
3813   interface(MEMORY_INTER) %{
3814     base(0x4);   // RSP
3815     index(0x4);  // No Index
3816     scale(0x0);  // No Scale
3817     disp($reg);  // Stack Offset
3818   %}
3819 %}
3820 
3821 operand stackSlotD(sRegD reg)
3822 %{
3823   constraint(ALLOC_IN_RC(stack_slots));
3824   // No match rule because this operand is only generated in matching
3825 
3826   format %{ "[$reg]" %}
3827   interface(MEMORY_INTER) %{
3828     base(0x4);   // RSP
3829     index(0x4);  // No Index
3830     scale(0x0);  // No Scale
3831     disp($reg);  // Stack Offset
3832   %}
3833 %}
3834 operand stackSlotL(sRegL reg)
3835 %{
3836   constraint(ALLOC_IN_RC(stack_slots));
3837   // No match rule because this operand is only generated in matching
3838 
3839   format %{ "[$reg]" %}
3840   interface(MEMORY_INTER) %{
3841     base(0x4);   // RSP
3842     index(0x4);  // No Index
3843     scale(0x0);  // No Scale
3844     disp($reg);  // Stack Offset
3845   %}
3846 %}
3847 
3848 //----------Conditional Branch Operands----------------------------------------
3849 // Comparison Op  - This is the operation of the comparison, and is limited to
3850 //                  the following set of codes:
3851 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
3852 //
3853 // Other attributes of the comparison, such as unsignedness, are specified
3854 // by the comparison instruction that sets a condition code flags register.
3855 // That result is represented by a flags operand whose subtype is appropriate
3856 // to the unsignedness (etc.) of the comparison.
3857 //
3858 // Later, the instruction which matches both the Comparison Op (a Bool) and
3859 // the flags (produced by the Cmp) specifies the coding of the comparison op
3860 // by matching a specific subtype of Bool operand below, such as cmpOpU.
3861 
3862 // Comparision Code
3863 operand cmpOp()
3864 %{
3865   match(Bool);
3866 
3867   format %{ "" %}
3868   interface(COND_INTER) %{
3869     equal(0x4, "e");
3870     not_equal(0x5, "ne");
3871     less(0xC, "l");
3872     greater_equal(0xD, "ge");
3873     less_equal(0xE, "le");
3874     greater(0xF, "g");
3875     overflow(0x0, "o");
3876     no_overflow(0x1, "no");
3877   %}
3878 %}
3879 
3880 // Comparison Code, unsigned compare.  Used by FP also, with
3881 // C2 (unordered) turned into GT or LT already.  The other bits
3882 // C0 and C3 are turned into Carry & Zero flags.
3883 operand cmpOpU()
3884 %{
3885   match(Bool);
3886 
3887   format %{ "" %}
3888   interface(COND_INTER) %{
3889     equal(0x4, "e");
3890     not_equal(0x5, "ne");
3891     less(0x2, "b");
3892     greater_equal(0x3, "nb");
3893     less_equal(0x6, "be");
3894     greater(0x7, "nbe");
3895     overflow(0x0, "o");
3896     no_overflow(0x1, "no");
3897   %}
3898 %}
3899 
3900 
3901 // Floating comparisons that don't require any fixup for the unordered case
3902 operand cmpOpUCF() %{
3903   match(Bool);
3904   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
3905             n->as_Bool()->_test._test == BoolTest::ge ||
3906             n->as_Bool()->_test._test == BoolTest::le ||
3907             n->as_Bool()->_test._test == BoolTest::gt);
3908   format %{ "" %}
3909   interface(COND_INTER) %{
3910     equal(0x4, "e");
3911     not_equal(0x5, "ne");
3912     less(0x2, "b");
3913     greater_equal(0x3, "nb");
3914     less_equal(0x6, "be");
3915     greater(0x7, "nbe");
3916     overflow(0x0, "o");
3917     no_overflow(0x1, "no");
3918   %}
3919 %}
3920 
3921 
3922 // Floating comparisons that can be fixed up with extra conditional jumps
3923 operand cmpOpUCF2() %{
3924   match(Bool);
3925   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
3926             n->as_Bool()->_test._test == BoolTest::eq);
3927   format %{ "" %}
3928   interface(COND_INTER) %{
3929     equal(0x4, "e");
3930     not_equal(0x5, "ne");
3931     less(0x2, "b");
3932     greater_equal(0x3, "nb");
3933     less_equal(0x6, "be");
3934     greater(0x7, "nbe");
3935     overflow(0x0, "o");
3936     no_overflow(0x1, "no");
3937   %}
3938 %}
3939 
3940 
3941 //----------OPERAND CLASSES----------------------------------------------------
3942 // Operand Classes are groups of operands that are used as to simplify
3943 // instruction definitions by not requiring the AD writer to specify separate
3944 // instructions for every form of operand when the instruction accepts
3945 // multiple operand types with the same basic encoding and format.  The classic
3946 // case of this is memory operands.
3947 
3948 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
3949                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
3950                indCompressedOopOffset,
3951                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
3952                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
3953                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
3954 
3955 //----------PIPELINE-----------------------------------------------------------
3956 // Rules which define the behavior of the target architectures pipeline.
3957 pipeline %{
3958 
3959 //----------ATTRIBUTES---------------------------------------------------------
3960 attributes %{
3961   variable_size_instructions;        // Fixed size instructions
3962   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
3963   instruction_unit_size = 1;         // An instruction is 1 bytes long
3964   instruction_fetch_unit_size = 16;  // The processor fetches one line
3965   instruction_fetch_units = 1;       // of 16 bytes
3966 
3967   // List of nop instructions
3968   nops( MachNop );
3969 %}
3970 
3971 //----------RESOURCES----------------------------------------------------------
3972 // Resources are the functional units available to the machine
3973 
3974 // Generic P2/P3 pipeline
3975 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
3976 // 3 instructions decoded per cycle.
3977 // 2 load/store ops per cycle, 1 branch, 1 FPU,
3978 // 3 ALU op, only ALU0 handles mul instructions.
3979 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
3980            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
3981            BR, FPU,
3982            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
3983 
3984 //----------PIPELINE DESCRIPTION-----------------------------------------------
3985 // Pipeline Description specifies the stages in the machine's pipeline
3986 
3987 // Generic P2/P3 pipeline
3988 pipe_desc(S0, S1, S2, S3, S4, S5);
3989 
3990 //----------PIPELINE CLASSES---------------------------------------------------
3991 // Pipeline Classes describe the stages in which input and output are
3992 // referenced by the hardware pipeline.
3993 
3994 // Naming convention: ialu or fpu
3995 // Then: _reg
3996 // Then: _reg if there is a 2nd register
3997 // Then: _long if it's a pair of instructions implementing a long
3998 // Then: _fat if it requires the big decoder
3999 //   Or: _mem if it requires the big decoder and a memory unit.
4000 
4001 // Integer ALU reg operation
4002 pipe_class ialu_reg(rRegI dst)
4003 %{
4004     single_instruction;
4005     dst    : S4(write);
4006     dst    : S3(read);
4007     DECODE : S0;        // any decoder
4008     ALU    : S3;        // any alu
4009 %}
4010 
4011 // Long ALU reg operation
4012 pipe_class ialu_reg_long(rRegL dst)
4013 %{
4014     instruction_count(2);
4015     dst    : S4(write);
4016     dst    : S3(read);
4017     DECODE : S0(2);     // any 2 decoders
4018     ALU    : S3(2);     // both alus
4019 %}
4020 
4021 // Integer ALU reg operation using big decoder
4022 pipe_class ialu_reg_fat(rRegI dst)
4023 %{
4024     single_instruction;
4025     dst    : S4(write);
4026     dst    : S3(read);
4027     D0     : S0;        // big decoder only
4028     ALU    : S3;        // any alu
4029 %}
4030 
4031 // Long ALU reg operation using big decoder
4032 pipe_class ialu_reg_long_fat(rRegL dst)
4033 %{
4034     instruction_count(2);
4035     dst    : S4(write);
4036     dst    : S3(read);
4037     D0     : S0(2);     // big decoder only; twice
4038     ALU    : S3(2);     // any 2 alus
4039 %}
4040 
4041 // Integer ALU reg-reg operation
4042 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4043 %{
4044     single_instruction;
4045     dst    : S4(write);
4046     src    : S3(read);
4047     DECODE : S0;        // any decoder
4048     ALU    : S3;        // any alu
4049 %}
4050 
4051 // Long ALU reg-reg operation
4052 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4053 %{
4054     instruction_count(2);
4055     dst    : S4(write);
4056     src    : S3(read);
4057     DECODE : S0(2);     // any 2 decoders
4058     ALU    : S3(2);     // both alus
4059 %}
4060 
4061 // Integer ALU reg-reg operation
4062 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4063 %{
4064     single_instruction;
4065     dst    : S4(write);
4066     src    : S3(read);
4067     D0     : S0;        // big decoder only
4068     ALU    : S3;        // any alu
4069 %}
4070 
4071 // Long ALU reg-reg operation
4072 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4073 %{
4074     instruction_count(2);
4075     dst    : S4(write);
4076     src    : S3(read);
4077     D0     : S0(2);     // big decoder only; twice
4078     ALU    : S3(2);     // both alus
4079 %}
4080 
4081 // Integer ALU reg-mem operation
4082 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4083 %{
4084     single_instruction;
4085     dst    : S5(write);
4086     mem    : S3(read);
4087     D0     : S0;        // big decoder only
4088     ALU    : S4;        // any alu
4089     MEM    : S3;        // any mem
4090 %}
4091 
4092 // Integer mem operation (prefetch)
4093 pipe_class ialu_mem(memory mem)
4094 %{
4095     single_instruction;
4096     mem    : S3(read);
4097     D0     : S0;        // big decoder only
4098     MEM    : S3;        // any mem
4099 %}
4100 
4101 // Integer Store to Memory
4102 pipe_class ialu_mem_reg(memory mem, rRegI src)
4103 %{
4104     single_instruction;
4105     mem    : S3(read);
4106     src    : S5(read);
4107     D0     : S0;        // big decoder only
4108     ALU    : S4;        // any alu
4109     MEM    : S3;
4110 %}
4111 
4112 // // Long Store to Memory
4113 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4114 // %{
4115 //     instruction_count(2);
4116 //     mem    : S3(read);
4117 //     src    : S5(read);
4118 //     D0     : S0(2);          // big decoder only; twice
4119 //     ALU    : S4(2);     // any 2 alus
4120 //     MEM    : S3(2);  // Both mems
4121 // %}
4122 
4123 // Integer Store to Memory
4124 pipe_class ialu_mem_imm(memory mem)
4125 %{
4126     single_instruction;
4127     mem    : S3(read);
4128     D0     : S0;        // big decoder only
4129     ALU    : S4;        // any alu
4130     MEM    : S3;
4131 %}
4132 
4133 // Integer ALU0 reg-reg operation
4134 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4135 %{
4136     single_instruction;
4137     dst    : S4(write);
4138     src    : S3(read);
4139     D0     : S0;        // Big decoder only
4140     ALU0   : S3;        // only alu0
4141 %}
4142 
4143 // Integer ALU0 reg-mem operation
4144 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4145 %{
4146     single_instruction;
4147     dst    : S5(write);
4148     mem    : S3(read);
4149     D0     : S0;        // big decoder only
4150     ALU0   : S4;        // ALU0 only
4151     MEM    : S3;        // any mem
4152 %}
4153 
4154 // Integer ALU reg-reg operation
4155 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4156 %{
4157     single_instruction;
4158     cr     : S4(write);
4159     src1   : S3(read);
4160     src2   : S3(read);
4161     DECODE : S0;        // any decoder
4162     ALU    : S3;        // any alu
4163 %}
4164 
4165 // Integer ALU reg-imm operation
4166 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4167 %{
4168     single_instruction;
4169     cr     : S4(write);
4170     src1   : S3(read);
4171     DECODE : S0;        // any decoder
4172     ALU    : S3;        // any alu
4173 %}
4174 
4175 // Integer ALU reg-mem operation
4176 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4177 %{
4178     single_instruction;
4179     cr     : S4(write);
4180     src1   : S3(read);
4181     src2   : S3(read);
4182     D0     : S0;        // big decoder only
4183     ALU    : S4;        // any alu
4184     MEM    : S3;
4185 %}
4186 
4187 // Conditional move reg-reg
4188 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4189 %{
4190     instruction_count(4);
4191     y      : S4(read);
4192     q      : S3(read);
4193     p      : S3(read);
4194     DECODE : S0(4);     // any decoder
4195 %}
4196 
4197 // Conditional move reg-reg
4198 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4199 %{
4200     single_instruction;
4201     dst    : S4(write);
4202     src    : S3(read);
4203     cr     : S3(read);
4204     DECODE : S0;        // any decoder
4205 %}
4206 
4207 // Conditional move reg-mem
4208 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4209 %{
4210     single_instruction;
4211     dst    : S4(write);
4212     src    : S3(read);
4213     cr     : S3(read);
4214     DECODE : S0;        // any decoder
4215     MEM    : S3;
4216 %}
4217 
4218 // Conditional move reg-reg long
4219 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4220 %{
4221     single_instruction;
4222     dst    : S4(write);
4223     src    : S3(read);
4224     cr     : S3(read);
4225     DECODE : S0(2);     // any 2 decoders
4226 %}
4227 
4228 // XXX
4229 // // Conditional move double reg-reg
4230 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4231 // %{
4232 //     single_instruction;
4233 //     dst    : S4(write);
4234 //     src    : S3(read);
4235 //     cr     : S3(read);
4236 //     DECODE : S0;     // any decoder
4237 // %}
4238 
4239 // Float reg-reg operation
4240 pipe_class fpu_reg(regD dst)
4241 %{
4242     instruction_count(2);
4243     dst    : S3(read);
4244     DECODE : S0(2);     // any 2 decoders
4245     FPU    : S3;
4246 %}
4247 
4248 // Float reg-reg operation
4249 pipe_class fpu_reg_reg(regD dst, regD src)
4250 %{
4251     instruction_count(2);
4252     dst    : S4(write);
4253     src    : S3(read);
4254     DECODE : S0(2);     // any 2 decoders
4255     FPU    : S3;
4256 %}
4257 
4258 // Float reg-reg operation
4259 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4260 %{
4261     instruction_count(3);
4262     dst    : S4(write);
4263     src1   : S3(read);
4264     src2   : S3(read);
4265     DECODE : S0(3);     // any 3 decoders
4266     FPU    : S3(2);
4267 %}
4268 
4269 // Float reg-reg operation
4270 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4271 %{
4272     instruction_count(4);
4273     dst    : S4(write);
4274     src1   : S3(read);
4275     src2   : S3(read);
4276     src3   : S3(read);
4277     DECODE : S0(4);     // any 3 decoders
4278     FPU    : S3(2);
4279 %}
4280 
4281 // Float reg-reg operation
4282 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4283 %{
4284     instruction_count(4);
4285     dst    : S4(write);
4286     src1   : S3(read);
4287     src2   : S3(read);
4288     src3   : S3(read);
4289     DECODE : S1(3);     // any 3 decoders
4290     D0     : S0;        // Big decoder only
4291     FPU    : S3(2);
4292     MEM    : S3;
4293 %}
4294 
4295 // Float reg-mem operation
4296 pipe_class fpu_reg_mem(regD dst, memory mem)
4297 %{
4298     instruction_count(2);
4299     dst    : S5(write);
4300     mem    : S3(read);
4301     D0     : S0;        // big decoder only
4302     DECODE : S1;        // any decoder for FPU POP
4303     FPU    : S4;
4304     MEM    : S3;        // any mem
4305 %}
4306 
4307 // Float reg-mem operation
4308 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4309 %{
4310     instruction_count(3);
4311     dst    : S5(write);
4312     src1   : S3(read);
4313     mem    : S3(read);
4314     D0     : S0;        // big decoder only
4315     DECODE : S1(2);     // any decoder for FPU POP
4316     FPU    : S4;
4317     MEM    : S3;        // any mem
4318 %}
4319 
4320 // Float mem-reg operation
4321 pipe_class fpu_mem_reg(memory mem, regD src)
4322 %{
4323     instruction_count(2);
4324     src    : S5(read);
4325     mem    : S3(read);
4326     DECODE : S0;        // any decoder for FPU PUSH
4327     D0     : S1;        // big decoder only
4328     FPU    : S4;
4329     MEM    : S3;        // any mem
4330 %}
4331 
4332 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4333 %{
4334     instruction_count(3);
4335     src1   : S3(read);
4336     src2   : S3(read);
4337     mem    : S3(read);
4338     DECODE : S0(2);     // any decoder for FPU PUSH
4339     D0     : S1;        // big decoder only
4340     FPU    : S4;
4341     MEM    : S3;        // any mem
4342 %}
4343 
4344 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4345 %{
4346     instruction_count(3);
4347     src1   : S3(read);
4348     src2   : S3(read);
4349     mem    : S4(read);
4350     DECODE : S0;        // any decoder for FPU PUSH
4351     D0     : S0(2);     // big decoder only
4352     FPU    : S4;
4353     MEM    : S3(2);     // any mem
4354 %}
4355 
4356 pipe_class fpu_mem_mem(memory dst, memory src1)
4357 %{
4358     instruction_count(2);
4359     src1   : S3(read);
4360     dst    : S4(read);
4361     D0     : S0(2);     // big decoder only
4362     MEM    : S3(2);     // any mem
4363 %}
4364 
4365 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4366 %{
4367     instruction_count(3);
4368     src1   : S3(read);
4369     src2   : S3(read);
4370     dst    : S4(read);
4371     D0     : S0(3);     // big decoder only
4372     FPU    : S4;
4373     MEM    : S3(3);     // any mem
4374 %}
4375 
4376 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4377 %{
4378     instruction_count(3);
4379     src1   : S4(read);
4380     mem    : S4(read);
4381     DECODE : S0;        // any decoder for FPU PUSH
4382     D0     : S0(2);     // big decoder only
4383     FPU    : S4;
4384     MEM    : S3(2);     // any mem
4385 %}
4386 
4387 // Float load constant
4388 pipe_class fpu_reg_con(regD dst)
4389 %{
4390     instruction_count(2);
4391     dst    : S5(write);
4392     D0     : S0;        // big decoder only for the load
4393     DECODE : S1;        // any decoder for FPU POP
4394     FPU    : S4;
4395     MEM    : S3;        // any mem
4396 %}
4397 
4398 // Float load constant
4399 pipe_class fpu_reg_reg_con(regD dst, regD src)
4400 %{
4401     instruction_count(3);
4402     dst    : S5(write);
4403     src    : S3(read);
4404     D0     : S0;        // big decoder only for the load
4405     DECODE : S1(2);     // any decoder for FPU POP
4406     FPU    : S4;
4407     MEM    : S3;        // any mem
4408 %}
4409 
4410 // UnConditional branch
4411 pipe_class pipe_jmp(label labl)
4412 %{
4413     single_instruction;
4414     BR   : S3;
4415 %}
4416 
4417 // Conditional branch
4418 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4419 %{
4420     single_instruction;
4421     cr    : S1(read);
4422     BR    : S3;
4423 %}
4424 
4425 // Allocation idiom
4426 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4427 %{
4428     instruction_count(1); force_serialization;
4429     fixed_latency(6);
4430     heap_ptr : S3(read);
4431     DECODE   : S0(3);
4432     D0       : S2;
4433     MEM      : S3;
4434     ALU      : S3(2);
4435     dst      : S5(write);
4436     BR       : S5;
4437 %}
4438 
4439 // Generic big/slow expanded idiom
4440 pipe_class pipe_slow()
4441 %{
4442     instruction_count(10); multiple_bundles; force_serialization;
4443     fixed_latency(100);
4444     D0  : S0(2);
4445     MEM : S3(2);
4446 %}
4447 
4448 // The real do-nothing guy
4449 pipe_class empty()
4450 %{
4451     instruction_count(0);
4452 %}
4453 
4454 // Define the class for the Nop node
4455 define
4456 %{
4457    MachNop = empty;
4458 %}
4459 
4460 %}
4461 
4462 //----------INSTRUCTIONS-------------------------------------------------------
4463 //
4464 // match      -- States which machine-independent subtree may be replaced
4465 //               by this instruction.
4466 // ins_cost   -- The estimated cost of this instruction is used by instruction
4467 //               selection to identify a minimum cost tree of machine
4468 //               instructions that matches a tree of machine-independent
4469 //               instructions.
4470 // format     -- A string providing the disassembly for this instruction.
4471 //               The value of an instruction's operand may be inserted
4472 //               by referring to it with a '$' prefix.
4473 // opcode     -- Three instruction opcodes may be provided.  These are referred
4474 //               to within an encode class as $primary, $secondary, and $tertiary
4475 //               rrspectively.  The primary opcode is commonly used to
4476 //               indicate the type of machine instruction, while secondary
4477 //               and tertiary are often used for prefix options or addressing
4478 //               modes.
4479 // ins_encode -- A list of encode classes with parameters. The encode class
4480 //               name must have been defined in an 'enc_class' specification
4481 //               in the encode section of the architecture description.
4482 
4483 
4484 //----------Load/Store/Move Instructions---------------------------------------
4485 //----------Load Instructions--------------------------------------------------
4486 
4487 // Load Byte (8 bit signed)
4488 instruct loadB(rRegI dst, memory mem)
4489 %{
4490   match(Set dst (LoadB mem));
4491 
4492   ins_cost(125);
4493   format %{ "movsbl  $dst, $mem\t# byte" %}
4494 
4495   ins_encode %{
4496     __ movsbl($dst$$Register, $mem$$Address);
4497   %}
4498 
4499   ins_pipe(ialu_reg_mem);
4500 %}
4501 
4502 // Load Byte (8 bit signed) into Long Register
4503 instruct loadB2L(rRegL dst, memory mem)
4504 %{
4505   match(Set dst (ConvI2L (LoadB mem)));
4506 
4507   ins_cost(125);
4508   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4509 
4510   ins_encode %{
4511     __ movsbq($dst$$Register, $mem$$Address);
4512   %}
4513 
4514   ins_pipe(ialu_reg_mem);
4515 %}
4516 
4517 // Load Unsigned Byte (8 bit UNsigned)
4518 instruct loadUB(rRegI dst, memory mem)
4519 %{
4520   match(Set dst (LoadUB mem));
4521 
4522   ins_cost(125);
4523   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4524 
4525   ins_encode %{
4526     __ movzbl($dst$$Register, $mem$$Address);
4527   %}
4528 
4529   ins_pipe(ialu_reg_mem);
4530 %}
4531 
4532 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4533 instruct loadUB2L(rRegL dst, memory mem)
4534 %{
4535   match(Set dst (ConvI2L (LoadUB mem)));
4536 
4537   ins_cost(125);
4538   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4539 
4540   ins_encode %{
4541     __ movzbq($dst$$Register, $mem$$Address);
4542   %}
4543 
4544   ins_pipe(ialu_reg_mem);
4545 %}
4546 
4547 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4548 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4549   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4550   effect(KILL cr);
4551 
4552   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4553             "andl    $dst, $mask" %}
4554   ins_encode %{
4555     Register Rdst = $dst$$Register;
4556     __ movzbq(Rdst, $mem$$Address);
4557     __ andl(Rdst, $mask$$constant);
4558   %}
4559   ins_pipe(ialu_reg_mem);
4560 %}
4561 
4562 // Load Short (16 bit signed)
4563 instruct loadS(rRegI dst, memory mem)
4564 %{
4565   match(Set dst (LoadS mem));
4566 
4567   ins_cost(125);
4568   format %{ "movswl $dst, $mem\t# short" %}
4569 
4570   ins_encode %{
4571     __ movswl($dst$$Register, $mem$$Address);
4572   %}
4573 
4574   ins_pipe(ialu_reg_mem);
4575 %}
4576 
4577 // Load Short (16 bit signed) to Byte (8 bit signed)
4578 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4579   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4580 
4581   ins_cost(125);
4582   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4583   ins_encode %{
4584     __ movsbl($dst$$Register, $mem$$Address);
4585   %}
4586   ins_pipe(ialu_reg_mem);
4587 %}
4588 
4589 // Load Short (16 bit signed) into Long Register
4590 instruct loadS2L(rRegL dst, memory mem)
4591 %{
4592   match(Set dst (ConvI2L (LoadS mem)));
4593 
4594   ins_cost(125);
4595   format %{ "movswq $dst, $mem\t# short -> long" %}
4596 
4597   ins_encode %{
4598     __ movswq($dst$$Register, $mem$$Address);
4599   %}
4600 
4601   ins_pipe(ialu_reg_mem);
4602 %}
4603 
4604 // Load Unsigned Short/Char (16 bit UNsigned)
4605 instruct loadUS(rRegI dst, memory mem)
4606 %{
4607   match(Set dst (LoadUS mem));
4608 
4609   ins_cost(125);
4610   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4611 
4612   ins_encode %{
4613     __ movzwl($dst$$Register, $mem$$Address);
4614   %}
4615 
4616   ins_pipe(ialu_reg_mem);
4617 %}
4618 
4619 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4620 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4621   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4622 
4623   ins_cost(125);
4624   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4625   ins_encode %{
4626     __ movsbl($dst$$Register, $mem$$Address);
4627   %}
4628   ins_pipe(ialu_reg_mem);
4629 %}
4630 
4631 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4632 instruct loadUS2L(rRegL dst, memory mem)
4633 %{
4634   match(Set dst (ConvI2L (LoadUS mem)));
4635 
4636   ins_cost(125);
4637   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4638 
4639   ins_encode %{
4640     __ movzwq($dst$$Register, $mem$$Address);
4641   %}
4642 
4643   ins_pipe(ialu_reg_mem);
4644 %}
4645 
4646 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4647 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4648   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4649 
4650   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4651   ins_encode %{
4652     __ movzbq($dst$$Register, $mem$$Address);
4653   %}
4654   ins_pipe(ialu_reg_mem);
4655 %}
4656 
4657 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4658 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4659   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4660   effect(KILL cr);
4661 
4662   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4663             "andl    $dst, $mask" %}
4664   ins_encode %{
4665     Register Rdst = $dst$$Register;
4666     __ movzwq(Rdst, $mem$$Address);
4667     __ andl(Rdst, $mask$$constant);
4668   %}
4669   ins_pipe(ialu_reg_mem);
4670 %}
4671 
4672 // Load Integer
4673 instruct loadI(rRegI dst, memory mem)
4674 %{
4675   match(Set dst (LoadI mem));
4676 
4677   ins_cost(125);
4678   format %{ "movl    $dst, $mem\t# int" %}
4679 
4680   ins_encode %{
4681     __ movl($dst$$Register, $mem$$Address);
4682   %}
4683 
4684   ins_pipe(ialu_reg_mem);
4685 %}
4686 
4687 // Load Integer (32 bit signed) to Byte (8 bit signed)
4688 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4689   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4690 
4691   ins_cost(125);
4692   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4693   ins_encode %{
4694     __ movsbl($dst$$Register, $mem$$Address);
4695   %}
4696   ins_pipe(ialu_reg_mem);
4697 %}
4698 
4699 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4700 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4701   match(Set dst (AndI (LoadI mem) mask));
4702 
4703   ins_cost(125);
4704   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4705   ins_encode %{
4706     __ movzbl($dst$$Register, $mem$$Address);
4707   %}
4708   ins_pipe(ialu_reg_mem);
4709 %}
4710 
4711 // Load Integer (32 bit signed) to Short (16 bit signed)
4712 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4713   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4714 
4715   ins_cost(125);
4716   format %{ "movswl  $dst, $mem\t# int -> short" %}
4717   ins_encode %{
4718     __ movswl($dst$$Register, $mem$$Address);
4719   %}
4720   ins_pipe(ialu_reg_mem);
4721 %}
4722 
4723 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4724 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4725   match(Set dst (AndI (LoadI mem) mask));
4726 
4727   ins_cost(125);
4728   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4729   ins_encode %{
4730     __ movzwl($dst$$Register, $mem$$Address);
4731   %}
4732   ins_pipe(ialu_reg_mem);
4733 %}
4734 
4735 // Load Integer into Long Register
4736 instruct loadI2L(rRegL dst, memory mem)
4737 %{
4738   match(Set dst (ConvI2L (LoadI mem)));
4739 
4740   ins_cost(125);
4741   format %{ "movslq  $dst, $mem\t# int -> long" %}
4742 
4743   ins_encode %{
4744     __ movslq($dst$$Register, $mem$$Address);
4745   %}
4746 
4747   ins_pipe(ialu_reg_mem);
4748 %}
4749 
4750 // Load Integer with mask 0xFF into Long Register
4751 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4752   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4753 
4754   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4755   ins_encode %{
4756     __ movzbq($dst$$Register, $mem$$Address);
4757   %}
4758   ins_pipe(ialu_reg_mem);
4759 %}
4760 
4761 // Load Integer with mask 0xFFFF into Long Register
4762 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4763   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4764 
4765   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4766   ins_encode %{
4767     __ movzwq($dst$$Register, $mem$$Address);
4768   %}
4769   ins_pipe(ialu_reg_mem);
4770 %}
4771 
4772 // Load Integer with a 31-bit mask into Long Register
4773 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4774   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4775   effect(KILL cr);
4776 
4777   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4778             "andl    $dst, $mask" %}
4779   ins_encode %{
4780     Register Rdst = $dst$$Register;
4781     __ movl(Rdst, $mem$$Address);
4782     __ andl(Rdst, $mask$$constant);
4783   %}
4784   ins_pipe(ialu_reg_mem);
4785 %}
4786 
4787 // Load Unsigned Integer into Long Register
4788 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
4789 %{
4790   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
4791 
4792   ins_cost(125);
4793   format %{ "movl    $dst, $mem\t# uint -> long" %}
4794 
4795   ins_encode %{
4796     __ movl($dst$$Register, $mem$$Address);
4797   %}
4798 
4799   ins_pipe(ialu_reg_mem);
4800 %}
4801 
4802 // Load Long
4803 instruct loadL(rRegL dst, memory mem)
4804 %{
4805   match(Set dst (LoadL mem));
4806 
4807   ins_cost(125);
4808   format %{ "movq    $dst, $mem\t# long" %}
4809 
4810   ins_encode %{
4811     __ movq($dst$$Register, $mem$$Address);
4812   %}
4813 
4814   ins_pipe(ialu_reg_mem); // XXX
4815 %}
4816 
4817 // Load Range
4818 instruct loadRange(rRegI dst, memory mem)
4819 %{
4820   match(Set dst (LoadRange mem));
4821 
4822   ins_cost(125); // XXX
4823   format %{ "movl    $dst, $mem\t# range" %}
4824   opcode(0x8B);
4825   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
4826   ins_pipe(ialu_reg_mem);
4827 %}
4828 
4829 // Load Pointer
4830 instruct loadP(rRegP dst, memory mem)
4831 %{
4832   match(Set dst (LoadP mem));
4833 
4834   ins_cost(125); // XXX
4835   format %{ "movq    $dst, $mem\t# ptr" %}
4836   opcode(0x8B);
4837   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4838   ins_pipe(ialu_reg_mem); // XXX
4839 %}
4840 
4841 // Load Compressed Pointer
4842 instruct loadN(rRegN dst, memory mem)
4843 %{
4844    match(Set dst (LoadN mem));
4845 
4846    ins_cost(125); // XXX
4847    format %{ "movl    $dst, $mem\t# compressed ptr" %}
4848    ins_encode %{
4849      __ movl($dst$$Register, $mem$$Address);
4850    %}
4851    ins_pipe(ialu_reg_mem); // XXX
4852 %}
4853 
4854 
4855 // Load Klass Pointer
4856 instruct loadKlass(rRegP dst, memory mem)
4857 %{
4858   match(Set dst (LoadKlass mem));
4859 
4860   ins_cost(125); // XXX
4861   format %{ "movq    $dst, $mem\t# class" %}
4862   opcode(0x8B);
4863   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4864   ins_pipe(ialu_reg_mem); // XXX
4865 %}
4866 
4867 // Load narrow Klass Pointer
4868 instruct loadNKlass(rRegN dst, memory mem)
4869 %{
4870   match(Set dst (LoadNKlass mem));
4871 
4872   ins_cost(125); // XXX
4873   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
4874   ins_encode %{
4875     __ movl($dst$$Register, $mem$$Address);
4876   %}
4877   ins_pipe(ialu_reg_mem); // XXX
4878 %}
4879 
4880 // Load Float
4881 instruct loadF(regF dst, memory mem)
4882 %{
4883   match(Set dst (LoadF mem));
4884 
4885   ins_cost(145); // XXX
4886   format %{ "movss   $dst, $mem\t# float" %}
4887   ins_encode %{
4888     __ movflt($dst$$XMMRegister, $mem$$Address);
4889   %}
4890   ins_pipe(pipe_slow); // XXX
4891 %}
4892 
4893 // Load Double
4894 instruct loadD_partial(regD dst, memory mem)
4895 %{
4896   predicate(!UseXmmLoadAndClearUpper);
4897   match(Set dst (LoadD mem));
4898 
4899   ins_cost(145); // XXX
4900   format %{ "movlpd  $dst, $mem\t# double" %}
4901   ins_encode %{
4902     __ movdbl($dst$$XMMRegister, $mem$$Address);
4903   %}
4904   ins_pipe(pipe_slow); // XXX
4905 %}
4906 
4907 instruct loadD(regD dst, memory mem)
4908 %{
4909   predicate(UseXmmLoadAndClearUpper);
4910   match(Set dst (LoadD mem));
4911 
4912   ins_cost(145); // XXX
4913   format %{ "movsd   $dst, $mem\t# double" %}
4914   ins_encode %{
4915     __ movdbl($dst$$XMMRegister, $mem$$Address);
4916   %}
4917   ins_pipe(pipe_slow); // XXX
4918 %}
4919 
4920 // Load Effective Address
4921 instruct leaP8(rRegP dst, indOffset8 mem)
4922 %{
4923   match(Set dst mem);
4924 
4925   ins_cost(110); // XXX
4926   format %{ "leaq    $dst, $mem\t# ptr 8" %}
4927   opcode(0x8D);
4928   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4929   ins_pipe(ialu_reg_reg_fat);
4930 %}
4931 
4932 instruct leaP32(rRegP dst, indOffset32 mem)
4933 %{
4934   match(Set dst mem);
4935 
4936   ins_cost(110);
4937   format %{ "leaq    $dst, $mem\t# ptr 32" %}
4938   opcode(0x8D);
4939   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4940   ins_pipe(ialu_reg_reg_fat);
4941 %}
4942 
4943 // instruct leaPIdx(rRegP dst, indIndex mem)
4944 // %{
4945 //   match(Set dst mem);
4946 
4947 //   ins_cost(110);
4948 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
4949 //   opcode(0x8D);
4950 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4951 //   ins_pipe(ialu_reg_reg_fat);
4952 // %}
4953 
4954 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
4955 %{
4956   match(Set dst mem);
4957 
4958   ins_cost(110);
4959   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
4960   opcode(0x8D);
4961   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4962   ins_pipe(ialu_reg_reg_fat);
4963 %}
4964 
4965 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
4966 %{
4967   match(Set dst mem);
4968 
4969   ins_cost(110);
4970   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
4971   opcode(0x8D);
4972   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4973   ins_pipe(ialu_reg_reg_fat);
4974 %}
4975 
4976 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
4977 %{
4978   match(Set dst mem);
4979 
4980   ins_cost(110);
4981   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
4982   opcode(0x8D);
4983   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4984   ins_pipe(ialu_reg_reg_fat);
4985 %}
4986 
4987 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
4988 %{
4989   match(Set dst mem);
4990 
4991   ins_cost(110);
4992   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
4993   opcode(0x8D);
4994   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
4995   ins_pipe(ialu_reg_reg_fat);
4996 %}
4997 
4998 // Load Effective Address which uses Narrow (32-bits) oop
4999 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5000 %{
5001   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5002   match(Set dst mem);
5003 
5004   ins_cost(110);
5005   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5006   opcode(0x8D);
5007   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5008   ins_pipe(ialu_reg_reg_fat);
5009 %}
5010 
5011 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5012 %{
5013   predicate(Universe::narrow_oop_shift() == 0);
5014   match(Set dst mem);
5015 
5016   ins_cost(110); // XXX
5017   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5018   opcode(0x8D);
5019   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5020   ins_pipe(ialu_reg_reg_fat);
5021 %}
5022 
5023 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5024 %{
5025   predicate(Universe::narrow_oop_shift() == 0);
5026   match(Set dst mem);
5027 
5028   ins_cost(110);
5029   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5030   opcode(0x8D);
5031   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5032   ins_pipe(ialu_reg_reg_fat);
5033 %}
5034 
5035 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5036 %{
5037   predicate(Universe::narrow_oop_shift() == 0);
5038   match(Set dst mem);
5039 
5040   ins_cost(110);
5041   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5042   opcode(0x8D);
5043   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5044   ins_pipe(ialu_reg_reg_fat);
5045 %}
5046 
5047 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5048 %{
5049   predicate(Universe::narrow_oop_shift() == 0);
5050   match(Set dst mem);
5051 
5052   ins_cost(110);
5053   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5054   opcode(0x8D);
5055   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5056   ins_pipe(ialu_reg_reg_fat);
5057 %}
5058 
5059 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5060 %{
5061   predicate(Universe::narrow_oop_shift() == 0);
5062   match(Set dst mem);
5063 
5064   ins_cost(110);
5065   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5066   opcode(0x8D);
5067   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5068   ins_pipe(ialu_reg_reg_fat);
5069 %}
5070 
5071 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5072 %{
5073   predicate(Universe::narrow_oop_shift() == 0);
5074   match(Set dst mem);
5075 
5076   ins_cost(110);
5077   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5078   opcode(0x8D);
5079   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5080   ins_pipe(ialu_reg_reg_fat);
5081 %}
5082 
5083 instruct loadConI(rRegI dst, immI src)
5084 %{
5085   match(Set dst src);
5086 
5087   format %{ "movl    $dst, $src\t# int" %}
5088   ins_encode(load_immI(dst, src));
5089   ins_pipe(ialu_reg_fat); // XXX
5090 %}
5091 
5092 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5093 %{
5094   match(Set dst src);
5095   effect(KILL cr);
5096 
5097   ins_cost(50);
5098   format %{ "xorl    $dst, $dst\t# int" %}
5099   opcode(0x33); /* + rd */
5100   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5101   ins_pipe(ialu_reg);
5102 %}
5103 
5104 instruct loadConL(rRegL dst, immL src)
5105 %{
5106   match(Set dst src);
5107 
5108   ins_cost(150);
5109   format %{ "movq    $dst, $src\t# long" %}
5110   ins_encode(load_immL(dst, src));
5111   ins_pipe(ialu_reg);
5112 %}
5113 
5114 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5115 %{
5116   match(Set dst src);
5117   effect(KILL cr);
5118 
5119   ins_cost(50);
5120   format %{ "xorl    $dst, $dst\t# long" %}
5121   opcode(0x33); /* + rd */
5122   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5123   ins_pipe(ialu_reg); // XXX
5124 %}
5125 
5126 instruct loadConUL32(rRegL dst, immUL32 src)
5127 %{
5128   match(Set dst src);
5129 
5130   ins_cost(60);
5131   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5132   ins_encode(load_immUL32(dst, src));
5133   ins_pipe(ialu_reg);
5134 %}
5135 
5136 instruct loadConL32(rRegL dst, immL32 src)
5137 %{
5138   match(Set dst src);
5139 
5140   ins_cost(70);
5141   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5142   ins_encode(load_immL32(dst, src));
5143   ins_pipe(ialu_reg);
5144 %}
5145 
5146 instruct loadConP(rRegP dst, immP con) %{
5147   match(Set dst con);
5148 
5149   format %{ "movq    $dst, $con\t# ptr" %}
5150   ins_encode(load_immP(dst, con));
5151   ins_pipe(ialu_reg_fat); // XXX
5152 %}
5153 
5154 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5155 %{
5156   match(Set dst src);
5157   effect(KILL cr);
5158 
5159   ins_cost(50);
5160   format %{ "xorl    $dst, $dst\t# ptr" %}
5161   opcode(0x33); /* + rd */
5162   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5163   ins_pipe(ialu_reg);
5164 %}
5165 
5166 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5167 %{
5168   match(Set dst src);
5169   effect(KILL cr);
5170 
5171   ins_cost(60);
5172   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5173   ins_encode(load_immP31(dst, src));
5174   ins_pipe(ialu_reg);
5175 %}
5176 
5177 instruct loadConF(regF dst, immF con) %{
5178   match(Set dst con);
5179   ins_cost(125);
5180   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5181   ins_encode %{
5182     __ movflt($dst$$XMMRegister, $constantaddress($con));
5183   %}
5184   ins_pipe(pipe_slow);
5185 %}
5186 
5187 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5188   match(Set dst src);
5189   effect(KILL cr);
5190   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5191   ins_encode %{
5192     __ xorq($dst$$Register, $dst$$Register);
5193   %}
5194   ins_pipe(ialu_reg);
5195 %}
5196 
5197 instruct loadConN(rRegN dst, immN src) %{
5198   match(Set dst src);
5199 
5200   ins_cost(125);
5201   format %{ "movl    $dst, $src\t# compressed ptr" %}
5202   ins_encode %{
5203     address con = (address)$src$$constant;
5204     if (con == NULL) {
5205       ShouldNotReachHere();
5206     } else {
5207       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5208     }
5209   %}
5210   ins_pipe(ialu_reg_fat); // XXX
5211 %}
5212 
5213 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5214   match(Set dst src);
5215 
5216   ins_cost(125);
5217   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5218   ins_encode %{
5219     address con = (address)$src$$constant;
5220     if (con == NULL) {
5221       ShouldNotReachHere();
5222     } else {
5223       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5224     }
5225   %}
5226   ins_pipe(ialu_reg_fat); // XXX
5227 %}
5228 
5229 instruct loadConF0(regF dst, immF0 src)
5230 %{
5231   match(Set dst src);
5232   ins_cost(100);
5233 
5234   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5235   ins_encode %{
5236     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5237   %}
5238   ins_pipe(pipe_slow);
5239 %}
5240 
5241 // Use the same format since predicate() can not be used here.
5242 instruct loadConD(regD dst, immD con) %{
5243   match(Set dst con);
5244   ins_cost(125);
5245   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5246   ins_encode %{
5247     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5248   %}
5249   ins_pipe(pipe_slow);
5250 %}
5251 
5252 instruct loadConD0(regD dst, immD0 src)
5253 %{
5254   match(Set dst src);
5255   ins_cost(100);
5256 
5257   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5258   ins_encode %{
5259     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5260   %}
5261   ins_pipe(pipe_slow);
5262 %}
5263 
5264 instruct loadSSI(rRegI dst, stackSlotI src)
5265 %{
5266   match(Set dst src);
5267 
5268   ins_cost(125);
5269   format %{ "movl    $dst, $src\t# int stk" %}
5270   opcode(0x8B);
5271   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5272   ins_pipe(ialu_reg_mem);
5273 %}
5274 
5275 instruct loadSSL(rRegL dst, stackSlotL src)
5276 %{
5277   match(Set dst src);
5278 
5279   ins_cost(125);
5280   format %{ "movq    $dst, $src\t# long stk" %}
5281   opcode(0x8B);
5282   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5283   ins_pipe(ialu_reg_mem);
5284 %}
5285 
5286 instruct loadSSP(rRegP dst, stackSlotP src)
5287 %{
5288   match(Set dst src);
5289 
5290   ins_cost(125);
5291   format %{ "movq    $dst, $src\t# ptr stk" %}
5292   opcode(0x8B);
5293   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5294   ins_pipe(ialu_reg_mem);
5295 %}
5296 
5297 instruct loadSSF(regF dst, stackSlotF src)
5298 %{
5299   match(Set dst src);
5300 
5301   ins_cost(125);
5302   format %{ "movss   $dst, $src\t# float stk" %}
5303   ins_encode %{
5304     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5305   %}
5306   ins_pipe(pipe_slow); // XXX
5307 %}
5308 
5309 // Use the same format since predicate() can not be used here.
5310 instruct loadSSD(regD dst, stackSlotD src)
5311 %{
5312   match(Set dst src);
5313 
5314   ins_cost(125);
5315   format %{ "movsd   $dst, $src\t# double stk" %}
5316   ins_encode  %{
5317     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5318   %}
5319   ins_pipe(pipe_slow); // XXX
5320 %}
5321 
5322 // Prefetch instructions for allocation.
5323 // Must be safe to execute with invalid address (cannot fault).
5324 
5325 instruct prefetchAlloc( memory mem ) %{
5326   predicate(AllocatePrefetchInstr==3);
5327   match(PrefetchAllocation mem);
5328   ins_cost(125);
5329 
5330   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5331   ins_encode %{
5332     __ prefetchw($mem$$Address);
5333   %}
5334   ins_pipe(ialu_mem);
5335 %}
5336 
5337 instruct prefetchAllocNTA( memory mem ) %{
5338   predicate(AllocatePrefetchInstr==0);
5339   match(PrefetchAllocation mem);
5340   ins_cost(125);
5341 
5342   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5343   ins_encode %{
5344     __ prefetchnta($mem$$Address);
5345   %}
5346   ins_pipe(ialu_mem);
5347 %}
5348 
5349 instruct prefetchAllocT0( memory mem ) %{
5350   predicate(AllocatePrefetchInstr==1);
5351   match(PrefetchAllocation mem);
5352   ins_cost(125);
5353 
5354   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5355   ins_encode %{
5356     __ prefetcht0($mem$$Address);
5357   %}
5358   ins_pipe(ialu_mem);
5359 %}
5360 
5361 instruct prefetchAllocT2( memory mem ) %{
5362   predicate(AllocatePrefetchInstr==2);
5363   match(PrefetchAllocation mem);
5364   ins_cost(125);
5365 
5366   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5367   ins_encode %{
5368     __ prefetcht2($mem$$Address);
5369   %}
5370   ins_pipe(ialu_mem);
5371 %}
5372 
5373 //----------Store Instructions-------------------------------------------------
5374 
5375 // Store Byte
5376 instruct storeB(memory mem, rRegI src)
5377 %{
5378   match(Set mem (StoreB mem src));
5379 
5380   ins_cost(125); // XXX
5381   format %{ "movb    $mem, $src\t# byte" %}
5382   opcode(0x88);
5383   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5384   ins_pipe(ialu_mem_reg);
5385 %}
5386 
5387 // Store Char/Short
5388 instruct storeC(memory mem, rRegI src)
5389 %{
5390   match(Set mem (StoreC mem src));
5391 
5392   ins_cost(125); // XXX
5393   format %{ "movw    $mem, $src\t# char/short" %}
5394   opcode(0x89);
5395   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5396   ins_pipe(ialu_mem_reg);
5397 %}
5398 
5399 // Store Integer
5400 instruct storeI(memory mem, rRegI src)
5401 %{
5402   match(Set mem (StoreI mem src));
5403 
5404   ins_cost(125); // XXX
5405   format %{ "movl    $mem, $src\t# int" %}
5406   opcode(0x89);
5407   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5408   ins_pipe(ialu_mem_reg);
5409 %}
5410 
5411 // Store Long
5412 instruct storeL(memory mem, rRegL src)
5413 %{
5414   match(Set mem (StoreL mem src));
5415 
5416   ins_cost(125); // XXX
5417   format %{ "movq    $mem, $src\t# long" %}
5418   opcode(0x89);
5419   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5420   ins_pipe(ialu_mem_reg); // XXX
5421 %}
5422 
5423 // Store Pointer
5424 instruct storeP(memory mem, any_RegP src)
5425 %{
5426   match(Set mem (StoreP mem src));
5427 
5428   ins_cost(125); // XXX
5429   format %{ "movq    $mem, $src\t# ptr" %}
5430   opcode(0x89);
5431   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5432   ins_pipe(ialu_mem_reg);
5433 %}
5434 
5435 instruct storeImmP0(memory mem, immP0 zero)
5436 %{
5437   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5438   match(Set mem (StoreP mem zero));
5439 
5440   ins_cost(125); // XXX
5441   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5442   ins_encode %{
5443     __ movq($mem$$Address, r12);
5444   %}
5445   ins_pipe(ialu_mem_reg);
5446 %}
5447 
5448 // Store NULL Pointer, mark word, or other simple pointer constant.
5449 instruct storeImmP(memory mem, immP31 src)
5450 %{
5451   match(Set mem (StoreP mem src));
5452 
5453   ins_cost(150); // XXX
5454   format %{ "movq    $mem, $src\t# ptr" %}
5455   opcode(0xC7); /* C7 /0 */
5456   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5457   ins_pipe(ialu_mem_imm);
5458 %}
5459 
5460 // Store Compressed Pointer
5461 instruct storeN(memory mem, rRegN src)
5462 %{
5463   match(Set mem (StoreN mem src));
5464 
5465   ins_cost(125); // XXX
5466   format %{ "movl    $mem, $src\t# compressed ptr" %}
5467   ins_encode %{
5468     __ movl($mem$$Address, $src$$Register);
5469   %}
5470   ins_pipe(ialu_mem_reg);
5471 %}
5472 
5473 instruct storeNKlass(memory mem, rRegN src)
5474 %{
5475   match(Set mem (StoreNKlass mem src));
5476 
5477   ins_cost(125); // XXX
5478   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5479   ins_encode %{
5480     __ movl($mem$$Address, $src$$Register);
5481   %}
5482   ins_pipe(ialu_mem_reg);
5483 %}
5484 
5485 instruct storeImmN0(memory mem, immN0 zero)
5486 %{
5487   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5488   match(Set mem (StoreN mem zero));
5489 
5490   ins_cost(125); // XXX
5491   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5492   ins_encode %{
5493     __ movl($mem$$Address, r12);
5494   %}
5495   ins_pipe(ialu_mem_reg);
5496 %}
5497 
5498 instruct storeImmN(memory mem, immN src)
5499 %{
5500   match(Set mem (StoreN mem src));
5501 
5502   ins_cost(150); // XXX
5503   format %{ "movl    $mem, $src\t# compressed ptr" %}
5504   ins_encode %{
5505     address con = (address)$src$$constant;
5506     if (con == NULL) {
5507       __ movl($mem$$Address, (int32_t)0);
5508     } else {
5509       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5510     }
5511   %}
5512   ins_pipe(ialu_mem_imm);
5513 %}
5514 
5515 instruct storeImmNKlass(memory mem, immNKlass src)
5516 %{
5517   match(Set mem (StoreNKlass mem src));
5518 
5519   ins_cost(150); // XXX
5520   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5521   ins_encode %{
5522     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5523   %}
5524   ins_pipe(ialu_mem_imm);
5525 %}
5526 
5527 // Store Integer Immediate
5528 instruct storeImmI0(memory mem, immI0 zero)
5529 %{
5530   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5531   match(Set mem (StoreI mem zero));
5532 
5533   ins_cost(125); // XXX
5534   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5535   ins_encode %{
5536     __ movl($mem$$Address, r12);
5537   %}
5538   ins_pipe(ialu_mem_reg);
5539 %}
5540 
5541 instruct storeImmI(memory mem, immI src)
5542 %{
5543   match(Set mem (StoreI mem src));
5544 
5545   ins_cost(150);
5546   format %{ "movl    $mem, $src\t# int" %}
5547   opcode(0xC7); /* C7 /0 */
5548   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5549   ins_pipe(ialu_mem_imm);
5550 %}
5551 
5552 // Store Long Immediate
5553 instruct storeImmL0(memory mem, immL0 zero)
5554 %{
5555   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5556   match(Set mem (StoreL mem zero));
5557 
5558   ins_cost(125); // XXX
5559   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5560   ins_encode %{
5561     __ movq($mem$$Address, r12);
5562   %}
5563   ins_pipe(ialu_mem_reg);
5564 %}
5565 
5566 instruct storeImmL(memory mem, immL32 src)
5567 %{
5568   match(Set mem (StoreL mem src));
5569 
5570   ins_cost(150);
5571   format %{ "movq    $mem, $src\t# long" %}
5572   opcode(0xC7); /* C7 /0 */
5573   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5574   ins_pipe(ialu_mem_imm);
5575 %}
5576 
5577 // Store Short/Char Immediate
5578 instruct storeImmC0(memory mem, immI0 zero)
5579 %{
5580   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5581   match(Set mem (StoreC mem zero));
5582 
5583   ins_cost(125); // XXX
5584   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5585   ins_encode %{
5586     __ movw($mem$$Address, r12);
5587   %}
5588   ins_pipe(ialu_mem_reg);
5589 %}
5590 
5591 instruct storeImmI16(memory mem, immI16 src)
5592 %{
5593   predicate(UseStoreImmI16);
5594   match(Set mem (StoreC mem src));
5595 
5596   ins_cost(150);
5597   format %{ "movw    $mem, $src\t# short/char" %}
5598   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5599   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5600   ins_pipe(ialu_mem_imm);
5601 %}
5602 
5603 // Store Byte Immediate
5604 instruct storeImmB0(memory mem, immI0 zero)
5605 %{
5606   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5607   match(Set mem (StoreB mem zero));
5608 
5609   ins_cost(125); // XXX
5610   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5611   ins_encode %{
5612     __ movb($mem$$Address, r12);
5613   %}
5614   ins_pipe(ialu_mem_reg);
5615 %}
5616 
5617 instruct storeImmB(memory mem, immI8 src)
5618 %{
5619   match(Set mem (StoreB mem src));
5620 
5621   ins_cost(150); // XXX
5622   format %{ "movb    $mem, $src\t# byte" %}
5623   opcode(0xC6); /* C6 /0 */
5624   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5625   ins_pipe(ialu_mem_imm);
5626 %}
5627 
5628 // Store CMS card-mark Immediate
5629 instruct storeImmCM0_reg(memory mem, immI0 zero)
5630 %{
5631   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5632   match(Set mem (StoreCM mem zero));
5633 
5634   ins_cost(125); // XXX
5635   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5636   ins_encode %{
5637     __ movb($mem$$Address, r12);
5638   %}
5639   ins_pipe(ialu_mem_reg);
5640 %}
5641 
5642 instruct storeImmCM0(memory mem, immI0 src)
5643 %{
5644   match(Set mem (StoreCM mem src));
5645 
5646   ins_cost(150); // XXX
5647   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5648   opcode(0xC6); /* C6 /0 */
5649   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5650   ins_pipe(ialu_mem_imm);
5651 %}
5652 
5653 // Store Float
5654 instruct storeF(memory mem, regF src)
5655 %{
5656   match(Set mem (StoreF mem src));
5657 
5658   ins_cost(95); // XXX
5659   format %{ "movss   $mem, $src\t# float" %}
5660   ins_encode %{
5661     __ movflt($mem$$Address, $src$$XMMRegister);
5662   %}
5663   ins_pipe(pipe_slow); // XXX
5664 %}
5665 
5666 // Store immediate Float value (it is faster than store from XMM register)
5667 instruct storeF0(memory mem, immF0 zero)
5668 %{
5669   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5670   match(Set mem (StoreF mem zero));
5671 
5672   ins_cost(25); // XXX
5673   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5674   ins_encode %{
5675     __ movl($mem$$Address, r12);
5676   %}
5677   ins_pipe(ialu_mem_reg);
5678 %}
5679 
5680 instruct storeF_imm(memory mem, immF src)
5681 %{
5682   match(Set mem (StoreF mem src));
5683 
5684   ins_cost(50);
5685   format %{ "movl    $mem, $src\t# float" %}
5686   opcode(0xC7); /* C7 /0 */
5687   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5688   ins_pipe(ialu_mem_imm);
5689 %}
5690 
5691 // Store Double
5692 instruct storeD(memory mem, regD src)
5693 %{
5694   match(Set mem (StoreD mem src));
5695 
5696   ins_cost(95); // XXX
5697   format %{ "movsd   $mem, $src\t# double" %}
5698   ins_encode %{
5699     __ movdbl($mem$$Address, $src$$XMMRegister);
5700   %}
5701   ins_pipe(pipe_slow); // XXX
5702 %}
5703 
5704 // Store immediate double 0.0 (it is faster than store from XMM register)
5705 instruct storeD0_imm(memory mem, immD0 src)
5706 %{
5707   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5708   match(Set mem (StoreD mem src));
5709 
5710   ins_cost(50);
5711   format %{ "movq    $mem, $src\t# double 0." %}
5712   opcode(0xC7); /* C7 /0 */
5713   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5714   ins_pipe(ialu_mem_imm);
5715 %}
5716 
5717 instruct storeD0(memory mem, immD0 zero)
5718 %{
5719   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5720   match(Set mem (StoreD mem zero));
5721 
5722   ins_cost(25); // XXX
5723   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5724   ins_encode %{
5725     __ movq($mem$$Address, r12);
5726   %}
5727   ins_pipe(ialu_mem_reg);
5728 %}
5729 
5730 instruct storeSSI(stackSlotI dst, rRegI src)
5731 %{
5732   match(Set dst src);
5733 
5734   ins_cost(100);
5735   format %{ "movl    $dst, $src\t# int stk" %}
5736   opcode(0x89);
5737   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5738   ins_pipe( ialu_mem_reg );
5739 %}
5740 
5741 instruct storeSSL(stackSlotL dst, rRegL src)
5742 %{
5743   match(Set dst src);
5744 
5745   ins_cost(100);
5746   format %{ "movq    $dst, $src\t# long stk" %}
5747   opcode(0x89);
5748   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5749   ins_pipe(ialu_mem_reg);
5750 %}
5751 
5752 instruct storeSSP(stackSlotP dst, rRegP src)
5753 %{
5754   match(Set dst src);
5755 
5756   ins_cost(100);
5757   format %{ "movq    $dst, $src\t# ptr stk" %}
5758   opcode(0x89);
5759   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
5760   ins_pipe(ialu_mem_reg);
5761 %}
5762 
5763 instruct storeSSF(stackSlotF dst, regF src)
5764 %{
5765   match(Set dst src);
5766 
5767   ins_cost(95); // XXX
5768   format %{ "movss   $dst, $src\t# float stk" %}
5769   ins_encode %{
5770     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
5771   %}
5772   ins_pipe(pipe_slow); // XXX
5773 %}
5774 
5775 instruct storeSSD(stackSlotD dst, regD src)
5776 %{
5777   match(Set dst src);
5778 
5779   ins_cost(95); // XXX
5780   format %{ "movsd   $dst, $src\t# double stk" %}
5781   ins_encode %{
5782     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
5783   %}
5784   ins_pipe(pipe_slow); // XXX
5785 %}
5786 
5787 //----------BSWAP Instructions-------------------------------------------------
5788 instruct bytes_reverse_int(rRegI dst) %{
5789   match(Set dst (ReverseBytesI dst));
5790 
5791   format %{ "bswapl  $dst" %}
5792   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
5793   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
5794   ins_pipe( ialu_reg );
5795 %}
5796 
5797 instruct bytes_reverse_long(rRegL dst) %{
5798   match(Set dst (ReverseBytesL dst));
5799 
5800   format %{ "bswapq  $dst" %}
5801   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
5802   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
5803   ins_pipe( ialu_reg);
5804 %}
5805 
5806 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
5807   match(Set dst (ReverseBytesUS dst));
5808   effect(KILL cr);
5809 
5810   format %{ "bswapl  $dst\n\t"
5811             "shrl    $dst,16\n\t" %}
5812   ins_encode %{
5813     __ bswapl($dst$$Register);
5814     __ shrl($dst$$Register, 16);
5815   %}
5816   ins_pipe( ialu_reg );
5817 %}
5818 
5819 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
5820   match(Set dst (ReverseBytesS dst));
5821   effect(KILL cr);
5822 
5823   format %{ "bswapl  $dst\n\t"
5824             "sar     $dst,16\n\t" %}
5825   ins_encode %{
5826     __ bswapl($dst$$Register);
5827     __ sarl($dst$$Register, 16);
5828   %}
5829   ins_pipe( ialu_reg );
5830 %}
5831 
5832 //---------- Zeros Count Instructions ------------------------------------------
5833 
5834 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5835   predicate(UseCountLeadingZerosInstruction);
5836   match(Set dst (CountLeadingZerosI src));
5837   effect(KILL cr);
5838 
5839   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
5840   ins_encode %{
5841     __ lzcntl($dst$$Register, $src$$Register);
5842   %}
5843   ins_pipe(ialu_reg);
5844 %}
5845 
5846 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
5847   predicate(!UseCountLeadingZerosInstruction);
5848   match(Set dst (CountLeadingZerosI src));
5849   effect(KILL cr);
5850 
5851   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
5852             "jnz     skip\n\t"
5853             "movl    $dst, -1\n"
5854       "skip:\n\t"
5855             "negl    $dst\n\t"
5856             "addl    $dst, 31" %}
5857   ins_encode %{
5858     Register Rdst = $dst$$Register;
5859     Register Rsrc = $src$$Register;
5860     Label skip;
5861     __ bsrl(Rdst, Rsrc);
5862     __ jccb(Assembler::notZero, skip);
5863     __ movl(Rdst, -1);
5864     __ bind(skip);
5865     __ negl(Rdst);
5866     __ addl(Rdst, BitsPerInt - 1);
5867   %}
5868   ins_pipe(ialu_reg);
5869 %}
5870 
5871 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5872   predicate(UseCountLeadingZerosInstruction);
5873   match(Set dst (CountLeadingZerosL src));
5874   effect(KILL cr);
5875 
5876   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
5877   ins_encode %{
5878     __ lzcntq($dst$$Register, $src$$Register);
5879   %}
5880   ins_pipe(ialu_reg);
5881 %}
5882 
5883 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
5884   predicate(!UseCountLeadingZerosInstruction);
5885   match(Set dst (CountLeadingZerosL src));
5886   effect(KILL cr);
5887 
5888   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
5889             "jnz     skip\n\t"
5890             "movl    $dst, -1\n"
5891       "skip:\n\t"
5892             "negl    $dst\n\t"
5893             "addl    $dst, 63" %}
5894   ins_encode %{
5895     Register Rdst = $dst$$Register;
5896     Register Rsrc = $src$$Register;
5897     Label skip;
5898     __ bsrq(Rdst, Rsrc);
5899     __ jccb(Assembler::notZero, skip);
5900     __ movl(Rdst, -1);
5901     __ bind(skip);
5902     __ negl(Rdst);
5903     __ addl(Rdst, BitsPerLong - 1);
5904   %}
5905   ins_pipe(ialu_reg);
5906 %}
5907 
5908 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
5909   predicate(UseCountTrailingZerosInstruction);
5910   match(Set dst (CountTrailingZerosI src));
5911   effect(KILL cr);
5912 
5913   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
5914   ins_encode %{
5915     __ tzcntl($dst$$Register, $src$$Register);
5916   %}
5917   ins_pipe(ialu_reg);
5918 %}
5919 
5920 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
5921   predicate(!UseCountTrailingZerosInstruction);
5922   match(Set dst (CountTrailingZerosI src));
5923   effect(KILL cr);
5924 
5925   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
5926             "jnz     done\n\t"
5927             "movl    $dst, 32\n"
5928       "done:" %}
5929   ins_encode %{
5930     Register Rdst = $dst$$Register;
5931     Label done;
5932     __ bsfl(Rdst, $src$$Register);
5933     __ jccb(Assembler::notZero, done);
5934     __ movl(Rdst, BitsPerInt);
5935     __ bind(done);
5936   %}
5937   ins_pipe(ialu_reg);
5938 %}
5939 
5940 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
5941   predicate(UseCountTrailingZerosInstruction);
5942   match(Set dst (CountTrailingZerosL src));
5943   effect(KILL cr);
5944 
5945   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
5946   ins_encode %{
5947     __ tzcntq($dst$$Register, $src$$Register);
5948   %}
5949   ins_pipe(ialu_reg);
5950 %}
5951 
5952 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
5953   predicate(!UseCountTrailingZerosInstruction);
5954   match(Set dst (CountTrailingZerosL src));
5955   effect(KILL cr);
5956 
5957   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
5958             "jnz     done\n\t"
5959             "movl    $dst, 64\n"
5960       "done:" %}
5961   ins_encode %{
5962     Register Rdst = $dst$$Register;
5963     Label done;
5964     __ bsfq(Rdst, $src$$Register);
5965     __ jccb(Assembler::notZero, done);
5966     __ movl(Rdst, BitsPerLong);
5967     __ bind(done);
5968   %}
5969   ins_pipe(ialu_reg);
5970 %}
5971 
5972 
5973 //---------- Population Count Instructions -------------------------------------
5974 
5975 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
5976   predicate(UsePopCountInstruction);
5977   match(Set dst (PopCountI src));
5978   effect(KILL cr);
5979 
5980   format %{ "popcnt  $dst, $src" %}
5981   ins_encode %{
5982     __ popcntl($dst$$Register, $src$$Register);
5983   %}
5984   ins_pipe(ialu_reg);
5985 %}
5986 
5987 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
5988   predicate(UsePopCountInstruction);
5989   match(Set dst (PopCountI (LoadI mem)));
5990   effect(KILL cr);
5991 
5992   format %{ "popcnt  $dst, $mem" %}
5993   ins_encode %{
5994     __ popcntl($dst$$Register, $mem$$Address);
5995   %}
5996   ins_pipe(ialu_reg);
5997 %}
5998 
5999 // Note: Long.bitCount(long) returns an int.
6000 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6001   predicate(UsePopCountInstruction);
6002   match(Set dst (PopCountL src));
6003   effect(KILL cr);
6004 
6005   format %{ "popcnt  $dst, $src" %}
6006   ins_encode %{
6007     __ popcntq($dst$$Register, $src$$Register);
6008   %}
6009   ins_pipe(ialu_reg);
6010 %}
6011 
6012 // Note: Long.bitCount(long) returns an int.
6013 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6014   predicate(UsePopCountInstruction);
6015   match(Set dst (PopCountL (LoadL mem)));
6016   effect(KILL cr);
6017 
6018   format %{ "popcnt  $dst, $mem" %}
6019   ins_encode %{
6020     __ popcntq($dst$$Register, $mem$$Address);
6021   %}
6022   ins_pipe(ialu_reg);
6023 %}
6024 
6025 
6026 //----------MemBar Instructions-----------------------------------------------
6027 // Memory barrier flavors
6028 
6029 instruct membar_acquire()
6030 %{
6031   match(MemBarAcquire);
6032   match(LoadFence);
6033   ins_cost(0);
6034 
6035   size(0);
6036   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6037   ins_encode();
6038   ins_pipe(empty);
6039 %}
6040 
6041 instruct membar_acquire_lock()
6042 %{
6043   match(MemBarAcquireLock);
6044   ins_cost(0);
6045 
6046   size(0);
6047   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6048   ins_encode();
6049   ins_pipe(empty);
6050 %}
6051 
6052 instruct membar_release()
6053 %{
6054   match(MemBarRelease);
6055   match(StoreFence);
6056   ins_cost(0);
6057 
6058   size(0);
6059   format %{ "MEMBAR-release ! (empty encoding)" %}
6060   ins_encode();
6061   ins_pipe(empty);
6062 %}
6063 
6064 instruct membar_release_lock()
6065 %{
6066   match(MemBarReleaseLock);
6067   ins_cost(0);
6068 
6069   size(0);
6070   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6071   ins_encode();
6072   ins_pipe(empty);
6073 %}
6074 
6075 instruct membar_volatile(rFlagsReg cr) %{
6076   match(MemBarVolatile);
6077   effect(KILL cr);
6078   ins_cost(400);
6079 
6080   format %{
6081     $$template
6082     if (os::is_MP()) {
6083       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6084     } else {
6085       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6086     }
6087   %}
6088   ins_encode %{
6089     __ membar(Assembler::StoreLoad);
6090   %}
6091   ins_pipe(pipe_slow);
6092 %}
6093 
6094 instruct unnecessary_membar_volatile()
6095 %{
6096   match(MemBarVolatile);
6097   predicate(Matcher::post_store_load_barrier(n));
6098   ins_cost(0);
6099 
6100   size(0);
6101   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6102   ins_encode();
6103   ins_pipe(empty);
6104 %}
6105 
6106 instruct membar_storestore() %{
6107   match(MemBarStoreStore);
6108   ins_cost(0);
6109 
6110   size(0);
6111   format %{ "MEMBAR-storestore (empty encoding)" %}
6112   ins_encode( );
6113   ins_pipe(empty);
6114 %}
6115 
6116 //----------Move Instructions--------------------------------------------------
6117 
6118 instruct castX2P(rRegP dst, rRegL src)
6119 %{
6120   match(Set dst (CastX2P src));
6121 
6122   format %{ "movq    $dst, $src\t# long->ptr" %}
6123   ins_encode %{
6124     if ($dst$$reg != $src$$reg) {
6125       __ movptr($dst$$Register, $src$$Register);
6126     }
6127   %}
6128   ins_pipe(ialu_reg_reg); // XXX
6129 %}
6130 
6131 instruct castP2X(rRegL dst, rRegP src)
6132 %{
6133   match(Set dst (CastP2X src));
6134 
6135   format %{ "movq    $dst, $src\t# ptr -> long" %}
6136   ins_encode %{
6137     if ($dst$$reg != $src$$reg) {
6138       __ movptr($dst$$Register, $src$$Register);
6139     }
6140   %}
6141   ins_pipe(ialu_reg_reg); // XXX
6142 %}
6143 
6144 // Convert oop into int for vectors alignment masking
6145 instruct convP2I(rRegI dst, rRegP src)
6146 %{
6147   match(Set dst (ConvL2I (CastP2X src)));
6148 
6149   format %{ "movl    $dst, $src\t# ptr -> int" %}
6150   ins_encode %{
6151     __ movl($dst$$Register, $src$$Register);
6152   %}
6153   ins_pipe(ialu_reg_reg); // XXX
6154 %}
6155 
6156 // Convert compressed oop into int for vectors alignment masking
6157 // in case of 32bit oops (heap < 4Gb).
6158 instruct convN2I(rRegI dst, rRegN src)
6159 %{
6160   predicate(Universe::narrow_oop_shift() == 0);
6161   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6162 
6163   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6164   ins_encode %{
6165     __ movl($dst$$Register, $src$$Register);
6166   %}
6167   ins_pipe(ialu_reg_reg); // XXX
6168 %}
6169 
6170 // Convert oop pointer into compressed form
6171 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6172   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6173   match(Set dst (EncodeP src));
6174   effect(KILL cr);
6175   format %{ "encode_heap_oop $dst,$src" %}
6176   ins_encode %{
6177     Register s = $src$$Register;
6178     Register d = $dst$$Register;
6179     if (s != d) {
6180       __ movq(d, s);
6181     }
6182     __ encode_heap_oop(d);
6183   %}
6184   ins_pipe(ialu_reg_long);
6185 %}
6186 
6187 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6188   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6189   match(Set dst (EncodeP src));
6190   effect(KILL cr);
6191   format %{ "encode_heap_oop_not_null $dst,$src" %}
6192   ins_encode %{
6193     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6194   %}
6195   ins_pipe(ialu_reg_long);
6196 %}
6197 
6198 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6199   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6200             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6201   match(Set dst (DecodeN src));
6202   effect(KILL cr);
6203   format %{ "decode_heap_oop $dst,$src" %}
6204   ins_encode %{
6205     Register s = $src$$Register;
6206     Register d = $dst$$Register;
6207     if (s != d) {
6208       __ movq(d, s);
6209     }
6210     __ decode_heap_oop(d);
6211   %}
6212   ins_pipe(ialu_reg_long);
6213 %}
6214 
6215 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6216   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6217             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6218   match(Set dst (DecodeN src));
6219   effect(KILL cr);
6220   format %{ "decode_heap_oop_not_null $dst,$src" %}
6221   ins_encode %{
6222     Register s = $src$$Register;
6223     Register d = $dst$$Register;
6224     if (s != d) {
6225       __ decode_heap_oop_not_null(d, s);
6226     } else {
6227       __ decode_heap_oop_not_null(d);
6228     }
6229   %}
6230   ins_pipe(ialu_reg_long);
6231 %}
6232 
6233 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6234   match(Set dst (EncodePKlass src));
6235   effect(KILL cr);
6236   format %{ "encode_klass_not_null $dst,$src" %}
6237   ins_encode %{
6238     __ encode_klass_not_null($dst$$Register, $src$$Register);
6239   %}
6240   ins_pipe(ialu_reg_long);
6241 %}
6242 
6243 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6244   match(Set dst (DecodeNKlass src));
6245   effect(KILL cr);
6246   format %{ "decode_klass_not_null $dst,$src" %}
6247   ins_encode %{
6248     Register s = $src$$Register;
6249     Register d = $dst$$Register;
6250     if (s != d) {
6251       __ decode_klass_not_null(d, s);
6252     } else {
6253       __ decode_klass_not_null(d);
6254     }
6255   %}
6256   ins_pipe(ialu_reg_long);
6257 %}
6258 
6259 
6260 //----------Conditional Move---------------------------------------------------
6261 // Jump
6262 // dummy instruction for generating temp registers
6263 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6264   match(Jump (LShiftL switch_val shift));
6265   ins_cost(350);
6266   predicate(false);
6267   effect(TEMP dest);
6268 
6269   format %{ "leaq    $dest, [$constantaddress]\n\t"
6270             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6271   ins_encode %{
6272     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6273     // to do that and the compiler is using that register as one it can allocate.
6274     // So we build it all by hand.
6275     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6276     // ArrayAddress dispatch(table, index);
6277     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6278     __ lea($dest$$Register, $constantaddress);
6279     __ jmp(dispatch);
6280   %}
6281   ins_pipe(pipe_jmp);
6282 %}
6283 
6284 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6285   match(Jump (AddL (LShiftL switch_val shift) offset));
6286   ins_cost(350);
6287   effect(TEMP dest);
6288 
6289   format %{ "leaq    $dest, [$constantaddress]\n\t"
6290             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6291   ins_encode %{
6292     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6293     // to do that and the compiler is using that register as one it can allocate.
6294     // So we build it all by hand.
6295     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6296     // ArrayAddress dispatch(table, index);
6297     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6298     __ lea($dest$$Register, $constantaddress);
6299     __ jmp(dispatch);
6300   %}
6301   ins_pipe(pipe_jmp);
6302 %}
6303 
6304 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6305   match(Jump switch_val);
6306   ins_cost(350);
6307   effect(TEMP dest);
6308 
6309   format %{ "leaq    $dest, [$constantaddress]\n\t"
6310             "jmp     [$dest + $switch_val]\n\t" %}
6311   ins_encode %{
6312     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6313     // to do that and the compiler is using that register as one it can allocate.
6314     // So we build it all by hand.
6315     // Address index(noreg, switch_reg, Address::times_1);
6316     // ArrayAddress dispatch(table, index);
6317     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6318     __ lea($dest$$Register, $constantaddress);
6319     __ jmp(dispatch);
6320   %}
6321   ins_pipe(pipe_jmp);
6322 %}
6323 
6324 // Conditional move
6325 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6326 %{
6327   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6328 
6329   ins_cost(200); // XXX
6330   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6331   opcode(0x0F, 0x40);
6332   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6333   ins_pipe(pipe_cmov_reg);
6334 %}
6335 
6336 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6337   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6338 
6339   ins_cost(200); // XXX
6340   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6341   opcode(0x0F, 0x40);
6342   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6343   ins_pipe(pipe_cmov_reg);
6344 %}
6345 
6346 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6347   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6348   ins_cost(200);
6349   expand %{
6350     cmovI_regU(cop, cr, dst, src);
6351   %}
6352 %}
6353 
6354 // Conditional move
6355 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6356   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6357 
6358   ins_cost(250); // XXX
6359   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6360   opcode(0x0F, 0x40);
6361   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6362   ins_pipe(pipe_cmov_mem);
6363 %}
6364 
6365 // Conditional move
6366 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6367 %{
6368   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6369 
6370   ins_cost(250); // XXX
6371   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6372   opcode(0x0F, 0x40);
6373   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6374   ins_pipe(pipe_cmov_mem);
6375 %}
6376 
6377 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6378   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6379   ins_cost(250);
6380   expand %{
6381     cmovI_memU(cop, cr, dst, src);
6382   %}
6383 %}
6384 
6385 // Conditional move
6386 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6387 %{
6388   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6389 
6390   ins_cost(200); // XXX
6391   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6392   opcode(0x0F, 0x40);
6393   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6394   ins_pipe(pipe_cmov_reg);
6395 %}
6396 
6397 // Conditional move
6398 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6399 %{
6400   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6401 
6402   ins_cost(200); // XXX
6403   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6404   opcode(0x0F, 0x40);
6405   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6406   ins_pipe(pipe_cmov_reg);
6407 %}
6408 
6409 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6410   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6411   ins_cost(200);
6412   expand %{
6413     cmovN_regU(cop, cr, dst, src);
6414   %}
6415 %}
6416 
6417 // Conditional move
6418 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6419 %{
6420   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6421 
6422   ins_cost(200); // XXX
6423   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6424   opcode(0x0F, 0x40);
6425   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6426   ins_pipe(pipe_cmov_reg);  // XXX
6427 %}
6428 
6429 // Conditional move
6430 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6431 %{
6432   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6433 
6434   ins_cost(200); // XXX
6435   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6436   opcode(0x0F, 0x40);
6437   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6438   ins_pipe(pipe_cmov_reg); // XXX
6439 %}
6440 
6441 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6442   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6443   ins_cost(200);
6444   expand %{
6445     cmovP_regU(cop, cr, dst, src);
6446   %}
6447 %}
6448 
6449 // DISABLED: Requires the ADLC to emit a bottom_type call that
6450 // correctly meets the two pointer arguments; one is an incoming
6451 // register but the other is a memory operand.  ALSO appears to
6452 // be buggy with implicit null checks.
6453 //
6454 //// Conditional move
6455 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6456 //%{
6457 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6458 //  ins_cost(250);
6459 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6460 //  opcode(0x0F,0x40);
6461 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6462 //  ins_pipe( pipe_cmov_mem );
6463 //%}
6464 //
6465 //// Conditional move
6466 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6467 //%{
6468 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6469 //  ins_cost(250);
6470 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6471 //  opcode(0x0F,0x40);
6472 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6473 //  ins_pipe( pipe_cmov_mem );
6474 //%}
6475 
6476 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6477 %{
6478   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6479 
6480   ins_cost(200); // XXX
6481   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6482   opcode(0x0F, 0x40);
6483   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6484   ins_pipe(pipe_cmov_reg);  // XXX
6485 %}
6486 
6487 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6488 %{
6489   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6490 
6491   ins_cost(200); // XXX
6492   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6493   opcode(0x0F, 0x40);
6494   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6495   ins_pipe(pipe_cmov_mem);  // XXX
6496 %}
6497 
6498 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6499 %{
6500   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6501 
6502   ins_cost(200); // XXX
6503   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6504   opcode(0x0F, 0x40);
6505   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6506   ins_pipe(pipe_cmov_reg); // XXX
6507 %}
6508 
6509 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6510   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6511   ins_cost(200);
6512   expand %{
6513     cmovL_regU(cop, cr, dst, src);
6514   %}
6515 %}
6516 
6517 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6518 %{
6519   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6520 
6521   ins_cost(200); // XXX
6522   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6523   opcode(0x0F, 0x40);
6524   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6525   ins_pipe(pipe_cmov_mem); // XXX
6526 %}
6527 
6528 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6529   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6530   ins_cost(200);
6531   expand %{
6532     cmovL_memU(cop, cr, dst, src);
6533   %}
6534 %}
6535 
6536 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6537 %{
6538   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6539 
6540   ins_cost(200); // XXX
6541   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6542             "movss     $dst, $src\n"
6543     "skip:" %}
6544   ins_encode %{
6545     Label Lskip;
6546     // Invert sense of branch from sense of CMOV
6547     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6548     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6549     __ bind(Lskip);
6550   %}
6551   ins_pipe(pipe_slow);
6552 %}
6553 
6554 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6555 // %{
6556 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6557 
6558 //   ins_cost(200); // XXX
6559 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6560 //             "movss     $dst, $src\n"
6561 //     "skip:" %}
6562 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6563 //   ins_pipe(pipe_slow);
6564 // %}
6565 
6566 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6567 %{
6568   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6569 
6570   ins_cost(200); // XXX
6571   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6572             "movss     $dst, $src\n"
6573     "skip:" %}
6574   ins_encode %{
6575     Label Lskip;
6576     // Invert sense of branch from sense of CMOV
6577     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6578     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6579     __ bind(Lskip);
6580   %}
6581   ins_pipe(pipe_slow);
6582 %}
6583 
6584 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6585   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6586   ins_cost(200);
6587   expand %{
6588     cmovF_regU(cop, cr, dst, src);
6589   %}
6590 %}
6591 
6592 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6593 %{
6594   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6595 
6596   ins_cost(200); // XXX
6597   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6598             "movsd     $dst, $src\n"
6599     "skip:" %}
6600   ins_encode %{
6601     Label Lskip;
6602     // Invert sense of branch from sense of CMOV
6603     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6604     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6605     __ bind(Lskip);
6606   %}
6607   ins_pipe(pipe_slow);
6608 %}
6609 
6610 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6611 %{
6612   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6613 
6614   ins_cost(200); // XXX
6615   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6616             "movsd     $dst, $src\n"
6617     "skip:" %}
6618   ins_encode %{
6619     Label Lskip;
6620     // Invert sense of branch from sense of CMOV
6621     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6622     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6623     __ bind(Lskip);
6624   %}
6625   ins_pipe(pipe_slow);
6626 %}
6627 
6628 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6629   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6630   ins_cost(200);
6631   expand %{
6632     cmovD_regU(cop, cr, dst, src);
6633   %}
6634 %}
6635 
6636 //----------Arithmetic Instructions--------------------------------------------
6637 //----------Addition Instructions----------------------------------------------
6638 
6639 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6640 %{
6641   match(Set dst (AddI dst src));
6642   effect(KILL cr);
6643 
6644   format %{ "addl    $dst, $src\t# int" %}
6645   opcode(0x03);
6646   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6647   ins_pipe(ialu_reg_reg);
6648 %}
6649 
6650 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6651 %{
6652   match(Set dst (AddI dst src));
6653   effect(KILL cr);
6654 
6655   format %{ "addl    $dst, $src\t# int" %}
6656   opcode(0x81, 0x00); /* /0 id */
6657   ins_encode(OpcSErm(dst, src), Con8or32(src));
6658   ins_pipe( ialu_reg );
6659 %}
6660 
6661 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6662 %{
6663   match(Set dst (AddI dst (LoadI src)));
6664   effect(KILL cr);
6665 
6666   ins_cost(125); // XXX
6667   format %{ "addl    $dst, $src\t# int" %}
6668   opcode(0x03);
6669   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6670   ins_pipe(ialu_reg_mem);
6671 %}
6672 
6673 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6674 %{
6675   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6676   effect(KILL cr);
6677 
6678   ins_cost(150); // XXX
6679   format %{ "addl    $dst, $src\t# int" %}
6680   opcode(0x01); /* Opcode 01 /r */
6681   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6682   ins_pipe(ialu_mem_reg);
6683 %}
6684 
6685 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6686 %{
6687   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6688   effect(KILL cr);
6689 
6690   ins_cost(125); // XXX
6691   format %{ "addl    $dst, $src\t# int" %}
6692   opcode(0x81); /* Opcode 81 /0 id */
6693   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6694   ins_pipe(ialu_mem_imm);
6695 %}
6696 
6697 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6698 %{
6699   predicate(UseIncDec);
6700   match(Set dst (AddI dst src));
6701   effect(KILL cr);
6702 
6703   format %{ "incl    $dst\t# int" %}
6704   opcode(0xFF, 0x00); // FF /0
6705   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6706   ins_pipe(ialu_reg);
6707 %}
6708 
6709 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6710 %{
6711   predicate(UseIncDec);
6712   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6713   effect(KILL cr);
6714 
6715   ins_cost(125); // XXX
6716   format %{ "incl    $dst\t# int" %}
6717   opcode(0xFF); /* Opcode FF /0 */
6718   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6719   ins_pipe(ialu_mem_imm);
6720 %}
6721 
6722 // XXX why does that use AddI
6723 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6724 %{
6725   predicate(UseIncDec);
6726   match(Set dst (AddI dst src));
6727   effect(KILL cr);
6728 
6729   format %{ "decl    $dst\t# int" %}
6730   opcode(0xFF, 0x01); // FF /1
6731   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6732   ins_pipe(ialu_reg);
6733 %}
6734 
6735 // XXX why does that use AddI
6736 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6737 %{
6738   predicate(UseIncDec);
6739   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6740   effect(KILL cr);
6741 
6742   ins_cost(125); // XXX
6743   format %{ "decl    $dst\t# int" %}
6744   opcode(0xFF); /* Opcode FF /1 */
6745   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
6746   ins_pipe(ialu_mem_imm);
6747 %}
6748 
6749 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
6750 %{
6751   match(Set dst (AddI src0 src1));
6752 
6753   ins_cost(110);
6754   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
6755   opcode(0x8D); /* 0x8D /r */
6756   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6757   ins_pipe(ialu_reg_reg);
6758 %}
6759 
6760 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
6761 %{
6762   match(Set dst (AddL dst src));
6763   effect(KILL cr);
6764 
6765   format %{ "addq    $dst, $src\t# long" %}
6766   opcode(0x03);
6767   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6768   ins_pipe(ialu_reg_reg);
6769 %}
6770 
6771 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
6772 %{
6773   match(Set dst (AddL dst src));
6774   effect(KILL cr);
6775 
6776   format %{ "addq    $dst, $src\t# long" %}
6777   opcode(0x81, 0x00); /* /0 id */
6778   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6779   ins_pipe( ialu_reg );
6780 %}
6781 
6782 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
6783 %{
6784   match(Set dst (AddL dst (LoadL src)));
6785   effect(KILL cr);
6786 
6787   ins_cost(125); // XXX
6788   format %{ "addq    $dst, $src\t# long" %}
6789   opcode(0x03);
6790   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6791   ins_pipe(ialu_reg_mem);
6792 %}
6793 
6794 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
6795 %{
6796   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6797   effect(KILL cr);
6798 
6799   ins_cost(150); // XXX
6800   format %{ "addq    $dst, $src\t# long" %}
6801   opcode(0x01); /* Opcode 01 /r */
6802   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6803   ins_pipe(ialu_mem_reg);
6804 %}
6805 
6806 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
6807 %{
6808   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6809   effect(KILL cr);
6810 
6811   ins_cost(125); // XXX
6812   format %{ "addq    $dst, $src\t# long" %}
6813   opcode(0x81); /* Opcode 81 /0 id */
6814   ins_encode(REX_mem_wide(dst),
6815              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6816   ins_pipe(ialu_mem_imm);
6817 %}
6818 
6819 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
6820 %{
6821   predicate(UseIncDec);
6822   match(Set dst (AddL dst src));
6823   effect(KILL cr);
6824 
6825   format %{ "incq    $dst\t# long" %}
6826   opcode(0xFF, 0x00); // FF /0
6827   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6828   ins_pipe(ialu_reg);
6829 %}
6830 
6831 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
6832 %{
6833   predicate(UseIncDec);
6834   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6835   effect(KILL cr);
6836 
6837   ins_cost(125); // XXX
6838   format %{ "incq    $dst\t# long" %}
6839   opcode(0xFF); /* Opcode FF /0 */
6840   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
6841   ins_pipe(ialu_mem_imm);
6842 %}
6843 
6844 // XXX why does that use AddL
6845 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
6846 %{
6847   predicate(UseIncDec);
6848   match(Set dst (AddL dst src));
6849   effect(KILL cr);
6850 
6851   format %{ "decq    $dst\t# long" %}
6852   opcode(0xFF, 0x01); // FF /1
6853   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
6854   ins_pipe(ialu_reg);
6855 %}
6856 
6857 // XXX why does that use AddL
6858 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
6859 %{
6860   predicate(UseIncDec);
6861   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
6862   effect(KILL cr);
6863 
6864   ins_cost(125); // XXX
6865   format %{ "decq    $dst\t# long" %}
6866   opcode(0xFF); /* Opcode FF /1 */
6867   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
6868   ins_pipe(ialu_mem_imm);
6869 %}
6870 
6871 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
6872 %{
6873   match(Set dst (AddL src0 src1));
6874 
6875   ins_cost(110);
6876   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
6877   opcode(0x8D); /* 0x8D /r */
6878   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
6879   ins_pipe(ialu_reg_reg);
6880 %}
6881 
6882 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
6883 %{
6884   match(Set dst (AddP dst src));
6885   effect(KILL cr);
6886 
6887   format %{ "addq    $dst, $src\t# ptr" %}
6888   opcode(0x03);
6889   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
6890   ins_pipe(ialu_reg_reg);
6891 %}
6892 
6893 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
6894 %{
6895   match(Set dst (AddP dst src));
6896   effect(KILL cr);
6897 
6898   format %{ "addq    $dst, $src\t# ptr" %}
6899   opcode(0x81, 0x00); /* /0 id */
6900   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
6901   ins_pipe( ialu_reg );
6902 %}
6903 
6904 // XXX addP mem ops ????
6905 
6906 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
6907 %{
6908   match(Set dst (AddP src0 src1));
6909 
6910   ins_cost(110);
6911   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
6912   opcode(0x8D); /* 0x8D /r */
6913   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
6914   ins_pipe(ialu_reg_reg);
6915 %}
6916 
6917 instruct checkCastPP(rRegP dst)
6918 %{
6919   match(Set dst (CheckCastPP dst));
6920 
6921   size(0);
6922   format %{ "# checkcastPP of $dst" %}
6923   ins_encode(/* empty encoding */);
6924   ins_pipe(empty);
6925 %}
6926 
6927 instruct castPP(rRegP dst)
6928 %{
6929   match(Set dst (CastPP dst));
6930 
6931   size(0);
6932   format %{ "# castPP of $dst" %}
6933   ins_encode(/* empty encoding */);
6934   ins_pipe(empty);
6935 %}
6936 
6937 instruct castII(rRegI dst)
6938 %{
6939   match(Set dst (CastII dst));
6940 
6941   size(0);
6942   format %{ "# castII of $dst" %}
6943   ins_encode(/* empty encoding */);
6944   ins_cost(0);
6945   ins_pipe(empty);
6946 %}
6947 
6948 // LoadP-locked same as a regular LoadP when used with compare-swap
6949 instruct loadPLocked(rRegP dst, memory mem)
6950 %{
6951   match(Set dst (LoadPLocked mem));
6952 
6953   ins_cost(125); // XXX
6954   format %{ "movq    $dst, $mem\t# ptr locked" %}
6955   opcode(0x8B);
6956   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6957   ins_pipe(ialu_reg_mem); // XXX
6958 %}
6959 
6960 // Conditional-store of the updated heap-top.
6961 // Used during allocation of the shared heap.
6962 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
6963 
6964 instruct storePConditional(memory heap_top_ptr,
6965                            rax_RegP oldval, rRegP newval,
6966                            rFlagsReg cr)
6967 %{
6968   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
6969 
6970   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
6971             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
6972   opcode(0x0F, 0xB1);
6973   ins_encode(lock_prefix,
6974              REX_reg_mem_wide(newval, heap_top_ptr),
6975              OpcP, OpcS,
6976              reg_mem(newval, heap_top_ptr));
6977   ins_pipe(pipe_cmpxchg);
6978 %}
6979 
6980 // Conditional-store of an int value.
6981 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
6982 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
6983 %{
6984   match(Set cr (StoreIConditional mem (Binary oldval newval)));
6985   effect(KILL oldval);
6986 
6987   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
6988   opcode(0x0F, 0xB1);
6989   ins_encode(lock_prefix,
6990              REX_reg_mem(newval, mem),
6991              OpcP, OpcS,
6992              reg_mem(newval, mem));
6993   ins_pipe(pipe_cmpxchg);
6994 %}
6995 
6996 // Conditional-store of a long value.
6997 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
6998 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
6999 %{
7000   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7001   effect(KILL oldval);
7002 
7003   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7004   opcode(0x0F, 0xB1);
7005   ins_encode(lock_prefix,
7006              REX_reg_mem_wide(newval, mem),
7007              OpcP, OpcS,
7008              reg_mem(newval, mem));
7009   ins_pipe(pipe_cmpxchg);
7010 %}
7011 
7012 
7013 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7014 instruct compareAndSwapP(rRegI res,
7015                          memory mem_ptr,
7016                          rax_RegP oldval, rRegP newval,
7017                          rFlagsReg cr)
7018 %{
7019   predicate(VM_Version::supports_cx8());
7020   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7021   effect(KILL cr, KILL oldval);
7022 
7023   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7024             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7025             "sete    $res\n\t"
7026             "movzbl  $res, $res" %}
7027   opcode(0x0F, 0xB1);
7028   ins_encode(lock_prefix,
7029              REX_reg_mem_wide(newval, mem_ptr),
7030              OpcP, OpcS,
7031              reg_mem(newval, mem_ptr),
7032              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7033              REX_reg_breg(res, res), // movzbl
7034              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7035   ins_pipe( pipe_cmpxchg );
7036 %}
7037 
7038 instruct compareAndSwapL(rRegI res,
7039                          memory mem_ptr,
7040                          rax_RegL oldval, rRegL newval,
7041                          rFlagsReg cr)
7042 %{
7043   predicate(VM_Version::supports_cx8());
7044   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7045   effect(KILL cr, KILL oldval);
7046 
7047   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7048             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7049             "sete    $res\n\t"
7050             "movzbl  $res, $res" %}
7051   opcode(0x0F, 0xB1);
7052   ins_encode(lock_prefix,
7053              REX_reg_mem_wide(newval, mem_ptr),
7054              OpcP, OpcS,
7055              reg_mem(newval, mem_ptr),
7056              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7057              REX_reg_breg(res, res), // movzbl
7058              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7059   ins_pipe( pipe_cmpxchg );
7060 %}
7061 
7062 instruct compareAndSwapI(rRegI res,
7063                          memory mem_ptr,
7064                          rax_RegI oldval, rRegI newval,
7065                          rFlagsReg cr)
7066 %{
7067   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7068   effect(KILL cr, KILL oldval);
7069 
7070   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7071             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7072             "sete    $res\n\t"
7073             "movzbl  $res, $res" %}
7074   opcode(0x0F, 0xB1);
7075   ins_encode(lock_prefix,
7076              REX_reg_mem(newval, mem_ptr),
7077              OpcP, OpcS,
7078              reg_mem(newval, mem_ptr),
7079              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7080              REX_reg_breg(res, res), // movzbl
7081              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7082   ins_pipe( pipe_cmpxchg );
7083 %}
7084 
7085 
7086 instruct compareAndSwapN(rRegI res,
7087                           memory mem_ptr,
7088                           rax_RegN oldval, rRegN newval,
7089                           rFlagsReg cr) %{
7090   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7091   effect(KILL cr, KILL oldval);
7092 
7093   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7094             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7095             "sete    $res\n\t"
7096             "movzbl  $res, $res" %}
7097   opcode(0x0F, 0xB1);
7098   ins_encode(lock_prefix,
7099              REX_reg_mem(newval, mem_ptr),
7100              OpcP, OpcS,
7101              reg_mem(newval, mem_ptr),
7102              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7103              REX_reg_breg(res, res), // movzbl
7104              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7105   ins_pipe( pipe_cmpxchg );
7106 %}
7107 
7108 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7109   predicate(n->as_LoadStore()->result_not_used());
7110   match(Set dummy (GetAndAddI mem add));
7111   effect(KILL cr);
7112   format %{ "ADDL  [$mem],$add" %}
7113   ins_encode %{
7114     if (os::is_MP()) { __ lock(); }
7115     __ addl($mem$$Address, $add$$constant);
7116   %}
7117   ins_pipe( pipe_cmpxchg );
7118 %}
7119 
7120 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7121   match(Set newval (GetAndAddI mem newval));
7122   effect(KILL cr);
7123   format %{ "XADDL  [$mem],$newval" %}
7124   ins_encode %{
7125     if (os::is_MP()) { __ lock(); }
7126     __ xaddl($mem$$Address, $newval$$Register);
7127   %}
7128   ins_pipe( pipe_cmpxchg );
7129 %}
7130 
7131 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7132   predicate(n->as_LoadStore()->result_not_used());
7133   match(Set dummy (GetAndAddL mem add));
7134   effect(KILL cr);
7135   format %{ "ADDQ  [$mem],$add" %}
7136   ins_encode %{
7137     if (os::is_MP()) { __ lock(); }
7138     __ addq($mem$$Address, $add$$constant);
7139   %}
7140   ins_pipe( pipe_cmpxchg );
7141 %}
7142 
7143 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7144   match(Set newval (GetAndAddL mem newval));
7145   effect(KILL cr);
7146   format %{ "XADDQ  [$mem],$newval" %}
7147   ins_encode %{
7148     if (os::is_MP()) { __ lock(); }
7149     __ xaddq($mem$$Address, $newval$$Register);
7150   %}
7151   ins_pipe( pipe_cmpxchg );
7152 %}
7153 
7154 instruct xchgI( memory mem, rRegI newval) %{
7155   match(Set newval (GetAndSetI mem newval));
7156   format %{ "XCHGL  $newval,[$mem]" %}
7157   ins_encode %{
7158     __ xchgl($newval$$Register, $mem$$Address);
7159   %}
7160   ins_pipe( pipe_cmpxchg );
7161 %}
7162 
7163 instruct xchgL( memory mem, rRegL newval) %{
7164   match(Set newval (GetAndSetL mem newval));
7165   format %{ "XCHGL  $newval,[$mem]" %}
7166   ins_encode %{
7167     __ xchgq($newval$$Register, $mem$$Address);
7168   %}
7169   ins_pipe( pipe_cmpxchg );
7170 %}
7171 
7172 instruct xchgP( memory mem, rRegP newval) %{
7173   match(Set newval (GetAndSetP mem newval));
7174   format %{ "XCHGQ  $newval,[$mem]" %}
7175   ins_encode %{
7176     __ xchgq($newval$$Register, $mem$$Address);
7177   %}
7178   ins_pipe( pipe_cmpxchg );
7179 %}
7180 
7181 instruct xchgN( memory mem, rRegN newval) %{
7182   match(Set newval (GetAndSetN mem newval));
7183   format %{ "XCHGL  $newval,$mem]" %}
7184   ins_encode %{
7185     __ xchgl($newval$$Register, $mem$$Address);
7186   %}
7187   ins_pipe( pipe_cmpxchg );
7188 %}
7189 
7190 //----------Subtraction Instructions-------------------------------------------
7191 
7192 // Integer Subtraction Instructions
7193 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7194 %{
7195   match(Set dst (SubI dst src));
7196   effect(KILL cr);
7197 
7198   format %{ "subl    $dst, $src\t# int" %}
7199   opcode(0x2B);
7200   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7201   ins_pipe(ialu_reg_reg);
7202 %}
7203 
7204 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7205 %{
7206   match(Set dst (SubI dst src));
7207   effect(KILL cr);
7208 
7209   format %{ "subl    $dst, $src\t# int" %}
7210   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7211   ins_encode(OpcSErm(dst, src), Con8or32(src));
7212   ins_pipe(ialu_reg);
7213 %}
7214 
7215 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7216 %{
7217   match(Set dst (SubI dst (LoadI src)));
7218   effect(KILL cr);
7219 
7220   ins_cost(125);
7221   format %{ "subl    $dst, $src\t# int" %}
7222   opcode(0x2B);
7223   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7224   ins_pipe(ialu_reg_mem);
7225 %}
7226 
7227 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7228 %{
7229   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7230   effect(KILL cr);
7231 
7232   ins_cost(150);
7233   format %{ "subl    $dst, $src\t# int" %}
7234   opcode(0x29); /* Opcode 29 /r */
7235   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7236   ins_pipe(ialu_mem_reg);
7237 %}
7238 
7239 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7240 %{
7241   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7242   effect(KILL cr);
7243 
7244   ins_cost(125); // XXX
7245   format %{ "subl    $dst, $src\t# int" %}
7246   opcode(0x81); /* Opcode 81 /5 id */
7247   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7248   ins_pipe(ialu_mem_imm);
7249 %}
7250 
7251 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7252 %{
7253   match(Set dst (SubL dst src));
7254   effect(KILL cr);
7255 
7256   format %{ "subq    $dst, $src\t# long" %}
7257   opcode(0x2B);
7258   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7259   ins_pipe(ialu_reg_reg);
7260 %}
7261 
7262 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7263 %{
7264   match(Set dst (SubL dst src));
7265   effect(KILL cr);
7266 
7267   format %{ "subq    $dst, $src\t# long" %}
7268   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7269   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7270   ins_pipe(ialu_reg);
7271 %}
7272 
7273 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7274 %{
7275   match(Set dst (SubL dst (LoadL src)));
7276   effect(KILL cr);
7277 
7278   ins_cost(125);
7279   format %{ "subq    $dst, $src\t# long" %}
7280   opcode(0x2B);
7281   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7282   ins_pipe(ialu_reg_mem);
7283 %}
7284 
7285 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7286 %{
7287   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7288   effect(KILL cr);
7289 
7290   ins_cost(150);
7291   format %{ "subq    $dst, $src\t# long" %}
7292   opcode(0x29); /* Opcode 29 /r */
7293   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7294   ins_pipe(ialu_mem_reg);
7295 %}
7296 
7297 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7298 %{
7299   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7300   effect(KILL cr);
7301 
7302   ins_cost(125); // XXX
7303   format %{ "subq    $dst, $src\t# long" %}
7304   opcode(0x81); /* Opcode 81 /5 id */
7305   ins_encode(REX_mem_wide(dst),
7306              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7307   ins_pipe(ialu_mem_imm);
7308 %}
7309 
7310 // Subtract from a pointer
7311 // XXX hmpf???
7312 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7313 %{
7314   match(Set dst (AddP dst (SubI zero src)));
7315   effect(KILL cr);
7316 
7317   format %{ "subq    $dst, $src\t# ptr - int" %}
7318   opcode(0x2B);
7319   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7320   ins_pipe(ialu_reg_reg);
7321 %}
7322 
7323 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7324 %{
7325   match(Set dst (SubI zero dst));
7326   effect(KILL cr);
7327 
7328   format %{ "negl    $dst\t# int" %}
7329   opcode(0xF7, 0x03);  // Opcode F7 /3
7330   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7331   ins_pipe(ialu_reg);
7332 %}
7333 
7334 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7335 %{
7336   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7337   effect(KILL cr);
7338 
7339   format %{ "negl    $dst\t# int" %}
7340   opcode(0xF7, 0x03);  // Opcode F7 /3
7341   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7342   ins_pipe(ialu_reg);
7343 %}
7344 
7345 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7346 %{
7347   match(Set dst (SubL zero dst));
7348   effect(KILL cr);
7349 
7350   format %{ "negq    $dst\t# long" %}
7351   opcode(0xF7, 0x03);  // Opcode F7 /3
7352   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7353   ins_pipe(ialu_reg);
7354 %}
7355 
7356 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7357 %{
7358   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7359   effect(KILL cr);
7360 
7361   format %{ "negq    $dst\t# long" %}
7362   opcode(0xF7, 0x03);  // Opcode F7 /3
7363   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7364   ins_pipe(ialu_reg);
7365 %}
7366 
7367 //----------Multiplication/Division Instructions-------------------------------
7368 // Integer Multiplication Instructions
7369 // Multiply Register
7370 
7371 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7372 %{
7373   match(Set dst (MulI dst src));
7374   effect(KILL cr);
7375 
7376   ins_cost(300);
7377   format %{ "imull   $dst, $src\t# int" %}
7378   opcode(0x0F, 0xAF);
7379   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7380   ins_pipe(ialu_reg_reg_alu0);
7381 %}
7382 
7383 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7384 %{
7385   match(Set dst (MulI src imm));
7386   effect(KILL cr);
7387 
7388   ins_cost(300);
7389   format %{ "imull   $dst, $src, $imm\t# int" %}
7390   opcode(0x69); /* 69 /r id */
7391   ins_encode(REX_reg_reg(dst, src),
7392              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7393   ins_pipe(ialu_reg_reg_alu0);
7394 %}
7395 
7396 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7397 %{
7398   match(Set dst (MulI dst (LoadI src)));
7399   effect(KILL cr);
7400 
7401   ins_cost(350);
7402   format %{ "imull   $dst, $src\t# int" %}
7403   opcode(0x0F, 0xAF);
7404   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7405   ins_pipe(ialu_reg_mem_alu0);
7406 %}
7407 
7408 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7409 %{
7410   match(Set dst (MulI (LoadI src) imm));
7411   effect(KILL cr);
7412 
7413   ins_cost(300);
7414   format %{ "imull   $dst, $src, $imm\t# int" %}
7415   opcode(0x69); /* 69 /r id */
7416   ins_encode(REX_reg_mem(dst, src),
7417              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7418   ins_pipe(ialu_reg_mem_alu0);
7419 %}
7420 
7421 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7422 %{
7423   match(Set dst (MulL dst src));
7424   effect(KILL cr);
7425 
7426   ins_cost(300);
7427   format %{ "imulq   $dst, $src\t# long" %}
7428   opcode(0x0F, 0xAF);
7429   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7430   ins_pipe(ialu_reg_reg_alu0);
7431 %}
7432 
7433 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7434 %{
7435   match(Set dst (MulL src imm));
7436   effect(KILL cr);
7437 
7438   ins_cost(300);
7439   format %{ "imulq   $dst, $src, $imm\t# long" %}
7440   opcode(0x69); /* 69 /r id */
7441   ins_encode(REX_reg_reg_wide(dst, src),
7442              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7443   ins_pipe(ialu_reg_reg_alu0);
7444 %}
7445 
7446 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7447 %{
7448   match(Set dst (MulL dst (LoadL src)));
7449   effect(KILL cr);
7450 
7451   ins_cost(350);
7452   format %{ "imulq   $dst, $src\t# long" %}
7453   opcode(0x0F, 0xAF);
7454   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7455   ins_pipe(ialu_reg_mem_alu0);
7456 %}
7457 
7458 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7459 %{
7460   match(Set dst (MulL (LoadL src) imm));
7461   effect(KILL cr);
7462 
7463   ins_cost(300);
7464   format %{ "imulq   $dst, $src, $imm\t# long" %}
7465   opcode(0x69); /* 69 /r id */
7466   ins_encode(REX_reg_mem_wide(dst, src),
7467              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7468   ins_pipe(ialu_reg_mem_alu0);
7469 %}
7470 
7471 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7472 %{
7473   match(Set dst (MulHiL src rax));
7474   effect(USE_KILL rax, KILL cr);
7475 
7476   ins_cost(300);
7477   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7478   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7479   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7480   ins_pipe(ialu_reg_reg_alu0);
7481 %}
7482 
7483 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7484                    rFlagsReg cr)
7485 %{
7486   match(Set rax (DivI rax div));
7487   effect(KILL rdx, KILL cr);
7488 
7489   ins_cost(30*100+10*100); // XXX
7490   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7491             "jne,s   normal\n\t"
7492             "xorl    rdx, rdx\n\t"
7493             "cmpl    $div, -1\n\t"
7494             "je,s    done\n"
7495     "normal: cdql\n\t"
7496             "idivl   $div\n"
7497     "done:"        %}
7498   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7499   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7500   ins_pipe(ialu_reg_reg_alu0);
7501 %}
7502 
7503 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7504                    rFlagsReg cr)
7505 %{
7506   match(Set rax (DivL rax div));
7507   effect(KILL rdx, KILL cr);
7508 
7509   ins_cost(30*100+10*100); // XXX
7510   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7511             "cmpq    rax, rdx\n\t"
7512             "jne,s   normal\n\t"
7513             "xorl    rdx, rdx\n\t"
7514             "cmpq    $div, -1\n\t"
7515             "je,s    done\n"
7516     "normal: cdqq\n\t"
7517             "idivq   $div\n"
7518     "done:"        %}
7519   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7520   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7521   ins_pipe(ialu_reg_reg_alu0);
7522 %}
7523 
7524 // Integer DIVMOD with Register, both quotient and mod results
7525 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7526                              rFlagsReg cr)
7527 %{
7528   match(DivModI rax div);
7529   effect(KILL cr);
7530 
7531   ins_cost(30*100+10*100); // XXX
7532   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7533             "jne,s   normal\n\t"
7534             "xorl    rdx, rdx\n\t"
7535             "cmpl    $div, -1\n\t"
7536             "je,s    done\n"
7537     "normal: cdql\n\t"
7538             "idivl   $div\n"
7539     "done:"        %}
7540   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7541   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7542   ins_pipe(pipe_slow);
7543 %}
7544 
7545 // Long DIVMOD with Register, both quotient and mod results
7546 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7547                              rFlagsReg cr)
7548 %{
7549   match(DivModL rax div);
7550   effect(KILL cr);
7551 
7552   ins_cost(30*100+10*100); // XXX
7553   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7554             "cmpq    rax, rdx\n\t"
7555             "jne,s   normal\n\t"
7556             "xorl    rdx, rdx\n\t"
7557             "cmpq    $div, -1\n\t"
7558             "je,s    done\n"
7559     "normal: cdqq\n\t"
7560             "idivq   $div\n"
7561     "done:"        %}
7562   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7563   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7564   ins_pipe(pipe_slow);
7565 %}
7566 
7567 //----------- DivL-By-Constant-Expansions--------------------------------------
7568 // DivI cases are handled by the compiler
7569 
7570 // Magic constant, reciprocal of 10
7571 instruct loadConL_0x6666666666666667(rRegL dst)
7572 %{
7573   effect(DEF dst);
7574 
7575   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7576   ins_encode(load_immL(dst, 0x6666666666666667));
7577   ins_pipe(ialu_reg);
7578 %}
7579 
7580 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7581 %{
7582   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7583 
7584   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7585   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7586   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7587   ins_pipe(ialu_reg_reg_alu0);
7588 %}
7589 
7590 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7591 %{
7592   effect(USE_DEF dst, KILL cr);
7593 
7594   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7595   opcode(0xC1, 0x7); /* C1 /7 ib */
7596   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7597   ins_pipe(ialu_reg);
7598 %}
7599 
7600 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7601 %{
7602   effect(USE_DEF dst, KILL cr);
7603 
7604   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7605   opcode(0xC1, 0x7); /* C1 /7 ib */
7606   ins_encode(reg_opc_imm_wide(dst, 0x2));
7607   ins_pipe(ialu_reg);
7608 %}
7609 
7610 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7611 %{
7612   match(Set dst (DivL src div));
7613 
7614   ins_cost((5+8)*100);
7615   expand %{
7616     rax_RegL rax;                     // Killed temp
7617     rFlagsReg cr;                     // Killed
7618     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7619     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7620     sarL_rReg_63(src, cr);            // sarq  src, 63
7621     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7622     subL_rReg(dst, src, cr);          // subl  rdx, src
7623   %}
7624 %}
7625 
7626 //-----------------------------------------------------------------------------
7627 
7628 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7629                    rFlagsReg cr)
7630 %{
7631   match(Set rdx (ModI rax div));
7632   effect(KILL rax, KILL cr);
7633 
7634   ins_cost(300); // XXX
7635   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7636             "jne,s   normal\n\t"
7637             "xorl    rdx, rdx\n\t"
7638             "cmpl    $div, -1\n\t"
7639             "je,s    done\n"
7640     "normal: cdql\n\t"
7641             "idivl   $div\n"
7642     "done:"        %}
7643   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7644   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7645   ins_pipe(ialu_reg_reg_alu0);
7646 %}
7647 
7648 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7649                    rFlagsReg cr)
7650 %{
7651   match(Set rdx (ModL rax div));
7652   effect(KILL rax, KILL cr);
7653 
7654   ins_cost(300); // XXX
7655   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7656             "cmpq    rax, rdx\n\t"
7657             "jne,s   normal\n\t"
7658             "xorl    rdx, rdx\n\t"
7659             "cmpq    $div, -1\n\t"
7660             "je,s    done\n"
7661     "normal: cdqq\n\t"
7662             "idivq   $div\n"
7663     "done:"        %}
7664   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7665   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7666   ins_pipe(ialu_reg_reg_alu0);
7667 %}
7668 
7669 // Integer Shift Instructions
7670 // Shift Left by one
7671 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7672 %{
7673   match(Set dst (LShiftI dst shift));
7674   effect(KILL cr);
7675 
7676   format %{ "sall    $dst, $shift" %}
7677   opcode(0xD1, 0x4); /* D1 /4 */
7678   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7679   ins_pipe(ialu_reg);
7680 %}
7681 
7682 // Shift Left by one
7683 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7684 %{
7685   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7686   effect(KILL cr);
7687 
7688   format %{ "sall    $dst, $shift\t" %}
7689   opcode(0xD1, 0x4); /* D1 /4 */
7690   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7691   ins_pipe(ialu_mem_imm);
7692 %}
7693 
7694 // Shift Left by 8-bit immediate
7695 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7696 %{
7697   match(Set dst (LShiftI dst shift));
7698   effect(KILL cr);
7699 
7700   format %{ "sall    $dst, $shift" %}
7701   opcode(0xC1, 0x4); /* C1 /4 ib */
7702   ins_encode(reg_opc_imm(dst, shift));
7703   ins_pipe(ialu_reg);
7704 %}
7705 
7706 // Shift Left by 8-bit immediate
7707 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7708 %{
7709   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7710   effect(KILL cr);
7711 
7712   format %{ "sall    $dst, $shift" %}
7713   opcode(0xC1, 0x4); /* C1 /4 ib */
7714   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7715   ins_pipe(ialu_mem_imm);
7716 %}
7717 
7718 // Shift Left by variable
7719 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7720 %{
7721   match(Set dst (LShiftI dst shift));
7722   effect(KILL cr);
7723 
7724   format %{ "sall    $dst, $shift" %}
7725   opcode(0xD3, 0x4); /* D3 /4 */
7726   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7727   ins_pipe(ialu_reg_reg);
7728 %}
7729 
7730 // Shift Left by variable
7731 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7732 %{
7733   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7734   effect(KILL cr);
7735 
7736   format %{ "sall    $dst, $shift" %}
7737   opcode(0xD3, 0x4); /* D3 /4 */
7738   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7739   ins_pipe(ialu_mem_reg);
7740 %}
7741 
7742 // Arithmetic shift right by one
7743 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7744 %{
7745   match(Set dst (RShiftI dst shift));
7746   effect(KILL cr);
7747 
7748   format %{ "sarl    $dst, $shift" %}
7749   opcode(0xD1, 0x7); /* D1 /7 */
7750   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7751   ins_pipe(ialu_reg);
7752 %}
7753 
7754 // Arithmetic shift right by one
7755 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7756 %{
7757   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7758   effect(KILL cr);
7759 
7760   format %{ "sarl    $dst, $shift" %}
7761   opcode(0xD1, 0x7); /* D1 /7 */
7762   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7763   ins_pipe(ialu_mem_imm);
7764 %}
7765 
7766 // Arithmetic Shift Right by 8-bit immediate
7767 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7768 %{
7769   match(Set dst (RShiftI dst shift));
7770   effect(KILL cr);
7771 
7772   format %{ "sarl    $dst, $shift" %}
7773   opcode(0xC1, 0x7); /* C1 /7 ib */
7774   ins_encode(reg_opc_imm(dst, shift));
7775   ins_pipe(ialu_mem_imm);
7776 %}
7777 
7778 // Arithmetic Shift Right by 8-bit immediate
7779 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7780 %{
7781   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7782   effect(KILL cr);
7783 
7784   format %{ "sarl    $dst, $shift" %}
7785   opcode(0xC1, 0x7); /* C1 /7 ib */
7786   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7787   ins_pipe(ialu_mem_imm);
7788 %}
7789 
7790 // Arithmetic Shift Right by variable
7791 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7792 %{
7793   match(Set dst (RShiftI dst shift));
7794   effect(KILL cr);
7795 
7796   format %{ "sarl    $dst, $shift" %}
7797   opcode(0xD3, 0x7); /* D3 /7 */
7798   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7799   ins_pipe(ialu_reg_reg);
7800 %}
7801 
7802 // Arithmetic Shift Right by variable
7803 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7804 %{
7805   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
7806   effect(KILL cr);
7807 
7808   format %{ "sarl    $dst, $shift" %}
7809   opcode(0xD3, 0x7); /* D3 /7 */
7810   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7811   ins_pipe(ialu_mem_reg);
7812 %}
7813 
7814 // Logical shift right by one
7815 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7816 %{
7817   match(Set dst (URShiftI dst shift));
7818   effect(KILL cr);
7819 
7820   format %{ "shrl    $dst, $shift" %}
7821   opcode(0xD1, 0x5); /* D1 /5 */
7822   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7823   ins_pipe(ialu_reg);
7824 %}
7825 
7826 // Logical shift right by one
7827 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7828 %{
7829   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7830   effect(KILL cr);
7831 
7832   format %{ "shrl    $dst, $shift" %}
7833   opcode(0xD1, 0x5); /* D1 /5 */
7834   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7835   ins_pipe(ialu_mem_imm);
7836 %}
7837 
7838 // Logical Shift Right by 8-bit immediate
7839 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7840 %{
7841   match(Set dst (URShiftI dst shift));
7842   effect(KILL cr);
7843 
7844   format %{ "shrl    $dst, $shift" %}
7845   opcode(0xC1, 0x5); /* C1 /5 ib */
7846   ins_encode(reg_opc_imm(dst, shift));
7847   ins_pipe(ialu_reg);
7848 %}
7849 
7850 // Logical Shift Right by 8-bit immediate
7851 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7852 %{
7853   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7854   effect(KILL cr);
7855 
7856   format %{ "shrl    $dst, $shift" %}
7857   opcode(0xC1, 0x5); /* C1 /5 ib */
7858   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7859   ins_pipe(ialu_mem_imm);
7860 %}
7861 
7862 // Logical Shift Right by variable
7863 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7864 %{
7865   match(Set dst (URShiftI dst shift));
7866   effect(KILL cr);
7867 
7868   format %{ "shrl    $dst, $shift" %}
7869   opcode(0xD3, 0x5); /* D3 /5 */
7870   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7871   ins_pipe(ialu_reg_reg);
7872 %}
7873 
7874 // Logical Shift Right by variable
7875 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7876 %{
7877   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
7878   effect(KILL cr);
7879 
7880   format %{ "shrl    $dst, $shift" %}
7881   opcode(0xD3, 0x5); /* D3 /5 */
7882   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7883   ins_pipe(ialu_mem_reg);
7884 %}
7885 
7886 // Long Shift Instructions
7887 // Shift Left by one
7888 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
7889 %{
7890   match(Set dst (LShiftL dst shift));
7891   effect(KILL cr);
7892 
7893   format %{ "salq    $dst, $shift" %}
7894   opcode(0xD1, 0x4); /* D1 /4 */
7895   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7896   ins_pipe(ialu_reg);
7897 %}
7898 
7899 // Shift Left by one
7900 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7901 %{
7902   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
7903   effect(KILL cr);
7904 
7905   format %{ "salq    $dst, $shift" %}
7906   opcode(0xD1, 0x4); /* D1 /4 */
7907   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7908   ins_pipe(ialu_mem_imm);
7909 %}
7910 
7911 // Shift Left by 8-bit immediate
7912 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
7913 %{
7914   match(Set dst (LShiftL dst shift));
7915   effect(KILL cr);
7916 
7917   format %{ "salq    $dst, $shift" %}
7918   opcode(0xC1, 0x4); /* C1 /4 ib */
7919   ins_encode(reg_opc_imm_wide(dst, shift));
7920   ins_pipe(ialu_reg);
7921 %}
7922 
7923 // Shift Left by 8-bit immediate
7924 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7925 %{
7926   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
7927   effect(KILL cr);
7928 
7929   format %{ "salq    $dst, $shift" %}
7930   opcode(0xC1, 0x4); /* C1 /4 ib */
7931   ins_encode(REX_mem_wide(dst), OpcP,
7932              RM_opc_mem(secondary, dst), Con8or32(shift));
7933   ins_pipe(ialu_mem_imm);
7934 %}
7935 
7936 // Shift Left by variable
7937 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
7938 %{
7939   match(Set dst (LShiftL dst shift));
7940   effect(KILL cr);
7941 
7942   format %{ "salq    $dst, $shift" %}
7943   opcode(0xD3, 0x4); /* D3 /4 */
7944   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7945   ins_pipe(ialu_reg_reg);
7946 %}
7947 
7948 // Shift Left by variable
7949 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7950 %{
7951   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
7952   effect(KILL cr);
7953 
7954   format %{ "salq    $dst, $shift" %}
7955   opcode(0xD3, 0x4); /* D3 /4 */
7956   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7957   ins_pipe(ialu_mem_reg);
7958 %}
7959 
7960 // Arithmetic shift right by one
7961 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
7962 %{
7963   match(Set dst (RShiftL dst shift));
7964   effect(KILL cr);
7965 
7966   format %{ "sarq    $dst, $shift" %}
7967   opcode(0xD1, 0x7); /* D1 /7 */
7968   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7969   ins_pipe(ialu_reg);
7970 %}
7971 
7972 // Arithmetic shift right by one
7973 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7974 %{
7975   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
7976   effect(KILL cr);
7977 
7978   format %{ "sarq    $dst, $shift" %}
7979   opcode(0xD1, 0x7); /* D1 /7 */
7980   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7981   ins_pipe(ialu_mem_imm);
7982 %}
7983 
7984 // Arithmetic Shift Right by 8-bit immediate
7985 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
7986 %{
7987   match(Set dst (RShiftL dst shift));
7988   effect(KILL cr);
7989 
7990   format %{ "sarq    $dst, $shift" %}
7991   opcode(0xC1, 0x7); /* C1 /7 ib */
7992   ins_encode(reg_opc_imm_wide(dst, shift));
7993   ins_pipe(ialu_mem_imm);
7994 %}
7995 
7996 // Arithmetic Shift Right by 8-bit immediate
7997 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7998 %{
7999   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8000   effect(KILL cr);
8001 
8002   format %{ "sarq    $dst, $shift" %}
8003   opcode(0xC1, 0x7); /* C1 /7 ib */
8004   ins_encode(REX_mem_wide(dst), OpcP,
8005              RM_opc_mem(secondary, dst), Con8or32(shift));
8006   ins_pipe(ialu_mem_imm);
8007 %}
8008 
8009 // Arithmetic Shift Right by variable
8010 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8011 %{
8012   match(Set dst (RShiftL dst shift));
8013   effect(KILL cr);
8014 
8015   format %{ "sarq    $dst, $shift" %}
8016   opcode(0xD3, 0x7); /* D3 /7 */
8017   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8018   ins_pipe(ialu_reg_reg);
8019 %}
8020 
8021 // Arithmetic Shift Right by variable
8022 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8023 %{
8024   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8025   effect(KILL cr);
8026 
8027   format %{ "sarq    $dst, $shift" %}
8028   opcode(0xD3, 0x7); /* D3 /7 */
8029   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8030   ins_pipe(ialu_mem_reg);
8031 %}
8032 
8033 // Logical shift right by one
8034 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8035 %{
8036   match(Set dst (URShiftL dst shift));
8037   effect(KILL cr);
8038 
8039   format %{ "shrq    $dst, $shift" %}
8040   opcode(0xD1, 0x5); /* D1 /5 */
8041   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8042   ins_pipe(ialu_reg);
8043 %}
8044 
8045 // Logical shift right by one
8046 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8047 %{
8048   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8049   effect(KILL cr);
8050 
8051   format %{ "shrq    $dst, $shift" %}
8052   opcode(0xD1, 0x5); /* D1 /5 */
8053   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8054   ins_pipe(ialu_mem_imm);
8055 %}
8056 
8057 // Logical Shift Right by 8-bit immediate
8058 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8059 %{
8060   match(Set dst (URShiftL dst shift));
8061   effect(KILL cr);
8062 
8063   format %{ "shrq    $dst, $shift" %}
8064   opcode(0xC1, 0x5); /* C1 /5 ib */
8065   ins_encode(reg_opc_imm_wide(dst, shift));
8066   ins_pipe(ialu_reg);
8067 %}
8068 
8069 
8070 // Logical Shift Right by 8-bit immediate
8071 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8072 %{
8073   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8074   effect(KILL cr);
8075 
8076   format %{ "shrq    $dst, $shift" %}
8077   opcode(0xC1, 0x5); /* C1 /5 ib */
8078   ins_encode(REX_mem_wide(dst), OpcP,
8079              RM_opc_mem(secondary, dst), Con8or32(shift));
8080   ins_pipe(ialu_mem_imm);
8081 %}
8082 
8083 // Logical Shift Right by variable
8084 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8085 %{
8086   match(Set dst (URShiftL dst shift));
8087   effect(KILL cr);
8088 
8089   format %{ "shrq    $dst, $shift" %}
8090   opcode(0xD3, 0x5); /* D3 /5 */
8091   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8092   ins_pipe(ialu_reg_reg);
8093 %}
8094 
8095 // Logical Shift Right by variable
8096 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8097 %{
8098   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8099   effect(KILL cr);
8100 
8101   format %{ "shrq    $dst, $shift" %}
8102   opcode(0xD3, 0x5); /* D3 /5 */
8103   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8104   ins_pipe(ialu_mem_reg);
8105 %}
8106 
8107 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8108 // This idiom is used by the compiler for the i2b bytecode.
8109 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8110 %{
8111   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8112 
8113   format %{ "movsbl  $dst, $src\t# i2b" %}
8114   opcode(0x0F, 0xBE);
8115   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8116   ins_pipe(ialu_reg_reg);
8117 %}
8118 
8119 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8120 // This idiom is used by the compiler the i2s bytecode.
8121 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8122 %{
8123   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8124 
8125   format %{ "movswl  $dst, $src\t# i2s" %}
8126   opcode(0x0F, 0xBF);
8127   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8128   ins_pipe(ialu_reg_reg);
8129 %}
8130 
8131 // ROL/ROR instructions
8132 
8133 // ROL expand
8134 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8135   effect(KILL cr, USE_DEF dst);
8136 
8137   format %{ "roll    $dst" %}
8138   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8139   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8140   ins_pipe(ialu_reg);
8141 %}
8142 
8143 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8144   effect(USE_DEF dst, USE shift, KILL cr);
8145 
8146   format %{ "roll    $dst, $shift" %}
8147   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8148   ins_encode( reg_opc_imm(dst, shift) );
8149   ins_pipe(ialu_reg);
8150 %}
8151 
8152 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8153 %{
8154   effect(USE_DEF dst, USE shift, KILL cr);
8155 
8156   format %{ "roll    $dst, $shift" %}
8157   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8158   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8159   ins_pipe(ialu_reg_reg);
8160 %}
8161 // end of ROL expand
8162 
8163 // Rotate Left by one
8164 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8165 %{
8166   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8167 
8168   expand %{
8169     rolI_rReg_imm1(dst, cr);
8170   %}
8171 %}
8172 
8173 // Rotate Left by 8-bit immediate
8174 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8175 %{
8176   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8177   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8178 
8179   expand %{
8180     rolI_rReg_imm8(dst, lshift, cr);
8181   %}
8182 %}
8183 
8184 // Rotate Left by variable
8185 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8186 %{
8187   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8188 
8189   expand %{
8190     rolI_rReg_CL(dst, shift, cr);
8191   %}
8192 %}
8193 
8194 // Rotate Left by variable
8195 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8196 %{
8197   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8198 
8199   expand %{
8200     rolI_rReg_CL(dst, shift, cr);
8201   %}
8202 %}
8203 
8204 // ROR expand
8205 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8206 %{
8207   effect(USE_DEF dst, KILL cr);
8208 
8209   format %{ "rorl    $dst" %}
8210   opcode(0xD1, 0x1); /* D1 /1 */
8211   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8212   ins_pipe(ialu_reg);
8213 %}
8214 
8215 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8216 %{
8217   effect(USE_DEF dst, USE shift, KILL cr);
8218 
8219   format %{ "rorl    $dst, $shift" %}
8220   opcode(0xC1, 0x1); /* C1 /1 ib */
8221   ins_encode(reg_opc_imm(dst, shift));
8222   ins_pipe(ialu_reg);
8223 %}
8224 
8225 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8226 %{
8227   effect(USE_DEF dst, USE shift, KILL cr);
8228 
8229   format %{ "rorl    $dst, $shift" %}
8230   opcode(0xD3, 0x1); /* D3 /1 */
8231   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8232   ins_pipe(ialu_reg_reg);
8233 %}
8234 // end of ROR expand
8235 
8236 // Rotate Right by one
8237 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8238 %{
8239   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8240 
8241   expand %{
8242     rorI_rReg_imm1(dst, cr);
8243   %}
8244 %}
8245 
8246 // Rotate Right by 8-bit immediate
8247 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8248 %{
8249   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8250   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8251 
8252   expand %{
8253     rorI_rReg_imm8(dst, rshift, cr);
8254   %}
8255 %}
8256 
8257 // Rotate Right by variable
8258 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8259 %{
8260   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8261 
8262   expand %{
8263     rorI_rReg_CL(dst, shift, cr);
8264   %}
8265 %}
8266 
8267 // Rotate Right by variable
8268 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8269 %{
8270   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8271 
8272   expand %{
8273     rorI_rReg_CL(dst, shift, cr);
8274   %}
8275 %}
8276 
8277 // for long rotate
8278 // ROL expand
8279 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8280   effect(USE_DEF dst, KILL cr);
8281 
8282   format %{ "rolq    $dst" %}
8283   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8284   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8285   ins_pipe(ialu_reg);
8286 %}
8287 
8288 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8289   effect(USE_DEF dst, USE shift, KILL cr);
8290 
8291   format %{ "rolq    $dst, $shift" %}
8292   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8293   ins_encode( reg_opc_imm_wide(dst, shift) );
8294   ins_pipe(ialu_reg);
8295 %}
8296 
8297 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8298 %{
8299   effect(USE_DEF dst, USE shift, KILL cr);
8300 
8301   format %{ "rolq    $dst, $shift" %}
8302   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8303   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8304   ins_pipe(ialu_reg_reg);
8305 %}
8306 // end of ROL expand
8307 
8308 // Rotate Left by one
8309 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8310 %{
8311   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8312 
8313   expand %{
8314     rolL_rReg_imm1(dst, cr);
8315   %}
8316 %}
8317 
8318 // Rotate Left by 8-bit immediate
8319 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8320 %{
8321   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8322   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8323 
8324   expand %{
8325     rolL_rReg_imm8(dst, lshift, cr);
8326   %}
8327 %}
8328 
8329 // Rotate Left by variable
8330 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8331 %{
8332   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8333 
8334   expand %{
8335     rolL_rReg_CL(dst, shift, cr);
8336   %}
8337 %}
8338 
8339 // Rotate Left by variable
8340 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8341 %{
8342   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8343 
8344   expand %{
8345     rolL_rReg_CL(dst, shift, cr);
8346   %}
8347 %}
8348 
8349 // ROR expand
8350 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8351 %{
8352   effect(USE_DEF dst, KILL cr);
8353 
8354   format %{ "rorq    $dst" %}
8355   opcode(0xD1, 0x1); /* D1 /1 */
8356   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8357   ins_pipe(ialu_reg);
8358 %}
8359 
8360 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8361 %{
8362   effect(USE_DEF dst, USE shift, KILL cr);
8363 
8364   format %{ "rorq    $dst, $shift" %}
8365   opcode(0xC1, 0x1); /* C1 /1 ib */
8366   ins_encode(reg_opc_imm_wide(dst, shift));
8367   ins_pipe(ialu_reg);
8368 %}
8369 
8370 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8371 %{
8372   effect(USE_DEF dst, USE shift, KILL cr);
8373 
8374   format %{ "rorq    $dst, $shift" %}
8375   opcode(0xD3, 0x1); /* D3 /1 */
8376   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8377   ins_pipe(ialu_reg_reg);
8378 %}
8379 // end of ROR expand
8380 
8381 // Rotate Right by one
8382 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8383 %{
8384   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8385 
8386   expand %{
8387     rorL_rReg_imm1(dst, cr);
8388   %}
8389 %}
8390 
8391 // Rotate Right by 8-bit immediate
8392 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8393 %{
8394   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8395   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8396 
8397   expand %{
8398     rorL_rReg_imm8(dst, rshift, cr);
8399   %}
8400 %}
8401 
8402 // Rotate Right by variable
8403 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8404 %{
8405   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8406 
8407   expand %{
8408     rorL_rReg_CL(dst, shift, cr);
8409   %}
8410 %}
8411 
8412 // Rotate Right by variable
8413 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8414 %{
8415   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8416 
8417   expand %{
8418     rorL_rReg_CL(dst, shift, cr);
8419   %}
8420 %}
8421 
8422 // Logical Instructions
8423 
8424 // Integer Logical Instructions
8425 
8426 // And Instructions
8427 // And Register with Register
8428 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8429 %{
8430   match(Set dst (AndI dst src));
8431   effect(KILL cr);
8432 
8433   format %{ "andl    $dst, $src\t# int" %}
8434   opcode(0x23);
8435   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8436   ins_pipe(ialu_reg_reg);
8437 %}
8438 
8439 // And Register with Immediate 255
8440 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8441 %{
8442   match(Set dst (AndI dst src));
8443 
8444   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8445   opcode(0x0F, 0xB6);
8446   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8447   ins_pipe(ialu_reg);
8448 %}
8449 
8450 // And Register with Immediate 255 and promote to long
8451 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8452 %{
8453   match(Set dst (ConvI2L (AndI src mask)));
8454 
8455   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8456   opcode(0x0F, 0xB6);
8457   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8458   ins_pipe(ialu_reg);
8459 %}
8460 
8461 // And Register with Immediate 65535
8462 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8463 %{
8464   match(Set dst (AndI dst src));
8465 
8466   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8467   opcode(0x0F, 0xB7);
8468   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8469   ins_pipe(ialu_reg);
8470 %}
8471 
8472 // And Register with Immediate 65535 and promote to long
8473 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8474 %{
8475   match(Set dst (ConvI2L (AndI src mask)));
8476 
8477   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8478   opcode(0x0F, 0xB7);
8479   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8480   ins_pipe(ialu_reg);
8481 %}
8482 
8483 // And Register with Immediate
8484 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8485 %{
8486   match(Set dst (AndI dst src));
8487   effect(KILL cr);
8488 
8489   format %{ "andl    $dst, $src\t# int" %}
8490   opcode(0x81, 0x04); /* Opcode 81 /4 */
8491   ins_encode(OpcSErm(dst, src), Con8or32(src));
8492   ins_pipe(ialu_reg);
8493 %}
8494 
8495 // And Register with Memory
8496 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8497 %{
8498   match(Set dst (AndI dst (LoadI src)));
8499   effect(KILL cr);
8500 
8501   ins_cost(125);
8502   format %{ "andl    $dst, $src\t# int" %}
8503   opcode(0x23);
8504   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8505   ins_pipe(ialu_reg_mem);
8506 %}
8507 
8508 // And Memory with Register
8509 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8510 %{
8511   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8512   effect(KILL cr);
8513 
8514   ins_cost(150);
8515   format %{ "andl    $dst, $src\t# int" %}
8516   opcode(0x21); /* Opcode 21 /r */
8517   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8518   ins_pipe(ialu_mem_reg);
8519 %}
8520 
8521 // And Memory with Immediate
8522 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8523 %{
8524   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8525   effect(KILL cr);
8526 
8527   ins_cost(125);
8528   format %{ "andl    $dst, $src\t# int" %}
8529   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8530   ins_encode(REX_mem(dst), OpcSE(src),
8531              RM_opc_mem(secondary, dst), Con8or32(src));
8532   ins_pipe(ialu_mem_imm);
8533 %}
8534 
8535 // BMI1 instructions
8536 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8537   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8538   predicate(UseBMI1Instructions);
8539   effect(KILL cr);
8540 
8541   ins_cost(125);
8542   format %{ "andnl  $dst, $src1, $src2" %}
8543 
8544   ins_encode %{
8545     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8546   %}
8547   ins_pipe(ialu_reg_mem);
8548 %}
8549 
8550 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8551   match(Set dst (AndI (XorI src1 minus_1) src2));
8552   predicate(UseBMI1Instructions);
8553   effect(KILL cr);
8554 
8555   format %{ "andnl  $dst, $src1, $src2" %}
8556 
8557   ins_encode %{
8558     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8559   %}
8560   ins_pipe(ialu_reg);
8561 %}
8562 
8563 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8564   match(Set dst (AndI (SubI imm_zero src) src));
8565   predicate(UseBMI1Instructions);
8566   effect(KILL cr);
8567 
8568   format %{ "blsil  $dst, $src" %}
8569 
8570   ins_encode %{
8571     __ blsil($dst$$Register, $src$$Register);
8572   %}
8573   ins_pipe(ialu_reg);
8574 %}
8575 
8576 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8577   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8578   predicate(UseBMI1Instructions);
8579   effect(KILL cr);
8580 
8581   ins_cost(125);
8582   format %{ "blsil  $dst, $src" %}
8583 
8584   ins_encode %{
8585     __ blsil($dst$$Register, $src$$Address);
8586   %}
8587   ins_pipe(ialu_reg_mem);
8588 %}
8589 
8590 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8591 %{
8592   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8593   predicate(UseBMI1Instructions);
8594   effect(KILL cr);
8595 
8596   ins_cost(125);
8597   format %{ "blsmskl $dst, $src" %}
8598 
8599   ins_encode %{
8600     __ blsmskl($dst$$Register, $src$$Address);
8601   %}
8602   ins_pipe(ialu_reg_mem);
8603 %}
8604 
8605 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8606 %{
8607   match(Set dst (XorI (AddI src minus_1) src));
8608   predicate(UseBMI1Instructions);
8609   effect(KILL cr);
8610 
8611   format %{ "blsmskl $dst, $src" %}
8612 
8613   ins_encode %{
8614     __ blsmskl($dst$$Register, $src$$Register);
8615   %}
8616 
8617   ins_pipe(ialu_reg);
8618 %}
8619 
8620 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8621 %{
8622   match(Set dst (AndI (AddI src minus_1) src) );
8623   predicate(UseBMI1Instructions);
8624   effect(KILL cr);
8625 
8626   format %{ "blsrl  $dst, $src" %}
8627 
8628   ins_encode %{
8629     __ blsrl($dst$$Register, $src$$Register);
8630   %}
8631 
8632   ins_pipe(ialu_reg_mem);
8633 %}
8634 
8635 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8636 %{
8637   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8638   predicate(UseBMI1Instructions);
8639   effect(KILL cr);
8640 
8641   ins_cost(125);
8642   format %{ "blsrl  $dst, $src" %}
8643 
8644   ins_encode %{
8645     __ blsrl($dst$$Register, $src$$Address);
8646   %}
8647 
8648   ins_pipe(ialu_reg);
8649 %}
8650 
8651 // Or Instructions
8652 // Or Register with Register
8653 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8654 %{
8655   match(Set dst (OrI dst src));
8656   effect(KILL cr);
8657 
8658   format %{ "orl     $dst, $src\t# int" %}
8659   opcode(0x0B);
8660   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8661   ins_pipe(ialu_reg_reg);
8662 %}
8663 
8664 // Or Register with Immediate
8665 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8666 %{
8667   match(Set dst (OrI dst src));
8668   effect(KILL cr);
8669 
8670   format %{ "orl     $dst, $src\t# int" %}
8671   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8672   ins_encode(OpcSErm(dst, src), Con8or32(src));
8673   ins_pipe(ialu_reg);
8674 %}
8675 
8676 // Or Register with Memory
8677 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8678 %{
8679   match(Set dst (OrI dst (LoadI src)));
8680   effect(KILL cr);
8681 
8682   ins_cost(125);
8683   format %{ "orl     $dst, $src\t# int" %}
8684   opcode(0x0B);
8685   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8686   ins_pipe(ialu_reg_mem);
8687 %}
8688 
8689 // Or Memory with Register
8690 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8691 %{
8692   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8693   effect(KILL cr);
8694 
8695   ins_cost(150);
8696   format %{ "orl     $dst, $src\t# int" %}
8697   opcode(0x09); /* Opcode 09 /r */
8698   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8699   ins_pipe(ialu_mem_reg);
8700 %}
8701 
8702 // Or Memory with Immediate
8703 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8704 %{
8705   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8706   effect(KILL cr);
8707 
8708   ins_cost(125);
8709   format %{ "orl     $dst, $src\t# int" %}
8710   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8711   ins_encode(REX_mem(dst), OpcSE(src),
8712              RM_opc_mem(secondary, dst), Con8or32(src));
8713   ins_pipe(ialu_mem_imm);
8714 %}
8715 
8716 // Xor Instructions
8717 // Xor Register with Register
8718 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8719 %{
8720   match(Set dst (XorI dst src));
8721   effect(KILL cr);
8722 
8723   format %{ "xorl    $dst, $src\t# int" %}
8724   opcode(0x33);
8725   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8726   ins_pipe(ialu_reg_reg);
8727 %}
8728 
8729 // Xor Register with Immediate -1
8730 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8731   match(Set dst (XorI dst imm));
8732 
8733   format %{ "not    $dst" %}
8734   ins_encode %{
8735      __ notl($dst$$Register);
8736   %}
8737   ins_pipe(ialu_reg);
8738 %}
8739 
8740 // Xor Register with Immediate
8741 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8742 %{
8743   match(Set dst (XorI dst src));
8744   effect(KILL cr);
8745 
8746   format %{ "xorl    $dst, $src\t# int" %}
8747   opcode(0x81, 0x06); /* Opcode 81 /6 id */
8748   ins_encode(OpcSErm(dst, src), Con8or32(src));
8749   ins_pipe(ialu_reg);
8750 %}
8751 
8752 // Xor Register with Memory
8753 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8754 %{
8755   match(Set dst (XorI dst (LoadI src)));
8756   effect(KILL cr);
8757 
8758   ins_cost(125);
8759   format %{ "xorl    $dst, $src\t# int" %}
8760   opcode(0x33);
8761   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8762   ins_pipe(ialu_reg_mem);
8763 %}
8764 
8765 // Xor Memory with Register
8766 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8767 %{
8768   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8769   effect(KILL cr);
8770 
8771   ins_cost(150);
8772   format %{ "xorl    $dst, $src\t# int" %}
8773   opcode(0x31); /* Opcode 31 /r */
8774   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8775   ins_pipe(ialu_mem_reg);
8776 %}
8777 
8778 // Xor Memory with Immediate
8779 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
8780 %{
8781   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
8782   effect(KILL cr);
8783 
8784   ins_cost(125);
8785   format %{ "xorl    $dst, $src\t# int" %}
8786   opcode(0x81, 0x6); /* Opcode 81 /6 id */
8787   ins_encode(REX_mem(dst), OpcSE(src),
8788              RM_opc_mem(secondary, dst), Con8or32(src));
8789   ins_pipe(ialu_mem_imm);
8790 %}
8791 
8792 
8793 // Long Logical Instructions
8794 
8795 // And Instructions
8796 // And Register with Register
8797 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8798 %{
8799   match(Set dst (AndL dst src));
8800   effect(KILL cr);
8801 
8802   format %{ "andq    $dst, $src\t# long" %}
8803   opcode(0x23);
8804   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8805   ins_pipe(ialu_reg_reg);
8806 %}
8807 
8808 // And Register with Immediate 255
8809 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
8810 %{
8811   match(Set dst (AndL dst src));
8812 
8813   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
8814   opcode(0x0F, 0xB6);
8815   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8816   ins_pipe(ialu_reg);
8817 %}
8818 
8819 // And Register with Immediate 65535
8820 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
8821 %{
8822   match(Set dst (AndL dst src));
8823 
8824   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
8825   opcode(0x0F, 0xB7);
8826   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8827   ins_pipe(ialu_reg);
8828 %}
8829 
8830 // And Register with Immediate
8831 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
8832 %{
8833   match(Set dst (AndL dst src));
8834   effect(KILL cr);
8835 
8836   format %{ "andq    $dst, $src\t# long" %}
8837   opcode(0x81, 0x04); /* Opcode 81 /4 */
8838   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8839   ins_pipe(ialu_reg);
8840 %}
8841 
8842 // And Register with Memory
8843 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8844 %{
8845   match(Set dst (AndL dst (LoadL src)));
8846   effect(KILL cr);
8847 
8848   ins_cost(125);
8849   format %{ "andq    $dst, $src\t# long" %}
8850   opcode(0x23);
8851   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8852   ins_pipe(ialu_reg_mem);
8853 %}
8854 
8855 // And Memory with Register
8856 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8857 %{
8858   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8859   effect(KILL cr);
8860 
8861   ins_cost(150);
8862   format %{ "andq    $dst, $src\t# long" %}
8863   opcode(0x21); /* Opcode 21 /r */
8864   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8865   ins_pipe(ialu_mem_reg);
8866 %}
8867 
8868 // And Memory with Immediate
8869 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8870 %{
8871   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
8872   effect(KILL cr);
8873 
8874   ins_cost(125);
8875   format %{ "andq    $dst, $src\t# long" %}
8876   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8877   ins_encode(REX_mem_wide(dst), OpcSE(src),
8878              RM_opc_mem(secondary, dst), Con8or32(src));
8879   ins_pipe(ialu_mem_imm);
8880 %}
8881 
8882 // BMI1 instructions
8883 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
8884   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
8885   predicate(UseBMI1Instructions);
8886   effect(KILL cr);
8887 
8888   ins_cost(125);
8889   format %{ "andnq  $dst, $src1, $src2" %}
8890 
8891   ins_encode %{
8892     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
8893   %}
8894   ins_pipe(ialu_reg_mem);
8895 %}
8896 
8897 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
8898   match(Set dst (AndL (XorL src1 minus_1) src2));
8899   predicate(UseBMI1Instructions);
8900   effect(KILL cr);
8901 
8902   format %{ "andnq  $dst, $src1, $src2" %}
8903 
8904   ins_encode %{
8905   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
8906   %}
8907   ins_pipe(ialu_reg_mem);
8908 %}
8909 
8910 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
8911   match(Set dst (AndL (SubL imm_zero src) src));
8912   predicate(UseBMI1Instructions);
8913   effect(KILL cr);
8914 
8915   format %{ "blsiq  $dst, $src" %}
8916 
8917   ins_encode %{
8918     __ blsiq($dst$$Register, $src$$Register);
8919   %}
8920   ins_pipe(ialu_reg);
8921 %}
8922 
8923 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
8924   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
8925   predicate(UseBMI1Instructions);
8926   effect(KILL cr);
8927 
8928   ins_cost(125);
8929   format %{ "blsiq  $dst, $src" %}
8930 
8931   ins_encode %{
8932     __ blsiq($dst$$Register, $src$$Address);
8933   %}
8934   ins_pipe(ialu_reg_mem);
8935 %}
8936 
8937 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
8938 %{
8939   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
8940   predicate(UseBMI1Instructions);
8941   effect(KILL cr);
8942 
8943   ins_cost(125);
8944   format %{ "blsmskq $dst, $src" %}
8945 
8946   ins_encode %{
8947     __ blsmskq($dst$$Register, $src$$Address);
8948   %}
8949   ins_pipe(ialu_reg_mem);
8950 %}
8951 
8952 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
8953 %{
8954   match(Set dst (XorL (AddL src minus_1) src));
8955   predicate(UseBMI1Instructions);
8956   effect(KILL cr);
8957 
8958   format %{ "blsmskq $dst, $src" %}
8959 
8960   ins_encode %{
8961     __ blsmskq($dst$$Register, $src$$Register);
8962   %}
8963 
8964   ins_pipe(ialu_reg);
8965 %}
8966 
8967 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
8968 %{
8969   match(Set dst (AndL (AddL src minus_1) src) );
8970   predicate(UseBMI1Instructions);
8971   effect(KILL cr);
8972 
8973   format %{ "blsrq  $dst, $src" %}
8974 
8975   ins_encode %{
8976     __ blsrq($dst$$Register, $src$$Register);
8977   %}
8978 
8979   ins_pipe(ialu_reg);
8980 %}
8981 
8982 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
8983 %{
8984   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
8985   predicate(UseBMI1Instructions);
8986   effect(KILL cr);
8987 
8988   ins_cost(125);
8989   format %{ "blsrq  $dst, $src" %}
8990 
8991   ins_encode %{
8992     __ blsrq($dst$$Register, $src$$Address);
8993   %}
8994 
8995   ins_pipe(ialu_reg);
8996 %}
8997 
8998 // Or Instructions
8999 // Or Register with Register
9000 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9001 %{
9002   match(Set dst (OrL dst src));
9003   effect(KILL cr);
9004 
9005   format %{ "orq     $dst, $src\t# long" %}
9006   opcode(0x0B);
9007   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9008   ins_pipe(ialu_reg_reg);
9009 %}
9010 
9011 // Use any_RegP to match R15 (TLS register) without spilling.
9012 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9013   match(Set dst (OrL dst (CastP2X src)));
9014   effect(KILL cr);
9015 
9016   format %{ "orq     $dst, $src\t# long" %}
9017   opcode(0x0B);
9018   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9019   ins_pipe(ialu_reg_reg);
9020 %}
9021 
9022 
9023 // Or Register with Immediate
9024 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9025 %{
9026   match(Set dst (OrL dst src));
9027   effect(KILL cr);
9028 
9029   format %{ "orq     $dst, $src\t# long" %}
9030   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9031   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9032   ins_pipe(ialu_reg);
9033 %}
9034 
9035 // Or Register with Memory
9036 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9037 %{
9038   match(Set dst (OrL dst (LoadL src)));
9039   effect(KILL cr);
9040 
9041   ins_cost(125);
9042   format %{ "orq     $dst, $src\t# long" %}
9043   opcode(0x0B);
9044   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9045   ins_pipe(ialu_reg_mem);
9046 %}
9047 
9048 // Or Memory with Register
9049 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9050 %{
9051   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9052   effect(KILL cr);
9053 
9054   ins_cost(150);
9055   format %{ "orq     $dst, $src\t# long" %}
9056   opcode(0x09); /* Opcode 09 /r */
9057   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9058   ins_pipe(ialu_mem_reg);
9059 %}
9060 
9061 // Or Memory with Immediate
9062 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9063 %{
9064   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9065   effect(KILL cr);
9066 
9067   ins_cost(125);
9068   format %{ "orq     $dst, $src\t# long" %}
9069   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9070   ins_encode(REX_mem_wide(dst), OpcSE(src),
9071              RM_opc_mem(secondary, dst), Con8or32(src));
9072   ins_pipe(ialu_mem_imm);
9073 %}
9074 
9075 // Xor Instructions
9076 // Xor Register with Register
9077 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9078 %{
9079   match(Set dst (XorL dst src));
9080   effect(KILL cr);
9081 
9082   format %{ "xorq    $dst, $src\t# long" %}
9083   opcode(0x33);
9084   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9085   ins_pipe(ialu_reg_reg);
9086 %}
9087 
9088 // Xor Register with Immediate -1
9089 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9090   match(Set dst (XorL dst imm));
9091 
9092   format %{ "notq   $dst" %}
9093   ins_encode %{
9094      __ notq($dst$$Register);
9095   %}
9096   ins_pipe(ialu_reg);
9097 %}
9098 
9099 // Xor Register with Immediate
9100 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9101 %{
9102   match(Set dst (XorL dst src));
9103   effect(KILL cr);
9104 
9105   format %{ "xorq    $dst, $src\t# long" %}
9106   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9107   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9108   ins_pipe(ialu_reg);
9109 %}
9110 
9111 // Xor Register with Memory
9112 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9113 %{
9114   match(Set dst (XorL dst (LoadL src)));
9115   effect(KILL cr);
9116 
9117   ins_cost(125);
9118   format %{ "xorq    $dst, $src\t# long" %}
9119   opcode(0x33);
9120   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9121   ins_pipe(ialu_reg_mem);
9122 %}
9123 
9124 // Xor Memory with Register
9125 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9126 %{
9127   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9128   effect(KILL cr);
9129 
9130   ins_cost(150);
9131   format %{ "xorq    $dst, $src\t# long" %}
9132   opcode(0x31); /* Opcode 31 /r */
9133   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9134   ins_pipe(ialu_mem_reg);
9135 %}
9136 
9137 // Xor Memory with Immediate
9138 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9139 %{
9140   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9141   effect(KILL cr);
9142 
9143   ins_cost(125);
9144   format %{ "xorq    $dst, $src\t# long" %}
9145   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9146   ins_encode(REX_mem_wide(dst), OpcSE(src),
9147              RM_opc_mem(secondary, dst), Con8or32(src));
9148   ins_pipe(ialu_mem_imm);
9149 %}
9150 
9151 // Convert Int to Boolean
9152 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9153 %{
9154   match(Set dst (Conv2B src));
9155   effect(KILL cr);
9156 
9157   format %{ "testl   $src, $src\t# ci2b\n\t"
9158             "setnz   $dst\n\t"
9159             "movzbl  $dst, $dst" %}
9160   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9161              setNZ_reg(dst),
9162              REX_reg_breg(dst, dst), // movzbl
9163              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9164   ins_pipe(pipe_slow); // XXX
9165 %}
9166 
9167 // Convert Pointer to Boolean
9168 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9169 %{
9170   match(Set dst (Conv2B src));
9171   effect(KILL cr);
9172 
9173   format %{ "testq   $src, $src\t# cp2b\n\t"
9174             "setnz   $dst\n\t"
9175             "movzbl  $dst, $dst" %}
9176   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9177              setNZ_reg(dst),
9178              REX_reg_breg(dst, dst), // movzbl
9179              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9180   ins_pipe(pipe_slow); // XXX
9181 %}
9182 
9183 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9184 %{
9185   match(Set dst (CmpLTMask p q));
9186   effect(KILL cr);
9187 
9188   ins_cost(400);
9189   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9190             "setlt   $dst\n\t"
9191             "movzbl  $dst, $dst\n\t"
9192             "negl    $dst" %}
9193   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9194              setLT_reg(dst),
9195              REX_reg_breg(dst, dst), // movzbl
9196              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9197              neg_reg(dst));
9198   ins_pipe(pipe_slow);
9199 %}
9200 
9201 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9202 %{
9203   match(Set dst (CmpLTMask dst zero));
9204   effect(KILL cr);
9205 
9206   ins_cost(100);
9207   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9208   ins_encode %{
9209   __ sarl($dst$$Register, 31);
9210   %}
9211   ins_pipe(ialu_reg);
9212 %}
9213 
9214 /* Better to save a register than avoid a branch */
9215 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9216 %{
9217   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9218   effect(KILL cr);
9219   ins_cost(300);
9220   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9221             "jge    done\n\t"
9222             "addl   $p,$y\n"
9223             "done:  " %}
9224   ins_encode %{
9225     Register Rp = $p$$Register;
9226     Register Rq = $q$$Register;
9227     Register Ry = $y$$Register;
9228     Label done;
9229     __ subl(Rp, Rq);
9230     __ jccb(Assembler::greaterEqual, done);
9231     __ addl(Rp, Ry);
9232     __ bind(done);
9233   %}
9234   ins_pipe(pipe_cmplt);
9235 %}
9236 
9237 /* Better to save a register than avoid a branch */
9238 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9239 %{
9240   match(Set y (AndI (CmpLTMask p q) y));
9241   effect(KILL cr);
9242 
9243   ins_cost(300);
9244 
9245   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9246             "jlt      done\n\t"
9247             "xorl     $y, $y\n"
9248             "done:  " %}
9249   ins_encode %{
9250     Register Rp = $p$$Register;
9251     Register Rq = $q$$Register;
9252     Register Ry = $y$$Register;
9253     Label done;
9254     __ cmpl(Rp, Rq);
9255     __ jccb(Assembler::less, done);
9256     __ xorl(Ry, Ry);
9257     __ bind(done);
9258   %}
9259   ins_pipe(pipe_cmplt);
9260 %}
9261 
9262 
9263 //---------- FP Instructions------------------------------------------------
9264 
9265 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9266 %{
9267   match(Set cr (CmpF src1 src2));
9268 
9269   ins_cost(145);
9270   format %{ "ucomiss $src1, $src2\n\t"
9271             "jnp,s   exit\n\t"
9272             "pushfq\t# saw NaN, set CF\n\t"
9273             "andq    [rsp], #0xffffff2b\n\t"
9274             "popfq\n"
9275     "exit:" %}
9276   ins_encode %{
9277     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9278     emit_cmpfp_fixup(_masm);
9279   %}
9280   ins_pipe(pipe_slow);
9281 %}
9282 
9283 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9284   match(Set cr (CmpF src1 src2));
9285 
9286   ins_cost(100);
9287   format %{ "ucomiss $src1, $src2" %}
9288   ins_encode %{
9289     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9290   %}
9291   ins_pipe(pipe_slow);
9292 %}
9293 
9294 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9295 %{
9296   match(Set cr (CmpF src1 (LoadF src2)));
9297 
9298   ins_cost(145);
9299   format %{ "ucomiss $src1, $src2\n\t"
9300             "jnp,s   exit\n\t"
9301             "pushfq\t# saw NaN, set CF\n\t"
9302             "andq    [rsp], #0xffffff2b\n\t"
9303             "popfq\n"
9304     "exit:" %}
9305   ins_encode %{
9306     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9307     emit_cmpfp_fixup(_masm);
9308   %}
9309   ins_pipe(pipe_slow);
9310 %}
9311 
9312 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9313   match(Set cr (CmpF src1 (LoadF src2)));
9314 
9315   ins_cost(100);
9316   format %{ "ucomiss $src1, $src2" %}
9317   ins_encode %{
9318     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9319   %}
9320   ins_pipe(pipe_slow);
9321 %}
9322 
9323 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9324   match(Set cr (CmpF src con));
9325 
9326   ins_cost(145);
9327   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9328             "jnp,s   exit\n\t"
9329             "pushfq\t# saw NaN, set CF\n\t"
9330             "andq    [rsp], #0xffffff2b\n\t"
9331             "popfq\n"
9332     "exit:" %}
9333   ins_encode %{
9334     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9335     emit_cmpfp_fixup(_masm);
9336   %}
9337   ins_pipe(pipe_slow);
9338 %}
9339 
9340 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9341   match(Set cr (CmpF src con));
9342   ins_cost(100);
9343   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9344   ins_encode %{
9345     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9346   %}
9347   ins_pipe(pipe_slow);
9348 %}
9349 
9350 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9351 %{
9352   match(Set cr (CmpD src1 src2));
9353 
9354   ins_cost(145);
9355   format %{ "ucomisd $src1, $src2\n\t"
9356             "jnp,s   exit\n\t"
9357             "pushfq\t# saw NaN, set CF\n\t"
9358             "andq    [rsp], #0xffffff2b\n\t"
9359             "popfq\n"
9360     "exit:" %}
9361   ins_encode %{
9362     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9363     emit_cmpfp_fixup(_masm);
9364   %}
9365   ins_pipe(pipe_slow);
9366 %}
9367 
9368 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9369   match(Set cr (CmpD src1 src2));
9370 
9371   ins_cost(100);
9372   format %{ "ucomisd $src1, $src2 test" %}
9373   ins_encode %{
9374     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9375   %}
9376   ins_pipe(pipe_slow);
9377 %}
9378 
9379 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9380 %{
9381   match(Set cr (CmpD src1 (LoadD src2)));
9382 
9383   ins_cost(145);
9384   format %{ "ucomisd $src1, $src2\n\t"
9385             "jnp,s   exit\n\t"
9386             "pushfq\t# saw NaN, set CF\n\t"
9387             "andq    [rsp], #0xffffff2b\n\t"
9388             "popfq\n"
9389     "exit:" %}
9390   ins_encode %{
9391     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9392     emit_cmpfp_fixup(_masm);
9393   %}
9394   ins_pipe(pipe_slow);
9395 %}
9396 
9397 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9398   match(Set cr (CmpD src1 (LoadD src2)));
9399 
9400   ins_cost(100);
9401   format %{ "ucomisd $src1, $src2" %}
9402   ins_encode %{
9403     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9404   %}
9405   ins_pipe(pipe_slow);
9406 %}
9407 
9408 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9409   match(Set cr (CmpD src con));
9410 
9411   ins_cost(145);
9412   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9413             "jnp,s   exit\n\t"
9414             "pushfq\t# saw NaN, set CF\n\t"
9415             "andq    [rsp], #0xffffff2b\n\t"
9416             "popfq\n"
9417     "exit:" %}
9418   ins_encode %{
9419     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9420     emit_cmpfp_fixup(_masm);
9421   %}
9422   ins_pipe(pipe_slow);
9423 %}
9424 
9425 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9426   match(Set cr (CmpD src con));
9427   ins_cost(100);
9428   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9429   ins_encode %{
9430     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9431   %}
9432   ins_pipe(pipe_slow);
9433 %}
9434 
9435 // Compare into -1,0,1
9436 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9437 %{
9438   match(Set dst (CmpF3 src1 src2));
9439   effect(KILL cr);
9440 
9441   ins_cost(275);
9442   format %{ "ucomiss $src1, $src2\n\t"
9443             "movl    $dst, #-1\n\t"
9444             "jp,s    done\n\t"
9445             "jb,s    done\n\t"
9446             "setne   $dst\n\t"
9447             "movzbl  $dst, $dst\n"
9448     "done:" %}
9449   ins_encode %{
9450     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9451     emit_cmpfp3(_masm, $dst$$Register);
9452   %}
9453   ins_pipe(pipe_slow);
9454 %}
9455 
9456 // Compare into -1,0,1
9457 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9458 %{
9459   match(Set dst (CmpF3 src1 (LoadF src2)));
9460   effect(KILL cr);
9461 
9462   ins_cost(275);
9463   format %{ "ucomiss $src1, $src2\n\t"
9464             "movl    $dst, #-1\n\t"
9465             "jp,s    done\n\t"
9466             "jb,s    done\n\t"
9467             "setne   $dst\n\t"
9468             "movzbl  $dst, $dst\n"
9469     "done:" %}
9470   ins_encode %{
9471     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9472     emit_cmpfp3(_masm, $dst$$Register);
9473   %}
9474   ins_pipe(pipe_slow);
9475 %}
9476 
9477 // Compare into -1,0,1
9478 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9479   match(Set dst (CmpF3 src con));
9480   effect(KILL cr);
9481 
9482   ins_cost(275);
9483   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9484             "movl    $dst, #-1\n\t"
9485             "jp,s    done\n\t"
9486             "jb,s    done\n\t"
9487             "setne   $dst\n\t"
9488             "movzbl  $dst, $dst\n"
9489     "done:" %}
9490   ins_encode %{
9491     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9492     emit_cmpfp3(_masm, $dst$$Register);
9493   %}
9494   ins_pipe(pipe_slow);
9495 %}
9496 
9497 // Compare into -1,0,1
9498 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9499 %{
9500   match(Set dst (CmpD3 src1 src2));
9501   effect(KILL cr);
9502 
9503   ins_cost(275);
9504   format %{ "ucomisd $src1, $src2\n\t"
9505             "movl    $dst, #-1\n\t"
9506             "jp,s    done\n\t"
9507             "jb,s    done\n\t"
9508             "setne   $dst\n\t"
9509             "movzbl  $dst, $dst\n"
9510     "done:" %}
9511   ins_encode %{
9512     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9513     emit_cmpfp3(_masm, $dst$$Register);
9514   %}
9515   ins_pipe(pipe_slow);
9516 %}
9517 
9518 // Compare into -1,0,1
9519 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9520 %{
9521   match(Set dst (CmpD3 src1 (LoadD src2)));
9522   effect(KILL cr);
9523 
9524   ins_cost(275);
9525   format %{ "ucomisd $src1, $src2\n\t"
9526             "movl    $dst, #-1\n\t"
9527             "jp,s    done\n\t"
9528             "jb,s    done\n\t"
9529             "setne   $dst\n\t"
9530             "movzbl  $dst, $dst\n"
9531     "done:" %}
9532   ins_encode %{
9533     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9534     emit_cmpfp3(_masm, $dst$$Register);
9535   %}
9536   ins_pipe(pipe_slow);
9537 %}
9538 
9539 // Compare into -1,0,1
9540 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9541   match(Set dst (CmpD3 src con));
9542   effect(KILL cr);
9543 
9544   ins_cost(275);
9545   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9546             "movl    $dst, #-1\n\t"
9547             "jp,s    done\n\t"
9548             "jb,s    done\n\t"
9549             "setne   $dst\n\t"
9550             "movzbl  $dst, $dst\n"
9551     "done:" %}
9552   ins_encode %{
9553     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9554     emit_cmpfp3(_masm, $dst$$Register);
9555   %}
9556   ins_pipe(pipe_slow);
9557 %}
9558 
9559 // -----------Trig and Trancendental Instructions------------------------------
9560 instruct cosD_reg(regD dst) %{
9561   match(Set dst (CosD dst));
9562 
9563   format %{ "dcos   $dst\n\t" %}
9564   opcode(0xD9, 0xFF);
9565   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9566   ins_pipe( pipe_slow );
9567 %}
9568 
9569 instruct sinD_reg(regD dst) %{
9570   match(Set dst (SinD dst));
9571 
9572   format %{ "dsin   $dst\n\t" %}
9573   opcode(0xD9, 0xFE);
9574   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9575   ins_pipe( pipe_slow );
9576 %}
9577 
9578 instruct tanD_reg(regD dst) %{
9579   match(Set dst (TanD dst));
9580 
9581   format %{ "dtan   $dst\n\t" %}
9582   ins_encode( Push_SrcXD(dst),
9583               Opcode(0xD9), Opcode(0xF2),   //fptan
9584               Opcode(0xDD), Opcode(0xD8),   //fstp st
9585               Push_ResultXD(dst) );
9586   ins_pipe( pipe_slow );
9587 %}
9588 
9589 instruct log10D_reg(regD dst) %{
9590   // The source and result Double operands in XMM registers
9591   match(Set dst (Log10D dst));
9592   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9593   // fyl2x        ; compute log_10(2) * log_2(x)
9594   format %{ "fldlg2\t\t\t#Log10\n\t"
9595             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9596          %}
9597    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9598               Push_SrcXD(dst),
9599               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9600               Push_ResultXD(dst));
9601 
9602   ins_pipe( pipe_slow );
9603 %}
9604 
9605 instruct logD_reg(regD dst) %{
9606   // The source and result Double operands in XMM registers
9607   match(Set dst (LogD dst));
9608   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9609   // fyl2x        ; compute log_e(2) * log_2(x)
9610   format %{ "fldln2\t\t\t#Log_e\n\t"
9611             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9612          %}
9613   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9614               Push_SrcXD(dst),
9615               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9616               Push_ResultXD(dst));
9617   ins_pipe( pipe_slow );
9618 %}
9619 
9620 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9621   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9622   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9623   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9624   ins_encode %{
9625     __ subptr(rsp, 8);
9626     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9627     __ fld_d(Address(rsp, 0));
9628     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9629     __ fld_d(Address(rsp, 0));
9630     __ fast_pow();
9631     __ fstp_d(Address(rsp, 0));
9632     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9633     __ addptr(rsp, 8);
9634   %}
9635   ins_pipe( pipe_slow );
9636 %}
9637 
9638 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9639   match(Set dst (ExpD src));
9640   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9641   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9642   ins_encode %{
9643     __ subptr(rsp, 8);
9644     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9645     __ fld_d(Address(rsp, 0));
9646     __ fast_exp();
9647     __ fstp_d(Address(rsp, 0));
9648     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9649     __ addptr(rsp, 8);
9650   %}
9651   ins_pipe( pipe_slow );
9652 %}
9653 
9654 //----------Arithmetic Conversion Instructions---------------------------------
9655 
9656 instruct roundFloat_nop(regF dst)
9657 %{
9658   match(Set dst (RoundFloat dst));
9659 
9660   ins_cost(0);
9661   ins_encode();
9662   ins_pipe(empty);
9663 %}
9664 
9665 instruct roundDouble_nop(regD dst)
9666 %{
9667   match(Set dst (RoundDouble dst));
9668 
9669   ins_cost(0);
9670   ins_encode();
9671   ins_pipe(empty);
9672 %}
9673 
9674 instruct convF2D_reg_reg(regD dst, regF src)
9675 %{
9676   match(Set dst (ConvF2D src));
9677 
9678   format %{ "cvtss2sd $dst, $src" %}
9679   ins_encode %{
9680     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9681   %}
9682   ins_pipe(pipe_slow); // XXX
9683 %}
9684 
9685 instruct convF2D_reg_mem(regD dst, memory src)
9686 %{
9687   match(Set dst (ConvF2D (LoadF src)));
9688 
9689   format %{ "cvtss2sd $dst, $src" %}
9690   ins_encode %{
9691     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9692   %}
9693   ins_pipe(pipe_slow); // XXX
9694 %}
9695 
9696 instruct convD2F_reg_reg(regF dst, regD src)
9697 %{
9698   match(Set dst (ConvD2F src));
9699 
9700   format %{ "cvtsd2ss $dst, $src" %}
9701   ins_encode %{
9702     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9703   %}
9704   ins_pipe(pipe_slow); // XXX
9705 %}
9706 
9707 instruct convD2F_reg_mem(regF dst, memory src)
9708 %{
9709   match(Set dst (ConvD2F (LoadD src)));
9710 
9711   format %{ "cvtsd2ss $dst, $src" %}
9712   ins_encode %{
9713     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9714   %}
9715   ins_pipe(pipe_slow); // XXX
9716 %}
9717 
9718 // XXX do mem variants
9719 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9720 %{
9721   match(Set dst (ConvF2I src));
9722   effect(KILL cr);
9723 
9724   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9725             "cmpl    $dst, #0x80000000\n\t"
9726             "jne,s   done\n\t"
9727             "subq    rsp, #8\n\t"
9728             "movss   [rsp], $src\n\t"
9729             "call    f2i_fixup\n\t"
9730             "popq    $dst\n"
9731     "done:   "%}
9732   ins_encode %{
9733     Label done;
9734     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9735     __ cmpl($dst$$Register, 0x80000000);
9736     __ jccb(Assembler::notEqual, done);
9737     __ subptr(rsp, 8);
9738     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9739     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9740     __ pop($dst$$Register);
9741     __ bind(done);
9742   %}
9743   ins_pipe(pipe_slow);
9744 %}
9745 
9746 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9747 %{
9748   match(Set dst (ConvF2L src));
9749   effect(KILL cr);
9750 
9751   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9752             "cmpq    $dst, [0x8000000000000000]\n\t"
9753             "jne,s   done\n\t"
9754             "subq    rsp, #8\n\t"
9755             "movss   [rsp], $src\n\t"
9756             "call    f2l_fixup\n\t"
9757             "popq    $dst\n"
9758     "done:   "%}
9759   ins_encode %{
9760     Label done;
9761     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9762     __ cmp64($dst$$Register,
9763              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9764     __ jccb(Assembler::notEqual, done);
9765     __ subptr(rsp, 8);
9766     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9767     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9768     __ pop($dst$$Register);
9769     __ bind(done);
9770   %}
9771   ins_pipe(pipe_slow);
9772 %}
9773 
9774 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9775 %{
9776   match(Set dst (ConvD2I src));
9777   effect(KILL cr);
9778 
9779   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9780             "cmpl    $dst, #0x80000000\n\t"
9781             "jne,s   done\n\t"
9782             "subq    rsp, #8\n\t"
9783             "movsd   [rsp], $src\n\t"
9784             "call    d2i_fixup\n\t"
9785             "popq    $dst\n"
9786     "done:   "%}
9787   ins_encode %{
9788     Label done;
9789     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
9790     __ cmpl($dst$$Register, 0x80000000);
9791     __ jccb(Assembler::notEqual, done);
9792     __ subptr(rsp, 8);
9793     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9794     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
9795     __ pop($dst$$Register);
9796     __ bind(done);
9797   %}
9798   ins_pipe(pipe_slow);
9799 %}
9800 
9801 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
9802 %{
9803   match(Set dst (ConvD2L src));
9804   effect(KILL cr);
9805 
9806   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
9807             "cmpq    $dst, [0x8000000000000000]\n\t"
9808             "jne,s   done\n\t"
9809             "subq    rsp, #8\n\t"
9810             "movsd   [rsp], $src\n\t"
9811             "call    d2l_fixup\n\t"
9812             "popq    $dst\n"
9813     "done:   "%}
9814   ins_encode %{
9815     Label done;
9816     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
9817     __ cmp64($dst$$Register,
9818              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9819     __ jccb(Assembler::notEqual, done);
9820     __ subptr(rsp, 8);
9821     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9822     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
9823     __ pop($dst$$Register);
9824     __ bind(done);
9825   %}
9826   ins_pipe(pipe_slow);
9827 %}
9828 
9829 instruct convI2F_reg_reg(regF dst, rRegI src)
9830 %{
9831   predicate(!UseXmmI2F);
9832   match(Set dst (ConvI2F src));
9833 
9834   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9835   ins_encode %{
9836     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
9837   %}
9838   ins_pipe(pipe_slow); // XXX
9839 %}
9840 
9841 instruct convI2F_reg_mem(regF dst, memory src)
9842 %{
9843   match(Set dst (ConvI2F (LoadI src)));
9844 
9845   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
9846   ins_encode %{
9847     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
9848   %}
9849   ins_pipe(pipe_slow); // XXX
9850 %}
9851 
9852 instruct convI2D_reg_reg(regD dst, rRegI src)
9853 %{
9854   predicate(!UseXmmI2D);
9855   match(Set dst (ConvI2D src));
9856 
9857   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9858   ins_encode %{
9859     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
9860   %}
9861   ins_pipe(pipe_slow); // XXX
9862 %}
9863 
9864 instruct convI2D_reg_mem(regD dst, memory src)
9865 %{
9866   match(Set dst (ConvI2D (LoadI src)));
9867 
9868   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
9869   ins_encode %{
9870     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
9871   %}
9872   ins_pipe(pipe_slow); // XXX
9873 %}
9874 
9875 instruct convXI2F_reg(regF dst, rRegI src)
9876 %{
9877   predicate(UseXmmI2F);
9878   match(Set dst (ConvI2F src));
9879 
9880   format %{ "movdl $dst, $src\n\t"
9881             "cvtdq2psl $dst, $dst\t# i2f" %}
9882   ins_encode %{
9883     __ movdl($dst$$XMMRegister, $src$$Register);
9884     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
9885   %}
9886   ins_pipe(pipe_slow); // XXX
9887 %}
9888 
9889 instruct convXI2D_reg(regD dst, rRegI src)
9890 %{
9891   predicate(UseXmmI2D);
9892   match(Set dst (ConvI2D src));
9893 
9894   format %{ "movdl $dst, $src\n\t"
9895             "cvtdq2pdl $dst, $dst\t# i2d" %}
9896   ins_encode %{
9897     __ movdl($dst$$XMMRegister, $src$$Register);
9898     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
9899   %}
9900   ins_pipe(pipe_slow); // XXX
9901 %}
9902 
9903 instruct convL2F_reg_reg(regF dst, rRegL src)
9904 %{
9905   match(Set dst (ConvL2F src));
9906 
9907   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
9908   ins_encode %{
9909     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
9910   %}
9911   ins_pipe(pipe_slow); // XXX
9912 %}
9913 
9914 instruct convL2F_reg_mem(regF dst, memory src)
9915 %{
9916   match(Set dst (ConvL2F (LoadL src)));
9917 
9918   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
9919   ins_encode %{
9920     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
9921   %}
9922   ins_pipe(pipe_slow); // XXX
9923 %}
9924 
9925 instruct convL2D_reg_reg(regD dst, rRegL src)
9926 %{
9927   match(Set dst (ConvL2D src));
9928 
9929   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
9930   ins_encode %{
9931     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
9932   %}
9933   ins_pipe(pipe_slow); // XXX
9934 %}
9935 
9936 instruct convL2D_reg_mem(regD dst, memory src)
9937 %{
9938   match(Set dst (ConvL2D (LoadL src)));
9939 
9940   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
9941   ins_encode %{
9942     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
9943   %}
9944   ins_pipe(pipe_slow); // XXX
9945 %}
9946 
9947 instruct convI2L_reg_reg(rRegL dst, rRegI src)
9948 %{
9949   match(Set dst (ConvI2L src));
9950 
9951   ins_cost(125);
9952   format %{ "movslq  $dst, $src\t# i2l" %}
9953   ins_encode %{
9954     __ movslq($dst$$Register, $src$$Register);
9955   %}
9956   ins_pipe(ialu_reg_reg);
9957 %}
9958 
9959 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
9960 // %{
9961 //   match(Set dst (ConvI2L src));
9962 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
9963 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
9964 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
9965 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
9966 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
9967 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
9968 
9969 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
9970 //   ins_encode(enc_copy(dst, src));
9971 // //   opcode(0x63); // needs REX.W
9972 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
9973 //   ins_pipe(ialu_reg_reg);
9974 // %}
9975 
9976 // Zero-extend convert int to long
9977 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
9978 %{
9979   match(Set dst (AndL (ConvI2L src) mask));
9980 
9981   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
9982   ins_encode %{
9983     if ($dst$$reg != $src$$reg) {
9984       __ movl($dst$$Register, $src$$Register);
9985     }
9986   %}
9987   ins_pipe(ialu_reg_reg);
9988 %}
9989 
9990 // Zero-extend convert int to long
9991 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
9992 %{
9993   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
9994 
9995   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
9996   ins_encode %{
9997     __ movl($dst$$Register, $src$$Address);
9998   %}
9999   ins_pipe(ialu_reg_mem);
10000 %}
10001 
10002 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10003 %{
10004   match(Set dst (AndL src mask));
10005 
10006   format %{ "movl    $dst, $src\t# zero-extend long" %}
10007   ins_encode %{
10008     __ movl($dst$$Register, $src$$Register);
10009   %}
10010   ins_pipe(ialu_reg_reg);
10011 %}
10012 
10013 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10014 %{
10015   match(Set dst (ConvL2I src));
10016 
10017   format %{ "movl    $dst, $src\t# l2i" %}
10018   ins_encode %{
10019     __ movl($dst$$Register, $src$$Register);
10020   %}
10021   ins_pipe(ialu_reg_reg);
10022 %}
10023 
10024 
10025 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10026   match(Set dst (MoveF2I src));
10027   effect(DEF dst, USE src);
10028 
10029   ins_cost(125);
10030   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10031   ins_encode %{
10032     __ movl($dst$$Register, Address(rsp, $src$$disp));
10033   %}
10034   ins_pipe(ialu_reg_mem);
10035 %}
10036 
10037 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10038   match(Set dst (MoveI2F src));
10039   effect(DEF dst, USE src);
10040 
10041   ins_cost(125);
10042   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10043   ins_encode %{
10044     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10045   %}
10046   ins_pipe(pipe_slow);
10047 %}
10048 
10049 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10050   match(Set dst (MoveD2L src));
10051   effect(DEF dst, USE src);
10052 
10053   ins_cost(125);
10054   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10055   ins_encode %{
10056     __ movq($dst$$Register, Address(rsp, $src$$disp));
10057   %}
10058   ins_pipe(ialu_reg_mem);
10059 %}
10060 
10061 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10062   predicate(!UseXmmLoadAndClearUpper);
10063   match(Set dst (MoveL2D src));
10064   effect(DEF dst, USE src);
10065 
10066   ins_cost(125);
10067   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10068   ins_encode %{
10069     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10070   %}
10071   ins_pipe(pipe_slow);
10072 %}
10073 
10074 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10075   predicate(UseXmmLoadAndClearUpper);
10076   match(Set dst (MoveL2D src));
10077   effect(DEF dst, USE src);
10078 
10079   ins_cost(125);
10080   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10081   ins_encode %{
10082     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10083   %}
10084   ins_pipe(pipe_slow);
10085 %}
10086 
10087 
10088 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10089   match(Set dst (MoveF2I src));
10090   effect(DEF dst, USE src);
10091 
10092   ins_cost(95); // XXX
10093   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10094   ins_encode %{
10095     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10096   %}
10097   ins_pipe(pipe_slow);
10098 %}
10099 
10100 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10101   match(Set dst (MoveI2F src));
10102   effect(DEF dst, USE src);
10103 
10104   ins_cost(100);
10105   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10106   ins_encode %{
10107     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10108   %}
10109   ins_pipe( ialu_mem_reg );
10110 %}
10111 
10112 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10113   match(Set dst (MoveD2L src));
10114   effect(DEF dst, USE src);
10115 
10116   ins_cost(95); // XXX
10117   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10118   ins_encode %{
10119     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10120   %}
10121   ins_pipe(pipe_slow);
10122 %}
10123 
10124 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10125   match(Set dst (MoveL2D src));
10126   effect(DEF dst, USE src);
10127 
10128   ins_cost(100);
10129   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10130   ins_encode %{
10131     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10132   %}
10133   ins_pipe(ialu_mem_reg);
10134 %}
10135 
10136 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10137   match(Set dst (MoveF2I src));
10138   effect(DEF dst, USE src);
10139   ins_cost(85);
10140   format %{ "movd    $dst,$src\t# MoveF2I" %}
10141   ins_encode %{
10142     __ movdl($dst$$Register, $src$$XMMRegister);
10143   %}
10144   ins_pipe( pipe_slow );
10145 %}
10146 
10147 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10148   match(Set dst (MoveD2L src));
10149   effect(DEF dst, USE src);
10150   ins_cost(85);
10151   format %{ "movd    $dst,$src\t# MoveD2L" %}
10152   ins_encode %{
10153     __ movdq($dst$$Register, $src$$XMMRegister);
10154   %}
10155   ins_pipe( pipe_slow );
10156 %}
10157 
10158 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10159   match(Set dst (MoveI2F src));
10160   effect(DEF dst, USE src);
10161   ins_cost(100);
10162   format %{ "movd    $dst,$src\t# MoveI2F" %}
10163   ins_encode %{
10164     __ movdl($dst$$XMMRegister, $src$$Register);
10165   %}
10166   ins_pipe( pipe_slow );
10167 %}
10168 
10169 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10170   match(Set dst (MoveL2D src));
10171   effect(DEF dst, USE src);
10172   ins_cost(100);
10173   format %{ "movd    $dst,$src\t# MoveL2D" %}
10174   ins_encode %{
10175      __ movdq($dst$$XMMRegister, $src$$Register);
10176   %}
10177   ins_pipe( pipe_slow );
10178 %}
10179 
10180 
10181 // =======================================================================
10182 // fast clearing of an array
10183 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10184                   rFlagsReg cr)
10185 %{
10186   predicate(!UseFastStosb);
10187   match(Set dummy (ClearArray cnt base));
10188   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10189 
10190   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10191             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10192   ins_encode %{ 
10193     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10194   %}
10195   ins_pipe(pipe_slow);
10196 %}
10197 
10198 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10199                         rFlagsReg cr)
10200 %{
10201   predicate(UseFastStosb);
10202   match(Set dummy (ClearArray cnt base));
10203   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10204   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10205             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10206             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10207   ins_encode %{ 
10208     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10209   %}
10210   ins_pipe( pipe_slow );
10211 %}
10212 
10213 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10214                         rax_RegI result, regD tmp1, rFlagsReg cr)
10215 %{
10216   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10217   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10218 
10219   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10220   ins_encode %{
10221     __ string_compare($str1$$Register, $str2$$Register,
10222                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10223                       $tmp1$$XMMRegister);
10224   %}
10225   ins_pipe( pipe_slow );
10226 %}
10227 
10228 // fast search of substring with known size.
10229 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10230                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10231 %{
10232   predicate(UseSSE42Intrinsics);
10233   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10234   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10235 
10236   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10237   ins_encode %{
10238     int icnt2 = (int)$int_cnt2$$constant;
10239     if (icnt2 >= 8) {
10240       // IndexOf for constant substrings with size >= 8 elements
10241       // which don't need to be loaded through stack.
10242       __ string_indexofC8($str1$$Register, $str2$$Register,
10243                           $cnt1$$Register, $cnt2$$Register,
10244                           icnt2, $result$$Register,
10245                           $vec$$XMMRegister, $tmp$$Register);
10246     } else {
10247       // Small strings are loaded through stack if they cross page boundary.
10248       __ string_indexof($str1$$Register, $str2$$Register,
10249                         $cnt1$$Register, $cnt2$$Register,
10250                         icnt2, $result$$Register,
10251                         $vec$$XMMRegister, $tmp$$Register);
10252     }
10253   %}
10254   ins_pipe( pipe_slow );
10255 %}
10256 
10257 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10258                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10259 %{
10260   predicate(UseSSE42Intrinsics);
10261   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10262   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10263 
10264   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10265   ins_encode %{
10266     __ string_indexof($str1$$Register, $str2$$Register,
10267                       $cnt1$$Register, $cnt2$$Register,
10268                       (-1), $result$$Register,
10269                       $vec$$XMMRegister, $tmp$$Register);
10270   %}
10271   ins_pipe( pipe_slow );
10272 %}
10273 
10274 // fast string equals
10275 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10276                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10277 %{
10278   match(Set result (StrEquals (Binary str1 str2) cnt));
10279   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10280 
10281   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10282   ins_encode %{
10283     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10284                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10285                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10286   %}
10287   ins_pipe( pipe_slow );
10288 %}
10289 
10290 // fast array equals
10291 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10292                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10293 %{
10294   match(Set result (AryEq ary1 ary2));
10295   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10296   //ins_cost(300);
10297 
10298   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10299   ins_encode %{
10300     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10301                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10302                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10303   %}
10304   ins_pipe( pipe_slow );
10305 %}
10306 
10307 // encode char[] to byte[] in ISO_8859_1
10308 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10309                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10310                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10311   match(Set result (EncodeISOArray src (Binary dst len)));
10312   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10313 
10314   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10315   ins_encode %{
10316     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10317                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10318                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10319   %}
10320   ins_pipe( pipe_slow );
10321 %}
10322 
10323 //----------Overflow Math Instructions-----------------------------------------
10324 
10325 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10326 %{
10327   match(Set cr (OverflowAddI op1 op2));
10328   effect(DEF cr, USE_KILL op1, USE op2);
10329 
10330   format %{ "addl    $op1, $op2\t# overflow check int" %}
10331 
10332   ins_encode %{
10333     __ addl($op1$$Register, $op2$$Register);
10334   %}
10335   ins_pipe(ialu_reg_reg);
10336 %}
10337 
10338 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10339 %{
10340   match(Set cr (OverflowAddI op1 op2));
10341   effect(DEF cr, USE_KILL op1, USE op2);
10342 
10343   format %{ "addl    $op1, $op2\t# overflow check int" %}
10344 
10345   ins_encode %{
10346     __ addl($op1$$Register, $op2$$constant);
10347   %}
10348   ins_pipe(ialu_reg_reg);
10349 %}
10350 
10351 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10352 %{
10353   match(Set cr (OverflowAddL op1 op2));
10354   effect(DEF cr, USE_KILL op1, USE op2);
10355 
10356   format %{ "addq    $op1, $op2\t# overflow check long" %}
10357   ins_encode %{
10358     __ addq($op1$$Register, $op2$$Register);
10359   %}
10360   ins_pipe(ialu_reg_reg);
10361 %}
10362 
10363 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10364 %{
10365   match(Set cr (OverflowAddL op1 op2));
10366   effect(DEF cr, USE_KILL op1, USE op2);
10367 
10368   format %{ "addq    $op1, $op2\t# overflow check long" %}
10369   ins_encode %{
10370     __ addq($op1$$Register, $op2$$constant);
10371   %}
10372   ins_pipe(ialu_reg_reg);
10373 %}
10374 
10375 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10376 %{
10377   match(Set cr (OverflowSubI op1 op2));
10378 
10379   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10380   ins_encode %{
10381     __ cmpl($op1$$Register, $op2$$Register);
10382   %}
10383   ins_pipe(ialu_reg_reg);
10384 %}
10385 
10386 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10387 %{
10388   match(Set cr (OverflowSubI op1 op2));
10389 
10390   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10391   ins_encode %{
10392     __ cmpl($op1$$Register, $op2$$constant);
10393   %}
10394   ins_pipe(ialu_reg_reg);
10395 %}
10396 
10397 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10398 %{
10399   match(Set cr (OverflowSubL op1 op2));
10400 
10401   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10402   ins_encode %{
10403     __ cmpq($op1$$Register, $op2$$Register);
10404   %}
10405   ins_pipe(ialu_reg_reg);
10406 %}
10407 
10408 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10409 %{
10410   match(Set cr (OverflowSubL op1 op2));
10411 
10412   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10413   ins_encode %{
10414     __ cmpq($op1$$Register, $op2$$constant);
10415   %}
10416   ins_pipe(ialu_reg_reg);
10417 %}
10418 
10419 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10420 %{
10421   match(Set cr (OverflowSubI zero op2));
10422   effect(DEF cr, USE_KILL op2);
10423 
10424   format %{ "negl    $op2\t# overflow check int" %}
10425   ins_encode %{
10426     __ negl($op2$$Register);
10427   %}
10428   ins_pipe(ialu_reg_reg);
10429 %}
10430 
10431 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10432 %{
10433   match(Set cr (OverflowSubL zero op2));
10434   effect(DEF cr, USE_KILL op2);
10435 
10436   format %{ "negq    $op2\t# overflow check long" %}
10437   ins_encode %{
10438     __ negq($op2$$Register);
10439   %}
10440   ins_pipe(ialu_reg_reg);
10441 %}
10442 
10443 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10444 %{
10445   match(Set cr (OverflowMulI op1 op2));
10446   effect(DEF cr, USE_KILL op1, USE op2);
10447 
10448   format %{ "imull    $op1, $op2\t# overflow check int" %}
10449   ins_encode %{
10450     __ imull($op1$$Register, $op2$$Register);
10451   %}
10452   ins_pipe(ialu_reg_reg_alu0);
10453 %}
10454 
10455 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10456 %{
10457   match(Set cr (OverflowMulI op1 op2));
10458   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10459 
10460   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10461   ins_encode %{
10462     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10463   %}
10464   ins_pipe(ialu_reg_reg_alu0);
10465 %}
10466 
10467 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10468 %{
10469   match(Set cr (OverflowMulL op1 op2));
10470   effect(DEF cr, USE_KILL op1, USE op2);
10471 
10472   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10473   ins_encode %{
10474     __ imulq($op1$$Register, $op2$$Register);
10475   %}
10476   ins_pipe(ialu_reg_reg_alu0);
10477 %}
10478 
10479 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10480 %{
10481   match(Set cr (OverflowMulL op1 op2));
10482   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10483 
10484   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10485   ins_encode %{
10486     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10487   %}
10488   ins_pipe(ialu_reg_reg_alu0);
10489 %}
10490 
10491 
10492 //----------Control Flow Instructions------------------------------------------
10493 // Signed compare Instructions
10494 
10495 // XXX more variants!!
10496 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10497 %{
10498   match(Set cr (CmpI op1 op2));
10499   effect(DEF cr, USE op1, USE op2);
10500 
10501   format %{ "cmpl    $op1, $op2" %}
10502   opcode(0x3B);  /* Opcode 3B /r */
10503   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10504   ins_pipe(ialu_cr_reg_reg);
10505 %}
10506 
10507 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10508 %{
10509   match(Set cr (CmpI op1 op2));
10510 
10511   format %{ "cmpl    $op1, $op2" %}
10512   opcode(0x81, 0x07); /* Opcode 81 /7 */
10513   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10514   ins_pipe(ialu_cr_reg_imm);
10515 %}
10516 
10517 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10518 %{
10519   match(Set cr (CmpI op1 (LoadI op2)));
10520 
10521   ins_cost(500); // XXX
10522   format %{ "cmpl    $op1, $op2" %}
10523   opcode(0x3B); /* Opcode 3B /r */
10524   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10525   ins_pipe(ialu_cr_reg_mem);
10526 %}
10527 
10528 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10529 %{
10530   match(Set cr (CmpI src zero));
10531 
10532   format %{ "testl   $src, $src" %}
10533   opcode(0x85);
10534   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10535   ins_pipe(ialu_cr_reg_imm);
10536 %}
10537 
10538 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10539 %{
10540   match(Set cr (CmpI (AndI src con) zero));
10541 
10542   format %{ "testl   $src, $con" %}
10543   opcode(0xF7, 0x00);
10544   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10545   ins_pipe(ialu_cr_reg_imm);
10546 %}
10547 
10548 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10549 %{
10550   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10551 
10552   format %{ "testl   $src, $mem" %}
10553   opcode(0x85);
10554   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10555   ins_pipe(ialu_cr_reg_mem);
10556 %}
10557 
10558 // Unsigned compare Instructions; really, same as signed except they
10559 // produce an rFlagsRegU instead of rFlagsReg.
10560 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10561 %{
10562   match(Set cr (CmpU op1 op2));
10563 
10564   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10565   opcode(0x3B); /* Opcode 3B /r */
10566   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10567   ins_pipe(ialu_cr_reg_reg);
10568 %}
10569 
10570 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10571 %{
10572   match(Set cr (CmpU op1 op2));
10573 
10574   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10575   opcode(0x81,0x07); /* Opcode 81 /7 */
10576   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10577   ins_pipe(ialu_cr_reg_imm);
10578 %}
10579 
10580 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10581 %{
10582   match(Set cr (CmpU op1 (LoadI op2)));
10583 
10584   ins_cost(500); // XXX
10585   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10586   opcode(0x3B); /* Opcode 3B /r */
10587   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10588   ins_pipe(ialu_cr_reg_mem);
10589 %}
10590 
10591 // // // Cisc-spilled version of cmpU_rReg
10592 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10593 // //%{
10594 // //  match(Set cr (CmpU (LoadI op1) op2));
10595 // //
10596 // //  format %{ "CMPu   $op1,$op2" %}
10597 // //  ins_cost(500);
10598 // //  opcode(0x39);  /* Opcode 39 /r */
10599 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10600 // //%}
10601 
10602 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10603 %{
10604   match(Set cr (CmpU src zero));
10605 
10606   format %{ "testl  $src, $src\t# unsigned" %}
10607   opcode(0x85);
10608   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10609   ins_pipe(ialu_cr_reg_imm);
10610 %}
10611 
10612 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10613 %{
10614   match(Set cr (CmpP op1 op2));
10615 
10616   format %{ "cmpq    $op1, $op2\t# ptr" %}
10617   opcode(0x3B); /* Opcode 3B /r */
10618   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10619   ins_pipe(ialu_cr_reg_reg);
10620 %}
10621 
10622 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10623 %{
10624   match(Set cr (CmpP op1 (LoadP op2)));
10625 
10626   ins_cost(500); // XXX
10627   format %{ "cmpq    $op1, $op2\t# ptr" %}
10628   opcode(0x3B); /* Opcode 3B /r */
10629   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10630   ins_pipe(ialu_cr_reg_mem);
10631 %}
10632 
10633 // // // Cisc-spilled version of cmpP_rReg
10634 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10635 // //%{
10636 // //  match(Set cr (CmpP (LoadP op1) op2));
10637 // //
10638 // //  format %{ "CMPu   $op1,$op2" %}
10639 // //  ins_cost(500);
10640 // //  opcode(0x39);  /* Opcode 39 /r */
10641 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10642 // //%}
10643 
10644 // XXX this is generalized by compP_rReg_mem???
10645 // Compare raw pointer (used in out-of-heap check).
10646 // Only works because non-oop pointers must be raw pointers
10647 // and raw pointers have no anti-dependencies.
10648 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10649 %{
10650   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10651   match(Set cr (CmpP op1 (LoadP op2)));
10652 
10653   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10654   opcode(0x3B); /* Opcode 3B /r */
10655   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10656   ins_pipe(ialu_cr_reg_mem);
10657 %}
10658 
10659 // This will generate a signed flags result. This should be OK since
10660 // any compare to a zero should be eq/neq.
10661 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10662 %{
10663   match(Set cr (CmpP src zero));
10664 
10665   format %{ "testq   $src, $src\t# ptr" %}
10666   opcode(0x85);
10667   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10668   ins_pipe(ialu_cr_reg_imm);
10669 %}
10670 
10671 // This will generate a signed flags result. This should be OK since
10672 // any compare to a zero should be eq/neq.
10673 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10674 %{
10675   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10676   match(Set cr (CmpP (LoadP op) zero));
10677 
10678   ins_cost(500); // XXX
10679   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10680   opcode(0xF7); /* Opcode F7 /0 */
10681   ins_encode(REX_mem_wide(op),
10682              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10683   ins_pipe(ialu_cr_reg_imm);
10684 %}
10685 
10686 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10687 %{
10688   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10689   match(Set cr (CmpP (LoadP mem) zero));
10690 
10691   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10692   ins_encode %{
10693     __ cmpq(r12, $mem$$Address);
10694   %}
10695   ins_pipe(ialu_cr_reg_mem);
10696 %}
10697 
10698 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10699 %{
10700   match(Set cr (CmpN op1 op2));
10701 
10702   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10703   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10704   ins_pipe(ialu_cr_reg_reg);
10705 %}
10706 
10707 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10708 %{
10709   match(Set cr (CmpN src (LoadN mem)));
10710 
10711   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10712   ins_encode %{
10713     __ cmpl($src$$Register, $mem$$Address);
10714   %}
10715   ins_pipe(ialu_cr_reg_mem);
10716 %}
10717 
10718 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10719   match(Set cr (CmpN op1 op2));
10720 
10721   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10722   ins_encode %{
10723     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10724   %}
10725   ins_pipe(ialu_cr_reg_imm);
10726 %}
10727 
10728 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10729 %{
10730   match(Set cr (CmpN src (LoadN mem)));
10731 
10732   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10733   ins_encode %{
10734     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10735   %}
10736   ins_pipe(ialu_cr_reg_mem);
10737 %}
10738 
10739 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10740   match(Set cr (CmpN op1 op2));
10741 
10742   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10743   ins_encode %{
10744     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10745   %}
10746   ins_pipe(ialu_cr_reg_imm);
10747 %}
10748 
10749 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10750 %{
10751   match(Set cr (CmpN src (LoadNKlass mem)));
10752 
10753   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10754   ins_encode %{
10755     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10756   %}
10757   ins_pipe(ialu_cr_reg_mem);
10758 %}
10759 
10760 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10761   match(Set cr (CmpN src zero));
10762 
10763   format %{ "testl   $src, $src\t# compressed ptr" %}
10764   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10765   ins_pipe(ialu_cr_reg_imm);
10766 %}
10767 
10768 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10769 %{
10770   predicate(Universe::narrow_oop_base() != NULL);
10771   match(Set cr (CmpN (LoadN mem) zero));
10772 
10773   ins_cost(500); // XXX
10774   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10775   ins_encode %{
10776     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10777   %}
10778   ins_pipe(ialu_cr_reg_mem);
10779 %}
10780 
10781 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10782 %{
10783   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10784   match(Set cr (CmpN (LoadN mem) zero));
10785 
10786   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10787   ins_encode %{
10788     __ cmpl(r12, $mem$$Address);
10789   %}
10790   ins_pipe(ialu_cr_reg_mem);
10791 %}
10792 
10793 // Yanked all unsigned pointer compare operations.
10794 // Pointer compares are done with CmpP which is already unsigned.
10795 
10796 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10797 %{
10798   match(Set cr (CmpL op1 op2));
10799 
10800   format %{ "cmpq    $op1, $op2" %}
10801   opcode(0x3B);  /* Opcode 3B /r */
10802   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10803   ins_pipe(ialu_cr_reg_reg);
10804 %}
10805 
10806 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10807 %{
10808   match(Set cr (CmpL op1 op2));
10809 
10810   format %{ "cmpq    $op1, $op2" %}
10811   opcode(0x81, 0x07); /* Opcode 81 /7 */
10812   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
10813   ins_pipe(ialu_cr_reg_imm);
10814 %}
10815 
10816 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
10817 %{
10818   match(Set cr (CmpL op1 (LoadL op2)));
10819 
10820   format %{ "cmpq    $op1, $op2" %}
10821   opcode(0x3B); /* Opcode 3B /r */
10822   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10823   ins_pipe(ialu_cr_reg_mem);
10824 %}
10825 
10826 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
10827 %{
10828   match(Set cr (CmpL src zero));
10829 
10830   format %{ "testq   $src, $src" %}
10831   opcode(0x85);
10832   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10833   ins_pipe(ialu_cr_reg_imm);
10834 %}
10835 
10836 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
10837 %{
10838   match(Set cr (CmpL (AndL src con) zero));
10839 
10840   format %{ "testq   $src, $con\t# long" %}
10841   opcode(0xF7, 0x00);
10842   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
10843   ins_pipe(ialu_cr_reg_imm);
10844 %}
10845 
10846 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
10847 %{
10848   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
10849 
10850   format %{ "testq   $src, $mem" %}
10851   opcode(0x85);
10852   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
10853   ins_pipe(ialu_cr_reg_mem);
10854 %}
10855 
10856 // Manifest a CmpL result in an integer register.  Very painful.
10857 // This is the test to avoid.
10858 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
10859 %{
10860   match(Set dst (CmpL3 src1 src2));
10861   effect(KILL flags);
10862 
10863   ins_cost(275); // XXX
10864   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
10865             "movl    $dst, -1\n\t"
10866             "jl,s    done\n\t"
10867             "setne   $dst\n\t"
10868             "movzbl  $dst, $dst\n\t"
10869     "done:" %}
10870   ins_encode(cmpl3_flag(src1, src2, dst));
10871   ins_pipe(pipe_slow);
10872 %}
10873 
10874 //----------Max and Min--------------------------------------------------------
10875 // Min Instructions
10876 
10877 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
10878 %{
10879   effect(USE_DEF dst, USE src, USE cr);
10880 
10881   format %{ "cmovlgt $dst, $src\t# min" %}
10882   opcode(0x0F, 0x4F);
10883   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10884   ins_pipe(pipe_cmov_reg);
10885 %}
10886 
10887 
10888 instruct minI_rReg(rRegI dst, rRegI src)
10889 %{
10890   match(Set dst (MinI dst src));
10891 
10892   ins_cost(200);
10893   expand %{
10894     rFlagsReg cr;
10895     compI_rReg(cr, dst, src);
10896     cmovI_reg_g(dst, src, cr);
10897   %}
10898 %}
10899 
10900 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
10901 %{
10902   effect(USE_DEF dst, USE src, USE cr);
10903 
10904   format %{ "cmovllt $dst, $src\t# max" %}
10905   opcode(0x0F, 0x4C);
10906   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
10907   ins_pipe(pipe_cmov_reg);
10908 %}
10909 
10910 
10911 instruct maxI_rReg(rRegI dst, rRegI src)
10912 %{
10913   match(Set dst (MaxI dst src));
10914 
10915   ins_cost(200);
10916   expand %{
10917     rFlagsReg cr;
10918     compI_rReg(cr, dst, src);
10919     cmovI_reg_l(dst, src, cr);
10920   %}
10921 %}
10922 
10923 // ============================================================================
10924 // Branch Instructions
10925 
10926 // Jump Direct - Label defines a relative address from JMP+1
10927 instruct jmpDir(label labl)
10928 %{
10929   match(Goto);
10930   effect(USE labl);
10931 
10932   ins_cost(300);
10933   format %{ "jmp     $labl" %}
10934   size(5);
10935   ins_encode %{
10936     Label* L = $labl$$label;
10937     __ jmp(*L, false); // Always long jump
10938   %}
10939   ins_pipe(pipe_jmp);
10940 %}
10941 
10942 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10943 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
10944 %{
10945   match(If cop cr);
10946   effect(USE labl);
10947 
10948   ins_cost(300);
10949   format %{ "j$cop     $labl" %}
10950   size(6);
10951   ins_encode %{
10952     Label* L = $labl$$label;
10953     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10954   %}
10955   ins_pipe(pipe_jcc);
10956 %}
10957 
10958 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10959 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
10960 %{
10961   match(CountedLoopEnd cop cr);
10962   effect(USE labl);
10963 
10964   ins_cost(300);
10965   format %{ "j$cop     $labl\t# loop end" %}
10966   size(6);
10967   ins_encode %{
10968     Label* L = $labl$$label;
10969     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10970   %}
10971   ins_pipe(pipe_jcc);
10972 %}
10973 
10974 // Jump Direct Conditional - Label defines a relative address from Jcc+1
10975 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
10976   match(CountedLoopEnd cop cmp);
10977   effect(USE labl);
10978 
10979   ins_cost(300);
10980   format %{ "j$cop,u   $labl\t# loop end" %}
10981   size(6);
10982   ins_encode %{
10983     Label* L = $labl$$label;
10984     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10985   %}
10986   ins_pipe(pipe_jcc);
10987 %}
10988 
10989 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
10990   match(CountedLoopEnd cop cmp);
10991   effect(USE labl);
10992 
10993   ins_cost(200);
10994   format %{ "j$cop,u   $labl\t# loop end" %}
10995   size(6);
10996   ins_encode %{
10997     Label* L = $labl$$label;
10998     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
10999   %}
11000   ins_pipe(pipe_jcc);
11001 %}
11002 
11003 // Jump Direct Conditional - using unsigned comparison
11004 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11005   match(If cop cmp);
11006   effect(USE labl);
11007 
11008   ins_cost(300);
11009   format %{ "j$cop,u  $labl" %}
11010   size(6);
11011   ins_encode %{
11012     Label* L = $labl$$label;
11013     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11014   %}
11015   ins_pipe(pipe_jcc);
11016 %}
11017 
11018 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11019   match(If cop cmp);
11020   effect(USE labl);
11021 
11022   ins_cost(200);
11023   format %{ "j$cop,u  $labl" %}
11024   size(6);
11025   ins_encode %{
11026     Label* L = $labl$$label;
11027     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11028   %}
11029   ins_pipe(pipe_jcc);
11030 %}
11031 
11032 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11033   match(If cop cmp);
11034   effect(USE labl);
11035 
11036   ins_cost(200);
11037   format %{ $$template
11038     if ($cop$$cmpcode == Assembler::notEqual) {
11039       $$emit$$"jp,u   $labl\n\t"
11040       $$emit$$"j$cop,u   $labl"
11041     } else {
11042       $$emit$$"jp,u   done\n\t"
11043       $$emit$$"j$cop,u   $labl\n\t"
11044       $$emit$$"done:"
11045     }
11046   %}
11047   ins_encode %{
11048     Label* l = $labl$$label;
11049     if ($cop$$cmpcode == Assembler::notEqual) {
11050       __ jcc(Assembler::parity, *l, false);
11051       __ jcc(Assembler::notEqual, *l, false);
11052     } else if ($cop$$cmpcode == Assembler::equal) {
11053       Label done;
11054       __ jccb(Assembler::parity, done);
11055       __ jcc(Assembler::equal, *l, false);
11056       __ bind(done);
11057     } else {
11058        ShouldNotReachHere();
11059     }
11060   %}
11061   ins_pipe(pipe_jcc);
11062 %}
11063 
11064 // ============================================================================
11065 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11066 // superklass array for an instance of the superklass.  Set a hidden
11067 // internal cache on a hit (cache is checked with exposed code in
11068 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11069 // encoding ALSO sets flags.
11070 
11071 instruct partialSubtypeCheck(rdi_RegP result,
11072                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11073                              rFlagsReg cr)
11074 %{
11075   match(Set result (PartialSubtypeCheck sub super));
11076   effect(KILL rcx, KILL cr);
11077 
11078   ins_cost(1100);  // slightly larger than the next version
11079   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11080             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11081             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11082             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11083             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11084             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11085             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11086     "miss:\t" %}
11087 
11088   opcode(0x1); // Force a XOR of RDI
11089   ins_encode(enc_PartialSubtypeCheck());
11090   ins_pipe(pipe_slow);
11091 %}
11092 
11093 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11094                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11095                                      immP0 zero,
11096                                      rdi_RegP result)
11097 %{
11098   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11099   effect(KILL rcx, KILL result);
11100 
11101   ins_cost(1000);
11102   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11103             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11104             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11105             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11106             "jne,s   miss\t\t# Missed: flags nz\n\t"
11107             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11108     "miss:\t" %}
11109 
11110   opcode(0x0); // No need to XOR RDI
11111   ins_encode(enc_PartialSubtypeCheck());
11112   ins_pipe(pipe_slow);
11113 %}
11114 
11115 // ============================================================================
11116 // Branch Instructions -- short offset versions
11117 //
11118 // These instructions are used to replace jumps of a long offset (the default
11119 // match) with jumps of a shorter offset.  These instructions are all tagged
11120 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11121 // match rules in general matching.  Instead, the ADLC generates a conversion
11122 // method in the MachNode which can be used to do in-place replacement of the
11123 // long variant with the shorter variant.  The compiler will determine if a
11124 // branch can be taken by the is_short_branch_offset() predicate in the machine
11125 // specific code section of the file.
11126 
11127 // Jump Direct - Label defines a relative address from JMP+1
11128 instruct jmpDir_short(label labl) %{
11129   match(Goto);
11130   effect(USE labl);
11131 
11132   ins_cost(300);
11133   format %{ "jmp,s   $labl" %}
11134   size(2);
11135   ins_encode %{
11136     Label* L = $labl$$label;
11137     __ jmpb(*L);
11138   %}
11139   ins_pipe(pipe_jmp);
11140   ins_short_branch(1);
11141 %}
11142 
11143 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11144 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11145   match(If cop cr);
11146   effect(USE labl);
11147 
11148   ins_cost(300);
11149   format %{ "j$cop,s   $labl" %}
11150   size(2);
11151   ins_encode %{
11152     Label* L = $labl$$label;
11153     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11154   %}
11155   ins_pipe(pipe_jcc);
11156   ins_short_branch(1);
11157 %}
11158 
11159 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11160 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11161   match(CountedLoopEnd cop cr);
11162   effect(USE labl);
11163 
11164   ins_cost(300);
11165   format %{ "j$cop,s   $labl\t# loop end" %}
11166   size(2);
11167   ins_encode %{
11168     Label* L = $labl$$label;
11169     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11170   %}
11171   ins_pipe(pipe_jcc);
11172   ins_short_branch(1);
11173 %}
11174 
11175 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11176 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11177   match(CountedLoopEnd cop cmp);
11178   effect(USE labl);
11179 
11180   ins_cost(300);
11181   format %{ "j$cop,us  $labl\t# loop end" %}
11182   size(2);
11183   ins_encode %{
11184     Label* L = $labl$$label;
11185     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11186   %}
11187   ins_pipe(pipe_jcc);
11188   ins_short_branch(1);
11189 %}
11190 
11191 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11192   match(CountedLoopEnd cop cmp);
11193   effect(USE labl);
11194 
11195   ins_cost(300);
11196   format %{ "j$cop,us  $labl\t# loop end" %}
11197   size(2);
11198   ins_encode %{
11199     Label* L = $labl$$label;
11200     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11201   %}
11202   ins_pipe(pipe_jcc);
11203   ins_short_branch(1);
11204 %}
11205 
11206 // Jump Direct Conditional - using unsigned comparison
11207 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11208   match(If cop cmp);
11209   effect(USE labl);
11210 
11211   ins_cost(300);
11212   format %{ "j$cop,us  $labl" %}
11213   size(2);
11214   ins_encode %{
11215     Label* L = $labl$$label;
11216     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11217   %}
11218   ins_pipe(pipe_jcc);
11219   ins_short_branch(1);
11220 %}
11221 
11222 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11223   match(If cop cmp);
11224   effect(USE labl);
11225 
11226   ins_cost(300);
11227   format %{ "j$cop,us  $labl" %}
11228   size(2);
11229   ins_encode %{
11230     Label* L = $labl$$label;
11231     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11232   %}
11233   ins_pipe(pipe_jcc);
11234   ins_short_branch(1);
11235 %}
11236 
11237 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11238   match(If cop cmp);
11239   effect(USE labl);
11240 
11241   ins_cost(300);
11242   format %{ $$template
11243     if ($cop$$cmpcode == Assembler::notEqual) {
11244       $$emit$$"jp,u,s   $labl\n\t"
11245       $$emit$$"j$cop,u,s   $labl"
11246     } else {
11247       $$emit$$"jp,u,s   done\n\t"
11248       $$emit$$"j$cop,u,s  $labl\n\t"
11249       $$emit$$"done:"
11250     }
11251   %}
11252   size(4);
11253   ins_encode %{
11254     Label* l = $labl$$label;
11255     if ($cop$$cmpcode == Assembler::notEqual) {
11256       __ jccb(Assembler::parity, *l);
11257       __ jccb(Assembler::notEqual, *l);
11258     } else if ($cop$$cmpcode == Assembler::equal) {
11259       Label done;
11260       __ jccb(Assembler::parity, done);
11261       __ jccb(Assembler::equal, *l);
11262       __ bind(done);
11263     } else {
11264        ShouldNotReachHere();
11265     }
11266   %}
11267   ins_pipe(pipe_jcc);
11268   ins_short_branch(1);
11269 %}
11270 
11271 // ============================================================================
11272 // inlined locking and unlocking
11273 
11274 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11275   predicate(Compile::current()->use_rtm());
11276   match(Set cr (FastLock object box));
11277   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11278   ins_cost(300);
11279   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11280   ins_encode %{
11281     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11282                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11283                  _counters, _rtm_counters, _stack_rtm_counters,
11284                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11285                  true, ra_->C->profile_rtm());
11286   %}
11287   ins_pipe(pipe_slow);
11288 %}
11289 
11290 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11291   predicate(!Compile::current()->use_rtm());
11292   match(Set cr (FastLock object box));
11293   effect(TEMP tmp, TEMP scr, USE_KILL box);
11294   ins_cost(300);
11295   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11296   ins_encode %{
11297     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11298                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11299   %}
11300   ins_pipe(pipe_slow);
11301 %}
11302 
11303 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11304   match(Set cr (FastUnlock object box));
11305   effect(TEMP tmp, USE_KILL box);
11306   ins_cost(300);
11307   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11308   ins_encode %{
11309     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11310   %}
11311   ins_pipe(pipe_slow);
11312 %}
11313 
11314 
11315 // ============================================================================
11316 // Safepoint Instructions
11317 instruct safePoint_poll(rFlagsReg cr)
11318 %{
11319   predicate(!Assembler::is_polling_page_far());
11320   match(SafePoint);
11321   effect(KILL cr);
11322 
11323   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11324             "# Safepoint: poll for GC" %}
11325   ins_cost(125);
11326   ins_encode %{
11327     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11328     __ testl(rax, addr);
11329   %}
11330   ins_pipe(ialu_reg_mem);
11331 %}
11332 
11333 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11334 %{
11335   predicate(Assembler::is_polling_page_far());
11336   match(SafePoint poll);
11337   effect(KILL cr, USE poll);
11338 
11339   format %{ "testl  rax, [$poll]\t"
11340             "# Safepoint: poll for GC" %}
11341   ins_cost(125);
11342   ins_encode %{
11343     __ relocate(relocInfo::poll_type);
11344     __ testl(rax, Address($poll$$Register, 0));
11345   %}
11346   ins_pipe(ialu_reg_mem);
11347 %}
11348 
11349 // ============================================================================
11350 // Procedure Call/Return Instructions
11351 // Call Java Static Instruction
11352 // Note: If this code changes, the corresponding ret_addr_offset() and
11353 //       compute_padding() functions will have to be adjusted.
11354 instruct CallStaticJavaDirect(method meth) %{
11355   match(CallStaticJava);
11356   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
11357   effect(USE meth);
11358 
11359   ins_cost(300);
11360   format %{ "call,static " %}
11361   opcode(0xE8); /* E8 cd */
11362   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11363   ins_pipe(pipe_slow);
11364   ins_alignment(4);
11365 %}
11366 
11367 // Call Java Static Instruction (method handle version)
11368 // Note: If this code changes, the corresponding ret_addr_offset() and
11369 //       compute_padding() functions will have to be adjusted.
11370 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
11371   match(CallStaticJava);
11372   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
11373   effect(USE meth);
11374   // RBP is saved by all callees (for interpreter stack correction).
11375   // We use it here for a similar purpose, in {preserve,restore}_SP.
11376 
11377   ins_cost(300);
11378   format %{ "call,static/MethodHandle " %}
11379   opcode(0xE8); /* E8 cd */
11380   ins_encode(clear_avx, preserve_SP,
11381              Java_Static_Call(meth),
11382              restore_SP,
11383              call_epilog);
11384   ins_pipe(pipe_slow);
11385   ins_alignment(4);
11386 %}
11387 
11388 // Call Java Dynamic Instruction
11389 // Note: If this code changes, the corresponding ret_addr_offset() and
11390 //       compute_padding() functions will have to be adjusted.
11391 instruct CallDynamicJavaDirect(method meth)
11392 %{
11393   match(CallDynamicJava);
11394   effect(USE meth);
11395 
11396   ins_cost(300);
11397   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11398             "call,dynamic " %}
11399   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11400   ins_pipe(pipe_slow);
11401   ins_alignment(4);
11402 %}
11403 
11404 // Call Runtime Instruction
11405 instruct CallRuntimeDirect(method meth)
11406 %{
11407   match(CallRuntime);
11408   effect(USE meth);
11409 
11410   ins_cost(300);
11411   format %{ "call,runtime " %}
11412   ins_encode(clear_avx, Java_To_Runtime(meth));
11413   ins_pipe(pipe_slow);
11414 %}
11415 
11416 // Call runtime without safepoint
11417 instruct CallLeafDirect(method meth)
11418 %{
11419   match(CallLeaf);
11420   effect(USE meth);
11421 
11422   ins_cost(300);
11423   format %{ "call_leaf,runtime " %}
11424   ins_encode(clear_avx, Java_To_Runtime(meth));
11425   ins_pipe(pipe_slow);
11426 %}
11427 
11428 // Call runtime without safepoint
11429 instruct CallLeafNoFPDirect(method meth)
11430 %{
11431   match(CallLeafNoFP);
11432   effect(USE meth);
11433 
11434   ins_cost(300);
11435   format %{ "call_leaf_nofp,runtime " %}
11436   ins_encode(Java_To_Runtime(meth));
11437   ins_pipe(pipe_slow);
11438 %}
11439 
11440 // Return Instruction
11441 // Remove the return address & jump to it.
11442 // Notice: We always emit a nop after a ret to make sure there is room
11443 // for safepoint patching
11444 instruct Ret()
11445 %{
11446   match(Return);
11447 
11448   format %{ "ret" %}
11449   opcode(0xC3);
11450   ins_encode(OpcP);
11451   ins_pipe(pipe_jmp);
11452 %}
11453 
11454 // Tail Call; Jump from runtime stub to Java code.
11455 // Also known as an 'interprocedural jump'.
11456 // Target of jump will eventually return to caller.
11457 // TailJump below removes the return address.
11458 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11459 %{
11460   match(TailCall jump_target method_oop);
11461 
11462   ins_cost(300);
11463   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11464   opcode(0xFF, 0x4); /* Opcode FF /4 */
11465   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11466   ins_pipe(pipe_jmp);
11467 %}
11468 
11469 // Tail Jump; remove the return address; jump to target.
11470 // TailCall above leaves the return address around.
11471 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11472 %{
11473   match(TailJump jump_target ex_oop);
11474 
11475   ins_cost(300);
11476   format %{ "popq    rdx\t# pop return address\n\t"
11477             "jmp     $jump_target" %}
11478   opcode(0xFF, 0x4); /* Opcode FF /4 */
11479   ins_encode(Opcode(0x5a), // popq rdx
11480              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11481   ins_pipe(pipe_jmp);
11482 %}
11483 
11484 // Create exception oop: created by stack-crawling runtime code.
11485 // Created exception is now available to this handler, and is setup
11486 // just prior to jumping to this handler.  No code emitted.
11487 instruct CreateException(rax_RegP ex_oop)
11488 %{
11489   match(Set ex_oop (CreateEx));
11490 
11491   size(0);
11492   // use the following format syntax
11493   format %{ "# exception oop is in rax; no code emitted" %}
11494   ins_encode();
11495   ins_pipe(empty);
11496 %}
11497 
11498 // Rethrow exception:
11499 // The exception oop will come in the first argument position.
11500 // Then JUMP (not call) to the rethrow stub code.
11501 instruct RethrowException()
11502 %{
11503   match(Rethrow);
11504 
11505   // use the following format syntax
11506   format %{ "jmp     rethrow_stub" %}
11507   ins_encode(enc_rethrow);
11508   ins_pipe(pipe_jmp);
11509 %}
11510 
11511 
11512 // ============================================================================
11513 // This name is KNOWN by the ADLC and cannot be changed.
11514 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11515 // for this guy.
11516 instruct tlsLoadP(r15_RegP dst) %{
11517   match(Set dst (ThreadLocal));
11518   effect(DEF dst);
11519 
11520   size(0);
11521   format %{ "# TLS is in R15" %}
11522   ins_encode( /*empty encoding*/ );
11523   ins_pipe(ialu_reg_reg);
11524 %}
11525 
11526 
11527 //----------PEEPHOLE RULES-----------------------------------------------------
11528 // These must follow all instruction definitions as they use the names
11529 // defined in the instructions definitions.
11530 //
11531 // peepmatch ( root_instr_name [preceding_instruction]* );
11532 //
11533 // peepconstraint %{
11534 // (instruction_number.operand_name relational_op instruction_number.operand_name
11535 //  [, ...] );
11536 // // instruction numbers are zero-based using left to right order in peepmatch
11537 //
11538 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11539 // // provide an instruction_number.operand_name for each operand that appears
11540 // // in the replacement instruction's match rule
11541 //
11542 // ---------VM FLAGS---------------------------------------------------------
11543 //
11544 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11545 //
11546 // Each peephole rule is given an identifying number starting with zero and
11547 // increasing by one in the order seen by the parser.  An individual peephole
11548 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11549 // on the command-line.
11550 //
11551 // ---------CURRENT LIMITATIONS----------------------------------------------
11552 //
11553 // Only match adjacent instructions in same basic block
11554 // Only equality constraints
11555 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11556 // Only one replacement instruction
11557 //
11558 // ---------EXAMPLE----------------------------------------------------------
11559 //
11560 // // pertinent parts of existing instructions in architecture description
11561 // instruct movI(rRegI dst, rRegI src)
11562 // %{
11563 //   match(Set dst (CopyI src));
11564 // %}
11565 //
11566 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11567 // %{
11568 //   match(Set dst (AddI dst src));
11569 //   effect(KILL cr);
11570 // %}
11571 //
11572 // // Change (inc mov) to lea
11573 // peephole %{
11574 //   // increment preceeded by register-register move
11575 //   peepmatch ( incI_rReg movI );
11576 //   // require that the destination register of the increment
11577 //   // match the destination register of the move
11578 //   peepconstraint ( 0.dst == 1.dst );
11579 //   // construct a replacement instruction that sets
11580 //   // the destination to ( move's source register + one )
11581 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11582 // %}
11583 //
11584 
11585 // Implementation no longer uses movX instructions since
11586 // machine-independent system no longer uses CopyX nodes.
11587 //
11588 // peephole
11589 // %{
11590 //   peepmatch (incI_rReg movI);
11591 //   peepconstraint (0.dst == 1.dst);
11592 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11593 // %}
11594 
11595 // peephole
11596 // %{
11597 //   peepmatch (decI_rReg movI);
11598 //   peepconstraint (0.dst == 1.dst);
11599 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11600 // %}
11601 
11602 // peephole
11603 // %{
11604 //   peepmatch (addI_rReg_imm movI);
11605 //   peepconstraint (0.dst == 1.dst);
11606 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11607 // %}
11608 
11609 // peephole
11610 // %{
11611 //   peepmatch (incL_rReg movL);
11612 //   peepconstraint (0.dst == 1.dst);
11613 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11614 // %}
11615 
11616 // peephole
11617 // %{
11618 //   peepmatch (decL_rReg movL);
11619 //   peepconstraint (0.dst == 1.dst);
11620 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11621 // %}
11622 
11623 // peephole
11624 // %{
11625 //   peepmatch (addL_rReg_imm movL);
11626 //   peepconstraint (0.dst == 1.dst);
11627 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11628 // %}
11629 
11630 // peephole
11631 // %{
11632 //   peepmatch (addP_rReg_imm movP);
11633 //   peepconstraint (0.dst == 1.dst);
11634 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11635 // %}
11636 
11637 // // Change load of spilled value to only a spill
11638 // instruct storeI(memory mem, rRegI src)
11639 // %{
11640 //   match(Set mem (StoreI mem src));
11641 // %}
11642 //
11643 // instruct loadI(rRegI dst, memory mem)
11644 // %{
11645 //   match(Set dst (LoadI mem));
11646 // %}
11647 //
11648 
11649 peephole
11650 %{
11651   peepmatch (loadI storeI);
11652   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11653   peepreplace (storeI(1.mem 1.mem 1.src));
11654 %}
11655 
11656 peephole
11657 %{
11658   peepmatch (loadL storeL);
11659   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11660   peepreplace (storeL(1.mem 1.mem 1.src));
11661 %}
11662 
11663 //----------SMARTSPILL RULES---------------------------------------------------
11664 // These must follow all instruction definitions as they use the names
11665 // defined in the instructions definitions.