rev 5902 : 8027754: Enable loop optimizations for loops with MathExact inside

   1 //
   2 // Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Class for all pointer registers (including RSP)
 170 reg_class any_reg(RAX, RAX_H,
 171                   RDX, RDX_H,
 172                   RBP, RBP_H,
 173                   RDI, RDI_H,
 174                   RSI, RSI_H,
 175                   RCX, RCX_H,
 176                   RBX, RBX_H,
 177                   RSP, RSP_H,
 178                   R8,  R8_H,
 179                   R9,  R9_H,
 180                   R10, R10_H,
 181                   R11, R11_H,
 182                   R12, R12_H,
 183                   R13, R13_H,
 184                   R14, R14_H,
 185                   R15, R15_H);
 186 
 187 // Class for all pointer registers except RSP
 188 reg_class ptr_reg(RAX, RAX_H,
 189                   RDX, RDX_H,
 190                   RBP, RBP_H,
 191                   RDI, RDI_H,
 192                   RSI, RSI_H,
 193                   RCX, RCX_H,
 194                   RBX, RBX_H,
 195                   R8,  R8_H,
 196                   R9,  R9_H,
 197                   R10, R10_H,
 198                   R11, R11_H,
 199                   R13, R13_H,
 200                   R14, R14_H);
 201 
 202 // Class for all pointer registers except RAX and RSP
 203 reg_class ptr_no_rax_reg(RDX, RDX_H,
 204                          RBP, RBP_H,
 205                          RDI, RDI_H,
 206                          RSI, RSI_H,
 207                          RCX, RCX_H,
 208                          RBX, RBX_H,
 209                          R8,  R8_H,
 210                          R9,  R9_H,
 211                          R10, R10_H,
 212                          R11, R11_H,
 213                          R13, R13_H,
 214                          R14, R14_H);
 215 
 216 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 217                          RAX, RAX_H,
 218                          RDI, RDI_H,
 219                          RSI, RSI_H,
 220                          RCX, RCX_H,
 221                          RBX, RBX_H,
 222                          R8,  R8_H,
 223                          R9,  R9_H,
 224                          R10, R10_H,
 225                          R11, R11_H,
 226                          R13, R13_H,
 227                          R14, R14_H);
 228 
 229 // Class for all pointer registers except RAX, RBX and RSP
 230 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 231                              RBP, RBP_H,
 232                              RDI, RDI_H,
 233                              RSI, RSI_H,
 234                              RCX, RCX_H,
 235                              R8,  R8_H,
 236                              R9,  R9_H,
 237                              R10, R10_H,
 238                              R11, R11_H,
 239                              R13, R13_H,
 240                              R14, R14_H);
 241 
 242 // Singleton class for RAX pointer register
 243 reg_class ptr_rax_reg(RAX, RAX_H);
 244 
 245 // Singleton class for RBX pointer register
 246 reg_class ptr_rbx_reg(RBX, RBX_H);
 247 
 248 // Singleton class for RSI pointer register
 249 reg_class ptr_rsi_reg(RSI, RSI_H);
 250 
 251 // Singleton class for RDI pointer register
 252 reg_class ptr_rdi_reg(RDI, RDI_H);
 253 
 254 // Singleton class for RBP pointer register
 255 reg_class ptr_rbp_reg(RBP, RBP_H);
 256 
 257 // Singleton class for stack pointer
 258 reg_class ptr_rsp_reg(RSP, RSP_H);
 259 
 260 // Singleton class for TLS pointer
 261 reg_class ptr_r15_reg(R15, R15_H);
 262 
 263 // Class for all long registers (except RSP)
 264 reg_class long_reg(RAX, RAX_H,
 265                    RDX, RDX_H,
 266                    RBP, RBP_H,
 267                    RDI, RDI_H,
 268                    RSI, RSI_H,
 269                    RCX, RCX_H,
 270                    RBX, RBX_H,
 271                    R8,  R8_H,
 272                    R9,  R9_H,
 273                    R10, R10_H,
 274                    R11, R11_H,
 275                    R13, R13_H,
 276                    R14, R14_H);
 277 
 278 // Class for all long registers except RAX, RDX (and RSP)
 279 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 280                               RDI, RDI_H,
 281                               RSI, RSI_H,
 282                               RCX, RCX_H,
 283                               RBX, RBX_H,
 284                               R8,  R8_H,
 285                               R9,  R9_H,
 286                               R10, R10_H,
 287                               R11, R11_H,
 288                               R13, R13_H,
 289                               R14, R14_H);
 290 
 291 // Class for all long registers except RCX (and RSP)
 292 reg_class long_no_rcx_reg(RBP, RBP_H,
 293                           RDI, RDI_H,
 294                           RSI, RSI_H,
 295                           RAX, RAX_H,
 296                           RDX, RDX_H,
 297                           RBX, RBX_H,
 298                           R8,  R8_H,
 299                           R9,  R9_H,
 300                           R10, R10_H,
 301                           R11, R11_H,
 302                           R13, R13_H,
 303                           R14, R14_H);
 304 
 305 // Class for all long registers except RAX (and RSP)
 306 reg_class long_no_rax_reg(RBP, RBP_H,
 307                           RDX, RDX_H,
 308                           RDI, RDI_H,
 309                           RSI, RSI_H,
 310                           RCX, RCX_H,
 311                           RBX, RBX_H,
 312                           R8,  R8_H,
 313                           R9,  R9_H,
 314                           R10, R10_H,
 315                           R11, R11_H,
 316                           R13, R13_H,
 317                           R14, R14_H);
 318 
 319 // Singleton class for RAX long register
 320 reg_class long_rax_reg(RAX, RAX_H);
 321 
 322 // Singleton class for RCX long register
 323 reg_class long_rcx_reg(RCX, RCX_H);
 324 
 325 // Singleton class for RDX long register
 326 reg_class long_rdx_reg(RDX, RDX_H);
 327 
 328 // Class for all int registers (except RSP)
 329 reg_class int_reg(RAX,
 330                   RDX,
 331                   RBP,
 332                   RDI,
 333                   RSI,
 334                   RCX,
 335                   RBX,
 336                   R8,
 337                   R9,
 338                   R10,
 339                   R11,
 340                   R13,
 341                   R14);
 342 
 343 // Class for all int registers except RCX (and RSP)
 344 reg_class int_no_rcx_reg(RAX,
 345                          RDX,
 346                          RBP,
 347                          RDI,
 348                          RSI,
 349                          RBX,
 350                          R8,
 351                          R9,
 352                          R10,
 353                          R11,
 354                          R13,
 355                          R14);
 356 
 357 // Class for all int registers except RAX, RDX (and RSP)
 358 reg_class int_no_rax_rdx_reg(RBP,
 359                              RDI,
 360                              RSI,
 361                              RCX,
 362                              RBX,
 363                              R8,
 364                              R9,
 365                              R10,
 366                              R11,
 367                              R13,
 368                              R14);
 369 
 370 // Singleton class for RAX int register
 371 reg_class int_rax_reg(RAX);
 372 
 373 // Singleton class for RBX int register
 374 reg_class int_rbx_reg(RBX);
 375 
 376 // Singleton class for RCX int register
 377 reg_class int_rcx_reg(RCX);
 378 
 379 // Singleton class for RCX int register
 380 reg_class int_rdx_reg(RDX);
 381 
 382 // Singleton class for RCX int register
 383 reg_class int_rdi_reg(RDI);
 384 
 385 // Singleton class for instruction pointer
 386 // reg_class ip_reg(RIP);
 387 
 388 %}
 389 
 390 //----------SOURCE BLOCK-------------------------------------------------------
 391 // This is a block of C++ code which provides values, functions, and
 392 // definitions necessary in the rest of the architecture description
 393 source %{
 394 #define   RELOC_IMM64    Assembler::imm_operand
 395 #define   RELOC_DISP32   Assembler::disp32_operand
 396 
 397 #define __ _masm.
 398 
 399 static int preserve_SP_size() {
 400   return 3;  // rex.w, op, rm(reg/reg)
 401 }
 402 static int clear_avx_size() {
 403   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 404 }
 405 
 406 // !!!!! Special hack to get all types of calls to specify the byte offset
 407 //       from the start of the call to the point where the return address
 408 //       will point.
 409 int MachCallStaticJavaNode::ret_addr_offset()
 410 {
 411   int offset = 5; // 5 bytes from start of call to where return address points
 412   offset += clear_avx_size();
 413   if (_method_handle_invoke)
 414     offset += preserve_SP_size();
 415   return offset;
 416 }
 417 
 418 int MachCallDynamicJavaNode::ret_addr_offset()
 419 {
 420   int offset = 15; // 15 bytes from start of call to where return address points
 421   offset += clear_avx_size();
 422   return offset;
 423 }
 424 
 425 int MachCallRuntimeNode::ret_addr_offset() {
 426   int offset = 13; // movq r10,#addr; callq (r10)
 427   offset += clear_avx_size();
 428   return offset;
 429 }
 430 
 431 // Indicate if the safepoint node needs the polling page as an input,
 432 // it does if the polling page is more than disp32 away.
 433 bool SafePointNode::needs_polling_address_input()
 434 {
 435   return Assembler::is_polling_page_far();
 436 }
 437 
 438 //
 439 // Compute padding required for nodes which need alignment
 440 //
 441 
 442 // The address of the call instruction needs to be 4-byte aligned to
 443 // ensure that it does not span a cache line so that it can be patched.
 444 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 445 {
 446   current_offset += clear_avx_size(); // skip vzeroupper
 447   current_offset += 1; // skip call opcode byte
 448   return round_to(current_offset, alignment_required()) - current_offset;
 449 }
 450 
 451 // The address of the call instruction needs to be 4-byte aligned to
 452 // ensure that it does not span a cache line so that it can be patched.
 453 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 454 {
 455   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 456   current_offset += clear_avx_size(); // skip vzeroupper
 457   current_offset += 1; // skip call opcode byte
 458   return round_to(current_offset, alignment_required()) - current_offset;
 459 }
 460 
 461 // The address of the call instruction needs to be 4-byte aligned to
 462 // ensure that it does not span a cache line so that it can be patched.
 463 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 464 {
 465   current_offset += clear_avx_size(); // skip vzeroupper
 466   current_offset += 11; // skip movq instruction + call opcode byte
 467   return round_to(current_offset, alignment_required()) - current_offset;
 468 }
 469 
 470 // EMIT_RM()
 471 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 472   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 473   cbuf.insts()->emit_int8(c);
 474 }
 475 
 476 // EMIT_CC()
 477 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 478   unsigned char c = (unsigned char) (f1 | f2);
 479   cbuf.insts()->emit_int8(c);
 480 }
 481 
 482 // EMIT_OPCODE()
 483 void emit_opcode(CodeBuffer &cbuf, int code) {
 484   cbuf.insts()->emit_int8((unsigned char) code);
 485 }
 486 
 487 // EMIT_OPCODE() w/ relocation information
 488 void emit_opcode(CodeBuffer &cbuf,
 489                  int code, relocInfo::relocType reloc, int offset, int format)
 490 {
 491   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 492   emit_opcode(cbuf, code);
 493 }
 494 
 495 // EMIT_D8()
 496 void emit_d8(CodeBuffer &cbuf, int d8) {
 497   cbuf.insts()->emit_int8((unsigned char) d8);
 498 }
 499 
 500 // EMIT_D16()
 501 void emit_d16(CodeBuffer &cbuf, int d16) {
 502   cbuf.insts()->emit_int16(d16);
 503 }
 504 
 505 // EMIT_D32()
 506 void emit_d32(CodeBuffer &cbuf, int d32) {
 507   cbuf.insts()->emit_int32(d32);
 508 }
 509 
 510 // EMIT_D64()
 511 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 512   cbuf.insts()->emit_int64(d64);
 513 }
 514 
 515 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 516 void emit_d32_reloc(CodeBuffer& cbuf,
 517                     int d32,
 518                     relocInfo::relocType reloc,
 519                     int format)
 520 {
 521   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 522   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 523   cbuf.insts()->emit_int32(d32);
 524 }
 525 
 526 // emit 32 bit value and construct relocation entry from RelocationHolder
 527 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 528 #ifdef ASSERT
 529   if (rspec.reloc()->type() == relocInfo::oop_type &&
 530       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 531     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 532     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 533   }
 534 #endif
 535   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 536   cbuf.insts()->emit_int32(d32);
 537 }
 538 
 539 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 540   address next_ip = cbuf.insts_end() + 4;
 541   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 542                  external_word_Relocation::spec(addr),
 543                  RELOC_DISP32);
 544 }
 545 
 546 
 547 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 548 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 549   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 550   cbuf.insts()->emit_int64(d64);
 551 }
 552 
 553 // emit 64 bit value and construct relocation entry from RelocationHolder
 554 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 555 #ifdef ASSERT
 556   if (rspec.reloc()->type() == relocInfo::oop_type &&
 557       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 558     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 559     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 560            "cannot embed scavengable oops in code");
 561   }
 562 #endif
 563   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 564   cbuf.insts()->emit_int64(d64);
 565 }
 566 
 567 // Access stack slot for load or store
 568 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 569 {
 570   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 571   if (-0x80 <= disp && disp < 0x80) {
 572     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 573     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 574     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 575   } else {
 576     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 577     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 578     emit_d32(cbuf, disp);     // Displacement // R/M byte
 579   }
 580 }
 581 
 582    // rRegI ereg, memory mem) %{    // emit_reg_mem
 583 void encode_RegMem(CodeBuffer &cbuf,
 584                    int reg,
 585                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 586 {
 587   assert(disp_reloc == relocInfo::none, "cannot have disp");
 588   int regenc = reg & 7;
 589   int baseenc = base & 7;
 590   int indexenc = index & 7;
 591 
 592   // There is no index & no scale, use form without SIB byte
 593   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 594     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 595     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 596       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 597     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 598       // If 8-bit displacement, mode 0x1
 599       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 600       emit_d8(cbuf, disp);
 601     } else {
 602       // If 32-bit displacement
 603       if (base == -1) { // Special flag for absolute address
 604         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 605         if (disp_reloc != relocInfo::none) {
 606           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 607         } else {
 608           emit_d32(cbuf, disp);
 609         }
 610       } else {
 611         // Normal base + offset
 612         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 613         if (disp_reloc != relocInfo::none) {
 614           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 615         } else {
 616           emit_d32(cbuf, disp);
 617         }
 618       }
 619     }
 620   } else {
 621     // Else, encode with the SIB byte
 622     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 623     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 624       // If no displacement
 625       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 626       emit_rm(cbuf, scale, indexenc, baseenc);
 627     } else {
 628       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 629         // If 8-bit displacement, mode 0x1
 630         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 631         emit_rm(cbuf, scale, indexenc, baseenc);
 632         emit_d8(cbuf, disp);
 633       } else {
 634         // If 32-bit displacement
 635         if (base == 0x04 ) {
 636           emit_rm(cbuf, 0x2, regenc, 0x4);
 637           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 638         } else {
 639           emit_rm(cbuf, 0x2, regenc, 0x4);
 640           emit_rm(cbuf, scale, indexenc, baseenc); // *
 641         }
 642         if (disp_reloc != relocInfo::none) {
 643           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 644         } else {
 645           emit_d32(cbuf, disp);
 646         }
 647       }
 648     }
 649   }
 650 }
 651 
 652 // This could be in MacroAssembler but it's fairly C2 specific
 653 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 654   Label exit;
 655   __ jccb(Assembler::noParity, exit);
 656   __ pushf();
 657   //
 658   // comiss/ucomiss instructions set ZF,PF,CF flags and
 659   // zero OF,AF,SF for NaN values.
 660   // Fixup flags by zeroing ZF,PF so that compare of NaN
 661   // values returns 'less than' result (CF is set).
 662   // Leave the rest of flags unchanged.
 663   //
 664   //    7 6 5 4 3 2 1 0
 665   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 666   //    0 0 1 0 1 0 1 1   (0x2B)
 667   //
 668   __ andq(Address(rsp, 0), 0xffffff2b);
 669   __ popf();
 670   __ bind(exit);
 671 }
 672 
 673 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 674   Label done;
 675   __ movl(dst, -1);
 676   __ jcc(Assembler::parity, done);
 677   __ jcc(Assembler::below, done);
 678   __ setb(Assembler::notEqual, dst);
 679   __ movzbl(dst, dst);
 680   __ bind(done);
 681 }
 682 
 683 
 684 //=============================================================================
 685 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 686 
 687 int Compile::ConstantTable::calculate_table_base_offset() const {
 688   return 0;  // absolute addressing, no offset
 689 }
 690 
 691 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 692 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 693   ShouldNotReachHere();
 694 }
 695 
 696 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 697   // Empty encoding
 698 }
 699 
 700 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 701   return 0;
 702 }
 703 
 704 #ifndef PRODUCT
 705 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 706   st->print("# MachConstantBaseNode (empty encoding)");
 707 }
 708 #endif
 709 
 710 
 711 //=============================================================================
 712 #ifndef PRODUCT
 713 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 714   Compile* C = ra_->C;
 715 
 716   int framesize = C->frame_slots() << LogBytesPerInt;
 717   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 718   // Remove wordSize for return addr which is already pushed.
 719   framesize -= wordSize;
 720 
 721   if (C->need_stack_bang(framesize)) {
 722     framesize -= wordSize;
 723     st->print("# stack bang");
 724     st->print("\n\t");
 725     st->print("pushq   rbp\t# Save rbp");
 726     if (framesize) {
 727       st->print("\n\t");
 728       st->print("subq    rsp, #%d\t# Create frame",framesize);
 729     }
 730   } else {
 731     st->print("subq    rsp, #%d\t# Create frame",framesize);
 732     st->print("\n\t");
 733     framesize -= wordSize;
 734     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 735   }
 736 
 737   if (VerifyStackAtCalls) {
 738     st->print("\n\t");
 739     framesize -= wordSize;
 740     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 741 #ifdef ASSERT
 742     st->print("\n\t");
 743     st->print("# stack alignment check");
 744 #endif
 745   }
 746   st->cr();
 747 }
 748 #endif
 749 
 750 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 751   Compile* C = ra_->C;
 752   MacroAssembler _masm(&cbuf);
 753 
 754   int framesize = C->frame_slots() << LogBytesPerInt;
 755 
 756   __ verified_entry(framesize, C->need_stack_bang(framesize), false);
 757 
 758   C->set_frame_complete(cbuf.insts_size());
 759 
 760   if (C->has_mach_constant_base_node()) {
 761     // NOTE: We set the table base offset here because users might be
 762     // emitted before MachConstantBaseNode.
 763     Compile::ConstantTable& constant_table = C->constant_table();
 764     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 765   }
 766 }
 767 
 768 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 769 {
 770   return MachNode::size(ra_); // too many variables; just compute it
 771                               // the hard way
 772 }
 773 
 774 int MachPrologNode::reloc() const
 775 {
 776   return 0; // a large enough number
 777 }
 778 
 779 //=============================================================================
 780 #ifndef PRODUCT
 781 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 782 {
 783   Compile* C = ra_->C;
 784   if (C->max_vector_size() > 16) {
 785     st->print("vzeroupper");
 786     st->cr(); st->print("\t");
 787   }
 788 
 789   int framesize = C->frame_slots() << LogBytesPerInt;
 790   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 791   // Remove word for return adr already pushed
 792   // and RBP
 793   framesize -= 2*wordSize;
 794 
 795   if (framesize) {
 796     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 797     st->print("\t");
 798   }
 799 
 800   st->print_cr("popq   rbp");
 801   if (do_polling() && C->is_method_compilation()) {
 802     st->print("\t");
 803     if (Assembler::is_polling_page_far()) {
 804       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 805                    "testl  rax, [rscratch1]\t"
 806                    "# Safepoint: poll for GC");
 807     } else {
 808       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 809                    "# Safepoint: poll for GC");
 810     }
 811   }
 812 }
 813 #endif
 814 
 815 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 816 {
 817   Compile* C = ra_->C;
 818   if (C->max_vector_size() > 16) {
 819     // Clear upper bits of YMM registers when current compiled code uses
 820     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 821     MacroAssembler _masm(&cbuf);
 822     __ vzeroupper();
 823   }
 824 
 825   int framesize = C->frame_slots() << LogBytesPerInt;
 826   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 827   // Remove word for return adr already pushed
 828   // and RBP
 829   framesize -= 2*wordSize;
 830 
 831   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 832 
 833   if (framesize) {
 834     emit_opcode(cbuf, Assembler::REX_W);
 835     if (framesize < 0x80) {
 836       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 837       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 838       emit_d8(cbuf, framesize);
 839     } else {
 840       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 841       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 842       emit_d32(cbuf, framesize);
 843     }
 844   }
 845 
 846   // popq rbp
 847   emit_opcode(cbuf, 0x58 | RBP_enc);
 848 
 849   if (do_polling() && C->is_method_compilation()) {
 850     MacroAssembler _masm(&cbuf);
 851     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 852     if (Assembler::is_polling_page_far()) {
 853       __ lea(rscratch1, polling_page);
 854       __ relocate(relocInfo::poll_return_type);
 855       __ testl(rax, Address(rscratch1, 0));
 856     } else {
 857       __ testl(rax, polling_page);
 858     }
 859   }
 860 }
 861 
 862 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
 863 {
 864   return MachNode::size(ra_); // too many variables; just compute it
 865                               // the hard way
 866 }
 867 
 868 int MachEpilogNode::reloc() const
 869 {
 870   return 2; // a large enough number
 871 }
 872 
 873 const Pipeline* MachEpilogNode::pipeline() const
 874 {
 875   return MachNode::pipeline_class();
 876 }
 877 
 878 int MachEpilogNode::safepoint_offset() const
 879 {
 880   return 0;
 881 }
 882 
 883 //=============================================================================
 884 
 885 enum RC {
 886   rc_bad,
 887   rc_int,
 888   rc_float,
 889   rc_stack
 890 };
 891 
 892 static enum RC rc_class(OptoReg::Name reg)
 893 {
 894   if( !OptoReg::is_valid(reg)  ) return rc_bad;
 895 
 896   if (OptoReg::is_stack(reg)) return rc_stack;
 897 
 898   VMReg r = OptoReg::as_VMReg(reg);
 899 
 900   if (r->is_Register()) return rc_int;
 901 
 902   assert(r->is_XMMRegister(), "must be");
 903   return rc_float;
 904 }
 905 
 906 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
 907 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
 908                           int src_hi, int dst_hi, uint ireg, outputStream* st);
 909 
 910 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
 911                             int stack_offset, int reg, uint ireg, outputStream* st);
 912 
 913 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
 914                                       int dst_offset, uint ireg, outputStream* st) {
 915   if (cbuf) {
 916     MacroAssembler _masm(cbuf);
 917     switch (ireg) {
 918     case Op_VecS:
 919       __ movq(Address(rsp, -8), rax);
 920       __ movl(rax, Address(rsp, src_offset));
 921       __ movl(Address(rsp, dst_offset), rax);
 922       __ movq(rax, Address(rsp, -8));
 923       break;
 924     case Op_VecD:
 925       __ pushq(Address(rsp, src_offset));
 926       __ popq (Address(rsp, dst_offset));
 927       break;
 928     case Op_VecX:
 929       __ pushq(Address(rsp, src_offset));
 930       __ popq (Address(rsp, dst_offset));
 931       __ pushq(Address(rsp, src_offset+8));
 932       __ popq (Address(rsp, dst_offset+8));
 933       break;
 934     case Op_VecY:
 935       __ vmovdqu(Address(rsp, -32), xmm0);
 936       __ vmovdqu(xmm0, Address(rsp, src_offset));
 937       __ vmovdqu(Address(rsp, dst_offset), xmm0);
 938       __ vmovdqu(xmm0, Address(rsp, -32));
 939       break;
 940     default:
 941       ShouldNotReachHere();
 942     }
 943 #ifndef PRODUCT
 944   } else {
 945     switch (ireg) {
 946     case Op_VecS:
 947       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
 948                 "movl    rax, [rsp + #%d]\n\t"
 949                 "movl    [rsp + #%d], rax\n\t"
 950                 "movq    rax, [rsp - #8]",
 951                 src_offset, dst_offset);
 952       break;
 953     case Op_VecD:
 954       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
 955                 "popq    [rsp + #%d]",
 956                 src_offset, dst_offset);
 957       break;
 958      case Op_VecX:
 959       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
 960                 "popq    [rsp + #%d]\n\t"
 961                 "pushq   [rsp + #%d]\n\t"
 962                 "popq    [rsp + #%d]",
 963                 src_offset, dst_offset, src_offset+8, dst_offset+8);
 964       break;
 965     case Op_VecY:
 966       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
 967                 "vmovdqu xmm0, [rsp + #%d]\n\t"
 968                 "vmovdqu [rsp + #%d], xmm0\n\t"
 969                 "vmovdqu xmm0, [rsp - #32]",
 970                 src_offset, dst_offset);
 971       break;
 972     default:
 973       ShouldNotReachHere();
 974     }
 975 #endif
 976   }
 977 }
 978 
 979 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
 980                                        PhaseRegAlloc* ra_,
 981                                        bool do_size,
 982                                        outputStream* st) const {
 983   assert(cbuf != NULL || st  != NULL, "sanity");
 984   // Get registers to move
 985   OptoReg::Name src_second = ra_->get_reg_second(in(1));
 986   OptoReg::Name src_first = ra_->get_reg_first(in(1));
 987   OptoReg::Name dst_second = ra_->get_reg_second(this);
 988   OptoReg::Name dst_first = ra_->get_reg_first(this);
 989 
 990   enum RC src_second_rc = rc_class(src_second);
 991   enum RC src_first_rc = rc_class(src_first);
 992   enum RC dst_second_rc = rc_class(dst_second);
 993   enum RC dst_first_rc = rc_class(dst_first);
 994 
 995   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
 996          "must move at least 1 register" );
 997 
 998   if (src_first == dst_first && src_second == dst_second) {
 999     // Self copy, no move
1000     return 0;
1001   }
1002   if (bottom_type()->isa_vect() != NULL) {
1003     uint ireg = ideal_reg();
1004     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1005     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY), "sanity");
1006     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1007       // mem -> mem
1008       int src_offset = ra_->reg2offset(src_first);
1009       int dst_offset = ra_->reg2offset(dst_first);
1010       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1011     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1012       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1013     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1014       int stack_offset = ra_->reg2offset(dst_first);
1015       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1016     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1017       int stack_offset = ra_->reg2offset(src_first);
1018       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1019     } else {
1020       ShouldNotReachHere();
1021     }
1022     return 0;
1023   }
1024   if (src_first_rc == rc_stack) {
1025     // mem ->
1026     if (dst_first_rc == rc_stack) {
1027       // mem -> mem
1028       assert(src_second != dst_first, "overlap");
1029       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1030           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1031         // 64-bit
1032         int src_offset = ra_->reg2offset(src_first);
1033         int dst_offset = ra_->reg2offset(dst_first);
1034         if (cbuf) {
1035           MacroAssembler _masm(cbuf);
1036           __ pushq(Address(rsp, src_offset));
1037           __ popq (Address(rsp, dst_offset));
1038 #ifndef PRODUCT
1039         } else {
1040           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1041                     "popq    [rsp + #%d]",
1042                      src_offset, dst_offset);
1043 #endif
1044         }
1045       } else {
1046         // 32-bit
1047         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1048         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1049         // No pushl/popl, so:
1050         int src_offset = ra_->reg2offset(src_first);
1051         int dst_offset = ra_->reg2offset(dst_first);
1052         if (cbuf) {
1053           MacroAssembler _masm(cbuf);
1054           __ movq(Address(rsp, -8), rax);
1055           __ movl(rax, Address(rsp, src_offset));
1056           __ movl(Address(rsp, dst_offset), rax);
1057           __ movq(rax, Address(rsp, -8));
1058 #ifndef PRODUCT
1059         } else {
1060           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1061                     "movl    rax, [rsp + #%d]\n\t"
1062                     "movl    [rsp + #%d], rax\n\t"
1063                     "movq    rax, [rsp - #8]",
1064                      src_offset, dst_offset);
1065 #endif
1066         }
1067       }
1068       return 0;
1069     } else if (dst_first_rc == rc_int) {
1070       // mem -> gpr
1071       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1072           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1073         // 64-bit
1074         int offset = ra_->reg2offset(src_first);
1075         if (cbuf) {
1076           MacroAssembler _masm(cbuf);
1077           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1078 #ifndef PRODUCT
1079         } else {
1080           st->print("movq    %s, [rsp + #%d]\t# spill",
1081                      Matcher::regName[dst_first],
1082                      offset);
1083 #endif
1084         }
1085       } else {
1086         // 32-bit
1087         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1088         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1089         int offset = ra_->reg2offset(src_first);
1090         if (cbuf) {
1091           MacroAssembler _masm(cbuf);
1092           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1093 #ifndef PRODUCT
1094         } else {
1095           st->print("movl    %s, [rsp + #%d]\t# spill",
1096                      Matcher::regName[dst_first],
1097                      offset);
1098 #endif
1099         }
1100       }
1101       return 0;
1102     } else if (dst_first_rc == rc_float) {
1103       // mem-> xmm
1104       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1105           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1106         // 64-bit
1107         int offset = ra_->reg2offset(src_first);
1108         if (cbuf) {
1109           MacroAssembler _masm(cbuf);
1110           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1111 #ifndef PRODUCT
1112         } else {
1113           st->print("%s  %s, [rsp + #%d]\t# spill",
1114                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1115                      Matcher::regName[dst_first],
1116                      offset);
1117 #endif
1118         }
1119       } else {
1120         // 32-bit
1121         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1122         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1123         int offset = ra_->reg2offset(src_first);
1124         if (cbuf) {
1125           MacroAssembler _masm(cbuf);
1126           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1127 #ifndef PRODUCT
1128         } else {
1129           st->print("movss   %s, [rsp + #%d]\t# spill",
1130                      Matcher::regName[dst_first],
1131                      offset);
1132 #endif
1133         }
1134       }
1135       return 0;
1136     }
1137   } else if (src_first_rc == rc_int) {
1138     // gpr ->
1139     if (dst_first_rc == rc_stack) {
1140       // gpr -> mem
1141       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1142           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1143         // 64-bit
1144         int offset = ra_->reg2offset(dst_first);
1145         if (cbuf) {
1146           MacroAssembler _masm(cbuf);
1147           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1148 #ifndef PRODUCT
1149         } else {
1150           st->print("movq    [rsp + #%d], %s\t# spill",
1151                      offset,
1152                      Matcher::regName[src_first]);
1153 #endif
1154         }
1155       } else {
1156         // 32-bit
1157         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1158         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1159         int offset = ra_->reg2offset(dst_first);
1160         if (cbuf) {
1161           MacroAssembler _masm(cbuf);
1162           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1163 #ifndef PRODUCT
1164         } else {
1165           st->print("movl    [rsp + #%d], %s\t# spill",
1166                      offset,
1167                      Matcher::regName[src_first]);
1168 #endif
1169         }
1170       }
1171       return 0;
1172     } else if (dst_first_rc == rc_int) {
1173       // gpr -> gpr
1174       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1175           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1176         // 64-bit
1177         if (cbuf) {
1178           MacroAssembler _masm(cbuf);
1179           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1180                   as_Register(Matcher::_regEncode[src_first]));
1181 #ifndef PRODUCT
1182         } else {
1183           st->print("movq    %s, %s\t# spill",
1184                      Matcher::regName[dst_first],
1185                      Matcher::regName[src_first]);
1186 #endif
1187         }
1188         return 0;
1189       } else {
1190         // 32-bit
1191         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1192         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1193         if (cbuf) {
1194           MacroAssembler _masm(cbuf);
1195           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1196                   as_Register(Matcher::_regEncode[src_first]));
1197 #ifndef PRODUCT
1198         } else {
1199           st->print("movl    %s, %s\t# spill",
1200                      Matcher::regName[dst_first],
1201                      Matcher::regName[src_first]);
1202 #endif
1203         }
1204         return 0;
1205       }
1206     } else if (dst_first_rc == rc_float) {
1207       // gpr -> xmm
1208       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1209           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1210         // 64-bit
1211         if (cbuf) {
1212           MacroAssembler _masm(cbuf);
1213           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1214 #ifndef PRODUCT
1215         } else {
1216           st->print("movdq   %s, %s\t# spill",
1217                      Matcher::regName[dst_first],
1218                      Matcher::regName[src_first]);
1219 #endif
1220         }
1221       } else {
1222         // 32-bit
1223         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1224         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1228 #ifndef PRODUCT
1229         } else {
1230           st->print("movdl   %s, %s\t# spill",
1231                      Matcher::regName[dst_first],
1232                      Matcher::regName[src_first]);
1233 #endif
1234         }
1235       }
1236       return 0;
1237     }
1238   } else if (src_first_rc == rc_float) {
1239     // xmm ->
1240     if (dst_first_rc == rc_stack) {
1241       // xmm -> mem
1242       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1243           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1244         // 64-bit
1245         int offset = ra_->reg2offset(dst_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("movsd   [rsp + #%d], %s\t# spill",
1252                      offset,
1253                      Matcher::regName[src_first]);
1254 #endif
1255         }
1256       } else {
1257         // 32-bit
1258         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1259         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1260         int offset = ra_->reg2offset(dst_first);
1261         if (cbuf) {
1262           MacroAssembler _masm(cbuf);
1263           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1264 #ifndef PRODUCT
1265         } else {
1266           st->print("movss   [rsp + #%d], %s\t# spill",
1267                      offset,
1268                      Matcher::regName[src_first]);
1269 #endif
1270         }
1271       }
1272       return 0;
1273     } else if (dst_first_rc == rc_int) {
1274       // xmm -> gpr
1275       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1276           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1277         // 64-bit
1278         if (cbuf) {
1279           MacroAssembler _masm(cbuf);
1280           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1281 #ifndef PRODUCT
1282         } else {
1283           st->print("movdq   %s, %s\t# spill",
1284                      Matcher::regName[dst_first],
1285                      Matcher::regName[src_first]);
1286 #endif
1287         }
1288       } else {
1289         // 32-bit
1290         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1291         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1292         if (cbuf) {
1293           MacroAssembler _masm(cbuf);
1294           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1295 #ifndef PRODUCT
1296         } else {
1297           st->print("movdl   %s, %s\t# spill",
1298                      Matcher::regName[dst_first],
1299                      Matcher::regName[src_first]);
1300 #endif
1301         }
1302       }
1303       return 0;
1304     } else if (dst_first_rc == rc_float) {
1305       // xmm -> xmm
1306       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1307           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1308         // 64-bit
1309         if (cbuf) {
1310           MacroAssembler _masm(cbuf);
1311           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1312 #ifndef PRODUCT
1313         } else {
1314           st->print("%s  %s, %s\t# spill",
1315                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1316                      Matcher::regName[dst_first],
1317                      Matcher::regName[src_first]);
1318 #endif
1319         }
1320       } else {
1321         // 32-bit
1322         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1323         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1324         if (cbuf) {
1325           MacroAssembler _masm(cbuf);
1326           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1327 #ifndef PRODUCT
1328         } else {
1329           st->print("%s  %s, %s\t# spill",
1330                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1331                      Matcher::regName[dst_first],
1332                      Matcher::regName[src_first]);
1333 #endif
1334         }
1335       }
1336       return 0;
1337     }
1338   }
1339 
1340   assert(0," foo ");
1341   Unimplemented();
1342   return 0;
1343 }
1344 
1345 #ifndef PRODUCT
1346 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1347   implementation(NULL, ra_, false, st);
1348 }
1349 #endif
1350 
1351 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1352   implementation(&cbuf, ra_, false, NULL);
1353 }
1354 
1355 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1356   return MachNode::size(ra_);
1357 }
1358 
1359 //=============================================================================
1360 #ifndef PRODUCT
1361 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1362 {
1363   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1364   int reg = ra_->get_reg_first(this);
1365   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1366             Matcher::regName[reg], offset);
1367 }
1368 #endif
1369 
1370 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1371 {
1372   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1373   int reg = ra_->get_encode(this);
1374   if (offset >= 0x80) {
1375     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1376     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1377     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1378     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1379     emit_d32(cbuf, offset);
1380   } else {
1381     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1382     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1383     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1384     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1385     emit_d8(cbuf, offset);
1386   }
1387 }
1388 
1389 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1390 {
1391   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1392   return (offset < 0x80) ? 5 : 8; // REX
1393 }
1394 
1395 //=============================================================================
1396 #ifndef PRODUCT
1397 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1398 {
1399   if (UseCompressedClassPointers) {
1400     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1401     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1402     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1403   } else {
1404     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1405                  "# Inline cache check");
1406   }
1407   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1408   st->print_cr("\tnop\t# nops to align entry point");
1409 }
1410 #endif
1411 
1412 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1413 {
1414   MacroAssembler masm(&cbuf);
1415   uint insts_size = cbuf.insts_size();
1416   if (UseCompressedClassPointers) {
1417     masm.load_klass(rscratch1, j_rarg0);
1418     masm.cmpptr(rax, rscratch1);
1419   } else {
1420     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1421   }
1422 
1423   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1424 
1425   /* WARNING these NOPs are critical so that verified entry point is properly
1426      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1427   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1428   if (OptoBreakpoint) {
1429     // Leave space for int3
1430     nops_cnt -= 1;
1431   }
1432   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1433   if (nops_cnt > 0)
1434     masm.nop(nops_cnt);
1435 }
1436 
1437 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1438 {
1439   return MachNode::size(ra_); // too many variables; just compute it
1440                               // the hard way
1441 }
1442 
1443 
1444 //=============================================================================
1445 uint size_exception_handler()
1446 {
1447   // NativeCall instruction size is the same as NativeJump.
1448   // Note that this value is also credited (in output.cpp) to
1449   // the size of the code section.
1450   return NativeJump::instruction_size;
1451 }
1452 
1453 // Emit exception handler code.
1454 int emit_exception_handler(CodeBuffer& cbuf)
1455 {
1456 
1457   // Note that the code buffer's insts_mark is always relative to insts.
1458   // That's why we must use the macroassembler to generate a handler.
1459   MacroAssembler _masm(&cbuf);
1460   address base =
1461   __ start_a_stub(size_exception_handler());
1462   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1463   int offset = __ offset();
1464   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1465   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1466   __ end_a_stub();
1467   return offset;
1468 }
1469 
1470 uint size_deopt_handler()
1471 {
1472   // three 5 byte instructions
1473   return 15;
1474 }
1475 
1476 // Emit deopt handler code.
1477 int emit_deopt_handler(CodeBuffer& cbuf)
1478 {
1479 
1480   // Note that the code buffer's insts_mark is always relative to insts.
1481   // That's why we must use the macroassembler to generate a handler.
1482   MacroAssembler _masm(&cbuf);
1483   address base =
1484   __ start_a_stub(size_deopt_handler());
1485   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1486   int offset = __ offset();
1487   address the_pc = (address) __ pc();
1488   Label next;
1489   // push a "the_pc" on the stack without destroying any registers
1490   // as they all may be live.
1491 
1492   // push address of "next"
1493   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1494   __ bind(next);
1495   // adjust it so it matches "the_pc"
1496   __ subptr(Address(rsp, 0), __ offset() - offset);
1497   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1498   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1499   __ end_a_stub();
1500   return offset;
1501 }
1502 
1503 int Matcher::regnum_to_fpu_offset(int regnum)
1504 {
1505   return regnum - 32; // The FP registers are in the second chunk
1506 }
1507 
1508 // This is UltraSparc specific, true just means we have fast l2f conversion
1509 const bool Matcher::convL2FSupported(void) {
1510   return true;
1511 }
1512 
1513 // Is this branch offset short enough that a short branch can be used?
1514 //
1515 // NOTE: If the platform does not provide any short branch variants, then
1516 //       this method should return false for offset 0.
1517 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1518   // The passed offset is relative to address of the branch.
1519   // On 86 a branch displacement is calculated relative to address
1520   // of a next instruction.
1521   offset -= br_size;
1522 
1523   // the short version of jmpConUCF2 contains multiple branches,
1524   // making the reach slightly less
1525   if (rule == jmpConUCF2_rule)
1526     return (-126 <= offset && offset <= 125);
1527   return (-128 <= offset && offset <= 127);
1528 }
1529 
1530 const bool Matcher::isSimpleConstant64(jlong value) {
1531   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1532   //return value == (int) value;  // Cf. storeImmL and immL32.
1533 
1534   // Probably always true, even if a temp register is required.
1535   return true;
1536 }
1537 
1538 // The ecx parameter to rep stosq for the ClearArray node is in words.
1539 const bool Matcher::init_array_count_is_in_bytes = false;
1540 
1541 // Threshold size for cleararray.
1542 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1543 
1544 // No additional cost for CMOVL.
1545 const int Matcher::long_cmove_cost() { return 0; }
1546 
1547 // No CMOVF/CMOVD with SSE2
1548 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1549 
1550 // Does the CPU require late expand (see block.cpp for description of late expand)?
1551 const bool Matcher::require_postalloc_expand = false;
1552 
1553 // Should the Matcher clone shifts on addressing modes, expecting them
1554 // to be subsumed into complex addressing expressions or compute them
1555 // into registers?  True for Intel but false for most RISCs
1556 const bool Matcher::clone_shift_expressions = true;
1557 
1558 // Do we need to mask the count passed to shift instructions or does
1559 // the cpu only look at the lower 5/6 bits anyway?
1560 const bool Matcher::need_masked_shift_count = false;
1561 
1562 bool Matcher::narrow_oop_use_complex_address() {
1563   assert(UseCompressedOops, "only for compressed oops code");
1564   return (LogMinObjAlignmentInBytes <= 3);
1565 }
1566 
1567 bool Matcher::narrow_klass_use_complex_address() {
1568   assert(UseCompressedClassPointers, "only for compressed klass code");
1569   return (LogKlassAlignmentInBytes <= 3);
1570 }
1571 
1572 // Is it better to copy float constants, or load them directly from
1573 // memory?  Intel can load a float constant from a direct address,
1574 // requiring no extra registers.  Most RISCs will have to materialize
1575 // an address into a register first, so they would do better to copy
1576 // the constant from stack.
1577 const bool Matcher::rematerialize_float_constants = true; // XXX
1578 
1579 // If CPU can load and store mis-aligned doubles directly then no
1580 // fixup is needed.  Else we split the double into 2 integer pieces
1581 // and move it piece-by-piece.  Only happens when passing doubles into
1582 // C code as the Java calling convention forces doubles to be aligned.
1583 const bool Matcher::misaligned_doubles_ok = true;
1584 
1585 // No-op on amd64
1586 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1587 
1588 // Advertise here if the CPU requires explicit rounding operations to
1589 // implement the UseStrictFP mode.
1590 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1591 
1592 // Are floats conerted to double when stored to stack during deoptimization?
1593 // On x64 it is stored without convertion so we can use normal access.
1594 bool Matcher::float_in_double() { return false; }
1595 
1596 // Do ints take an entire long register or just half?
1597 const bool Matcher::int_in_long = true;
1598 
1599 // Return whether or not this register is ever used as an argument.
1600 // This function is used on startup to build the trampoline stubs in
1601 // generateOptoStub.  Registers not mentioned will be killed by the VM
1602 // call in the trampoline, and arguments in those registers not be
1603 // available to the callee.
1604 bool Matcher::can_be_java_arg(int reg)
1605 {
1606   return
1607     reg ==  RDI_num || reg == RDI_H_num ||
1608     reg ==  RSI_num || reg == RSI_H_num ||
1609     reg ==  RDX_num || reg == RDX_H_num ||
1610     reg ==  RCX_num || reg == RCX_H_num ||
1611     reg ==   R8_num || reg ==  R8_H_num ||
1612     reg ==   R9_num || reg ==  R9_H_num ||
1613     reg ==  R12_num || reg == R12_H_num ||
1614     reg == XMM0_num || reg == XMM0b_num ||
1615     reg == XMM1_num || reg == XMM1b_num ||
1616     reg == XMM2_num || reg == XMM2b_num ||
1617     reg == XMM3_num || reg == XMM3b_num ||
1618     reg == XMM4_num || reg == XMM4b_num ||
1619     reg == XMM5_num || reg == XMM5b_num ||
1620     reg == XMM6_num || reg == XMM6b_num ||
1621     reg == XMM7_num || reg == XMM7b_num;
1622 }
1623 
1624 bool Matcher::is_spillable_arg(int reg)
1625 {
1626   return can_be_java_arg(reg);
1627 }
1628 
1629 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1630   // In 64 bit mode a code which use multiply when
1631   // devisor is constant is faster than hardware
1632   // DIV instruction (it uses MulHiL).
1633   return false;
1634 }
1635 
1636 // Register for DIVI projection of divmodI
1637 RegMask Matcher::divI_proj_mask() {
1638   return INT_RAX_REG_mask();
1639 }
1640 
1641 // Register for MODI projection of divmodI
1642 RegMask Matcher::modI_proj_mask() {
1643   return INT_RDX_REG_mask();
1644 }
1645 
1646 // Register for DIVL projection of divmodL
1647 RegMask Matcher::divL_proj_mask() {
1648   return LONG_RAX_REG_mask();
1649 }
1650 
1651 // Register for MODL projection of divmodL
1652 RegMask Matcher::modL_proj_mask() {
1653   return LONG_RDX_REG_mask();
1654 }
1655 
1656 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1657   return PTR_RBP_REG_mask();
1658 }
1659 
1660 const RegMask Matcher::mathExactI_result_proj_mask() {
1661   return INT_RAX_REG_mask();
1662 }
1663 
1664 const RegMask Matcher::mathExactL_result_proj_mask() {
1665   return LONG_RAX_REG_mask();
1666 }
1667 
1668 const RegMask Matcher::mathExactI_flags_proj_mask() {
1669   return INT_FLAGS_mask();
1670 }
1671 
1672 %}
1673 
1674 //----------ENCODING BLOCK-----------------------------------------------------
1675 // This block specifies the encoding classes used by the compiler to
1676 // output byte streams.  Encoding classes are parameterized macros
1677 // used by Machine Instruction Nodes in order to generate the bit
1678 // encoding of the instruction.  Operands specify their base encoding
1679 // interface with the interface keyword.  There are currently
1680 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1681 // COND_INTER.  REG_INTER causes an operand to generate a function
1682 // which returns its register number when queried.  CONST_INTER causes
1683 // an operand to generate a function which returns the value of the
1684 // constant when queried.  MEMORY_INTER causes an operand to generate
1685 // four functions which return the Base Register, the Index Register,
1686 // the Scale Value, and the Offset Value of the operand when queried.
1687 // COND_INTER causes an operand to generate six functions which return
1688 // the encoding code (ie - encoding bits for the instruction)
1689 // associated with each basic boolean condition for a conditional
1690 // instruction.
1691 //
1692 // Instructions specify two basic values for encoding.  Again, a
1693 // function is available to check if the constant displacement is an
1694 // oop. They use the ins_encode keyword to specify their encoding
1695 // classes (which must be a sequence of enc_class names, and their
1696 // parameters, specified in the encoding block), and they use the
1697 // opcode keyword to specify, in order, their primary, secondary, and
1698 // tertiary opcode.  Only the opcode sections which a particular
1699 // instruction needs for encoding need to be specified.
1700 encode %{
1701   // Build emit functions for each basic byte or larger field in the
1702   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1703   // from C++ code in the enc_class source block.  Emit functions will
1704   // live in the main source block for now.  In future, we can
1705   // generalize this by adding a syntax that specifies the sizes of
1706   // fields in an order, so that the adlc can build the emit functions
1707   // automagically
1708 
1709   // Emit primary opcode
1710   enc_class OpcP
1711   %{
1712     emit_opcode(cbuf, $primary);
1713   %}
1714 
1715   // Emit secondary opcode
1716   enc_class OpcS
1717   %{
1718     emit_opcode(cbuf, $secondary);
1719   %}
1720 
1721   // Emit tertiary opcode
1722   enc_class OpcT
1723   %{
1724     emit_opcode(cbuf, $tertiary);
1725   %}
1726 
1727   // Emit opcode directly
1728   enc_class Opcode(immI d8)
1729   %{
1730     emit_opcode(cbuf, $d8$$constant);
1731   %}
1732 
1733   // Emit size prefix
1734   enc_class SizePrefix
1735   %{
1736     emit_opcode(cbuf, 0x66);
1737   %}
1738 
1739   enc_class reg(rRegI reg)
1740   %{
1741     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1742   %}
1743 
1744   enc_class reg_reg(rRegI dst, rRegI src)
1745   %{
1746     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1747   %}
1748 
1749   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1750   %{
1751     emit_opcode(cbuf, $opcode$$constant);
1752     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1753   %}
1754 
1755   enc_class cdql_enc(no_rax_rdx_RegI div)
1756   %{
1757     // Full implementation of Java idiv and irem; checks for
1758     // special case as described in JVM spec., p.243 & p.271.
1759     //
1760     //         normal case                           special case
1761     //
1762     // input : rax: dividend                         min_int
1763     //         reg: divisor                          -1
1764     //
1765     // output: rax: quotient  (= rax idiv reg)       min_int
1766     //         rdx: remainder (= rax irem reg)       0
1767     //
1768     //  Code sequnce:
1769     //
1770     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1771     //    5:   75 07/08                jne    e <normal>
1772     //    7:   33 d2                   xor    %edx,%edx
1773     //  [div >= 8 -> offset + 1]
1774     //  [REX_B]
1775     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1776     //    c:   74 03/04                je     11 <done>
1777     // 000000000000000e <normal>:
1778     //    e:   99                      cltd
1779     //  [div >= 8 -> offset + 1]
1780     //  [REX_B]
1781     //    f:   f7 f9                   idiv   $div
1782     // 0000000000000011 <done>:
1783 
1784     // cmp    $0x80000000,%eax
1785     emit_opcode(cbuf, 0x3d);
1786     emit_d8(cbuf, 0x00);
1787     emit_d8(cbuf, 0x00);
1788     emit_d8(cbuf, 0x00);
1789     emit_d8(cbuf, 0x80);
1790 
1791     // jne    e <normal>
1792     emit_opcode(cbuf, 0x75);
1793     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1794 
1795     // xor    %edx,%edx
1796     emit_opcode(cbuf, 0x33);
1797     emit_d8(cbuf, 0xD2);
1798 
1799     // cmp    $0xffffffffffffffff,%ecx
1800     if ($div$$reg >= 8) {
1801       emit_opcode(cbuf, Assembler::REX_B);
1802     }
1803     emit_opcode(cbuf, 0x83);
1804     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1805     emit_d8(cbuf, 0xFF);
1806 
1807     // je     11 <done>
1808     emit_opcode(cbuf, 0x74);
1809     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1810 
1811     // <normal>
1812     // cltd
1813     emit_opcode(cbuf, 0x99);
1814 
1815     // idivl (note: must be emitted by the user of this rule)
1816     // <done>
1817   %}
1818 
1819   enc_class cdqq_enc(no_rax_rdx_RegL div)
1820   %{
1821     // Full implementation of Java ldiv and lrem; checks for
1822     // special case as described in JVM spec., p.243 & p.271.
1823     //
1824     //         normal case                           special case
1825     //
1826     // input : rax: dividend                         min_long
1827     //         reg: divisor                          -1
1828     //
1829     // output: rax: quotient  (= rax idiv reg)       min_long
1830     //         rdx: remainder (= rax irem reg)       0
1831     //
1832     //  Code sequnce:
1833     //
1834     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1835     //    7:   00 00 80
1836     //    a:   48 39 d0                cmp    %rdx,%rax
1837     //    d:   75 08                   jne    17 <normal>
1838     //    f:   33 d2                   xor    %edx,%edx
1839     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1840     //   15:   74 05                   je     1c <done>
1841     // 0000000000000017 <normal>:
1842     //   17:   48 99                   cqto
1843     //   19:   48 f7 f9                idiv   $div
1844     // 000000000000001c <done>:
1845 
1846     // mov    $0x8000000000000000,%rdx
1847     emit_opcode(cbuf, Assembler::REX_W);
1848     emit_opcode(cbuf, 0xBA);
1849     emit_d8(cbuf, 0x00);
1850     emit_d8(cbuf, 0x00);
1851     emit_d8(cbuf, 0x00);
1852     emit_d8(cbuf, 0x00);
1853     emit_d8(cbuf, 0x00);
1854     emit_d8(cbuf, 0x00);
1855     emit_d8(cbuf, 0x00);
1856     emit_d8(cbuf, 0x80);
1857 
1858     // cmp    %rdx,%rax
1859     emit_opcode(cbuf, Assembler::REX_W);
1860     emit_opcode(cbuf, 0x39);
1861     emit_d8(cbuf, 0xD0);
1862 
1863     // jne    17 <normal>
1864     emit_opcode(cbuf, 0x75);
1865     emit_d8(cbuf, 0x08);
1866 
1867     // xor    %edx,%edx
1868     emit_opcode(cbuf, 0x33);
1869     emit_d8(cbuf, 0xD2);
1870 
1871     // cmp    $0xffffffffffffffff,$div
1872     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1873     emit_opcode(cbuf, 0x83);
1874     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1875     emit_d8(cbuf, 0xFF);
1876 
1877     // je     1e <done>
1878     emit_opcode(cbuf, 0x74);
1879     emit_d8(cbuf, 0x05);
1880 
1881     // <normal>
1882     // cqto
1883     emit_opcode(cbuf, Assembler::REX_W);
1884     emit_opcode(cbuf, 0x99);
1885 
1886     // idivq (note: must be emitted by the user of this rule)
1887     // <done>
1888   %}
1889 
1890   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1891   enc_class OpcSE(immI imm)
1892   %{
1893     // Emit primary opcode and set sign-extend bit
1894     // Check for 8-bit immediate, and set sign extend bit in opcode
1895     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1896       emit_opcode(cbuf, $primary | 0x02);
1897     } else {
1898       // 32-bit immediate
1899       emit_opcode(cbuf, $primary);
1900     }
1901   %}
1902 
1903   enc_class OpcSErm(rRegI dst, immI imm)
1904   %{
1905     // OpcSEr/m
1906     int dstenc = $dst$$reg;
1907     if (dstenc >= 8) {
1908       emit_opcode(cbuf, Assembler::REX_B);
1909       dstenc -= 8;
1910     }
1911     // Emit primary opcode and set sign-extend bit
1912     // Check for 8-bit immediate, and set sign extend bit in opcode
1913     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1914       emit_opcode(cbuf, $primary | 0x02);
1915     } else {
1916       // 32-bit immediate
1917       emit_opcode(cbuf, $primary);
1918     }
1919     // Emit r/m byte with secondary opcode, after primary opcode.
1920     emit_rm(cbuf, 0x3, $secondary, dstenc);
1921   %}
1922 
1923   enc_class OpcSErm_wide(rRegL dst, immI imm)
1924   %{
1925     // OpcSEr/m
1926     int dstenc = $dst$$reg;
1927     if (dstenc < 8) {
1928       emit_opcode(cbuf, Assembler::REX_W);
1929     } else {
1930       emit_opcode(cbuf, Assembler::REX_WB);
1931       dstenc -= 8;
1932     }
1933     // Emit primary opcode and set sign-extend bit
1934     // Check for 8-bit immediate, and set sign extend bit in opcode
1935     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1936       emit_opcode(cbuf, $primary | 0x02);
1937     } else {
1938       // 32-bit immediate
1939       emit_opcode(cbuf, $primary);
1940     }
1941     // Emit r/m byte with secondary opcode, after primary opcode.
1942     emit_rm(cbuf, 0x3, $secondary, dstenc);
1943   %}
1944 
1945   enc_class Con8or32(immI imm)
1946   %{
1947     // Check for 8-bit immediate, and set sign extend bit in opcode
1948     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1949       $$$emit8$imm$$constant;
1950     } else {
1951       // 32-bit immediate
1952       $$$emit32$imm$$constant;
1953     }
1954   %}
1955 
1956   enc_class opc2_reg(rRegI dst)
1957   %{
1958     // BSWAP
1959     emit_cc(cbuf, $secondary, $dst$$reg);
1960   %}
1961 
1962   enc_class opc3_reg(rRegI dst)
1963   %{
1964     // BSWAP
1965     emit_cc(cbuf, $tertiary, $dst$$reg);
1966   %}
1967 
1968   enc_class reg_opc(rRegI div)
1969   %{
1970     // INC, DEC, IDIV, IMOD, JMP indirect, ...
1971     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
1972   %}
1973 
1974   enc_class enc_cmov(cmpOp cop)
1975   %{
1976     // CMOV
1977     $$$emit8$primary;
1978     emit_cc(cbuf, $secondary, $cop$$cmpcode);
1979   %}
1980 
1981   enc_class enc_PartialSubtypeCheck()
1982   %{
1983     Register Rrdi = as_Register(RDI_enc); // result register
1984     Register Rrax = as_Register(RAX_enc); // super class
1985     Register Rrcx = as_Register(RCX_enc); // killed
1986     Register Rrsi = as_Register(RSI_enc); // sub class
1987     Label miss;
1988     const bool set_cond_codes = true;
1989 
1990     MacroAssembler _masm(&cbuf);
1991     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
1992                                      NULL, &miss,
1993                                      /*set_cond_codes:*/ true);
1994     if ($primary) {
1995       __ xorptr(Rrdi, Rrdi);
1996     }
1997     __ bind(miss);
1998   %}
1999 
2000   enc_class clear_avx %{
2001     debug_only(int off0 = cbuf.insts_size());
2002     if (ra_->C->max_vector_size() > 16) {
2003       // Clear upper bits of YMM registers when current compiled code uses
2004       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2005       MacroAssembler _masm(&cbuf);
2006       __ vzeroupper();
2007     }
2008     debug_only(int off1 = cbuf.insts_size());
2009     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2010   %}
2011 
2012   enc_class Java_To_Runtime(method meth) %{
2013     // No relocation needed
2014     MacroAssembler _masm(&cbuf);
2015     __ mov64(r10, (int64_t) $meth$$method);
2016     __ call(r10);
2017   %}
2018 
2019   enc_class Java_To_Interpreter(method meth)
2020   %{
2021     // CALL Java_To_Interpreter
2022     // This is the instruction starting address for relocation info.
2023     cbuf.set_insts_mark();
2024     $$$emit8$primary;
2025     // CALL directly to the runtime
2026     emit_d32_reloc(cbuf,
2027                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2028                    runtime_call_Relocation::spec(),
2029                    RELOC_DISP32);
2030   %}
2031 
2032   enc_class Java_Static_Call(method meth)
2033   %{
2034     // JAVA STATIC CALL
2035     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2036     // determine who we intended to call.
2037     cbuf.set_insts_mark();
2038     $$$emit8$primary;
2039 
2040     if (!_method) {
2041       emit_d32_reloc(cbuf,
2042                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2043                      runtime_call_Relocation::spec(),
2044                      RELOC_DISP32);
2045     } else if (_optimized_virtual) {
2046       emit_d32_reloc(cbuf,
2047                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2048                      opt_virtual_call_Relocation::spec(),
2049                      RELOC_DISP32);
2050     } else {
2051       emit_d32_reloc(cbuf,
2052                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2053                      static_call_Relocation::spec(),
2054                      RELOC_DISP32);
2055     }
2056     if (_method) {
2057       // Emit stub for static call.
2058       CompiledStaticCall::emit_to_interp_stub(cbuf);
2059     }
2060   %}
2061 
2062   enc_class Java_Dynamic_Call(method meth) %{
2063     MacroAssembler _masm(&cbuf);
2064     __ ic_call((address)$meth$$method);
2065   %}
2066 
2067   enc_class Java_Compiled_Call(method meth)
2068   %{
2069     // JAVA COMPILED CALL
2070     int disp = in_bytes(Method:: from_compiled_offset());
2071 
2072     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2073     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2074 
2075     // callq *disp(%rax)
2076     cbuf.set_insts_mark();
2077     $$$emit8$primary;
2078     if (disp < 0x80) {
2079       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2080       emit_d8(cbuf, disp); // Displacement
2081     } else {
2082       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2083       emit_d32(cbuf, disp); // Displacement
2084     }
2085   %}
2086 
2087   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2088   %{
2089     // SAL, SAR, SHR
2090     int dstenc = $dst$$reg;
2091     if (dstenc >= 8) {
2092       emit_opcode(cbuf, Assembler::REX_B);
2093       dstenc -= 8;
2094     }
2095     $$$emit8$primary;
2096     emit_rm(cbuf, 0x3, $secondary, dstenc);
2097     $$$emit8$shift$$constant;
2098   %}
2099 
2100   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2101   %{
2102     // SAL, SAR, SHR
2103     int dstenc = $dst$$reg;
2104     if (dstenc < 8) {
2105       emit_opcode(cbuf, Assembler::REX_W);
2106     } else {
2107       emit_opcode(cbuf, Assembler::REX_WB);
2108       dstenc -= 8;
2109     }
2110     $$$emit8$primary;
2111     emit_rm(cbuf, 0x3, $secondary, dstenc);
2112     $$$emit8$shift$$constant;
2113   %}
2114 
2115   enc_class load_immI(rRegI dst, immI src)
2116   %{
2117     int dstenc = $dst$$reg;
2118     if (dstenc >= 8) {
2119       emit_opcode(cbuf, Assembler::REX_B);
2120       dstenc -= 8;
2121     }
2122     emit_opcode(cbuf, 0xB8 | dstenc);
2123     $$$emit32$src$$constant;
2124   %}
2125 
2126   enc_class load_immL(rRegL dst, immL src)
2127   %{
2128     int dstenc = $dst$$reg;
2129     if (dstenc < 8) {
2130       emit_opcode(cbuf, Assembler::REX_W);
2131     } else {
2132       emit_opcode(cbuf, Assembler::REX_WB);
2133       dstenc -= 8;
2134     }
2135     emit_opcode(cbuf, 0xB8 | dstenc);
2136     emit_d64(cbuf, $src$$constant);
2137   %}
2138 
2139   enc_class load_immUL32(rRegL dst, immUL32 src)
2140   %{
2141     // same as load_immI, but this time we care about zeroes in the high word
2142     int dstenc = $dst$$reg;
2143     if (dstenc >= 8) {
2144       emit_opcode(cbuf, Assembler::REX_B);
2145       dstenc -= 8;
2146     }
2147     emit_opcode(cbuf, 0xB8 | dstenc);
2148     $$$emit32$src$$constant;
2149   %}
2150 
2151   enc_class load_immL32(rRegL dst, immL32 src)
2152   %{
2153     int dstenc = $dst$$reg;
2154     if (dstenc < 8) {
2155       emit_opcode(cbuf, Assembler::REX_W);
2156     } else {
2157       emit_opcode(cbuf, Assembler::REX_WB);
2158       dstenc -= 8;
2159     }
2160     emit_opcode(cbuf, 0xC7);
2161     emit_rm(cbuf, 0x03, 0x00, dstenc);
2162     $$$emit32$src$$constant;
2163   %}
2164 
2165   enc_class load_immP31(rRegP dst, immP32 src)
2166   %{
2167     // same as load_immI, but this time we care about zeroes in the high word
2168     int dstenc = $dst$$reg;
2169     if (dstenc >= 8) {
2170       emit_opcode(cbuf, Assembler::REX_B);
2171       dstenc -= 8;
2172     }
2173     emit_opcode(cbuf, 0xB8 | dstenc);
2174     $$$emit32$src$$constant;
2175   %}
2176 
2177   enc_class load_immP(rRegP dst, immP src)
2178   %{
2179     int dstenc = $dst$$reg;
2180     if (dstenc < 8) {
2181       emit_opcode(cbuf, Assembler::REX_W);
2182     } else {
2183       emit_opcode(cbuf, Assembler::REX_WB);
2184       dstenc -= 8;
2185     }
2186     emit_opcode(cbuf, 0xB8 | dstenc);
2187     // This next line should be generated from ADLC
2188     if ($src->constant_reloc() != relocInfo::none) {
2189       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2190     } else {
2191       emit_d64(cbuf, $src$$constant);
2192     }
2193   %}
2194 
2195   enc_class Con32(immI src)
2196   %{
2197     // Output immediate
2198     $$$emit32$src$$constant;
2199   %}
2200 
2201   enc_class Con32F_as_bits(immF src)
2202   %{
2203     // Output Float immediate bits
2204     jfloat jf = $src$$constant;
2205     jint jf_as_bits = jint_cast(jf);
2206     emit_d32(cbuf, jf_as_bits);
2207   %}
2208 
2209   enc_class Con16(immI src)
2210   %{
2211     // Output immediate
2212     $$$emit16$src$$constant;
2213   %}
2214 
2215   // How is this different from Con32??? XXX
2216   enc_class Con_d32(immI src)
2217   %{
2218     emit_d32(cbuf,$src$$constant);
2219   %}
2220 
2221   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2222     // Output immediate memory reference
2223     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2224     emit_d32(cbuf, 0x00);
2225   %}
2226 
2227   enc_class lock_prefix()
2228   %{
2229     if (os::is_MP()) {
2230       emit_opcode(cbuf, 0xF0); // lock
2231     }
2232   %}
2233 
2234   enc_class REX_mem(memory mem)
2235   %{
2236     if ($mem$$base >= 8) {
2237       if ($mem$$index < 8) {
2238         emit_opcode(cbuf, Assembler::REX_B);
2239       } else {
2240         emit_opcode(cbuf, Assembler::REX_XB);
2241       }
2242     } else {
2243       if ($mem$$index >= 8) {
2244         emit_opcode(cbuf, Assembler::REX_X);
2245       }
2246     }
2247   %}
2248 
2249   enc_class REX_mem_wide(memory mem)
2250   %{
2251     if ($mem$$base >= 8) {
2252       if ($mem$$index < 8) {
2253         emit_opcode(cbuf, Assembler::REX_WB);
2254       } else {
2255         emit_opcode(cbuf, Assembler::REX_WXB);
2256       }
2257     } else {
2258       if ($mem$$index < 8) {
2259         emit_opcode(cbuf, Assembler::REX_W);
2260       } else {
2261         emit_opcode(cbuf, Assembler::REX_WX);
2262       }
2263     }
2264   %}
2265 
2266   // for byte regs
2267   enc_class REX_breg(rRegI reg)
2268   %{
2269     if ($reg$$reg >= 4) {
2270       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2271     }
2272   %}
2273 
2274   // for byte regs
2275   enc_class REX_reg_breg(rRegI dst, rRegI src)
2276   %{
2277     if ($dst$$reg < 8) {
2278       if ($src$$reg >= 4) {
2279         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2280       }
2281     } else {
2282       if ($src$$reg < 8) {
2283         emit_opcode(cbuf, Assembler::REX_R);
2284       } else {
2285         emit_opcode(cbuf, Assembler::REX_RB);
2286       }
2287     }
2288   %}
2289 
2290   // for byte regs
2291   enc_class REX_breg_mem(rRegI reg, memory mem)
2292   %{
2293     if ($reg$$reg < 8) {
2294       if ($mem$$base < 8) {
2295         if ($mem$$index >= 8) {
2296           emit_opcode(cbuf, Assembler::REX_X);
2297         } else if ($reg$$reg >= 4) {
2298           emit_opcode(cbuf, Assembler::REX);
2299         }
2300       } else {
2301         if ($mem$$index < 8) {
2302           emit_opcode(cbuf, Assembler::REX_B);
2303         } else {
2304           emit_opcode(cbuf, Assembler::REX_XB);
2305         }
2306       }
2307     } else {
2308       if ($mem$$base < 8) {
2309         if ($mem$$index < 8) {
2310           emit_opcode(cbuf, Assembler::REX_R);
2311         } else {
2312           emit_opcode(cbuf, Assembler::REX_RX);
2313         }
2314       } else {
2315         if ($mem$$index < 8) {
2316           emit_opcode(cbuf, Assembler::REX_RB);
2317         } else {
2318           emit_opcode(cbuf, Assembler::REX_RXB);
2319         }
2320       }
2321     }
2322   %}
2323 
2324   enc_class REX_reg(rRegI reg)
2325   %{
2326     if ($reg$$reg >= 8) {
2327       emit_opcode(cbuf, Assembler::REX_B);
2328     }
2329   %}
2330 
2331   enc_class REX_reg_wide(rRegI reg)
2332   %{
2333     if ($reg$$reg < 8) {
2334       emit_opcode(cbuf, Assembler::REX_W);
2335     } else {
2336       emit_opcode(cbuf, Assembler::REX_WB);
2337     }
2338   %}
2339 
2340   enc_class REX_reg_reg(rRegI dst, rRegI src)
2341   %{
2342     if ($dst$$reg < 8) {
2343       if ($src$$reg >= 8) {
2344         emit_opcode(cbuf, Assembler::REX_B);
2345       }
2346     } else {
2347       if ($src$$reg < 8) {
2348         emit_opcode(cbuf, Assembler::REX_R);
2349       } else {
2350         emit_opcode(cbuf, Assembler::REX_RB);
2351       }
2352     }
2353   %}
2354 
2355   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2356   %{
2357     if ($dst$$reg < 8) {
2358       if ($src$$reg < 8) {
2359         emit_opcode(cbuf, Assembler::REX_W);
2360       } else {
2361         emit_opcode(cbuf, Assembler::REX_WB);
2362       }
2363     } else {
2364       if ($src$$reg < 8) {
2365         emit_opcode(cbuf, Assembler::REX_WR);
2366       } else {
2367         emit_opcode(cbuf, Assembler::REX_WRB);
2368       }
2369     }
2370   %}
2371 
2372   enc_class REX_reg_mem(rRegI reg, memory mem)
2373   %{
2374     if ($reg$$reg < 8) {
2375       if ($mem$$base < 8) {
2376         if ($mem$$index >= 8) {
2377           emit_opcode(cbuf, Assembler::REX_X);
2378         }
2379       } else {
2380         if ($mem$$index < 8) {
2381           emit_opcode(cbuf, Assembler::REX_B);
2382         } else {
2383           emit_opcode(cbuf, Assembler::REX_XB);
2384         }
2385       }
2386     } else {
2387       if ($mem$$base < 8) {
2388         if ($mem$$index < 8) {
2389           emit_opcode(cbuf, Assembler::REX_R);
2390         } else {
2391           emit_opcode(cbuf, Assembler::REX_RX);
2392         }
2393       } else {
2394         if ($mem$$index < 8) {
2395           emit_opcode(cbuf, Assembler::REX_RB);
2396         } else {
2397           emit_opcode(cbuf, Assembler::REX_RXB);
2398         }
2399       }
2400     }
2401   %}
2402 
2403   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2404   %{
2405     if ($reg$$reg < 8) {
2406       if ($mem$$base < 8) {
2407         if ($mem$$index < 8) {
2408           emit_opcode(cbuf, Assembler::REX_W);
2409         } else {
2410           emit_opcode(cbuf, Assembler::REX_WX);
2411         }
2412       } else {
2413         if ($mem$$index < 8) {
2414           emit_opcode(cbuf, Assembler::REX_WB);
2415         } else {
2416           emit_opcode(cbuf, Assembler::REX_WXB);
2417         }
2418       }
2419     } else {
2420       if ($mem$$base < 8) {
2421         if ($mem$$index < 8) {
2422           emit_opcode(cbuf, Assembler::REX_WR);
2423         } else {
2424           emit_opcode(cbuf, Assembler::REX_WRX);
2425         }
2426       } else {
2427         if ($mem$$index < 8) {
2428           emit_opcode(cbuf, Assembler::REX_WRB);
2429         } else {
2430           emit_opcode(cbuf, Assembler::REX_WRXB);
2431         }
2432       }
2433     }
2434   %}
2435 
2436   enc_class reg_mem(rRegI ereg, memory mem)
2437   %{
2438     // High registers handle in encode_RegMem
2439     int reg = $ereg$$reg;
2440     int base = $mem$$base;
2441     int index = $mem$$index;
2442     int scale = $mem$$scale;
2443     int disp = $mem$$disp;
2444     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2445 
2446     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2447   %}
2448 
2449   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2450   %{
2451     int rm_byte_opcode = $rm_opcode$$constant;
2452 
2453     // High registers handle in encode_RegMem
2454     int base = $mem$$base;
2455     int index = $mem$$index;
2456     int scale = $mem$$scale;
2457     int displace = $mem$$disp;
2458 
2459     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2460                                             // working with static
2461                                             // globals
2462     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2463                   disp_reloc);
2464   %}
2465 
2466   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2467   %{
2468     int reg_encoding = $dst$$reg;
2469     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2470     int index        = 0x04;            // 0x04 indicates no index
2471     int scale        = 0x00;            // 0x00 indicates no scale
2472     int displace     = $src1$$constant; // 0x00 indicates no displacement
2473     relocInfo::relocType disp_reloc = relocInfo::none;
2474     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2475                   disp_reloc);
2476   %}
2477 
2478   enc_class neg_reg(rRegI dst)
2479   %{
2480     int dstenc = $dst$$reg;
2481     if (dstenc >= 8) {
2482       emit_opcode(cbuf, Assembler::REX_B);
2483       dstenc -= 8;
2484     }
2485     // NEG $dst
2486     emit_opcode(cbuf, 0xF7);
2487     emit_rm(cbuf, 0x3, 0x03, dstenc);
2488   %}
2489 
2490   enc_class neg_reg_wide(rRegI dst)
2491   %{
2492     int dstenc = $dst$$reg;
2493     if (dstenc < 8) {
2494       emit_opcode(cbuf, Assembler::REX_W);
2495     } else {
2496       emit_opcode(cbuf, Assembler::REX_WB);
2497       dstenc -= 8;
2498     }
2499     // NEG $dst
2500     emit_opcode(cbuf, 0xF7);
2501     emit_rm(cbuf, 0x3, 0x03, dstenc);
2502   %}
2503 
2504   enc_class setLT_reg(rRegI dst)
2505   %{
2506     int dstenc = $dst$$reg;
2507     if (dstenc >= 8) {
2508       emit_opcode(cbuf, Assembler::REX_B);
2509       dstenc -= 8;
2510     } else if (dstenc >= 4) {
2511       emit_opcode(cbuf, Assembler::REX);
2512     }
2513     // SETLT $dst
2514     emit_opcode(cbuf, 0x0F);
2515     emit_opcode(cbuf, 0x9C);
2516     emit_rm(cbuf, 0x3, 0x0, dstenc);
2517   %}
2518 
2519   enc_class setNZ_reg(rRegI dst)
2520   %{
2521     int dstenc = $dst$$reg;
2522     if (dstenc >= 8) {
2523       emit_opcode(cbuf, Assembler::REX_B);
2524       dstenc -= 8;
2525     } else if (dstenc >= 4) {
2526       emit_opcode(cbuf, Assembler::REX);
2527     }
2528     // SETNZ $dst
2529     emit_opcode(cbuf, 0x0F);
2530     emit_opcode(cbuf, 0x95);
2531     emit_rm(cbuf, 0x3, 0x0, dstenc);
2532   %}
2533 
2534 
2535   // Compare the lonogs and set -1, 0, or 1 into dst
2536   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2537   %{
2538     int src1enc = $src1$$reg;
2539     int src2enc = $src2$$reg;
2540     int dstenc = $dst$$reg;
2541 
2542     // cmpq $src1, $src2
2543     if (src1enc < 8) {
2544       if (src2enc < 8) {
2545         emit_opcode(cbuf, Assembler::REX_W);
2546       } else {
2547         emit_opcode(cbuf, Assembler::REX_WB);
2548       }
2549     } else {
2550       if (src2enc < 8) {
2551         emit_opcode(cbuf, Assembler::REX_WR);
2552       } else {
2553         emit_opcode(cbuf, Assembler::REX_WRB);
2554       }
2555     }
2556     emit_opcode(cbuf, 0x3B);
2557     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2558 
2559     // movl $dst, -1
2560     if (dstenc >= 8) {
2561       emit_opcode(cbuf, Assembler::REX_B);
2562     }
2563     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2564     emit_d32(cbuf, -1);
2565 
2566     // jl,s done
2567     emit_opcode(cbuf, 0x7C);
2568     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2569 
2570     // setne $dst
2571     if (dstenc >= 4) {
2572       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2573     }
2574     emit_opcode(cbuf, 0x0F);
2575     emit_opcode(cbuf, 0x95);
2576     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2577 
2578     // movzbl $dst, $dst
2579     if (dstenc >= 4) {
2580       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2581     }
2582     emit_opcode(cbuf, 0x0F);
2583     emit_opcode(cbuf, 0xB6);
2584     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2585   %}
2586 
2587   enc_class Push_ResultXD(regD dst) %{
2588     MacroAssembler _masm(&cbuf);
2589     __ fstp_d(Address(rsp, 0));
2590     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2591     __ addptr(rsp, 8);
2592   %}
2593 
2594   enc_class Push_SrcXD(regD src) %{
2595     MacroAssembler _masm(&cbuf);
2596     __ subptr(rsp, 8);
2597     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2598     __ fld_d(Address(rsp, 0));
2599   %}
2600 
2601 
2602   // obj: object to lock
2603   // box: box address (header location) -- killed
2604   // tmp: rax -- killed
2605   // scr: rbx -- killed
2606   //
2607   // What follows is a direct transliteration of fast_lock() and fast_unlock()
2608   // from i486.ad.  See that file for comments.
2609   // TODO: where possible switch from movq (r, 0) to movl(r,0) and
2610   // use the shorter encoding.  (Movl clears the high-order 32-bits).
2611 
2612 
2613   enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
2614   %{
2615     Register objReg = as_Register((int)$obj$$reg);
2616     Register boxReg = as_Register((int)$box$$reg);
2617     Register tmpReg = as_Register($tmp$$reg);
2618     Register scrReg = as_Register($scr$$reg);
2619     MacroAssembler masm(&cbuf);
2620 
2621     // Verify uniqueness of register assignments -- necessary but not sufficient
2622     assert (objReg != boxReg && objReg != tmpReg &&
2623             objReg != scrReg && tmpReg != scrReg, "invariant") ;
2624 
2625     if (_counters != NULL) {
2626       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
2627     }
2628     if (EmitSync & 1) {
2629         // Without cast to int32_t a movptr will destroy r10 which is typically obj
2630         masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2631         masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
2632     } else
2633     if (EmitSync & 2) {
2634         Label DONE_LABEL;
2635         if (UseBiasedLocking) {
2636            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
2637           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
2638         }
2639         // QQQ was movl...
2640         masm.movptr(tmpReg, 0x1);
2641         masm.orptr(tmpReg, Address(objReg, 0));
2642         masm.movptr(Address(boxReg, 0), tmpReg);
2643         if (os::is_MP()) {
2644           masm.lock();
2645         }
2646         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2647         masm.jcc(Assembler::equal, DONE_LABEL);
2648 
2649         // Recursive locking
2650         masm.subptr(tmpReg, rsp);
2651         masm.andptr(tmpReg, 7 - os::vm_page_size());
2652         masm.movptr(Address(boxReg, 0), tmpReg);
2653 
2654         masm.bind(DONE_LABEL);
2655         masm.nop(); // avoid branch to branch
2656     } else {
2657         Label DONE_LABEL, IsInflated, Egress;
2658 
2659         masm.movptr(tmpReg, Address(objReg, 0)) ;
2660         masm.testl (tmpReg, 0x02) ;         // inflated vs stack-locked|neutral|biased
2661         masm.jcc   (Assembler::notZero, IsInflated) ;
2662 
2663         // it's stack-locked, biased or neutral
2664         // TODO: optimize markword triage order to reduce the number of
2665         // conditional branches in the most common cases.
2666         // Beware -- there's a subtle invariant that fetch of the markword
2667         // at [FETCH], below, will never observe a biased encoding (*101b).
2668         // If this invariant is not held we'll suffer exclusion (safety) failure.
2669 
2670         if (UseBiasedLocking && !UseOptoBiasInlining) {
2671           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
2672           masm.movptr(tmpReg, Address(objReg, 0)) ;        // [FETCH]
2673         }
2674 
2675         // was q will it destroy high?
2676         masm.orl   (tmpReg, 1) ;
2677         masm.movptr(Address(boxReg, 0), tmpReg) ;
2678         if (os::is_MP()) { masm.lock(); }
2679         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2680         if (_counters != NULL) {
2681            masm.cond_inc32(Assembler::equal,
2682                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2683         }
2684         masm.jcc   (Assembler::equal, DONE_LABEL);
2685 
2686         // Recursive locking
2687         masm.subptr(tmpReg, rsp);
2688         masm.andptr(tmpReg, 7 - os::vm_page_size());
2689         masm.movptr(Address(boxReg, 0), tmpReg);
2690         if (_counters != NULL) {
2691            masm.cond_inc32(Assembler::equal,
2692                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2693         }
2694         masm.jmp   (DONE_LABEL) ;
2695 
2696         masm.bind  (IsInflated) ;
2697         // It's inflated
2698 
2699         // TODO: someday avoid the ST-before-CAS penalty by
2700         // relocating (deferring) the following ST.
2701         // We should also think about trying a CAS without having
2702         // fetched _owner.  If the CAS is successful we may
2703         // avoid an RTO->RTS upgrade on the $line.
2704         // Without cast to int32_t a movptr will destroy r10 which is typically obj
2705         masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2706 
2707         masm.mov    (boxReg, tmpReg) ;
2708         masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2709         masm.testptr(tmpReg, tmpReg) ;
2710         masm.jcc    (Assembler::notZero, DONE_LABEL) ;
2711 
2712         // It's inflated and appears unlocked
2713         if (os::is_MP()) { masm.lock(); }
2714         masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2715         // Intentional fall-through into DONE_LABEL ...
2716 
2717         masm.bind  (DONE_LABEL) ;
2718         masm.nop   () ;                 // avoid jmp to jmp
2719     }
2720   %}
2721 
2722   // obj: object to unlock
2723   // box: box address (displaced header location), killed
2724   // RBX: killed tmp; cannot be obj nor box
2725   enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
2726   %{
2727 
2728     Register objReg = as_Register($obj$$reg);
2729     Register boxReg = as_Register($box$$reg);
2730     Register tmpReg = as_Register($tmp$$reg);
2731     MacroAssembler masm(&cbuf);
2732 
2733     if (EmitSync & 4) {
2734        masm.cmpptr(rsp, 0) ;
2735     } else
2736     if (EmitSync & 8) {
2737        Label DONE_LABEL;
2738        if (UseBiasedLocking) {
2739          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2740        }
2741 
2742        // Check whether the displaced header is 0
2743        //(=> recursive unlock)
2744        masm.movptr(tmpReg, Address(boxReg, 0));
2745        masm.testptr(tmpReg, tmpReg);
2746        masm.jcc(Assembler::zero, DONE_LABEL);
2747 
2748        // If not recursive lock, reset the header to displaced header
2749        if (os::is_MP()) {
2750          masm.lock();
2751        }
2752        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2753        masm.bind(DONE_LABEL);
2754        masm.nop(); // avoid branch to branch
2755     } else {
2756        Label DONE_LABEL, Stacked, CheckSucc ;
2757 
2758        if (UseBiasedLocking && !UseOptoBiasInlining) {
2759          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2760        }
2761 
2762        masm.movptr(tmpReg, Address(objReg, 0)) ;
2763        masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
2764        masm.jcc   (Assembler::zero, DONE_LABEL) ;
2765        masm.testl (tmpReg, 0x02) ;
2766        masm.jcc   (Assembler::zero, Stacked) ;
2767 
2768        // It's inflated
2769        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2770        masm.xorptr(boxReg, r15_thread) ;
2771        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
2772        masm.jcc   (Assembler::notZero, DONE_LABEL) ;
2773        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
2774        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
2775        masm.jcc   (Assembler::notZero, CheckSucc) ;
2776        masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2777        masm.jmp   (DONE_LABEL) ;
2778 
2779        if ((EmitSync & 65536) == 0) {
2780          Label LSuccess, LGoSlowPath ;
2781          masm.bind  (CheckSucc) ;
2782          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2783          masm.jcc   (Assembler::zero, LGoSlowPath) ;
2784 
2785          // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
2786          // the explicit ST;MEMBAR combination, but masm doesn't currently support
2787          // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc
2788          // are all faster when the write buffer is populated.
2789          masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2790          if (os::is_MP()) {
2791             masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
2792          }
2793          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2794          masm.jcc   (Assembler::notZero, LSuccess) ;
2795 
2796          masm.movptr (boxReg, (int32_t)NULL_WORD) ;                   // box is really EAX
2797          if (os::is_MP()) { masm.lock(); }
2798          masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
2799          masm.jcc   (Assembler::notEqual, LSuccess) ;
2800          // Intentional fall-through into slow-path
2801 
2802          masm.bind  (LGoSlowPath) ;
2803          masm.orl   (boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
2804          masm.jmp   (DONE_LABEL) ;
2805 
2806          masm.bind  (LSuccess) ;
2807          masm.testl (boxReg, 0) ;                      // set ICC.ZF=1 to indicate success
2808          masm.jmp   (DONE_LABEL) ;
2809        }
2810 
2811        masm.bind  (Stacked) ;
2812        masm.movptr(tmpReg, Address (boxReg, 0)) ;      // re-fetch
2813        if (os::is_MP()) { masm.lock(); }
2814        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2815 
2816        if (EmitSync & 65536) {
2817           masm.bind (CheckSucc) ;
2818        }
2819        masm.bind(DONE_LABEL);
2820        if (EmitSync & 32768) {
2821           masm.nop();                      // avoid branch to branch
2822        }
2823     }
2824   %}
2825 
2826 
2827   enc_class enc_rethrow()
2828   %{
2829     cbuf.set_insts_mark();
2830     emit_opcode(cbuf, 0xE9); // jmp entry
2831     emit_d32_reloc(cbuf,
2832                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2833                    runtime_call_Relocation::spec(),
2834                    RELOC_DISP32);
2835   %}
2836 
2837 %}
2838 
2839 
2840 
2841 //----------FRAME--------------------------------------------------------------
2842 // Definition of frame structure and management information.
2843 //
2844 //  S T A C K   L A Y O U T    Allocators stack-slot number
2845 //                             |   (to get allocators register number
2846 //  G  Owned by    |        |  v    add OptoReg::stack0())
2847 //  r   CALLER     |        |
2848 //  o     |        +--------+      pad to even-align allocators stack-slot
2849 //  w     V        |  pad0  |        numbers; owned by CALLER
2850 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2851 //  h     ^        |   in   |  5
2852 //        |        |  args  |  4   Holes in incoming args owned by SELF
2853 //  |     |        |        |  3
2854 //  |     |        +--------+
2855 //  V     |        | old out|      Empty on Intel, window on Sparc
2856 //        |    old |preserve|      Must be even aligned.
2857 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2858 //        |        |   in   |  3   area for Intel ret address
2859 //     Owned by    |preserve|      Empty on Sparc.
2860 //       SELF      +--------+
2861 //        |        |  pad2  |  2   pad to align old SP
2862 //        |        +--------+  1
2863 //        |        | locks  |  0
2864 //        |        +--------+----> OptoReg::stack0(), even aligned
2865 //        |        |  pad1  | 11   pad to align new SP
2866 //        |        +--------+
2867 //        |        |        | 10
2868 //        |        | spills |  9   spills
2869 //        V        |        |  8   (pad0 slot for callee)
2870 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2871 //        ^        |  out   |  7
2872 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2873 //     Owned by    +--------+
2874 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2875 //        |    new |preserve|      Must be even-aligned.
2876 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2877 //        |        |        |
2878 //
2879 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2880 //         known from SELF's arguments and the Java calling convention.
2881 //         Region 6-7 is determined per call site.
2882 // Note 2: If the calling convention leaves holes in the incoming argument
2883 //         area, those holes are owned by SELF.  Holes in the outgoing area
2884 //         are owned by the CALLEE.  Holes should not be nessecary in the
2885 //         incoming area, as the Java calling convention is completely under
2886 //         the control of the AD file.  Doubles can be sorted and packed to
2887 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2888 //         varargs C calling conventions.
2889 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2890 //         even aligned with pad0 as needed.
2891 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2892 //         region 6-11 is even aligned; it may be padded out more so that
2893 //         the region from SP to FP meets the minimum stack alignment.
2894 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2895 //         alignment.  Region 11, pad1, may be dynamically extended so that
2896 //         SP meets the minimum alignment.
2897 
2898 frame
2899 %{
2900   // What direction does stack grow in (assumed to be same for C & Java)
2901   stack_direction(TOWARDS_LOW);
2902 
2903   // These three registers define part of the calling convention
2904   // between compiled code and the interpreter.
2905   inline_cache_reg(RAX);                // Inline Cache Register
2906   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2907                                         // calling interpreter
2908 
2909   // Optional: name the operand used by cisc-spilling to access
2910   // [stack_pointer + offset]
2911   cisc_spilling_operand_name(indOffset32);
2912 
2913   // Number of stack slots consumed by locking an object
2914   sync_stack_slots(2);
2915 
2916   // Compiled code's Frame Pointer
2917   frame_pointer(RSP);
2918 
2919   // Interpreter stores its frame pointer in a register which is
2920   // stored to the stack by I2CAdaptors.
2921   // I2CAdaptors convert from interpreted java to compiled java.
2922   interpreter_frame_pointer(RBP);
2923 
2924   // Stack alignment requirement
2925   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2926 
2927   // Number of stack slots between incoming argument block and the start of
2928   // a new frame.  The PROLOG must add this many slots to the stack.  The
2929   // EPILOG must remove this many slots.  amd64 needs two slots for
2930   // return address.
2931   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2932 
2933   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2934   // for calls to C.  Supports the var-args backing area for register parms.
2935   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2936 
2937   // The after-PROLOG location of the return address.  Location of
2938   // return address specifies a type (REG or STACK) and a number
2939   // representing the register number (i.e. - use a register name) or
2940   // stack slot.
2941   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2942   // Otherwise, it is above the locks and verification slot and alignment word
2943   return_addr(STACK - 2 +
2944               round_to((Compile::current()->in_preserve_stack_slots() +
2945                         Compile::current()->fixed_slots()),
2946                        stack_alignment_in_slots()));
2947 
2948   // Body of function which returns an integer array locating
2949   // arguments either in registers or in stack slots.  Passed an array
2950   // of ideal registers called "sig" and a "length" count.  Stack-slot
2951   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2952   // arguments for a CALLEE.  Incoming stack arguments are
2953   // automatically biased by the preserve_stack_slots field above.
2954 
2955   calling_convention
2956   %{
2957     // No difference between ingoing/outgoing just pass false
2958     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2959   %}
2960 
2961   c_calling_convention
2962   %{
2963     // This is obviously always outgoing
2964     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2965   %}
2966 
2967   // Location of compiled Java return values.  Same as C for now.
2968   return_value
2969   %{
2970     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2971            "only return normal values");
2972 
2973     static const int lo[Op_RegL + 1] = {
2974       0,
2975       0,
2976       RAX_num,  // Op_RegN
2977       RAX_num,  // Op_RegI
2978       RAX_num,  // Op_RegP
2979       XMM0_num, // Op_RegF
2980       XMM0_num, // Op_RegD
2981       RAX_num   // Op_RegL
2982     };
2983     static const int hi[Op_RegL + 1] = {
2984       0,
2985       0,
2986       OptoReg::Bad, // Op_RegN
2987       OptoReg::Bad, // Op_RegI
2988       RAX_H_num,    // Op_RegP
2989       OptoReg::Bad, // Op_RegF
2990       XMM0b_num,    // Op_RegD
2991       RAX_H_num     // Op_RegL
2992     };
2993     // Excluded flags and vector registers.
2994     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 5, "missing type");
2995     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2996   %}
2997 %}
2998 
2999 //----------ATTRIBUTES---------------------------------------------------------
3000 //----------Operand Attributes-------------------------------------------------
3001 op_attrib op_cost(0);        // Required cost attribute
3002 
3003 //----------Instruction Attributes---------------------------------------------
3004 ins_attrib ins_cost(100);       // Required cost attribute
3005 ins_attrib ins_size(8);         // Required size attribute (in bits)
3006 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3007                                 // a non-matching short branch variant
3008                                 // of some long branch?
3009 ins_attrib ins_alignment(1);    // Required alignment attribute (must
3010                                 // be a power of 2) specifies the
3011                                 // alignment that some part of the
3012                                 // instruction (not necessarily the
3013                                 // start) requires.  If > 1, a
3014                                 // compute_padding() function must be
3015                                 // provided for the instruction
3016 
3017 //----------OPERANDS-----------------------------------------------------------
3018 // Operand definitions must precede instruction definitions for correct parsing
3019 // in the ADLC because operands constitute user defined types which are used in
3020 // instruction definitions.
3021 
3022 //----------Simple Operands----------------------------------------------------
3023 // Immediate Operands
3024 // Integer Immediate
3025 operand immI()
3026 %{
3027   match(ConI);
3028 
3029   op_cost(10);
3030   format %{ %}
3031   interface(CONST_INTER);
3032 %}
3033 
3034 // Constant for test vs zero
3035 operand immI0()
3036 %{
3037   predicate(n->get_int() == 0);
3038   match(ConI);
3039 
3040   op_cost(0);
3041   format %{ %}
3042   interface(CONST_INTER);
3043 %}
3044 
3045 // Constant for increment
3046 operand immI1()
3047 %{
3048   predicate(n->get_int() == 1);
3049   match(ConI);
3050 
3051   op_cost(0);
3052   format %{ %}
3053   interface(CONST_INTER);
3054 %}
3055 
3056 // Constant for decrement
3057 operand immI_M1()
3058 %{
3059   predicate(n->get_int() == -1);
3060   match(ConI);
3061 
3062   op_cost(0);
3063   format %{ %}
3064   interface(CONST_INTER);
3065 %}
3066 
3067 // Valid scale values for addressing modes
3068 operand immI2()
3069 %{
3070   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3071   match(ConI);
3072 
3073   format %{ %}
3074   interface(CONST_INTER);
3075 %}
3076 
3077 operand immI8()
3078 %{
3079   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
3080   match(ConI);
3081 
3082   op_cost(5);
3083   format %{ %}
3084   interface(CONST_INTER);
3085 %}
3086 
3087 operand immI16()
3088 %{
3089   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3090   match(ConI);
3091 
3092   op_cost(10);
3093   format %{ %}
3094   interface(CONST_INTER);
3095 %}
3096 
3097 // Int Immediate non-negative
3098 operand immU31()
3099 %{
3100   predicate(n->get_int() >= 0);
3101   match(ConI);
3102 
3103   op_cost(0);
3104   format %{ %}
3105   interface(CONST_INTER);
3106 %}
3107 
3108 // Constant for long shifts
3109 operand immI_32()
3110 %{
3111   predicate( n->get_int() == 32 );
3112   match(ConI);
3113 
3114   op_cost(0);
3115   format %{ %}
3116   interface(CONST_INTER);
3117 %}
3118 
3119 // Constant for long shifts
3120 operand immI_64()
3121 %{
3122   predicate( n->get_int() == 64 );
3123   match(ConI);
3124 
3125   op_cost(0);
3126   format %{ %}
3127   interface(CONST_INTER);
3128 %}
3129 
3130 // Pointer Immediate
3131 operand immP()
3132 %{
3133   match(ConP);
3134 
3135   op_cost(10);
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // NULL Pointer Immediate
3141 operand immP0()
3142 %{
3143   predicate(n->get_ptr() == 0);
3144   match(ConP);
3145 
3146   op_cost(5);
3147   format %{ %}
3148   interface(CONST_INTER);
3149 %}
3150 
3151 // Pointer Immediate
3152 operand immN() %{
3153   match(ConN);
3154 
3155   op_cost(10);
3156   format %{ %}
3157   interface(CONST_INTER);
3158 %}
3159 
3160 operand immNKlass() %{
3161   match(ConNKlass);
3162 
3163   op_cost(10);
3164   format %{ %}
3165   interface(CONST_INTER);
3166 %}
3167 
3168 // NULL Pointer Immediate
3169 operand immN0() %{
3170   predicate(n->get_narrowcon() == 0);
3171   match(ConN);
3172 
3173   op_cost(5);
3174   format %{ %}
3175   interface(CONST_INTER);
3176 %}
3177 
3178 operand immP31()
3179 %{
3180   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3181             && (n->get_ptr() >> 31) == 0);
3182   match(ConP);
3183 
3184   op_cost(5);
3185   format %{ %}
3186   interface(CONST_INTER);
3187 %}
3188 
3189 
3190 // Long Immediate
3191 operand immL()
3192 %{
3193   match(ConL);
3194 
3195   op_cost(20);
3196   format %{ %}
3197   interface(CONST_INTER);
3198 %}
3199 
3200 // Long Immediate 8-bit
3201 operand immL8()
3202 %{
3203   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3204   match(ConL);
3205 
3206   op_cost(5);
3207   format %{ %}
3208   interface(CONST_INTER);
3209 %}
3210 
3211 // Long Immediate 32-bit unsigned
3212 operand immUL32()
3213 %{
3214   predicate(n->get_long() == (unsigned int) (n->get_long()));
3215   match(ConL);
3216 
3217   op_cost(10);
3218   format %{ %}
3219   interface(CONST_INTER);
3220 %}
3221 
3222 // Long Immediate 32-bit signed
3223 operand immL32()
3224 %{
3225   predicate(n->get_long() == (int) (n->get_long()));
3226   match(ConL);
3227 
3228   op_cost(15);
3229   format %{ %}
3230   interface(CONST_INTER);
3231 %}
3232 
3233 // Long Immediate zero
3234 operand immL0()
3235 %{
3236   predicate(n->get_long() == 0L);
3237   match(ConL);
3238 
3239   op_cost(10);
3240   format %{ %}
3241   interface(CONST_INTER);
3242 %}
3243 
3244 // Constant for increment
3245 operand immL1()
3246 %{
3247   predicate(n->get_long() == 1);
3248   match(ConL);
3249 
3250   format %{ %}
3251   interface(CONST_INTER);
3252 %}
3253 
3254 // Constant for decrement
3255 operand immL_M1()
3256 %{
3257   predicate(n->get_long() == -1);
3258   match(ConL);
3259 
3260   format %{ %}
3261   interface(CONST_INTER);
3262 %}
3263 
3264 // Long Immediate: the value 10
3265 operand immL10()
3266 %{
3267   predicate(n->get_long() == 10);
3268   match(ConL);
3269 
3270   format %{ %}
3271   interface(CONST_INTER);
3272 %}
3273 
3274 // Long immediate from 0 to 127.
3275 // Used for a shorter form of long mul by 10.
3276 operand immL_127()
3277 %{
3278   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3279   match(ConL);
3280 
3281   op_cost(10);
3282   format %{ %}
3283   interface(CONST_INTER);
3284 %}
3285 
3286 // Long Immediate: low 32-bit mask
3287 operand immL_32bits()
3288 %{
3289   predicate(n->get_long() == 0xFFFFFFFFL);
3290   match(ConL);
3291   op_cost(20);
3292 
3293   format %{ %}
3294   interface(CONST_INTER);
3295 %}
3296 
3297 // Float Immediate zero
3298 operand immF0()
3299 %{
3300   predicate(jint_cast(n->getf()) == 0);
3301   match(ConF);
3302 
3303   op_cost(5);
3304   format %{ %}
3305   interface(CONST_INTER);
3306 %}
3307 
3308 // Float Immediate
3309 operand immF()
3310 %{
3311   match(ConF);
3312 
3313   op_cost(15);
3314   format %{ %}
3315   interface(CONST_INTER);
3316 %}
3317 
3318 // Double Immediate zero
3319 operand immD0()
3320 %{
3321   predicate(jlong_cast(n->getd()) == 0);
3322   match(ConD);
3323 
3324   op_cost(5);
3325   format %{ %}
3326   interface(CONST_INTER);
3327 %}
3328 
3329 // Double Immediate
3330 operand immD()
3331 %{
3332   match(ConD);
3333 
3334   op_cost(15);
3335   format %{ %}
3336   interface(CONST_INTER);
3337 %}
3338 
3339 // Immediates for special shifts (sign extend)
3340 
3341 // Constants for increment
3342 operand immI_16()
3343 %{
3344   predicate(n->get_int() == 16);
3345   match(ConI);
3346 
3347   format %{ %}
3348   interface(CONST_INTER);
3349 %}
3350 
3351 operand immI_24()
3352 %{
3353   predicate(n->get_int() == 24);
3354   match(ConI);
3355 
3356   format %{ %}
3357   interface(CONST_INTER);
3358 %}
3359 
3360 // Constant for byte-wide masking
3361 operand immI_255()
3362 %{
3363   predicate(n->get_int() == 255);
3364   match(ConI);
3365 
3366   format %{ %}
3367   interface(CONST_INTER);
3368 %}
3369 
3370 // Constant for short-wide masking
3371 operand immI_65535()
3372 %{
3373   predicate(n->get_int() == 65535);
3374   match(ConI);
3375 
3376   format %{ %}
3377   interface(CONST_INTER);
3378 %}
3379 
3380 // Constant for byte-wide masking
3381 operand immL_255()
3382 %{
3383   predicate(n->get_long() == 255);
3384   match(ConL);
3385 
3386   format %{ %}
3387   interface(CONST_INTER);
3388 %}
3389 
3390 // Constant for short-wide masking
3391 operand immL_65535()
3392 %{
3393   predicate(n->get_long() == 65535);
3394   match(ConL);
3395 
3396   format %{ %}
3397   interface(CONST_INTER);
3398 %}
3399 
3400 // Register Operands
3401 // Integer Register
3402 operand rRegI()
3403 %{
3404   constraint(ALLOC_IN_RC(int_reg));
3405   match(RegI);
3406 
3407   match(rax_RegI);
3408   match(rbx_RegI);
3409   match(rcx_RegI);
3410   match(rdx_RegI);
3411   match(rdi_RegI);
3412 
3413   format %{ %}
3414   interface(REG_INTER);
3415 %}
3416 
3417 // Special Registers
3418 operand rax_RegI()
3419 %{
3420   constraint(ALLOC_IN_RC(int_rax_reg));
3421   match(RegI);
3422   match(rRegI);
3423 
3424   format %{ "RAX" %}
3425   interface(REG_INTER);
3426 %}
3427 
3428 // Special Registers
3429 operand rbx_RegI()
3430 %{
3431   constraint(ALLOC_IN_RC(int_rbx_reg));
3432   match(RegI);
3433   match(rRegI);
3434 
3435   format %{ "RBX" %}
3436   interface(REG_INTER);
3437 %}
3438 
3439 operand rcx_RegI()
3440 %{
3441   constraint(ALLOC_IN_RC(int_rcx_reg));
3442   match(RegI);
3443   match(rRegI);
3444 
3445   format %{ "RCX" %}
3446   interface(REG_INTER);
3447 %}
3448 
3449 operand rdx_RegI()
3450 %{
3451   constraint(ALLOC_IN_RC(int_rdx_reg));
3452   match(RegI);
3453   match(rRegI);
3454 
3455   format %{ "RDX" %}
3456   interface(REG_INTER);
3457 %}
3458 
3459 operand rdi_RegI()
3460 %{
3461   constraint(ALLOC_IN_RC(int_rdi_reg));
3462   match(RegI);
3463   match(rRegI);
3464 
3465   format %{ "RDI" %}
3466   interface(REG_INTER);
3467 %}
3468 
3469 operand no_rcx_RegI()
3470 %{
3471   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3472   match(RegI);
3473   match(rax_RegI);
3474   match(rbx_RegI);
3475   match(rdx_RegI);
3476   match(rdi_RegI);
3477 
3478   format %{ %}
3479   interface(REG_INTER);
3480 %}
3481 
3482 operand no_rax_rdx_RegI()
3483 %{
3484   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3485   match(RegI);
3486   match(rbx_RegI);
3487   match(rcx_RegI);
3488   match(rdi_RegI);
3489 
3490   format %{ %}
3491   interface(REG_INTER);
3492 %}
3493 
3494 // Pointer Register
3495 operand any_RegP()
3496 %{
3497   constraint(ALLOC_IN_RC(any_reg));
3498   match(RegP);
3499   match(rax_RegP);
3500   match(rbx_RegP);
3501   match(rdi_RegP);
3502   match(rsi_RegP);
3503   match(rbp_RegP);
3504   match(r15_RegP);
3505   match(rRegP);
3506 
3507   format %{ %}
3508   interface(REG_INTER);
3509 %}
3510 
3511 operand rRegP()
3512 %{
3513   constraint(ALLOC_IN_RC(ptr_reg));
3514   match(RegP);
3515   match(rax_RegP);
3516   match(rbx_RegP);
3517   match(rdi_RegP);
3518   match(rsi_RegP);
3519   match(rbp_RegP);
3520   match(r15_RegP);  // See Q&A below about r15_RegP.
3521 
3522   format %{ %}
3523   interface(REG_INTER);
3524 %}
3525 
3526 operand rRegN() %{
3527   constraint(ALLOC_IN_RC(int_reg));
3528   match(RegN);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3535 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3536 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
3537 // The output of an instruction is controlled by the allocator, which respects
3538 // register class masks, not match rules.  Unless an instruction mentions
3539 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3540 // by the allocator as an input.
3541 
3542 operand no_rax_RegP()
3543 %{
3544   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3545   match(RegP);
3546   match(rbx_RegP);
3547   match(rsi_RegP);
3548   match(rdi_RegP);
3549 
3550   format %{ %}
3551   interface(REG_INTER);
3552 %}
3553 
3554 operand no_rbp_RegP()
3555 %{
3556   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
3557   match(RegP);
3558   match(rbx_RegP);
3559   match(rsi_RegP);
3560   match(rdi_RegP);
3561 
3562   format %{ %}
3563   interface(REG_INTER);
3564 %}
3565 
3566 operand no_rax_rbx_RegP()
3567 %{
3568   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3569   match(RegP);
3570   match(rsi_RegP);
3571   match(rdi_RegP);
3572 
3573   format %{ %}
3574   interface(REG_INTER);
3575 %}
3576 
3577 // Special Registers
3578 // Return a pointer value
3579 operand rax_RegP()
3580 %{
3581   constraint(ALLOC_IN_RC(ptr_rax_reg));
3582   match(RegP);
3583   match(rRegP);
3584 
3585   format %{ %}
3586   interface(REG_INTER);
3587 %}
3588 
3589 // Special Registers
3590 // Return a compressed pointer value
3591 operand rax_RegN()
3592 %{
3593   constraint(ALLOC_IN_RC(int_rax_reg));
3594   match(RegN);
3595   match(rRegN);
3596 
3597   format %{ %}
3598   interface(REG_INTER);
3599 %}
3600 
3601 // Used in AtomicAdd
3602 operand rbx_RegP()
3603 %{
3604   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3605   match(RegP);
3606   match(rRegP);
3607 
3608   format %{ %}
3609   interface(REG_INTER);
3610 %}
3611 
3612 operand rsi_RegP()
3613 %{
3614   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3615   match(RegP);
3616   match(rRegP);
3617 
3618   format %{ %}
3619   interface(REG_INTER);
3620 %}
3621 
3622 // Used in rep stosq
3623 operand rdi_RegP()
3624 %{
3625   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3626   match(RegP);
3627   match(rRegP);
3628 
3629   format %{ %}
3630   interface(REG_INTER);
3631 %}
3632 
3633 operand rbp_RegP()
3634 %{
3635   constraint(ALLOC_IN_RC(ptr_rbp_reg));
3636   match(RegP);
3637   match(rRegP);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 operand r15_RegP()
3644 %{
3645   constraint(ALLOC_IN_RC(ptr_r15_reg));
3646   match(RegP);
3647   match(rRegP);
3648 
3649   format %{ %}
3650   interface(REG_INTER);
3651 %}
3652 
3653 operand rRegL()
3654 %{
3655   constraint(ALLOC_IN_RC(long_reg));
3656   match(RegL);
3657   match(rax_RegL);
3658   match(rdx_RegL);
3659 
3660   format %{ %}
3661   interface(REG_INTER);
3662 %}
3663 
3664 // Special Registers
3665 operand no_rax_rdx_RegL()
3666 %{
3667   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3668   match(RegL);
3669   match(rRegL);
3670 
3671   format %{ %}
3672   interface(REG_INTER);
3673 %}
3674 
3675 operand no_rax_RegL()
3676 %{
3677   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3678   match(RegL);
3679   match(rRegL);
3680   match(rdx_RegL);
3681 
3682   format %{ %}
3683   interface(REG_INTER);
3684 %}
3685 
3686 operand no_rcx_RegL()
3687 %{
3688   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3689   match(RegL);
3690   match(rRegL);
3691 
3692   format %{ %}
3693   interface(REG_INTER);
3694 %}
3695 
3696 operand rax_RegL()
3697 %{
3698   constraint(ALLOC_IN_RC(long_rax_reg));
3699   match(RegL);
3700   match(rRegL);
3701 
3702   format %{ "RAX" %}
3703   interface(REG_INTER);
3704 %}
3705 
3706 operand rcx_RegL()
3707 %{
3708   constraint(ALLOC_IN_RC(long_rcx_reg));
3709   match(RegL);
3710   match(rRegL);
3711 
3712   format %{ %}
3713   interface(REG_INTER);
3714 %}
3715 
3716 operand rdx_RegL()
3717 %{
3718   constraint(ALLOC_IN_RC(long_rdx_reg));
3719   match(RegL);
3720   match(rRegL);
3721 
3722   format %{ %}
3723   interface(REG_INTER);
3724 %}
3725 
3726 // Flags register, used as output of compare instructions
3727 operand rFlagsReg()
3728 %{
3729   constraint(ALLOC_IN_RC(int_flags));
3730   match(RegFlags);
3731 
3732   format %{ "RFLAGS" %}
3733   interface(REG_INTER);
3734 %}
3735 
3736 // Flags register, used as output of FLOATING POINT compare instructions
3737 operand rFlagsRegU()
3738 %{
3739   constraint(ALLOC_IN_RC(int_flags));
3740   match(RegFlags);
3741 
3742   format %{ "RFLAGS_U" %}
3743   interface(REG_INTER);
3744 %}
3745 
3746 operand rFlagsRegUCF() %{
3747   constraint(ALLOC_IN_RC(int_flags));
3748   match(RegFlags);
3749   predicate(false);
3750 
3751   format %{ "RFLAGS_U_CF" %}
3752   interface(REG_INTER);
3753 %}
3754 
3755 // Float register operands
3756 operand regF()
3757 %{
3758   constraint(ALLOC_IN_RC(float_reg));
3759   match(RegF);
3760 
3761   format %{ %}
3762   interface(REG_INTER);
3763 %}
3764 
3765 // Double register operands
3766 operand regD()
3767 %{
3768   constraint(ALLOC_IN_RC(double_reg));
3769   match(RegD);
3770 
3771   format %{ %}
3772   interface(REG_INTER);
3773 %}
3774 
3775 //----------Memory Operands----------------------------------------------------
3776 // Direct Memory Operand
3777 // operand direct(immP addr)
3778 // %{
3779 //   match(addr);
3780 
3781 //   format %{ "[$addr]" %}
3782 //   interface(MEMORY_INTER) %{
3783 //     base(0xFFFFFFFF);
3784 //     index(0x4);
3785 //     scale(0x0);
3786 //     disp($addr);
3787 //   %}
3788 // %}
3789 
3790 // Indirect Memory Operand
3791 operand indirect(any_RegP reg)
3792 %{
3793   constraint(ALLOC_IN_RC(ptr_reg));
3794   match(reg);
3795 
3796   format %{ "[$reg]" %}
3797   interface(MEMORY_INTER) %{
3798     base($reg);
3799     index(0x4);
3800     scale(0x0);
3801     disp(0x0);
3802   %}
3803 %}
3804 
3805 // Indirect Memory Plus Short Offset Operand
3806 operand indOffset8(any_RegP reg, immL8 off)
3807 %{
3808   constraint(ALLOC_IN_RC(ptr_reg));
3809   match(AddP reg off);
3810 
3811   format %{ "[$reg + $off (8-bit)]" %}
3812   interface(MEMORY_INTER) %{
3813     base($reg);
3814     index(0x4);
3815     scale(0x0);
3816     disp($off);
3817   %}
3818 %}
3819 
3820 // Indirect Memory Plus Long Offset Operand
3821 operand indOffset32(any_RegP reg, immL32 off)
3822 %{
3823   constraint(ALLOC_IN_RC(ptr_reg));
3824   match(AddP reg off);
3825 
3826   format %{ "[$reg + $off (32-bit)]" %}
3827   interface(MEMORY_INTER) %{
3828     base($reg);
3829     index(0x4);
3830     scale(0x0);
3831     disp($off);
3832   %}
3833 %}
3834 
3835 // Indirect Memory Plus Index Register Plus Offset Operand
3836 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3837 %{
3838   constraint(ALLOC_IN_RC(ptr_reg));
3839   match(AddP (AddP reg lreg) off);
3840 
3841   op_cost(10);
3842   format %{"[$reg + $off + $lreg]" %}
3843   interface(MEMORY_INTER) %{
3844     base($reg);
3845     index($lreg);
3846     scale(0x0);
3847     disp($off);
3848   %}
3849 %}
3850 
3851 // Indirect Memory Plus Index Register Plus Offset Operand
3852 operand indIndex(any_RegP reg, rRegL lreg)
3853 %{
3854   constraint(ALLOC_IN_RC(ptr_reg));
3855   match(AddP reg lreg);
3856 
3857   op_cost(10);
3858   format %{"[$reg + $lreg]" %}
3859   interface(MEMORY_INTER) %{
3860     base($reg);
3861     index($lreg);
3862     scale(0x0);
3863     disp(0x0);
3864   %}
3865 %}
3866 
3867 // Indirect Memory Times Scale Plus Index Register
3868 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3869 %{
3870   constraint(ALLOC_IN_RC(ptr_reg));
3871   match(AddP reg (LShiftL lreg scale));
3872 
3873   op_cost(10);
3874   format %{"[$reg + $lreg << $scale]" %}
3875   interface(MEMORY_INTER) %{
3876     base($reg);
3877     index($lreg);
3878     scale($scale);
3879     disp(0x0);
3880   %}
3881 %}
3882 
3883 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3884 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3885 %{
3886   constraint(ALLOC_IN_RC(ptr_reg));
3887   match(AddP (AddP reg (LShiftL lreg scale)) off);
3888 
3889   op_cost(10);
3890   format %{"[$reg + $off + $lreg << $scale]" %}
3891   interface(MEMORY_INTER) %{
3892     base($reg);
3893     index($lreg);
3894     scale($scale);
3895     disp($off);
3896   %}
3897 %}
3898 
3899 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3900 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3901 %{
3902   constraint(ALLOC_IN_RC(ptr_reg));
3903   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3904   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3905 
3906   op_cost(10);
3907   format %{"[$reg + $off + $idx << $scale]" %}
3908   interface(MEMORY_INTER) %{
3909     base($reg);
3910     index($idx);
3911     scale($scale);
3912     disp($off);
3913   %}
3914 %}
3915 
3916 // Indirect Narrow Oop Plus Offset Operand
3917 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3918 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3919 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3920   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3921   constraint(ALLOC_IN_RC(ptr_reg));
3922   match(AddP (DecodeN reg) off);
3923 
3924   op_cost(10);
3925   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3926   interface(MEMORY_INTER) %{
3927     base(0xc); // R12
3928     index($reg);
3929     scale(0x3);
3930     disp($off);
3931   %}
3932 %}
3933 
3934 // Indirect Memory Operand
3935 operand indirectNarrow(rRegN reg)
3936 %{
3937   predicate(Universe::narrow_oop_shift() == 0);
3938   constraint(ALLOC_IN_RC(ptr_reg));
3939   match(DecodeN reg);
3940 
3941   format %{ "[$reg]" %}
3942   interface(MEMORY_INTER) %{
3943     base($reg);
3944     index(0x4);
3945     scale(0x0);
3946     disp(0x0);
3947   %}
3948 %}
3949 
3950 // Indirect Memory Plus Short Offset Operand
3951 operand indOffset8Narrow(rRegN reg, immL8 off)
3952 %{
3953   predicate(Universe::narrow_oop_shift() == 0);
3954   constraint(ALLOC_IN_RC(ptr_reg));
3955   match(AddP (DecodeN reg) off);
3956 
3957   format %{ "[$reg + $off (8-bit)]" %}
3958   interface(MEMORY_INTER) %{
3959     base($reg);
3960     index(0x4);
3961     scale(0x0);
3962     disp($off);
3963   %}
3964 %}
3965 
3966 // Indirect Memory Plus Long Offset Operand
3967 operand indOffset32Narrow(rRegN reg, immL32 off)
3968 %{
3969   predicate(Universe::narrow_oop_shift() == 0);
3970   constraint(ALLOC_IN_RC(ptr_reg));
3971   match(AddP (DecodeN reg) off);
3972 
3973   format %{ "[$reg + $off (32-bit)]" %}
3974   interface(MEMORY_INTER) %{
3975     base($reg);
3976     index(0x4);
3977     scale(0x0);
3978     disp($off);
3979   %}
3980 %}
3981 
3982 // Indirect Memory Plus Index Register Plus Offset Operand
3983 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3984 %{
3985   predicate(Universe::narrow_oop_shift() == 0);
3986   constraint(ALLOC_IN_RC(ptr_reg));
3987   match(AddP (AddP (DecodeN reg) lreg) off);
3988 
3989   op_cost(10);
3990   format %{"[$reg + $off + $lreg]" %}
3991   interface(MEMORY_INTER) %{
3992     base($reg);
3993     index($lreg);
3994     scale(0x0);
3995     disp($off);
3996   %}
3997 %}
3998 
3999 // Indirect Memory Plus Index Register Plus Offset Operand
4000 operand indIndexNarrow(rRegN reg, rRegL lreg)
4001 %{
4002   predicate(Universe::narrow_oop_shift() == 0);
4003   constraint(ALLOC_IN_RC(ptr_reg));
4004   match(AddP (DecodeN reg) lreg);
4005 
4006   op_cost(10);
4007   format %{"[$reg + $lreg]" %}
4008   interface(MEMORY_INTER) %{
4009     base($reg);
4010     index($lreg);
4011     scale(0x0);
4012     disp(0x0);
4013   %}
4014 %}
4015 
4016 // Indirect Memory Times Scale Plus Index Register
4017 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4018 %{
4019   predicate(Universe::narrow_oop_shift() == 0);
4020   constraint(ALLOC_IN_RC(ptr_reg));
4021   match(AddP (DecodeN reg) (LShiftL lreg scale));
4022 
4023   op_cost(10);
4024   format %{"[$reg + $lreg << $scale]" %}
4025   interface(MEMORY_INTER) %{
4026     base($reg);
4027     index($lreg);
4028     scale($scale);
4029     disp(0x0);
4030   %}
4031 %}
4032 
4033 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4034 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4035 %{
4036   predicate(Universe::narrow_oop_shift() == 0);
4037   constraint(ALLOC_IN_RC(ptr_reg));
4038   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4039 
4040   op_cost(10);
4041   format %{"[$reg + $off + $lreg << $scale]" %}
4042   interface(MEMORY_INTER) %{
4043     base($reg);
4044     index($lreg);
4045     scale($scale);
4046     disp($off);
4047   %}
4048 %}
4049 
4050 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4051 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4052 %{
4053   constraint(ALLOC_IN_RC(ptr_reg));
4054   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4055   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4056 
4057   op_cost(10);
4058   format %{"[$reg + $off + $idx << $scale]" %}
4059   interface(MEMORY_INTER) %{
4060     base($reg);
4061     index($idx);
4062     scale($scale);
4063     disp($off);
4064   %}
4065 %}
4066 
4067 //----------Special Memory Operands--------------------------------------------
4068 // Stack Slot Operand - This operand is used for loading and storing temporary
4069 //                      values on the stack where a match requires a value to
4070 //                      flow through memory.
4071 operand stackSlotP(sRegP reg)
4072 %{
4073   constraint(ALLOC_IN_RC(stack_slots));
4074   // No match rule because this operand is only generated in matching
4075 
4076   format %{ "[$reg]" %}
4077   interface(MEMORY_INTER) %{
4078     base(0x4);   // RSP
4079     index(0x4);  // No Index
4080     scale(0x0);  // No Scale
4081     disp($reg);  // Stack Offset
4082   %}
4083 %}
4084 
4085 operand stackSlotI(sRegI reg)
4086 %{
4087   constraint(ALLOC_IN_RC(stack_slots));
4088   // No match rule because this operand is only generated in matching
4089 
4090   format %{ "[$reg]" %}
4091   interface(MEMORY_INTER) %{
4092     base(0x4);   // RSP
4093     index(0x4);  // No Index
4094     scale(0x0);  // No Scale
4095     disp($reg);  // Stack Offset
4096   %}
4097 %}
4098 
4099 operand stackSlotF(sRegF reg)
4100 %{
4101   constraint(ALLOC_IN_RC(stack_slots));
4102   // No match rule because this operand is only generated in matching
4103 
4104   format %{ "[$reg]" %}
4105   interface(MEMORY_INTER) %{
4106     base(0x4);   // RSP
4107     index(0x4);  // No Index
4108     scale(0x0);  // No Scale
4109     disp($reg);  // Stack Offset
4110   %}
4111 %}
4112 
4113 operand stackSlotD(sRegD reg)
4114 %{
4115   constraint(ALLOC_IN_RC(stack_slots));
4116   // No match rule because this operand is only generated in matching
4117 
4118   format %{ "[$reg]" %}
4119   interface(MEMORY_INTER) %{
4120     base(0x4);   // RSP
4121     index(0x4);  // No Index
4122     scale(0x0);  // No Scale
4123     disp($reg);  // Stack Offset
4124   %}
4125 %}
4126 operand stackSlotL(sRegL reg)
4127 %{
4128   constraint(ALLOC_IN_RC(stack_slots));
4129   // No match rule because this operand is only generated in matching
4130 
4131   format %{ "[$reg]" %}
4132   interface(MEMORY_INTER) %{
4133     base(0x4);   // RSP
4134     index(0x4);  // No Index
4135     scale(0x0);  // No Scale
4136     disp($reg);  // Stack Offset
4137   %}
4138 %}
4139 
4140 //----------Conditional Branch Operands----------------------------------------
4141 // Comparison Op  - This is the operation of the comparison, and is limited to
4142 //                  the following set of codes:
4143 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4144 //
4145 // Other attributes of the comparison, such as unsignedness, are specified
4146 // by the comparison instruction that sets a condition code flags register.
4147 // That result is represented by a flags operand whose subtype is appropriate
4148 // to the unsignedness (etc.) of the comparison.
4149 //
4150 // Later, the instruction which matches both the Comparison Op (a Bool) and
4151 // the flags (produced by the Cmp) specifies the coding of the comparison op
4152 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4153 
4154 // Comparision Code
4155 operand cmpOp()
4156 %{
4157   match(Bool);
4158 
4159   format %{ "" %}
4160   interface(COND_INTER) %{
4161     equal(0x4, "e");
4162     not_equal(0x5, "ne");
4163     less(0xC, "l");
4164     greater_equal(0xD, "ge");
4165     less_equal(0xE, "le");
4166     greater(0xF, "g");
4167     overflow(0x0, "o");
4168     no_overflow(0x1, "no");
4169   %}
4170 %}
4171 
4172 // Comparison Code, unsigned compare.  Used by FP also, with
4173 // C2 (unordered) turned into GT or LT already.  The other bits
4174 // C0 and C3 are turned into Carry & Zero flags.
4175 operand cmpOpU()
4176 %{
4177   match(Bool);
4178 
4179   format %{ "" %}
4180   interface(COND_INTER) %{
4181     equal(0x4, "e");
4182     not_equal(0x5, "ne");
4183     less(0x2, "b");
4184     greater_equal(0x3, "nb");
4185     less_equal(0x6, "be");
4186     greater(0x7, "nbe");
4187     overflow(0x0, "o");
4188     no_overflow(0x1, "no");
4189   %}
4190 %}
4191 
4192 
4193 // Floating comparisons that don't require any fixup for the unordered case
4194 operand cmpOpUCF() %{
4195   match(Bool);
4196   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4197             n->as_Bool()->_test._test == BoolTest::ge ||
4198             n->as_Bool()->_test._test == BoolTest::le ||
4199             n->as_Bool()->_test._test == BoolTest::gt);
4200   format %{ "" %}
4201   interface(COND_INTER) %{
4202     equal(0x4, "e");
4203     not_equal(0x5, "ne");
4204     less(0x2, "b");
4205     greater_equal(0x3, "nb");
4206     less_equal(0x6, "be");
4207     greater(0x7, "nbe");
4208     overflow(0x0, "o");
4209     no_overflow(0x1, "no");
4210   %}
4211 %}
4212 
4213 
4214 // Floating comparisons that can be fixed up with extra conditional jumps
4215 operand cmpOpUCF2() %{
4216   match(Bool);
4217   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4218             n->as_Bool()->_test._test == BoolTest::eq);
4219   format %{ "" %}
4220   interface(COND_INTER) %{
4221     equal(0x4, "e");
4222     not_equal(0x5, "ne");
4223     less(0x2, "b");
4224     greater_equal(0x3, "nb");
4225     less_equal(0x6, "be");
4226     greater(0x7, "nbe");
4227     overflow(0x0, "o");
4228     no_overflow(0x1, "no");
4229   %}
4230 %}
4231 
4232 
4233 //----------OPERAND CLASSES----------------------------------------------------
4234 // Operand Classes are groups of operands that are used as to simplify
4235 // instruction definitions by not requiring the AD writer to specify separate
4236 // instructions for every form of operand when the instruction accepts
4237 // multiple operand types with the same basic encoding and format.  The classic
4238 // case of this is memory operands.
4239 
4240 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4241                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
4242                indCompressedOopOffset,
4243                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4244                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4245                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
4246 
4247 //----------PIPELINE-----------------------------------------------------------
4248 // Rules which define the behavior of the target architectures pipeline.
4249 pipeline %{
4250 
4251 //----------ATTRIBUTES---------------------------------------------------------
4252 attributes %{
4253   variable_size_instructions;        // Fixed size instructions
4254   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4255   instruction_unit_size = 1;         // An instruction is 1 bytes long
4256   instruction_fetch_unit_size = 16;  // The processor fetches one line
4257   instruction_fetch_units = 1;       // of 16 bytes
4258 
4259   // List of nop instructions
4260   nops( MachNop );
4261 %}
4262 
4263 //----------RESOURCES----------------------------------------------------------
4264 // Resources are the functional units available to the machine
4265 
4266 // Generic P2/P3 pipeline
4267 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4268 // 3 instructions decoded per cycle.
4269 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4270 // 3 ALU op, only ALU0 handles mul instructions.
4271 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4272            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4273            BR, FPU,
4274            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4275 
4276 //----------PIPELINE DESCRIPTION-----------------------------------------------
4277 // Pipeline Description specifies the stages in the machine's pipeline
4278 
4279 // Generic P2/P3 pipeline
4280 pipe_desc(S0, S1, S2, S3, S4, S5);
4281 
4282 //----------PIPELINE CLASSES---------------------------------------------------
4283 // Pipeline Classes describe the stages in which input and output are
4284 // referenced by the hardware pipeline.
4285 
4286 // Naming convention: ialu or fpu
4287 // Then: _reg
4288 // Then: _reg if there is a 2nd register
4289 // Then: _long if it's a pair of instructions implementing a long
4290 // Then: _fat if it requires the big decoder
4291 //   Or: _mem if it requires the big decoder and a memory unit.
4292 
4293 // Integer ALU reg operation
4294 pipe_class ialu_reg(rRegI dst)
4295 %{
4296     single_instruction;
4297     dst    : S4(write);
4298     dst    : S3(read);
4299     DECODE : S0;        // any decoder
4300     ALU    : S3;        // any alu
4301 %}
4302 
4303 // Long ALU reg operation
4304 pipe_class ialu_reg_long(rRegL dst)
4305 %{
4306     instruction_count(2);
4307     dst    : S4(write);
4308     dst    : S3(read);
4309     DECODE : S0(2);     // any 2 decoders
4310     ALU    : S3(2);     // both alus
4311 %}
4312 
4313 // Integer ALU reg operation using big decoder
4314 pipe_class ialu_reg_fat(rRegI dst)
4315 %{
4316     single_instruction;
4317     dst    : S4(write);
4318     dst    : S3(read);
4319     D0     : S0;        // big decoder only
4320     ALU    : S3;        // any alu
4321 %}
4322 
4323 // Long ALU reg operation using big decoder
4324 pipe_class ialu_reg_long_fat(rRegL dst)
4325 %{
4326     instruction_count(2);
4327     dst    : S4(write);
4328     dst    : S3(read);
4329     D0     : S0(2);     // big decoder only; twice
4330     ALU    : S3(2);     // any 2 alus
4331 %}
4332 
4333 // Integer ALU reg-reg operation
4334 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4335 %{
4336     single_instruction;
4337     dst    : S4(write);
4338     src    : S3(read);
4339     DECODE : S0;        // any decoder
4340     ALU    : S3;        // any alu
4341 %}
4342 
4343 // Long ALU reg-reg operation
4344 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4345 %{
4346     instruction_count(2);
4347     dst    : S4(write);
4348     src    : S3(read);
4349     DECODE : S0(2);     // any 2 decoders
4350     ALU    : S3(2);     // both alus
4351 %}
4352 
4353 // Integer ALU reg-reg operation
4354 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4355 %{
4356     single_instruction;
4357     dst    : S4(write);
4358     src    : S3(read);
4359     D0     : S0;        // big decoder only
4360     ALU    : S3;        // any alu
4361 %}
4362 
4363 // Long ALU reg-reg operation
4364 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4365 %{
4366     instruction_count(2);
4367     dst    : S4(write);
4368     src    : S3(read);
4369     D0     : S0(2);     // big decoder only; twice
4370     ALU    : S3(2);     // both alus
4371 %}
4372 
4373 // Integer ALU reg-mem operation
4374 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4375 %{
4376     single_instruction;
4377     dst    : S5(write);
4378     mem    : S3(read);
4379     D0     : S0;        // big decoder only
4380     ALU    : S4;        // any alu
4381     MEM    : S3;        // any mem
4382 %}
4383 
4384 // Integer mem operation (prefetch)
4385 pipe_class ialu_mem(memory mem)
4386 %{
4387     single_instruction;
4388     mem    : S3(read);
4389     D0     : S0;        // big decoder only
4390     MEM    : S3;        // any mem
4391 %}
4392 
4393 // Integer Store to Memory
4394 pipe_class ialu_mem_reg(memory mem, rRegI src)
4395 %{
4396     single_instruction;
4397     mem    : S3(read);
4398     src    : S5(read);
4399     D0     : S0;        // big decoder only
4400     ALU    : S4;        // any alu
4401     MEM    : S3;
4402 %}
4403 
4404 // // Long Store to Memory
4405 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4406 // %{
4407 //     instruction_count(2);
4408 //     mem    : S3(read);
4409 //     src    : S5(read);
4410 //     D0     : S0(2);          // big decoder only; twice
4411 //     ALU    : S4(2);     // any 2 alus
4412 //     MEM    : S3(2);  // Both mems
4413 // %}
4414 
4415 // Integer Store to Memory
4416 pipe_class ialu_mem_imm(memory mem)
4417 %{
4418     single_instruction;
4419     mem    : S3(read);
4420     D0     : S0;        // big decoder only
4421     ALU    : S4;        // any alu
4422     MEM    : S3;
4423 %}
4424 
4425 // Integer ALU0 reg-reg operation
4426 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4427 %{
4428     single_instruction;
4429     dst    : S4(write);
4430     src    : S3(read);
4431     D0     : S0;        // Big decoder only
4432     ALU0   : S3;        // only alu0
4433 %}
4434 
4435 // Integer ALU0 reg-mem operation
4436 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4437 %{
4438     single_instruction;
4439     dst    : S5(write);
4440     mem    : S3(read);
4441     D0     : S0;        // big decoder only
4442     ALU0   : S4;        // ALU0 only
4443     MEM    : S3;        // any mem
4444 %}
4445 
4446 // Integer ALU reg-reg operation
4447 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4448 %{
4449     single_instruction;
4450     cr     : S4(write);
4451     src1   : S3(read);
4452     src2   : S3(read);
4453     DECODE : S0;        // any decoder
4454     ALU    : S3;        // any alu
4455 %}
4456 
4457 // Integer ALU reg-imm operation
4458 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4459 %{
4460     single_instruction;
4461     cr     : S4(write);
4462     src1   : S3(read);
4463     DECODE : S0;        // any decoder
4464     ALU    : S3;        // any alu
4465 %}
4466 
4467 // Integer ALU reg-mem operation
4468 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4469 %{
4470     single_instruction;
4471     cr     : S4(write);
4472     src1   : S3(read);
4473     src2   : S3(read);
4474     D0     : S0;        // big decoder only
4475     ALU    : S4;        // any alu
4476     MEM    : S3;
4477 %}
4478 
4479 // Conditional move reg-reg
4480 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4481 %{
4482     instruction_count(4);
4483     y      : S4(read);
4484     q      : S3(read);
4485     p      : S3(read);
4486     DECODE : S0(4);     // any decoder
4487 %}
4488 
4489 // Conditional move reg-reg
4490 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4491 %{
4492     single_instruction;
4493     dst    : S4(write);
4494     src    : S3(read);
4495     cr     : S3(read);
4496     DECODE : S0;        // any decoder
4497 %}
4498 
4499 // Conditional move reg-mem
4500 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4501 %{
4502     single_instruction;
4503     dst    : S4(write);
4504     src    : S3(read);
4505     cr     : S3(read);
4506     DECODE : S0;        // any decoder
4507     MEM    : S3;
4508 %}
4509 
4510 // Conditional move reg-reg long
4511 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4512 %{
4513     single_instruction;
4514     dst    : S4(write);
4515     src    : S3(read);
4516     cr     : S3(read);
4517     DECODE : S0(2);     // any 2 decoders
4518 %}
4519 
4520 // XXX
4521 // // Conditional move double reg-reg
4522 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4523 // %{
4524 //     single_instruction;
4525 //     dst    : S4(write);
4526 //     src    : S3(read);
4527 //     cr     : S3(read);
4528 //     DECODE : S0;     // any decoder
4529 // %}
4530 
4531 // Float reg-reg operation
4532 pipe_class fpu_reg(regD dst)
4533 %{
4534     instruction_count(2);
4535     dst    : S3(read);
4536     DECODE : S0(2);     // any 2 decoders
4537     FPU    : S3;
4538 %}
4539 
4540 // Float reg-reg operation
4541 pipe_class fpu_reg_reg(regD dst, regD src)
4542 %{
4543     instruction_count(2);
4544     dst    : S4(write);
4545     src    : S3(read);
4546     DECODE : S0(2);     // any 2 decoders
4547     FPU    : S3;
4548 %}
4549 
4550 // Float reg-reg operation
4551 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4552 %{
4553     instruction_count(3);
4554     dst    : S4(write);
4555     src1   : S3(read);
4556     src2   : S3(read);
4557     DECODE : S0(3);     // any 3 decoders
4558     FPU    : S3(2);
4559 %}
4560 
4561 // Float reg-reg operation
4562 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4563 %{
4564     instruction_count(4);
4565     dst    : S4(write);
4566     src1   : S3(read);
4567     src2   : S3(read);
4568     src3   : S3(read);
4569     DECODE : S0(4);     // any 3 decoders
4570     FPU    : S3(2);
4571 %}
4572 
4573 // Float reg-reg operation
4574 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4575 %{
4576     instruction_count(4);
4577     dst    : S4(write);
4578     src1   : S3(read);
4579     src2   : S3(read);
4580     src3   : S3(read);
4581     DECODE : S1(3);     // any 3 decoders
4582     D0     : S0;        // Big decoder only
4583     FPU    : S3(2);
4584     MEM    : S3;
4585 %}
4586 
4587 // Float reg-mem operation
4588 pipe_class fpu_reg_mem(regD dst, memory mem)
4589 %{
4590     instruction_count(2);
4591     dst    : S5(write);
4592     mem    : S3(read);
4593     D0     : S0;        // big decoder only
4594     DECODE : S1;        // any decoder for FPU POP
4595     FPU    : S4;
4596     MEM    : S3;        // any mem
4597 %}
4598 
4599 // Float reg-mem operation
4600 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4601 %{
4602     instruction_count(3);
4603     dst    : S5(write);
4604     src1   : S3(read);
4605     mem    : S3(read);
4606     D0     : S0;        // big decoder only
4607     DECODE : S1(2);     // any decoder for FPU POP
4608     FPU    : S4;
4609     MEM    : S3;        // any mem
4610 %}
4611 
4612 // Float mem-reg operation
4613 pipe_class fpu_mem_reg(memory mem, regD src)
4614 %{
4615     instruction_count(2);
4616     src    : S5(read);
4617     mem    : S3(read);
4618     DECODE : S0;        // any decoder for FPU PUSH
4619     D0     : S1;        // big decoder only
4620     FPU    : S4;
4621     MEM    : S3;        // any mem
4622 %}
4623 
4624 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4625 %{
4626     instruction_count(3);
4627     src1   : S3(read);
4628     src2   : S3(read);
4629     mem    : S3(read);
4630     DECODE : S0(2);     // any decoder for FPU PUSH
4631     D0     : S1;        // big decoder only
4632     FPU    : S4;
4633     MEM    : S3;        // any mem
4634 %}
4635 
4636 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4637 %{
4638     instruction_count(3);
4639     src1   : S3(read);
4640     src2   : S3(read);
4641     mem    : S4(read);
4642     DECODE : S0;        // any decoder for FPU PUSH
4643     D0     : S0(2);     // big decoder only
4644     FPU    : S4;
4645     MEM    : S3(2);     // any mem
4646 %}
4647 
4648 pipe_class fpu_mem_mem(memory dst, memory src1)
4649 %{
4650     instruction_count(2);
4651     src1   : S3(read);
4652     dst    : S4(read);
4653     D0     : S0(2);     // big decoder only
4654     MEM    : S3(2);     // any mem
4655 %}
4656 
4657 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4658 %{
4659     instruction_count(3);
4660     src1   : S3(read);
4661     src2   : S3(read);
4662     dst    : S4(read);
4663     D0     : S0(3);     // big decoder only
4664     FPU    : S4;
4665     MEM    : S3(3);     // any mem
4666 %}
4667 
4668 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4669 %{
4670     instruction_count(3);
4671     src1   : S4(read);
4672     mem    : S4(read);
4673     DECODE : S0;        // any decoder for FPU PUSH
4674     D0     : S0(2);     // big decoder only
4675     FPU    : S4;
4676     MEM    : S3(2);     // any mem
4677 %}
4678 
4679 // Float load constant
4680 pipe_class fpu_reg_con(regD dst)
4681 %{
4682     instruction_count(2);
4683     dst    : S5(write);
4684     D0     : S0;        // big decoder only for the load
4685     DECODE : S1;        // any decoder for FPU POP
4686     FPU    : S4;
4687     MEM    : S3;        // any mem
4688 %}
4689 
4690 // Float load constant
4691 pipe_class fpu_reg_reg_con(regD dst, regD src)
4692 %{
4693     instruction_count(3);
4694     dst    : S5(write);
4695     src    : S3(read);
4696     D0     : S0;        // big decoder only for the load
4697     DECODE : S1(2);     // any decoder for FPU POP
4698     FPU    : S4;
4699     MEM    : S3;        // any mem
4700 %}
4701 
4702 // UnConditional branch
4703 pipe_class pipe_jmp(label labl)
4704 %{
4705     single_instruction;
4706     BR   : S3;
4707 %}
4708 
4709 // Conditional branch
4710 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4711 %{
4712     single_instruction;
4713     cr    : S1(read);
4714     BR    : S3;
4715 %}
4716 
4717 // Allocation idiom
4718 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4719 %{
4720     instruction_count(1); force_serialization;
4721     fixed_latency(6);
4722     heap_ptr : S3(read);
4723     DECODE   : S0(3);
4724     D0       : S2;
4725     MEM      : S3;
4726     ALU      : S3(2);
4727     dst      : S5(write);
4728     BR       : S5;
4729 %}
4730 
4731 // Generic big/slow expanded idiom
4732 pipe_class pipe_slow()
4733 %{
4734     instruction_count(10); multiple_bundles; force_serialization;
4735     fixed_latency(100);
4736     D0  : S0(2);
4737     MEM : S3(2);
4738 %}
4739 
4740 // The real do-nothing guy
4741 pipe_class empty()
4742 %{
4743     instruction_count(0);
4744 %}
4745 
4746 // Define the class for the Nop node
4747 define
4748 %{
4749    MachNop = empty;
4750 %}
4751 
4752 %}
4753 
4754 //----------INSTRUCTIONS-------------------------------------------------------
4755 //
4756 // match      -- States which machine-independent subtree may be replaced
4757 //               by this instruction.
4758 // ins_cost   -- The estimated cost of this instruction is used by instruction
4759 //               selection to identify a minimum cost tree of machine
4760 //               instructions that matches a tree of machine-independent
4761 //               instructions.
4762 // format     -- A string providing the disassembly for this instruction.
4763 //               The value of an instruction's operand may be inserted
4764 //               by referring to it with a '$' prefix.
4765 // opcode     -- Three instruction opcodes may be provided.  These are referred
4766 //               to within an encode class as $primary, $secondary, and $tertiary
4767 //               rrspectively.  The primary opcode is commonly used to
4768 //               indicate the type of machine instruction, while secondary
4769 //               and tertiary are often used for prefix options or addressing
4770 //               modes.
4771 // ins_encode -- A list of encode classes with parameters. The encode class
4772 //               name must have been defined in an 'enc_class' specification
4773 //               in the encode section of the architecture description.
4774 
4775 
4776 //----------Load/Store/Move Instructions---------------------------------------
4777 //----------Load Instructions--------------------------------------------------
4778 
4779 // Load Byte (8 bit signed)
4780 instruct loadB(rRegI dst, memory mem)
4781 %{
4782   match(Set dst (LoadB mem));
4783 
4784   ins_cost(125);
4785   format %{ "movsbl  $dst, $mem\t# byte" %}
4786 
4787   ins_encode %{
4788     __ movsbl($dst$$Register, $mem$$Address);
4789   %}
4790 
4791   ins_pipe(ialu_reg_mem);
4792 %}
4793 
4794 // Load Byte (8 bit signed) into Long Register
4795 instruct loadB2L(rRegL dst, memory mem)
4796 %{
4797   match(Set dst (ConvI2L (LoadB mem)));
4798 
4799   ins_cost(125);
4800   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4801 
4802   ins_encode %{
4803     __ movsbq($dst$$Register, $mem$$Address);
4804   %}
4805 
4806   ins_pipe(ialu_reg_mem);
4807 %}
4808 
4809 // Load Unsigned Byte (8 bit UNsigned)
4810 instruct loadUB(rRegI dst, memory mem)
4811 %{
4812   match(Set dst (LoadUB mem));
4813 
4814   ins_cost(125);
4815   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4816 
4817   ins_encode %{
4818     __ movzbl($dst$$Register, $mem$$Address);
4819   %}
4820 
4821   ins_pipe(ialu_reg_mem);
4822 %}
4823 
4824 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4825 instruct loadUB2L(rRegL dst, memory mem)
4826 %{
4827   match(Set dst (ConvI2L (LoadUB mem)));
4828 
4829   ins_cost(125);
4830   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4831 
4832   ins_encode %{
4833     __ movzbq($dst$$Register, $mem$$Address);
4834   %}
4835 
4836   ins_pipe(ialu_reg_mem);
4837 %}
4838 
4839 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
4840 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
4841   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4842   effect(KILL cr);
4843 
4844   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
4845             "andl    $dst, $mask" %}
4846   ins_encode %{
4847     Register Rdst = $dst$$Register;
4848     __ movzbq(Rdst, $mem$$Address);
4849     __ andl(Rdst, $mask$$constant);
4850   %}
4851   ins_pipe(ialu_reg_mem);
4852 %}
4853 
4854 // Load Short (16 bit signed)
4855 instruct loadS(rRegI dst, memory mem)
4856 %{
4857   match(Set dst (LoadS mem));
4858 
4859   ins_cost(125);
4860   format %{ "movswl $dst, $mem\t# short" %}
4861 
4862   ins_encode %{
4863     __ movswl($dst$$Register, $mem$$Address);
4864   %}
4865 
4866   ins_pipe(ialu_reg_mem);
4867 %}
4868 
4869 // Load Short (16 bit signed) to Byte (8 bit signed)
4870 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4871   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4872 
4873   ins_cost(125);
4874   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4875   ins_encode %{
4876     __ movsbl($dst$$Register, $mem$$Address);
4877   %}
4878   ins_pipe(ialu_reg_mem);
4879 %}
4880 
4881 // Load Short (16 bit signed) into Long Register
4882 instruct loadS2L(rRegL dst, memory mem)
4883 %{
4884   match(Set dst (ConvI2L (LoadS mem)));
4885 
4886   ins_cost(125);
4887   format %{ "movswq $dst, $mem\t# short -> long" %}
4888 
4889   ins_encode %{
4890     __ movswq($dst$$Register, $mem$$Address);
4891   %}
4892 
4893   ins_pipe(ialu_reg_mem);
4894 %}
4895 
4896 // Load Unsigned Short/Char (16 bit UNsigned)
4897 instruct loadUS(rRegI dst, memory mem)
4898 %{
4899   match(Set dst (LoadUS mem));
4900 
4901   ins_cost(125);
4902   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4903 
4904   ins_encode %{
4905     __ movzwl($dst$$Register, $mem$$Address);
4906   %}
4907 
4908   ins_pipe(ialu_reg_mem);
4909 %}
4910 
4911 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4912 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4913   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4914 
4915   ins_cost(125);
4916   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4917   ins_encode %{
4918     __ movsbl($dst$$Register, $mem$$Address);
4919   %}
4920   ins_pipe(ialu_reg_mem);
4921 %}
4922 
4923 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4924 instruct loadUS2L(rRegL dst, memory mem)
4925 %{
4926   match(Set dst (ConvI2L (LoadUS mem)));
4927 
4928   ins_cost(125);
4929   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4930 
4931   ins_encode %{
4932     __ movzwq($dst$$Register, $mem$$Address);
4933   %}
4934 
4935   ins_pipe(ialu_reg_mem);
4936 %}
4937 
4938 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4939 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4940   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4941 
4942   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4943   ins_encode %{
4944     __ movzbq($dst$$Register, $mem$$Address);
4945   %}
4946   ins_pipe(ialu_reg_mem);
4947 %}
4948 
4949 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
4950 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
4951   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4952   effect(KILL cr);
4953 
4954   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
4955             "andl    $dst, $mask" %}
4956   ins_encode %{
4957     Register Rdst = $dst$$Register;
4958     __ movzwq(Rdst, $mem$$Address);
4959     __ andl(Rdst, $mask$$constant);
4960   %}
4961   ins_pipe(ialu_reg_mem);
4962 %}
4963 
4964 // Load Integer
4965 instruct loadI(rRegI dst, memory mem)
4966 %{
4967   match(Set dst (LoadI mem));
4968 
4969   ins_cost(125);
4970   format %{ "movl    $dst, $mem\t# int" %}
4971 
4972   ins_encode %{
4973     __ movl($dst$$Register, $mem$$Address);
4974   %}
4975 
4976   ins_pipe(ialu_reg_mem);
4977 %}
4978 
4979 // Load Integer (32 bit signed) to Byte (8 bit signed)
4980 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4981   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4982 
4983   ins_cost(125);
4984   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4985   ins_encode %{
4986     __ movsbl($dst$$Register, $mem$$Address);
4987   %}
4988   ins_pipe(ialu_reg_mem);
4989 %}
4990 
4991 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4992 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4993   match(Set dst (AndI (LoadI mem) mask));
4994 
4995   ins_cost(125);
4996   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4997   ins_encode %{
4998     __ movzbl($dst$$Register, $mem$$Address);
4999   %}
5000   ins_pipe(ialu_reg_mem);
5001 %}
5002 
5003 // Load Integer (32 bit signed) to Short (16 bit signed)
5004 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5005   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5006 
5007   ins_cost(125);
5008   format %{ "movswl  $dst, $mem\t# int -> short" %}
5009   ins_encode %{
5010     __ movswl($dst$$Register, $mem$$Address);
5011   %}
5012   ins_pipe(ialu_reg_mem);
5013 %}
5014 
5015 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5016 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5017   match(Set dst (AndI (LoadI mem) mask));
5018 
5019   ins_cost(125);
5020   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5021   ins_encode %{
5022     __ movzwl($dst$$Register, $mem$$Address);
5023   %}
5024   ins_pipe(ialu_reg_mem);
5025 %}
5026 
5027 // Load Integer into Long Register
5028 instruct loadI2L(rRegL dst, memory mem)
5029 %{
5030   match(Set dst (ConvI2L (LoadI mem)));
5031 
5032   ins_cost(125);
5033   format %{ "movslq  $dst, $mem\t# int -> long" %}
5034 
5035   ins_encode %{
5036     __ movslq($dst$$Register, $mem$$Address);
5037   %}
5038 
5039   ins_pipe(ialu_reg_mem);
5040 %}
5041 
5042 // Load Integer with mask 0xFF into Long Register
5043 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5044   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5045 
5046   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5047   ins_encode %{
5048     __ movzbq($dst$$Register, $mem$$Address);
5049   %}
5050   ins_pipe(ialu_reg_mem);
5051 %}
5052 
5053 // Load Integer with mask 0xFFFF into Long Register
5054 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5055   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5056 
5057   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5058   ins_encode %{
5059     __ movzwq($dst$$Register, $mem$$Address);
5060   %}
5061   ins_pipe(ialu_reg_mem);
5062 %}
5063 
5064 // Load Integer with a 31-bit mask into Long Register
5065 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5066   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5067   effect(KILL cr);
5068 
5069   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5070             "andl    $dst, $mask" %}
5071   ins_encode %{
5072     Register Rdst = $dst$$Register;
5073     __ movl(Rdst, $mem$$Address);
5074     __ andl(Rdst, $mask$$constant);
5075   %}
5076   ins_pipe(ialu_reg_mem);
5077 %}
5078 
5079 // Load Unsigned Integer into Long Register
5080 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
5081 %{
5082   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5083 
5084   ins_cost(125);
5085   format %{ "movl    $dst, $mem\t# uint -> long" %}
5086 
5087   ins_encode %{
5088     __ movl($dst$$Register, $mem$$Address);
5089   %}
5090 
5091   ins_pipe(ialu_reg_mem);
5092 %}
5093 
5094 // Load Long
5095 instruct loadL(rRegL dst, memory mem)
5096 %{
5097   match(Set dst (LoadL mem));
5098 
5099   ins_cost(125);
5100   format %{ "movq    $dst, $mem\t# long" %}
5101 
5102   ins_encode %{
5103     __ movq($dst$$Register, $mem$$Address);
5104   %}
5105 
5106   ins_pipe(ialu_reg_mem); // XXX
5107 %}
5108 
5109 // Load Range
5110 instruct loadRange(rRegI dst, memory mem)
5111 %{
5112   match(Set dst (LoadRange mem));
5113 
5114   ins_cost(125); // XXX
5115   format %{ "movl    $dst, $mem\t# range" %}
5116   opcode(0x8B);
5117   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5118   ins_pipe(ialu_reg_mem);
5119 %}
5120 
5121 // Load Pointer
5122 instruct loadP(rRegP dst, memory mem)
5123 %{
5124   match(Set dst (LoadP mem));
5125 
5126   ins_cost(125); // XXX
5127   format %{ "movq    $dst, $mem\t# ptr" %}
5128   opcode(0x8B);
5129   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5130   ins_pipe(ialu_reg_mem); // XXX
5131 %}
5132 
5133 // Load Compressed Pointer
5134 instruct loadN(rRegN dst, memory mem)
5135 %{
5136    match(Set dst (LoadN mem));
5137 
5138    ins_cost(125); // XXX
5139    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5140    ins_encode %{
5141      __ movl($dst$$Register, $mem$$Address);
5142    %}
5143    ins_pipe(ialu_reg_mem); // XXX
5144 %}
5145 
5146 
5147 // Load Klass Pointer
5148 instruct loadKlass(rRegP dst, memory mem)
5149 %{
5150   match(Set dst (LoadKlass mem));
5151 
5152   ins_cost(125); // XXX
5153   format %{ "movq    $dst, $mem\t# class" %}
5154   opcode(0x8B);
5155   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5156   ins_pipe(ialu_reg_mem); // XXX
5157 %}
5158 
5159 // Load narrow Klass Pointer
5160 instruct loadNKlass(rRegN dst, memory mem)
5161 %{
5162   match(Set dst (LoadNKlass mem));
5163 
5164   ins_cost(125); // XXX
5165   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5166   ins_encode %{
5167     __ movl($dst$$Register, $mem$$Address);
5168   %}
5169   ins_pipe(ialu_reg_mem); // XXX
5170 %}
5171 
5172 // Load Float
5173 instruct loadF(regF dst, memory mem)
5174 %{
5175   match(Set dst (LoadF mem));
5176 
5177   ins_cost(145); // XXX
5178   format %{ "movss   $dst, $mem\t# float" %}
5179   ins_encode %{
5180     __ movflt($dst$$XMMRegister, $mem$$Address);
5181   %}
5182   ins_pipe(pipe_slow); // XXX
5183 %}
5184 
5185 // Load Double
5186 instruct loadD_partial(regD dst, memory mem)
5187 %{
5188   predicate(!UseXmmLoadAndClearUpper);
5189   match(Set dst (LoadD mem));
5190 
5191   ins_cost(145); // XXX
5192   format %{ "movlpd  $dst, $mem\t# double" %}
5193   ins_encode %{
5194     __ movdbl($dst$$XMMRegister, $mem$$Address);
5195   %}
5196   ins_pipe(pipe_slow); // XXX
5197 %}
5198 
5199 instruct loadD(regD dst, memory mem)
5200 %{
5201   predicate(UseXmmLoadAndClearUpper);
5202   match(Set dst (LoadD mem));
5203 
5204   ins_cost(145); // XXX
5205   format %{ "movsd   $dst, $mem\t# double" %}
5206   ins_encode %{
5207     __ movdbl($dst$$XMMRegister, $mem$$Address);
5208   %}
5209   ins_pipe(pipe_slow); // XXX
5210 %}
5211 
5212 // Load Effective Address
5213 instruct leaP8(rRegP dst, indOffset8 mem)
5214 %{
5215   match(Set dst mem);
5216 
5217   ins_cost(110); // XXX
5218   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5219   opcode(0x8D);
5220   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5221   ins_pipe(ialu_reg_reg_fat);
5222 %}
5223 
5224 instruct leaP32(rRegP dst, indOffset32 mem)
5225 %{
5226   match(Set dst mem);
5227 
5228   ins_cost(110);
5229   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5230   opcode(0x8D);
5231   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5232   ins_pipe(ialu_reg_reg_fat);
5233 %}
5234 
5235 // instruct leaPIdx(rRegP dst, indIndex mem)
5236 // %{
5237 //   match(Set dst mem);
5238 
5239 //   ins_cost(110);
5240 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5241 //   opcode(0x8D);
5242 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5243 //   ins_pipe(ialu_reg_reg_fat);
5244 // %}
5245 
5246 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5247 %{
5248   match(Set dst mem);
5249 
5250   ins_cost(110);
5251   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5252   opcode(0x8D);
5253   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5254   ins_pipe(ialu_reg_reg_fat);
5255 %}
5256 
5257 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5258 %{
5259   match(Set dst mem);
5260 
5261   ins_cost(110);
5262   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5263   opcode(0x8D);
5264   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5265   ins_pipe(ialu_reg_reg_fat);
5266 %}
5267 
5268 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5269 %{
5270   match(Set dst mem);
5271 
5272   ins_cost(110);
5273   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5274   opcode(0x8D);
5275   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5276   ins_pipe(ialu_reg_reg_fat);
5277 %}
5278 
5279 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5280 %{
5281   match(Set dst mem);
5282 
5283   ins_cost(110);
5284   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5285   opcode(0x8D);
5286   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5287   ins_pipe(ialu_reg_reg_fat);
5288 %}
5289 
5290 // Load Effective Address which uses Narrow (32-bits) oop
5291 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5292 %{
5293   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5294   match(Set dst mem);
5295 
5296   ins_cost(110);
5297   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5298   opcode(0x8D);
5299   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5300   ins_pipe(ialu_reg_reg_fat);
5301 %}
5302 
5303 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5304 %{
5305   predicate(Universe::narrow_oop_shift() == 0);
5306   match(Set dst mem);
5307 
5308   ins_cost(110); // XXX
5309   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5310   opcode(0x8D);
5311   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5312   ins_pipe(ialu_reg_reg_fat);
5313 %}
5314 
5315 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5316 %{
5317   predicate(Universe::narrow_oop_shift() == 0);
5318   match(Set dst mem);
5319 
5320   ins_cost(110);
5321   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5322   opcode(0x8D);
5323   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5324   ins_pipe(ialu_reg_reg_fat);
5325 %}
5326 
5327 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5328 %{
5329   predicate(Universe::narrow_oop_shift() == 0);
5330   match(Set dst mem);
5331 
5332   ins_cost(110);
5333   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5334   opcode(0x8D);
5335   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5336   ins_pipe(ialu_reg_reg_fat);
5337 %}
5338 
5339 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5340 %{
5341   predicate(Universe::narrow_oop_shift() == 0);
5342   match(Set dst mem);
5343 
5344   ins_cost(110);
5345   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5346   opcode(0x8D);
5347   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5348   ins_pipe(ialu_reg_reg_fat);
5349 %}
5350 
5351 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5352 %{
5353   predicate(Universe::narrow_oop_shift() == 0);
5354   match(Set dst mem);
5355 
5356   ins_cost(110);
5357   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5358   opcode(0x8D);
5359   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5360   ins_pipe(ialu_reg_reg_fat);
5361 %}
5362 
5363 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5364 %{
5365   predicate(Universe::narrow_oop_shift() == 0);
5366   match(Set dst mem);
5367 
5368   ins_cost(110);
5369   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5370   opcode(0x8D);
5371   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5372   ins_pipe(ialu_reg_reg_fat);
5373 %}
5374 
5375 instruct loadConI(rRegI dst, immI src)
5376 %{
5377   match(Set dst src);
5378 
5379   format %{ "movl    $dst, $src\t# int" %}
5380   ins_encode(load_immI(dst, src));
5381   ins_pipe(ialu_reg_fat); // XXX
5382 %}
5383 
5384 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5385 %{
5386   match(Set dst src);
5387   effect(KILL cr);
5388 
5389   ins_cost(50);
5390   format %{ "xorl    $dst, $dst\t# int" %}
5391   opcode(0x33); /* + rd */
5392   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5393   ins_pipe(ialu_reg);
5394 %}
5395 
5396 instruct loadConL(rRegL dst, immL src)
5397 %{
5398   match(Set dst src);
5399 
5400   ins_cost(150);
5401   format %{ "movq    $dst, $src\t# long" %}
5402   ins_encode(load_immL(dst, src));
5403   ins_pipe(ialu_reg);
5404 %}
5405 
5406 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5407 %{
5408   match(Set dst src);
5409   effect(KILL cr);
5410 
5411   ins_cost(50);
5412   format %{ "xorl    $dst, $dst\t# long" %}
5413   opcode(0x33); /* + rd */
5414   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5415   ins_pipe(ialu_reg); // XXX
5416 %}
5417 
5418 instruct loadConUL32(rRegL dst, immUL32 src)
5419 %{
5420   match(Set dst src);
5421 
5422   ins_cost(60);
5423   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5424   ins_encode(load_immUL32(dst, src));
5425   ins_pipe(ialu_reg);
5426 %}
5427 
5428 instruct loadConL32(rRegL dst, immL32 src)
5429 %{
5430   match(Set dst src);
5431 
5432   ins_cost(70);
5433   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5434   ins_encode(load_immL32(dst, src));
5435   ins_pipe(ialu_reg);
5436 %}
5437 
5438 instruct loadConP(rRegP dst, immP con) %{
5439   match(Set dst con);
5440 
5441   format %{ "movq    $dst, $con\t# ptr" %}
5442   ins_encode(load_immP(dst, con));
5443   ins_pipe(ialu_reg_fat); // XXX
5444 %}
5445 
5446 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5447 %{
5448   match(Set dst src);
5449   effect(KILL cr);
5450 
5451   ins_cost(50);
5452   format %{ "xorl    $dst, $dst\t# ptr" %}
5453   opcode(0x33); /* + rd */
5454   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5455   ins_pipe(ialu_reg);
5456 %}
5457 
5458 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5459 %{
5460   match(Set dst src);
5461   effect(KILL cr);
5462 
5463   ins_cost(60);
5464   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5465   ins_encode(load_immP31(dst, src));
5466   ins_pipe(ialu_reg);
5467 %}
5468 
5469 instruct loadConF(regF dst, immF con) %{
5470   match(Set dst con);
5471   ins_cost(125);
5472   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5473   ins_encode %{
5474     __ movflt($dst$$XMMRegister, $constantaddress($con));
5475   %}
5476   ins_pipe(pipe_slow);
5477 %}
5478 
5479 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5480   match(Set dst src);
5481   effect(KILL cr);
5482   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5483   ins_encode %{
5484     __ xorq($dst$$Register, $dst$$Register);
5485   %}
5486   ins_pipe(ialu_reg);
5487 %}
5488 
5489 instruct loadConN(rRegN dst, immN src) %{
5490   match(Set dst src);
5491 
5492   ins_cost(125);
5493   format %{ "movl    $dst, $src\t# compressed ptr" %}
5494   ins_encode %{
5495     address con = (address)$src$$constant;
5496     if (con == NULL) {
5497       ShouldNotReachHere();
5498     } else {
5499       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5500     }
5501   %}
5502   ins_pipe(ialu_reg_fat); // XXX
5503 %}
5504 
5505 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5506   match(Set dst src);
5507 
5508   ins_cost(125);
5509   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5510   ins_encode %{
5511     address con = (address)$src$$constant;
5512     if (con == NULL) {
5513       ShouldNotReachHere();
5514     } else {
5515       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5516     }
5517   %}
5518   ins_pipe(ialu_reg_fat); // XXX
5519 %}
5520 
5521 instruct loadConF0(regF dst, immF0 src)
5522 %{
5523   match(Set dst src);
5524   ins_cost(100);
5525 
5526   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5527   ins_encode %{
5528     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5529   %}
5530   ins_pipe(pipe_slow);
5531 %}
5532 
5533 // Use the same format since predicate() can not be used here.
5534 instruct loadConD(regD dst, immD con) %{
5535   match(Set dst con);
5536   ins_cost(125);
5537   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5538   ins_encode %{
5539     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5540   %}
5541   ins_pipe(pipe_slow);
5542 %}
5543 
5544 instruct loadConD0(regD dst, immD0 src)
5545 %{
5546   match(Set dst src);
5547   ins_cost(100);
5548 
5549   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5550   ins_encode %{
5551     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5552   %}
5553   ins_pipe(pipe_slow);
5554 %}
5555 
5556 instruct loadSSI(rRegI dst, stackSlotI src)
5557 %{
5558   match(Set dst src);
5559 
5560   ins_cost(125);
5561   format %{ "movl    $dst, $src\t# int stk" %}
5562   opcode(0x8B);
5563   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5564   ins_pipe(ialu_reg_mem);
5565 %}
5566 
5567 instruct loadSSL(rRegL dst, stackSlotL src)
5568 %{
5569   match(Set dst src);
5570 
5571   ins_cost(125);
5572   format %{ "movq    $dst, $src\t# long stk" %}
5573   opcode(0x8B);
5574   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5575   ins_pipe(ialu_reg_mem);
5576 %}
5577 
5578 instruct loadSSP(rRegP dst, stackSlotP src)
5579 %{
5580   match(Set dst src);
5581 
5582   ins_cost(125);
5583   format %{ "movq    $dst, $src\t# ptr stk" %}
5584   opcode(0x8B);
5585   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5586   ins_pipe(ialu_reg_mem);
5587 %}
5588 
5589 instruct loadSSF(regF dst, stackSlotF src)
5590 %{
5591   match(Set dst src);
5592 
5593   ins_cost(125);
5594   format %{ "movss   $dst, $src\t# float stk" %}
5595   ins_encode %{
5596     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5597   %}
5598   ins_pipe(pipe_slow); // XXX
5599 %}
5600 
5601 // Use the same format since predicate() can not be used here.
5602 instruct loadSSD(regD dst, stackSlotD src)
5603 %{
5604   match(Set dst src);
5605 
5606   ins_cost(125);
5607   format %{ "movsd   $dst, $src\t# double stk" %}
5608   ins_encode  %{
5609     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5610   %}
5611   ins_pipe(pipe_slow); // XXX
5612 %}
5613 
5614 // Prefetch instructions.
5615 // Must be safe to execute with invalid address (cannot fault).
5616 
5617 instruct prefetchr( memory mem ) %{
5618   predicate(ReadPrefetchInstr==3);
5619   match(PrefetchRead mem);
5620   ins_cost(125);
5621 
5622   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
5623   ins_encode %{
5624     __ prefetchr($mem$$Address);
5625   %}
5626   ins_pipe(ialu_mem);
5627 %}
5628 
5629 instruct prefetchrNTA( memory mem ) %{
5630   predicate(ReadPrefetchInstr==0);
5631   match(PrefetchRead mem);
5632   ins_cost(125);
5633 
5634   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
5635   ins_encode %{
5636     __ prefetchnta($mem$$Address);
5637   %}
5638   ins_pipe(ialu_mem);
5639 %}
5640 
5641 instruct prefetchrT0( memory mem ) %{
5642   predicate(ReadPrefetchInstr==1);
5643   match(PrefetchRead mem);
5644   ins_cost(125);
5645 
5646   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
5647   ins_encode %{
5648     __ prefetcht0($mem$$Address);
5649   %}
5650   ins_pipe(ialu_mem);
5651 %}
5652 
5653 instruct prefetchrT2( memory mem ) %{
5654   predicate(ReadPrefetchInstr==2);
5655   match(PrefetchRead mem);
5656   ins_cost(125);
5657 
5658   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
5659   ins_encode %{
5660     __ prefetcht2($mem$$Address);
5661   %}
5662   ins_pipe(ialu_mem);
5663 %}
5664 
5665 instruct prefetchwNTA( memory mem ) %{
5666   match(PrefetchWrite mem);
5667   ins_cost(125);
5668 
5669   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
5670   ins_encode %{
5671     __ prefetchnta($mem$$Address);
5672   %}
5673   ins_pipe(ialu_mem);
5674 %}
5675 
5676 // Prefetch instructions for allocation.
5677 
5678 instruct prefetchAlloc( memory mem ) %{
5679   predicate(AllocatePrefetchInstr==3);
5680   match(PrefetchAllocation mem);
5681   ins_cost(125);
5682 
5683   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5684   ins_encode %{
5685     __ prefetchw($mem$$Address);
5686   %}
5687   ins_pipe(ialu_mem);
5688 %}
5689 
5690 instruct prefetchAllocNTA( memory mem ) %{
5691   predicate(AllocatePrefetchInstr==0);
5692   match(PrefetchAllocation mem);
5693   ins_cost(125);
5694 
5695   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5696   ins_encode %{
5697     __ prefetchnta($mem$$Address);
5698   %}
5699   ins_pipe(ialu_mem);
5700 %}
5701 
5702 instruct prefetchAllocT0( memory mem ) %{
5703   predicate(AllocatePrefetchInstr==1);
5704   match(PrefetchAllocation mem);
5705   ins_cost(125);
5706 
5707   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5708   ins_encode %{
5709     __ prefetcht0($mem$$Address);
5710   %}
5711   ins_pipe(ialu_mem);
5712 %}
5713 
5714 instruct prefetchAllocT2( memory mem ) %{
5715   predicate(AllocatePrefetchInstr==2);
5716   match(PrefetchAllocation mem);
5717   ins_cost(125);
5718 
5719   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5720   ins_encode %{
5721     __ prefetcht2($mem$$Address);
5722   %}
5723   ins_pipe(ialu_mem);
5724 %}
5725 
5726 //----------Store Instructions-------------------------------------------------
5727 
5728 // Store Byte
5729 instruct storeB(memory mem, rRegI src)
5730 %{
5731   match(Set mem (StoreB mem src));
5732 
5733   ins_cost(125); // XXX
5734   format %{ "movb    $mem, $src\t# byte" %}
5735   opcode(0x88);
5736   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5737   ins_pipe(ialu_mem_reg);
5738 %}
5739 
5740 // Store Char/Short
5741 instruct storeC(memory mem, rRegI src)
5742 %{
5743   match(Set mem (StoreC mem src));
5744 
5745   ins_cost(125); // XXX
5746   format %{ "movw    $mem, $src\t# char/short" %}
5747   opcode(0x89);
5748   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5749   ins_pipe(ialu_mem_reg);
5750 %}
5751 
5752 // Store Integer
5753 instruct storeI(memory mem, rRegI src)
5754 %{
5755   match(Set mem (StoreI mem src));
5756 
5757   ins_cost(125); // XXX
5758   format %{ "movl    $mem, $src\t# int" %}
5759   opcode(0x89);
5760   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5761   ins_pipe(ialu_mem_reg);
5762 %}
5763 
5764 // Store Long
5765 instruct storeL(memory mem, rRegL src)
5766 %{
5767   match(Set mem (StoreL mem src));
5768 
5769   ins_cost(125); // XXX
5770   format %{ "movq    $mem, $src\t# long" %}
5771   opcode(0x89);
5772   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5773   ins_pipe(ialu_mem_reg); // XXX
5774 %}
5775 
5776 // Store Pointer
5777 instruct storeP(memory mem, any_RegP src)
5778 %{
5779   match(Set mem (StoreP mem src));
5780 
5781   ins_cost(125); // XXX
5782   format %{ "movq    $mem, $src\t# ptr" %}
5783   opcode(0x89);
5784   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5785   ins_pipe(ialu_mem_reg);
5786 %}
5787 
5788 instruct storeImmP0(memory mem, immP0 zero)
5789 %{
5790   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5791   match(Set mem (StoreP mem zero));
5792 
5793   ins_cost(125); // XXX
5794   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5795   ins_encode %{
5796     __ movq($mem$$Address, r12);
5797   %}
5798   ins_pipe(ialu_mem_reg);
5799 %}
5800 
5801 // Store NULL Pointer, mark word, or other simple pointer constant.
5802 instruct storeImmP(memory mem, immP31 src)
5803 %{
5804   match(Set mem (StoreP mem src));
5805 
5806   ins_cost(150); // XXX
5807   format %{ "movq    $mem, $src\t# ptr" %}
5808   opcode(0xC7); /* C7 /0 */
5809   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5810   ins_pipe(ialu_mem_imm);
5811 %}
5812 
5813 // Store Compressed Pointer
5814 instruct storeN(memory mem, rRegN src)
5815 %{
5816   match(Set mem (StoreN mem src));
5817 
5818   ins_cost(125); // XXX
5819   format %{ "movl    $mem, $src\t# compressed ptr" %}
5820   ins_encode %{
5821     __ movl($mem$$Address, $src$$Register);
5822   %}
5823   ins_pipe(ialu_mem_reg);
5824 %}
5825 
5826 instruct storeNKlass(memory mem, rRegN src)
5827 %{
5828   match(Set mem (StoreNKlass mem src));
5829 
5830   ins_cost(125); // XXX
5831   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5832   ins_encode %{
5833     __ movl($mem$$Address, $src$$Register);
5834   %}
5835   ins_pipe(ialu_mem_reg);
5836 %}
5837 
5838 instruct storeImmN0(memory mem, immN0 zero)
5839 %{
5840   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5841   match(Set mem (StoreN mem zero));
5842 
5843   ins_cost(125); // XXX
5844   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5845   ins_encode %{
5846     __ movl($mem$$Address, r12);
5847   %}
5848   ins_pipe(ialu_mem_reg);
5849 %}
5850 
5851 instruct storeImmN(memory mem, immN src)
5852 %{
5853   match(Set mem (StoreN mem src));
5854 
5855   ins_cost(150); // XXX
5856   format %{ "movl    $mem, $src\t# compressed ptr" %}
5857   ins_encode %{
5858     address con = (address)$src$$constant;
5859     if (con == NULL) {
5860       __ movl($mem$$Address, (int32_t)0);
5861     } else {
5862       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5863     }
5864   %}
5865   ins_pipe(ialu_mem_imm);
5866 %}
5867 
5868 instruct storeImmNKlass(memory mem, immNKlass src)
5869 %{
5870   match(Set mem (StoreNKlass mem src));
5871 
5872   ins_cost(150); // XXX
5873   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5874   ins_encode %{
5875     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5876   %}
5877   ins_pipe(ialu_mem_imm);
5878 %}
5879 
5880 // Store Integer Immediate
5881 instruct storeImmI0(memory mem, immI0 zero)
5882 %{
5883   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5884   match(Set mem (StoreI mem zero));
5885 
5886   ins_cost(125); // XXX
5887   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5888   ins_encode %{
5889     __ movl($mem$$Address, r12);
5890   %}
5891   ins_pipe(ialu_mem_reg);
5892 %}
5893 
5894 instruct storeImmI(memory mem, immI src)
5895 %{
5896   match(Set mem (StoreI mem src));
5897 
5898   ins_cost(150);
5899   format %{ "movl    $mem, $src\t# int" %}
5900   opcode(0xC7); /* C7 /0 */
5901   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5902   ins_pipe(ialu_mem_imm);
5903 %}
5904 
5905 // Store Long Immediate
5906 instruct storeImmL0(memory mem, immL0 zero)
5907 %{
5908   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5909   match(Set mem (StoreL mem zero));
5910 
5911   ins_cost(125); // XXX
5912   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5913   ins_encode %{
5914     __ movq($mem$$Address, r12);
5915   %}
5916   ins_pipe(ialu_mem_reg);
5917 %}
5918 
5919 instruct storeImmL(memory mem, immL32 src)
5920 %{
5921   match(Set mem (StoreL mem src));
5922 
5923   ins_cost(150);
5924   format %{ "movq    $mem, $src\t# long" %}
5925   opcode(0xC7); /* C7 /0 */
5926   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5927   ins_pipe(ialu_mem_imm);
5928 %}
5929 
5930 // Store Short/Char Immediate
5931 instruct storeImmC0(memory mem, immI0 zero)
5932 %{
5933   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5934   match(Set mem (StoreC mem zero));
5935 
5936   ins_cost(125); // XXX
5937   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5938   ins_encode %{
5939     __ movw($mem$$Address, r12);
5940   %}
5941   ins_pipe(ialu_mem_reg);
5942 %}
5943 
5944 instruct storeImmI16(memory mem, immI16 src)
5945 %{
5946   predicate(UseStoreImmI16);
5947   match(Set mem (StoreC mem src));
5948 
5949   ins_cost(150);
5950   format %{ "movw    $mem, $src\t# short/char" %}
5951   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5952   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5953   ins_pipe(ialu_mem_imm);
5954 %}
5955 
5956 // Store Byte Immediate
5957 instruct storeImmB0(memory mem, immI0 zero)
5958 %{
5959   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5960   match(Set mem (StoreB mem zero));
5961 
5962   ins_cost(125); // XXX
5963   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5964   ins_encode %{
5965     __ movb($mem$$Address, r12);
5966   %}
5967   ins_pipe(ialu_mem_reg);
5968 %}
5969 
5970 instruct storeImmB(memory mem, immI8 src)
5971 %{
5972   match(Set mem (StoreB mem src));
5973 
5974   ins_cost(150); // XXX
5975   format %{ "movb    $mem, $src\t# byte" %}
5976   opcode(0xC6); /* C6 /0 */
5977   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5978   ins_pipe(ialu_mem_imm);
5979 %}
5980 
5981 // Store CMS card-mark Immediate
5982 instruct storeImmCM0_reg(memory mem, immI0 zero)
5983 %{
5984   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5985   match(Set mem (StoreCM mem zero));
5986 
5987   ins_cost(125); // XXX
5988   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5989   ins_encode %{
5990     __ movb($mem$$Address, r12);
5991   %}
5992   ins_pipe(ialu_mem_reg);
5993 %}
5994 
5995 instruct storeImmCM0(memory mem, immI0 src)
5996 %{
5997   match(Set mem (StoreCM mem src));
5998 
5999   ins_cost(150); // XXX
6000   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6001   opcode(0xC6); /* C6 /0 */
6002   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6003   ins_pipe(ialu_mem_imm);
6004 %}
6005 
6006 // Store Float
6007 instruct storeF(memory mem, regF src)
6008 %{
6009   match(Set mem (StoreF mem src));
6010 
6011   ins_cost(95); // XXX
6012   format %{ "movss   $mem, $src\t# float" %}
6013   ins_encode %{
6014     __ movflt($mem$$Address, $src$$XMMRegister);
6015   %}
6016   ins_pipe(pipe_slow); // XXX
6017 %}
6018 
6019 // Store immediate Float value (it is faster than store from XMM register)
6020 instruct storeF0(memory mem, immF0 zero)
6021 %{
6022   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6023   match(Set mem (StoreF mem zero));
6024 
6025   ins_cost(25); // XXX
6026   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6027   ins_encode %{
6028     __ movl($mem$$Address, r12);
6029   %}
6030   ins_pipe(ialu_mem_reg);
6031 %}
6032 
6033 instruct storeF_imm(memory mem, immF src)
6034 %{
6035   match(Set mem (StoreF mem src));
6036 
6037   ins_cost(50);
6038   format %{ "movl    $mem, $src\t# float" %}
6039   opcode(0xC7); /* C7 /0 */
6040   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6041   ins_pipe(ialu_mem_imm);
6042 %}
6043 
6044 // Store Double
6045 instruct storeD(memory mem, regD src)
6046 %{
6047   match(Set mem (StoreD mem src));
6048 
6049   ins_cost(95); // XXX
6050   format %{ "movsd   $mem, $src\t# double" %}
6051   ins_encode %{
6052     __ movdbl($mem$$Address, $src$$XMMRegister);
6053   %}
6054   ins_pipe(pipe_slow); // XXX
6055 %}
6056 
6057 // Store immediate double 0.0 (it is faster than store from XMM register)
6058 instruct storeD0_imm(memory mem, immD0 src)
6059 %{
6060   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6061   match(Set mem (StoreD mem src));
6062 
6063   ins_cost(50);
6064   format %{ "movq    $mem, $src\t# double 0." %}
6065   opcode(0xC7); /* C7 /0 */
6066   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6067   ins_pipe(ialu_mem_imm);
6068 %}
6069 
6070 instruct storeD0(memory mem, immD0 zero)
6071 %{
6072   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6073   match(Set mem (StoreD mem zero));
6074 
6075   ins_cost(25); // XXX
6076   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6077   ins_encode %{
6078     __ movq($mem$$Address, r12);
6079   %}
6080   ins_pipe(ialu_mem_reg);
6081 %}
6082 
6083 instruct storeSSI(stackSlotI dst, rRegI src)
6084 %{
6085   match(Set dst src);
6086 
6087   ins_cost(100);
6088   format %{ "movl    $dst, $src\t# int stk" %}
6089   opcode(0x89);
6090   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6091   ins_pipe( ialu_mem_reg );
6092 %}
6093 
6094 instruct storeSSL(stackSlotL dst, rRegL src)
6095 %{
6096   match(Set dst src);
6097 
6098   ins_cost(100);
6099   format %{ "movq    $dst, $src\t# long stk" %}
6100   opcode(0x89);
6101   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6102   ins_pipe(ialu_mem_reg);
6103 %}
6104 
6105 instruct storeSSP(stackSlotP dst, rRegP src)
6106 %{
6107   match(Set dst src);
6108 
6109   ins_cost(100);
6110   format %{ "movq    $dst, $src\t# ptr stk" %}
6111   opcode(0x89);
6112   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6113   ins_pipe(ialu_mem_reg);
6114 %}
6115 
6116 instruct storeSSF(stackSlotF dst, regF src)
6117 %{
6118   match(Set dst src);
6119 
6120   ins_cost(95); // XXX
6121   format %{ "movss   $dst, $src\t# float stk" %}
6122   ins_encode %{
6123     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6124   %}
6125   ins_pipe(pipe_slow); // XXX
6126 %}
6127 
6128 instruct storeSSD(stackSlotD dst, regD src)
6129 %{
6130   match(Set dst src);
6131 
6132   ins_cost(95); // XXX
6133   format %{ "movsd   $dst, $src\t# double stk" %}
6134   ins_encode %{
6135     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6136   %}
6137   ins_pipe(pipe_slow); // XXX
6138 %}
6139 
6140 //----------BSWAP Instructions-------------------------------------------------
6141 instruct bytes_reverse_int(rRegI dst) %{
6142   match(Set dst (ReverseBytesI dst));
6143 
6144   format %{ "bswapl  $dst" %}
6145   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6146   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6147   ins_pipe( ialu_reg );
6148 %}
6149 
6150 instruct bytes_reverse_long(rRegL dst) %{
6151   match(Set dst (ReverseBytesL dst));
6152 
6153   format %{ "bswapq  $dst" %}
6154   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6155   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6156   ins_pipe( ialu_reg);
6157 %}
6158 
6159 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6160   match(Set dst (ReverseBytesUS dst));
6161   effect(KILL cr);
6162 
6163   format %{ "bswapl  $dst\n\t"
6164             "shrl    $dst,16\n\t" %}
6165   ins_encode %{
6166     __ bswapl($dst$$Register);
6167     __ shrl($dst$$Register, 16);
6168   %}
6169   ins_pipe( ialu_reg );
6170 %}
6171 
6172 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6173   match(Set dst (ReverseBytesS dst));
6174   effect(KILL cr);
6175 
6176   format %{ "bswapl  $dst\n\t"
6177             "sar     $dst,16\n\t" %}
6178   ins_encode %{
6179     __ bswapl($dst$$Register);
6180     __ sarl($dst$$Register, 16);
6181   %}
6182   ins_pipe( ialu_reg );
6183 %}
6184 
6185 //---------- Zeros Count Instructions ------------------------------------------
6186 
6187 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6188   predicate(UseCountLeadingZerosInstruction);
6189   match(Set dst (CountLeadingZerosI src));
6190   effect(KILL cr);
6191 
6192   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6193   ins_encode %{
6194     __ lzcntl($dst$$Register, $src$$Register);
6195   %}
6196   ins_pipe(ialu_reg);
6197 %}
6198 
6199 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6200   predicate(!UseCountLeadingZerosInstruction);
6201   match(Set dst (CountLeadingZerosI src));
6202   effect(KILL cr);
6203 
6204   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6205             "jnz     skip\n\t"
6206             "movl    $dst, -1\n"
6207       "skip:\n\t"
6208             "negl    $dst\n\t"
6209             "addl    $dst, 31" %}
6210   ins_encode %{
6211     Register Rdst = $dst$$Register;
6212     Register Rsrc = $src$$Register;
6213     Label skip;
6214     __ bsrl(Rdst, Rsrc);
6215     __ jccb(Assembler::notZero, skip);
6216     __ movl(Rdst, -1);
6217     __ bind(skip);
6218     __ negl(Rdst);
6219     __ addl(Rdst, BitsPerInt - 1);
6220   %}
6221   ins_pipe(ialu_reg);
6222 %}
6223 
6224 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6225   predicate(UseCountLeadingZerosInstruction);
6226   match(Set dst (CountLeadingZerosL src));
6227   effect(KILL cr);
6228 
6229   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6230   ins_encode %{
6231     __ lzcntq($dst$$Register, $src$$Register);
6232   %}
6233   ins_pipe(ialu_reg);
6234 %}
6235 
6236 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6237   predicate(!UseCountLeadingZerosInstruction);
6238   match(Set dst (CountLeadingZerosL src));
6239   effect(KILL cr);
6240 
6241   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6242             "jnz     skip\n\t"
6243             "movl    $dst, -1\n"
6244       "skip:\n\t"
6245             "negl    $dst\n\t"
6246             "addl    $dst, 63" %}
6247   ins_encode %{
6248     Register Rdst = $dst$$Register;
6249     Register Rsrc = $src$$Register;
6250     Label skip;
6251     __ bsrq(Rdst, Rsrc);
6252     __ jccb(Assembler::notZero, skip);
6253     __ movl(Rdst, -1);
6254     __ bind(skip);
6255     __ negl(Rdst);
6256     __ addl(Rdst, BitsPerLong - 1);
6257   %}
6258   ins_pipe(ialu_reg);
6259 %}
6260 
6261 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6262   match(Set dst (CountTrailingZerosI src));
6263   effect(KILL cr);
6264 
6265   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6266             "jnz     done\n\t"
6267             "movl    $dst, 32\n"
6268       "done:" %}
6269   ins_encode %{
6270     Register Rdst = $dst$$Register;
6271     Label done;
6272     __ bsfl(Rdst, $src$$Register);
6273     __ jccb(Assembler::notZero, done);
6274     __ movl(Rdst, BitsPerInt);
6275     __ bind(done);
6276   %}
6277   ins_pipe(ialu_reg);
6278 %}
6279 
6280 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6281   match(Set dst (CountTrailingZerosL src));
6282   effect(KILL cr);
6283 
6284   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6285             "jnz     done\n\t"
6286             "movl    $dst, 64\n"
6287       "done:" %}
6288   ins_encode %{
6289     Register Rdst = $dst$$Register;
6290     Label done;
6291     __ bsfq(Rdst, $src$$Register);
6292     __ jccb(Assembler::notZero, done);
6293     __ movl(Rdst, BitsPerLong);
6294     __ bind(done);
6295   %}
6296   ins_pipe(ialu_reg);
6297 %}
6298 
6299 
6300 //---------- Population Count Instructions -------------------------------------
6301 
6302 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6303   predicate(UsePopCountInstruction);
6304   match(Set dst (PopCountI src));
6305   effect(KILL cr);
6306 
6307   format %{ "popcnt  $dst, $src" %}
6308   ins_encode %{
6309     __ popcntl($dst$$Register, $src$$Register);
6310   %}
6311   ins_pipe(ialu_reg);
6312 %}
6313 
6314 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6315   predicate(UsePopCountInstruction);
6316   match(Set dst (PopCountI (LoadI mem)));
6317   effect(KILL cr);
6318 
6319   format %{ "popcnt  $dst, $mem" %}
6320   ins_encode %{
6321     __ popcntl($dst$$Register, $mem$$Address);
6322   %}
6323   ins_pipe(ialu_reg);
6324 %}
6325 
6326 // Note: Long.bitCount(long) returns an int.
6327 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6328   predicate(UsePopCountInstruction);
6329   match(Set dst (PopCountL src));
6330   effect(KILL cr);
6331 
6332   format %{ "popcnt  $dst, $src" %}
6333   ins_encode %{
6334     __ popcntq($dst$$Register, $src$$Register);
6335   %}
6336   ins_pipe(ialu_reg);
6337 %}
6338 
6339 // Note: Long.bitCount(long) returns an int.
6340 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6341   predicate(UsePopCountInstruction);
6342   match(Set dst (PopCountL (LoadL mem)));
6343   effect(KILL cr);
6344 
6345   format %{ "popcnt  $dst, $mem" %}
6346   ins_encode %{
6347     __ popcntq($dst$$Register, $mem$$Address);
6348   %}
6349   ins_pipe(ialu_reg);
6350 %}
6351 
6352 
6353 //----------MemBar Instructions-----------------------------------------------
6354 // Memory barrier flavors
6355 
6356 instruct membar_acquire()
6357 %{
6358   match(MemBarAcquire);
6359   match(LoadFence);
6360   ins_cost(0);
6361 
6362   size(0);
6363   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6364   ins_encode();
6365   ins_pipe(empty);
6366 %}
6367 
6368 instruct membar_acquire_lock()
6369 %{
6370   match(MemBarAcquireLock);
6371   ins_cost(0);
6372 
6373   size(0);
6374   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6375   ins_encode();
6376   ins_pipe(empty);
6377 %}
6378 
6379 instruct membar_release()
6380 %{
6381   match(MemBarRelease);
6382   match(StoreFence);
6383   ins_cost(0);
6384 
6385   size(0);
6386   format %{ "MEMBAR-release ! (empty encoding)" %}
6387   ins_encode();
6388   ins_pipe(empty);
6389 %}
6390 
6391 instruct membar_release_lock()
6392 %{
6393   match(MemBarReleaseLock);
6394   ins_cost(0);
6395 
6396   size(0);
6397   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6398   ins_encode();
6399   ins_pipe(empty);
6400 %}
6401 
6402 instruct membar_volatile(rFlagsReg cr) %{
6403   match(MemBarVolatile);
6404   effect(KILL cr);
6405   ins_cost(400);
6406 
6407   format %{
6408     $$template
6409     if (os::is_MP()) {
6410       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6411     } else {
6412       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6413     }
6414   %}
6415   ins_encode %{
6416     __ membar(Assembler::StoreLoad);
6417   %}
6418   ins_pipe(pipe_slow);
6419 %}
6420 
6421 instruct unnecessary_membar_volatile()
6422 %{
6423   match(MemBarVolatile);
6424   predicate(Matcher::post_store_load_barrier(n));
6425   ins_cost(0);
6426 
6427   size(0);
6428   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6429   ins_encode();
6430   ins_pipe(empty);
6431 %}
6432 
6433 instruct membar_storestore() %{
6434   match(MemBarStoreStore);
6435   ins_cost(0);
6436 
6437   size(0);
6438   format %{ "MEMBAR-storestore (empty encoding)" %}
6439   ins_encode( );
6440   ins_pipe(empty);
6441 %}
6442 
6443 //----------Move Instructions--------------------------------------------------
6444 
6445 instruct castX2P(rRegP dst, rRegL src)
6446 %{
6447   match(Set dst (CastX2P src));
6448 
6449   format %{ "movq    $dst, $src\t# long->ptr" %}
6450   ins_encode %{
6451     if ($dst$$reg != $src$$reg) {
6452       __ movptr($dst$$Register, $src$$Register);
6453     }
6454   %}
6455   ins_pipe(ialu_reg_reg); // XXX
6456 %}
6457 
6458 instruct castP2X(rRegL dst, rRegP src)
6459 %{
6460   match(Set dst (CastP2X src));
6461 
6462   format %{ "movq    $dst, $src\t# ptr -> long" %}
6463   ins_encode %{
6464     if ($dst$$reg != $src$$reg) {
6465       __ movptr($dst$$Register, $src$$Register);
6466     }
6467   %}
6468   ins_pipe(ialu_reg_reg); // XXX
6469 %}
6470 
6471 // Convert oop into int for vectors alignment masking
6472 instruct convP2I(rRegI dst, rRegP src)
6473 %{
6474   match(Set dst (ConvL2I (CastP2X src)));
6475 
6476   format %{ "movl    $dst, $src\t# ptr -> int" %}
6477   ins_encode %{
6478     __ movl($dst$$Register, $src$$Register);
6479   %}
6480   ins_pipe(ialu_reg_reg); // XXX
6481 %}
6482 
6483 // Convert compressed oop into int for vectors alignment masking
6484 // in case of 32bit oops (heap < 4Gb).
6485 instruct convN2I(rRegI dst, rRegN src)
6486 %{
6487   predicate(Universe::narrow_oop_shift() == 0);
6488   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6489 
6490   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6491   ins_encode %{
6492     __ movl($dst$$Register, $src$$Register);
6493   %}
6494   ins_pipe(ialu_reg_reg); // XXX
6495 %}
6496 
6497 // Convert oop pointer into compressed form
6498 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6499   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6500   match(Set dst (EncodeP src));
6501   effect(KILL cr);
6502   format %{ "encode_heap_oop $dst,$src" %}
6503   ins_encode %{
6504     Register s = $src$$Register;
6505     Register d = $dst$$Register;
6506     if (s != d) {
6507       __ movq(d, s);
6508     }
6509     __ encode_heap_oop(d);
6510   %}
6511   ins_pipe(ialu_reg_long);
6512 %}
6513 
6514 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6515   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6516   match(Set dst (EncodeP src));
6517   effect(KILL cr);
6518   format %{ "encode_heap_oop_not_null $dst,$src" %}
6519   ins_encode %{
6520     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6521   %}
6522   ins_pipe(ialu_reg_long);
6523 %}
6524 
6525 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6526   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6527             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6528   match(Set dst (DecodeN src));
6529   effect(KILL cr);
6530   format %{ "decode_heap_oop $dst,$src" %}
6531   ins_encode %{
6532     Register s = $src$$Register;
6533     Register d = $dst$$Register;
6534     if (s != d) {
6535       __ movq(d, s);
6536     }
6537     __ decode_heap_oop(d);
6538   %}
6539   ins_pipe(ialu_reg_long);
6540 %}
6541 
6542 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6543   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6544             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6545   match(Set dst (DecodeN src));
6546   effect(KILL cr);
6547   format %{ "decode_heap_oop_not_null $dst,$src" %}
6548   ins_encode %{
6549     Register s = $src$$Register;
6550     Register d = $dst$$Register;
6551     if (s != d) {
6552       __ decode_heap_oop_not_null(d, s);
6553     } else {
6554       __ decode_heap_oop_not_null(d);
6555     }
6556   %}
6557   ins_pipe(ialu_reg_long);
6558 %}
6559 
6560 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6561   match(Set dst (EncodePKlass src));
6562   effect(KILL cr);
6563   format %{ "encode_klass_not_null $dst,$src" %}
6564   ins_encode %{
6565     __ encode_klass_not_null($dst$$Register, $src$$Register);
6566   %}
6567   ins_pipe(ialu_reg_long);
6568 %}
6569 
6570 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6571   match(Set dst (DecodeNKlass src));
6572   effect(KILL cr);
6573   format %{ "decode_klass_not_null $dst,$src" %}
6574   ins_encode %{
6575     Register s = $src$$Register;
6576     Register d = $dst$$Register;
6577     if (s != d) {
6578       __ decode_klass_not_null(d, s);
6579     } else {
6580       __ decode_klass_not_null(d);
6581     }
6582   %}
6583   ins_pipe(ialu_reg_long);
6584 %}
6585 
6586 
6587 //----------Conditional Move---------------------------------------------------
6588 // Jump
6589 // dummy instruction for generating temp registers
6590 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6591   match(Jump (LShiftL switch_val shift));
6592   ins_cost(350);
6593   predicate(false);
6594   effect(TEMP dest);
6595 
6596   format %{ "leaq    $dest, [$constantaddress]\n\t"
6597             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6598   ins_encode %{
6599     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6600     // to do that and the compiler is using that register as one it can allocate.
6601     // So we build it all by hand.
6602     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6603     // ArrayAddress dispatch(table, index);
6604     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6605     __ lea($dest$$Register, $constantaddress);
6606     __ jmp(dispatch);
6607   %}
6608   ins_pipe(pipe_jmp);
6609 %}
6610 
6611 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6612   match(Jump (AddL (LShiftL switch_val shift) offset));
6613   ins_cost(350);
6614   effect(TEMP dest);
6615 
6616   format %{ "leaq    $dest, [$constantaddress]\n\t"
6617             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6618   ins_encode %{
6619     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6620     // to do that and the compiler is using that register as one it can allocate.
6621     // So we build it all by hand.
6622     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6623     // ArrayAddress dispatch(table, index);
6624     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6625     __ lea($dest$$Register, $constantaddress);
6626     __ jmp(dispatch);
6627   %}
6628   ins_pipe(pipe_jmp);
6629 %}
6630 
6631 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6632   match(Jump switch_val);
6633   ins_cost(350);
6634   effect(TEMP dest);
6635 
6636   format %{ "leaq    $dest, [$constantaddress]\n\t"
6637             "jmp     [$dest + $switch_val]\n\t" %}
6638   ins_encode %{
6639     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6640     // to do that and the compiler is using that register as one it can allocate.
6641     // So we build it all by hand.
6642     // Address index(noreg, switch_reg, Address::times_1);
6643     // ArrayAddress dispatch(table, index);
6644     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6645     __ lea($dest$$Register, $constantaddress);
6646     __ jmp(dispatch);
6647   %}
6648   ins_pipe(pipe_jmp);
6649 %}
6650 
6651 // Conditional move
6652 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6653 %{
6654   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6655 
6656   ins_cost(200); // XXX
6657   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6658   opcode(0x0F, 0x40);
6659   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6660   ins_pipe(pipe_cmov_reg);
6661 %}
6662 
6663 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6664   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6665 
6666   ins_cost(200); // XXX
6667   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6668   opcode(0x0F, 0x40);
6669   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6670   ins_pipe(pipe_cmov_reg);
6671 %}
6672 
6673 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6674   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6675   ins_cost(200);
6676   expand %{
6677     cmovI_regU(cop, cr, dst, src);
6678   %}
6679 %}
6680 
6681 // Conditional move
6682 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6683   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6684 
6685   ins_cost(250); // XXX
6686   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6687   opcode(0x0F, 0x40);
6688   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6689   ins_pipe(pipe_cmov_mem);
6690 %}
6691 
6692 // Conditional move
6693 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6694 %{
6695   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6696 
6697   ins_cost(250); // XXX
6698   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6699   opcode(0x0F, 0x40);
6700   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6701   ins_pipe(pipe_cmov_mem);
6702 %}
6703 
6704 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6705   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6706   ins_cost(250);
6707   expand %{
6708     cmovI_memU(cop, cr, dst, src);
6709   %}
6710 %}
6711 
6712 // Conditional move
6713 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6714 %{
6715   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6716 
6717   ins_cost(200); // XXX
6718   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6719   opcode(0x0F, 0x40);
6720   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6721   ins_pipe(pipe_cmov_reg);
6722 %}
6723 
6724 // Conditional move
6725 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6726 %{
6727   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6728 
6729   ins_cost(200); // XXX
6730   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6731   opcode(0x0F, 0x40);
6732   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6733   ins_pipe(pipe_cmov_reg);
6734 %}
6735 
6736 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6737   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6738   ins_cost(200);
6739   expand %{
6740     cmovN_regU(cop, cr, dst, src);
6741   %}
6742 %}
6743 
6744 // Conditional move
6745 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6746 %{
6747   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6748 
6749   ins_cost(200); // XXX
6750   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6751   opcode(0x0F, 0x40);
6752   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6753   ins_pipe(pipe_cmov_reg);  // XXX
6754 %}
6755 
6756 // Conditional move
6757 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6758 %{
6759   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6760 
6761   ins_cost(200); // XXX
6762   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6763   opcode(0x0F, 0x40);
6764   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6765   ins_pipe(pipe_cmov_reg); // XXX
6766 %}
6767 
6768 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6769   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6770   ins_cost(200);
6771   expand %{
6772     cmovP_regU(cop, cr, dst, src);
6773   %}
6774 %}
6775 
6776 // DISABLED: Requires the ADLC to emit a bottom_type call that
6777 // correctly meets the two pointer arguments; one is an incoming
6778 // register but the other is a memory operand.  ALSO appears to
6779 // be buggy with implicit null checks.
6780 //
6781 //// Conditional move
6782 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6783 //%{
6784 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6785 //  ins_cost(250);
6786 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6787 //  opcode(0x0F,0x40);
6788 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6789 //  ins_pipe( pipe_cmov_mem );
6790 //%}
6791 //
6792 //// Conditional move
6793 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6794 //%{
6795 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6796 //  ins_cost(250);
6797 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6798 //  opcode(0x0F,0x40);
6799 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6800 //  ins_pipe( pipe_cmov_mem );
6801 //%}
6802 
6803 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6804 %{
6805   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6806 
6807   ins_cost(200); // XXX
6808   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6809   opcode(0x0F, 0x40);
6810   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6811   ins_pipe(pipe_cmov_reg);  // XXX
6812 %}
6813 
6814 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6815 %{
6816   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6817 
6818   ins_cost(200); // XXX
6819   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6820   opcode(0x0F, 0x40);
6821   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6822   ins_pipe(pipe_cmov_mem);  // XXX
6823 %}
6824 
6825 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6826 %{
6827   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6828 
6829   ins_cost(200); // XXX
6830   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6831   opcode(0x0F, 0x40);
6832   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6833   ins_pipe(pipe_cmov_reg); // XXX
6834 %}
6835 
6836 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6837   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6838   ins_cost(200);
6839   expand %{
6840     cmovL_regU(cop, cr, dst, src);
6841   %}
6842 %}
6843 
6844 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6845 %{
6846   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6847 
6848   ins_cost(200); // XXX
6849   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6850   opcode(0x0F, 0x40);
6851   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6852   ins_pipe(pipe_cmov_mem); // XXX
6853 %}
6854 
6855 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6856   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6857   ins_cost(200);
6858   expand %{
6859     cmovL_memU(cop, cr, dst, src);
6860   %}
6861 %}
6862 
6863 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6864 %{
6865   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6866 
6867   ins_cost(200); // XXX
6868   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6869             "movss     $dst, $src\n"
6870     "skip:" %}
6871   ins_encode %{
6872     Label Lskip;
6873     // Invert sense of branch from sense of CMOV
6874     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6875     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6876     __ bind(Lskip);
6877   %}
6878   ins_pipe(pipe_slow);
6879 %}
6880 
6881 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6882 // %{
6883 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6884 
6885 //   ins_cost(200); // XXX
6886 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6887 //             "movss     $dst, $src\n"
6888 //     "skip:" %}
6889 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6890 //   ins_pipe(pipe_slow);
6891 // %}
6892 
6893 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6894 %{
6895   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6896 
6897   ins_cost(200); // XXX
6898   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6899             "movss     $dst, $src\n"
6900     "skip:" %}
6901   ins_encode %{
6902     Label Lskip;
6903     // Invert sense of branch from sense of CMOV
6904     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6905     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6906     __ bind(Lskip);
6907   %}
6908   ins_pipe(pipe_slow);
6909 %}
6910 
6911 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6912   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6913   ins_cost(200);
6914   expand %{
6915     cmovF_regU(cop, cr, dst, src);
6916   %}
6917 %}
6918 
6919 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6920 %{
6921   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6922 
6923   ins_cost(200); // XXX
6924   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6925             "movsd     $dst, $src\n"
6926     "skip:" %}
6927   ins_encode %{
6928     Label Lskip;
6929     // Invert sense of branch from sense of CMOV
6930     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6931     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6932     __ bind(Lskip);
6933   %}
6934   ins_pipe(pipe_slow);
6935 %}
6936 
6937 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6938 %{
6939   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6940 
6941   ins_cost(200); // XXX
6942   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6943             "movsd     $dst, $src\n"
6944     "skip:" %}
6945   ins_encode %{
6946     Label Lskip;
6947     // Invert sense of branch from sense of CMOV
6948     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6949     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6950     __ bind(Lskip);
6951   %}
6952   ins_pipe(pipe_slow);
6953 %}
6954 
6955 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6956   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6957   ins_cost(200);
6958   expand %{
6959     cmovD_regU(cop, cr, dst, src);
6960   %}
6961 %}
6962 
6963 //----------Arithmetic Instructions--------------------------------------------
6964 //----------Addition Instructions----------------------------------------------
6965 
6966 instruct addExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
6967 %{
6968   match(AddExactI dst src);
6969   effect(DEF cr);
6970 
6971   format %{ "addl    $dst, $src\t# addExact int" %}
6972   ins_encode %{
6973     __ addl($dst$$Register, $src$$Register);
6974   %}
6975   ins_pipe(ialu_reg_reg);
6976 %}
6977 
6978 instruct addExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr)
6979 %{
6980   match(AddExactI dst src);
6981   effect(DEF cr);
6982 
6983   format %{ "addl    $dst, $src\t# addExact int" %}
6984   ins_encode %{
6985     __ addl($dst$$Register, $src$$constant);
6986   %}
6987   ins_pipe(ialu_reg_reg);
6988 %}
6989 
6990 instruct addExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
6991 %{
6992   match(AddExactI dst (LoadI src));
6993   effect(DEF cr);
6994 
6995   ins_cost(125); // XXX
6996   format %{ "addl    $dst, $src\t# addExact int" %}
6997   ins_encode %{
6998     __ addl($dst$$Register, $src$$Address);
6999   %}
7000 
7001   ins_pipe(ialu_reg_mem);
7002 %}
7003 
7004 instruct addExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
7005 %{
7006   match(AddExactL dst src);
7007   effect(DEF cr);
7008 
7009   format %{ "addq    $dst, $src\t# addExact long" %}
7010   ins_encode %{
7011     __ addq($dst$$Register, $src$$Register);
7012   %}
7013   ins_pipe(ialu_reg_reg);
7014 %}
7015 
7016 instruct addExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr)
7017 %{
7018   match(AddExactL dst src);
7019   effect(DEF cr);
7020 
7021   format %{ "addq    $dst, $src\t# addExact long" %}
7022   ins_encode %{
7023     __ addq($dst$$Register, $src$$constant);
7024   %}
7025   ins_pipe(ialu_reg_reg);
7026 %}
7027 
7028 instruct addExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr)
7029 %{
7030   match(AddExactL dst (LoadL src));
7031   effect(DEF cr);
7032 
7033   ins_cost(125); // XXX
7034   format %{ "addq    $dst, $src\t# addExact long" %}
7035   ins_encode %{
7036     __ addq($dst$$Register, $src$$Address);
7037   %}
7038 
7039   ins_pipe(ialu_reg_mem);
7040 %}
7041 
7042 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7043 %{
7044   match(Set dst (AddI dst src));
7045   effect(KILL cr);
7046 
7047   format %{ "addl    $dst, $src\t# int" %}
7048   opcode(0x03);
7049   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7050   ins_pipe(ialu_reg_reg);
7051 %}
7052 
7053 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7054 %{
7055   match(Set dst (AddI dst src));
7056   effect(KILL cr);
7057 
7058   format %{ "addl    $dst, $src\t# int" %}
7059   opcode(0x81, 0x00); /* /0 id */
7060   ins_encode(OpcSErm(dst, src), Con8or32(src));
7061   ins_pipe( ialu_reg );
7062 %}
7063 
7064 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7065 %{
7066   match(Set dst (AddI dst (LoadI src)));
7067   effect(KILL cr);
7068 
7069   ins_cost(125); // XXX
7070   format %{ "addl    $dst, $src\t# int" %}
7071   opcode(0x03);
7072   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7073   ins_pipe(ialu_reg_mem);
7074 %}
7075 
7076 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7077 %{
7078   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7079   effect(KILL cr);
7080 
7081   ins_cost(150); // XXX
7082   format %{ "addl    $dst, $src\t# int" %}
7083   opcode(0x01); /* Opcode 01 /r */
7084   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7085   ins_pipe(ialu_mem_reg);
7086 %}
7087 
7088 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7089 %{
7090   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7091   effect(KILL cr);
7092 
7093   ins_cost(125); // XXX
7094   format %{ "addl    $dst, $src\t# int" %}
7095   opcode(0x81); /* Opcode 81 /0 id */
7096   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7097   ins_pipe(ialu_mem_imm);
7098 %}
7099 
7100 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7101 %{
7102   predicate(UseIncDec);
7103   match(Set dst (AddI dst src));
7104   effect(KILL cr);
7105 
7106   format %{ "incl    $dst\t# int" %}
7107   opcode(0xFF, 0x00); // FF /0
7108   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7109   ins_pipe(ialu_reg);
7110 %}
7111 
7112 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7113 %{
7114   predicate(UseIncDec);
7115   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7116   effect(KILL cr);
7117 
7118   ins_cost(125); // XXX
7119   format %{ "incl    $dst\t# int" %}
7120   opcode(0xFF); /* Opcode FF /0 */
7121   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7122   ins_pipe(ialu_mem_imm);
7123 %}
7124 
7125 // XXX why does that use AddI
7126 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7127 %{
7128   predicate(UseIncDec);
7129   match(Set dst (AddI dst src));
7130   effect(KILL cr);
7131 
7132   format %{ "decl    $dst\t# int" %}
7133   opcode(0xFF, 0x01); // FF /1
7134   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7135   ins_pipe(ialu_reg);
7136 %}
7137 
7138 // XXX why does that use AddI
7139 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7140 %{
7141   predicate(UseIncDec);
7142   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7143   effect(KILL cr);
7144 
7145   ins_cost(125); // XXX
7146   format %{ "decl    $dst\t# int" %}
7147   opcode(0xFF); /* Opcode FF /1 */
7148   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7149   ins_pipe(ialu_mem_imm);
7150 %}
7151 
7152 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7153 %{
7154   match(Set dst (AddI src0 src1));
7155 
7156   ins_cost(110);
7157   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7158   opcode(0x8D); /* 0x8D /r */
7159   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7160   ins_pipe(ialu_reg_reg);
7161 %}
7162 
7163 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7164 %{
7165   match(Set dst (AddL dst src));
7166   effect(KILL cr);
7167 
7168   format %{ "addq    $dst, $src\t# long" %}
7169   opcode(0x03);
7170   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7171   ins_pipe(ialu_reg_reg);
7172 %}
7173 
7174 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7175 %{
7176   match(Set dst (AddL dst src));
7177   effect(KILL cr);
7178 
7179   format %{ "addq    $dst, $src\t# long" %}
7180   opcode(0x81, 0x00); /* /0 id */
7181   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7182   ins_pipe( ialu_reg );
7183 %}
7184 
7185 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7186 %{
7187   match(Set dst (AddL dst (LoadL src)));
7188   effect(KILL cr);
7189 
7190   ins_cost(125); // XXX
7191   format %{ "addq    $dst, $src\t# long" %}
7192   opcode(0x03);
7193   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7194   ins_pipe(ialu_reg_mem);
7195 %}
7196 
7197 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7198 %{
7199   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7200   effect(KILL cr);
7201 
7202   ins_cost(150); // XXX
7203   format %{ "addq    $dst, $src\t# long" %}
7204   opcode(0x01); /* Opcode 01 /r */
7205   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7206   ins_pipe(ialu_mem_reg);
7207 %}
7208 
7209 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7210 %{
7211   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7212   effect(KILL cr);
7213 
7214   ins_cost(125); // XXX
7215   format %{ "addq    $dst, $src\t# long" %}
7216   opcode(0x81); /* Opcode 81 /0 id */
7217   ins_encode(REX_mem_wide(dst),
7218              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7219   ins_pipe(ialu_mem_imm);
7220 %}
7221 
7222 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7223 %{
7224   predicate(UseIncDec);
7225   match(Set dst (AddL dst src));
7226   effect(KILL cr);
7227 
7228   format %{ "incq    $dst\t# long" %}
7229   opcode(0xFF, 0x00); // FF /0
7230   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7231   ins_pipe(ialu_reg);
7232 %}
7233 
7234 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7235 %{
7236   predicate(UseIncDec);
7237   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7238   effect(KILL cr);
7239 
7240   ins_cost(125); // XXX
7241   format %{ "incq    $dst\t# long" %}
7242   opcode(0xFF); /* Opcode FF /0 */
7243   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7244   ins_pipe(ialu_mem_imm);
7245 %}
7246 
7247 // XXX why does that use AddL
7248 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7249 %{
7250   predicate(UseIncDec);
7251   match(Set dst (AddL dst src));
7252   effect(KILL cr);
7253 
7254   format %{ "decq    $dst\t# long" %}
7255   opcode(0xFF, 0x01); // FF /1
7256   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7257   ins_pipe(ialu_reg);
7258 %}
7259 
7260 // XXX why does that use AddL
7261 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7262 %{
7263   predicate(UseIncDec);
7264   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7265   effect(KILL cr);
7266 
7267   ins_cost(125); // XXX
7268   format %{ "decq    $dst\t# long" %}
7269   opcode(0xFF); /* Opcode FF /1 */
7270   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7271   ins_pipe(ialu_mem_imm);
7272 %}
7273 
7274 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7275 %{
7276   match(Set dst (AddL src0 src1));
7277 
7278   ins_cost(110);
7279   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7280   opcode(0x8D); /* 0x8D /r */
7281   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7282   ins_pipe(ialu_reg_reg);
7283 %}
7284 
7285 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7286 %{
7287   match(Set dst (AddP dst src));
7288   effect(KILL cr);
7289 
7290   format %{ "addq    $dst, $src\t# ptr" %}
7291   opcode(0x03);
7292   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7293   ins_pipe(ialu_reg_reg);
7294 %}
7295 
7296 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7297 %{
7298   match(Set dst (AddP dst src));
7299   effect(KILL cr);
7300 
7301   format %{ "addq    $dst, $src\t# ptr" %}
7302   opcode(0x81, 0x00); /* /0 id */
7303   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7304   ins_pipe( ialu_reg );
7305 %}
7306 
7307 // XXX addP mem ops ????
7308 
7309 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7310 %{
7311   match(Set dst (AddP src0 src1));
7312 
7313   ins_cost(110);
7314   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7315   opcode(0x8D); /* 0x8D /r */
7316   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7317   ins_pipe(ialu_reg_reg);
7318 %}
7319 
7320 instruct checkCastPP(rRegP dst)
7321 %{
7322   match(Set dst (CheckCastPP dst));
7323 
7324   size(0);
7325   format %{ "# checkcastPP of $dst" %}
7326   ins_encode(/* empty encoding */);
7327   ins_pipe(empty);
7328 %}
7329 
7330 instruct castPP(rRegP dst)
7331 %{
7332   match(Set dst (CastPP dst));
7333 
7334   size(0);
7335   format %{ "# castPP of $dst" %}
7336   ins_encode(/* empty encoding */);
7337   ins_pipe(empty);
7338 %}
7339 
7340 instruct castII(rRegI dst)
7341 %{
7342   match(Set dst (CastII dst));
7343 
7344   size(0);
7345   format %{ "# castII of $dst" %}
7346   ins_encode(/* empty encoding */);
7347   ins_cost(0);
7348   ins_pipe(empty);
7349 %}
7350 
7351 // LoadP-locked same as a regular LoadP when used with compare-swap
7352 instruct loadPLocked(rRegP dst, memory mem)
7353 %{
7354   match(Set dst (LoadPLocked mem));
7355 
7356   ins_cost(125); // XXX
7357   format %{ "movq    $dst, $mem\t# ptr locked" %}
7358   opcode(0x8B);
7359   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7360   ins_pipe(ialu_reg_mem); // XXX
7361 %}
7362 
7363 // Conditional-store of the updated heap-top.
7364 // Used during allocation of the shared heap.
7365 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7366 
7367 instruct storePConditional(memory heap_top_ptr,
7368                            rax_RegP oldval, rRegP newval,
7369                            rFlagsReg cr)
7370 %{
7371   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7372 
7373   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7374             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7375   opcode(0x0F, 0xB1);
7376   ins_encode(lock_prefix,
7377              REX_reg_mem_wide(newval, heap_top_ptr),
7378              OpcP, OpcS,
7379              reg_mem(newval, heap_top_ptr));
7380   ins_pipe(pipe_cmpxchg);
7381 %}
7382 
7383 // Conditional-store of an int value.
7384 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7385 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7386 %{
7387   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7388   effect(KILL oldval);
7389 
7390   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7391   opcode(0x0F, 0xB1);
7392   ins_encode(lock_prefix,
7393              REX_reg_mem(newval, mem),
7394              OpcP, OpcS,
7395              reg_mem(newval, mem));
7396   ins_pipe(pipe_cmpxchg);
7397 %}
7398 
7399 // Conditional-store of a long value.
7400 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7401 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7402 %{
7403   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7404   effect(KILL oldval);
7405 
7406   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7407   opcode(0x0F, 0xB1);
7408   ins_encode(lock_prefix,
7409              REX_reg_mem_wide(newval, mem),
7410              OpcP, OpcS,
7411              reg_mem(newval, mem));
7412   ins_pipe(pipe_cmpxchg);
7413 %}
7414 
7415 
7416 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7417 instruct compareAndSwapP(rRegI res,
7418                          memory mem_ptr,
7419                          rax_RegP oldval, rRegP newval,
7420                          rFlagsReg cr)
7421 %{
7422   predicate(VM_Version::supports_cx8());
7423   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7424   effect(KILL cr, KILL oldval);
7425 
7426   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7427             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7428             "sete    $res\n\t"
7429             "movzbl  $res, $res" %}
7430   opcode(0x0F, 0xB1);
7431   ins_encode(lock_prefix,
7432              REX_reg_mem_wide(newval, mem_ptr),
7433              OpcP, OpcS,
7434              reg_mem(newval, mem_ptr),
7435              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7436              REX_reg_breg(res, res), // movzbl
7437              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7438   ins_pipe( pipe_cmpxchg );
7439 %}
7440 
7441 instruct compareAndSwapL(rRegI res,
7442                          memory mem_ptr,
7443                          rax_RegL oldval, rRegL newval,
7444                          rFlagsReg cr)
7445 %{
7446   predicate(VM_Version::supports_cx8());
7447   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7448   effect(KILL cr, KILL oldval);
7449 
7450   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7451             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7452             "sete    $res\n\t"
7453             "movzbl  $res, $res" %}
7454   opcode(0x0F, 0xB1);
7455   ins_encode(lock_prefix,
7456              REX_reg_mem_wide(newval, mem_ptr),
7457              OpcP, OpcS,
7458              reg_mem(newval, mem_ptr),
7459              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7460              REX_reg_breg(res, res), // movzbl
7461              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7462   ins_pipe( pipe_cmpxchg );
7463 %}
7464 
7465 instruct compareAndSwapI(rRegI res,
7466                          memory mem_ptr,
7467                          rax_RegI oldval, rRegI newval,
7468                          rFlagsReg cr)
7469 %{
7470   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7471   effect(KILL cr, KILL oldval);
7472 
7473   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7474             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7475             "sete    $res\n\t"
7476             "movzbl  $res, $res" %}
7477   opcode(0x0F, 0xB1);
7478   ins_encode(lock_prefix,
7479              REX_reg_mem(newval, mem_ptr),
7480              OpcP, OpcS,
7481              reg_mem(newval, mem_ptr),
7482              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7483              REX_reg_breg(res, res), // movzbl
7484              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7485   ins_pipe( pipe_cmpxchg );
7486 %}
7487 
7488 
7489 instruct compareAndSwapN(rRegI res,
7490                           memory mem_ptr,
7491                           rax_RegN oldval, rRegN newval,
7492                           rFlagsReg cr) %{
7493   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7494   effect(KILL cr, KILL oldval);
7495 
7496   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7497             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7498             "sete    $res\n\t"
7499             "movzbl  $res, $res" %}
7500   opcode(0x0F, 0xB1);
7501   ins_encode(lock_prefix,
7502              REX_reg_mem(newval, mem_ptr),
7503              OpcP, OpcS,
7504              reg_mem(newval, mem_ptr),
7505              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7506              REX_reg_breg(res, res), // movzbl
7507              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7508   ins_pipe( pipe_cmpxchg );
7509 %}
7510 
7511 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7512   predicate(n->as_LoadStore()->result_not_used());
7513   match(Set dummy (GetAndAddI mem add));
7514   effect(KILL cr);
7515   format %{ "ADDL  [$mem],$add" %}
7516   ins_encode %{
7517     if (os::is_MP()) { __ lock(); }
7518     __ addl($mem$$Address, $add$$constant);
7519   %}
7520   ins_pipe( pipe_cmpxchg );
7521 %}
7522 
7523 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7524   match(Set newval (GetAndAddI mem newval));
7525   effect(KILL cr);
7526   format %{ "XADDL  [$mem],$newval" %}
7527   ins_encode %{
7528     if (os::is_MP()) { __ lock(); }
7529     __ xaddl($mem$$Address, $newval$$Register);
7530   %}
7531   ins_pipe( pipe_cmpxchg );
7532 %}
7533 
7534 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7535   predicate(n->as_LoadStore()->result_not_used());
7536   match(Set dummy (GetAndAddL mem add));
7537   effect(KILL cr);
7538   format %{ "ADDQ  [$mem],$add" %}
7539   ins_encode %{
7540     if (os::is_MP()) { __ lock(); }
7541     __ addq($mem$$Address, $add$$constant);
7542   %}
7543   ins_pipe( pipe_cmpxchg );
7544 %}
7545 
7546 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7547   match(Set newval (GetAndAddL mem newval));
7548   effect(KILL cr);
7549   format %{ "XADDQ  [$mem],$newval" %}
7550   ins_encode %{
7551     if (os::is_MP()) { __ lock(); }
7552     __ xaddq($mem$$Address, $newval$$Register);
7553   %}
7554   ins_pipe( pipe_cmpxchg );
7555 %}
7556 
7557 instruct xchgI( memory mem, rRegI newval) %{
7558   match(Set newval (GetAndSetI mem newval));
7559   format %{ "XCHGL  $newval,[$mem]" %}
7560   ins_encode %{
7561     __ xchgl($newval$$Register, $mem$$Address);
7562   %}
7563   ins_pipe( pipe_cmpxchg );
7564 %}
7565 
7566 instruct xchgL( memory mem, rRegL newval) %{
7567   match(Set newval (GetAndSetL mem newval));
7568   format %{ "XCHGL  $newval,[$mem]" %}
7569   ins_encode %{
7570     __ xchgq($newval$$Register, $mem$$Address);
7571   %}
7572   ins_pipe( pipe_cmpxchg );
7573 %}
7574 
7575 instruct xchgP( memory mem, rRegP newval) %{
7576   match(Set newval (GetAndSetP mem newval));
7577   format %{ "XCHGQ  $newval,[$mem]" %}
7578   ins_encode %{
7579     __ xchgq($newval$$Register, $mem$$Address);
7580   %}
7581   ins_pipe( pipe_cmpxchg );
7582 %}
7583 
7584 instruct xchgN( memory mem, rRegN newval) %{
7585   match(Set newval (GetAndSetN mem newval));
7586   format %{ "XCHGL  $newval,$mem]" %}
7587   ins_encode %{
7588     __ xchgl($newval$$Register, $mem$$Address);
7589   %}
7590   ins_pipe( pipe_cmpxchg );
7591 %}
7592 
7593 //----------Subtraction Instructions-------------------------------------------
7594 
7595 // Integer Subtraction Instructions
7596 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7597 %{
7598   match(Set dst (SubI dst src));
7599   effect(KILL cr);
7600 
7601   format %{ "subl    $dst, $src\t# int" %}
7602   opcode(0x2B);
7603   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7604   ins_pipe(ialu_reg_reg);
7605 %}
7606 
7607 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7608 %{
7609   match(Set dst (SubI dst src));
7610   effect(KILL cr);
7611 
7612   format %{ "subl    $dst, $src\t# int" %}
7613   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7614   ins_encode(OpcSErm(dst, src), Con8or32(src));
7615   ins_pipe(ialu_reg);
7616 %}
7617 
7618 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7619 %{
7620   match(Set dst (SubI dst (LoadI src)));
7621   effect(KILL cr);
7622 
7623   ins_cost(125);
7624   format %{ "subl    $dst, $src\t# int" %}
7625   opcode(0x2B);
7626   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7627   ins_pipe(ialu_reg_mem);
7628 %}
7629 
7630 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7631 %{
7632   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7633   effect(KILL cr);
7634 
7635   ins_cost(150);
7636   format %{ "subl    $dst, $src\t# int" %}
7637   opcode(0x29); /* Opcode 29 /r */
7638   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7639   ins_pipe(ialu_mem_reg);
7640 %}
7641 
7642 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7643 %{
7644   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7645   effect(KILL cr);
7646 
7647   ins_cost(125); // XXX
7648   format %{ "subl    $dst, $src\t# int" %}
7649   opcode(0x81); /* Opcode 81 /5 id */
7650   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7651   ins_pipe(ialu_mem_imm);
7652 %}
7653 
7654 instruct subExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
7655 %{
7656   match(SubExactI dst src);
7657   effect(DEF cr);
7658 
7659   format %{ "subl    $dst, $src\t# subExact int" %}
7660   ins_encode %{
7661     __ subl($dst$$Register, $src$$Register);
7662   %}
7663   ins_pipe(ialu_reg_reg);
7664 %}
7665 
7666 instruct subExactI_rReg_imm(rax_RegI dst, immI src, rFlagsReg cr)
7667 %{
7668   match(SubExactI dst src);
7669   effect(DEF cr);
7670 
7671   format %{ "subl    $dst, $src\t# subExact int" %}
7672   ins_encode %{
7673     __ subl($dst$$Register, $src$$constant);
7674   %}
7675   ins_pipe(ialu_reg_reg);
7676 %}
7677 
7678 instruct subExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
7679 %{
7680   match(SubExactI dst (LoadI src));
7681   effect(DEF cr);
7682 
7683   ins_cost(125);
7684   format %{ "subl    $dst, $src\t# subExact int" %}
7685   ins_encode %{
7686     __ subl($dst$$Register, $src$$Address);
7687   %}
7688   ins_pipe(ialu_reg_mem);
7689 %}
7690 
7691 instruct subExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
7692 %{
7693   match(SubExactL dst src);
7694   effect(DEF cr);
7695 
7696   format %{ "subq    $dst, $src\t# subExact long" %}
7697   ins_encode %{
7698     __ subq($dst$$Register, $src$$Register);
7699   %}
7700   ins_pipe(ialu_reg_reg);
7701 %}
7702 
7703 instruct subExactL_rReg_imm(rax_RegL dst, immL32 src, rFlagsReg cr)
7704 %{
7705   match(SubExactL dst (LoadL src));
7706   effect(DEF cr);
7707 
7708   format %{ "subq    $dst, $src\t# subExact long" %}
7709   ins_encode %{
7710     __ subq($dst$$Register, $src$$constant);
7711   %}
7712   ins_pipe(ialu_reg_reg);
7713 %}
7714 
7715 instruct subExactL_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
7716 %{
7717   match(SubExactI dst src);
7718   effect(DEF cr);
7719 
7720   ins_cost(125);
7721   format %{ "subq    $dst, $src\t# subExact long" %}
7722   ins_encode %{
7723     __ subq($dst$$Register, $src$$Address);
7724   %}
7725   ins_pipe(ialu_reg_mem);
7726 %}
7727 
7728 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7729 %{
7730   match(Set dst (SubL dst src));
7731   effect(KILL cr);
7732 
7733   format %{ "subq    $dst, $src\t# long" %}
7734   opcode(0x2B);
7735   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7736   ins_pipe(ialu_reg_reg);
7737 %}
7738 
7739 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7740 %{
7741   match(Set dst (SubL dst src));
7742   effect(KILL cr);
7743 
7744   format %{ "subq    $dst, $src\t# long" %}
7745   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7746   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7747   ins_pipe(ialu_reg);
7748 %}
7749 
7750 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7751 %{
7752   match(Set dst (SubL dst (LoadL src)));
7753   effect(KILL cr);
7754 
7755   ins_cost(125);
7756   format %{ "subq    $dst, $src\t# long" %}
7757   opcode(0x2B);
7758   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7759   ins_pipe(ialu_reg_mem);
7760 %}
7761 
7762 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7763 %{
7764   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7765   effect(KILL cr);
7766 
7767   ins_cost(150);
7768   format %{ "subq    $dst, $src\t# long" %}
7769   opcode(0x29); /* Opcode 29 /r */
7770   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7771   ins_pipe(ialu_mem_reg);
7772 %}
7773 
7774 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7775 %{
7776   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7777   effect(KILL cr);
7778 
7779   ins_cost(125); // XXX
7780   format %{ "subq    $dst, $src\t# long" %}
7781   opcode(0x81); /* Opcode 81 /5 id */
7782   ins_encode(REX_mem_wide(dst),
7783              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7784   ins_pipe(ialu_mem_imm);
7785 %}
7786 
7787 // Subtract from a pointer
7788 // XXX hmpf???
7789 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7790 %{
7791   match(Set dst (AddP dst (SubI zero src)));
7792   effect(KILL cr);
7793 
7794   format %{ "subq    $dst, $src\t# ptr - int" %}
7795   opcode(0x2B);
7796   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7797   ins_pipe(ialu_reg_reg);
7798 %}
7799 
7800 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7801 %{
7802   match(Set dst (SubI zero dst));
7803   effect(KILL cr);
7804 
7805   format %{ "negl    $dst\t# int" %}
7806   opcode(0xF7, 0x03);  // Opcode F7 /3
7807   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7808   ins_pipe(ialu_reg);
7809 %}
7810 
7811 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7812 %{
7813   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7814   effect(KILL cr);
7815 
7816   format %{ "negl    $dst\t# int" %}
7817   opcode(0xF7, 0x03);  // Opcode F7 /3
7818   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7819   ins_pipe(ialu_reg);
7820 %}
7821 
7822 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7823 %{
7824   match(Set dst (SubL zero dst));
7825   effect(KILL cr);
7826 
7827   format %{ "negq    $dst\t# long" %}
7828   opcode(0xF7, 0x03);  // Opcode F7 /3
7829   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7830   ins_pipe(ialu_reg);
7831 %}
7832 
7833 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7834 %{
7835   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7836   effect(KILL cr);
7837 
7838   format %{ "negq    $dst\t# long" %}
7839   opcode(0xF7, 0x03);  // Opcode F7 /3
7840   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7841   ins_pipe(ialu_reg);
7842 %}
7843 
7844 instruct negExactI_rReg(rax_RegI dst, rFlagsReg cr)
7845 %{
7846   match(NegExactI dst);
7847   effect(KILL cr);
7848 
7849   format %{ "negl    $dst\t# negExact int" %}
7850   ins_encode %{
7851     __ negl($dst$$Register);
7852   %}
7853   ins_pipe(ialu_reg);
7854 %}
7855 
7856 instruct negExactL_rReg(rax_RegL dst, rFlagsReg cr)
7857 %{
7858   match(NegExactL dst);
7859   effect(KILL cr);
7860 
7861   format %{ "negq    $dst\t# negExact long" %}
7862   ins_encode %{
7863     __ negq($dst$$Register);
7864   %}
7865   ins_pipe(ialu_reg);
7866 %}
7867 
7868 
7869 //----------Multiplication/Division Instructions-------------------------------
7870 // Integer Multiplication Instructions
7871 // Multiply Register
7872 
7873 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7874 %{
7875   match(Set dst (MulI dst src));
7876   effect(KILL cr);
7877 
7878   ins_cost(300);
7879   format %{ "imull   $dst, $src\t# int" %}
7880   opcode(0x0F, 0xAF);
7881   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7882   ins_pipe(ialu_reg_reg_alu0);
7883 %}
7884 
7885 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7886 %{
7887   match(Set dst (MulI src imm));
7888   effect(KILL cr);
7889 
7890   ins_cost(300);
7891   format %{ "imull   $dst, $src, $imm\t# int" %}
7892   opcode(0x69); /* 69 /r id */
7893   ins_encode(REX_reg_reg(dst, src),
7894              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7895   ins_pipe(ialu_reg_reg_alu0);
7896 %}
7897 
7898 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7899 %{
7900   match(Set dst (MulI dst (LoadI src)));
7901   effect(KILL cr);
7902 
7903   ins_cost(350);
7904   format %{ "imull   $dst, $src\t# int" %}
7905   opcode(0x0F, 0xAF);
7906   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7907   ins_pipe(ialu_reg_mem_alu0);
7908 %}
7909 
7910 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7911 %{
7912   match(Set dst (MulI (LoadI src) imm));
7913   effect(KILL cr);
7914 
7915   ins_cost(300);
7916   format %{ "imull   $dst, $src, $imm\t# int" %}
7917   opcode(0x69); /* 69 /r id */
7918   ins_encode(REX_reg_mem(dst, src),
7919              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7920   ins_pipe(ialu_reg_mem_alu0);
7921 %}
7922 
7923 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7924 %{
7925   match(Set dst (MulL dst src));
7926   effect(KILL cr);
7927 
7928   ins_cost(300);
7929   format %{ "imulq   $dst, $src\t# long" %}
7930   opcode(0x0F, 0xAF);
7931   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7932   ins_pipe(ialu_reg_reg_alu0);
7933 %}
7934 
7935 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7936 %{
7937   match(Set dst (MulL src imm));
7938   effect(KILL cr);
7939 
7940   ins_cost(300);
7941   format %{ "imulq   $dst, $src, $imm\t# long" %}
7942   opcode(0x69); /* 69 /r id */
7943   ins_encode(REX_reg_reg_wide(dst, src),
7944              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7945   ins_pipe(ialu_reg_reg_alu0);
7946 %}
7947 
7948 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7949 %{
7950   match(Set dst (MulL dst (LoadL src)));
7951   effect(KILL cr);
7952 
7953   ins_cost(350);
7954   format %{ "imulq   $dst, $src\t# long" %}
7955   opcode(0x0F, 0xAF);
7956   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7957   ins_pipe(ialu_reg_mem_alu0);
7958 %}
7959 
7960 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7961 %{
7962   match(Set dst (MulL (LoadL src) imm));
7963   effect(KILL cr);
7964 
7965   ins_cost(300);
7966   format %{ "imulq   $dst, $src, $imm\t# long" %}
7967   opcode(0x69); /* 69 /r id */
7968   ins_encode(REX_reg_mem_wide(dst, src),
7969              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7970   ins_pipe(ialu_reg_mem_alu0);
7971 %}
7972 
7973 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7974 %{
7975   match(Set dst (MulHiL src rax));
7976   effect(USE_KILL rax, KILL cr);
7977 
7978   ins_cost(300);
7979   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7980   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7981   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7982   ins_pipe(ialu_reg_reg_alu0);
7983 %}
7984 
7985 
7986 instruct mulExactI_rReg(rax_RegI dst, rRegI src, rFlagsReg cr)
7987 %{
7988   match(MulExactI dst src);
7989   effect(DEF cr);
7990 
7991   ins_cost(300);
7992   format %{ "imull   $dst, $src\t# mulExact int" %}
7993   ins_encode %{
7994     __ imull($dst$$Register, $src$$Register);
7995   %}
7996   ins_pipe(ialu_reg_reg_alu0);
7997 %}
7998 
7999 
8000 instruct mulExactI_rReg_imm(rax_RegI dst, rRegI src, immI imm, rFlagsReg cr)
8001 %{
8002   match(MulExactI src imm);
8003   effect(DEF cr);
8004 
8005   ins_cost(300);
8006   format %{ "imull   $dst, $src, $imm\t# mulExact int" %}
8007   ins_encode %{
8008     __ imull($dst$$Register, $src$$Register, $imm$$constant);
8009   %}
8010   ins_pipe(ialu_reg_reg_alu0);
8011 %}
8012 
8013 instruct mulExactI_rReg_mem(rax_RegI dst, memory src, rFlagsReg cr)
8014 %{
8015   match(MulExactI dst (LoadI src));
8016   effect(DEF cr);
8017 
8018   ins_cost(350);
8019   format %{ "imull   $dst, $src\t# mulExact int" %}
8020   ins_encode %{
8021     __ imull($dst$$Register, $src$$Address);
8022   %}
8023   ins_pipe(ialu_reg_mem_alu0);
8024 %}
8025 
8026 instruct mulExactL_rReg(rax_RegL dst, rRegL src, rFlagsReg cr)
8027 %{
8028   match(MulExactL dst src);
8029   effect(DEF cr);
8030 
8031   ins_cost(300);
8032   format %{ "imulq   $dst, $src\t# mulExact long" %}
8033   ins_encode %{
8034     __ imulq($dst$$Register, $src$$Register);
8035   %}
8036   ins_pipe(ialu_reg_reg_alu0);
8037 %}
8038 
8039 instruct mulExactL_rReg_imm(rax_RegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8040 %{
8041   match(MulExactL src imm);
8042   effect(DEF cr);
8043 
8044   ins_cost(300);
8045   format %{ "imulq   $dst, $src, $imm\t# mulExact long" %}
8046   ins_encode %{
8047     __ imulq($dst$$Register, $src$$Register, $imm$$constant);
8048   %}
8049   ins_pipe(ialu_reg_reg_alu0);
8050 %}
8051 
8052 instruct mulExactL_rReg_mem(rax_RegL dst, memory src, rFlagsReg cr)
8053 %{
8054   match(MulExactL dst (LoadL src));
8055   effect(DEF cr);
8056 
8057   ins_cost(350);
8058   format %{ "imulq   $dst, $src\t# mulExact long" %}
8059   ins_encode %{
8060     __ imulq($dst$$Register, $src$$Address);
8061   %}
8062   ins_pipe(ialu_reg_mem_alu0);
8063 %}
8064 
8065 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8066                    rFlagsReg cr)
8067 %{
8068   match(Set rax (DivI rax div));
8069   effect(KILL rdx, KILL cr);
8070 
8071   ins_cost(30*100+10*100); // XXX
8072   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8073             "jne,s   normal\n\t"
8074             "xorl    rdx, rdx\n\t"
8075             "cmpl    $div, -1\n\t"
8076             "je,s    done\n"
8077     "normal: cdql\n\t"
8078             "idivl   $div\n"
8079     "done:"        %}
8080   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8081   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8082   ins_pipe(ialu_reg_reg_alu0);
8083 %}
8084 
8085 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8086                    rFlagsReg cr)
8087 %{
8088   match(Set rax (DivL rax div));
8089   effect(KILL rdx, KILL cr);
8090 
8091   ins_cost(30*100+10*100); // XXX
8092   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8093             "cmpq    rax, rdx\n\t"
8094             "jne,s   normal\n\t"
8095             "xorl    rdx, rdx\n\t"
8096             "cmpq    $div, -1\n\t"
8097             "je,s    done\n"
8098     "normal: cdqq\n\t"
8099             "idivq   $div\n"
8100     "done:"        %}
8101   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8102   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8103   ins_pipe(ialu_reg_reg_alu0);
8104 %}
8105 
8106 // Integer DIVMOD with Register, both quotient and mod results
8107 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8108                              rFlagsReg cr)
8109 %{
8110   match(DivModI rax div);
8111   effect(KILL cr);
8112 
8113   ins_cost(30*100+10*100); // XXX
8114   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8115             "jne,s   normal\n\t"
8116             "xorl    rdx, rdx\n\t"
8117             "cmpl    $div, -1\n\t"
8118             "je,s    done\n"
8119     "normal: cdql\n\t"
8120             "idivl   $div\n"
8121     "done:"        %}
8122   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8123   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8124   ins_pipe(pipe_slow);
8125 %}
8126 
8127 // Long DIVMOD with Register, both quotient and mod results
8128 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8129                              rFlagsReg cr)
8130 %{
8131   match(DivModL rax div);
8132   effect(KILL cr);
8133 
8134   ins_cost(30*100+10*100); // XXX
8135   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8136             "cmpq    rax, rdx\n\t"
8137             "jne,s   normal\n\t"
8138             "xorl    rdx, rdx\n\t"
8139             "cmpq    $div, -1\n\t"
8140             "je,s    done\n"
8141     "normal: cdqq\n\t"
8142             "idivq   $div\n"
8143     "done:"        %}
8144   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8145   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8146   ins_pipe(pipe_slow);
8147 %}
8148 
8149 //----------- DivL-By-Constant-Expansions--------------------------------------
8150 // DivI cases are handled by the compiler
8151 
8152 // Magic constant, reciprocal of 10
8153 instruct loadConL_0x6666666666666667(rRegL dst)
8154 %{
8155   effect(DEF dst);
8156 
8157   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8158   ins_encode(load_immL(dst, 0x6666666666666667));
8159   ins_pipe(ialu_reg);
8160 %}
8161 
8162 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8163 %{
8164   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8165 
8166   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8167   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8168   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8169   ins_pipe(ialu_reg_reg_alu0);
8170 %}
8171 
8172 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8173 %{
8174   effect(USE_DEF dst, KILL cr);
8175 
8176   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8177   opcode(0xC1, 0x7); /* C1 /7 ib */
8178   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8179   ins_pipe(ialu_reg);
8180 %}
8181 
8182 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8183 %{
8184   effect(USE_DEF dst, KILL cr);
8185 
8186   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8187   opcode(0xC1, 0x7); /* C1 /7 ib */
8188   ins_encode(reg_opc_imm_wide(dst, 0x2));
8189   ins_pipe(ialu_reg);
8190 %}
8191 
8192 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8193 %{
8194   match(Set dst (DivL src div));
8195 
8196   ins_cost((5+8)*100);
8197   expand %{
8198     rax_RegL rax;                     // Killed temp
8199     rFlagsReg cr;                     // Killed
8200     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8201     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8202     sarL_rReg_63(src, cr);            // sarq  src, 63
8203     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8204     subL_rReg(dst, src, cr);          // subl  rdx, src
8205   %}
8206 %}
8207 
8208 //-----------------------------------------------------------------------------
8209 
8210 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8211                    rFlagsReg cr)
8212 %{
8213   match(Set rdx (ModI rax div));
8214   effect(KILL rax, KILL cr);
8215 
8216   ins_cost(300); // XXX
8217   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8218             "jne,s   normal\n\t"
8219             "xorl    rdx, rdx\n\t"
8220             "cmpl    $div, -1\n\t"
8221             "je,s    done\n"
8222     "normal: cdql\n\t"
8223             "idivl   $div\n"
8224     "done:"        %}
8225   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8226   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8227   ins_pipe(ialu_reg_reg_alu0);
8228 %}
8229 
8230 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8231                    rFlagsReg cr)
8232 %{
8233   match(Set rdx (ModL rax div));
8234   effect(KILL rax, KILL cr);
8235 
8236   ins_cost(300); // XXX
8237   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8238             "cmpq    rax, rdx\n\t"
8239             "jne,s   normal\n\t"
8240             "xorl    rdx, rdx\n\t"
8241             "cmpq    $div, -1\n\t"
8242             "je,s    done\n"
8243     "normal: cdqq\n\t"
8244             "idivq   $div\n"
8245     "done:"        %}
8246   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8247   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8248   ins_pipe(ialu_reg_reg_alu0);
8249 %}
8250 
8251 // Integer Shift Instructions
8252 // Shift Left by one
8253 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8254 %{
8255   match(Set dst (LShiftI dst shift));
8256   effect(KILL cr);
8257 
8258   format %{ "sall    $dst, $shift" %}
8259   opcode(0xD1, 0x4); /* D1 /4 */
8260   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8261   ins_pipe(ialu_reg);
8262 %}
8263 
8264 // Shift Left by one
8265 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8266 %{
8267   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8268   effect(KILL cr);
8269 
8270   format %{ "sall    $dst, $shift\t" %}
8271   opcode(0xD1, 0x4); /* D1 /4 */
8272   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8273   ins_pipe(ialu_mem_imm);
8274 %}
8275 
8276 // Shift Left by 8-bit immediate
8277 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8278 %{
8279   match(Set dst (LShiftI dst shift));
8280   effect(KILL cr);
8281 
8282   format %{ "sall    $dst, $shift" %}
8283   opcode(0xC1, 0x4); /* C1 /4 ib */
8284   ins_encode(reg_opc_imm(dst, shift));
8285   ins_pipe(ialu_reg);
8286 %}
8287 
8288 // Shift Left by 8-bit immediate
8289 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8290 %{
8291   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8292   effect(KILL cr);
8293 
8294   format %{ "sall    $dst, $shift" %}
8295   opcode(0xC1, 0x4); /* C1 /4 ib */
8296   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8297   ins_pipe(ialu_mem_imm);
8298 %}
8299 
8300 // Shift Left by variable
8301 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8302 %{
8303   match(Set dst (LShiftI dst shift));
8304   effect(KILL cr);
8305 
8306   format %{ "sall    $dst, $shift" %}
8307   opcode(0xD3, 0x4); /* D3 /4 */
8308   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8309   ins_pipe(ialu_reg_reg);
8310 %}
8311 
8312 // Shift Left by variable
8313 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8314 %{
8315   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8316   effect(KILL cr);
8317 
8318   format %{ "sall    $dst, $shift" %}
8319   opcode(0xD3, 0x4); /* D3 /4 */
8320   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8321   ins_pipe(ialu_mem_reg);
8322 %}
8323 
8324 // Arithmetic shift right by one
8325 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8326 %{
8327   match(Set dst (RShiftI dst shift));
8328   effect(KILL cr);
8329 
8330   format %{ "sarl    $dst, $shift" %}
8331   opcode(0xD1, 0x7); /* D1 /7 */
8332   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8333   ins_pipe(ialu_reg);
8334 %}
8335 
8336 // Arithmetic shift right by one
8337 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8338 %{
8339   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8340   effect(KILL cr);
8341 
8342   format %{ "sarl    $dst, $shift" %}
8343   opcode(0xD1, 0x7); /* D1 /7 */
8344   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8345   ins_pipe(ialu_mem_imm);
8346 %}
8347 
8348 // Arithmetic Shift Right by 8-bit immediate
8349 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8350 %{
8351   match(Set dst (RShiftI dst shift));
8352   effect(KILL cr);
8353 
8354   format %{ "sarl    $dst, $shift" %}
8355   opcode(0xC1, 0x7); /* C1 /7 ib */
8356   ins_encode(reg_opc_imm(dst, shift));
8357   ins_pipe(ialu_mem_imm);
8358 %}
8359 
8360 // Arithmetic Shift Right by 8-bit immediate
8361 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8362 %{
8363   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8364   effect(KILL cr);
8365 
8366   format %{ "sarl    $dst, $shift" %}
8367   opcode(0xC1, 0x7); /* C1 /7 ib */
8368   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8369   ins_pipe(ialu_mem_imm);
8370 %}
8371 
8372 // Arithmetic Shift Right by variable
8373 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8374 %{
8375   match(Set dst (RShiftI dst shift));
8376   effect(KILL cr);
8377 
8378   format %{ "sarl    $dst, $shift" %}
8379   opcode(0xD3, 0x7); /* D3 /7 */
8380   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8381   ins_pipe(ialu_reg_reg);
8382 %}
8383 
8384 // Arithmetic Shift Right by variable
8385 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8386 %{
8387   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8388   effect(KILL cr);
8389 
8390   format %{ "sarl    $dst, $shift" %}
8391   opcode(0xD3, 0x7); /* D3 /7 */
8392   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8393   ins_pipe(ialu_mem_reg);
8394 %}
8395 
8396 // Logical shift right by one
8397 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8398 %{
8399   match(Set dst (URShiftI dst shift));
8400   effect(KILL cr);
8401 
8402   format %{ "shrl    $dst, $shift" %}
8403   opcode(0xD1, 0x5); /* D1 /5 */
8404   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8405   ins_pipe(ialu_reg);
8406 %}
8407 
8408 // Logical shift right by one
8409 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8410 %{
8411   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8412   effect(KILL cr);
8413 
8414   format %{ "shrl    $dst, $shift" %}
8415   opcode(0xD1, 0x5); /* D1 /5 */
8416   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8417   ins_pipe(ialu_mem_imm);
8418 %}
8419 
8420 // Logical Shift Right by 8-bit immediate
8421 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8422 %{
8423   match(Set dst (URShiftI dst shift));
8424   effect(KILL cr);
8425 
8426   format %{ "shrl    $dst, $shift" %}
8427   opcode(0xC1, 0x5); /* C1 /5 ib */
8428   ins_encode(reg_opc_imm(dst, shift));
8429   ins_pipe(ialu_reg);
8430 %}
8431 
8432 // Logical Shift Right by 8-bit immediate
8433 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8434 %{
8435   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8436   effect(KILL cr);
8437 
8438   format %{ "shrl    $dst, $shift" %}
8439   opcode(0xC1, 0x5); /* C1 /5 ib */
8440   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8441   ins_pipe(ialu_mem_imm);
8442 %}
8443 
8444 // Logical Shift Right by variable
8445 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8446 %{
8447   match(Set dst (URShiftI dst shift));
8448   effect(KILL cr);
8449 
8450   format %{ "shrl    $dst, $shift" %}
8451   opcode(0xD3, 0x5); /* D3 /5 */
8452   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8453   ins_pipe(ialu_reg_reg);
8454 %}
8455 
8456 // Logical Shift Right by variable
8457 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8458 %{
8459   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8460   effect(KILL cr);
8461 
8462   format %{ "shrl    $dst, $shift" %}
8463   opcode(0xD3, 0x5); /* D3 /5 */
8464   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8465   ins_pipe(ialu_mem_reg);
8466 %}
8467 
8468 // Long Shift Instructions
8469 // Shift Left by one
8470 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8471 %{
8472   match(Set dst (LShiftL dst shift));
8473   effect(KILL cr);
8474 
8475   format %{ "salq    $dst, $shift" %}
8476   opcode(0xD1, 0x4); /* D1 /4 */
8477   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8478   ins_pipe(ialu_reg);
8479 %}
8480 
8481 // Shift Left by one
8482 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8483 %{
8484   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8485   effect(KILL cr);
8486 
8487   format %{ "salq    $dst, $shift" %}
8488   opcode(0xD1, 0x4); /* D1 /4 */
8489   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8490   ins_pipe(ialu_mem_imm);
8491 %}
8492 
8493 // Shift Left by 8-bit immediate
8494 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8495 %{
8496   match(Set dst (LShiftL dst shift));
8497   effect(KILL cr);
8498 
8499   format %{ "salq    $dst, $shift" %}
8500   opcode(0xC1, 0x4); /* C1 /4 ib */
8501   ins_encode(reg_opc_imm_wide(dst, shift));
8502   ins_pipe(ialu_reg);
8503 %}
8504 
8505 // Shift Left by 8-bit immediate
8506 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8507 %{
8508   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8509   effect(KILL cr);
8510 
8511   format %{ "salq    $dst, $shift" %}
8512   opcode(0xC1, 0x4); /* C1 /4 ib */
8513   ins_encode(REX_mem_wide(dst), OpcP,
8514              RM_opc_mem(secondary, dst), Con8or32(shift));
8515   ins_pipe(ialu_mem_imm);
8516 %}
8517 
8518 // Shift Left by variable
8519 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8520 %{
8521   match(Set dst (LShiftL dst shift));
8522   effect(KILL cr);
8523 
8524   format %{ "salq    $dst, $shift" %}
8525   opcode(0xD3, 0x4); /* D3 /4 */
8526   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8527   ins_pipe(ialu_reg_reg);
8528 %}
8529 
8530 // Shift Left by variable
8531 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8532 %{
8533   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8534   effect(KILL cr);
8535 
8536   format %{ "salq    $dst, $shift" %}
8537   opcode(0xD3, 0x4); /* D3 /4 */
8538   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8539   ins_pipe(ialu_mem_reg);
8540 %}
8541 
8542 // Arithmetic shift right by one
8543 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8544 %{
8545   match(Set dst (RShiftL dst shift));
8546   effect(KILL cr);
8547 
8548   format %{ "sarq    $dst, $shift" %}
8549   opcode(0xD1, 0x7); /* D1 /7 */
8550   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8551   ins_pipe(ialu_reg);
8552 %}
8553 
8554 // Arithmetic shift right by one
8555 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8556 %{
8557   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8558   effect(KILL cr);
8559 
8560   format %{ "sarq    $dst, $shift" %}
8561   opcode(0xD1, 0x7); /* D1 /7 */
8562   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8563   ins_pipe(ialu_mem_imm);
8564 %}
8565 
8566 // Arithmetic Shift Right by 8-bit immediate
8567 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8568 %{
8569   match(Set dst (RShiftL dst shift));
8570   effect(KILL cr);
8571 
8572   format %{ "sarq    $dst, $shift" %}
8573   opcode(0xC1, 0x7); /* C1 /7 ib */
8574   ins_encode(reg_opc_imm_wide(dst, shift));
8575   ins_pipe(ialu_mem_imm);
8576 %}
8577 
8578 // Arithmetic Shift Right by 8-bit immediate
8579 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8580 %{
8581   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8582   effect(KILL cr);
8583 
8584   format %{ "sarq    $dst, $shift" %}
8585   opcode(0xC1, 0x7); /* C1 /7 ib */
8586   ins_encode(REX_mem_wide(dst), OpcP,
8587              RM_opc_mem(secondary, dst), Con8or32(shift));
8588   ins_pipe(ialu_mem_imm);
8589 %}
8590 
8591 // Arithmetic Shift Right by variable
8592 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8593 %{
8594   match(Set dst (RShiftL dst shift));
8595   effect(KILL cr);
8596 
8597   format %{ "sarq    $dst, $shift" %}
8598   opcode(0xD3, 0x7); /* D3 /7 */
8599   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8600   ins_pipe(ialu_reg_reg);
8601 %}
8602 
8603 // Arithmetic Shift Right by variable
8604 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8605 %{
8606   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8607   effect(KILL cr);
8608 
8609   format %{ "sarq    $dst, $shift" %}
8610   opcode(0xD3, 0x7); /* D3 /7 */
8611   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8612   ins_pipe(ialu_mem_reg);
8613 %}
8614 
8615 // Logical shift right by one
8616 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8617 %{
8618   match(Set dst (URShiftL dst shift));
8619   effect(KILL cr);
8620 
8621   format %{ "shrq    $dst, $shift" %}
8622   opcode(0xD1, 0x5); /* D1 /5 */
8623   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8624   ins_pipe(ialu_reg);
8625 %}
8626 
8627 // Logical shift right by one
8628 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8629 %{
8630   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8631   effect(KILL cr);
8632 
8633   format %{ "shrq    $dst, $shift" %}
8634   opcode(0xD1, 0x5); /* D1 /5 */
8635   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8636   ins_pipe(ialu_mem_imm);
8637 %}
8638 
8639 // Logical Shift Right by 8-bit immediate
8640 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8641 %{
8642   match(Set dst (URShiftL dst shift));
8643   effect(KILL cr);
8644 
8645   format %{ "shrq    $dst, $shift" %}
8646   opcode(0xC1, 0x5); /* C1 /5 ib */
8647   ins_encode(reg_opc_imm_wide(dst, shift));
8648   ins_pipe(ialu_reg);
8649 %}
8650 
8651 
8652 // Logical Shift Right by 8-bit immediate
8653 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8654 %{
8655   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8656   effect(KILL cr);
8657 
8658   format %{ "shrq    $dst, $shift" %}
8659   opcode(0xC1, 0x5); /* C1 /5 ib */
8660   ins_encode(REX_mem_wide(dst), OpcP,
8661              RM_opc_mem(secondary, dst), Con8or32(shift));
8662   ins_pipe(ialu_mem_imm);
8663 %}
8664 
8665 // Logical Shift Right by variable
8666 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8667 %{
8668   match(Set dst (URShiftL dst shift));
8669   effect(KILL cr);
8670 
8671   format %{ "shrq    $dst, $shift" %}
8672   opcode(0xD3, 0x5); /* D3 /5 */
8673   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8674   ins_pipe(ialu_reg_reg);
8675 %}
8676 
8677 // Logical Shift Right by variable
8678 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8679 %{
8680   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8681   effect(KILL cr);
8682 
8683   format %{ "shrq    $dst, $shift" %}
8684   opcode(0xD3, 0x5); /* D3 /5 */
8685   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8686   ins_pipe(ialu_mem_reg);
8687 %}
8688 
8689 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8690 // This idiom is used by the compiler for the i2b bytecode.
8691 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8692 %{
8693   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8694 
8695   format %{ "movsbl  $dst, $src\t# i2b" %}
8696   opcode(0x0F, 0xBE);
8697   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8698   ins_pipe(ialu_reg_reg);
8699 %}
8700 
8701 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8702 // This idiom is used by the compiler the i2s bytecode.
8703 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8704 %{
8705   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8706 
8707   format %{ "movswl  $dst, $src\t# i2s" %}
8708   opcode(0x0F, 0xBF);
8709   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8710   ins_pipe(ialu_reg_reg);
8711 %}
8712 
8713 // ROL/ROR instructions
8714 
8715 // ROL expand
8716 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8717   effect(KILL cr, USE_DEF dst);
8718 
8719   format %{ "roll    $dst" %}
8720   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8721   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8722   ins_pipe(ialu_reg);
8723 %}
8724 
8725 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8726   effect(USE_DEF dst, USE shift, KILL cr);
8727 
8728   format %{ "roll    $dst, $shift" %}
8729   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8730   ins_encode( reg_opc_imm(dst, shift) );
8731   ins_pipe(ialu_reg);
8732 %}
8733 
8734 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8735 %{
8736   effect(USE_DEF dst, USE shift, KILL cr);
8737 
8738   format %{ "roll    $dst, $shift" %}
8739   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8740   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8741   ins_pipe(ialu_reg_reg);
8742 %}
8743 // end of ROL expand
8744 
8745 // Rotate Left by one
8746 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8747 %{
8748   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8749 
8750   expand %{
8751     rolI_rReg_imm1(dst, cr);
8752   %}
8753 %}
8754 
8755 // Rotate Left by 8-bit immediate
8756 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8757 %{
8758   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8759   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8760 
8761   expand %{
8762     rolI_rReg_imm8(dst, lshift, cr);
8763   %}
8764 %}
8765 
8766 // Rotate Left by variable
8767 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8768 %{
8769   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8770 
8771   expand %{
8772     rolI_rReg_CL(dst, shift, cr);
8773   %}
8774 %}
8775 
8776 // Rotate Left by variable
8777 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8778 %{
8779   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8780 
8781   expand %{
8782     rolI_rReg_CL(dst, shift, cr);
8783   %}
8784 %}
8785 
8786 // ROR expand
8787 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8788 %{
8789   effect(USE_DEF dst, KILL cr);
8790 
8791   format %{ "rorl    $dst" %}
8792   opcode(0xD1, 0x1); /* D1 /1 */
8793   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8794   ins_pipe(ialu_reg);
8795 %}
8796 
8797 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8798 %{
8799   effect(USE_DEF dst, USE shift, KILL cr);
8800 
8801   format %{ "rorl    $dst, $shift" %}
8802   opcode(0xC1, 0x1); /* C1 /1 ib */
8803   ins_encode(reg_opc_imm(dst, shift));
8804   ins_pipe(ialu_reg);
8805 %}
8806 
8807 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8808 %{
8809   effect(USE_DEF dst, USE shift, KILL cr);
8810 
8811   format %{ "rorl    $dst, $shift" %}
8812   opcode(0xD3, 0x1); /* D3 /1 */
8813   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8814   ins_pipe(ialu_reg_reg);
8815 %}
8816 // end of ROR expand
8817 
8818 // Rotate Right by one
8819 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8820 %{
8821   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8822 
8823   expand %{
8824     rorI_rReg_imm1(dst, cr);
8825   %}
8826 %}
8827 
8828 // Rotate Right by 8-bit immediate
8829 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8830 %{
8831   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8832   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8833 
8834   expand %{
8835     rorI_rReg_imm8(dst, rshift, cr);
8836   %}
8837 %}
8838 
8839 // Rotate Right by variable
8840 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8841 %{
8842   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8843 
8844   expand %{
8845     rorI_rReg_CL(dst, shift, cr);
8846   %}
8847 %}
8848 
8849 // Rotate Right by variable
8850 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8851 %{
8852   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8853 
8854   expand %{
8855     rorI_rReg_CL(dst, shift, cr);
8856   %}
8857 %}
8858 
8859 // for long rotate
8860 // ROL expand
8861 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8862   effect(USE_DEF dst, KILL cr);
8863 
8864   format %{ "rolq    $dst" %}
8865   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8866   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8867   ins_pipe(ialu_reg);
8868 %}
8869 
8870 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8871   effect(USE_DEF dst, USE shift, KILL cr);
8872 
8873   format %{ "rolq    $dst, $shift" %}
8874   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8875   ins_encode( reg_opc_imm_wide(dst, shift) );
8876   ins_pipe(ialu_reg);
8877 %}
8878 
8879 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8880 %{
8881   effect(USE_DEF dst, USE shift, KILL cr);
8882 
8883   format %{ "rolq    $dst, $shift" %}
8884   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8885   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8886   ins_pipe(ialu_reg_reg);
8887 %}
8888 // end of ROL expand
8889 
8890 // Rotate Left by one
8891 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8892 %{
8893   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8894 
8895   expand %{
8896     rolL_rReg_imm1(dst, cr);
8897   %}
8898 %}
8899 
8900 // Rotate Left by 8-bit immediate
8901 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8902 %{
8903   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8904   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8905 
8906   expand %{
8907     rolL_rReg_imm8(dst, lshift, cr);
8908   %}
8909 %}
8910 
8911 // Rotate Left by variable
8912 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8913 %{
8914   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8915 
8916   expand %{
8917     rolL_rReg_CL(dst, shift, cr);
8918   %}
8919 %}
8920 
8921 // Rotate Left by variable
8922 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8923 %{
8924   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8925 
8926   expand %{
8927     rolL_rReg_CL(dst, shift, cr);
8928   %}
8929 %}
8930 
8931 // ROR expand
8932 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8933 %{
8934   effect(USE_DEF dst, KILL cr);
8935 
8936   format %{ "rorq    $dst" %}
8937   opcode(0xD1, 0x1); /* D1 /1 */
8938   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8939   ins_pipe(ialu_reg);
8940 %}
8941 
8942 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8943 %{
8944   effect(USE_DEF dst, USE shift, KILL cr);
8945 
8946   format %{ "rorq    $dst, $shift" %}
8947   opcode(0xC1, 0x1); /* C1 /1 ib */
8948   ins_encode(reg_opc_imm_wide(dst, shift));
8949   ins_pipe(ialu_reg);
8950 %}
8951 
8952 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8953 %{
8954   effect(USE_DEF dst, USE shift, KILL cr);
8955 
8956   format %{ "rorq    $dst, $shift" %}
8957   opcode(0xD3, 0x1); /* D3 /1 */
8958   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8959   ins_pipe(ialu_reg_reg);
8960 %}
8961 // end of ROR expand
8962 
8963 // Rotate Right by one
8964 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8965 %{
8966   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8967 
8968   expand %{
8969     rorL_rReg_imm1(dst, cr);
8970   %}
8971 %}
8972 
8973 // Rotate Right by 8-bit immediate
8974 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8975 %{
8976   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8977   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8978 
8979   expand %{
8980     rorL_rReg_imm8(dst, rshift, cr);
8981   %}
8982 %}
8983 
8984 // Rotate Right by variable
8985 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8986 %{
8987   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8988 
8989   expand %{
8990     rorL_rReg_CL(dst, shift, cr);
8991   %}
8992 %}
8993 
8994 // Rotate Right by variable
8995 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8996 %{
8997   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8998 
8999   expand %{
9000     rorL_rReg_CL(dst, shift, cr);
9001   %}
9002 %}
9003 
9004 // Logical Instructions
9005 
9006 // Integer Logical Instructions
9007 
9008 // And Instructions
9009 // And Register with Register
9010 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9011 %{
9012   match(Set dst (AndI dst src));
9013   effect(KILL cr);
9014 
9015   format %{ "andl    $dst, $src\t# int" %}
9016   opcode(0x23);
9017   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9018   ins_pipe(ialu_reg_reg);
9019 %}
9020 
9021 // And Register with Immediate 255
9022 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9023 %{
9024   match(Set dst (AndI dst src));
9025 
9026   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9027   opcode(0x0F, 0xB6);
9028   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9029   ins_pipe(ialu_reg);
9030 %}
9031 
9032 // And Register with Immediate 255 and promote to long
9033 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9034 %{
9035   match(Set dst (ConvI2L (AndI src mask)));
9036 
9037   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9038   opcode(0x0F, 0xB6);
9039   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9040   ins_pipe(ialu_reg);
9041 %}
9042 
9043 // And Register with Immediate 65535
9044 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9045 %{
9046   match(Set dst (AndI dst src));
9047 
9048   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9049   opcode(0x0F, 0xB7);
9050   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9051   ins_pipe(ialu_reg);
9052 %}
9053 
9054 // And Register with Immediate 65535 and promote to long
9055 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9056 %{
9057   match(Set dst (ConvI2L (AndI src mask)));
9058 
9059   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9060   opcode(0x0F, 0xB7);
9061   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9062   ins_pipe(ialu_reg);
9063 %}
9064 
9065 // And Register with Immediate
9066 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9067 %{
9068   match(Set dst (AndI dst src));
9069   effect(KILL cr);
9070 
9071   format %{ "andl    $dst, $src\t# int" %}
9072   opcode(0x81, 0x04); /* Opcode 81 /4 */
9073   ins_encode(OpcSErm(dst, src), Con8or32(src));
9074   ins_pipe(ialu_reg);
9075 %}
9076 
9077 // And Register with Memory
9078 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9079 %{
9080   match(Set dst (AndI dst (LoadI src)));
9081   effect(KILL cr);
9082 
9083   ins_cost(125);
9084   format %{ "andl    $dst, $src\t# int" %}
9085   opcode(0x23);
9086   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9087   ins_pipe(ialu_reg_mem);
9088 %}
9089 
9090 // And Memory with Register
9091 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9092 %{
9093   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9094   effect(KILL cr);
9095 
9096   ins_cost(150);
9097   format %{ "andl    $dst, $src\t# int" %}
9098   opcode(0x21); /* Opcode 21 /r */
9099   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9100   ins_pipe(ialu_mem_reg);
9101 %}
9102 
9103 // And Memory with Immediate
9104 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9105 %{
9106   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9107   effect(KILL cr);
9108 
9109   ins_cost(125);
9110   format %{ "andl    $dst, $src\t# int" %}
9111   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9112   ins_encode(REX_mem(dst), OpcSE(src),
9113              RM_opc_mem(secondary, dst), Con8or32(src));
9114   ins_pipe(ialu_mem_imm);
9115 %}
9116 
9117 // Or Instructions
9118 // Or Register with Register
9119 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9120 %{
9121   match(Set dst (OrI dst src));
9122   effect(KILL cr);
9123 
9124   format %{ "orl     $dst, $src\t# int" %}
9125   opcode(0x0B);
9126   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9127   ins_pipe(ialu_reg_reg);
9128 %}
9129 
9130 // Or Register with Immediate
9131 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9132 %{
9133   match(Set dst (OrI dst src));
9134   effect(KILL cr);
9135 
9136   format %{ "orl     $dst, $src\t# int" %}
9137   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9138   ins_encode(OpcSErm(dst, src), Con8or32(src));
9139   ins_pipe(ialu_reg);
9140 %}
9141 
9142 // Or Register with Memory
9143 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9144 %{
9145   match(Set dst (OrI dst (LoadI src)));
9146   effect(KILL cr);
9147 
9148   ins_cost(125);
9149   format %{ "orl     $dst, $src\t# int" %}
9150   opcode(0x0B);
9151   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9152   ins_pipe(ialu_reg_mem);
9153 %}
9154 
9155 // Or Memory with Register
9156 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9157 %{
9158   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9159   effect(KILL cr);
9160 
9161   ins_cost(150);
9162   format %{ "orl     $dst, $src\t# int" %}
9163   opcode(0x09); /* Opcode 09 /r */
9164   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9165   ins_pipe(ialu_mem_reg);
9166 %}
9167 
9168 // Or Memory with Immediate
9169 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9170 %{
9171   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9172   effect(KILL cr);
9173 
9174   ins_cost(125);
9175   format %{ "orl     $dst, $src\t# int" %}
9176   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9177   ins_encode(REX_mem(dst), OpcSE(src),
9178              RM_opc_mem(secondary, dst), Con8or32(src));
9179   ins_pipe(ialu_mem_imm);
9180 %}
9181 
9182 // Xor Instructions
9183 // Xor Register with Register
9184 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9185 %{
9186   match(Set dst (XorI dst src));
9187   effect(KILL cr);
9188 
9189   format %{ "xorl    $dst, $src\t# int" %}
9190   opcode(0x33);
9191   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9192   ins_pipe(ialu_reg_reg);
9193 %}
9194 
9195 // Xor Register with Immediate -1
9196 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9197   match(Set dst (XorI dst imm));
9198 
9199   format %{ "not    $dst" %}
9200   ins_encode %{
9201      __ notl($dst$$Register);
9202   %}
9203   ins_pipe(ialu_reg);
9204 %}
9205 
9206 // Xor Register with Immediate
9207 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9208 %{
9209   match(Set dst (XorI dst src));
9210   effect(KILL cr);
9211 
9212   format %{ "xorl    $dst, $src\t# int" %}
9213   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9214   ins_encode(OpcSErm(dst, src), Con8or32(src));
9215   ins_pipe(ialu_reg);
9216 %}
9217 
9218 // Xor Register with Memory
9219 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9220 %{
9221   match(Set dst (XorI dst (LoadI src)));
9222   effect(KILL cr);
9223 
9224   ins_cost(125);
9225   format %{ "xorl    $dst, $src\t# int" %}
9226   opcode(0x33);
9227   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9228   ins_pipe(ialu_reg_mem);
9229 %}
9230 
9231 // Xor Memory with Register
9232 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9233 %{
9234   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9235   effect(KILL cr);
9236 
9237   ins_cost(150);
9238   format %{ "xorl    $dst, $src\t# int" %}
9239   opcode(0x31); /* Opcode 31 /r */
9240   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9241   ins_pipe(ialu_mem_reg);
9242 %}
9243 
9244 // Xor Memory with Immediate
9245 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9246 %{
9247   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9248   effect(KILL cr);
9249 
9250   ins_cost(125);
9251   format %{ "xorl    $dst, $src\t# int" %}
9252   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9253   ins_encode(REX_mem(dst), OpcSE(src),
9254              RM_opc_mem(secondary, dst), Con8or32(src));
9255   ins_pipe(ialu_mem_imm);
9256 %}
9257 
9258 
9259 // Long Logical Instructions
9260 
9261 // And Instructions
9262 // And Register with Register
9263 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9264 %{
9265   match(Set dst (AndL dst src));
9266   effect(KILL cr);
9267 
9268   format %{ "andq    $dst, $src\t# long" %}
9269   opcode(0x23);
9270   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9271   ins_pipe(ialu_reg_reg);
9272 %}
9273 
9274 // And Register with Immediate 255
9275 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9276 %{
9277   match(Set dst (AndL dst src));
9278 
9279   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9280   opcode(0x0F, 0xB6);
9281   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9282   ins_pipe(ialu_reg);
9283 %}
9284 
9285 // And Register with Immediate 65535
9286 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9287 %{
9288   match(Set dst (AndL dst src));
9289 
9290   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9291   opcode(0x0F, 0xB7);
9292   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9293   ins_pipe(ialu_reg);
9294 %}
9295 
9296 // And Register with Immediate
9297 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9298 %{
9299   match(Set dst (AndL dst src));
9300   effect(KILL cr);
9301 
9302   format %{ "andq    $dst, $src\t# long" %}
9303   opcode(0x81, 0x04); /* Opcode 81 /4 */
9304   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9305   ins_pipe(ialu_reg);
9306 %}
9307 
9308 // And Register with Memory
9309 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9310 %{
9311   match(Set dst (AndL dst (LoadL src)));
9312   effect(KILL cr);
9313 
9314   ins_cost(125);
9315   format %{ "andq    $dst, $src\t# long" %}
9316   opcode(0x23);
9317   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9318   ins_pipe(ialu_reg_mem);
9319 %}
9320 
9321 // And Memory with Register
9322 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9323 %{
9324   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9325   effect(KILL cr);
9326 
9327   ins_cost(150);
9328   format %{ "andq    $dst, $src\t# long" %}
9329   opcode(0x21); /* Opcode 21 /r */
9330   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9331   ins_pipe(ialu_mem_reg);
9332 %}
9333 
9334 // And Memory with Immediate
9335 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9336 %{
9337   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9338   effect(KILL cr);
9339 
9340   ins_cost(125);
9341   format %{ "andq    $dst, $src\t# long" %}
9342   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9343   ins_encode(REX_mem_wide(dst), OpcSE(src),
9344              RM_opc_mem(secondary, dst), Con8or32(src));
9345   ins_pipe(ialu_mem_imm);
9346 %}
9347 
9348 // Or Instructions
9349 // Or Register with Register
9350 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9351 %{
9352   match(Set dst (OrL dst src));
9353   effect(KILL cr);
9354 
9355   format %{ "orq     $dst, $src\t# long" %}
9356   opcode(0x0B);
9357   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9358   ins_pipe(ialu_reg_reg);
9359 %}
9360 
9361 // Use any_RegP to match R15 (TLS register) without spilling.
9362 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9363   match(Set dst (OrL dst (CastP2X src)));
9364   effect(KILL cr);
9365 
9366   format %{ "orq     $dst, $src\t# long" %}
9367   opcode(0x0B);
9368   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9369   ins_pipe(ialu_reg_reg);
9370 %}
9371 
9372 
9373 // Or Register with Immediate
9374 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9375 %{
9376   match(Set dst (OrL dst src));
9377   effect(KILL cr);
9378 
9379   format %{ "orq     $dst, $src\t# long" %}
9380   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9381   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9382   ins_pipe(ialu_reg);
9383 %}
9384 
9385 // Or Register with Memory
9386 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9387 %{
9388   match(Set dst (OrL dst (LoadL src)));
9389   effect(KILL cr);
9390 
9391   ins_cost(125);
9392   format %{ "orq     $dst, $src\t# long" %}
9393   opcode(0x0B);
9394   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9395   ins_pipe(ialu_reg_mem);
9396 %}
9397 
9398 // Or Memory with Register
9399 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9400 %{
9401   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9402   effect(KILL cr);
9403 
9404   ins_cost(150);
9405   format %{ "orq     $dst, $src\t# long" %}
9406   opcode(0x09); /* Opcode 09 /r */
9407   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9408   ins_pipe(ialu_mem_reg);
9409 %}
9410 
9411 // Or Memory with Immediate
9412 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9413 %{
9414   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9415   effect(KILL cr);
9416 
9417   ins_cost(125);
9418   format %{ "orq     $dst, $src\t# long" %}
9419   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9420   ins_encode(REX_mem_wide(dst), OpcSE(src),
9421              RM_opc_mem(secondary, dst), Con8or32(src));
9422   ins_pipe(ialu_mem_imm);
9423 %}
9424 
9425 // Xor Instructions
9426 // Xor Register with Register
9427 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9428 %{
9429   match(Set dst (XorL dst src));
9430   effect(KILL cr);
9431 
9432   format %{ "xorq    $dst, $src\t# long" %}
9433   opcode(0x33);
9434   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9435   ins_pipe(ialu_reg_reg);
9436 %}
9437 
9438 // Xor Register with Immediate -1
9439 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9440   match(Set dst (XorL dst imm));
9441 
9442   format %{ "notq   $dst" %}
9443   ins_encode %{
9444      __ notq($dst$$Register);
9445   %}
9446   ins_pipe(ialu_reg);
9447 %}
9448 
9449 // Xor Register with Immediate
9450 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9451 %{
9452   match(Set dst (XorL dst src));
9453   effect(KILL cr);
9454 
9455   format %{ "xorq    $dst, $src\t# long" %}
9456   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9457   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9458   ins_pipe(ialu_reg);
9459 %}
9460 
9461 // Xor Register with Memory
9462 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9463 %{
9464   match(Set dst (XorL dst (LoadL src)));
9465   effect(KILL cr);
9466 
9467   ins_cost(125);
9468   format %{ "xorq    $dst, $src\t# long" %}
9469   opcode(0x33);
9470   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9471   ins_pipe(ialu_reg_mem);
9472 %}
9473 
9474 // Xor Memory with Register
9475 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9476 %{
9477   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9478   effect(KILL cr);
9479 
9480   ins_cost(150);
9481   format %{ "xorq    $dst, $src\t# long" %}
9482   opcode(0x31); /* Opcode 31 /r */
9483   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9484   ins_pipe(ialu_mem_reg);
9485 %}
9486 
9487 // Xor Memory with Immediate
9488 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9489 %{
9490   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9491   effect(KILL cr);
9492 
9493   ins_cost(125);
9494   format %{ "xorq    $dst, $src\t# long" %}
9495   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9496   ins_encode(REX_mem_wide(dst), OpcSE(src),
9497              RM_opc_mem(secondary, dst), Con8or32(src));
9498   ins_pipe(ialu_mem_imm);
9499 %}
9500 
9501 // Convert Int to Boolean
9502 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9503 %{
9504   match(Set dst (Conv2B src));
9505   effect(KILL cr);
9506 
9507   format %{ "testl   $src, $src\t# ci2b\n\t"
9508             "setnz   $dst\n\t"
9509             "movzbl  $dst, $dst" %}
9510   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9511              setNZ_reg(dst),
9512              REX_reg_breg(dst, dst), // movzbl
9513              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9514   ins_pipe(pipe_slow); // XXX
9515 %}
9516 
9517 // Convert Pointer to Boolean
9518 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9519 %{
9520   match(Set dst (Conv2B src));
9521   effect(KILL cr);
9522 
9523   format %{ "testq   $src, $src\t# cp2b\n\t"
9524             "setnz   $dst\n\t"
9525             "movzbl  $dst, $dst" %}
9526   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9527              setNZ_reg(dst),
9528              REX_reg_breg(dst, dst), // movzbl
9529              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9530   ins_pipe(pipe_slow); // XXX
9531 %}
9532 
9533 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9534 %{
9535   match(Set dst (CmpLTMask p q));
9536   effect(KILL cr);
9537 
9538   ins_cost(400);
9539   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9540             "setlt   $dst\n\t"
9541             "movzbl  $dst, $dst\n\t"
9542             "negl    $dst" %}
9543   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9544              setLT_reg(dst),
9545              REX_reg_breg(dst, dst), // movzbl
9546              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9547              neg_reg(dst));
9548   ins_pipe(pipe_slow);
9549 %}
9550 
9551 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9552 %{
9553   match(Set dst (CmpLTMask dst zero));
9554   effect(KILL cr);
9555 
9556   ins_cost(100);
9557   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9558   ins_encode %{
9559   __ sarl($dst$$Register, 31);
9560   %}
9561   ins_pipe(ialu_reg);
9562 %}
9563 
9564 /* Better to save a register than avoid a branch */
9565 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9566 %{
9567   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9568   effect(KILL cr);
9569   ins_cost(300);
9570   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9571             "jge    done\n\t"
9572             "addl   $p,$y\n"
9573             "done:  " %}
9574   ins_encode %{
9575     Register Rp = $p$$Register;
9576     Register Rq = $q$$Register;
9577     Register Ry = $y$$Register;
9578     Label done;
9579     __ subl(Rp, Rq);
9580     __ jccb(Assembler::greaterEqual, done);
9581     __ addl(Rp, Ry);
9582     __ bind(done);
9583   %}
9584   ins_pipe(pipe_cmplt);
9585 %}
9586 
9587 /* Better to save a register than avoid a branch */
9588 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9589 %{
9590   match(Set y (AndI (CmpLTMask p q) y));
9591   effect(KILL cr);
9592 
9593   ins_cost(300);
9594 
9595   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9596             "jlt      done\n\t"
9597             "xorl     $y, $y\n"
9598             "done:  " %}
9599   ins_encode %{
9600     Register Rp = $p$$Register;
9601     Register Rq = $q$$Register;
9602     Register Ry = $y$$Register;
9603     Label done;
9604     __ cmpl(Rp, Rq);
9605     __ jccb(Assembler::less, done);
9606     __ xorl(Ry, Ry);
9607     __ bind(done);
9608   %}
9609   ins_pipe(pipe_cmplt);
9610 %}
9611 
9612 
9613 //---------- FP Instructions------------------------------------------------
9614 
9615 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9616 %{
9617   match(Set cr (CmpF src1 src2));
9618 
9619   ins_cost(145);
9620   format %{ "ucomiss $src1, $src2\n\t"
9621             "jnp,s   exit\n\t"
9622             "pushfq\t# saw NaN, set CF\n\t"
9623             "andq    [rsp], #0xffffff2b\n\t"
9624             "popfq\n"
9625     "exit:" %}
9626   ins_encode %{
9627     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9628     emit_cmpfp_fixup(_masm);
9629   %}
9630   ins_pipe(pipe_slow);
9631 %}
9632 
9633 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9634   match(Set cr (CmpF src1 src2));
9635 
9636   ins_cost(100);
9637   format %{ "ucomiss $src1, $src2" %}
9638   ins_encode %{
9639     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9640   %}
9641   ins_pipe(pipe_slow);
9642 %}
9643 
9644 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9645 %{
9646   match(Set cr (CmpF src1 (LoadF src2)));
9647 
9648   ins_cost(145);
9649   format %{ "ucomiss $src1, $src2\n\t"
9650             "jnp,s   exit\n\t"
9651             "pushfq\t# saw NaN, set CF\n\t"
9652             "andq    [rsp], #0xffffff2b\n\t"
9653             "popfq\n"
9654     "exit:" %}
9655   ins_encode %{
9656     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9657     emit_cmpfp_fixup(_masm);
9658   %}
9659   ins_pipe(pipe_slow);
9660 %}
9661 
9662 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9663   match(Set cr (CmpF src1 (LoadF src2)));
9664 
9665   ins_cost(100);
9666   format %{ "ucomiss $src1, $src2" %}
9667   ins_encode %{
9668     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9669   %}
9670   ins_pipe(pipe_slow);
9671 %}
9672 
9673 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9674   match(Set cr (CmpF src con));
9675 
9676   ins_cost(145);
9677   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9678             "jnp,s   exit\n\t"
9679             "pushfq\t# saw NaN, set CF\n\t"
9680             "andq    [rsp], #0xffffff2b\n\t"
9681             "popfq\n"
9682     "exit:" %}
9683   ins_encode %{
9684     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9685     emit_cmpfp_fixup(_masm);
9686   %}
9687   ins_pipe(pipe_slow);
9688 %}
9689 
9690 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9691   match(Set cr (CmpF src con));
9692   ins_cost(100);
9693   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9694   ins_encode %{
9695     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9696   %}
9697   ins_pipe(pipe_slow);
9698 %}
9699 
9700 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9701 %{
9702   match(Set cr (CmpD src1 src2));
9703 
9704   ins_cost(145);
9705   format %{ "ucomisd $src1, $src2\n\t"
9706             "jnp,s   exit\n\t"
9707             "pushfq\t# saw NaN, set CF\n\t"
9708             "andq    [rsp], #0xffffff2b\n\t"
9709             "popfq\n"
9710     "exit:" %}
9711   ins_encode %{
9712     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9713     emit_cmpfp_fixup(_masm);
9714   %}
9715   ins_pipe(pipe_slow);
9716 %}
9717 
9718 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9719   match(Set cr (CmpD src1 src2));
9720 
9721   ins_cost(100);
9722   format %{ "ucomisd $src1, $src2 test" %}
9723   ins_encode %{
9724     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9725   %}
9726   ins_pipe(pipe_slow);
9727 %}
9728 
9729 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9730 %{
9731   match(Set cr (CmpD src1 (LoadD src2)));
9732 
9733   ins_cost(145);
9734   format %{ "ucomisd $src1, $src2\n\t"
9735             "jnp,s   exit\n\t"
9736             "pushfq\t# saw NaN, set CF\n\t"
9737             "andq    [rsp], #0xffffff2b\n\t"
9738             "popfq\n"
9739     "exit:" %}
9740   ins_encode %{
9741     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9742     emit_cmpfp_fixup(_masm);
9743   %}
9744   ins_pipe(pipe_slow);
9745 %}
9746 
9747 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9748   match(Set cr (CmpD src1 (LoadD src2)));
9749 
9750   ins_cost(100);
9751   format %{ "ucomisd $src1, $src2" %}
9752   ins_encode %{
9753     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9754   %}
9755   ins_pipe(pipe_slow);
9756 %}
9757 
9758 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9759   match(Set cr (CmpD src con));
9760 
9761   ins_cost(145);
9762   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9763             "jnp,s   exit\n\t"
9764             "pushfq\t# saw NaN, set CF\n\t"
9765             "andq    [rsp], #0xffffff2b\n\t"
9766             "popfq\n"
9767     "exit:" %}
9768   ins_encode %{
9769     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9770     emit_cmpfp_fixup(_masm);
9771   %}
9772   ins_pipe(pipe_slow);
9773 %}
9774 
9775 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9776   match(Set cr (CmpD src con));
9777   ins_cost(100);
9778   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9779   ins_encode %{
9780     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9781   %}
9782   ins_pipe(pipe_slow);
9783 %}
9784 
9785 // Compare into -1,0,1
9786 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9787 %{
9788   match(Set dst (CmpF3 src1 src2));
9789   effect(KILL cr);
9790 
9791   ins_cost(275);
9792   format %{ "ucomiss $src1, $src2\n\t"
9793             "movl    $dst, #-1\n\t"
9794             "jp,s    done\n\t"
9795             "jb,s    done\n\t"
9796             "setne   $dst\n\t"
9797             "movzbl  $dst, $dst\n"
9798     "done:" %}
9799   ins_encode %{
9800     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9801     emit_cmpfp3(_masm, $dst$$Register);
9802   %}
9803   ins_pipe(pipe_slow);
9804 %}
9805 
9806 // Compare into -1,0,1
9807 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9808 %{
9809   match(Set dst (CmpF3 src1 (LoadF src2)));
9810   effect(KILL cr);
9811 
9812   ins_cost(275);
9813   format %{ "ucomiss $src1, $src2\n\t"
9814             "movl    $dst, #-1\n\t"
9815             "jp,s    done\n\t"
9816             "jb,s    done\n\t"
9817             "setne   $dst\n\t"
9818             "movzbl  $dst, $dst\n"
9819     "done:" %}
9820   ins_encode %{
9821     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9822     emit_cmpfp3(_masm, $dst$$Register);
9823   %}
9824   ins_pipe(pipe_slow);
9825 %}
9826 
9827 // Compare into -1,0,1
9828 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9829   match(Set dst (CmpF3 src con));
9830   effect(KILL cr);
9831 
9832   ins_cost(275);
9833   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9834             "movl    $dst, #-1\n\t"
9835             "jp,s    done\n\t"
9836             "jb,s    done\n\t"
9837             "setne   $dst\n\t"
9838             "movzbl  $dst, $dst\n"
9839     "done:" %}
9840   ins_encode %{
9841     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9842     emit_cmpfp3(_masm, $dst$$Register);
9843   %}
9844   ins_pipe(pipe_slow);
9845 %}
9846 
9847 // Compare into -1,0,1
9848 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9849 %{
9850   match(Set dst (CmpD3 src1 src2));
9851   effect(KILL cr);
9852 
9853   ins_cost(275);
9854   format %{ "ucomisd $src1, $src2\n\t"
9855             "movl    $dst, #-1\n\t"
9856             "jp,s    done\n\t"
9857             "jb,s    done\n\t"
9858             "setne   $dst\n\t"
9859             "movzbl  $dst, $dst\n"
9860     "done:" %}
9861   ins_encode %{
9862     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9863     emit_cmpfp3(_masm, $dst$$Register);
9864   %}
9865   ins_pipe(pipe_slow);
9866 %}
9867 
9868 // Compare into -1,0,1
9869 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9870 %{
9871   match(Set dst (CmpD3 src1 (LoadD src2)));
9872   effect(KILL cr);
9873 
9874   ins_cost(275);
9875   format %{ "ucomisd $src1, $src2\n\t"
9876             "movl    $dst, #-1\n\t"
9877             "jp,s    done\n\t"
9878             "jb,s    done\n\t"
9879             "setne   $dst\n\t"
9880             "movzbl  $dst, $dst\n"
9881     "done:" %}
9882   ins_encode %{
9883     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9884     emit_cmpfp3(_masm, $dst$$Register);
9885   %}
9886   ins_pipe(pipe_slow);
9887 %}
9888 
9889 // Compare into -1,0,1
9890 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9891   match(Set dst (CmpD3 src con));
9892   effect(KILL cr);
9893 
9894   ins_cost(275);
9895   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9896             "movl    $dst, #-1\n\t"
9897             "jp,s    done\n\t"
9898             "jb,s    done\n\t"
9899             "setne   $dst\n\t"
9900             "movzbl  $dst, $dst\n"
9901     "done:" %}
9902   ins_encode %{
9903     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9904     emit_cmpfp3(_masm, $dst$$Register);
9905   %}
9906   ins_pipe(pipe_slow);
9907 %}
9908 
9909 // -----------Trig and Trancendental Instructions------------------------------
9910 instruct cosD_reg(regD dst) %{
9911   match(Set dst (CosD dst));
9912 
9913   format %{ "dcos   $dst\n\t" %}
9914   opcode(0xD9, 0xFF);
9915   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9916   ins_pipe( pipe_slow );
9917 %}
9918 
9919 instruct sinD_reg(regD dst) %{
9920   match(Set dst (SinD dst));
9921 
9922   format %{ "dsin   $dst\n\t" %}
9923   opcode(0xD9, 0xFE);
9924   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9925   ins_pipe( pipe_slow );
9926 %}
9927 
9928 instruct tanD_reg(regD dst) %{
9929   match(Set dst (TanD dst));
9930 
9931   format %{ "dtan   $dst\n\t" %}
9932   ins_encode( Push_SrcXD(dst),
9933               Opcode(0xD9), Opcode(0xF2),   //fptan
9934               Opcode(0xDD), Opcode(0xD8),   //fstp st
9935               Push_ResultXD(dst) );
9936   ins_pipe( pipe_slow );
9937 %}
9938 
9939 instruct log10D_reg(regD dst) %{
9940   // The source and result Double operands in XMM registers
9941   match(Set dst (Log10D dst));
9942   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9943   // fyl2x        ; compute log_10(2) * log_2(x)
9944   format %{ "fldlg2\t\t\t#Log10\n\t"
9945             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9946          %}
9947    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9948               Push_SrcXD(dst),
9949               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9950               Push_ResultXD(dst));
9951 
9952   ins_pipe( pipe_slow );
9953 %}
9954 
9955 instruct logD_reg(regD dst) %{
9956   // The source and result Double operands in XMM registers
9957   match(Set dst (LogD dst));
9958   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9959   // fyl2x        ; compute log_e(2) * log_2(x)
9960   format %{ "fldln2\t\t\t#Log_e\n\t"
9961             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9962          %}
9963   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9964               Push_SrcXD(dst),
9965               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9966               Push_ResultXD(dst));
9967   ins_pipe( pipe_slow );
9968 %}
9969 
9970 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9971   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9972   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9973   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9974   ins_encode %{
9975     __ subptr(rsp, 8);
9976     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9977     __ fld_d(Address(rsp, 0));
9978     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9979     __ fld_d(Address(rsp, 0));
9980     __ fast_pow();
9981     __ fstp_d(Address(rsp, 0));
9982     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9983     __ addptr(rsp, 8);
9984   %}
9985   ins_pipe( pipe_slow );
9986 %}
9987 
9988 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9989   match(Set dst (ExpD src));
9990   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9991   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9992   ins_encode %{
9993     __ subptr(rsp, 8);
9994     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9995     __ fld_d(Address(rsp, 0));
9996     __ fast_exp();
9997     __ fstp_d(Address(rsp, 0));
9998     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9999     __ addptr(rsp, 8);
10000   %}
10001   ins_pipe( pipe_slow );
10002 %}
10003 
10004 //----------Arithmetic Conversion Instructions---------------------------------
10005 
10006 instruct roundFloat_nop(regF dst)
10007 %{
10008   match(Set dst (RoundFloat dst));
10009 
10010   ins_cost(0);
10011   ins_encode();
10012   ins_pipe(empty);
10013 %}
10014 
10015 instruct roundDouble_nop(regD dst)
10016 %{
10017   match(Set dst (RoundDouble dst));
10018 
10019   ins_cost(0);
10020   ins_encode();
10021   ins_pipe(empty);
10022 %}
10023 
10024 instruct convF2D_reg_reg(regD dst, regF src)
10025 %{
10026   match(Set dst (ConvF2D src));
10027 
10028   format %{ "cvtss2sd $dst, $src" %}
10029   ins_encode %{
10030     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10031   %}
10032   ins_pipe(pipe_slow); // XXX
10033 %}
10034 
10035 instruct convF2D_reg_mem(regD dst, memory src)
10036 %{
10037   match(Set dst (ConvF2D (LoadF src)));
10038 
10039   format %{ "cvtss2sd $dst, $src" %}
10040   ins_encode %{
10041     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10042   %}
10043   ins_pipe(pipe_slow); // XXX
10044 %}
10045 
10046 instruct convD2F_reg_reg(regF dst, regD src)
10047 %{
10048   match(Set dst (ConvD2F src));
10049 
10050   format %{ "cvtsd2ss $dst, $src" %}
10051   ins_encode %{
10052     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10053   %}
10054   ins_pipe(pipe_slow); // XXX
10055 %}
10056 
10057 instruct convD2F_reg_mem(regF dst, memory src)
10058 %{
10059   match(Set dst (ConvD2F (LoadD src)));
10060 
10061   format %{ "cvtsd2ss $dst, $src" %}
10062   ins_encode %{
10063     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10064   %}
10065   ins_pipe(pipe_slow); // XXX
10066 %}
10067 
10068 // XXX do mem variants
10069 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10070 %{
10071   match(Set dst (ConvF2I src));
10072   effect(KILL cr);
10073 
10074   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10075             "cmpl    $dst, #0x80000000\n\t"
10076             "jne,s   done\n\t"
10077             "subq    rsp, #8\n\t"
10078             "movss   [rsp], $src\n\t"
10079             "call    f2i_fixup\n\t"
10080             "popq    $dst\n"
10081     "done:   "%}
10082   ins_encode %{
10083     Label done;
10084     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10085     __ cmpl($dst$$Register, 0x80000000);
10086     __ jccb(Assembler::notEqual, done);
10087     __ subptr(rsp, 8);
10088     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10089     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10090     __ pop($dst$$Register);
10091     __ bind(done);
10092   %}
10093   ins_pipe(pipe_slow);
10094 %}
10095 
10096 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10097 %{
10098   match(Set dst (ConvF2L src));
10099   effect(KILL cr);
10100 
10101   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10102             "cmpq    $dst, [0x8000000000000000]\n\t"
10103             "jne,s   done\n\t"
10104             "subq    rsp, #8\n\t"
10105             "movss   [rsp], $src\n\t"
10106             "call    f2l_fixup\n\t"
10107             "popq    $dst\n"
10108     "done:   "%}
10109   ins_encode %{
10110     Label done;
10111     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10112     __ cmp64($dst$$Register,
10113              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10114     __ jccb(Assembler::notEqual, done);
10115     __ subptr(rsp, 8);
10116     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10117     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10118     __ pop($dst$$Register);
10119     __ bind(done);
10120   %}
10121   ins_pipe(pipe_slow);
10122 %}
10123 
10124 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10125 %{
10126   match(Set dst (ConvD2I src));
10127   effect(KILL cr);
10128 
10129   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10130             "cmpl    $dst, #0x80000000\n\t"
10131             "jne,s   done\n\t"
10132             "subq    rsp, #8\n\t"
10133             "movsd   [rsp], $src\n\t"
10134             "call    d2i_fixup\n\t"
10135             "popq    $dst\n"
10136     "done:   "%}
10137   ins_encode %{
10138     Label done;
10139     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10140     __ cmpl($dst$$Register, 0x80000000);
10141     __ jccb(Assembler::notEqual, done);
10142     __ subptr(rsp, 8);
10143     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10144     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10145     __ pop($dst$$Register);
10146     __ bind(done);
10147   %}
10148   ins_pipe(pipe_slow);
10149 %}
10150 
10151 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10152 %{
10153   match(Set dst (ConvD2L src));
10154   effect(KILL cr);
10155 
10156   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10157             "cmpq    $dst, [0x8000000000000000]\n\t"
10158             "jne,s   done\n\t"
10159             "subq    rsp, #8\n\t"
10160             "movsd   [rsp], $src\n\t"
10161             "call    d2l_fixup\n\t"
10162             "popq    $dst\n"
10163     "done:   "%}
10164   ins_encode %{
10165     Label done;
10166     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10167     __ cmp64($dst$$Register,
10168              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10169     __ jccb(Assembler::notEqual, done);
10170     __ subptr(rsp, 8);
10171     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10172     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10173     __ pop($dst$$Register);
10174     __ bind(done);
10175   %}
10176   ins_pipe(pipe_slow);
10177 %}
10178 
10179 instruct convI2F_reg_reg(regF dst, rRegI src)
10180 %{
10181   predicate(!UseXmmI2F);
10182   match(Set dst (ConvI2F src));
10183 
10184   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10185   ins_encode %{
10186     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10187   %}
10188   ins_pipe(pipe_slow); // XXX
10189 %}
10190 
10191 instruct convI2F_reg_mem(regF dst, memory src)
10192 %{
10193   match(Set dst (ConvI2F (LoadI src)));
10194 
10195   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10196   ins_encode %{
10197     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10198   %}
10199   ins_pipe(pipe_slow); // XXX
10200 %}
10201 
10202 instruct convI2D_reg_reg(regD dst, rRegI src)
10203 %{
10204   predicate(!UseXmmI2D);
10205   match(Set dst (ConvI2D src));
10206 
10207   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10208   ins_encode %{
10209     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10210   %}
10211   ins_pipe(pipe_slow); // XXX
10212 %}
10213 
10214 instruct convI2D_reg_mem(regD dst, memory src)
10215 %{
10216   match(Set dst (ConvI2D (LoadI src)));
10217 
10218   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10219   ins_encode %{
10220     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10221   %}
10222   ins_pipe(pipe_slow); // XXX
10223 %}
10224 
10225 instruct convXI2F_reg(regF dst, rRegI src)
10226 %{
10227   predicate(UseXmmI2F);
10228   match(Set dst (ConvI2F src));
10229 
10230   format %{ "movdl $dst, $src\n\t"
10231             "cvtdq2psl $dst, $dst\t# i2f" %}
10232   ins_encode %{
10233     __ movdl($dst$$XMMRegister, $src$$Register);
10234     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10235   %}
10236   ins_pipe(pipe_slow); // XXX
10237 %}
10238 
10239 instruct convXI2D_reg(regD dst, rRegI src)
10240 %{
10241   predicate(UseXmmI2D);
10242   match(Set dst (ConvI2D src));
10243 
10244   format %{ "movdl $dst, $src\n\t"
10245             "cvtdq2pdl $dst, $dst\t# i2d" %}
10246   ins_encode %{
10247     __ movdl($dst$$XMMRegister, $src$$Register);
10248     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10249   %}
10250   ins_pipe(pipe_slow); // XXX
10251 %}
10252 
10253 instruct convL2F_reg_reg(regF dst, rRegL src)
10254 %{
10255   match(Set dst (ConvL2F src));
10256 
10257   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10258   ins_encode %{
10259     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10260   %}
10261   ins_pipe(pipe_slow); // XXX
10262 %}
10263 
10264 instruct convL2F_reg_mem(regF dst, memory src)
10265 %{
10266   match(Set dst (ConvL2F (LoadL src)));
10267 
10268   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10269   ins_encode %{
10270     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10271   %}
10272   ins_pipe(pipe_slow); // XXX
10273 %}
10274 
10275 instruct convL2D_reg_reg(regD dst, rRegL src)
10276 %{
10277   match(Set dst (ConvL2D src));
10278 
10279   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10280   ins_encode %{
10281     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10282   %}
10283   ins_pipe(pipe_slow); // XXX
10284 %}
10285 
10286 instruct convL2D_reg_mem(regD dst, memory src)
10287 %{
10288   match(Set dst (ConvL2D (LoadL src)));
10289 
10290   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10291   ins_encode %{
10292     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10293   %}
10294   ins_pipe(pipe_slow); // XXX
10295 %}
10296 
10297 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10298 %{
10299   match(Set dst (ConvI2L src));
10300 
10301   ins_cost(125);
10302   format %{ "movslq  $dst, $src\t# i2l" %}
10303   ins_encode %{
10304     __ movslq($dst$$Register, $src$$Register);
10305   %}
10306   ins_pipe(ialu_reg_reg);
10307 %}
10308 
10309 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10310 // %{
10311 //   match(Set dst (ConvI2L src));
10312 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10313 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10314 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10315 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10316 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10317 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10318 
10319 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10320 //   ins_encode(enc_copy(dst, src));
10321 // //   opcode(0x63); // needs REX.W
10322 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10323 //   ins_pipe(ialu_reg_reg);
10324 // %}
10325 
10326 // Zero-extend convert int to long
10327 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10328 %{
10329   match(Set dst (AndL (ConvI2L src) mask));
10330 
10331   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10332   ins_encode %{
10333     if ($dst$$reg != $src$$reg) {
10334       __ movl($dst$$Register, $src$$Register);
10335     }
10336   %}
10337   ins_pipe(ialu_reg_reg);
10338 %}
10339 
10340 // Zero-extend convert int to long
10341 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10342 %{
10343   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10344 
10345   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10346   ins_encode %{
10347     __ movl($dst$$Register, $src$$Address);
10348   %}
10349   ins_pipe(ialu_reg_mem);
10350 %}
10351 
10352 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10353 %{
10354   match(Set dst (AndL src mask));
10355 
10356   format %{ "movl    $dst, $src\t# zero-extend long" %}
10357   ins_encode %{
10358     __ movl($dst$$Register, $src$$Register);
10359   %}
10360   ins_pipe(ialu_reg_reg);
10361 %}
10362 
10363 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10364 %{
10365   match(Set dst (ConvL2I src));
10366 
10367   format %{ "movl    $dst, $src\t# l2i" %}
10368   ins_encode %{
10369     __ movl($dst$$Register, $src$$Register);
10370   %}
10371   ins_pipe(ialu_reg_reg);
10372 %}
10373 
10374 
10375 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10376   match(Set dst (MoveF2I src));
10377   effect(DEF dst, USE src);
10378 
10379   ins_cost(125);
10380   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10381   ins_encode %{
10382     __ movl($dst$$Register, Address(rsp, $src$$disp));
10383   %}
10384   ins_pipe(ialu_reg_mem);
10385 %}
10386 
10387 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10388   match(Set dst (MoveI2F src));
10389   effect(DEF dst, USE src);
10390 
10391   ins_cost(125);
10392   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10393   ins_encode %{
10394     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10395   %}
10396   ins_pipe(pipe_slow);
10397 %}
10398 
10399 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10400   match(Set dst (MoveD2L src));
10401   effect(DEF dst, USE src);
10402 
10403   ins_cost(125);
10404   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10405   ins_encode %{
10406     __ movq($dst$$Register, Address(rsp, $src$$disp));
10407   %}
10408   ins_pipe(ialu_reg_mem);
10409 %}
10410 
10411 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10412   predicate(!UseXmmLoadAndClearUpper);
10413   match(Set dst (MoveL2D src));
10414   effect(DEF dst, USE src);
10415 
10416   ins_cost(125);
10417   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10418   ins_encode %{
10419     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10420   %}
10421   ins_pipe(pipe_slow);
10422 %}
10423 
10424 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10425   predicate(UseXmmLoadAndClearUpper);
10426   match(Set dst (MoveL2D src));
10427   effect(DEF dst, USE src);
10428 
10429   ins_cost(125);
10430   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10431   ins_encode %{
10432     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10433   %}
10434   ins_pipe(pipe_slow);
10435 %}
10436 
10437 
10438 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10439   match(Set dst (MoveF2I src));
10440   effect(DEF dst, USE src);
10441 
10442   ins_cost(95); // XXX
10443   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10444   ins_encode %{
10445     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10446   %}
10447   ins_pipe(pipe_slow);
10448 %}
10449 
10450 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10451   match(Set dst (MoveI2F src));
10452   effect(DEF dst, USE src);
10453 
10454   ins_cost(100);
10455   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10456   ins_encode %{
10457     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10458   %}
10459   ins_pipe( ialu_mem_reg );
10460 %}
10461 
10462 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10463   match(Set dst (MoveD2L src));
10464   effect(DEF dst, USE src);
10465 
10466   ins_cost(95); // XXX
10467   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10468   ins_encode %{
10469     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10470   %}
10471   ins_pipe(pipe_slow);
10472 %}
10473 
10474 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10475   match(Set dst (MoveL2D src));
10476   effect(DEF dst, USE src);
10477 
10478   ins_cost(100);
10479   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10480   ins_encode %{
10481     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10482   %}
10483   ins_pipe(ialu_mem_reg);
10484 %}
10485 
10486 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10487   match(Set dst (MoveF2I src));
10488   effect(DEF dst, USE src);
10489   ins_cost(85);
10490   format %{ "movd    $dst,$src\t# MoveF2I" %}
10491   ins_encode %{
10492     __ movdl($dst$$Register, $src$$XMMRegister);
10493   %}
10494   ins_pipe( pipe_slow );
10495 %}
10496 
10497 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10498   match(Set dst (MoveD2L src));
10499   effect(DEF dst, USE src);
10500   ins_cost(85);
10501   format %{ "movd    $dst,$src\t# MoveD2L" %}
10502   ins_encode %{
10503     __ movdq($dst$$Register, $src$$XMMRegister);
10504   %}
10505   ins_pipe( pipe_slow );
10506 %}
10507 
10508 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10509   match(Set dst (MoveI2F src));
10510   effect(DEF dst, USE src);
10511   ins_cost(100);
10512   format %{ "movd    $dst,$src\t# MoveI2F" %}
10513   ins_encode %{
10514     __ movdl($dst$$XMMRegister, $src$$Register);
10515   %}
10516   ins_pipe( pipe_slow );
10517 %}
10518 
10519 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10520   match(Set dst (MoveL2D src));
10521   effect(DEF dst, USE src);
10522   ins_cost(100);
10523   format %{ "movd    $dst,$src\t# MoveL2D" %}
10524   ins_encode %{
10525      __ movdq($dst$$XMMRegister, $src$$Register);
10526   %}
10527   ins_pipe( pipe_slow );
10528 %}
10529 
10530 
10531 // =======================================================================
10532 // fast clearing of an array
10533 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10534                   rFlagsReg cr)
10535 %{
10536   predicate(!UseFastStosb);
10537   match(Set dummy (ClearArray cnt base));
10538   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10539 
10540   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10541             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10542   ins_encode %{ 
10543     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10544   %}
10545   ins_pipe(pipe_slow);
10546 %}
10547 
10548 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10549                         rFlagsReg cr)
10550 %{
10551   predicate(UseFastStosb);
10552   match(Set dummy (ClearArray cnt base));
10553   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10554   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10555             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10556             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10557   ins_encode %{ 
10558     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10559   %}
10560   ins_pipe( pipe_slow );
10561 %}
10562 
10563 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10564                         rax_RegI result, regD tmp1, rFlagsReg cr)
10565 %{
10566   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10567   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10568 
10569   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10570   ins_encode %{
10571     __ string_compare($str1$$Register, $str2$$Register,
10572                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10573                       $tmp1$$XMMRegister);
10574   %}
10575   ins_pipe( pipe_slow );
10576 %}
10577 
10578 // fast search of substring with known size.
10579 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10580                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10581 %{
10582   predicate(UseSSE42Intrinsics);
10583   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10584   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10585 
10586   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10587   ins_encode %{
10588     int icnt2 = (int)$int_cnt2$$constant;
10589     if (icnt2 >= 8) {
10590       // IndexOf for constant substrings with size >= 8 elements
10591       // which don't need to be loaded through stack.
10592       __ string_indexofC8($str1$$Register, $str2$$Register,
10593                           $cnt1$$Register, $cnt2$$Register,
10594                           icnt2, $result$$Register,
10595                           $vec$$XMMRegister, $tmp$$Register);
10596     } else {
10597       // Small strings are loaded through stack if they cross page boundary.
10598       __ string_indexof($str1$$Register, $str2$$Register,
10599                         $cnt1$$Register, $cnt2$$Register,
10600                         icnt2, $result$$Register,
10601                         $vec$$XMMRegister, $tmp$$Register);
10602     }
10603   %}
10604   ins_pipe( pipe_slow );
10605 %}
10606 
10607 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10608                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10609 %{
10610   predicate(UseSSE42Intrinsics);
10611   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10612   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10613 
10614   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10615   ins_encode %{
10616     __ string_indexof($str1$$Register, $str2$$Register,
10617                       $cnt1$$Register, $cnt2$$Register,
10618                       (-1), $result$$Register,
10619                       $vec$$XMMRegister, $tmp$$Register);
10620   %}
10621   ins_pipe( pipe_slow );
10622 %}
10623 
10624 // fast string equals
10625 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10626                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10627 %{
10628   match(Set result (StrEquals (Binary str1 str2) cnt));
10629   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10630 
10631   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10632   ins_encode %{
10633     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10634                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10635                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10636   %}
10637   ins_pipe( pipe_slow );
10638 %}
10639 
10640 // fast array equals
10641 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10642                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10643 %{
10644   match(Set result (AryEq ary1 ary2));
10645   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10646   //ins_cost(300);
10647 
10648   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10649   ins_encode %{
10650     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10651                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10652                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10653   %}
10654   ins_pipe( pipe_slow );
10655 %}
10656 
10657 // encode char[] to byte[] in ISO_8859_1
10658 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10659                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10660                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10661   match(Set result (EncodeISOArray src (Binary dst len)));
10662   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10663 
10664   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10665   ins_encode %{
10666     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10667                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10668                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10669   %}
10670   ins_pipe( pipe_slow );










































































































































































10671 %}
10672 
10673 
10674 //----------Control Flow Instructions------------------------------------------
10675 // Signed compare Instructions
10676 
10677 // XXX more variants!!
10678 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10679 %{
10680   match(Set cr (CmpI op1 op2));
10681   effect(DEF cr, USE op1, USE op2);
10682 
10683   format %{ "cmpl    $op1, $op2" %}
10684   opcode(0x3B);  /* Opcode 3B /r */
10685   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10686   ins_pipe(ialu_cr_reg_reg);
10687 %}
10688 
10689 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10690 %{
10691   match(Set cr (CmpI op1 op2));
10692 
10693   format %{ "cmpl    $op1, $op2" %}
10694   opcode(0x81, 0x07); /* Opcode 81 /7 */
10695   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10696   ins_pipe(ialu_cr_reg_imm);
10697 %}
10698 
10699 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10700 %{
10701   match(Set cr (CmpI op1 (LoadI op2)));
10702 
10703   ins_cost(500); // XXX
10704   format %{ "cmpl    $op1, $op2" %}
10705   opcode(0x3B); /* Opcode 3B /r */
10706   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10707   ins_pipe(ialu_cr_reg_mem);
10708 %}
10709 
10710 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10711 %{
10712   match(Set cr (CmpI src zero));
10713 
10714   format %{ "testl   $src, $src" %}
10715   opcode(0x85);
10716   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10717   ins_pipe(ialu_cr_reg_imm);
10718 %}
10719 
10720 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10721 %{
10722   match(Set cr (CmpI (AndI src con) zero));
10723 
10724   format %{ "testl   $src, $con" %}
10725   opcode(0xF7, 0x00);
10726   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10727   ins_pipe(ialu_cr_reg_imm);
10728 %}
10729 
10730 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10731 %{
10732   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10733 
10734   format %{ "testl   $src, $mem" %}
10735   opcode(0x85);
10736   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10737   ins_pipe(ialu_cr_reg_mem);
10738 %}
10739 
10740 // Unsigned compare Instructions; really, same as signed except they
10741 // produce an rFlagsRegU instead of rFlagsReg.
10742 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10743 %{
10744   match(Set cr (CmpU op1 op2));
10745 
10746   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10747   opcode(0x3B); /* Opcode 3B /r */
10748   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10749   ins_pipe(ialu_cr_reg_reg);
10750 %}
10751 
10752 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10753 %{
10754   match(Set cr (CmpU op1 op2));
10755 
10756   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10757   opcode(0x81,0x07); /* Opcode 81 /7 */
10758   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10759   ins_pipe(ialu_cr_reg_imm);
10760 %}
10761 
10762 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10763 %{
10764   match(Set cr (CmpU op1 (LoadI op2)));
10765 
10766   ins_cost(500); // XXX
10767   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10768   opcode(0x3B); /* Opcode 3B /r */
10769   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10770   ins_pipe(ialu_cr_reg_mem);
10771 %}
10772 
10773 // // // Cisc-spilled version of cmpU_rReg
10774 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10775 // //%{
10776 // //  match(Set cr (CmpU (LoadI op1) op2));
10777 // //
10778 // //  format %{ "CMPu   $op1,$op2" %}
10779 // //  ins_cost(500);
10780 // //  opcode(0x39);  /* Opcode 39 /r */
10781 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10782 // //%}
10783 
10784 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10785 %{
10786   match(Set cr (CmpU src zero));
10787 
10788   format %{ "testl  $src, $src\t# unsigned" %}
10789   opcode(0x85);
10790   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10791   ins_pipe(ialu_cr_reg_imm);
10792 %}
10793 
10794 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10795 %{
10796   match(Set cr (CmpP op1 op2));
10797 
10798   format %{ "cmpq    $op1, $op2\t# ptr" %}
10799   opcode(0x3B); /* Opcode 3B /r */
10800   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10801   ins_pipe(ialu_cr_reg_reg);
10802 %}
10803 
10804 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10805 %{
10806   match(Set cr (CmpP op1 (LoadP op2)));
10807 
10808   ins_cost(500); // XXX
10809   format %{ "cmpq    $op1, $op2\t# ptr" %}
10810   opcode(0x3B); /* Opcode 3B /r */
10811   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10812   ins_pipe(ialu_cr_reg_mem);
10813 %}
10814 
10815 // // // Cisc-spilled version of cmpP_rReg
10816 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10817 // //%{
10818 // //  match(Set cr (CmpP (LoadP op1) op2));
10819 // //
10820 // //  format %{ "CMPu   $op1,$op2" %}
10821 // //  ins_cost(500);
10822 // //  opcode(0x39);  /* Opcode 39 /r */
10823 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10824 // //%}
10825 
10826 // XXX this is generalized by compP_rReg_mem???
10827 // Compare raw pointer (used in out-of-heap check).
10828 // Only works because non-oop pointers must be raw pointers
10829 // and raw pointers have no anti-dependencies.
10830 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10831 %{
10832   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10833   match(Set cr (CmpP op1 (LoadP op2)));
10834 
10835   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10836   opcode(0x3B); /* Opcode 3B /r */
10837   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10838   ins_pipe(ialu_cr_reg_mem);
10839 %}
10840 
10841 // This will generate a signed flags result. This should be OK since
10842 // any compare to a zero should be eq/neq.
10843 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10844 %{
10845   match(Set cr (CmpP src zero));
10846 
10847   format %{ "testq   $src, $src\t# ptr" %}
10848   opcode(0x85);
10849   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10850   ins_pipe(ialu_cr_reg_imm);
10851 %}
10852 
10853 // This will generate a signed flags result. This should be OK since
10854 // any compare to a zero should be eq/neq.
10855 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10856 %{
10857   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10858   match(Set cr (CmpP (LoadP op) zero));
10859 
10860   ins_cost(500); // XXX
10861   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10862   opcode(0xF7); /* Opcode F7 /0 */
10863   ins_encode(REX_mem_wide(op),
10864              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10865   ins_pipe(ialu_cr_reg_imm);
10866 %}
10867 
10868 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10869 %{
10870   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10871   match(Set cr (CmpP (LoadP mem) zero));
10872 
10873   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10874   ins_encode %{
10875     __ cmpq(r12, $mem$$Address);
10876   %}
10877   ins_pipe(ialu_cr_reg_mem);
10878 %}
10879 
10880 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10881 %{
10882   match(Set cr (CmpN op1 op2));
10883 
10884   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10885   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10886   ins_pipe(ialu_cr_reg_reg);
10887 %}
10888 
10889 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10890 %{
10891   match(Set cr (CmpN src (LoadN mem)));
10892 
10893   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10894   ins_encode %{
10895     __ cmpl($src$$Register, $mem$$Address);
10896   %}
10897   ins_pipe(ialu_cr_reg_mem);
10898 %}
10899 
10900 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10901   match(Set cr (CmpN op1 op2));
10902 
10903   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10904   ins_encode %{
10905     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10906   %}
10907   ins_pipe(ialu_cr_reg_imm);
10908 %}
10909 
10910 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10911 %{
10912   match(Set cr (CmpN src (LoadN mem)));
10913 
10914   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10915   ins_encode %{
10916     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10917   %}
10918   ins_pipe(ialu_cr_reg_mem);
10919 %}
10920 
10921 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
10922   match(Set cr (CmpN op1 op2));
10923 
10924   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
10925   ins_encode %{
10926     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
10927   %}
10928   ins_pipe(ialu_cr_reg_imm);
10929 %}
10930 
10931 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
10932 %{
10933   match(Set cr (CmpN src (LoadNKlass mem)));
10934 
10935   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
10936   ins_encode %{
10937     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
10938   %}
10939   ins_pipe(ialu_cr_reg_mem);
10940 %}
10941 
10942 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
10943   match(Set cr (CmpN src zero));
10944 
10945   format %{ "testl   $src, $src\t# compressed ptr" %}
10946   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
10947   ins_pipe(ialu_cr_reg_imm);
10948 %}
10949 
10950 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
10951 %{
10952   predicate(Universe::narrow_oop_base() != NULL);
10953   match(Set cr (CmpN (LoadN mem) zero));
10954 
10955   ins_cost(500); // XXX
10956   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
10957   ins_encode %{
10958     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
10959   %}
10960   ins_pipe(ialu_cr_reg_mem);
10961 %}
10962 
10963 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
10964 %{
10965   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
10966   match(Set cr (CmpN (LoadN mem) zero));
10967 
10968   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
10969   ins_encode %{
10970     __ cmpl(r12, $mem$$Address);
10971   %}
10972   ins_pipe(ialu_cr_reg_mem);
10973 %}
10974 
10975 // Yanked all unsigned pointer compare operations.
10976 // Pointer compares are done with CmpP which is already unsigned.
10977 
10978 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10979 %{
10980   match(Set cr (CmpL op1 op2));
10981 
10982   format %{ "cmpq    $op1, $op2" %}
10983   opcode(0x3B);  /* Opcode 3B /r */
10984   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10985   ins_pipe(ialu_cr_reg_reg);
10986 %}
10987 
10988 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10989 %{
10990   match(Set cr (CmpL op1 op2));
10991 
10992   format %{ "cmpq    $op1, $op2" %}
10993   opcode(0x81, 0x07); /* Opcode 81 /7 */
10994   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
10995   ins_pipe(ialu_cr_reg_imm);
10996 %}
10997 
10998 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
10999 %{
11000   match(Set cr (CmpL op1 (LoadL op2)));
11001 
11002   format %{ "cmpq    $op1, $op2" %}
11003   opcode(0x3B); /* Opcode 3B /r */
11004   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11005   ins_pipe(ialu_cr_reg_mem);
11006 %}
11007 
11008 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11009 %{
11010   match(Set cr (CmpL src zero));
11011 
11012   format %{ "testq   $src, $src" %}
11013   opcode(0x85);
11014   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11015   ins_pipe(ialu_cr_reg_imm);
11016 %}
11017 
11018 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11019 %{
11020   match(Set cr (CmpL (AndL src con) zero));
11021 
11022   format %{ "testq   $src, $con\t# long" %}
11023   opcode(0xF7, 0x00);
11024   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11025   ins_pipe(ialu_cr_reg_imm);
11026 %}
11027 
11028 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11029 %{
11030   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11031 
11032   format %{ "testq   $src, $mem" %}
11033   opcode(0x85);
11034   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11035   ins_pipe(ialu_cr_reg_mem);
11036 %}
11037 
11038 // Manifest a CmpL result in an integer register.  Very painful.
11039 // This is the test to avoid.
11040 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11041 %{
11042   match(Set dst (CmpL3 src1 src2));
11043   effect(KILL flags);
11044 
11045   ins_cost(275); // XXX
11046   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11047             "movl    $dst, -1\n\t"
11048             "jl,s    done\n\t"
11049             "setne   $dst\n\t"
11050             "movzbl  $dst, $dst\n\t"
11051     "done:" %}
11052   ins_encode(cmpl3_flag(src1, src2, dst));
11053   ins_pipe(pipe_slow);
11054 %}
11055 
11056 //----------Max and Min--------------------------------------------------------
11057 // Min Instructions
11058 
11059 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11060 %{
11061   effect(USE_DEF dst, USE src, USE cr);
11062 
11063   format %{ "cmovlgt $dst, $src\t# min" %}
11064   opcode(0x0F, 0x4F);
11065   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11066   ins_pipe(pipe_cmov_reg);
11067 %}
11068 
11069 
11070 instruct minI_rReg(rRegI dst, rRegI src)
11071 %{
11072   match(Set dst (MinI dst src));
11073 
11074   ins_cost(200);
11075   expand %{
11076     rFlagsReg cr;
11077     compI_rReg(cr, dst, src);
11078     cmovI_reg_g(dst, src, cr);
11079   %}
11080 %}
11081 
11082 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11083 %{
11084   effect(USE_DEF dst, USE src, USE cr);
11085 
11086   format %{ "cmovllt $dst, $src\t# max" %}
11087   opcode(0x0F, 0x4C);
11088   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11089   ins_pipe(pipe_cmov_reg);
11090 %}
11091 
11092 
11093 instruct maxI_rReg(rRegI dst, rRegI src)
11094 %{
11095   match(Set dst (MaxI dst src));
11096 
11097   ins_cost(200);
11098   expand %{
11099     rFlagsReg cr;
11100     compI_rReg(cr, dst, src);
11101     cmovI_reg_l(dst, src, cr);
11102   %}
11103 %}
11104 
11105 // ============================================================================
11106 // Branch Instructions
11107 
11108 // Jump Direct - Label defines a relative address from JMP+1
11109 instruct jmpDir(label labl)
11110 %{
11111   match(Goto);
11112   effect(USE labl);
11113 
11114   ins_cost(300);
11115   format %{ "jmp     $labl" %}
11116   size(5);
11117   ins_encode %{
11118     Label* L = $labl$$label;
11119     __ jmp(*L, false); // Always long jump
11120   %}
11121   ins_pipe(pipe_jmp);
11122 %}
11123 
11124 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11125 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11126 %{
11127   match(If cop cr);
11128   effect(USE labl);
11129 
11130   ins_cost(300);
11131   format %{ "j$cop     $labl" %}
11132   size(6);
11133   ins_encode %{
11134     Label* L = $labl$$label;
11135     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11136   %}
11137   ins_pipe(pipe_jcc);
11138 %}
11139 
11140 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11141 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11142 %{
11143   match(CountedLoopEnd cop cr);
11144   effect(USE labl);
11145 
11146   ins_cost(300);
11147   format %{ "j$cop     $labl\t# loop end" %}
11148   size(6);
11149   ins_encode %{
11150     Label* L = $labl$$label;
11151     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11152   %}
11153   ins_pipe(pipe_jcc);
11154 %}
11155 
11156 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11157 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11158   match(CountedLoopEnd cop cmp);
11159   effect(USE labl);
11160 
11161   ins_cost(300);
11162   format %{ "j$cop,u   $labl\t# loop end" %}
11163   size(6);
11164   ins_encode %{
11165     Label* L = $labl$$label;
11166     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11167   %}
11168   ins_pipe(pipe_jcc);
11169 %}
11170 
11171 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11172   match(CountedLoopEnd cop cmp);
11173   effect(USE labl);
11174 
11175   ins_cost(200);
11176   format %{ "j$cop,u   $labl\t# loop end" %}
11177   size(6);
11178   ins_encode %{
11179     Label* L = $labl$$label;
11180     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11181   %}
11182   ins_pipe(pipe_jcc);
11183 %}
11184 
11185 // Jump Direct Conditional - using unsigned comparison
11186 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11187   match(If cop cmp);
11188   effect(USE labl);
11189 
11190   ins_cost(300);
11191   format %{ "j$cop,u  $labl" %}
11192   size(6);
11193   ins_encode %{
11194     Label* L = $labl$$label;
11195     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11196   %}
11197   ins_pipe(pipe_jcc);
11198 %}
11199 
11200 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11201   match(If cop cmp);
11202   effect(USE labl);
11203 
11204   ins_cost(200);
11205   format %{ "j$cop,u  $labl" %}
11206   size(6);
11207   ins_encode %{
11208     Label* L = $labl$$label;
11209     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11210   %}
11211   ins_pipe(pipe_jcc);
11212 %}
11213 
11214 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11215   match(If cop cmp);
11216   effect(USE labl);
11217 
11218   ins_cost(200);
11219   format %{ $$template
11220     if ($cop$$cmpcode == Assembler::notEqual) {
11221       $$emit$$"jp,u   $labl\n\t"
11222       $$emit$$"j$cop,u   $labl"
11223     } else {
11224       $$emit$$"jp,u   done\n\t"
11225       $$emit$$"j$cop,u   $labl\n\t"
11226       $$emit$$"done:"
11227     }
11228   %}
11229   ins_encode %{
11230     Label* l = $labl$$label;
11231     if ($cop$$cmpcode == Assembler::notEqual) {
11232       __ jcc(Assembler::parity, *l, false);
11233       __ jcc(Assembler::notEqual, *l, false);
11234     } else if ($cop$$cmpcode == Assembler::equal) {
11235       Label done;
11236       __ jccb(Assembler::parity, done);
11237       __ jcc(Assembler::equal, *l, false);
11238       __ bind(done);
11239     } else {
11240        ShouldNotReachHere();
11241     }
11242   %}
11243   ins_pipe(pipe_jcc);
11244 %}
11245 
11246 // ============================================================================
11247 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11248 // superklass array for an instance of the superklass.  Set a hidden
11249 // internal cache on a hit (cache is checked with exposed code in
11250 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11251 // encoding ALSO sets flags.
11252 
11253 instruct partialSubtypeCheck(rdi_RegP result,
11254                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11255                              rFlagsReg cr)
11256 %{
11257   match(Set result (PartialSubtypeCheck sub super));
11258   effect(KILL rcx, KILL cr);
11259 
11260   ins_cost(1100);  // slightly larger than the next version
11261   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11262             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11263             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11264             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11265             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11266             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11267             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11268     "miss:\t" %}
11269 
11270   opcode(0x1); // Force a XOR of RDI
11271   ins_encode(enc_PartialSubtypeCheck());
11272   ins_pipe(pipe_slow);
11273 %}
11274 
11275 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11276                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11277                                      immP0 zero,
11278                                      rdi_RegP result)
11279 %{
11280   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11281   effect(KILL rcx, KILL result);
11282 
11283   ins_cost(1000);
11284   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11285             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11286             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11287             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11288             "jne,s   miss\t\t# Missed: flags nz\n\t"
11289             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11290     "miss:\t" %}
11291 
11292   opcode(0x0); // No need to XOR RDI
11293   ins_encode(enc_PartialSubtypeCheck());
11294   ins_pipe(pipe_slow);
11295 %}
11296 
11297 // ============================================================================
11298 // Branch Instructions -- short offset versions
11299 //
11300 // These instructions are used to replace jumps of a long offset (the default
11301 // match) with jumps of a shorter offset.  These instructions are all tagged
11302 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11303 // match rules in general matching.  Instead, the ADLC generates a conversion
11304 // method in the MachNode which can be used to do in-place replacement of the
11305 // long variant with the shorter variant.  The compiler will determine if a
11306 // branch can be taken by the is_short_branch_offset() predicate in the machine
11307 // specific code section of the file.
11308 
11309 // Jump Direct - Label defines a relative address from JMP+1
11310 instruct jmpDir_short(label labl) %{
11311   match(Goto);
11312   effect(USE labl);
11313 
11314   ins_cost(300);
11315   format %{ "jmp,s   $labl" %}
11316   size(2);
11317   ins_encode %{
11318     Label* L = $labl$$label;
11319     __ jmpb(*L);
11320   %}
11321   ins_pipe(pipe_jmp);
11322   ins_short_branch(1);
11323 %}
11324 
11325 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11326 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11327   match(If cop cr);
11328   effect(USE labl);
11329 
11330   ins_cost(300);
11331   format %{ "j$cop,s   $labl" %}
11332   size(2);
11333   ins_encode %{
11334     Label* L = $labl$$label;
11335     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11336   %}
11337   ins_pipe(pipe_jcc);
11338   ins_short_branch(1);
11339 %}
11340 
11341 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11342 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11343   match(CountedLoopEnd cop cr);
11344   effect(USE labl);
11345 
11346   ins_cost(300);
11347   format %{ "j$cop,s   $labl\t# loop end" %}
11348   size(2);
11349   ins_encode %{
11350     Label* L = $labl$$label;
11351     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11352   %}
11353   ins_pipe(pipe_jcc);
11354   ins_short_branch(1);
11355 %}
11356 
11357 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11358 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11359   match(CountedLoopEnd cop cmp);
11360   effect(USE labl);
11361 
11362   ins_cost(300);
11363   format %{ "j$cop,us  $labl\t# loop end" %}
11364   size(2);
11365   ins_encode %{
11366     Label* L = $labl$$label;
11367     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11368   %}
11369   ins_pipe(pipe_jcc);
11370   ins_short_branch(1);
11371 %}
11372 
11373 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11374   match(CountedLoopEnd cop cmp);
11375   effect(USE labl);
11376 
11377   ins_cost(300);
11378   format %{ "j$cop,us  $labl\t# loop end" %}
11379   size(2);
11380   ins_encode %{
11381     Label* L = $labl$$label;
11382     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11383   %}
11384   ins_pipe(pipe_jcc);
11385   ins_short_branch(1);
11386 %}
11387 
11388 // Jump Direct Conditional - using unsigned comparison
11389 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11390   match(If cop cmp);
11391   effect(USE labl);
11392 
11393   ins_cost(300);
11394   format %{ "j$cop,us  $labl" %}
11395   size(2);
11396   ins_encode %{
11397     Label* L = $labl$$label;
11398     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11399   %}
11400   ins_pipe(pipe_jcc);
11401   ins_short_branch(1);
11402 %}
11403 
11404 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11405   match(If cop cmp);
11406   effect(USE labl);
11407 
11408   ins_cost(300);
11409   format %{ "j$cop,us  $labl" %}
11410   size(2);
11411   ins_encode %{
11412     Label* L = $labl$$label;
11413     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11414   %}
11415   ins_pipe(pipe_jcc);
11416   ins_short_branch(1);
11417 %}
11418 
11419 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11420   match(If cop cmp);
11421   effect(USE labl);
11422 
11423   ins_cost(300);
11424   format %{ $$template
11425     if ($cop$$cmpcode == Assembler::notEqual) {
11426       $$emit$$"jp,u,s   $labl\n\t"
11427       $$emit$$"j$cop,u,s   $labl"
11428     } else {
11429       $$emit$$"jp,u,s   done\n\t"
11430       $$emit$$"j$cop,u,s  $labl\n\t"
11431       $$emit$$"done:"
11432     }
11433   %}
11434   size(4);
11435   ins_encode %{
11436     Label* l = $labl$$label;
11437     if ($cop$$cmpcode == Assembler::notEqual) {
11438       __ jccb(Assembler::parity, *l);
11439       __ jccb(Assembler::notEqual, *l);
11440     } else if ($cop$$cmpcode == Assembler::equal) {
11441       Label done;
11442       __ jccb(Assembler::parity, done);
11443       __ jccb(Assembler::equal, *l);
11444       __ bind(done);
11445     } else {
11446        ShouldNotReachHere();
11447     }
11448   %}
11449   ins_pipe(pipe_jcc);
11450   ins_short_branch(1);
11451 %}
11452 
11453 // ============================================================================
11454 // inlined locking and unlocking
11455 
11456 instruct cmpFastLock(rFlagsReg cr,
11457                      rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
11458 %{
11459   match(Set cr (FastLock object box));
11460   effect(TEMP tmp, TEMP scr, USE_KILL box);
11461 
11462   ins_cost(300);
11463   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11464   ins_encode(Fast_Lock(object, box, tmp, scr));
11465   ins_pipe(pipe_slow);
11466 %}
11467 
11468 instruct cmpFastUnlock(rFlagsReg cr,
11469                        rRegP object, rax_RegP box, rRegP tmp)
11470 %{
11471   match(Set cr (FastUnlock object box));
11472   effect(TEMP tmp, USE_KILL box);
11473 
11474   ins_cost(300);
11475   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11476   ins_encode(Fast_Unlock(object, box, tmp));
11477   ins_pipe(pipe_slow);
11478 %}
11479 
11480 
11481 // ============================================================================
11482 // Safepoint Instructions
11483 instruct safePoint_poll(rFlagsReg cr)
11484 %{
11485   predicate(!Assembler::is_polling_page_far());
11486   match(SafePoint);
11487   effect(KILL cr);
11488 
11489   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11490             "# Safepoint: poll for GC" %}
11491   ins_cost(125);
11492   ins_encode %{
11493     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11494     __ testl(rax, addr);
11495   %}
11496   ins_pipe(ialu_reg_mem);
11497 %}
11498 
11499 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11500 %{
11501   predicate(Assembler::is_polling_page_far());
11502   match(SafePoint poll);
11503   effect(KILL cr, USE poll);
11504 
11505   format %{ "testl  rax, [$poll]\t"
11506             "# Safepoint: poll for GC" %}
11507   ins_cost(125);
11508   ins_encode %{
11509     __ relocate(relocInfo::poll_type);
11510     __ testl(rax, Address($poll$$Register, 0));
11511   %}
11512   ins_pipe(ialu_reg_mem);
11513 %}
11514 
11515 // ============================================================================
11516 // Procedure Call/Return Instructions
11517 // Call Java Static Instruction
11518 // Note: If this code changes, the corresponding ret_addr_offset() and
11519 //       compute_padding() functions will have to be adjusted.
11520 instruct CallStaticJavaDirect(method meth) %{
11521   match(CallStaticJava);
11522   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
11523   effect(USE meth);
11524 
11525   ins_cost(300);
11526   format %{ "call,static " %}
11527   opcode(0xE8); /* E8 cd */
11528   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11529   ins_pipe(pipe_slow);
11530   ins_alignment(4);
11531 %}
11532 
11533 // Call Java Static Instruction (method handle version)
11534 // Note: If this code changes, the corresponding ret_addr_offset() and
11535 //       compute_padding() functions will have to be adjusted.
11536 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
11537   match(CallStaticJava);
11538   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
11539   effect(USE meth);
11540   // RBP is saved by all callees (for interpreter stack correction).
11541   // We use it here for a similar purpose, in {preserve,restore}_SP.
11542 
11543   ins_cost(300);
11544   format %{ "call,static/MethodHandle " %}
11545   opcode(0xE8); /* E8 cd */
11546   ins_encode(clear_avx, preserve_SP,
11547              Java_Static_Call(meth),
11548              restore_SP,
11549              call_epilog);
11550   ins_pipe(pipe_slow);
11551   ins_alignment(4);
11552 %}
11553 
11554 // Call Java Dynamic Instruction
11555 // Note: If this code changes, the corresponding ret_addr_offset() and
11556 //       compute_padding() functions will have to be adjusted.
11557 instruct CallDynamicJavaDirect(method meth)
11558 %{
11559   match(CallDynamicJava);
11560   effect(USE meth);
11561 
11562   ins_cost(300);
11563   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11564             "call,dynamic " %}
11565   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11566   ins_pipe(pipe_slow);
11567   ins_alignment(4);
11568 %}
11569 
11570 // Call Runtime Instruction
11571 instruct CallRuntimeDirect(method meth)
11572 %{
11573   match(CallRuntime);
11574   effect(USE meth);
11575 
11576   ins_cost(300);
11577   format %{ "call,runtime " %}
11578   ins_encode(clear_avx, Java_To_Runtime(meth));
11579   ins_pipe(pipe_slow);
11580 %}
11581 
11582 // Call runtime without safepoint
11583 instruct CallLeafDirect(method meth)
11584 %{
11585   match(CallLeaf);
11586   effect(USE meth);
11587 
11588   ins_cost(300);
11589   format %{ "call_leaf,runtime " %}
11590   ins_encode(clear_avx, Java_To_Runtime(meth));
11591   ins_pipe(pipe_slow);
11592 %}
11593 
11594 // Call runtime without safepoint
11595 instruct CallLeafNoFPDirect(method meth)
11596 %{
11597   match(CallLeafNoFP);
11598   effect(USE meth);
11599 
11600   ins_cost(300);
11601   format %{ "call_leaf_nofp,runtime " %}
11602   ins_encode(Java_To_Runtime(meth));
11603   ins_pipe(pipe_slow);
11604 %}
11605 
11606 // Return Instruction
11607 // Remove the return address & jump to it.
11608 // Notice: We always emit a nop after a ret to make sure there is room
11609 // for safepoint patching
11610 instruct Ret()
11611 %{
11612   match(Return);
11613 
11614   format %{ "ret" %}
11615   opcode(0xC3);
11616   ins_encode(OpcP);
11617   ins_pipe(pipe_jmp);
11618 %}
11619 
11620 // Tail Call; Jump from runtime stub to Java code.
11621 // Also known as an 'interprocedural jump'.
11622 // Target of jump will eventually return to caller.
11623 // TailJump below removes the return address.
11624 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11625 %{
11626   match(TailCall jump_target method_oop);
11627 
11628   ins_cost(300);
11629   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11630   opcode(0xFF, 0x4); /* Opcode FF /4 */
11631   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11632   ins_pipe(pipe_jmp);
11633 %}
11634 
11635 // Tail Jump; remove the return address; jump to target.
11636 // TailCall above leaves the return address around.
11637 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11638 %{
11639   match(TailJump jump_target ex_oop);
11640 
11641   ins_cost(300);
11642   format %{ "popq    rdx\t# pop return address\n\t"
11643             "jmp     $jump_target" %}
11644   opcode(0xFF, 0x4); /* Opcode FF /4 */
11645   ins_encode(Opcode(0x5a), // popq rdx
11646              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11647   ins_pipe(pipe_jmp);
11648 %}
11649 
11650 // Create exception oop: created by stack-crawling runtime code.
11651 // Created exception is now available to this handler, and is setup
11652 // just prior to jumping to this handler.  No code emitted.
11653 instruct CreateException(rax_RegP ex_oop)
11654 %{
11655   match(Set ex_oop (CreateEx));
11656 
11657   size(0);
11658   // use the following format syntax
11659   format %{ "# exception oop is in rax; no code emitted" %}
11660   ins_encode();
11661   ins_pipe(empty);
11662 %}
11663 
11664 // Rethrow exception:
11665 // The exception oop will come in the first argument position.
11666 // Then JUMP (not call) to the rethrow stub code.
11667 instruct RethrowException()
11668 %{
11669   match(Rethrow);
11670 
11671   // use the following format syntax
11672   format %{ "jmp     rethrow_stub" %}
11673   ins_encode(enc_rethrow);
11674   ins_pipe(pipe_jmp);
11675 %}
11676 
11677 
11678 // ============================================================================
11679 // This name is KNOWN by the ADLC and cannot be changed.
11680 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11681 // for this guy.
11682 instruct tlsLoadP(r15_RegP dst) %{
11683   match(Set dst (ThreadLocal));
11684   effect(DEF dst);
11685 
11686   size(0);
11687   format %{ "# TLS is in R15" %}
11688   ins_encode( /*empty encoding*/ );
11689   ins_pipe(ialu_reg_reg);
11690 %}
11691 
11692 
11693 //----------PEEPHOLE RULES-----------------------------------------------------
11694 // These must follow all instruction definitions as they use the names
11695 // defined in the instructions definitions.
11696 //
11697 // peepmatch ( root_instr_name [preceding_instruction]* );
11698 //
11699 // peepconstraint %{
11700 // (instruction_number.operand_name relational_op instruction_number.operand_name
11701 //  [, ...] );
11702 // // instruction numbers are zero-based using left to right order in peepmatch
11703 //
11704 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11705 // // provide an instruction_number.operand_name for each operand that appears
11706 // // in the replacement instruction's match rule
11707 //
11708 // ---------VM FLAGS---------------------------------------------------------
11709 //
11710 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11711 //
11712 // Each peephole rule is given an identifying number starting with zero and
11713 // increasing by one in the order seen by the parser.  An individual peephole
11714 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11715 // on the command-line.
11716 //
11717 // ---------CURRENT LIMITATIONS----------------------------------------------
11718 //
11719 // Only match adjacent instructions in same basic block
11720 // Only equality constraints
11721 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11722 // Only one replacement instruction
11723 //
11724 // ---------EXAMPLE----------------------------------------------------------
11725 //
11726 // // pertinent parts of existing instructions in architecture description
11727 // instruct movI(rRegI dst, rRegI src)
11728 // %{
11729 //   match(Set dst (CopyI src));
11730 // %}
11731 //
11732 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11733 // %{
11734 //   match(Set dst (AddI dst src));
11735 //   effect(KILL cr);
11736 // %}
11737 //
11738 // // Change (inc mov) to lea
11739 // peephole %{
11740 //   // increment preceeded by register-register move
11741 //   peepmatch ( incI_rReg movI );
11742 //   // require that the destination register of the increment
11743 //   // match the destination register of the move
11744 //   peepconstraint ( 0.dst == 1.dst );
11745 //   // construct a replacement instruction that sets
11746 //   // the destination to ( move's source register + one )
11747 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11748 // %}
11749 //
11750 
11751 // Implementation no longer uses movX instructions since
11752 // machine-independent system no longer uses CopyX nodes.
11753 //
11754 // peephole
11755 // %{
11756 //   peepmatch (incI_rReg movI);
11757 //   peepconstraint (0.dst == 1.dst);
11758 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11759 // %}
11760 
11761 // peephole
11762 // %{
11763 //   peepmatch (decI_rReg movI);
11764 //   peepconstraint (0.dst == 1.dst);
11765 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11766 // %}
11767 
11768 // peephole
11769 // %{
11770 //   peepmatch (addI_rReg_imm movI);
11771 //   peepconstraint (0.dst == 1.dst);
11772 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11773 // %}
11774 
11775 // peephole
11776 // %{
11777 //   peepmatch (incL_rReg movL);
11778 //   peepconstraint (0.dst == 1.dst);
11779 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11780 // %}
11781 
11782 // peephole
11783 // %{
11784 //   peepmatch (decL_rReg movL);
11785 //   peepconstraint (0.dst == 1.dst);
11786 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11787 // %}
11788 
11789 // peephole
11790 // %{
11791 //   peepmatch (addL_rReg_imm movL);
11792 //   peepconstraint (0.dst == 1.dst);
11793 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11794 // %}
11795 
11796 // peephole
11797 // %{
11798 //   peepmatch (addP_rReg_imm movP);
11799 //   peepconstraint (0.dst == 1.dst);
11800 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11801 // %}
11802 
11803 // // Change load of spilled value to only a spill
11804 // instruct storeI(memory mem, rRegI src)
11805 // %{
11806 //   match(Set mem (StoreI mem src));
11807 // %}
11808 //
11809 // instruct loadI(rRegI dst, memory mem)
11810 // %{
11811 //   match(Set dst (LoadI mem));
11812 // %}
11813 //
11814 
11815 peephole
11816 %{
11817   peepmatch (loadI storeI);
11818   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11819   peepreplace (storeI(1.mem 1.mem 1.src));
11820 %}
11821 
11822 peephole
11823 %{
11824   peepmatch (loadL storeL);
11825   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11826   peepreplace (storeL(1.mem 1.mem 1.src));
11827 %}
11828 
11829 //----------SMARTSPILL RULES---------------------------------------------------
11830 // These must follow all instruction definitions as they use the names
11831 // defined in the instructions definitions.
--- EOF ---