1 //
   2 // Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // XMM registers.  128-bit registers or 4 words each, labeled (a)-d.
 135 // Word a in each register holds a Float, words ab hold a Double.  We
 136 // currently do not use the SIMD capabilities, so registers cd are
 137 // unused at the moment.
 138 // XMM8-XMM15 must be encoded with REX.
 139 // Linux ABI:   No register preserved across function calls
 140 //              XMM0-XMM7 might hold parameters
 141 // Windows ABI: XMM6-XMM15 preserved across function calls
 142 //              XMM0-XMM3 might hold parameters
 143 
 144 reg_def XMM0   (SOC, SOC, Op_RegF,  0, xmm0->as_VMReg());
 145 reg_def XMM0_H (SOC, SOC, Op_RegF,  0, xmm0->as_VMReg()->next());
 146 
 147 reg_def XMM1   (SOC, SOC, Op_RegF,  1, xmm1->as_VMReg());
 148 reg_def XMM1_H (SOC, SOC, Op_RegF,  1, xmm1->as_VMReg()->next());
 149 
 150 reg_def XMM2   (SOC, SOC, Op_RegF,  2, xmm2->as_VMReg());
 151 reg_def XMM2_H (SOC, SOC, Op_RegF,  2, xmm2->as_VMReg()->next());
 152 
 153 reg_def XMM3   (SOC, SOC, Op_RegF,  3, xmm3->as_VMReg());
 154 reg_def XMM3_H (SOC, SOC, Op_RegF,  3, xmm3->as_VMReg()->next());
 155 
 156 reg_def XMM4   (SOC, SOC, Op_RegF,  4, xmm4->as_VMReg());
 157 reg_def XMM4_H (SOC, SOC, Op_RegF,  4, xmm4->as_VMReg()->next());
 158 
 159 reg_def XMM5   (SOC, SOC, Op_RegF,  5, xmm5->as_VMReg());
 160 reg_def XMM5_H (SOC, SOC, Op_RegF,  5, xmm5->as_VMReg()->next());
 161 
 162 #ifdef _WIN64
 163 
 164 reg_def XMM6   (SOC, SOE, Op_RegF,  6, xmm6->as_VMReg());
 165 reg_def XMM6_H (SOC, SOE, Op_RegF,  6, xmm6->as_VMReg()->next());
 166 
 167 reg_def XMM7   (SOC, SOE, Op_RegF,  7, xmm7->as_VMReg());
 168 reg_def XMM7_H (SOC, SOE, Op_RegF,  7, xmm7->as_VMReg()->next());
 169 
 170 reg_def XMM8   (SOC, SOE, Op_RegF,  8, xmm8->as_VMReg());
 171 reg_def XMM8_H (SOC, SOE, Op_RegF,  8, xmm8->as_VMReg()->next());
 172 
 173 reg_def XMM9   (SOC, SOE, Op_RegF,  9, xmm9->as_VMReg());
 174 reg_def XMM9_H (SOC, SOE, Op_RegF,  9, xmm9->as_VMReg()->next());
 175 
 176 reg_def XMM10  (SOC, SOE, Op_RegF, 10, xmm10->as_VMReg());
 177 reg_def XMM10_H(SOC, SOE, Op_RegF, 10, xmm10->as_VMReg()->next());
 178 
 179 reg_def XMM11  (SOC, SOE, Op_RegF, 11, xmm11->as_VMReg());
 180 reg_def XMM11_H(SOC, SOE, Op_RegF, 11, xmm11->as_VMReg()->next());
 181 
 182 reg_def XMM12  (SOC, SOE, Op_RegF, 12, xmm12->as_VMReg());
 183 reg_def XMM12_H(SOC, SOE, Op_RegF, 12, xmm12->as_VMReg()->next());
 184 
 185 reg_def XMM13  (SOC, SOE, Op_RegF, 13, xmm13->as_VMReg());
 186 reg_def XMM13_H(SOC, SOE, Op_RegF, 13, xmm13->as_VMReg()->next());
 187 
 188 reg_def XMM14  (SOC, SOE, Op_RegF, 14, xmm14->as_VMReg());
 189 reg_def XMM14_H(SOC, SOE, Op_RegF, 14, xmm14->as_VMReg()->next());
 190 
 191 reg_def XMM15  (SOC, SOE, Op_RegF, 15, xmm15->as_VMReg());
 192 reg_def XMM15_H(SOC, SOE, Op_RegF, 15, xmm15->as_VMReg()->next());
 193 
 194 #else
 195 
 196 reg_def XMM6   (SOC, SOC, Op_RegF,  6, xmm6->as_VMReg());
 197 reg_def XMM6_H (SOC, SOC, Op_RegF,  6, xmm6->as_VMReg()->next());
 198 
 199 reg_def XMM7   (SOC, SOC, Op_RegF,  7, xmm7->as_VMReg());
 200 reg_def XMM7_H (SOC, SOC, Op_RegF,  7, xmm7->as_VMReg()->next());
 201 
 202 reg_def XMM8   (SOC, SOC, Op_RegF,  8, xmm8->as_VMReg());
 203 reg_def XMM8_H (SOC, SOC, Op_RegF,  8, xmm8->as_VMReg()->next());
 204 
 205 reg_def XMM9   (SOC, SOC, Op_RegF,  9, xmm9->as_VMReg());
 206 reg_def XMM9_H (SOC, SOC, Op_RegF,  9, xmm9->as_VMReg()->next());
 207 
 208 reg_def XMM10  (SOC, SOC, Op_RegF, 10, xmm10->as_VMReg());
 209 reg_def XMM10_H(SOC, SOC, Op_RegF, 10, xmm10->as_VMReg()->next());
 210 
 211 reg_def XMM11  (SOC, SOC, Op_RegF, 11, xmm11->as_VMReg());
 212 reg_def XMM11_H(SOC, SOC, Op_RegF, 11, xmm11->as_VMReg()->next());
 213 
 214 reg_def XMM12  (SOC, SOC, Op_RegF, 12, xmm12->as_VMReg());
 215 reg_def XMM12_H(SOC, SOC, Op_RegF, 12, xmm12->as_VMReg()->next());
 216 
 217 reg_def XMM13  (SOC, SOC, Op_RegF, 13, xmm13->as_VMReg());
 218 reg_def XMM13_H(SOC, SOC, Op_RegF, 13, xmm13->as_VMReg()->next());
 219 
 220 reg_def XMM14  (SOC, SOC, Op_RegF, 14, xmm14->as_VMReg());
 221 reg_def XMM14_H(SOC, SOC, Op_RegF, 14, xmm14->as_VMReg()->next());
 222 
 223 reg_def XMM15  (SOC, SOC, Op_RegF, 15, xmm15->as_VMReg());
 224 reg_def XMM15_H(SOC, SOC, Op_RegF, 15, xmm15->as_VMReg()->next());
 225 
 226 #endif // _WIN64
 227 
 228 reg_def RFLAGS(SOC, SOC, 0, 16, VMRegImpl::Bad());
 229 
 230 // Specify priority of register selection within phases of register
 231 // allocation.  Highest priority is first.  A useful heuristic is to
 232 // give registers a low priority when they are required by machine
 233 // instructions, like EAX and EDX on I486, and choose no-save registers
 234 // before save-on-call, & save-on-call before save-on-entry.  Registers
 235 // which participate in fixed calling sequences should come last.
 236 // Registers which are used as pairs must fall on an even boundary.
 237 
 238 alloc_class chunk0(R10,         R10_H,
 239                    R11,         R11_H,
 240                    R8,          R8_H,
 241                    R9,          R9_H,
 242                    R12,         R12_H,
 243                    RCX,         RCX_H,
 244                    RBX,         RBX_H,
 245                    RDI,         RDI_H,
 246                    RDX,         RDX_H,
 247                    RSI,         RSI_H,
 248                    RAX,         RAX_H,
 249                    RBP,         RBP_H,
 250                    R13,         R13_H,
 251                    R14,         R14_H,
 252                    R15,         R15_H,
 253                    RSP,         RSP_H);
 254 
 255 // XXX probably use 8-15 first on Linux
 256 alloc_class chunk1(XMM0,  XMM0_H,
 257                    XMM1,  XMM1_H,
 258                    XMM2,  XMM2_H,
 259                    XMM3,  XMM3_H,
 260                    XMM4,  XMM4_H,
 261                    XMM5,  XMM5_H,
 262                    XMM6,  XMM6_H,
 263                    XMM7,  XMM7_H,
 264                    XMM8,  XMM8_H,
 265                    XMM9,  XMM9_H,
 266                    XMM10, XMM10_H,
 267                    XMM11, XMM11_H,
 268                    XMM12, XMM12_H,
 269                    XMM13, XMM13_H,
 270                    XMM14, XMM14_H,
 271                    XMM15, XMM15_H);
 272 
 273 alloc_class chunk2(RFLAGS);
 274 
 275 
 276 //----------Architecture Description Register Classes--------------------------
 277 // Several register classes are automatically defined based upon information in
 278 // this architecture description.
 279 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 280 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 281 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 282 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 283 //
 284 
 285 // Class for all pointer registers (including RSP)
 286 reg_class any_reg(RAX, RAX_H,
 287                   RDX, RDX_H,
 288                   RBP, RBP_H,
 289                   RDI, RDI_H,
 290                   RSI, RSI_H,
 291                   RCX, RCX_H,
 292                   RBX, RBX_H,
 293                   RSP, RSP_H,
 294                   R8,  R8_H,
 295                   R9,  R9_H,
 296                   R10, R10_H,
 297                   R11, R11_H,
 298                   R12, R12_H,
 299                   R13, R13_H,
 300                   R14, R14_H,
 301                   R15, R15_H);
 302 
 303 // Class for all pointer registers except RSP
 304 reg_class ptr_reg(RAX, RAX_H,
 305                   RDX, RDX_H,
 306                   RBP, RBP_H,
 307                   RDI, RDI_H,
 308                   RSI, RSI_H,
 309                   RCX, RCX_H,
 310                   RBX, RBX_H,
 311                   R8,  R8_H,
 312                   R9,  R9_H,
 313                   R10, R10_H,
 314                   R11, R11_H,
 315                   R13, R13_H,
 316                   R14, R14_H);
 317 
 318 // Class for all pointer registers except RAX and RSP
 319 reg_class ptr_no_rax_reg(RDX, RDX_H,
 320                          RBP, RBP_H,
 321                          RDI, RDI_H,
 322                          RSI, RSI_H,
 323                          RCX, RCX_H,
 324                          RBX, RBX_H,
 325                          R8,  R8_H,
 326                          R9,  R9_H,
 327                          R10, R10_H,
 328                          R11, R11_H,
 329                          R13, R13_H,
 330                          R14, R14_H);
 331 
 332 reg_class ptr_no_rbp_reg(RDX, RDX_H,
 333                          RAX, RAX_H,
 334                          RDI, RDI_H,
 335                          RSI, RSI_H,
 336                          RCX, RCX_H,
 337                          RBX, RBX_H,
 338                          R8,  R8_H,
 339                          R9,  R9_H,
 340                          R10, R10_H,
 341                          R11, R11_H,
 342                          R13, R13_H,
 343                          R14, R14_H);
 344 
 345 // Class for all pointer registers except RAX, RBX and RSP
 346 reg_class ptr_no_rax_rbx_reg(RDX, RDX_H,
 347                              RBP, RBP_H,
 348                              RDI, RDI_H,
 349                              RSI, RSI_H,
 350                              RCX, RCX_H,
 351                              R8,  R8_H,
 352                              R9,  R9_H,
 353                              R10, R10_H,
 354                              R11, R11_H,
 355                              R13, R13_H,
 356                              R14, R14_H);
 357 
 358 // Singleton class for RAX pointer register
 359 reg_class ptr_rax_reg(RAX, RAX_H);
 360 
 361 // Singleton class for RBX pointer register
 362 reg_class ptr_rbx_reg(RBX, RBX_H);
 363 
 364 // Singleton class for RSI pointer register
 365 reg_class ptr_rsi_reg(RSI, RSI_H);
 366 
 367 // Singleton class for RDI pointer register
 368 reg_class ptr_rdi_reg(RDI, RDI_H);
 369 
 370 // Singleton class for RBP pointer register
 371 reg_class ptr_rbp_reg(RBP, RBP_H);
 372 
 373 // Singleton class for stack pointer
 374 reg_class ptr_rsp_reg(RSP, RSP_H);
 375 
 376 // Singleton class for TLS pointer
 377 reg_class ptr_r15_reg(R15, R15_H);
 378 
 379 // Class for all long registers (except RSP)
 380 reg_class long_reg(RAX, RAX_H,
 381                    RDX, RDX_H,
 382                    RBP, RBP_H,
 383                    RDI, RDI_H,
 384                    RSI, RSI_H,
 385                    RCX, RCX_H,
 386                    RBX, RBX_H,
 387                    R8,  R8_H,
 388                    R9,  R9_H,
 389                    R10, R10_H,
 390                    R11, R11_H,
 391                    R13, R13_H,
 392                    R14, R14_H);
 393 
 394 // Class for all long registers except RAX, RDX (and RSP)
 395 reg_class long_no_rax_rdx_reg(RBP, RBP_H,
 396                               RDI, RDI_H,
 397                               RSI, RSI_H,
 398                               RCX, RCX_H,
 399                               RBX, RBX_H,
 400                               R8,  R8_H,
 401                               R9,  R9_H,
 402                               R10, R10_H,
 403                               R11, R11_H,
 404                               R13, R13_H,
 405                               R14, R14_H);
 406 
 407 // Class for all long registers except RCX (and RSP)
 408 reg_class long_no_rcx_reg(RBP, RBP_H,
 409                           RDI, RDI_H,
 410                           RSI, RSI_H,
 411                           RAX, RAX_H,
 412                           RDX, RDX_H,
 413                           RBX, RBX_H,
 414                           R8,  R8_H,
 415                           R9,  R9_H,
 416                           R10, R10_H,
 417                           R11, R11_H,
 418                           R13, R13_H,
 419                           R14, R14_H);
 420 
 421 // Class for all long registers except RAX (and RSP)
 422 reg_class long_no_rax_reg(RBP, RBP_H,
 423                           RDX, RDX_H,
 424                           RDI, RDI_H,
 425                           RSI, RSI_H,
 426                           RCX, RCX_H,
 427                           RBX, RBX_H,
 428                           R8,  R8_H,
 429                           R9,  R9_H,
 430                           R10, R10_H,
 431                           R11, R11_H,
 432                           R13, R13_H,
 433                           R14, R14_H);
 434 
 435 // Singleton class for RAX long register
 436 reg_class long_rax_reg(RAX, RAX_H);
 437 
 438 // Singleton class for RCX long register
 439 reg_class long_rcx_reg(RCX, RCX_H);
 440 
 441 // Singleton class for RDX long register
 442 reg_class long_rdx_reg(RDX, RDX_H);
 443 
 444 // Class for all int registers (except RSP)
 445 reg_class int_reg(RAX,
 446                   RDX,
 447                   RBP,
 448                   RDI,
 449                   RSI,
 450                   RCX,
 451                   RBX,
 452                   R8,
 453                   R9,
 454                   R10,
 455                   R11,
 456                   R13,
 457                   R14);
 458 
 459 // Class for all int registers except RCX (and RSP)
 460 reg_class int_no_rcx_reg(RAX,
 461                          RDX,
 462                          RBP,
 463                          RDI,
 464                          RSI,
 465                          RBX,
 466                          R8,
 467                          R9,
 468                          R10,
 469                          R11,
 470                          R13,
 471                          R14);
 472 
 473 // Class for all int registers except RAX, RDX (and RSP)
 474 reg_class int_no_rax_rdx_reg(RBP,
 475                              RDI,
 476                              RSI,
 477                              RCX,
 478                              RBX,
 479                              R8,
 480                              R9,
 481                              R10,
 482                              R11,
 483                              R13,
 484                              R14);
 485 
 486 // Singleton class for RAX int register
 487 reg_class int_rax_reg(RAX);
 488 
 489 // Singleton class for RBX int register
 490 reg_class int_rbx_reg(RBX);
 491 
 492 // Singleton class for RCX int register
 493 reg_class int_rcx_reg(RCX);
 494 
 495 // Singleton class for RCX int register
 496 reg_class int_rdx_reg(RDX);
 497 
 498 // Singleton class for RCX int register
 499 reg_class int_rdi_reg(RDI);
 500 
 501 // Singleton class for instruction pointer
 502 // reg_class ip_reg(RIP);
 503 
 504 // Singleton class for condition codes
 505 reg_class int_flags(RFLAGS);
 506 
 507 // Class for all float registers
 508 reg_class float_reg(XMM0,
 509                     XMM1,
 510                     XMM2,
 511                     XMM3,
 512                     XMM4,
 513                     XMM5,
 514                     XMM6,
 515                     XMM7,
 516                     XMM8,
 517                     XMM9,
 518                     XMM10,
 519                     XMM11,
 520                     XMM12,
 521                     XMM13,
 522                     XMM14,
 523                     XMM15);
 524 
 525 // Class for all double registers
 526 reg_class double_reg(XMM0,  XMM0_H,
 527                      XMM1,  XMM1_H,
 528                      XMM2,  XMM2_H,
 529                      XMM3,  XMM3_H,
 530                      XMM4,  XMM4_H,
 531                      XMM5,  XMM5_H,
 532                      XMM6,  XMM6_H,
 533                      XMM7,  XMM7_H,
 534                      XMM8,  XMM8_H,
 535                      XMM9,  XMM9_H,
 536                      XMM10, XMM10_H,
 537                      XMM11, XMM11_H,
 538                      XMM12, XMM12_H,
 539                      XMM13, XMM13_H,
 540                      XMM14, XMM14_H,
 541                      XMM15, XMM15_H);
 542 %}
 543 
 544 
 545 //----------SOURCE BLOCK-------------------------------------------------------
 546 // This is a block of C++ code which provides values, functions, and
 547 // definitions necessary in the rest of the architecture description
 548 source %{
 549 #define   RELOC_IMM64    Assembler::imm_operand
 550 #define   RELOC_DISP32   Assembler::disp32_operand
 551 
 552 #define __ _masm.
 553 
 554 static int preserve_SP_size() {
 555   return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
 556 }
 557 
 558 // !!!!! Special hack to get all types of calls to specify the byte offset
 559 //       from the start of the call to the point where the return address
 560 //       will point.
 561 int MachCallStaticJavaNode::ret_addr_offset()
 562 {
 563   int offset = 5; // 5 bytes from start of call to where return address points
 564   if (_method_handle_invoke)
 565     offset += preserve_SP_size();
 566   return offset;
 567 }
 568 
 569 int MachCallDynamicJavaNode::ret_addr_offset()
 570 {
 571   return 15; // 15 bytes from start of call to where return address points
 572 }
 573 
 574 // In os_cpu .ad file
 575 // int MachCallRuntimeNode::ret_addr_offset()
 576 
 577 // Indicate if the safepoint node needs the polling page as an input,
 578 // it does if the polling page is more than disp32 away.
 579 bool SafePointNode::needs_polling_address_input()
 580 {
 581   return Assembler::is_polling_page_far();
 582 }
 583 
 584 //
 585 // Compute padding required for nodes which need alignment
 586 //
 587 
 588 // The address of the call instruction needs to be 4-byte aligned to
 589 // ensure that it does not span a cache line so that it can be patched.
 590 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 591 {
 592   current_offset += 1; // skip call opcode byte
 593   return round_to(current_offset, alignment_required()) - current_offset;
 594 }
 595 
 596 // The address of the call instruction needs to be 4-byte aligned to
 597 // ensure that it does not span a cache line so that it can be patched.
 598 int CallStaticJavaHandleNode::compute_padding(int current_offset) const
 599 {
 600   current_offset += preserve_SP_size();   // skip mov rbp, rsp
 601   current_offset += 1; // skip call opcode byte
 602   return round_to(current_offset, alignment_required()) - current_offset;
 603 }
 604 
 605 // The address of the call instruction needs to be 4-byte aligned to
 606 // ensure that it does not span a cache line so that it can be patched.
 607 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 608 {
 609   current_offset += 11; // skip movq instruction + call opcode byte
 610   return round_to(current_offset, alignment_required()) - current_offset;
 611 }
 612 
 613 #ifndef PRODUCT
 614 void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const
 615 {
 616   st->print("INT3");
 617 }
 618 #endif
 619 
 620 // EMIT_RM()
 621 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 622   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 623   cbuf.insts()->emit_int8(c);
 624 }
 625 
 626 // EMIT_CC()
 627 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 628   unsigned char c = (unsigned char) (f1 | f2);
 629   cbuf.insts()->emit_int8(c);
 630 }
 631 
 632 // EMIT_OPCODE()
 633 void emit_opcode(CodeBuffer &cbuf, int code) {
 634   cbuf.insts()->emit_int8((unsigned char) code);
 635 }
 636 
 637 // EMIT_OPCODE() w/ relocation information
 638 void emit_opcode(CodeBuffer &cbuf,
 639                  int code, relocInfo::relocType reloc, int offset, int format)
 640 {
 641   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 642   emit_opcode(cbuf, code);
 643 }
 644 
 645 // EMIT_D8()
 646 void emit_d8(CodeBuffer &cbuf, int d8) {
 647   cbuf.insts()->emit_int8((unsigned char) d8);
 648 }
 649 
 650 // EMIT_D16()
 651 void emit_d16(CodeBuffer &cbuf, int d16) {
 652   cbuf.insts()->emit_int16(d16);
 653 }
 654 
 655 // EMIT_D32()
 656 void emit_d32(CodeBuffer &cbuf, int d32) {
 657   cbuf.insts()->emit_int32(d32);
 658 }
 659 
 660 // EMIT_D64()
 661 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 662   cbuf.insts()->emit_int64(d64);
 663 }
 664 
 665 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 666 void emit_d32_reloc(CodeBuffer& cbuf,
 667                     int d32,
 668                     relocInfo::relocType reloc,
 669                     int format)
 670 {
 671   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 672   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 673   cbuf.insts()->emit_int32(d32);
 674 }
 675 
 676 // emit 32 bit value and construct relocation entry from RelocationHolder
 677 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 678 #ifdef ASSERT
 679   if (rspec.reloc()->type() == relocInfo::oop_type &&
 680       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 681     assert(oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 682   }
 683 #endif
 684   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 685   cbuf.insts()->emit_int32(d32);
 686 }
 687 
 688 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 689   address next_ip = cbuf.insts_end() + 4;
 690   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 691                  external_word_Relocation::spec(addr),
 692                  RELOC_DISP32);
 693 }
 694 
 695 
 696 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 697 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 698   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 699   cbuf.insts()->emit_int64(d64);
 700 }
 701 
 702 // emit 64 bit value and construct relocation entry from RelocationHolder
 703 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 704 #ifdef ASSERT
 705   if (rspec.reloc()->type() == relocInfo::oop_type &&
 706       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 707     assert(oop(d64)->is_oop() && (ScavengeRootsInCode || !oop(d64)->is_scavengable()),
 708            "cannot embed scavengable oops in code");
 709   }
 710 #endif
 711   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 712   cbuf.insts()->emit_int64(d64);
 713 }
 714 
 715 // Access stack slot for load or store
 716 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 717 {
 718   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 719   if (-0x80 <= disp && disp < 0x80) {
 720     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 721     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 722     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 723   } else {
 724     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 725     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 726     emit_d32(cbuf, disp);     // Displacement // R/M byte
 727   }
 728 }
 729 
 730    // rRegI ereg, memory mem) %{    // emit_reg_mem
 731 void encode_RegMem(CodeBuffer &cbuf,
 732                    int reg,
 733                    int base, int index, int scale, int disp, bool disp_is_oop)
 734 {
 735   assert(!disp_is_oop, "cannot have disp");
 736   int regenc = reg & 7;
 737   int baseenc = base & 7;
 738   int indexenc = index & 7;
 739 
 740   // There is no index & no scale, use form without SIB byte
 741   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 742     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 743     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 744       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 745     } else if (-0x80 <= disp && disp < 0x80 && !disp_is_oop) {
 746       // If 8-bit displacement, mode 0x1
 747       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 748       emit_d8(cbuf, disp);
 749     } else {
 750       // If 32-bit displacement
 751       if (base == -1) { // Special flag for absolute address
 752         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 753         if (disp_is_oop) {
 754           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 755         } else {
 756           emit_d32(cbuf, disp);
 757         }
 758       } else {
 759         // Normal base + offset
 760         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 761         if (disp_is_oop) {
 762           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 763         } else {
 764           emit_d32(cbuf, disp);
 765         }
 766       }
 767     }
 768   } else {
 769     // Else, encode with the SIB byte
 770     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 771     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 772       // If no displacement
 773       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 774       emit_rm(cbuf, scale, indexenc, baseenc);
 775     } else {
 776       if (-0x80 <= disp && disp < 0x80 && !disp_is_oop) {
 777         // If 8-bit displacement, mode 0x1
 778         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 779         emit_rm(cbuf, scale, indexenc, baseenc);
 780         emit_d8(cbuf, disp);
 781       } else {
 782         // If 32-bit displacement
 783         if (base == 0x04 ) {
 784           emit_rm(cbuf, 0x2, regenc, 0x4);
 785           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 786         } else {
 787           emit_rm(cbuf, 0x2, regenc, 0x4);
 788           emit_rm(cbuf, scale, indexenc, baseenc); // *
 789         }
 790         if (disp_is_oop) {
 791           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 792         } else {
 793           emit_d32(cbuf, disp);
 794         }
 795       }
 796     }
 797   }
 798 }
 799 
 800 void encode_copy(CodeBuffer &cbuf, int dstenc, int srcenc)
 801 {
 802   if (dstenc != srcenc) {
 803     if (dstenc < 8) {
 804       if (srcenc >= 8) {
 805         emit_opcode(cbuf, Assembler::REX_B);
 806         srcenc -= 8;
 807       }
 808     } else {
 809       if (srcenc < 8) {
 810         emit_opcode(cbuf, Assembler::REX_R);
 811       } else {
 812         emit_opcode(cbuf, Assembler::REX_RB);
 813         srcenc -= 8;
 814       }
 815       dstenc -= 8;
 816     }
 817 
 818     emit_opcode(cbuf, 0x8B);
 819     emit_rm(cbuf, 0x3, dstenc, srcenc);
 820   }
 821 }
 822 
 823 void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
 824   if( dst_encoding == src_encoding ) {
 825     // reg-reg copy, use an empty encoding
 826   } else {
 827     MacroAssembler _masm(&cbuf);
 828 
 829     __ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
 830   }
 831 }
 832 
 833 // This could be in MacroAssembler but it's fairly C2 specific
 834 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 835   Label exit;
 836   __ jccb(Assembler::noParity, exit);
 837   __ pushf();
 838   __ andq(Address(rsp, 0), 0xffffff2b);
 839   __ popf();
 840   __ bind(exit);
 841   __ nop(); // (target for branch to avoid branch to branch)
 842 }
 843 
 844 
 845 //=============================================================================
 846 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 847 
 848 int Compile::ConstantTable::calculate_table_base_offset() const {
 849   return 0;  // absolute addressing, no offset
 850 }
 851 
 852 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 853   // Empty encoding
 854 }
 855 
 856 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 857   return 0;
 858 }
 859 
 860 #ifndef PRODUCT
 861 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 862   st->print("# MachConstantBaseNode (empty encoding)");
 863 }
 864 #endif
 865 
 866 
 867 //=============================================================================
 868 #ifndef PRODUCT
 869 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 870 {
 871   Compile* C = ra_->C;
 872 
 873   int framesize = C->frame_slots() << LogBytesPerInt;
 874   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 875   // Remove wordSize for return adr already pushed
 876   // and another for the RBP we are going to save
 877   framesize -= 2*wordSize;
 878   bool need_nop = true;
 879 
 880   // Calls to C2R adapters often do not accept exceptional returns.
 881   // We require that their callers must bang for them.  But be
 882   // careful, because some VM calls (such as call site linkage) can
 883   // use several kilobytes of stack.  But the stack safety zone should
 884   // account for that.  See bugs 4446381, 4468289, 4497237.
 885   if (C->need_stack_bang(framesize)) {
 886     st->print_cr("# stack bang"); st->print("\t");
 887     need_nop = false;
 888   }
 889   st->print_cr("pushq   rbp"); st->print("\t");
 890 
 891   if (VerifyStackAtCalls) {
 892     // Majik cookie to verify stack depth
 893     st->print_cr("pushq   0xffffffffbadb100d"
 894                   "\t# Majik cookie for stack depth check");
 895     st->print("\t");
 896     framesize -= wordSize; // Remove 2 for cookie
 897     need_nop = false;
 898   }
 899 
 900   if (framesize) {
 901     st->print("subq    rsp, #%d\t# Create frame", framesize);
 902     if (framesize < 0x80 && need_nop) {
 903       st->print("\n\tnop\t# nop for patch_verified_entry");
 904     }
 905   }
 906 }
 907 #endif
 908 
 909 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const
 910 {
 911   Compile* C = ra_->C;
 912 
 913   // WARNING: Initial instruction MUST be 5 bytes or longer so that
 914   // NativeJump::patch_verified_entry will be able to patch out the entry
 915   // code safely. The fldcw is ok at 6 bytes, the push to verify stack
 916   // depth is ok at 5 bytes, the frame allocation can be either 3 or
 917   // 6 bytes. So if we don't do the fldcw or the push then we must
 918   // use the 6 byte frame allocation even if we have no frame. :-(
 919   // If method sets FPU control word do it now
 920 
 921   int framesize = C->frame_slots() << LogBytesPerInt;
 922   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 923   // Remove wordSize for return adr already pushed
 924   // and another for the RBP we are going to save
 925   framesize -= 2*wordSize;
 926   bool need_nop = true;
 927 
 928   // Calls to C2R adapters often do not accept exceptional returns.
 929   // We require that their callers must bang for them.  But be
 930   // careful, because some VM calls (such as call site linkage) can
 931   // use several kilobytes of stack.  But the stack safety zone should
 932   // account for that.  See bugs 4446381, 4468289, 4497237.
 933   if (C->need_stack_bang(framesize)) {
 934     MacroAssembler masm(&cbuf);
 935     masm.generate_stack_overflow_check(framesize);
 936     need_nop = false;
 937   }
 938 
 939   // We always push rbp so that on return to interpreter rbp will be
 940   // restored correctly and we can correct the stack.
 941   emit_opcode(cbuf, 0x50 | RBP_enc);
 942 
 943   if (VerifyStackAtCalls) {
 944     // Majik cookie to verify stack depth
 945     emit_opcode(cbuf, 0x68); // pushq (sign-extended) 0xbadb100d
 946     emit_d32(cbuf, 0xbadb100d);
 947     framesize -= wordSize; // Remove 2 for cookie
 948     need_nop = false;
 949   }
 950 
 951   if (framesize) {
 952     emit_opcode(cbuf, Assembler::REX_W);
 953     if (framesize < 0x80) {
 954       emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
 955       emit_rm(cbuf, 0x3, 0x05, RSP_enc);
 956       emit_d8(cbuf, framesize);
 957       if (need_nop) {
 958         emit_opcode(cbuf, 0x90); // nop
 959       }
 960     } else {
 961       emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
 962       emit_rm(cbuf, 0x3, 0x05, RSP_enc);
 963       emit_d32(cbuf, framesize);
 964     }
 965   }
 966 
 967   C->set_frame_complete(cbuf.insts_size());
 968 
 969 #ifdef ASSERT
 970   if (VerifyStackAtCalls) {
 971     Label L;
 972     MacroAssembler masm(&cbuf);
 973     masm.push(rax);
 974     masm.mov(rax, rsp);
 975     masm.andptr(rax, StackAlignmentInBytes-1);
 976     masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
 977     masm.pop(rax);
 978     masm.jcc(Assembler::equal, L);
 979     masm.stop("Stack is not properly aligned!");
 980     masm.bind(L);
 981   }
 982 #endif
 983 
 984   if (C->has_mach_constant_base_node()) {
 985     // NOTE: We set the table base offset here because users might be
 986     // emitted before MachConstantBaseNode.
 987     Compile::ConstantTable& constant_table = C->constant_table();
 988     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 989   }
 990 }
 991 
 992 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 993 {
 994   return MachNode::size(ra_); // too many variables; just compute it
 995                               // the hard way
 996 }
 997 
 998 int MachPrologNode::reloc() const
 999 {
1000   return 0; // a large enough number
1001 }
1002 
1003 //=============================================================================
1004 #ifndef PRODUCT
1005 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1006 {
1007   Compile* C = ra_->C;
1008   int framesize = C->frame_slots() << LogBytesPerInt;
1009   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1010   // Remove word for return adr already pushed
1011   // and RBP
1012   framesize -= 2*wordSize;
1013 
1014   if (framesize) {
1015     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
1016     st->print("\t");
1017   }
1018 
1019   st->print_cr("popq   rbp");
1020   if (do_polling() && C->is_method_compilation()) {
1021     st->print("\t");
1022     if (Assembler::is_polling_page_far()) {
1023       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
1024                    "testl  rax, [rscratch1]\t"
1025                    "# Safepoint: poll for GC");
1026     } else {
1027       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
1028                    "# Safepoint: poll for GC");
1029     }
1030   }
1031 }
1032 #endif
1033 
1034 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1035 {
1036   Compile* C = ra_->C;
1037   int framesize = C->frame_slots() << LogBytesPerInt;
1038   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1039   // Remove word for return adr already pushed
1040   // and RBP
1041   framesize -= 2*wordSize;
1042 
1043   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1044 
1045   if (framesize) {
1046     emit_opcode(cbuf, Assembler::REX_W);
1047     if (framesize < 0x80) {
1048       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
1049       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1050       emit_d8(cbuf, framesize);
1051     } else {
1052       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1053       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1054       emit_d32(cbuf, framesize);
1055     }
1056   }
1057 
1058   // popq rbp
1059   emit_opcode(cbuf, 0x58 | RBP_enc);
1060 
1061   if (do_polling() && C->is_method_compilation()) {
1062     MacroAssembler _masm(&cbuf);
1063     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1064     if (Assembler::is_polling_page_far()) {
1065       __ lea(rscratch1, polling_page);
1066       __ relocate(relocInfo::poll_return_type);
1067       __ testl(rax, Address(rscratch1, 0));
1068     } else {
1069       __ testl(rax, polling_page);
1070     }
1071   }
1072 }
1073 
1074 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1075 {
1076   return MachNode::size(ra_); // too many variables; just compute it
1077                               // the hard way
1078 }
1079 
1080 int MachEpilogNode::reloc() const
1081 {
1082   return 2; // a large enough number
1083 }
1084 
1085 const Pipeline* MachEpilogNode::pipeline() const
1086 {
1087   return MachNode::pipeline_class();
1088 }
1089 
1090 int MachEpilogNode::safepoint_offset() const
1091 {
1092   return 0;
1093 }
1094 
1095 //=============================================================================
1096 
1097 enum RC {
1098   rc_bad,
1099   rc_int,
1100   rc_float,
1101   rc_stack
1102 };
1103 
1104 static enum RC rc_class(OptoReg::Name reg)
1105 {
1106   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1107 
1108   if (OptoReg::is_stack(reg)) return rc_stack;
1109 
1110   VMReg r = OptoReg::as_VMReg(reg);
1111 
1112   if (r->is_Register()) return rc_int;
1113 
1114   assert(r->is_XMMRegister(), "must be");
1115   return rc_float;
1116 }
1117 
1118 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1119                                        PhaseRegAlloc* ra_,
1120                                        bool do_size,
1121                                        outputStream* st) const
1122 {
1123 
1124   // Get registers to move
1125   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1126   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1127   OptoReg::Name dst_second = ra_->get_reg_second(this);
1128   OptoReg::Name dst_first = ra_->get_reg_first(this);
1129 
1130   enum RC src_second_rc = rc_class(src_second);
1131   enum RC src_first_rc = rc_class(src_first);
1132   enum RC dst_second_rc = rc_class(dst_second);
1133   enum RC dst_first_rc = rc_class(dst_first);
1134 
1135   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1136          "must move at least 1 register" );
1137 
1138   if (src_first == dst_first && src_second == dst_second) {
1139     // Self copy, no move
1140     return 0;
1141   } else if (src_first_rc == rc_stack) {
1142     // mem ->
1143     if (dst_first_rc == rc_stack) {
1144       // mem -> mem
1145       assert(src_second != dst_first, "overlap");
1146       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1147           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1148         // 64-bit
1149         int src_offset = ra_->reg2offset(src_first);
1150         int dst_offset = ra_->reg2offset(dst_first);
1151         if (cbuf) {
1152           emit_opcode(*cbuf, 0xFF);
1153           encode_RegMem(*cbuf, RSI_enc, RSP_enc, 0x4, 0, src_offset, false);
1154 
1155           emit_opcode(*cbuf, 0x8F);
1156           encode_RegMem(*cbuf, RAX_enc, RSP_enc, 0x4, 0, dst_offset, false);
1157 
1158 #ifndef PRODUCT
1159         } else if (!do_size) {
1160           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1161                      "popq    [rsp + #%d]",
1162                      src_offset,
1163                      dst_offset);
1164 #endif
1165         }
1166         return
1167           3 + ((src_offset == 0) ? 0 : (src_offset < 0x80 ? 1 : 4)) +
1168           3 + ((dst_offset == 0) ? 0 : (dst_offset < 0x80 ? 1 : 4));
1169       } else {
1170         // 32-bit
1171         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1172         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1173         // No pushl/popl, so:
1174         int src_offset = ra_->reg2offset(src_first);
1175         int dst_offset = ra_->reg2offset(dst_first);
1176         if (cbuf) {
1177           emit_opcode(*cbuf, Assembler::REX_W);
1178           emit_opcode(*cbuf, 0x89);
1179           emit_opcode(*cbuf, 0x44);
1180           emit_opcode(*cbuf, 0x24);
1181           emit_opcode(*cbuf, 0xF8);
1182 
1183           emit_opcode(*cbuf, 0x8B);
1184           encode_RegMem(*cbuf,
1185                         RAX_enc,
1186                         RSP_enc, 0x4, 0, src_offset,
1187                         false);
1188 
1189           emit_opcode(*cbuf, 0x89);
1190           encode_RegMem(*cbuf,
1191                         RAX_enc,
1192                         RSP_enc, 0x4, 0, dst_offset,
1193                         false);
1194 
1195           emit_opcode(*cbuf, Assembler::REX_W);
1196           emit_opcode(*cbuf, 0x8B);
1197           emit_opcode(*cbuf, 0x44);
1198           emit_opcode(*cbuf, 0x24);
1199           emit_opcode(*cbuf, 0xF8);
1200 
1201 #ifndef PRODUCT
1202         } else if (!do_size) {
1203           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1204                      "movl    rax, [rsp + #%d]\n\t"
1205                      "movl    [rsp + #%d], rax\n\t"
1206                      "movq    rax, [rsp - #8]",
1207                      src_offset,
1208                      dst_offset);
1209 #endif
1210         }
1211         return
1212           5 + // movq
1213           3 + ((src_offset == 0) ? 0 : (src_offset < 0x80 ? 1 : 4)) + // movl
1214           3 + ((dst_offset == 0) ? 0 : (dst_offset < 0x80 ? 1 : 4)) + // movl
1215           5; // movq
1216       }
1217     } else if (dst_first_rc == rc_int) {
1218       // mem -> gpr
1219       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1220           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1221         // 64-bit
1222         int offset = ra_->reg2offset(src_first);
1223         if (cbuf) {
1224           if (Matcher::_regEncode[dst_first] < 8) {
1225             emit_opcode(*cbuf, Assembler::REX_W);
1226           } else {
1227             emit_opcode(*cbuf, Assembler::REX_WR);
1228           }
1229           emit_opcode(*cbuf, 0x8B);
1230           encode_RegMem(*cbuf,
1231                         Matcher::_regEncode[dst_first],
1232                         RSP_enc, 0x4, 0, offset,
1233                         false);
1234 #ifndef PRODUCT
1235         } else if (!do_size) {
1236           st->print("movq    %s, [rsp + #%d]\t# spill",
1237                      Matcher::regName[dst_first],
1238                      offset);
1239 #endif
1240         }
1241         return
1242           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) + 4; // REX
1243       } else {
1244         // 32-bit
1245         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1246         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1247         int offset = ra_->reg2offset(src_first);
1248         if (cbuf) {
1249           if (Matcher::_regEncode[dst_first] >= 8) {
1250             emit_opcode(*cbuf, Assembler::REX_R);
1251           }
1252           emit_opcode(*cbuf, 0x8B);
1253           encode_RegMem(*cbuf,
1254                         Matcher::_regEncode[dst_first],
1255                         RSP_enc, 0x4, 0, offset,
1256                         false);
1257 #ifndef PRODUCT
1258         } else if (!do_size) {
1259           st->print("movl    %s, [rsp + #%d]\t# spill",
1260                      Matcher::regName[dst_first],
1261                      offset);
1262 #endif
1263         }
1264         return
1265           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1266           ((Matcher::_regEncode[dst_first] < 8)
1267            ? 3
1268            : 4); // REX
1269       }
1270     } else if (dst_first_rc == rc_float) {
1271       // mem-> xmm
1272       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1273           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1274         // 64-bit
1275         int offset = ra_->reg2offset(src_first);
1276         if (cbuf) {
1277           emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
1278           if (Matcher::_regEncode[dst_first] >= 8) {
1279             emit_opcode(*cbuf, Assembler::REX_R);
1280           }
1281           emit_opcode(*cbuf, 0x0F);
1282           emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
1283           encode_RegMem(*cbuf,
1284                         Matcher::_regEncode[dst_first],
1285                         RSP_enc, 0x4, 0, offset,
1286                         false);
1287 #ifndef PRODUCT
1288         } else if (!do_size) {
1289           st->print("%s  %s, [rsp + #%d]\t# spill",
1290                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1291                      Matcher::regName[dst_first],
1292                      offset);
1293 #endif
1294         }
1295         return
1296           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1297           ((Matcher::_regEncode[dst_first] < 8)
1298            ? 5
1299            : 6); // REX
1300       } else {
1301         // 32-bit
1302         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1303         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1304         int offset = ra_->reg2offset(src_first);
1305         if (cbuf) {
1306           emit_opcode(*cbuf, 0xF3);
1307           if (Matcher::_regEncode[dst_first] >= 8) {
1308             emit_opcode(*cbuf, Assembler::REX_R);
1309           }
1310           emit_opcode(*cbuf, 0x0F);
1311           emit_opcode(*cbuf, 0x10);
1312           encode_RegMem(*cbuf,
1313                         Matcher::_regEncode[dst_first],
1314                         RSP_enc, 0x4, 0, offset,
1315                         false);
1316 #ifndef PRODUCT
1317         } else if (!do_size) {
1318           st->print("movss   %s, [rsp + #%d]\t# spill",
1319                      Matcher::regName[dst_first],
1320                      offset);
1321 #endif
1322         }
1323         return
1324           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1325           ((Matcher::_regEncode[dst_first] < 8)
1326            ? 5
1327            : 6); // REX
1328       }
1329     }
1330   } else if (src_first_rc == rc_int) {
1331     // gpr ->
1332     if (dst_first_rc == rc_stack) {
1333       // gpr -> mem
1334       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1335           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1336         // 64-bit
1337         int offset = ra_->reg2offset(dst_first);
1338         if (cbuf) {
1339           if (Matcher::_regEncode[src_first] < 8) {
1340             emit_opcode(*cbuf, Assembler::REX_W);
1341           } else {
1342             emit_opcode(*cbuf, Assembler::REX_WR);
1343           }
1344           emit_opcode(*cbuf, 0x89);
1345           encode_RegMem(*cbuf,
1346                         Matcher::_regEncode[src_first],
1347                         RSP_enc, 0x4, 0, offset,
1348                         false);
1349 #ifndef PRODUCT
1350         } else if (!do_size) {
1351           st->print("movq    [rsp + #%d], %s\t# spill",
1352                      offset,
1353                      Matcher::regName[src_first]);
1354 #endif
1355         }
1356         return ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) + 4; // REX
1357       } else {
1358         // 32-bit
1359         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1360         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1361         int offset = ra_->reg2offset(dst_first);
1362         if (cbuf) {
1363           if (Matcher::_regEncode[src_first] >= 8) {
1364             emit_opcode(*cbuf, Assembler::REX_R);
1365           }
1366           emit_opcode(*cbuf, 0x89);
1367           encode_RegMem(*cbuf,
1368                         Matcher::_regEncode[src_first],
1369                         RSP_enc, 0x4, 0, offset,
1370                         false);
1371 #ifndef PRODUCT
1372         } else if (!do_size) {
1373           st->print("movl    [rsp + #%d], %s\t# spill",
1374                      offset,
1375                      Matcher::regName[src_first]);
1376 #endif
1377         }
1378         return
1379           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1380           ((Matcher::_regEncode[src_first] < 8)
1381            ? 3
1382            : 4); // REX
1383       }
1384     } else if (dst_first_rc == rc_int) {
1385       // gpr -> gpr
1386       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1387           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1388         // 64-bit
1389         if (cbuf) {
1390           if (Matcher::_regEncode[dst_first] < 8) {
1391             if (Matcher::_regEncode[src_first] < 8) {
1392               emit_opcode(*cbuf, Assembler::REX_W);
1393             } else {
1394               emit_opcode(*cbuf, Assembler::REX_WB);
1395             }
1396           } else {
1397             if (Matcher::_regEncode[src_first] < 8) {
1398               emit_opcode(*cbuf, Assembler::REX_WR);
1399             } else {
1400               emit_opcode(*cbuf, Assembler::REX_WRB);
1401             }
1402           }
1403           emit_opcode(*cbuf, 0x8B);
1404           emit_rm(*cbuf, 0x3,
1405                   Matcher::_regEncode[dst_first] & 7,
1406                   Matcher::_regEncode[src_first] & 7);
1407 #ifndef PRODUCT
1408         } else if (!do_size) {
1409           st->print("movq    %s, %s\t# spill",
1410                      Matcher::regName[dst_first],
1411                      Matcher::regName[src_first]);
1412 #endif
1413         }
1414         return 3; // REX
1415       } else {
1416         // 32-bit
1417         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1418         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1419         if (cbuf) {
1420           if (Matcher::_regEncode[dst_first] < 8) {
1421             if (Matcher::_regEncode[src_first] >= 8) {
1422               emit_opcode(*cbuf, Assembler::REX_B);
1423             }
1424           } else {
1425             if (Matcher::_regEncode[src_first] < 8) {
1426               emit_opcode(*cbuf, Assembler::REX_R);
1427             } else {
1428               emit_opcode(*cbuf, Assembler::REX_RB);
1429             }
1430           }
1431           emit_opcode(*cbuf, 0x8B);
1432           emit_rm(*cbuf, 0x3,
1433                   Matcher::_regEncode[dst_first] & 7,
1434                   Matcher::_regEncode[src_first] & 7);
1435 #ifndef PRODUCT
1436         } else if (!do_size) {
1437           st->print("movl    %s, %s\t# spill",
1438                      Matcher::regName[dst_first],
1439                      Matcher::regName[src_first]);
1440 #endif
1441         }
1442         return
1443           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
1444           ? 2
1445           : 3; // REX
1446       }
1447     } else if (dst_first_rc == rc_float) {
1448       // gpr -> xmm
1449       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1450           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1451         // 64-bit
1452         if (cbuf) {
1453           emit_opcode(*cbuf, 0x66);
1454           if (Matcher::_regEncode[dst_first] < 8) {
1455             if (Matcher::_regEncode[src_first] < 8) {
1456               emit_opcode(*cbuf, Assembler::REX_W);
1457             } else {
1458               emit_opcode(*cbuf, Assembler::REX_WB);
1459             }
1460           } else {
1461             if (Matcher::_regEncode[src_first] < 8) {
1462               emit_opcode(*cbuf, Assembler::REX_WR);
1463             } else {
1464               emit_opcode(*cbuf, Assembler::REX_WRB);
1465             }
1466           }
1467           emit_opcode(*cbuf, 0x0F);
1468           emit_opcode(*cbuf, 0x6E);
1469           emit_rm(*cbuf, 0x3,
1470                   Matcher::_regEncode[dst_first] & 7,
1471                   Matcher::_regEncode[src_first] & 7);
1472 #ifndef PRODUCT
1473         } else if (!do_size) {
1474           st->print("movdq   %s, %s\t# spill",
1475                      Matcher::regName[dst_first],
1476                      Matcher::regName[src_first]);
1477 #endif
1478         }
1479         return 5; // REX
1480       } else {
1481         // 32-bit
1482         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1483         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1484         if (cbuf) {
1485           emit_opcode(*cbuf, 0x66);
1486           if (Matcher::_regEncode[dst_first] < 8) {
1487             if (Matcher::_regEncode[src_first] >= 8) {
1488               emit_opcode(*cbuf, Assembler::REX_B);
1489             }
1490           } else {
1491             if (Matcher::_regEncode[src_first] < 8) {
1492               emit_opcode(*cbuf, Assembler::REX_R);
1493             } else {
1494               emit_opcode(*cbuf, Assembler::REX_RB);
1495             }
1496           }
1497           emit_opcode(*cbuf, 0x0F);
1498           emit_opcode(*cbuf, 0x6E);
1499           emit_rm(*cbuf, 0x3,
1500                   Matcher::_regEncode[dst_first] & 7,
1501                   Matcher::_regEncode[src_first] & 7);
1502 #ifndef PRODUCT
1503         } else if (!do_size) {
1504           st->print("movdl   %s, %s\t# spill",
1505                      Matcher::regName[dst_first],
1506                      Matcher::regName[src_first]);
1507 #endif
1508         }
1509         return
1510           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
1511           ? 4
1512           : 5; // REX
1513       }
1514     }
1515   } else if (src_first_rc == rc_float) {
1516     // xmm ->
1517     if (dst_first_rc == rc_stack) {
1518       // xmm -> mem
1519       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1520           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1521         // 64-bit
1522         int offset = ra_->reg2offset(dst_first);
1523         if (cbuf) {
1524           emit_opcode(*cbuf, 0xF2);
1525           if (Matcher::_regEncode[src_first] >= 8) {
1526               emit_opcode(*cbuf, Assembler::REX_R);
1527           }
1528           emit_opcode(*cbuf, 0x0F);
1529           emit_opcode(*cbuf, 0x11);
1530           encode_RegMem(*cbuf,
1531                         Matcher::_regEncode[src_first],
1532                         RSP_enc, 0x4, 0, offset,
1533                         false);
1534 #ifndef PRODUCT
1535         } else if (!do_size) {
1536           st->print("movsd   [rsp + #%d], %s\t# spill",
1537                      offset,
1538                      Matcher::regName[src_first]);
1539 #endif
1540         }
1541         return
1542           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1543           ((Matcher::_regEncode[src_first] < 8)
1544            ? 5
1545            : 6); // REX
1546       } else {
1547         // 32-bit
1548         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1549         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1550         int offset = ra_->reg2offset(dst_first);
1551         if (cbuf) {
1552           emit_opcode(*cbuf, 0xF3);
1553           if (Matcher::_regEncode[src_first] >= 8) {
1554               emit_opcode(*cbuf, Assembler::REX_R);
1555           }
1556           emit_opcode(*cbuf, 0x0F);
1557           emit_opcode(*cbuf, 0x11);
1558           encode_RegMem(*cbuf,
1559                         Matcher::_regEncode[src_first],
1560                         RSP_enc, 0x4, 0, offset,
1561                         false);
1562 #ifndef PRODUCT
1563         } else if (!do_size) {
1564           st->print("movss   [rsp + #%d], %s\t# spill",
1565                      offset,
1566                      Matcher::regName[src_first]);
1567 #endif
1568         }
1569         return
1570           ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
1571           ((Matcher::_regEncode[src_first] < 8)
1572            ? 5
1573            : 6); // REX
1574       }
1575     } else if (dst_first_rc == rc_int) {
1576       // xmm -> gpr
1577       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1578           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1579         // 64-bit
1580         if (cbuf) {
1581           emit_opcode(*cbuf, 0x66);
1582           if (Matcher::_regEncode[dst_first] < 8) {
1583             if (Matcher::_regEncode[src_first] < 8) {
1584               emit_opcode(*cbuf, Assembler::REX_W);
1585             } else {
1586               emit_opcode(*cbuf, Assembler::REX_WR); // attention!
1587             }
1588           } else {
1589             if (Matcher::_regEncode[src_first] < 8) {
1590               emit_opcode(*cbuf, Assembler::REX_WB); // attention!
1591             } else {
1592               emit_opcode(*cbuf, Assembler::REX_WRB);
1593             }
1594           }
1595           emit_opcode(*cbuf, 0x0F);
1596           emit_opcode(*cbuf, 0x7E);
1597           emit_rm(*cbuf, 0x3,
1598                   Matcher::_regEncode[src_first] & 7,
1599                   Matcher::_regEncode[dst_first] & 7);
1600 #ifndef PRODUCT
1601         } else if (!do_size) {
1602           st->print("movdq   %s, %s\t# spill",
1603                      Matcher::regName[dst_first],
1604                      Matcher::regName[src_first]);
1605 #endif
1606         }
1607         return 5; // REX
1608       } else {
1609         // 32-bit
1610         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1611         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1612         if (cbuf) {
1613           emit_opcode(*cbuf, 0x66);
1614           if (Matcher::_regEncode[dst_first] < 8) {
1615             if (Matcher::_regEncode[src_first] >= 8) {
1616               emit_opcode(*cbuf, Assembler::REX_R); // attention!
1617             }
1618           } else {
1619             if (Matcher::_regEncode[src_first] < 8) {
1620               emit_opcode(*cbuf, Assembler::REX_B); // attention!
1621             } else {
1622               emit_opcode(*cbuf, Assembler::REX_RB);
1623             }
1624           }
1625           emit_opcode(*cbuf, 0x0F);
1626           emit_opcode(*cbuf, 0x7E);
1627           emit_rm(*cbuf, 0x3,
1628                   Matcher::_regEncode[src_first] & 7,
1629                   Matcher::_regEncode[dst_first] & 7);
1630 #ifndef PRODUCT
1631         } else if (!do_size) {
1632           st->print("movdl   %s, %s\t# spill",
1633                      Matcher::regName[dst_first],
1634                      Matcher::regName[src_first]);
1635 #endif
1636         }
1637         return
1638           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
1639           ? 4
1640           : 5; // REX
1641       }
1642     } else if (dst_first_rc == rc_float) {
1643       // xmm -> xmm
1644       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1645           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1646         // 64-bit
1647         if (cbuf) {
1648           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
1649           if (Matcher::_regEncode[dst_first] < 8) {
1650             if (Matcher::_regEncode[src_first] >= 8) {
1651               emit_opcode(*cbuf, Assembler::REX_B);
1652             }
1653           } else {
1654             if (Matcher::_regEncode[src_first] < 8) {
1655               emit_opcode(*cbuf, Assembler::REX_R);
1656             } else {
1657               emit_opcode(*cbuf, Assembler::REX_RB);
1658             }
1659           }
1660           emit_opcode(*cbuf, 0x0F);
1661           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
1662           emit_rm(*cbuf, 0x3,
1663                   Matcher::_regEncode[dst_first] & 7,
1664                   Matcher::_regEncode[src_first] & 7);
1665 #ifndef PRODUCT
1666         } else if (!do_size) {
1667           st->print("%s  %s, %s\t# spill",
1668                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1669                      Matcher::regName[dst_first],
1670                      Matcher::regName[src_first]);
1671 #endif
1672         }
1673         return
1674           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
1675           ? 4
1676           : 5; // REX
1677       } else {
1678         // 32-bit
1679         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1680         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1681         if (cbuf) {
1682           if (!UseXmmRegToRegMoveAll)
1683             emit_opcode(*cbuf, 0xF3);
1684           if (Matcher::_regEncode[dst_first] < 8) {
1685             if (Matcher::_regEncode[src_first] >= 8) {
1686               emit_opcode(*cbuf, Assembler::REX_B);
1687             }
1688           } else {
1689             if (Matcher::_regEncode[src_first] < 8) {
1690               emit_opcode(*cbuf, Assembler::REX_R);
1691             } else {
1692               emit_opcode(*cbuf, Assembler::REX_RB);
1693             }
1694           }
1695           emit_opcode(*cbuf, 0x0F);
1696           emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
1697           emit_rm(*cbuf, 0x3,
1698                   Matcher::_regEncode[dst_first] & 7,
1699                   Matcher::_regEncode[src_first] & 7);
1700 #ifndef PRODUCT
1701         } else if (!do_size) {
1702           st->print("%s  %s, %s\t# spill",
1703                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1704                      Matcher::regName[dst_first],
1705                      Matcher::regName[src_first]);
1706 #endif
1707         }
1708         return
1709           (Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
1710           ? (UseXmmRegToRegMoveAll ? 3 : 4)
1711           : (UseXmmRegToRegMoveAll ? 4 : 5); // REX
1712       }
1713     }
1714   }
1715 
1716   assert(0," foo ");
1717   Unimplemented();
1718 
1719   return 0;
1720 }
1721 
1722 #ifndef PRODUCT
1723 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const
1724 {
1725   implementation(NULL, ra_, false, st);
1726 }
1727 #endif
1728 
1729 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const
1730 {
1731   implementation(&cbuf, ra_, false, NULL);
1732 }
1733 
1734 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const
1735 {
1736   return implementation(NULL, ra_, true, NULL);
1737 }
1738 
1739 //=============================================================================
1740 #ifndef PRODUCT
1741 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const
1742 {
1743   st->print("nop \t# %d bytes pad for loops and calls", _count);
1744 }
1745 #endif
1746 
1747 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const
1748 {
1749   MacroAssembler _masm(&cbuf);
1750   __ nop(_count);
1751 }
1752 
1753 uint MachNopNode::size(PhaseRegAlloc*) const
1754 {
1755   return _count;
1756 }
1757 
1758 
1759 //=============================================================================
1760 #ifndef PRODUCT
1761 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1762 {
1763   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1764   int reg = ra_->get_reg_first(this);
1765   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1766             Matcher::regName[reg], offset);
1767 }
1768 #endif
1769 
1770 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1771 {
1772   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1773   int reg = ra_->get_encode(this);
1774   if (offset >= 0x80) {
1775     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1776     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1777     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1778     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1779     emit_d32(cbuf, offset);
1780   } else {
1781     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1782     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1783     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1784     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1785     emit_d8(cbuf, offset);
1786   }
1787 }
1788 
1789 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1790 {
1791   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1792   return (offset < 0x80) ? 5 : 8; // REX
1793 }
1794 
1795 //=============================================================================
1796 
1797 // emit call stub, compiled java to interpreter
1798 void emit_java_to_interp(CodeBuffer& cbuf)
1799 {
1800   // Stub is fixed up when the corresponding call is converted from
1801   // calling compiled code to calling interpreted code.
1802   // movq rbx, 0
1803   // jmp -5 # to self
1804 
1805   address mark = cbuf.insts_mark();  // get mark within main instrs section
1806 
1807   // Note that the code buffer's insts_mark is always relative to insts.
1808   // That's why we must use the macroassembler to generate a stub.
1809   MacroAssembler _masm(&cbuf);
1810 
1811   address base =
1812   __ start_a_stub(Compile::MAX_stubs_size);
1813   if (base == NULL)  return;  // CodeBuffer::expand failed
1814   // static stub relocation stores the instruction address of the call
1815   __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64);
1816   // static stub relocation also tags the methodOop in the code-stream.
1817   __ movoop(rbx, (jobject) NULL);  // method is zapped till fixup time
1818   // This is recognized as unresolved by relocs/nativeinst/ic code
1819   __ jump(RuntimeAddress(__ pc()));
1820 
1821   // Update current stubs pointer and restore insts_end.
1822   __ end_a_stub();
1823 }
1824 
1825 // size of call stub, compiled java to interpretor
1826 uint size_java_to_interp()
1827 {
1828   return 15;  // movq (1+1+8); jmp (1+4)
1829 }
1830 
1831 // relocation entries for call stub, compiled java to interpretor
1832 uint reloc_java_to_interp()
1833 {
1834   return 4; // 3 in emit_java_to_interp + 1 in Java_Static_Call
1835 }
1836 
1837 //=============================================================================
1838 #ifndef PRODUCT
1839 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1840 {
1841   if (UseCompressedOops) {
1842     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1843     if (Universe::narrow_oop_shift() != 0) {
1844       st->print_cr("\tdecode_heap_oop_not_null rscratch1, rscratch1");
1845     }
1846     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1847   } else {
1848     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1849                  "# Inline cache check");
1850   }
1851   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1852   st->print_cr("\tnop\t# nops to align entry point");
1853 }
1854 #endif
1855 
1856 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1857 {
1858   MacroAssembler masm(&cbuf);
1859   uint insts_size = cbuf.insts_size();
1860   if (UseCompressedOops) {
1861     masm.load_klass(rscratch1, j_rarg0);
1862     masm.cmpptr(rax, rscratch1);
1863   } else {
1864     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1865   }
1866 
1867   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1868 
1869   /* WARNING these NOPs are critical so that verified entry point is properly
1870      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1871   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1872   if (OptoBreakpoint) {
1873     // Leave space for int3
1874     nops_cnt -= 1;
1875   }
1876   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1877   if (nops_cnt > 0)
1878     masm.nop(nops_cnt);
1879 }
1880 
1881 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1882 {
1883   return MachNode::size(ra_); // too many variables; just compute it
1884                               // the hard way
1885 }
1886 
1887 
1888 //=============================================================================
1889 uint size_exception_handler()
1890 {
1891   // NativeCall instruction size is the same as NativeJump.
1892   // Note that this value is also credited (in output.cpp) to
1893   // the size of the code section.
1894   return NativeJump::instruction_size;
1895 }
1896 
1897 // Emit exception handler code.
1898 int emit_exception_handler(CodeBuffer& cbuf)
1899 {
1900 
1901   // Note that the code buffer's insts_mark is always relative to insts.
1902   // That's why we must use the macroassembler to generate a handler.
1903   MacroAssembler _masm(&cbuf);
1904   address base =
1905   __ start_a_stub(size_exception_handler());
1906   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1907   int offset = __ offset();
1908   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1909   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1910   __ end_a_stub();
1911   return offset;
1912 }
1913 
1914 uint size_deopt_handler()
1915 {
1916   // three 5 byte instructions
1917   return 15;
1918 }
1919 
1920 // Emit deopt handler code.
1921 int emit_deopt_handler(CodeBuffer& cbuf)
1922 {
1923 
1924   // Note that the code buffer's insts_mark is always relative to insts.
1925   // That's why we must use the macroassembler to generate a handler.
1926   MacroAssembler _masm(&cbuf);
1927   address base =
1928   __ start_a_stub(size_deopt_handler());
1929   if (base == NULL)  return 0;  // CodeBuffer::expand failed
1930   int offset = __ offset();
1931   address the_pc = (address) __ pc();
1932   Label next;
1933   // push a "the_pc" on the stack without destroying any registers
1934   // as they all may be live.
1935 
1936   // push address of "next"
1937   __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1938   __ bind(next);
1939   // adjust it so it matches "the_pc"
1940   __ subptr(Address(rsp, 0), __ offset() - offset);
1941   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1942   assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1943   __ end_a_stub();
1944   return offset;
1945 }
1946 
1947 
1948 const bool Matcher::match_rule_supported(int opcode) {
1949   if (!has_match_rule(opcode))
1950     return false;
1951 
1952   return true;  // Per default match rules are supported.
1953 }
1954 
1955 int Matcher::regnum_to_fpu_offset(int regnum)
1956 {
1957   return regnum - 32; // The FP registers are in the second chunk
1958 }
1959 
1960 // This is UltraSparc specific, true just means we have fast l2f conversion
1961 const bool Matcher::convL2FSupported(void) {
1962   return true;
1963 }
1964 
1965 // Vector width in bytes
1966 const uint Matcher::vector_width_in_bytes(void) {
1967   return 8;
1968 }
1969 
1970 // Vector ideal reg
1971 const uint Matcher::vector_ideal_reg(void) {
1972   return Op_RegD;
1973 }
1974 
1975 // Is this branch offset short enough that a short branch can be used?
1976 //
1977 // NOTE: If the platform does not provide any short branch variants, then
1978 //       this method should return false for offset 0.
1979 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1980   // The passed offset is relative to address of the branch.
1981   // On 86 a branch displacement is calculated relative to address
1982   // of a next instruction.
1983   offset -= br_size;
1984 
1985   // the short version of jmpConUCF2 contains multiple branches,
1986   // making the reach slightly less
1987   if (rule == jmpConUCF2_rule)
1988     return (-126 <= offset && offset <= 125);
1989   return (-128 <= offset && offset <= 127);
1990 }
1991 
1992 const bool Matcher::isSimpleConstant64(jlong value) {
1993   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1994   //return value == (int) value;  // Cf. storeImmL and immL32.
1995 
1996   // Probably always true, even if a temp register is required.
1997   return true;
1998 }
1999 
2000 // The ecx parameter to rep stosq for the ClearArray node is in words.
2001 const bool Matcher::init_array_count_is_in_bytes = false;
2002 
2003 // Threshold size for cleararray.
2004 const int Matcher::init_array_short_size = 8 * BytesPerLong;
2005 
2006 // No additional cost for CMOVL.
2007 const int Matcher::long_cmove_cost() { return 0; }
2008 
2009 // No CMOVF/CMOVD with SSE2
2010 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
2011 
2012 // Should the Matcher clone shifts on addressing modes, expecting them
2013 // to be subsumed into complex addressing expressions or compute them
2014 // into registers?  True for Intel but false for most RISCs
2015 const bool Matcher::clone_shift_expressions = true;
2016 
2017 // Do we need to mask the count passed to shift instructions or does
2018 // the cpu only look at the lower 5/6 bits anyway?
2019 const bool Matcher::need_masked_shift_count = false;
2020 
2021 bool Matcher::narrow_oop_use_complex_address() {
2022   assert(UseCompressedOops, "only for compressed oops code");
2023   return (LogMinObjAlignmentInBytes <= 3);
2024 }
2025 
2026 // Is it better to copy float constants, or load them directly from
2027 // memory?  Intel can load a float constant from a direct address,
2028 // requiring no extra registers.  Most RISCs will have to materialize
2029 // an address into a register first, so they would do better to copy
2030 // the constant from stack.
2031 const bool Matcher::rematerialize_float_constants = true; // XXX
2032 
2033 // If CPU can load and store mis-aligned doubles directly then no
2034 // fixup is needed.  Else we split the double into 2 integer pieces
2035 // and move it piece-by-piece.  Only happens when passing doubles into
2036 // C code as the Java calling convention forces doubles to be aligned.
2037 const bool Matcher::misaligned_doubles_ok = true;
2038 
2039 // No-op on amd64
2040 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
2041 
2042 // Advertise here if the CPU requires explicit rounding operations to
2043 // implement the UseStrictFP mode.
2044 const bool Matcher::strict_fp_requires_explicit_rounding = true;
2045 
2046 // Are floats conerted to double when stored to stack during deoptimization?
2047 // On x64 it is stored without convertion so we can use normal access.
2048 bool Matcher::float_in_double() { return false; }
2049 
2050 // Do ints take an entire long register or just half?
2051 const bool Matcher::int_in_long = true;
2052 
2053 // Return whether or not this register is ever used as an argument.
2054 // This function is used on startup to build the trampoline stubs in
2055 // generateOptoStub.  Registers not mentioned will be killed by the VM
2056 // call in the trampoline, and arguments in those registers not be
2057 // available to the callee.
2058 bool Matcher::can_be_java_arg(int reg)
2059 {
2060   return
2061     reg ==  RDI_num || reg ==  RDI_H_num ||
2062     reg ==  RSI_num || reg ==  RSI_H_num ||
2063     reg ==  RDX_num || reg ==  RDX_H_num ||
2064     reg ==  RCX_num || reg ==  RCX_H_num ||
2065     reg ==   R8_num || reg ==   R8_H_num ||
2066     reg ==   R9_num || reg ==   R9_H_num ||
2067     reg ==  R12_num || reg ==  R12_H_num ||
2068     reg == XMM0_num || reg == XMM0_H_num ||
2069     reg == XMM1_num || reg == XMM1_H_num ||
2070     reg == XMM2_num || reg == XMM2_H_num ||
2071     reg == XMM3_num || reg == XMM3_H_num ||
2072     reg == XMM4_num || reg == XMM4_H_num ||
2073     reg == XMM5_num || reg == XMM5_H_num ||
2074     reg == XMM6_num || reg == XMM6_H_num ||
2075     reg == XMM7_num || reg == XMM7_H_num;
2076 }
2077 
2078 bool Matcher::is_spillable_arg(int reg)
2079 {
2080   return can_be_java_arg(reg);
2081 }
2082 
2083 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
2084   // In 64 bit mode a code which use multiply when
2085   // devisor is constant is faster than hardware
2086   // DIV instruction (it uses MulHiL).
2087   return false;
2088 }
2089 
2090 // Register for DIVI projection of divmodI
2091 RegMask Matcher::divI_proj_mask() {
2092   return INT_RAX_REG_mask();
2093 }
2094 
2095 // Register for MODI projection of divmodI
2096 RegMask Matcher::modI_proj_mask() {
2097   return INT_RDX_REG_mask();
2098 }
2099 
2100 // Register for DIVL projection of divmodL
2101 RegMask Matcher::divL_proj_mask() {
2102   return LONG_RAX_REG_mask();
2103 }
2104 
2105 // Register for MODL projection of divmodL
2106 RegMask Matcher::modL_proj_mask() {
2107   return LONG_RDX_REG_mask();
2108 }
2109 
2110 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2111   return PTR_RBP_REG_mask();
2112 }
2113 
2114 static Address build_address(int b, int i, int s, int d) {
2115   Register index = as_Register(i);
2116   Address::ScaleFactor scale = (Address::ScaleFactor)s;
2117   if (index == rsp) {
2118     index = noreg;
2119     scale = Address::no_scale;
2120   }
2121   Address addr(as_Register(b), index, scale, d);
2122   return addr;
2123 }
2124 
2125 %}
2126 
2127 //----------ENCODING BLOCK-----------------------------------------------------
2128 // This block specifies the encoding classes used by the compiler to
2129 // output byte streams.  Encoding classes are parameterized macros
2130 // used by Machine Instruction Nodes in order to generate the bit
2131 // encoding of the instruction.  Operands specify their base encoding
2132 // interface with the interface keyword.  There are currently
2133 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2134 // COND_INTER.  REG_INTER causes an operand to generate a function
2135 // which returns its register number when queried.  CONST_INTER causes
2136 // an operand to generate a function which returns the value of the
2137 // constant when queried.  MEMORY_INTER causes an operand to generate
2138 // four functions which return the Base Register, the Index Register,
2139 // the Scale Value, and the Offset Value of the operand when queried.
2140 // COND_INTER causes an operand to generate six functions which return
2141 // the encoding code (ie - encoding bits for the instruction)
2142 // associated with each basic boolean condition for a conditional
2143 // instruction.
2144 //
2145 // Instructions specify two basic values for encoding.  Again, a
2146 // function is available to check if the constant displacement is an
2147 // oop. They use the ins_encode keyword to specify their encoding
2148 // classes (which must be a sequence of enc_class names, and their
2149 // parameters, specified in the encoding block), and they use the
2150 // opcode keyword to specify, in order, their primary, secondary, and
2151 // tertiary opcode.  Only the opcode sections which a particular
2152 // instruction needs for encoding need to be specified.
2153 encode %{
2154   // Build emit functions for each basic byte or larger field in the
2155   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2156   // from C++ code in the enc_class source block.  Emit functions will
2157   // live in the main source block for now.  In future, we can
2158   // generalize this by adding a syntax that specifies the sizes of
2159   // fields in an order, so that the adlc can build the emit functions
2160   // automagically
2161 
2162   // Emit primary opcode
2163   enc_class OpcP
2164   %{
2165     emit_opcode(cbuf, $primary);
2166   %}
2167 
2168   // Emit secondary opcode
2169   enc_class OpcS
2170   %{
2171     emit_opcode(cbuf, $secondary);
2172   %}
2173 
2174   // Emit tertiary opcode
2175   enc_class OpcT
2176   %{
2177     emit_opcode(cbuf, $tertiary);
2178   %}
2179 
2180   // Emit opcode directly
2181   enc_class Opcode(immI d8)
2182   %{
2183     emit_opcode(cbuf, $d8$$constant);
2184   %}
2185 
2186   // Emit size prefix
2187   enc_class SizePrefix
2188   %{
2189     emit_opcode(cbuf, 0x66);
2190   %}
2191 
2192   enc_class reg(rRegI reg)
2193   %{
2194     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
2195   %}
2196 
2197   enc_class reg_reg(rRegI dst, rRegI src)
2198   %{
2199     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2200   %}
2201 
2202   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
2203   %{
2204     emit_opcode(cbuf, $opcode$$constant);
2205     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2206   %}
2207 
2208   enc_class cmpfp_fixup() %{
2209       MacroAssembler _masm(&cbuf);
2210       emit_cmpfp_fixup(_masm);
2211   %}
2212 
2213   enc_class cmpfp3(rRegI dst)
2214   %{
2215     int dstenc = $dst$$reg;
2216 
2217     // movl $dst, -1
2218     if (dstenc >= 8) {
2219       emit_opcode(cbuf, Assembler::REX_B);
2220     }
2221     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2222     emit_d32(cbuf, -1);
2223 
2224     // jp,s done
2225     emit_opcode(cbuf, 0x7A);
2226     emit_d8(cbuf, dstenc < 4 ? 0x08 : 0x0A);
2227 
2228     // jb,s done
2229     emit_opcode(cbuf, 0x72);
2230     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2231 
2232     // setne $dst
2233     if (dstenc >= 4) {
2234       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2235     }
2236     emit_opcode(cbuf, 0x0F);
2237     emit_opcode(cbuf, 0x95);
2238     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2239 
2240     // movzbl $dst, $dst
2241     if (dstenc >= 4) {
2242       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2243     }
2244     emit_opcode(cbuf, 0x0F);
2245     emit_opcode(cbuf, 0xB6);
2246     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2247   %}
2248 
2249   enc_class cdql_enc(no_rax_rdx_RegI div)
2250   %{
2251     // Full implementation of Java idiv and irem; checks for
2252     // special case as described in JVM spec., p.243 & p.271.
2253     //
2254     //         normal case                           special case
2255     //
2256     // input : rax: dividend                         min_int
2257     //         reg: divisor                          -1
2258     //
2259     // output: rax: quotient  (= rax idiv reg)       min_int
2260     //         rdx: remainder (= rax irem reg)       0
2261     //
2262     //  Code sequnce:
2263     //
2264     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
2265     //    5:   75 07/08                jne    e <normal>
2266     //    7:   33 d2                   xor    %edx,%edx
2267     //  [div >= 8 -> offset + 1]
2268     //  [REX_B]
2269     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
2270     //    c:   74 03/04                je     11 <done>
2271     // 000000000000000e <normal>:
2272     //    e:   99                      cltd
2273     //  [div >= 8 -> offset + 1]
2274     //  [REX_B]
2275     //    f:   f7 f9                   idiv   $div
2276     // 0000000000000011 <done>:
2277 
2278     // cmp    $0x80000000,%eax
2279     emit_opcode(cbuf, 0x3d);
2280     emit_d8(cbuf, 0x00);
2281     emit_d8(cbuf, 0x00);
2282     emit_d8(cbuf, 0x00);
2283     emit_d8(cbuf, 0x80);
2284 
2285     // jne    e <normal>
2286     emit_opcode(cbuf, 0x75);
2287     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
2288 
2289     // xor    %edx,%edx
2290     emit_opcode(cbuf, 0x33);
2291     emit_d8(cbuf, 0xD2);
2292 
2293     // cmp    $0xffffffffffffffff,%ecx
2294     if ($div$$reg >= 8) {
2295       emit_opcode(cbuf, Assembler::REX_B);
2296     }
2297     emit_opcode(cbuf, 0x83);
2298     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2299     emit_d8(cbuf, 0xFF);
2300 
2301     // je     11 <done>
2302     emit_opcode(cbuf, 0x74);
2303     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
2304 
2305     // <normal>
2306     // cltd
2307     emit_opcode(cbuf, 0x99);
2308 
2309     // idivl (note: must be emitted by the user of this rule)
2310     // <done>
2311   %}
2312 
2313   enc_class cdqq_enc(no_rax_rdx_RegL div)
2314   %{
2315     // Full implementation of Java ldiv and lrem; checks for
2316     // special case as described in JVM spec., p.243 & p.271.
2317     //
2318     //         normal case                           special case
2319     //
2320     // input : rax: dividend                         min_long
2321     //         reg: divisor                          -1
2322     //
2323     // output: rax: quotient  (= rax idiv reg)       min_long
2324     //         rdx: remainder (= rax irem reg)       0
2325     //
2326     //  Code sequnce:
2327     //
2328     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
2329     //    7:   00 00 80
2330     //    a:   48 39 d0                cmp    %rdx,%rax
2331     //    d:   75 08                   jne    17 <normal>
2332     //    f:   33 d2                   xor    %edx,%edx
2333     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
2334     //   15:   74 05                   je     1c <done>
2335     // 0000000000000017 <normal>:
2336     //   17:   48 99                   cqto
2337     //   19:   48 f7 f9                idiv   $div
2338     // 000000000000001c <done>:
2339 
2340     // mov    $0x8000000000000000,%rdx
2341     emit_opcode(cbuf, Assembler::REX_W);
2342     emit_opcode(cbuf, 0xBA);
2343     emit_d8(cbuf, 0x00);
2344     emit_d8(cbuf, 0x00);
2345     emit_d8(cbuf, 0x00);
2346     emit_d8(cbuf, 0x00);
2347     emit_d8(cbuf, 0x00);
2348     emit_d8(cbuf, 0x00);
2349     emit_d8(cbuf, 0x00);
2350     emit_d8(cbuf, 0x80);
2351 
2352     // cmp    %rdx,%rax
2353     emit_opcode(cbuf, Assembler::REX_W);
2354     emit_opcode(cbuf, 0x39);
2355     emit_d8(cbuf, 0xD0);
2356 
2357     // jne    17 <normal>
2358     emit_opcode(cbuf, 0x75);
2359     emit_d8(cbuf, 0x08);
2360 
2361     // xor    %edx,%edx
2362     emit_opcode(cbuf, 0x33);
2363     emit_d8(cbuf, 0xD2);
2364 
2365     // cmp    $0xffffffffffffffff,$div
2366     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
2367     emit_opcode(cbuf, 0x83);
2368     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2369     emit_d8(cbuf, 0xFF);
2370 
2371     // je     1e <done>
2372     emit_opcode(cbuf, 0x74);
2373     emit_d8(cbuf, 0x05);
2374 
2375     // <normal>
2376     // cqto
2377     emit_opcode(cbuf, Assembler::REX_W);
2378     emit_opcode(cbuf, 0x99);
2379 
2380     // idivq (note: must be emitted by the user of this rule)
2381     // <done>
2382   %}
2383 
2384   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2385   enc_class OpcSE(immI imm)
2386   %{
2387     // Emit primary opcode and set sign-extend bit
2388     // Check for 8-bit immediate, and set sign extend bit in opcode
2389     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2390       emit_opcode(cbuf, $primary | 0x02);
2391     } else {
2392       // 32-bit immediate
2393       emit_opcode(cbuf, $primary);
2394     }
2395   %}
2396 
2397   enc_class OpcSErm(rRegI dst, immI imm)
2398   %{
2399     // OpcSEr/m
2400     int dstenc = $dst$$reg;
2401     if (dstenc >= 8) {
2402       emit_opcode(cbuf, Assembler::REX_B);
2403       dstenc -= 8;
2404     }
2405     // Emit primary opcode and set sign-extend bit
2406     // Check for 8-bit immediate, and set sign extend bit in opcode
2407     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2408       emit_opcode(cbuf, $primary | 0x02);
2409     } else {
2410       // 32-bit immediate
2411       emit_opcode(cbuf, $primary);
2412     }
2413     // Emit r/m byte with secondary opcode, after primary opcode.
2414     emit_rm(cbuf, 0x3, $secondary, dstenc);
2415   %}
2416 
2417   enc_class OpcSErm_wide(rRegL dst, immI imm)
2418   %{
2419     // OpcSEr/m
2420     int dstenc = $dst$$reg;
2421     if (dstenc < 8) {
2422       emit_opcode(cbuf, Assembler::REX_W);
2423     } else {
2424       emit_opcode(cbuf, Assembler::REX_WB);
2425       dstenc -= 8;
2426     }
2427     // Emit primary opcode and set sign-extend bit
2428     // Check for 8-bit immediate, and set sign extend bit in opcode
2429     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2430       emit_opcode(cbuf, $primary | 0x02);
2431     } else {
2432       // 32-bit immediate
2433       emit_opcode(cbuf, $primary);
2434     }
2435     // Emit r/m byte with secondary opcode, after primary opcode.
2436     emit_rm(cbuf, 0x3, $secondary, dstenc);
2437   %}
2438 
2439   enc_class Con8or32(immI imm)
2440   %{
2441     // Check for 8-bit immediate, and set sign extend bit in opcode
2442     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2443       $$$emit8$imm$$constant;
2444     } else {
2445       // 32-bit immediate
2446       $$$emit32$imm$$constant;
2447     }
2448   %}
2449 
2450   enc_class opc2_reg(rRegI dst)
2451   %{
2452     // BSWAP
2453     emit_cc(cbuf, $secondary, $dst$$reg);
2454   %}
2455 
2456   enc_class opc3_reg(rRegI dst)
2457   %{
2458     // BSWAP
2459     emit_cc(cbuf, $tertiary, $dst$$reg);
2460   %}
2461 
2462   enc_class reg_opc(rRegI div)
2463   %{
2464     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2465     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2466   %}
2467 
2468   enc_class enc_cmov(cmpOp cop)
2469   %{
2470     // CMOV
2471     $$$emit8$primary;
2472     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2473   %}
2474 
2475   enc_class enc_cmovf_branch(cmpOp cop, regF dst, regF src)
2476   %{
2477     // Invert sense of branch from sense of cmov
2478     emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
2479     emit_d8(cbuf, ($dst$$reg < 8 && $src$$reg < 8)
2480                   ? (UseXmmRegToRegMoveAll ? 3 : 4)
2481                   : (UseXmmRegToRegMoveAll ? 4 : 5) ); // REX
2482     // UseXmmRegToRegMoveAll ? movaps(dst, src) : movss(dst, src)
2483     if (!UseXmmRegToRegMoveAll) emit_opcode(cbuf, 0xF3);
2484     if ($dst$$reg < 8) {
2485       if ($src$$reg >= 8) {
2486         emit_opcode(cbuf, Assembler::REX_B);
2487       }
2488     } else {
2489       if ($src$$reg < 8) {
2490         emit_opcode(cbuf, Assembler::REX_R);
2491       } else {
2492         emit_opcode(cbuf, Assembler::REX_RB);
2493       }
2494     }
2495     emit_opcode(cbuf, 0x0F);
2496     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
2497     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2498   %}
2499 
2500   enc_class enc_cmovd_branch(cmpOp cop, regD dst, regD src)
2501   %{
2502     // Invert sense of branch from sense of cmov
2503     emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
2504     emit_d8(cbuf, $dst$$reg < 8 && $src$$reg < 8 ? 4 : 5); // REX
2505 
2506     //  UseXmmRegToRegMoveAll ? movapd(dst, src) : movsd(dst, src)
2507     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
2508     if ($dst$$reg < 8) {
2509       if ($src$$reg >= 8) {
2510         emit_opcode(cbuf, Assembler::REX_B);
2511       }
2512     } else {
2513       if ($src$$reg < 8) {
2514         emit_opcode(cbuf, Assembler::REX_R);
2515       } else {
2516         emit_opcode(cbuf, Assembler::REX_RB);
2517       }
2518     }
2519     emit_opcode(cbuf, 0x0F);
2520     emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
2521     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2522   %}
2523 
2524   enc_class enc_PartialSubtypeCheck()
2525   %{
2526     Register Rrdi = as_Register(RDI_enc); // result register
2527     Register Rrax = as_Register(RAX_enc); // super class
2528     Register Rrcx = as_Register(RCX_enc); // killed
2529     Register Rrsi = as_Register(RSI_enc); // sub class
2530     Label miss;
2531     const bool set_cond_codes = true;
2532 
2533     MacroAssembler _masm(&cbuf);
2534     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2535                                      NULL, &miss,
2536                                      /*set_cond_codes:*/ true);
2537     if ($primary) {
2538       __ xorptr(Rrdi, Rrdi);
2539     }
2540     __ bind(miss);
2541   %}
2542 
2543   enc_class Java_To_Interpreter(method meth)
2544   %{
2545     // CALL Java_To_Interpreter
2546     // This is the instruction starting address for relocation info.
2547     cbuf.set_insts_mark();
2548     $$$emit8$primary;
2549     // CALL directly to the runtime
2550     emit_d32_reloc(cbuf,
2551                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2552                    runtime_call_Relocation::spec(),
2553                    RELOC_DISP32);
2554   %}
2555 
2556   enc_class preserve_SP %{
2557     debug_only(int off0 = cbuf.insts_size());
2558     MacroAssembler _masm(&cbuf);
2559     // RBP is preserved across all calls, even compiled calls.
2560     // Use it to preserve RSP in places where the callee might change the SP.
2561     __ movptr(rbp_mh_SP_save, rsp);
2562     debug_only(int off1 = cbuf.insts_size());
2563     assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
2564   %}
2565 
2566   enc_class restore_SP %{
2567     MacroAssembler _masm(&cbuf);
2568     __ movptr(rsp, rbp_mh_SP_save);
2569   %}
2570 
2571   enc_class Java_Static_Call(method meth)
2572   %{
2573     // JAVA STATIC CALL
2574     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2575     // determine who we intended to call.
2576     cbuf.set_insts_mark();
2577     $$$emit8$primary;
2578 
2579     if (!_method) {
2580       emit_d32_reloc(cbuf,
2581                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2582                      runtime_call_Relocation::spec(),
2583                      RELOC_DISP32);
2584     } else if (_optimized_virtual) {
2585       emit_d32_reloc(cbuf,
2586                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2587                      opt_virtual_call_Relocation::spec(),
2588                      RELOC_DISP32);
2589     } else {
2590       emit_d32_reloc(cbuf,
2591                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2592                      static_call_Relocation::spec(),
2593                      RELOC_DISP32);
2594     }
2595     if (_method) {
2596       // Emit stub for static call
2597       emit_java_to_interp(cbuf);
2598     }
2599   %}
2600 
2601   enc_class Java_Dynamic_Call(method meth)
2602   %{
2603     // JAVA DYNAMIC CALL
2604     // !!!!!
2605     // Generate  "movq rax, -1", placeholder instruction to load oop-info
2606     // emit_call_dynamic_prologue( cbuf );
2607     cbuf.set_insts_mark();
2608 
2609     // movq rax, -1
2610     emit_opcode(cbuf, Assembler::REX_W);
2611     emit_opcode(cbuf, 0xB8 | RAX_enc);
2612     emit_d64_reloc(cbuf,
2613                    (int64_t) Universe::non_oop_word(),
2614                    oop_Relocation::spec_for_immediate(), RELOC_IMM64);
2615     address virtual_call_oop_addr = cbuf.insts_mark();
2616     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to determine
2617     // who we intended to call.
2618     cbuf.set_insts_mark();
2619     $$$emit8$primary;
2620     emit_d32_reloc(cbuf,
2621                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2622                    virtual_call_Relocation::spec(virtual_call_oop_addr),
2623                    RELOC_DISP32);
2624   %}
2625 
2626   enc_class Java_Compiled_Call(method meth)
2627   %{
2628     // JAVA COMPILED CALL
2629     int disp = in_bytes(methodOopDesc:: from_compiled_offset());
2630 
2631     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2632     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2633 
2634     // callq *disp(%rax)
2635     cbuf.set_insts_mark();
2636     $$$emit8$primary;
2637     if (disp < 0x80) {
2638       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2639       emit_d8(cbuf, disp); // Displacement
2640     } else {
2641       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2642       emit_d32(cbuf, disp); // Displacement
2643     }
2644   %}
2645 
2646   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2647   %{
2648     // SAL, SAR, SHR
2649     int dstenc = $dst$$reg;
2650     if (dstenc >= 8) {
2651       emit_opcode(cbuf, Assembler::REX_B);
2652       dstenc -= 8;
2653     }
2654     $$$emit8$primary;
2655     emit_rm(cbuf, 0x3, $secondary, dstenc);
2656     $$$emit8$shift$$constant;
2657   %}
2658 
2659   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2660   %{
2661     // SAL, SAR, SHR
2662     int dstenc = $dst$$reg;
2663     if (dstenc < 8) {
2664       emit_opcode(cbuf, Assembler::REX_W);
2665     } else {
2666       emit_opcode(cbuf, Assembler::REX_WB);
2667       dstenc -= 8;
2668     }
2669     $$$emit8$primary;
2670     emit_rm(cbuf, 0x3, $secondary, dstenc);
2671     $$$emit8$shift$$constant;
2672   %}
2673 
2674   enc_class load_immI(rRegI dst, immI src)
2675   %{
2676     int dstenc = $dst$$reg;
2677     if (dstenc >= 8) {
2678       emit_opcode(cbuf, Assembler::REX_B);
2679       dstenc -= 8;
2680     }
2681     emit_opcode(cbuf, 0xB8 | dstenc);
2682     $$$emit32$src$$constant;
2683   %}
2684 
2685   enc_class load_immL(rRegL dst, immL src)
2686   %{
2687     int dstenc = $dst$$reg;
2688     if (dstenc < 8) {
2689       emit_opcode(cbuf, Assembler::REX_W);
2690     } else {
2691       emit_opcode(cbuf, Assembler::REX_WB);
2692       dstenc -= 8;
2693     }
2694     emit_opcode(cbuf, 0xB8 | dstenc);
2695     emit_d64(cbuf, $src$$constant);
2696   %}
2697 
2698   enc_class load_immUL32(rRegL dst, immUL32 src)
2699   %{
2700     // same as load_immI, but this time we care about zeroes in the high word
2701     int dstenc = $dst$$reg;
2702     if (dstenc >= 8) {
2703       emit_opcode(cbuf, Assembler::REX_B);
2704       dstenc -= 8;
2705     }
2706     emit_opcode(cbuf, 0xB8 | dstenc);
2707     $$$emit32$src$$constant;
2708   %}
2709 
2710   enc_class load_immL32(rRegL dst, immL32 src)
2711   %{
2712     int dstenc = $dst$$reg;
2713     if (dstenc < 8) {
2714       emit_opcode(cbuf, Assembler::REX_W);
2715     } else {
2716       emit_opcode(cbuf, Assembler::REX_WB);
2717       dstenc -= 8;
2718     }
2719     emit_opcode(cbuf, 0xC7);
2720     emit_rm(cbuf, 0x03, 0x00, dstenc);
2721     $$$emit32$src$$constant;
2722   %}
2723 
2724   enc_class load_immP31(rRegP dst, immP32 src)
2725   %{
2726     // same as load_immI, but this time we care about zeroes in the high word
2727     int dstenc = $dst$$reg;
2728     if (dstenc >= 8) {
2729       emit_opcode(cbuf, Assembler::REX_B);
2730       dstenc -= 8;
2731     }
2732     emit_opcode(cbuf, 0xB8 | dstenc);
2733     $$$emit32$src$$constant;
2734   %}
2735 
2736   enc_class load_immP(rRegP dst, immP src)
2737   %{
2738     int dstenc = $dst$$reg;
2739     if (dstenc < 8) {
2740       emit_opcode(cbuf, Assembler::REX_W);
2741     } else {
2742       emit_opcode(cbuf, Assembler::REX_WB);
2743       dstenc -= 8;
2744     }
2745     emit_opcode(cbuf, 0xB8 | dstenc);
2746     // This next line should be generated from ADLC
2747     if ($src->constant_is_oop()) {
2748       emit_d64_reloc(cbuf, $src$$constant, relocInfo::oop_type, RELOC_IMM64);
2749     } else {
2750       emit_d64(cbuf, $src$$constant);
2751     }
2752   %}
2753 
2754   // Encode a reg-reg copy.  If it is useless, then empty encoding.
2755   enc_class enc_copy(rRegI dst, rRegI src)
2756   %{
2757     encode_copy(cbuf, $dst$$reg, $src$$reg);
2758   %}
2759 
2760   // Encode xmm reg-reg copy.  If it is useless, then empty encoding.
2761   enc_class enc_CopyXD( RegD dst, RegD src ) %{
2762     encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
2763   %}
2764 
2765   enc_class enc_copy_always(rRegI dst, rRegI src)
2766   %{
2767     int srcenc = $src$$reg;
2768     int dstenc = $dst$$reg;
2769 
2770     if (dstenc < 8) {
2771       if (srcenc >= 8) {
2772         emit_opcode(cbuf, Assembler::REX_B);
2773         srcenc -= 8;
2774       }
2775     } else {
2776       if (srcenc < 8) {
2777         emit_opcode(cbuf, Assembler::REX_R);
2778       } else {
2779         emit_opcode(cbuf, Assembler::REX_RB);
2780         srcenc -= 8;
2781       }
2782       dstenc -= 8;
2783     }
2784 
2785     emit_opcode(cbuf, 0x8B);
2786     emit_rm(cbuf, 0x3, dstenc, srcenc);
2787   %}
2788 
2789   enc_class enc_copy_wide(rRegL dst, rRegL src)
2790   %{
2791     int srcenc = $src$$reg;
2792     int dstenc = $dst$$reg;
2793 
2794     if (dstenc != srcenc) {
2795       if (dstenc < 8) {
2796         if (srcenc < 8) {
2797           emit_opcode(cbuf, Assembler::REX_W);
2798         } else {
2799           emit_opcode(cbuf, Assembler::REX_WB);
2800           srcenc -= 8;
2801         }
2802       } else {
2803         if (srcenc < 8) {
2804           emit_opcode(cbuf, Assembler::REX_WR);
2805         } else {
2806           emit_opcode(cbuf, Assembler::REX_WRB);
2807           srcenc -= 8;
2808         }
2809         dstenc -= 8;
2810       }
2811       emit_opcode(cbuf, 0x8B);
2812       emit_rm(cbuf, 0x3, dstenc, srcenc);
2813     }
2814   %}
2815 
2816   enc_class Con32(immI src)
2817   %{
2818     // Output immediate
2819     $$$emit32$src$$constant;
2820   %}
2821 
2822   enc_class Con64(immL src)
2823   %{
2824     // Output immediate
2825     emit_d64($src$$constant);
2826   %}
2827 
2828   enc_class Con32F_as_bits(immF src)
2829   %{
2830     // Output Float immediate bits
2831     jfloat jf = $src$$constant;
2832     jint jf_as_bits = jint_cast(jf);
2833     emit_d32(cbuf, jf_as_bits);
2834   %}
2835 
2836   enc_class Con16(immI src)
2837   %{
2838     // Output immediate
2839     $$$emit16$src$$constant;
2840   %}
2841 
2842   // How is this different from Con32??? XXX
2843   enc_class Con_d32(immI src)
2844   %{
2845     emit_d32(cbuf,$src$$constant);
2846   %}
2847 
2848   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2849     // Output immediate memory reference
2850     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2851     emit_d32(cbuf, 0x00);
2852   %}
2853 
2854   enc_class lock_prefix()
2855   %{
2856     if (os::is_MP()) {
2857       emit_opcode(cbuf, 0xF0); // lock
2858     }
2859   %}
2860 
2861   enc_class REX_mem(memory mem)
2862   %{
2863     if ($mem$$base >= 8) {
2864       if ($mem$$index < 8) {
2865         emit_opcode(cbuf, Assembler::REX_B);
2866       } else {
2867         emit_opcode(cbuf, Assembler::REX_XB);
2868       }
2869     } else {
2870       if ($mem$$index >= 8) {
2871         emit_opcode(cbuf, Assembler::REX_X);
2872       }
2873     }
2874   %}
2875 
2876   enc_class REX_mem_wide(memory mem)
2877   %{
2878     if ($mem$$base >= 8) {
2879       if ($mem$$index < 8) {
2880         emit_opcode(cbuf, Assembler::REX_WB);
2881       } else {
2882         emit_opcode(cbuf, Assembler::REX_WXB);
2883       }
2884     } else {
2885       if ($mem$$index < 8) {
2886         emit_opcode(cbuf, Assembler::REX_W);
2887       } else {
2888         emit_opcode(cbuf, Assembler::REX_WX);
2889       }
2890     }
2891   %}
2892 
2893   // for byte regs
2894   enc_class REX_breg(rRegI reg)
2895   %{
2896     if ($reg$$reg >= 4) {
2897       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2898     }
2899   %}
2900 
2901   // for byte regs
2902   enc_class REX_reg_breg(rRegI dst, rRegI src)
2903   %{
2904     if ($dst$$reg < 8) {
2905       if ($src$$reg >= 4) {
2906         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2907       }
2908     } else {
2909       if ($src$$reg < 8) {
2910         emit_opcode(cbuf, Assembler::REX_R);
2911       } else {
2912         emit_opcode(cbuf, Assembler::REX_RB);
2913       }
2914     }
2915   %}
2916 
2917   // for byte regs
2918   enc_class REX_breg_mem(rRegI reg, memory mem)
2919   %{
2920     if ($reg$$reg < 8) {
2921       if ($mem$$base < 8) {
2922         if ($mem$$index >= 8) {
2923           emit_opcode(cbuf, Assembler::REX_X);
2924         } else if ($reg$$reg >= 4) {
2925           emit_opcode(cbuf, Assembler::REX);
2926         }
2927       } else {
2928         if ($mem$$index < 8) {
2929           emit_opcode(cbuf, Assembler::REX_B);
2930         } else {
2931           emit_opcode(cbuf, Assembler::REX_XB);
2932         }
2933       }
2934     } else {
2935       if ($mem$$base < 8) {
2936         if ($mem$$index < 8) {
2937           emit_opcode(cbuf, Assembler::REX_R);
2938         } else {
2939           emit_opcode(cbuf, Assembler::REX_RX);
2940         }
2941       } else {
2942         if ($mem$$index < 8) {
2943           emit_opcode(cbuf, Assembler::REX_RB);
2944         } else {
2945           emit_opcode(cbuf, Assembler::REX_RXB);
2946         }
2947       }
2948     }
2949   %}
2950 
2951   enc_class REX_reg(rRegI reg)
2952   %{
2953     if ($reg$$reg >= 8) {
2954       emit_opcode(cbuf, Assembler::REX_B);
2955     }
2956   %}
2957 
2958   enc_class REX_reg_wide(rRegI reg)
2959   %{
2960     if ($reg$$reg < 8) {
2961       emit_opcode(cbuf, Assembler::REX_W);
2962     } else {
2963       emit_opcode(cbuf, Assembler::REX_WB);
2964     }
2965   %}
2966 
2967   enc_class REX_reg_reg(rRegI dst, rRegI src)
2968   %{
2969     if ($dst$$reg < 8) {
2970       if ($src$$reg >= 8) {
2971         emit_opcode(cbuf, Assembler::REX_B);
2972       }
2973     } else {
2974       if ($src$$reg < 8) {
2975         emit_opcode(cbuf, Assembler::REX_R);
2976       } else {
2977         emit_opcode(cbuf, Assembler::REX_RB);
2978       }
2979     }
2980   %}
2981 
2982   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2983   %{
2984     if ($dst$$reg < 8) {
2985       if ($src$$reg < 8) {
2986         emit_opcode(cbuf, Assembler::REX_W);
2987       } else {
2988         emit_opcode(cbuf, Assembler::REX_WB);
2989       }
2990     } else {
2991       if ($src$$reg < 8) {
2992         emit_opcode(cbuf, Assembler::REX_WR);
2993       } else {
2994         emit_opcode(cbuf, Assembler::REX_WRB);
2995       }
2996     }
2997   %}
2998 
2999   enc_class REX_reg_mem(rRegI reg, memory mem)
3000   %{
3001     if ($reg$$reg < 8) {
3002       if ($mem$$base < 8) {
3003         if ($mem$$index >= 8) {
3004           emit_opcode(cbuf, Assembler::REX_X);
3005         }
3006       } else {
3007         if ($mem$$index < 8) {
3008           emit_opcode(cbuf, Assembler::REX_B);
3009         } else {
3010           emit_opcode(cbuf, Assembler::REX_XB);
3011         }
3012       }
3013     } else {
3014       if ($mem$$base < 8) {
3015         if ($mem$$index < 8) {
3016           emit_opcode(cbuf, Assembler::REX_R);
3017         } else {
3018           emit_opcode(cbuf, Assembler::REX_RX);
3019         }
3020       } else {
3021         if ($mem$$index < 8) {
3022           emit_opcode(cbuf, Assembler::REX_RB);
3023         } else {
3024           emit_opcode(cbuf, Assembler::REX_RXB);
3025         }
3026       }
3027     }
3028   %}
3029 
3030   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
3031   %{
3032     if ($reg$$reg < 8) {
3033       if ($mem$$base < 8) {
3034         if ($mem$$index < 8) {
3035           emit_opcode(cbuf, Assembler::REX_W);
3036         } else {
3037           emit_opcode(cbuf, Assembler::REX_WX);
3038         }
3039       } else {
3040         if ($mem$$index < 8) {
3041           emit_opcode(cbuf, Assembler::REX_WB);
3042         } else {
3043           emit_opcode(cbuf, Assembler::REX_WXB);
3044         }
3045       }
3046     } else {
3047       if ($mem$$base < 8) {
3048         if ($mem$$index < 8) {
3049           emit_opcode(cbuf, Assembler::REX_WR);
3050         } else {
3051           emit_opcode(cbuf, Assembler::REX_WRX);
3052         }
3053       } else {
3054         if ($mem$$index < 8) {
3055           emit_opcode(cbuf, Assembler::REX_WRB);
3056         } else {
3057           emit_opcode(cbuf, Assembler::REX_WRXB);
3058         }
3059       }
3060     }
3061   %}
3062 
3063   enc_class reg_mem(rRegI ereg, memory mem)
3064   %{
3065     // High registers handle in encode_RegMem
3066     int reg = $ereg$$reg;
3067     int base = $mem$$base;
3068     int index = $mem$$index;
3069     int scale = $mem$$scale;
3070     int disp = $mem$$disp;
3071     bool disp_is_oop = $mem->disp_is_oop();
3072 
3073     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_is_oop);
3074   %}
3075 
3076   enc_class RM_opc_mem(immI rm_opcode, memory mem)
3077   %{
3078     int rm_byte_opcode = $rm_opcode$$constant;
3079 
3080     // High registers handle in encode_RegMem
3081     int base = $mem$$base;
3082     int index = $mem$$index;
3083     int scale = $mem$$scale;
3084     int displace = $mem$$disp;
3085 
3086     bool disp_is_oop = $mem->disp_is_oop(); // disp-as-oop when
3087                                             // working with static
3088                                             // globals
3089     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
3090                   disp_is_oop);
3091   %}
3092 
3093   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
3094   %{
3095     int reg_encoding = $dst$$reg;
3096     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
3097     int index        = 0x04;            // 0x04 indicates no index
3098     int scale        = 0x00;            // 0x00 indicates no scale
3099     int displace     = $src1$$constant; // 0x00 indicates no displacement
3100     bool disp_is_oop = false;
3101     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
3102                   disp_is_oop);
3103   %}
3104 
3105   enc_class neg_reg(rRegI dst)
3106   %{
3107     int dstenc = $dst$$reg;
3108     if (dstenc >= 8) {
3109       emit_opcode(cbuf, Assembler::REX_B);
3110       dstenc -= 8;
3111     }
3112     // NEG $dst
3113     emit_opcode(cbuf, 0xF7);
3114     emit_rm(cbuf, 0x3, 0x03, dstenc);
3115   %}
3116 
3117   enc_class neg_reg_wide(rRegI dst)
3118   %{
3119     int dstenc = $dst$$reg;
3120     if (dstenc < 8) {
3121       emit_opcode(cbuf, Assembler::REX_W);
3122     } else {
3123       emit_opcode(cbuf, Assembler::REX_WB);
3124       dstenc -= 8;
3125     }
3126     // NEG $dst
3127     emit_opcode(cbuf, 0xF7);
3128     emit_rm(cbuf, 0x3, 0x03, dstenc);
3129   %}
3130 
3131   enc_class setLT_reg(rRegI dst)
3132   %{
3133     int dstenc = $dst$$reg;
3134     if (dstenc >= 8) {
3135       emit_opcode(cbuf, Assembler::REX_B);
3136       dstenc -= 8;
3137     } else if (dstenc >= 4) {
3138       emit_opcode(cbuf, Assembler::REX);
3139     }
3140     // SETLT $dst
3141     emit_opcode(cbuf, 0x0F);
3142     emit_opcode(cbuf, 0x9C);
3143     emit_rm(cbuf, 0x3, 0x0, dstenc);
3144   %}
3145 
3146   enc_class setNZ_reg(rRegI dst)
3147   %{
3148     int dstenc = $dst$$reg;
3149     if (dstenc >= 8) {
3150       emit_opcode(cbuf, Assembler::REX_B);
3151       dstenc -= 8;
3152     } else if (dstenc >= 4) {
3153       emit_opcode(cbuf, Assembler::REX);
3154     }
3155     // SETNZ $dst
3156     emit_opcode(cbuf, 0x0F);
3157     emit_opcode(cbuf, 0x95);
3158     emit_rm(cbuf, 0x3, 0x0, dstenc);
3159   %}
3160 
3161 
3162   // Compare the lonogs and set -1, 0, or 1 into dst
3163   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
3164   %{
3165     int src1enc = $src1$$reg;
3166     int src2enc = $src2$$reg;
3167     int dstenc = $dst$$reg;
3168 
3169     // cmpq $src1, $src2
3170     if (src1enc < 8) {
3171       if (src2enc < 8) {
3172         emit_opcode(cbuf, Assembler::REX_W);
3173       } else {
3174         emit_opcode(cbuf, Assembler::REX_WB);
3175       }
3176     } else {
3177       if (src2enc < 8) {
3178         emit_opcode(cbuf, Assembler::REX_WR);
3179       } else {
3180         emit_opcode(cbuf, Assembler::REX_WRB);
3181       }
3182     }
3183     emit_opcode(cbuf, 0x3B);
3184     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
3185 
3186     // movl $dst, -1
3187     if (dstenc >= 8) {
3188       emit_opcode(cbuf, Assembler::REX_B);
3189     }
3190     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
3191     emit_d32(cbuf, -1);
3192 
3193     // jl,s done
3194     emit_opcode(cbuf, 0x7C);
3195     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
3196 
3197     // setne $dst
3198     if (dstenc >= 4) {
3199       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
3200     }
3201     emit_opcode(cbuf, 0x0F);
3202     emit_opcode(cbuf, 0x95);
3203     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
3204 
3205     // movzbl $dst, $dst
3206     if (dstenc >= 4) {
3207       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
3208     }
3209     emit_opcode(cbuf, 0x0F);
3210     emit_opcode(cbuf, 0xB6);
3211     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
3212   %}
3213 
3214   enc_class Push_ResultXD(regD dst) %{
3215     int dstenc = $dst$$reg;
3216 
3217     store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [RSP]
3218 
3219     // UseXmmLoadAndClearUpper ? movsd dst,[rsp] : movlpd dst,[rsp]
3220     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
3221     if (dstenc >= 8) {
3222       emit_opcode(cbuf, Assembler::REX_R);
3223     }
3224     emit_opcode  (cbuf, 0x0F );
3225     emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12 );
3226     encode_RegMem(cbuf, dstenc, RSP_enc, 0x4, 0, 0, false);
3227 
3228     // add rsp,8
3229     emit_opcode(cbuf, Assembler::REX_W);
3230     emit_opcode(cbuf,0x83);
3231     emit_rm(cbuf,0x3, 0x0, RSP_enc);
3232     emit_d8(cbuf,0x08);
3233   %}
3234 
3235   enc_class Push_SrcXD(regD src) %{
3236     int srcenc = $src$$reg;
3237 
3238     // subq rsp,#8
3239     emit_opcode(cbuf, Assembler::REX_W);
3240     emit_opcode(cbuf, 0x83);
3241     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
3242     emit_d8(cbuf, 0x8);
3243 
3244     // movsd [rsp],src
3245     emit_opcode(cbuf, 0xF2);
3246     if (srcenc >= 8) {
3247       emit_opcode(cbuf, Assembler::REX_R);
3248     }
3249     emit_opcode(cbuf, 0x0F);
3250     emit_opcode(cbuf, 0x11);
3251     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false);
3252 
3253     // fldd [rsp]
3254     emit_opcode(cbuf, 0x66);
3255     emit_opcode(cbuf, 0xDD);
3256     encode_RegMem(cbuf, 0x0, RSP_enc, 0x4, 0, 0, false);
3257   %}
3258 
3259 
3260   enc_class movq_ld(regD dst, memory mem) %{
3261     MacroAssembler _masm(&cbuf);
3262     __ movq($dst$$XMMRegister, $mem$$Address);
3263   %}
3264 
3265   enc_class movq_st(memory mem, regD src) %{
3266     MacroAssembler _masm(&cbuf);
3267     __ movq($mem$$Address, $src$$XMMRegister);
3268   %}
3269 
3270   enc_class pshufd_8x8(regF dst, regF src) %{
3271     MacroAssembler _masm(&cbuf);
3272 
3273     encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
3274     __ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
3275     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
3276   %}
3277 
3278   enc_class pshufd_4x16(regF dst, regF src) %{
3279     MacroAssembler _masm(&cbuf);
3280 
3281     __ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
3282   %}
3283 
3284   enc_class pshufd(regD dst, regD src, int mode) %{
3285     MacroAssembler _masm(&cbuf);
3286 
3287     __ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
3288   %}
3289 
3290   enc_class pxor(regD dst, regD src) %{
3291     MacroAssembler _masm(&cbuf);
3292 
3293     __ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
3294   %}
3295 
3296   enc_class mov_i2x(regD dst, rRegI src) %{
3297     MacroAssembler _masm(&cbuf);
3298 
3299     __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
3300   %}
3301 
3302   // obj: object to lock
3303   // box: box address (header location) -- killed
3304   // tmp: rax -- killed
3305   // scr: rbx -- killed
3306   //
3307   // What follows is a direct transliteration of fast_lock() and fast_unlock()
3308   // from i486.ad.  See that file for comments.
3309   // TODO: where possible switch from movq (r, 0) to movl(r,0) and
3310   // use the shorter encoding.  (Movl clears the high-order 32-bits).
3311 
3312 
3313   enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
3314   %{
3315     Register objReg = as_Register((int)$obj$$reg);
3316     Register boxReg = as_Register((int)$box$$reg);
3317     Register tmpReg = as_Register($tmp$$reg);
3318     Register scrReg = as_Register($scr$$reg);
3319     MacroAssembler masm(&cbuf);
3320 
3321     // Verify uniqueness of register assignments -- necessary but not sufficient
3322     assert (objReg != boxReg && objReg != tmpReg &&
3323             objReg != scrReg && tmpReg != scrReg, "invariant") ;
3324 
3325     if (_counters != NULL) {
3326       masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
3327     }
3328     if (EmitSync & 1) {
3329         // Without cast to int32_t a movptr will destroy r10 which is typically obj
3330         masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
3331         masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
3332     } else
3333     if (EmitSync & 2) {
3334         Label DONE_LABEL;
3335         if (UseBiasedLocking) {
3336            // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
3337           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3338         }
3339         // QQQ was movl...
3340         masm.movptr(tmpReg, 0x1);
3341         masm.orptr(tmpReg, Address(objReg, 0));
3342         masm.movptr(Address(boxReg, 0), tmpReg);
3343         if (os::is_MP()) {
3344           masm.lock();
3345         }
3346         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
3347         masm.jcc(Assembler::equal, DONE_LABEL);
3348 
3349         // Recursive locking
3350         masm.subptr(tmpReg, rsp);
3351         masm.andptr(tmpReg, 7 - os::vm_page_size());
3352         masm.movptr(Address(boxReg, 0), tmpReg);
3353 
3354         masm.bind(DONE_LABEL);
3355         masm.nop(); // avoid branch to branch
3356     } else {
3357         Label DONE_LABEL, IsInflated, Egress;
3358 
3359         masm.movptr(tmpReg, Address(objReg, 0)) ;
3360         masm.testl (tmpReg, 0x02) ;         // inflated vs stack-locked|neutral|biased
3361         masm.jcc   (Assembler::notZero, IsInflated) ;
3362 
3363         // it's stack-locked, biased or neutral
3364         // TODO: optimize markword triage order to reduce the number of
3365         // conditional branches in the most common cases.
3366         // Beware -- there's a subtle invariant that fetch of the markword
3367         // at [FETCH], below, will never observe a biased encoding (*101b).
3368         // If this invariant is not held we'll suffer exclusion (safety) failure.
3369 
3370         if (UseBiasedLocking && !UseOptoBiasInlining) {
3371           masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
3372           masm.movptr(tmpReg, Address(objReg, 0)) ;        // [FETCH]
3373         }
3374 
3375         // was q will it destroy high?
3376         masm.orl   (tmpReg, 1) ;
3377         masm.movptr(Address(boxReg, 0), tmpReg) ;
3378         if (os::is_MP()) { masm.lock(); }
3379         masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
3380         if (_counters != NULL) {
3381            masm.cond_inc32(Assembler::equal,
3382                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
3383         }
3384         masm.jcc   (Assembler::equal, DONE_LABEL);
3385 
3386         // Recursive locking
3387         masm.subptr(tmpReg, rsp);
3388         masm.andptr(tmpReg, 7 - os::vm_page_size());
3389         masm.movptr(Address(boxReg, 0), tmpReg);
3390         if (_counters != NULL) {
3391            masm.cond_inc32(Assembler::equal,
3392                            ExternalAddress((address) _counters->fast_path_entry_count_addr()));
3393         }
3394         masm.jmp   (DONE_LABEL) ;
3395 
3396         masm.bind  (IsInflated) ;
3397         // It's inflated
3398 
3399         // TODO: someday avoid the ST-before-CAS penalty by
3400         // relocating (deferring) the following ST.
3401         // We should also think about trying a CAS without having
3402         // fetched _owner.  If the CAS is successful we may
3403         // avoid an RTO->RTS upgrade on the $line.
3404         // Without cast to int32_t a movptr will destroy r10 which is typically obj
3405         masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
3406 
3407         masm.mov    (boxReg, tmpReg) ;
3408         masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3409         masm.testptr(tmpReg, tmpReg) ;
3410         masm.jcc    (Assembler::notZero, DONE_LABEL) ;
3411 
3412         // It's inflated and appears unlocked
3413         if (os::is_MP()) { masm.lock(); }
3414         masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3415         // Intentional fall-through into DONE_LABEL ...
3416 
3417         masm.bind  (DONE_LABEL) ;
3418         masm.nop   () ;                 // avoid jmp to jmp
3419     }
3420   %}
3421 
3422   // obj: object to unlock
3423   // box: box address (displaced header location), killed
3424   // RBX: killed tmp; cannot be obj nor box
3425   enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
3426   %{
3427 
3428     Register objReg = as_Register($obj$$reg);
3429     Register boxReg = as_Register($box$$reg);
3430     Register tmpReg = as_Register($tmp$$reg);
3431     MacroAssembler masm(&cbuf);
3432 
3433     if (EmitSync & 4) {
3434        masm.cmpptr(rsp, 0) ;
3435     } else
3436     if (EmitSync & 8) {
3437        Label DONE_LABEL;
3438        if (UseBiasedLocking) {
3439          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3440        }
3441 
3442        // Check whether the displaced header is 0
3443        //(=> recursive unlock)
3444        masm.movptr(tmpReg, Address(boxReg, 0));
3445        masm.testptr(tmpReg, tmpReg);
3446        masm.jcc(Assembler::zero, DONE_LABEL);
3447 
3448        // If not recursive lock, reset the header to displaced header
3449        if (os::is_MP()) {
3450          masm.lock();
3451        }
3452        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
3453        masm.bind(DONE_LABEL);
3454        masm.nop(); // avoid branch to branch
3455     } else {
3456        Label DONE_LABEL, Stacked, CheckSucc ;
3457 
3458        if (UseBiasedLocking && !UseOptoBiasInlining) {
3459          masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3460        }
3461 
3462        masm.movptr(tmpReg, Address(objReg, 0)) ;
3463        masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
3464        masm.jcc   (Assembler::zero, DONE_LABEL) ;
3465        masm.testl (tmpReg, 0x02) ;
3466        masm.jcc   (Assembler::zero, Stacked) ;
3467 
3468        // It's inflated
3469        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3470        masm.xorptr(boxReg, r15_thread) ;
3471        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3472        masm.jcc   (Assembler::notZero, DONE_LABEL) ;
3473        masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
3474        masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
3475        masm.jcc   (Assembler::notZero, CheckSucc) ;
3476        masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3477        masm.jmp   (DONE_LABEL) ;
3478 
3479        if ((EmitSync & 65536) == 0) {
3480          Label LSuccess, LGoSlowPath ;
3481          masm.bind  (CheckSucc) ;
3482          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3483          masm.jcc   (Assembler::zero, LGoSlowPath) ;
3484 
3485          // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
3486          // the explicit ST;MEMBAR combination, but masm doesn't currently support
3487          // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc
3488          // are all faster when the write buffer is populated.
3489          masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3490          if (os::is_MP()) {
3491             masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
3492          }
3493          masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3494          masm.jcc   (Assembler::notZero, LSuccess) ;
3495 
3496          masm.movptr (boxReg, (int32_t)NULL_WORD) ;                   // box is really EAX
3497          if (os::is_MP()) { masm.lock(); }
3498          masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
3499          masm.jcc   (Assembler::notEqual, LSuccess) ;
3500          // Intentional fall-through into slow-path
3501 
3502          masm.bind  (LGoSlowPath) ;
3503          masm.orl   (boxReg, 1) ;                      // set ICC.ZF=0 to indicate failure
3504          masm.jmp   (DONE_LABEL) ;
3505 
3506          masm.bind  (LSuccess) ;
3507          masm.testl (boxReg, 0) ;                      // set ICC.ZF=1 to indicate success
3508          masm.jmp   (DONE_LABEL) ;
3509        }
3510 
3511        masm.bind  (Stacked) ;
3512        masm.movptr(tmpReg, Address (boxReg, 0)) ;      // re-fetch
3513        if (os::is_MP()) { masm.lock(); }
3514        masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
3515 
3516        if (EmitSync & 65536) {
3517           masm.bind (CheckSucc) ;
3518        }
3519        masm.bind(DONE_LABEL);
3520        if (EmitSync & 32768) {
3521           masm.nop();                      // avoid branch to branch
3522        }
3523     }
3524   %}
3525 
3526 
3527   enc_class enc_rethrow()
3528   %{
3529     cbuf.set_insts_mark();
3530     emit_opcode(cbuf, 0xE9); // jmp entry
3531     emit_d32_reloc(cbuf,
3532                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
3533                    runtime_call_Relocation::spec(),
3534                    RELOC_DISP32);
3535   %}
3536 
3537   enc_class absF_encoding(regF dst)
3538   %{
3539     int dstenc = $dst$$reg;
3540     address signmask_address = (address) StubRoutines::x86::float_sign_mask();
3541 
3542     cbuf.set_insts_mark();
3543     if (dstenc >= 8) {
3544       emit_opcode(cbuf, Assembler::REX_R);
3545       dstenc -= 8;
3546     }
3547     // XXX reg_mem doesn't support RIP-relative addressing yet
3548     emit_opcode(cbuf, 0x0F);
3549     emit_opcode(cbuf, 0x54);
3550     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
3551     emit_d32_reloc(cbuf, signmask_address);
3552   %}
3553 
3554   enc_class absD_encoding(regD dst)
3555   %{
3556     int dstenc = $dst$$reg;
3557     address signmask_address = (address) StubRoutines::x86::double_sign_mask();
3558 
3559     cbuf.set_insts_mark();
3560     emit_opcode(cbuf, 0x66);
3561     if (dstenc >= 8) {
3562       emit_opcode(cbuf, Assembler::REX_R);
3563       dstenc -= 8;
3564     }
3565     // XXX reg_mem doesn't support RIP-relative addressing yet
3566     emit_opcode(cbuf, 0x0F);
3567     emit_opcode(cbuf, 0x54);
3568     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
3569     emit_d32_reloc(cbuf, signmask_address);
3570   %}
3571 
3572   enc_class negF_encoding(regF dst)
3573   %{
3574     int dstenc = $dst$$reg;
3575     address signflip_address = (address) StubRoutines::x86::float_sign_flip();
3576 
3577     cbuf.set_insts_mark();
3578     if (dstenc >= 8) {
3579       emit_opcode(cbuf, Assembler::REX_R);
3580       dstenc -= 8;
3581     }
3582     // XXX reg_mem doesn't support RIP-relative addressing yet
3583     emit_opcode(cbuf, 0x0F);
3584     emit_opcode(cbuf, 0x57);
3585     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
3586     emit_d32_reloc(cbuf, signflip_address);
3587   %}
3588 
3589   enc_class negD_encoding(regD dst)
3590   %{
3591     int dstenc = $dst$$reg;
3592     address signflip_address = (address) StubRoutines::x86::double_sign_flip();
3593 
3594     cbuf.set_insts_mark();
3595     emit_opcode(cbuf, 0x66);
3596     if (dstenc >= 8) {
3597       emit_opcode(cbuf, Assembler::REX_R);
3598       dstenc -= 8;
3599     }
3600     // XXX reg_mem doesn't support RIP-relative addressing yet
3601     emit_opcode(cbuf, 0x0F);
3602     emit_opcode(cbuf, 0x57);
3603     emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
3604     emit_d32_reloc(cbuf, signflip_address);
3605   %}
3606 
3607   enc_class f2i_fixup(rRegI dst, regF src)
3608   %{
3609     int dstenc = $dst$$reg;
3610     int srcenc = $src$$reg;
3611 
3612     // cmpl $dst, #0x80000000
3613     if (dstenc >= 8) {
3614       emit_opcode(cbuf, Assembler::REX_B);
3615     }
3616     emit_opcode(cbuf, 0x81);
3617     emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
3618     emit_d32(cbuf, 0x80000000);
3619 
3620     // jne,s done
3621     emit_opcode(cbuf, 0x75);
3622     if (srcenc < 8 && dstenc < 8) {
3623       emit_d8(cbuf, 0xF);
3624     } else if (srcenc >= 8 && dstenc >= 8) {
3625       emit_d8(cbuf, 0x11);
3626     } else {
3627       emit_d8(cbuf, 0x10);
3628     }
3629 
3630     // subq rsp, #8
3631     emit_opcode(cbuf, Assembler::REX_W);
3632     emit_opcode(cbuf, 0x83);
3633     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
3634     emit_d8(cbuf, 8);
3635 
3636     // movss [rsp], $src
3637     emit_opcode(cbuf, 0xF3);
3638     if (srcenc >= 8) {
3639       emit_opcode(cbuf, Assembler::REX_R);
3640     }
3641     emit_opcode(cbuf, 0x0F);
3642     emit_opcode(cbuf, 0x11);
3643     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
3644 
3645     // call f2i_fixup
3646     cbuf.set_insts_mark();
3647     emit_opcode(cbuf, 0xE8);
3648     emit_d32_reloc(cbuf,
3649                    (int)
3650                    (StubRoutines::x86::f2i_fixup() - cbuf.insts_end() - 4),
3651                    runtime_call_Relocation::spec(),
3652                    RELOC_DISP32);
3653 
3654     // popq $dst
3655     if (dstenc >= 8) {
3656       emit_opcode(cbuf, Assembler::REX_B);
3657     }
3658     emit_opcode(cbuf, 0x58 | (dstenc & 7));
3659 
3660     // done:
3661   %}
3662 
3663   enc_class f2l_fixup(rRegL dst, regF src)
3664   %{
3665     int dstenc = $dst$$reg;
3666     int srcenc = $src$$reg;
3667     address const_address = (address) StubRoutines::x86::double_sign_flip();
3668 
3669     // cmpq $dst, [0x8000000000000000]
3670     cbuf.set_insts_mark();
3671     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
3672     emit_opcode(cbuf, 0x39);
3673     // XXX reg_mem doesn't support RIP-relative addressing yet
3674     emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
3675     emit_d32_reloc(cbuf, const_address);
3676 
3677 
3678     // jne,s done
3679     emit_opcode(cbuf, 0x75);
3680     if (srcenc < 8 && dstenc < 8) {
3681       emit_d8(cbuf, 0xF);
3682     } else if (srcenc >= 8 && dstenc >= 8) {
3683       emit_d8(cbuf, 0x11);
3684     } else {
3685       emit_d8(cbuf, 0x10);
3686     }
3687 
3688     // subq rsp, #8
3689     emit_opcode(cbuf, Assembler::REX_W);
3690     emit_opcode(cbuf, 0x83);
3691     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
3692     emit_d8(cbuf, 8);
3693 
3694     // movss [rsp], $src
3695     emit_opcode(cbuf, 0xF3);
3696     if (srcenc >= 8) {
3697       emit_opcode(cbuf, Assembler::REX_R);
3698     }
3699     emit_opcode(cbuf, 0x0F);
3700     emit_opcode(cbuf, 0x11);
3701     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
3702 
3703     // call f2l_fixup
3704     cbuf.set_insts_mark();
3705     emit_opcode(cbuf, 0xE8);
3706     emit_d32_reloc(cbuf,
3707                    (int)
3708                    (StubRoutines::x86::f2l_fixup() - cbuf.insts_end() - 4),
3709                    runtime_call_Relocation::spec(),
3710                    RELOC_DISP32);
3711 
3712     // popq $dst
3713     if (dstenc >= 8) {
3714       emit_opcode(cbuf, Assembler::REX_B);
3715     }
3716     emit_opcode(cbuf, 0x58 | (dstenc & 7));
3717 
3718     // done:
3719   %}
3720 
3721   enc_class d2i_fixup(rRegI dst, regD src)
3722   %{
3723     int dstenc = $dst$$reg;
3724     int srcenc = $src$$reg;
3725 
3726     // cmpl $dst, #0x80000000
3727     if (dstenc >= 8) {
3728       emit_opcode(cbuf, Assembler::REX_B);
3729     }
3730     emit_opcode(cbuf, 0x81);
3731     emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
3732     emit_d32(cbuf, 0x80000000);
3733 
3734     // jne,s done
3735     emit_opcode(cbuf, 0x75);
3736     if (srcenc < 8 && dstenc < 8) {
3737       emit_d8(cbuf, 0xF);
3738     } else if (srcenc >= 8 && dstenc >= 8) {
3739       emit_d8(cbuf, 0x11);
3740     } else {
3741       emit_d8(cbuf, 0x10);
3742     }
3743 
3744     // subq rsp, #8
3745     emit_opcode(cbuf, Assembler::REX_W);
3746     emit_opcode(cbuf, 0x83);
3747     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
3748     emit_d8(cbuf, 8);
3749 
3750     // movsd [rsp], $src
3751     emit_opcode(cbuf, 0xF2);
3752     if (srcenc >= 8) {
3753       emit_opcode(cbuf, Assembler::REX_R);
3754     }
3755     emit_opcode(cbuf, 0x0F);
3756     emit_opcode(cbuf, 0x11);
3757     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
3758 
3759     // call d2i_fixup
3760     cbuf.set_insts_mark();
3761     emit_opcode(cbuf, 0xE8);
3762     emit_d32_reloc(cbuf,
3763                    (int)
3764                    (StubRoutines::x86::d2i_fixup() - cbuf.insts_end() - 4),
3765                    runtime_call_Relocation::spec(),
3766                    RELOC_DISP32);
3767 
3768     // popq $dst
3769     if (dstenc >= 8) {
3770       emit_opcode(cbuf, Assembler::REX_B);
3771     }
3772     emit_opcode(cbuf, 0x58 | (dstenc & 7));
3773 
3774     // done:
3775   %}
3776 
3777   enc_class d2l_fixup(rRegL dst, regD src)
3778   %{
3779     int dstenc = $dst$$reg;
3780     int srcenc = $src$$reg;
3781     address const_address = (address) StubRoutines::x86::double_sign_flip();
3782 
3783     // cmpq $dst, [0x8000000000000000]
3784     cbuf.set_insts_mark();
3785     emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
3786     emit_opcode(cbuf, 0x39);
3787     // XXX reg_mem doesn't support RIP-relative addressing yet
3788     emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
3789     emit_d32_reloc(cbuf, const_address);
3790 
3791 
3792     // jne,s done
3793     emit_opcode(cbuf, 0x75);
3794     if (srcenc < 8 && dstenc < 8) {
3795       emit_d8(cbuf, 0xF);
3796     } else if (srcenc >= 8 && dstenc >= 8) {
3797       emit_d8(cbuf, 0x11);
3798     } else {
3799       emit_d8(cbuf, 0x10);
3800     }
3801 
3802     // subq rsp, #8
3803     emit_opcode(cbuf, Assembler::REX_W);
3804     emit_opcode(cbuf, 0x83);
3805     emit_rm(cbuf, 0x3, 0x5, RSP_enc);
3806     emit_d8(cbuf, 8);
3807 
3808     // movsd [rsp], $src
3809     emit_opcode(cbuf, 0xF2);
3810     if (srcenc >= 8) {
3811       emit_opcode(cbuf, Assembler::REX_R);
3812     }
3813     emit_opcode(cbuf, 0x0F);
3814     emit_opcode(cbuf, 0x11);
3815     encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
3816 
3817     // call d2l_fixup
3818     cbuf.set_insts_mark();
3819     emit_opcode(cbuf, 0xE8);
3820     emit_d32_reloc(cbuf,
3821                    (int)
3822                    (StubRoutines::x86::d2l_fixup() - cbuf.insts_end() - 4),
3823                    runtime_call_Relocation::spec(),
3824                    RELOC_DISP32);
3825 
3826     // popq $dst
3827     if (dstenc >= 8) {
3828       emit_opcode(cbuf, Assembler::REX_B);
3829     }
3830     emit_opcode(cbuf, 0x58 | (dstenc & 7));
3831 
3832     // done:
3833   %}
3834 %}
3835 
3836 
3837 
3838 //----------FRAME--------------------------------------------------------------
3839 // Definition of frame structure and management information.
3840 //
3841 //  S T A C K   L A Y O U T    Allocators stack-slot number
3842 //                             |   (to get allocators register number
3843 //  G  Owned by    |        |  v    add OptoReg::stack0())
3844 //  r   CALLER     |        |
3845 //  o     |        +--------+      pad to even-align allocators stack-slot
3846 //  w     V        |  pad0  |        numbers; owned by CALLER
3847 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
3848 //  h     ^        |   in   |  5
3849 //        |        |  args  |  4   Holes in incoming args owned by SELF
3850 //  |     |        |        |  3
3851 //  |     |        +--------+
3852 //  V     |        | old out|      Empty on Intel, window on Sparc
3853 //        |    old |preserve|      Must be even aligned.
3854 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
3855 //        |        |   in   |  3   area for Intel ret address
3856 //     Owned by    |preserve|      Empty on Sparc.
3857 //       SELF      +--------+
3858 //        |        |  pad2  |  2   pad to align old SP
3859 //        |        +--------+  1
3860 //        |        | locks  |  0
3861 //        |        +--------+----> OptoReg::stack0(), even aligned
3862 //        |        |  pad1  | 11   pad to align new SP
3863 //        |        +--------+
3864 //        |        |        | 10
3865 //        |        | spills |  9   spills
3866 //        V        |        |  8   (pad0 slot for callee)
3867 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
3868 //        ^        |  out   |  7
3869 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
3870 //     Owned by    +--------+
3871 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
3872 //        |    new |preserve|      Must be even-aligned.
3873 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
3874 //        |        |        |
3875 //
3876 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
3877 //         known from SELF's arguments and the Java calling convention.
3878 //         Region 6-7 is determined per call site.
3879 // Note 2: If the calling convention leaves holes in the incoming argument
3880 //         area, those holes are owned by SELF.  Holes in the outgoing area
3881 //         are owned by the CALLEE.  Holes should not be nessecary in the
3882 //         incoming area, as the Java calling convention is completely under
3883 //         the control of the AD file.  Doubles can be sorted and packed to
3884 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
3885 //         varargs C calling conventions.
3886 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
3887 //         even aligned with pad0 as needed.
3888 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
3889 //         region 6-11 is even aligned; it may be padded out more so that
3890 //         the region from SP to FP meets the minimum stack alignment.
3891 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3892 //         alignment.  Region 11, pad1, may be dynamically extended so that
3893 //         SP meets the minimum alignment.
3894 
3895 frame
3896 %{
3897   // What direction does stack grow in (assumed to be same for C & Java)
3898   stack_direction(TOWARDS_LOW);
3899 
3900   // These three registers define part of the calling convention
3901   // between compiled code and the interpreter.
3902   inline_cache_reg(RAX);                // Inline Cache Register
3903   interpreter_method_oop_reg(RBX);      // Method Oop Register when
3904                                         // calling interpreter
3905 
3906   // Optional: name the operand used by cisc-spilling to access
3907   // [stack_pointer + offset]
3908   cisc_spilling_operand_name(indOffset32);
3909 
3910   // Number of stack slots consumed by locking an object
3911   sync_stack_slots(2);
3912 
3913   // Compiled code's Frame Pointer
3914   frame_pointer(RSP);
3915 
3916   // Interpreter stores its frame pointer in a register which is
3917   // stored to the stack by I2CAdaptors.
3918   // I2CAdaptors convert from interpreted java to compiled java.
3919   interpreter_frame_pointer(RBP);
3920 
3921   // Stack alignment requirement
3922   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3923 
3924   // Number of stack slots between incoming argument block and the start of
3925   // a new frame.  The PROLOG must add this many slots to the stack.  The
3926   // EPILOG must remove this many slots.  amd64 needs two slots for
3927   // return address.
3928   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
3929 
3930   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3931   // for calls to C.  Supports the var-args backing area for register parms.
3932   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3933 
3934   // The after-PROLOG location of the return address.  Location of
3935   // return address specifies a type (REG or STACK) and a number
3936   // representing the register number (i.e. - use a register name) or
3937   // stack slot.
3938   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3939   // Otherwise, it is above the locks and verification slot and alignment word
3940   return_addr(STACK - 2 +
3941               round_to(2 + 2 * VerifyStackAtCalls +
3942                        Compile::current()->fixed_slots(),
3943                        WordsPerLong * 2));
3944 
3945   // Body of function which returns an integer array locating
3946   // arguments either in registers or in stack slots.  Passed an array
3947   // of ideal registers called "sig" and a "length" count.  Stack-slot
3948   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3949   // arguments for a CALLEE.  Incoming stack arguments are
3950   // automatically biased by the preserve_stack_slots field above.
3951 
3952   calling_convention
3953   %{
3954     // No difference between ingoing/outgoing just pass false
3955     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3956   %}
3957 
3958   c_calling_convention
3959   %{
3960     // This is obviously always outgoing
3961     (void) SharedRuntime::c_calling_convention(sig_bt, regs, length);
3962   %}
3963 
3964   // Location of compiled Java return values.  Same as C for now.
3965   return_value
3966   %{
3967     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3968            "only return normal values");
3969 
3970     static const int lo[Op_RegL + 1] = {
3971       0,
3972       0,
3973       RAX_num,  // Op_RegN
3974       RAX_num,  // Op_RegI
3975       RAX_num,  // Op_RegP
3976       XMM0_num, // Op_RegF
3977       XMM0_num, // Op_RegD
3978       RAX_num   // Op_RegL
3979     };
3980     static const int hi[Op_RegL + 1] = {
3981       0,
3982       0,
3983       OptoReg::Bad, // Op_RegN
3984       OptoReg::Bad, // Op_RegI
3985       RAX_H_num,    // Op_RegP
3986       OptoReg::Bad, // Op_RegF
3987       XMM0_H_num,   // Op_RegD
3988       RAX_H_num     // Op_RegL
3989     };
3990     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 1, "missing type");
3991     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3992   %}
3993 %}
3994 
3995 //----------ATTRIBUTES---------------------------------------------------------
3996 //----------Operand Attributes-------------------------------------------------
3997 op_attrib op_cost(0);        // Required cost attribute
3998 
3999 //----------Instruction Attributes---------------------------------------------
4000 ins_attrib ins_cost(100);       // Required cost attribute
4001 ins_attrib ins_size(8);         // Required size attribute (in bits)
4002 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4003                                 // a non-matching short branch variant
4004                                 // of some long branch?
4005 ins_attrib ins_alignment(1);    // Required alignment attribute (must
4006                                 // be a power of 2) specifies the
4007                                 // alignment that some part of the
4008                                 // instruction (not necessarily the
4009                                 // start) requires.  If > 1, a
4010                                 // compute_padding() function must be
4011                                 // provided for the instruction
4012 
4013 //----------OPERANDS-----------------------------------------------------------
4014 // Operand definitions must precede instruction definitions for correct parsing
4015 // in the ADLC because operands constitute user defined types which are used in
4016 // instruction definitions.
4017 
4018 //----------Simple Operands----------------------------------------------------
4019 // Immediate Operands
4020 // Integer Immediate
4021 operand immI()
4022 %{
4023   match(ConI);
4024 
4025   op_cost(10);
4026   format %{ %}
4027   interface(CONST_INTER);
4028 %}
4029 
4030 // Constant for test vs zero
4031 operand immI0()
4032 %{
4033   predicate(n->get_int() == 0);
4034   match(ConI);
4035 
4036   op_cost(0);
4037   format %{ %}
4038   interface(CONST_INTER);
4039 %}
4040 
4041 // Constant for increment
4042 operand immI1()
4043 %{
4044   predicate(n->get_int() == 1);
4045   match(ConI);
4046 
4047   op_cost(0);
4048   format %{ %}
4049   interface(CONST_INTER);
4050 %}
4051 
4052 // Constant for decrement
4053 operand immI_M1()
4054 %{
4055   predicate(n->get_int() == -1);
4056   match(ConI);
4057 
4058   op_cost(0);
4059   format %{ %}
4060   interface(CONST_INTER);
4061 %}
4062 
4063 // Valid scale values for addressing modes
4064 operand immI2()
4065 %{
4066   predicate(0 <= n->get_int() && (n->get_int() <= 3));
4067   match(ConI);
4068 
4069   format %{ %}
4070   interface(CONST_INTER);
4071 %}
4072 
4073 operand immI8()
4074 %{
4075   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
4076   match(ConI);
4077 
4078   op_cost(5);
4079   format %{ %}
4080   interface(CONST_INTER);
4081 %}
4082 
4083 operand immI16()
4084 %{
4085   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
4086   match(ConI);
4087 
4088   op_cost(10);
4089   format %{ %}
4090   interface(CONST_INTER);
4091 %}
4092 
4093 // Constant for long shifts
4094 operand immI_32()
4095 %{
4096   predicate( n->get_int() == 32 );
4097   match(ConI);
4098 
4099   op_cost(0);
4100   format %{ %}
4101   interface(CONST_INTER);
4102 %}
4103 
4104 // Constant for long shifts
4105 operand immI_64()
4106 %{
4107   predicate( n->get_int() == 64 );
4108   match(ConI);
4109 
4110   op_cost(0);
4111   format %{ %}
4112   interface(CONST_INTER);
4113 %}
4114 
4115 // Pointer Immediate
4116 operand immP()
4117 %{
4118   match(ConP);
4119 
4120   op_cost(10);
4121   format %{ %}
4122   interface(CONST_INTER);
4123 %}
4124 
4125 // NULL Pointer Immediate
4126 operand immP0()
4127 %{
4128   predicate(n->get_ptr() == 0);
4129   match(ConP);
4130 
4131   op_cost(5);
4132   format %{ %}
4133   interface(CONST_INTER);
4134 %}
4135 
4136 operand immP_poll() %{
4137   predicate(n->get_ptr() != 0 && n->get_ptr() == (intptr_t)os::get_polling_page());
4138   match(ConP);
4139 
4140   // formats are generated automatically for constants and base registers
4141   format %{ %}
4142   interface(CONST_INTER);
4143 %}
4144 
4145 // Pointer Immediate
4146 operand immN() %{
4147   match(ConN);
4148 
4149   op_cost(10);
4150   format %{ %}
4151   interface(CONST_INTER);
4152 %}
4153 
4154 // NULL Pointer Immediate
4155 operand immN0() %{
4156   predicate(n->get_narrowcon() == 0);
4157   match(ConN);
4158 
4159   op_cost(5);
4160   format %{ %}
4161   interface(CONST_INTER);
4162 %}
4163 
4164 operand immP31()
4165 %{
4166   predicate(!n->as_Type()->type()->isa_oopptr()
4167             && (n->get_ptr() >> 31) == 0);
4168   match(ConP);
4169 
4170   op_cost(5);
4171   format %{ %}
4172   interface(CONST_INTER);
4173 %}
4174 
4175 
4176 // Long Immediate
4177 operand immL()
4178 %{
4179   match(ConL);
4180 
4181   op_cost(20);
4182   format %{ %}
4183   interface(CONST_INTER);
4184 %}
4185 
4186 // Long Immediate 8-bit
4187 operand immL8()
4188 %{
4189   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
4190   match(ConL);
4191 
4192   op_cost(5);
4193   format %{ %}
4194   interface(CONST_INTER);
4195 %}
4196 
4197 // Long Immediate 32-bit unsigned
4198 operand immUL32()
4199 %{
4200   predicate(n->get_long() == (unsigned int) (n->get_long()));
4201   match(ConL);
4202 
4203   op_cost(10);
4204   format %{ %}
4205   interface(CONST_INTER);
4206 %}
4207 
4208 // Long Immediate 32-bit signed
4209 operand immL32()
4210 %{
4211   predicate(n->get_long() == (int) (n->get_long()));
4212   match(ConL);
4213 
4214   op_cost(15);
4215   format %{ %}
4216   interface(CONST_INTER);
4217 %}
4218 
4219 // Long Immediate zero
4220 operand immL0()
4221 %{
4222   predicate(n->get_long() == 0L);
4223   match(ConL);
4224 
4225   op_cost(10);
4226   format %{ %}
4227   interface(CONST_INTER);
4228 %}
4229 
4230 // Constant for increment
4231 operand immL1()
4232 %{
4233   predicate(n->get_long() == 1);
4234   match(ConL);
4235 
4236   format %{ %}
4237   interface(CONST_INTER);
4238 %}
4239 
4240 // Constant for decrement
4241 operand immL_M1()
4242 %{
4243   predicate(n->get_long() == -1);
4244   match(ConL);
4245 
4246   format %{ %}
4247   interface(CONST_INTER);
4248 %}
4249 
4250 // Long Immediate: the value 10
4251 operand immL10()
4252 %{
4253   predicate(n->get_long() == 10);
4254   match(ConL);
4255 
4256   format %{ %}
4257   interface(CONST_INTER);
4258 %}
4259 
4260 // Long immediate from 0 to 127.
4261 // Used for a shorter form of long mul by 10.
4262 operand immL_127()
4263 %{
4264   predicate(0 <= n->get_long() && n->get_long() < 0x80);
4265   match(ConL);
4266 
4267   op_cost(10);
4268   format %{ %}
4269   interface(CONST_INTER);
4270 %}
4271 
4272 // Long Immediate: low 32-bit mask
4273 operand immL_32bits()
4274 %{
4275   predicate(n->get_long() == 0xFFFFFFFFL);
4276   match(ConL);
4277   op_cost(20);
4278 
4279   format %{ %}
4280   interface(CONST_INTER);
4281 %}
4282 
4283 // Float Immediate zero
4284 operand immF0()
4285 %{
4286   predicate(jint_cast(n->getf()) == 0);
4287   match(ConF);
4288 
4289   op_cost(5);
4290   format %{ %}
4291   interface(CONST_INTER);
4292 %}
4293 
4294 // Float Immediate
4295 operand immF()
4296 %{
4297   match(ConF);
4298 
4299   op_cost(15);
4300   format %{ %}
4301   interface(CONST_INTER);
4302 %}
4303 
4304 // Double Immediate zero
4305 operand immD0()
4306 %{
4307   predicate(jlong_cast(n->getd()) == 0);
4308   match(ConD);
4309 
4310   op_cost(5);
4311   format %{ %}
4312   interface(CONST_INTER);
4313 %}
4314 
4315 // Double Immediate
4316 operand immD()
4317 %{
4318   match(ConD);
4319 
4320   op_cost(15);
4321   format %{ %}
4322   interface(CONST_INTER);
4323 %}
4324 
4325 // Immediates for special shifts (sign extend)
4326 
4327 // Constants for increment
4328 operand immI_16()
4329 %{
4330   predicate(n->get_int() == 16);
4331   match(ConI);
4332 
4333   format %{ %}
4334   interface(CONST_INTER);
4335 %}
4336 
4337 operand immI_24()
4338 %{
4339   predicate(n->get_int() == 24);
4340   match(ConI);
4341 
4342   format %{ %}
4343   interface(CONST_INTER);
4344 %}
4345 
4346 // Constant for byte-wide masking
4347 operand immI_255()
4348 %{
4349   predicate(n->get_int() == 255);
4350   match(ConI);
4351 
4352   format %{ %}
4353   interface(CONST_INTER);
4354 %}
4355 
4356 // Constant for short-wide masking
4357 operand immI_65535()
4358 %{
4359   predicate(n->get_int() == 65535);
4360   match(ConI);
4361 
4362   format %{ %}
4363   interface(CONST_INTER);
4364 %}
4365 
4366 // Constant for byte-wide masking
4367 operand immL_255()
4368 %{
4369   predicate(n->get_long() == 255);
4370   match(ConL);
4371 
4372   format %{ %}
4373   interface(CONST_INTER);
4374 %}
4375 
4376 // Constant for short-wide masking
4377 operand immL_65535()
4378 %{
4379   predicate(n->get_long() == 65535);
4380   match(ConL);
4381 
4382   format %{ %}
4383   interface(CONST_INTER);
4384 %}
4385 
4386 // Register Operands
4387 // Integer Register
4388 operand rRegI()
4389 %{
4390   constraint(ALLOC_IN_RC(int_reg));
4391   match(RegI);
4392 
4393   match(rax_RegI);
4394   match(rbx_RegI);
4395   match(rcx_RegI);
4396   match(rdx_RegI);
4397   match(rdi_RegI);
4398 
4399   format %{ %}
4400   interface(REG_INTER);
4401 %}
4402 
4403 // Special Registers
4404 operand rax_RegI()
4405 %{
4406   constraint(ALLOC_IN_RC(int_rax_reg));
4407   match(RegI);
4408   match(rRegI);
4409 
4410   format %{ "RAX" %}
4411   interface(REG_INTER);
4412 %}
4413 
4414 // Special Registers
4415 operand rbx_RegI()
4416 %{
4417   constraint(ALLOC_IN_RC(int_rbx_reg));
4418   match(RegI);
4419   match(rRegI);
4420 
4421   format %{ "RBX" %}
4422   interface(REG_INTER);
4423 %}
4424 
4425 operand rcx_RegI()
4426 %{
4427   constraint(ALLOC_IN_RC(int_rcx_reg));
4428   match(RegI);
4429   match(rRegI);
4430 
4431   format %{ "RCX" %}
4432   interface(REG_INTER);
4433 %}
4434 
4435 operand rdx_RegI()
4436 %{
4437   constraint(ALLOC_IN_RC(int_rdx_reg));
4438   match(RegI);
4439   match(rRegI);
4440 
4441   format %{ "RDX" %}
4442   interface(REG_INTER);
4443 %}
4444 
4445 operand rdi_RegI()
4446 %{
4447   constraint(ALLOC_IN_RC(int_rdi_reg));
4448   match(RegI);
4449   match(rRegI);
4450 
4451   format %{ "RDI" %}
4452   interface(REG_INTER);
4453 %}
4454 
4455 operand no_rcx_RegI()
4456 %{
4457   constraint(ALLOC_IN_RC(int_no_rcx_reg));
4458   match(RegI);
4459   match(rax_RegI);
4460   match(rbx_RegI);
4461   match(rdx_RegI);
4462   match(rdi_RegI);
4463 
4464   format %{ %}
4465   interface(REG_INTER);
4466 %}
4467 
4468 operand no_rax_rdx_RegI()
4469 %{
4470   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
4471   match(RegI);
4472   match(rbx_RegI);
4473   match(rcx_RegI);
4474   match(rdi_RegI);
4475 
4476   format %{ %}
4477   interface(REG_INTER);
4478 %}
4479 
4480 // Pointer Register
4481 operand any_RegP()
4482 %{
4483   constraint(ALLOC_IN_RC(any_reg));
4484   match(RegP);
4485   match(rax_RegP);
4486   match(rbx_RegP);
4487   match(rdi_RegP);
4488   match(rsi_RegP);
4489   match(rbp_RegP);
4490   match(r15_RegP);
4491   match(rRegP);
4492 
4493   format %{ %}
4494   interface(REG_INTER);
4495 %}
4496 
4497 operand rRegP()
4498 %{
4499   constraint(ALLOC_IN_RC(ptr_reg));
4500   match(RegP);
4501   match(rax_RegP);
4502   match(rbx_RegP);
4503   match(rdi_RegP);
4504   match(rsi_RegP);
4505   match(rbp_RegP);
4506   match(r15_RegP);  // See Q&A below about r15_RegP.
4507 
4508   format %{ %}
4509   interface(REG_INTER);
4510 %}
4511 
4512 operand rRegN() %{
4513   constraint(ALLOC_IN_RC(int_reg));
4514   match(RegN);
4515 
4516   format %{ %}
4517   interface(REG_INTER);
4518 %}
4519 
4520 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
4521 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
4522 // It's fine for an instruction input which expects rRegP to match a r15_RegP.
4523 // The output of an instruction is controlled by the allocator, which respects
4524 // register class masks, not match rules.  Unless an instruction mentions
4525 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
4526 // by the allocator as an input.
4527 
4528 operand no_rax_RegP()
4529 %{
4530   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
4531   match(RegP);
4532   match(rbx_RegP);
4533   match(rsi_RegP);
4534   match(rdi_RegP);
4535 
4536   format %{ %}
4537   interface(REG_INTER);
4538 %}
4539 
4540 operand no_rbp_RegP()
4541 %{
4542   constraint(ALLOC_IN_RC(ptr_no_rbp_reg));
4543   match(RegP);
4544   match(rbx_RegP);
4545   match(rsi_RegP);
4546   match(rdi_RegP);
4547 
4548   format %{ %}
4549   interface(REG_INTER);
4550 %}
4551 
4552 operand no_rax_rbx_RegP()
4553 %{
4554   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
4555   match(RegP);
4556   match(rsi_RegP);
4557   match(rdi_RegP);
4558 
4559   format %{ %}
4560   interface(REG_INTER);
4561 %}
4562 
4563 // Special Registers
4564 // Return a pointer value
4565 operand rax_RegP()
4566 %{
4567   constraint(ALLOC_IN_RC(ptr_rax_reg));
4568   match(RegP);
4569   match(rRegP);
4570 
4571   format %{ %}
4572   interface(REG_INTER);
4573 %}
4574 
4575 // Special Registers
4576 // Return a compressed pointer value
4577 operand rax_RegN()
4578 %{
4579   constraint(ALLOC_IN_RC(int_rax_reg));
4580   match(RegN);
4581   match(rRegN);
4582 
4583   format %{ %}
4584   interface(REG_INTER);
4585 %}
4586 
4587 // Used in AtomicAdd
4588 operand rbx_RegP()
4589 %{
4590   constraint(ALLOC_IN_RC(ptr_rbx_reg));
4591   match(RegP);
4592   match(rRegP);
4593 
4594   format %{ %}
4595   interface(REG_INTER);
4596 %}
4597 
4598 operand rsi_RegP()
4599 %{
4600   constraint(ALLOC_IN_RC(ptr_rsi_reg));
4601   match(RegP);
4602   match(rRegP);
4603 
4604   format %{ %}
4605   interface(REG_INTER);
4606 %}
4607 
4608 // Used in rep stosq
4609 operand rdi_RegP()
4610 %{
4611   constraint(ALLOC_IN_RC(ptr_rdi_reg));
4612   match(RegP);
4613   match(rRegP);
4614 
4615   format %{ %}
4616   interface(REG_INTER);
4617 %}
4618 
4619 operand rbp_RegP()
4620 %{
4621   constraint(ALLOC_IN_RC(ptr_rbp_reg));
4622   match(RegP);
4623   match(rRegP);
4624 
4625   format %{ %}
4626   interface(REG_INTER);
4627 %}
4628 
4629 operand r15_RegP()
4630 %{
4631   constraint(ALLOC_IN_RC(ptr_r15_reg));
4632   match(RegP);
4633   match(rRegP);
4634 
4635   format %{ %}
4636   interface(REG_INTER);
4637 %}
4638 
4639 operand rRegL()
4640 %{
4641   constraint(ALLOC_IN_RC(long_reg));
4642   match(RegL);
4643   match(rax_RegL);
4644   match(rdx_RegL);
4645 
4646   format %{ %}
4647   interface(REG_INTER);
4648 %}
4649 
4650 // Special Registers
4651 operand no_rax_rdx_RegL()
4652 %{
4653   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
4654   match(RegL);
4655   match(rRegL);
4656 
4657   format %{ %}
4658   interface(REG_INTER);
4659 %}
4660 
4661 operand no_rax_RegL()
4662 %{
4663   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
4664   match(RegL);
4665   match(rRegL);
4666   match(rdx_RegL);
4667 
4668   format %{ %}
4669   interface(REG_INTER);
4670 %}
4671 
4672 operand no_rcx_RegL()
4673 %{
4674   constraint(ALLOC_IN_RC(long_no_rcx_reg));
4675   match(RegL);
4676   match(rRegL);
4677 
4678   format %{ %}
4679   interface(REG_INTER);
4680 %}
4681 
4682 operand rax_RegL()
4683 %{
4684   constraint(ALLOC_IN_RC(long_rax_reg));
4685   match(RegL);
4686   match(rRegL);
4687 
4688   format %{ "RAX" %}
4689   interface(REG_INTER);
4690 %}
4691 
4692 operand rcx_RegL()
4693 %{
4694   constraint(ALLOC_IN_RC(long_rcx_reg));
4695   match(RegL);
4696   match(rRegL);
4697 
4698   format %{ %}
4699   interface(REG_INTER);
4700 %}
4701 
4702 operand rdx_RegL()
4703 %{
4704   constraint(ALLOC_IN_RC(long_rdx_reg));
4705   match(RegL);
4706   match(rRegL);
4707 
4708   format %{ %}
4709   interface(REG_INTER);
4710 %}
4711 
4712 // Flags register, used as output of compare instructions
4713 operand rFlagsReg()
4714 %{
4715   constraint(ALLOC_IN_RC(int_flags));
4716   match(RegFlags);
4717 
4718   format %{ "RFLAGS" %}
4719   interface(REG_INTER);
4720 %}
4721 
4722 // Flags register, used as output of FLOATING POINT compare instructions
4723 operand rFlagsRegU()
4724 %{
4725   constraint(ALLOC_IN_RC(int_flags));
4726   match(RegFlags);
4727 
4728   format %{ "RFLAGS_U" %}
4729   interface(REG_INTER);
4730 %}
4731 
4732 operand rFlagsRegUCF() %{
4733   constraint(ALLOC_IN_RC(int_flags));
4734   match(RegFlags);
4735   predicate(false);
4736 
4737   format %{ "RFLAGS_U_CF" %}
4738   interface(REG_INTER);
4739 %}
4740 
4741 // Float register operands
4742 operand regF()
4743 %{
4744   constraint(ALLOC_IN_RC(float_reg));
4745   match(RegF);
4746 
4747   format %{ %}
4748   interface(REG_INTER);
4749 %}
4750 
4751 // Double register operands
4752 operand regD()
4753 %{
4754   constraint(ALLOC_IN_RC(double_reg));
4755   match(RegD);
4756 
4757   format %{ %}
4758   interface(REG_INTER);
4759 %}
4760 
4761 
4762 //----------Memory Operands----------------------------------------------------
4763 // Direct Memory Operand
4764 // operand direct(immP addr)
4765 // %{
4766 //   match(addr);
4767 
4768 //   format %{ "[$addr]" %}
4769 //   interface(MEMORY_INTER) %{
4770 //     base(0xFFFFFFFF);
4771 //     index(0x4);
4772 //     scale(0x0);
4773 //     disp($addr);
4774 //   %}
4775 // %}
4776 
4777 // Indirect Memory Operand
4778 operand indirect(any_RegP reg)
4779 %{
4780   constraint(ALLOC_IN_RC(ptr_reg));
4781   match(reg);
4782 
4783   format %{ "[$reg]" %}
4784   interface(MEMORY_INTER) %{
4785     base($reg);
4786     index(0x4);
4787     scale(0x0);
4788     disp(0x0);
4789   %}
4790 %}
4791 
4792 // Indirect Memory Plus Short Offset Operand
4793 operand indOffset8(any_RegP reg, immL8 off)
4794 %{
4795   constraint(ALLOC_IN_RC(ptr_reg));
4796   match(AddP reg off);
4797 
4798   format %{ "[$reg + $off (8-bit)]" %}
4799   interface(MEMORY_INTER) %{
4800     base($reg);
4801     index(0x4);
4802     scale(0x0);
4803     disp($off);
4804   %}
4805 %}
4806 
4807 // Indirect Memory Plus Long Offset Operand
4808 operand indOffset32(any_RegP reg, immL32 off)
4809 %{
4810   constraint(ALLOC_IN_RC(ptr_reg));
4811   match(AddP reg off);
4812 
4813   format %{ "[$reg + $off (32-bit)]" %}
4814   interface(MEMORY_INTER) %{
4815     base($reg);
4816     index(0x4);
4817     scale(0x0);
4818     disp($off);
4819   %}
4820 %}
4821 
4822 // Indirect Memory Plus Index Register Plus Offset Operand
4823 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
4824 %{
4825   constraint(ALLOC_IN_RC(ptr_reg));
4826   match(AddP (AddP reg lreg) off);
4827 
4828   op_cost(10);
4829   format %{"[$reg + $off + $lreg]" %}
4830   interface(MEMORY_INTER) %{
4831     base($reg);
4832     index($lreg);
4833     scale(0x0);
4834     disp($off);
4835   %}
4836 %}
4837 
4838 // Indirect Memory Plus Index Register Plus Offset Operand
4839 operand indIndex(any_RegP reg, rRegL lreg)
4840 %{
4841   constraint(ALLOC_IN_RC(ptr_reg));
4842   match(AddP reg lreg);
4843 
4844   op_cost(10);
4845   format %{"[$reg + $lreg]" %}
4846   interface(MEMORY_INTER) %{
4847     base($reg);
4848     index($lreg);
4849     scale(0x0);
4850     disp(0x0);
4851   %}
4852 %}
4853 
4854 // Indirect Memory Times Scale Plus Index Register
4855 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
4856 %{
4857   constraint(ALLOC_IN_RC(ptr_reg));
4858   match(AddP reg (LShiftL lreg scale));
4859 
4860   op_cost(10);
4861   format %{"[$reg + $lreg << $scale]" %}
4862   interface(MEMORY_INTER) %{
4863     base($reg);
4864     index($lreg);
4865     scale($scale);
4866     disp(0x0);
4867   %}
4868 %}
4869 
4870 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4871 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
4872 %{
4873   constraint(ALLOC_IN_RC(ptr_reg));
4874   match(AddP (AddP reg (LShiftL lreg scale)) off);
4875 
4876   op_cost(10);
4877   format %{"[$reg + $off + $lreg << $scale]" %}
4878   interface(MEMORY_INTER) %{
4879     base($reg);
4880     index($lreg);
4881     scale($scale);
4882     disp($off);
4883   %}
4884 %}
4885 
4886 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4887 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
4888 %{
4889   constraint(ALLOC_IN_RC(ptr_reg));
4890   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4891   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
4892 
4893   op_cost(10);
4894   format %{"[$reg + $off + $idx << $scale]" %}
4895   interface(MEMORY_INTER) %{
4896     base($reg);
4897     index($idx);
4898     scale($scale);
4899     disp($off);
4900   %}
4901 %}
4902 
4903 // Indirect Narrow Oop Plus Offset Operand
4904 // Note: x86 architecture doesn't support "scale * index + offset" without a base
4905 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
4906 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
4907   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
4908   constraint(ALLOC_IN_RC(ptr_reg));
4909   match(AddP (DecodeN reg) off);
4910 
4911   op_cost(10);
4912   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
4913   interface(MEMORY_INTER) %{
4914     base(0xc); // R12
4915     index($reg);
4916     scale(0x3);
4917     disp($off);
4918   %}
4919 %}
4920 
4921 // Indirect Memory Operand
4922 operand indirectNarrow(rRegN reg)
4923 %{
4924   predicate(Universe::narrow_oop_shift() == 0);
4925   constraint(ALLOC_IN_RC(ptr_reg));
4926   match(DecodeN reg);
4927 
4928   format %{ "[$reg]" %}
4929   interface(MEMORY_INTER) %{
4930     base($reg);
4931     index(0x4);
4932     scale(0x0);
4933     disp(0x0);
4934   %}
4935 %}
4936 
4937 // Indirect Memory Plus Short Offset Operand
4938 operand indOffset8Narrow(rRegN reg, immL8 off)
4939 %{
4940   predicate(Universe::narrow_oop_shift() == 0);
4941   constraint(ALLOC_IN_RC(ptr_reg));
4942   match(AddP (DecodeN reg) off);
4943 
4944   format %{ "[$reg + $off (8-bit)]" %}
4945   interface(MEMORY_INTER) %{
4946     base($reg);
4947     index(0x4);
4948     scale(0x0);
4949     disp($off);
4950   %}
4951 %}
4952 
4953 // Indirect Memory Plus Long Offset Operand
4954 operand indOffset32Narrow(rRegN reg, immL32 off)
4955 %{
4956   predicate(Universe::narrow_oop_shift() == 0);
4957   constraint(ALLOC_IN_RC(ptr_reg));
4958   match(AddP (DecodeN reg) off);
4959 
4960   format %{ "[$reg + $off (32-bit)]" %}
4961   interface(MEMORY_INTER) %{
4962     base($reg);
4963     index(0x4);
4964     scale(0x0);
4965     disp($off);
4966   %}
4967 %}
4968 
4969 // Indirect Memory Plus Index Register Plus Offset Operand
4970 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4971 %{
4972   predicate(Universe::narrow_oop_shift() == 0);
4973   constraint(ALLOC_IN_RC(ptr_reg));
4974   match(AddP (AddP (DecodeN reg) lreg) off);
4975 
4976   op_cost(10);
4977   format %{"[$reg + $off + $lreg]" %}
4978   interface(MEMORY_INTER) %{
4979     base($reg);
4980     index($lreg);
4981     scale(0x0);
4982     disp($off);
4983   %}
4984 %}
4985 
4986 // Indirect Memory Plus Index Register Plus Offset Operand
4987 operand indIndexNarrow(rRegN reg, rRegL lreg)
4988 %{
4989   predicate(Universe::narrow_oop_shift() == 0);
4990   constraint(ALLOC_IN_RC(ptr_reg));
4991   match(AddP (DecodeN reg) lreg);
4992 
4993   op_cost(10);
4994   format %{"[$reg + $lreg]" %}
4995   interface(MEMORY_INTER) %{
4996     base($reg);
4997     index($lreg);
4998     scale(0x0);
4999     disp(0x0);
5000   %}
5001 %}
5002 
5003 // Indirect Memory Times Scale Plus Index Register
5004 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
5005 %{
5006   predicate(Universe::narrow_oop_shift() == 0);
5007   constraint(ALLOC_IN_RC(ptr_reg));
5008   match(AddP (DecodeN reg) (LShiftL lreg scale));
5009 
5010   op_cost(10);
5011   format %{"[$reg + $lreg << $scale]" %}
5012   interface(MEMORY_INTER) %{
5013     base($reg);
5014     index($lreg);
5015     scale($scale);
5016     disp(0x0);
5017   %}
5018 %}
5019 
5020 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
5021 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
5022 %{
5023   predicate(Universe::narrow_oop_shift() == 0);
5024   constraint(ALLOC_IN_RC(ptr_reg));
5025   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5026 
5027   op_cost(10);
5028   format %{"[$reg + $off + $lreg << $scale]" %}
5029   interface(MEMORY_INTER) %{
5030     base($reg);
5031     index($lreg);
5032     scale($scale);
5033     disp($off);
5034   %}
5035 %}
5036 
5037 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
5038 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
5039 %{
5040   constraint(ALLOC_IN_RC(ptr_reg));
5041   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
5042   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
5043 
5044   op_cost(10);
5045   format %{"[$reg + $off + $idx << $scale]" %}
5046   interface(MEMORY_INTER) %{
5047     base($reg);
5048     index($idx);
5049     scale($scale);
5050     disp($off);
5051   %}
5052 %}
5053 
5054 
5055 //----------Special Memory Operands--------------------------------------------
5056 // Stack Slot Operand - This operand is used for loading and storing temporary
5057 //                      values on the stack where a match requires a value to
5058 //                      flow through memory.
5059 operand stackSlotP(sRegP reg)
5060 %{
5061   constraint(ALLOC_IN_RC(stack_slots));
5062   // No match rule because this operand is only generated in matching
5063 
5064   format %{ "[$reg]" %}
5065   interface(MEMORY_INTER) %{
5066     base(0x4);   // RSP
5067     index(0x4);  // No Index
5068     scale(0x0);  // No Scale
5069     disp($reg);  // Stack Offset
5070   %}
5071 %}
5072 
5073 operand stackSlotI(sRegI reg)
5074 %{
5075   constraint(ALLOC_IN_RC(stack_slots));
5076   // No match rule because this operand is only generated in matching
5077 
5078   format %{ "[$reg]" %}
5079   interface(MEMORY_INTER) %{
5080     base(0x4);   // RSP
5081     index(0x4);  // No Index
5082     scale(0x0);  // No Scale
5083     disp($reg);  // Stack Offset
5084   %}
5085 %}
5086 
5087 operand stackSlotF(sRegF reg)
5088 %{
5089   constraint(ALLOC_IN_RC(stack_slots));
5090   // No match rule because this operand is only generated in matching
5091 
5092   format %{ "[$reg]" %}
5093   interface(MEMORY_INTER) %{
5094     base(0x4);   // RSP
5095     index(0x4);  // No Index
5096     scale(0x0);  // No Scale
5097     disp($reg);  // Stack Offset
5098   %}
5099 %}
5100 
5101 operand stackSlotD(sRegD reg)
5102 %{
5103   constraint(ALLOC_IN_RC(stack_slots));
5104   // No match rule because this operand is only generated in matching
5105 
5106   format %{ "[$reg]" %}
5107   interface(MEMORY_INTER) %{
5108     base(0x4);   // RSP
5109     index(0x4);  // No Index
5110     scale(0x0);  // No Scale
5111     disp($reg);  // Stack Offset
5112   %}
5113 %}
5114 operand stackSlotL(sRegL reg)
5115 %{
5116   constraint(ALLOC_IN_RC(stack_slots));
5117   // No match rule because this operand is only generated in matching
5118 
5119   format %{ "[$reg]" %}
5120   interface(MEMORY_INTER) %{
5121     base(0x4);   // RSP
5122     index(0x4);  // No Index
5123     scale(0x0);  // No Scale
5124     disp($reg);  // Stack Offset
5125   %}
5126 %}
5127 
5128 //----------Conditional Branch Operands----------------------------------------
5129 // Comparison Op  - This is the operation of the comparison, and is limited to
5130 //                  the following set of codes:
5131 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5132 //
5133 // Other attributes of the comparison, such as unsignedness, are specified
5134 // by the comparison instruction that sets a condition code flags register.
5135 // That result is represented by a flags operand whose subtype is appropriate
5136 // to the unsignedness (etc.) of the comparison.
5137 //
5138 // Later, the instruction which matches both the Comparison Op (a Bool) and
5139 // the flags (produced by the Cmp) specifies the coding of the comparison op
5140 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5141 
5142 // Comparision Code
5143 operand cmpOp()
5144 %{
5145   match(Bool);
5146 
5147   format %{ "" %}
5148   interface(COND_INTER) %{
5149     equal(0x4, "e");
5150     not_equal(0x5, "ne");
5151     less(0xC, "l");
5152     greater_equal(0xD, "ge");
5153     less_equal(0xE, "le");
5154     greater(0xF, "g");
5155   %}
5156 %}
5157 
5158 // Comparison Code, unsigned compare.  Used by FP also, with
5159 // C2 (unordered) turned into GT or LT already.  The other bits
5160 // C0 and C3 are turned into Carry & Zero flags.
5161 operand cmpOpU()
5162 %{
5163   match(Bool);
5164 
5165   format %{ "" %}
5166   interface(COND_INTER) %{
5167     equal(0x4, "e");
5168     not_equal(0x5, "ne");
5169     less(0x2, "b");
5170     greater_equal(0x3, "nb");
5171     less_equal(0x6, "be");
5172     greater(0x7, "nbe");
5173   %}
5174 %}
5175 
5176 
5177 // Floating comparisons that don't require any fixup for the unordered case
5178 operand cmpOpUCF() %{
5179   match(Bool);
5180   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
5181             n->as_Bool()->_test._test == BoolTest::ge ||
5182             n->as_Bool()->_test._test == BoolTest::le ||
5183             n->as_Bool()->_test._test == BoolTest::gt);
5184   format %{ "" %}
5185   interface(COND_INTER) %{
5186     equal(0x4, "e");
5187     not_equal(0x5, "ne");
5188     less(0x2, "b");
5189     greater_equal(0x3, "nb");
5190     less_equal(0x6, "be");
5191     greater(0x7, "nbe");
5192   %}
5193 %}
5194 
5195 
5196 // Floating comparisons that can be fixed up with extra conditional jumps
5197 operand cmpOpUCF2() %{
5198   match(Bool);
5199   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
5200             n->as_Bool()->_test._test == BoolTest::eq);
5201   format %{ "" %}
5202   interface(COND_INTER) %{
5203     equal(0x4, "e");
5204     not_equal(0x5, "ne");
5205     less(0x2, "b");
5206     greater_equal(0x3, "nb");
5207     less_equal(0x6, "be");
5208     greater(0x7, "nbe");
5209   %}
5210 %}
5211 
5212 
5213 //----------OPERAND CLASSES----------------------------------------------------
5214 // Operand Classes are groups of operands that are used as to simplify
5215 // instruction definitions by not requiring the AD writer to specify separate
5216 // instructions for every form of operand when the instruction accepts
5217 // multiple operand types with the same basic encoding and format.  The classic
5218 // case of this is memory operands.
5219 
5220 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
5221                indIndexScale, indIndexScaleOffset, indPosIndexScaleOffset,
5222                indCompressedOopOffset,
5223                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
5224                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
5225                indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
5226 
5227 //----------PIPELINE-----------------------------------------------------------
5228 // Rules which define the behavior of the target architectures pipeline.
5229 pipeline %{
5230 
5231 //----------ATTRIBUTES---------------------------------------------------------
5232 attributes %{
5233   variable_size_instructions;        // Fixed size instructions
5234   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
5235   instruction_unit_size = 1;         // An instruction is 1 bytes long
5236   instruction_fetch_unit_size = 16;  // The processor fetches one line
5237   instruction_fetch_units = 1;       // of 16 bytes
5238 
5239   // List of nop instructions
5240   nops( MachNop );
5241 %}
5242 
5243 //----------RESOURCES----------------------------------------------------------
5244 // Resources are the functional units available to the machine
5245 
5246 // Generic P2/P3 pipeline
5247 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
5248 // 3 instructions decoded per cycle.
5249 // 2 load/store ops per cycle, 1 branch, 1 FPU,
5250 // 3 ALU op, only ALU0 handles mul instructions.
5251 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
5252            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
5253            BR, FPU,
5254            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
5255 
5256 //----------PIPELINE DESCRIPTION-----------------------------------------------
5257 // Pipeline Description specifies the stages in the machine's pipeline
5258 
5259 // Generic P2/P3 pipeline
5260 pipe_desc(S0, S1, S2, S3, S4, S5);
5261 
5262 //----------PIPELINE CLASSES---------------------------------------------------
5263 // Pipeline Classes describe the stages in which input and output are
5264 // referenced by the hardware pipeline.
5265 
5266 // Naming convention: ialu or fpu
5267 // Then: _reg
5268 // Then: _reg if there is a 2nd register
5269 // Then: _long if it's a pair of instructions implementing a long
5270 // Then: _fat if it requires the big decoder
5271 //   Or: _mem if it requires the big decoder and a memory unit.
5272 
5273 // Integer ALU reg operation
5274 pipe_class ialu_reg(rRegI dst)
5275 %{
5276     single_instruction;
5277     dst    : S4(write);
5278     dst    : S3(read);
5279     DECODE : S0;        // any decoder
5280     ALU    : S3;        // any alu
5281 %}
5282 
5283 // Long ALU reg operation
5284 pipe_class ialu_reg_long(rRegL dst)
5285 %{
5286     instruction_count(2);
5287     dst    : S4(write);
5288     dst    : S3(read);
5289     DECODE : S0(2);     // any 2 decoders
5290     ALU    : S3(2);     // both alus
5291 %}
5292 
5293 // Integer ALU reg operation using big decoder
5294 pipe_class ialu_reg_fat(rRegI dst)
5295 %{
5296     single_instruction;
5297     dst    : S4(write);
5298     dst    : S3(read);
5299     D0     : S0;        // big decoder only
5300     ALU    : S3;        // any alu
5301 %}
5302 
5303 // Long ALU reg operation using big decoder
5304 pipe_class ialu_reg_long_fat(rRegL dst)
5305 %{
5306     instruction_count(2);
5307     dst    : S4(write);
5308     dst    : S3(read);
5309     D0     : S0(2);     // big decoder only; twice
5310     ALU    : S3(2);     // any 2 alus
5311 %}
5312 
5313 // Integer ALU reg-reg operation
5314 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
5315 %{
5316     single_instruction;
5317     dst    : S4(write);
5318     src    : S3(read);
5319     DECODE : S0;        // any decoder
5320     ALU    : S3;        // any alu
5321 %}
5322 
5323 // Long ALU reg-reg operation
5324 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
5325 %{
5326     instruction_count(2);
5327     dst    : S4(write);
5328     src    : S3(read);
5329     DECODE : S0(2);     // any 2 decoders
5330     ALU    : S3(2);     // both alus
5331 %}
5332 
5333 // Integer ALU reg-reg operation
5334 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
5335 %{
5336     single_instruction;
5337     dst    : S4(write);
5338     src    : S3(read);
5339     D0     : S0;        // big decoder only
5340     ALU    : S3;        // any alu
5341 %}
5342 
5343 // Long ALU reg-reg operation
5344 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
5345 %{
5346     instruction_count(2);
5347     dst    : S4(write);
5348     src    : S3(read);
5349     D0     : S0(2);     // big decoder only; twice
5350     ALU    : S3(2);     // both alus
5351 %}
5352 
5353 // Integer ALU reg-mem operation
5354 pipe_class ialu_reg_mem(rRegI dst, memory mem)
5355 %{
5356     single_instruction;
5357     dst    : S5(write);
5358     mem    : S3(read);
5359     D0     : S0;        // big decoder only
5360     ALU    : S4;        // any alu
5361     MEM    : S3;        // any mem
5362 %}
5363 
5364 // Integer mem operation (prefetch)
5365 pipe_class ialu_mem(memory mem)
5366 %{
5367     single_instruction;
5368     mem    : S3(read);
5369     D0     : S0;        // big decoder only
5370     MEM    : S3;        // any mem
5371 %}
5372 
5373 // Integer Store to Memory
5374 pipe_class ialu_mem_reg(memory mem, rRegI src)
5375 %{
5376     single_instruction;
5377     mem    : S3(read);
5378     src    : S5(read);
5379     D0     : S0;        // big decoder only
5380     ALU    : S4;        // any alu
5381     MEM    : S3;
5382 %}
5383 
5384 // // Long Store to Memory
5385 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
5386 // %{
5387 //     instruction_count(2);
5388 //     mem    : S3(read);
5389 //     src    : S5(read);
5390 //     D0     : S0(2);          // big decoder only; twice
5391 //     ALU    : S4(2);     // any 2 alus
5392 //     MEM    : S3(2);  // Both mems
5393 // %}
5394 
5395 // Integer Store to Memory
5396 pipe_class ialu_mem_imm(memory mem)
5397 %{
5398     single_instruction;
5399     mem    : S3(read);
5400     D0     : S0;        // big decoder only
5401     ALU    : S4;        // any alu
5402     MEM    : S3;
5403 %}
5404 
5405 // Integer ALU0 reg-reg operation
5406 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
5407 %{
5408     single_instruction;
5409     dst    : S4(write);
5410     src    : S3(read);
5411     D0     : S0;        // Big decoder only
5412     ALU0   : S3;        // only alu0
5413 %}
5414 
5415 // Integer ALU0 reg-mem operation
5416 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
5417 %{
5418     single_instruction;
5419     dst    : S5(write);
5420     mem    : S3(read);
5421     D0     : S0;        // big decoder only
5422     ALU0   : S4;        // ALU0 only
5423     MEM    : S3;        // any mem
5424 %}
5425 
5426 // Integer ALU reg-reg operation
5427 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
5428 %{
5429     single_instruction;
5430     cr     : S4(write);
5431     src1   : S3(read);
5432     src2   : S3(read);
5433     DECODE : S0;        // any decoder
5434     ALU    : S3;        // any alu
5435 %}
5436 
5437 // Integer ALU reg-imm operation
5438 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
5439 %{
5440     single_instruction;
5441     cr     : S4(write);
5442     src1   : S3(read);
5443     DECODE : S0;        // any decoder
5444     ALU    : S3;        // any alu
5445 %}
5446 
5447 // Integer ALU reg-mem operation
5448 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
5449 %{
5450     single_instruction;
5451     cr     : S4(write);
5452     src1   : S3(read);
5453     src2   : S3(read);
5454     D0     : S0;        // big decoder only
5455     ALU    : S4;        // any alu
5456     MEM    : S3;
5457 %}
5458 
5459 // Conditional move reg-reg
5460 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
5461 %{
5462     instruction_count(4);
5463     y      : S4(read);
5464     q      : S3(read);
5465     p      : S3(read);
5466     DECODE : S0(4);     // any decoder
5467 %}
5468 
5469 // Conditional move reg-reg
5470 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
5471 %{
5472     single_instruction;
5473     dst    : S4(write);
5474     src    : S3(read);
5475     cr     : S3(read);
5476     DECODE : S0;        // any decoder
5477 %}
5478 
5479 // Conditional move reg-mem
5480 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
5481 %{
5482     single_instruction;
5483     dst    : S4(write);
5484     src    : S3(read);
5485     cr     : S3(read);
5486     DECODE : S0;        // any decoder
5487     MEM    : S3;
5488 %}
5489 
5490 // Conditional move reg-reg long
5491 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
5492 %{
5493     single_instruction;
5494     dst    : S4(write);
5495     src    : S3(read);
5496     cr     : S3(read);
5497     DECODE : S0(2);     // any 2 decoders
5498 %}
5499 
5500 // XXX
5501 // // Conditional move double reg-reg
5502 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
5503 // %{
5504 //     single_instruction;
5505 //     dst    : S4(write);
5506 //     src    : S3(read);
5507 //     cr     : S3(read);
5508 //     DECODE : S0;     // any decoder
5509 // %}
5510 
5511 // Float reg-reg operation
5512 pipe_class fpu_reg(regD dst)
5513 %{
5514     instruction_count(2);
5515     dst    : S3(read);
5516     DECODE : S0(2);     // any 2 decoders
5517     FPU    : S3;
5518 %}
5519 
5520 // Float reg-reg operation
5521 pipe_class fpu_reg_reg(regD dst, regD src)
5522 %{
5523     instruction_count(2);
5524     dst    : S4(write);
5525     src    : S3(read);
5526     DECODE : S0(2);     // any 2 decoders
5527     FPU    : S3;
5528 %}
5529 
5530 // Float reg-reg operation
5531 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
5532 %{
5533     instruction_count(3);
5534     dst    : S4(write);
5535     src1   : S3(read);
5536     src2   : S3(read);
5537     DECODE : S0(3);     // any 3 decoders
5538     FPU    : S3(2);
5539 %}
5540 
5541 // Float reg-reg operation
5542 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
5543 %{
5544     instruction_count(4);
5545     dst    : S4(write);
5546     src1   : S3(read);
5547     src2   : S3(read);
5548     src3   : S3(read);
5549     DECODE : S0(4);     // any 3 decoders
5550     FPU    : S3(2);
5551 %}
5552 
5553 // Float reg-reg operation
5554 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
5555 %{
5556     instruction_count(4);
5557     dst    : S4(write);
5558     src1   : S3(read);
5559     src2   : S3(read);
5560     src3   : S3(read);
5561     DECODE : S1(3);     // any 3 decoders
5562     D0     : S0;        // Big decoder only
5563     FPU    : S3(2);
5564     MEM    : S3;
5565 %}
5566 
5567 // Float reg-mem operation
5568 pipe_class fpu_reg_mem(regD dst, memory mem)
5569 %{
5570     instruction_count(2);
5571     dst    : S5(write);
5572     mem    : S3(read);
5573     D0     : S0;        // big decoder only
5574     DECODE : S1;        // any decoder for FPU POP
5575     FPU    : S4;
5576     MEM    : S3;        // any mem
5577 %}
5578 
5579 // Float reg-mem operation
5580 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
5581 %{
5582     instruction_count(3);
5583     dst    : S5(write);
5584     src1   : S3(read);
5585     mem    : S3(read);
5586     D0     : S0;        // big decoder only
5587     DECODE : S1(2);     // any decoder for FPU POP
5588     FPU    : S4;
5589     MEM    : S3;        // any mem
5590 %}
5591 
5592 // Float mem-reg operation
5593 pipe_class fpu_mem_reg(memory mem, regD src)
5594 %{
5595     instruction_count(2);
5596     src    : S5(read);
5597     mem    : S3(read);
5598     DECODE : S0;        // any decoder for FPU PUSH
5599     D0     : S1;        // big decoder only
5600     FPU    : S4;
5601     MEM    : S3;        // any mem
5602 %}
5603 
5604 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
5605 %{
5606     instruction_count(3);
5607     src1   : S3(read);
5608     src2   : S3(read);
5609     mem    : S3(read);
5610     DECODE : S0(2);     // any decoder for FPU PUSH
5611     D0     : S1;        // big decoder only
5612     FPU    : S4;
5613     MEM    : S3;        // any mem
5614 %}
5615 
5616 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
5617 %{
5618     instruction_count(3);
5619     src1   : S3(read);
5620     src2   : S3(read);
5621     mem    : S4(read);
5622     DECODE : S0;        // any decoder for FPU PUSH
5623     D0     : S0(2);     // big decoder only
5624     FPU    : S4;
5625     MEM    : S3(2);     // any mem
5626 %}
5627 
5628 pipe_class fpu_mem_mem(memory dst, memory src1)
5629 %{
5630     instruction_count(2);
5631     src1   : S3(read);
5632     dst    : S4(read);
5633     D0     : S0(2);     // big decoder only
5634     MEM    : S3(2);     // any mem
5635 %}
5636 
5637 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
5638 %{
5639     instruction_count(3);
5640     src1   : S3(read);
5641     src2   : S3(read);
5642     dst    : S4(read);
5643     D0     : S0(3);     // big decoder only
5644     FPU    : S4;
5645     MEM    : S3(3);     // any mem
5646 %}
5647 
5648 pipe_class fpu_mem_reg_con(memory mem, regD src1)
5649 %{
5650     instruction_count(3);
5651     src1   : S4(read);
5652     mem    : S4(read);
5653     DECODE : S0;        // any decoder for FPU PUSH
5654     D0     : S0(2);     // big decoder only
5655     FPU    : S4;
5656     MEM    : S3(2);     // any mem
5657 %}
5658 
5659 // Float load constant
5660 pipe_class fpu_reg_con(regD dst)
5661 %{
5662     instruction_count(2);
5663     dst    : S5(write);
5664     D0     : S0;        // big decoder only for the load
5665     DECODE : S1;        // any decoder for FPU POP
5666     FPU    : S4;
5667     MEM    : S3;        // any mem
5668 %}
5669 
5670 // Float load constant
5671 pipe_class fpu_reg_reg_con(regD dst, regD src)
5672 %{
5673     instruction_count(3);
5674     dst    : S5(write);
5675     src    : S3(read);
5676     D0     : S0;        // big decoder only for the load
5677     DECODE : S1(2);     // any decoder for FPU POP
5678     FPU    : S4;
5679     MEM    : S3;        // any mem
5680 %}
5681 
5682 // UnConditional branch
5683 pipe_class pipe_jmp(label labl)
5684 %{
5685     single_instruction;
5686     BR   : S3;
5687 %}
5688 
5689 // Conditional branch
5690 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
5691 %{
5692     single_instruction;
5693     cr    : S1(read);
5694     BR    : S3;
5695 %}
5696 
5697 // Allocation idiom
5698 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
5699 %{
5700     instruction_count(1); force_serialization;
5701     fixed_latency(6);
5702     heap_ptr : S3(read);
5703     DECODE   : S0(3);
5704     D0       : S2;
5705     MEM      : S3;
5706     ALU      : S3(2);
5707     dst      : S5(write);
5708     BR       : S5;
5709 %}
5710 
5711 // Generic big/slow expanded idiom
5712 pipe_class pipe_slow()
5713 %{
5714     instruction_count(10); multiple_bundles; force_serialization;
5715     fixed_latency(100);
5716     D0  : S0(2);
5717     MEM : S3(2);
5718 %}
5719 
5720 // The real do-nothing guy
5721 pipe_class empty()
5722 %{
5723     instruction_count(0);
5724 %}
5725 
5726 // Define the class for the Nop node
5727 define
5728 %{
5729    MachNop = empty;
5730 %}
5731 
5732 %}
5733 
5734 //----------INSTRUCTIONS-------------------------------------------------------
5735 //
5736 // match      -- States which machine-independent subtree may be replaced
5737 //               by this instruction.
5738 // ins_cost   -- The estimated cost of this instruction is used by instruction
5739 //               selection to identify a minimum cost tree of machine
5740 //               instructions that matches a tree of machine-independent
5741 //               instructions.
5742 // format     -- A string providing the disassembly for this instruction.
5743 //               The value of an instruction's operand may be inserted
5744 //               by referring to it with a '$' prefix.
5745 // opcode     -- Three instruction opcodes may be provided.  These are referred
5746 //               to within an encode class as $primary, $secondary, and $tertiary
5747 //               rrspectively.  The primary opcode is commonly used to
5748 //               indicate the type of machine instruction, while secondary
5749 //               and tertiary are often used for prefix options or addressing
5750 //               modes.
5751 // ins_encode -- A list of encode classes with parameters. The encode class
5752 //               name must have been defined in an 'enc_class' specification
5753 //               in the encode section of the architecture description.
5754 
5755 
5756 //----------Load/Store/Move Instructions---------------------------------------
5757 //----------Load Instructions--------------------------------------------------
5758 
5759 // Load Byte (8 bit signed)
5760 instruct loadB(rRegI dst, memory mem)
5761 %{
5762   match(Set dst (LoadB mem));
5763 
5764   ins_cost(125);
5765   format %{ "movsbl  $dst, $mem\t# byte" %}
5766 
5767   ins_encode %{
5768     __ movsbl($dst$$Register, $mem$$Address);
5769   %}
5770 
5771   ins_pipe(ialu_reg_mem);
5772 %}
5773 
5774 // Load Byte (8 bit signed) into Long Register
5775 instruct loadB2L(rRegL dst, memory mem)
5776 %{
5777   match(Set dst (ConvI2L (LoadB mem)));
5778 
5779   ins_cost(125);
5780   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5781 
5782   ins_encode %{
5783     __ movsbq($dst$$Register, $mem$$Address);
5784   %}
5785 
5786   ins_pipe(ialu_reg_mem);
5787 %}
5788 
5789 // Load Unsigned Byte (8 bit UNsigned)
5790 instruct loadUB(rRegI dst, memory mem)
5791 %{
5792   match(Set dst (LoadUB mem));
5793 
5794   ins_cost(125);
5795   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5796 
5797   ins_encode %{
5798     __ movzbl($dst$$Register, $mem$$Address);
5799   %}
5800 
5801   ins_pipe(ialu_reg_mem);
5802 %}
5803 
5804 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5805 instruct loadUB2L(rRegL dst, memory mem)
5806 %{
5807   match(Set dst (ConvI2L (LoadUB mem)));
5808 
5809   ins_cost(125);
5810   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5811 
5812   ins_encode %{
5813     __ movzbq($dst$$Register, $mem$$Address);
5814   %}
5815 
5816   ins_pipe(ialu_reg_mem);
5817 %}
5818 
5819 // Load Unsigned Byte (8 bit UNsigned) with a 8-bit mask into Long Register
5820 instruct loadUB2L_immI8(rRegL dst, memory mem, immI8 mask, rFlagsReg cr) %{
5821   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5822   effect(KILL cr);
5823 
5824   format %{ "movzbq  $dst, $mem\t# ubyte & 8-bit mask -> long\n\t"
5825             "andl    $dst, $mask" %}
5826   ins_encode %{
5827     Register Rdst = $dst$$Register;
5828     __ movzbq(Rdst, $mem$$Address);
5829     __ andl(Rdst, $mask$$constant);
5830   %}
5831   ins_pipe(ialu_reg_mem);
5832 %}
5833 
5834 // Load Short (16 bit signed)
5835 instruct loadS(rRegI dst, memory mem)
5836 %{
5837   match(Set dst (LoadS mem));
5838 
5839   ins_cost(125);
5840   format %{ "movswl $dst, $mem\t# short" %}
5841 
5842   ins_encode %{
5843     __ movswl($dst$$Register, $mem$$Address);
5844   %}
5845 
5846   ins_pipe(ialu_reg_mem);
5847 %}
5848 
5849 // Load Short (16 bit signed) to Byte (8 bit signed)
5850 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5851   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5852 
5853   ins_cost(125);
5854   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5855   ins_encode %{
5856     __ movsbl($dst$$Register, $mem$$Address);
5857   %}
5858   ins_pipe(ialu_reg_mem);
5859 %}
5860 
5861 // Load Short (16 bit signed) into Long Register
5862 instruct loadS2L(rRegL dst, memory mem)
5863 %{
5864   match(Set dst (ConvI2L (LoadS mem)));
5865 
5866   ins_cost(125);
5867   format %{ "movswq $dst, $mem\t# short -> long" %}
5868 
5869   ins_encode %{
5870     __ movswq($dst$$Register, $mem$$Address);
5871   %}
5872 
5873   ins_pipe(ialu_reg_mem);
5874 %}
5875 
5876 // Load Unsigned Short/Char (16 bit UNsigned)
5877 instruct loadUS(rRegI dst, memory mem)
5878 %{
5879   match(Set dst (LoadUS mem));
5880 
5881   ins_cost(125);
5882   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5883 
5884   ins_encode %{
5885     __ movzwl($dst$$Register, $mem$$Address);
5886   %}
5887 
5888   ins_pipe(ialu_reg_mem);
5889 %}
5890 
5891 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5892 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5893   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5894 
5895   ins_cost(125);
5896   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5897   ins_encode %{
5898     __ movsbl($dst$$Register, $mem$$Address);
5899   %}
5900   ins_pipe(ialu_reg_mem);
5901 %}
5902 
5903 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5904 instruct loadUS2L(rRegL dst, memory mem)
5905 %{
5906   match(Set dst (ConvI2L (LoadUS mem)));
5907 
5908   ins_cost(125);
5909   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5910 
5911   ins_encode %{
5912     __ movzwq($dst$$Register, $mem$$Address);
5913   %}
5914 
5915   ins_pipe(ialu_reg_mem);
5916 %}
5917 
5918 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5919 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5920   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5921 
5922   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5923   ins_encode %{
5924     __ movzbq($dst$$Register, $mem$$Address);
5925   %}
5926   ins_pipe(ialu_reg_mem);
5927 %}
5928 
5929 // Load Unsigned Short/Char (16 bit UNsigned) with mask into Long Register
5930 instruct loadUS2L_immI16(rRegL dst, memory mem, immI16 mask, rFlagsReg cr) %{
5931   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5932   effect(KILL cr);
5933 
5934   format %{ "movzwq  $dst, $mem\t# ushort/char & 16-bit mask -> long\n\t"
5935             "andl    $dst, $mask" %}
5936   ins_encode %{
5937     Register Rdst = $dst$$Register;
5938     __ movzwq(Rdst, $mem$$Address);
5939     __ andl(Rdst, $mask$$constant);
5940   %}
5941   ins_pipe(ialu_reg_mem);
5942 %}
5943 
5944 // Load Integer
5945 instruct loadI(rRegI dst, memory mem)
5946 %{
5947   match(Set dst (LoadI mem));
5948 
5949   ins_cost(125);
5950   format %{ "movl    $dst, $mem\t# int" %}
5951 
5952   ins_encode %{
5953     __ movl($dst$$Register, $mem$$Address);
5954   %}
5955 
5956   ins_pipe(ialu_reg_mem);
5957 %}
5958 
5959 // Load Integer (32 bit signed) to Byte (8 bit signed)
5960 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5961   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5962 
5963   ins_cost(125);
5964   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5965   ins_encode %{
5966     __ movsbl($dst$$Register, $mem$$Address);
5967   %}
5968   ins_pipe(ialu_reg_mem);
5969 %}
5970 
5971 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5972 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5973   match(Set dst (AndI (LoadI mem) mask));
5974 
5975   ins_cost(125);
5976   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5977   ins_encode %{
5978     __ movzbl($dst$$Register, $mem$$Address);
5979   %}
5980   ins_pipe(ialu_reg_mem);
5981 %}
5982 
5983 // Load Integer (32 bit signed) to Short (16 bit signed)
5984 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5985   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5986 
5987   ins_cost(125);
5988   format %{ "movswl  $dst, $mem\t# int -> short" %}
5989   ins_encode %{
5990     __ movswl($dst$$Register, $mem$$Address);
5991   %}
5992   ins_pipe(ialu_reg_mem);
5993 %}
5994 
5995 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5996 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5997   match(Set dst (AndI (LoadI mem) mask));
5998 
5999   ins_cost(125);
6000   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
6001   ins_encode %{
6002     __ movzwl($dst$$Register, $mem$$Address);
6003   %}
6004   ins_pipe(ialu_reg_mem);
6005 %}
6006 
6007 // Load Integer into Long Register
6008 instruct loadI2L(rRegL dst, memory mem)
6009 %{
6010   match(Set dst (ConvI2L (LoadI mem)));
6011 
6012   ins_cost(125);
6013   format %{ "movslq  $dst, $mem\t# int -> long" %}
6014 
6015   ins_encode %{
6016     __ movslq($dst$$Register, $mem$$Address);
6017   %}
6018 
6019   ins_pipe(ialu_reg_mem);
6020 %}
6021 
6022 // Load Integer with mask 0xFF into Long Register
6023 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
6024   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6025 
6026   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
6027   ins_encode %{
6028     __ movzbq($dst$$Register, $mem$$Address);
6029   %}
6030   ins_pipe(ialu_reg_mem);
6031 %}
6032 
6033 // Load Integer with mask 0xFFFF into Long Register
6034 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
6035   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6036 
6037   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
6038   ins_encode %{
6039     __ movzwq($dst$$Register, $mem$$Address);
6040   %}
6041   ins_pipe(ialu_reg_mem);
6042 %}
6043 
6044 // Load Integer with a 32-bit mask into Long Register
6045 instruct loadI2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
6046   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
6047   effect(KILL cr);
6048 
6049   format %{ "movl    $dst, $mem\t# int & 32-bit mask -> long\n\t"
6050             "andl    $dst, $mask" %}
6051   ins_encode %{
6052     Register Rdst = $dst$$Register;
6053     __ movl(Rdst, $mem$$Address);
6054     __ andl(Rdst, $mask$$constant);
6055   %}
6056   ins_pipe(ialu_reg_mem);
6057 %}
6058 
6059 // Load Unsigned Integer into Long Register
6060 instruct loadUI2L(rRegL dst, memory mem)
6061 %{
6062   match(Set dst (LoadUI2L mem));
6063 
6064   ins_cost(125);
6065   format %{ "movl    $dst, $mem\t# uint -> long" %}
6066 
6067   ins_encode %{
6068     __ movl($dst$$Register, $mem$$Address);
6069   %}
6070 
6071   ins_pipe(ialu_reg_mem);
6072 %}
6073 
6074 // Load Long
6075 instruct loadL(rRegL dst, memory mem)
6076 %{
6077   match(Set dst (LoadL mem));
6078 
6079   ins_cost(125);
6080   format %{ "movq    $dst, $mem\t# long" %}
6081 
6082   ins_encode %{
6083     __ movq($dst$$Register, $mem$$Address);
6084   %}
6085 
6086   ins_pipe(ialu_reg_mem); // XXX
6087 %}
6088 
6089 // Load Range
6090 instruct loadRange(rRegI dst, memory mem)
6091 %{
6092   match(Set dst (LoadRange mem));
6093 
6094   ins_cost(125); // XXX
6095   format %{ "movl    $dst, $mem\t# range" %}
6096   opcode(0x8B);
6097   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
6098   ins_pipe(ialu_reg_mem);
6099 %}
6100 
6101 // Load Pointer
6102 instruct loadP(rRegP dst, memory mem)
6103 %{
6104   match(Set dst (LoadP mem));
6105 
6106   ins_cost(125); // XXX
6107   format %{ "movq    $dst, $mem\t# ptr" %}
6108   opcode(0x8B);
6109   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6110   ins_pipe(ialu_reg_mem); // XXX
6111 %}
6112 
6113 // Load Compressed Pointer
6114 instruct loadN(rRegN dst, memory mem)
6115 %{
6116    match(Set dst (LoadN mem));
6117 
6118    ins_cost(125); // XXX
6119    format %{ "movl    $dst, $mem\t# compressed ptr" %}
6120    ins_encode %{
6121      __ movl($dst$$Register, $mem$$Address);
6122    %}
6123    ins_pipe(ialu_reg_mem); // XXX
6124 %}
6125 
6126 
6127 // Load Klass Pointer
6128 instruct loadKlass(rRegP dst, memory mem)
6129 %{
6130   match(Set dst (LoadKlass mem));
6131 
6132   ins_cost(125); // XXX
6133   format %{ "movq    $dst, $mem\t# class" %}
6134   opcode(0x8B);
6135   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6136   ins_pipe(ialu_reg_mem); // XXX
6137 %}
6138 
6139 // Load narrow Klass Pointer
6140 instruct loadNKlass(rRegN dst, memory mem)
6141 %{
6142   match(Set dst (LoadNKlass mem));
6143 
6144   ins_cost(125); // XXX
6145   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
6146   ins_encode %{
6147     __ movl($dst$$Register, $mem$$Address);
6148   %}
6149   ins_pipe(ialu_reg_mem); // XXX
6150 %}
6151 
6152 // Load Float
6153 instruct loadF(regF dst, memory mem)
6154 %{
6155   match(Set dst (LoadF mem));
6156 
6157   ins_cost(145); // XXX
6158   format %{ "movss   $dst, $mem\t# float" %}
6159   opcode(0xF3, 0x0F, 0x10);
6160   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
6161   ins_pipe(pipe_slow); // XXX
6162 %}
6163 
6164 // Load Double
6165 instruct loadD_partial(regD dst, memory mem)
6166 %{
6167   predicate(!UseXmmLoadAndClearUpper);
6168   match(Set dst (LoadD mem));
6169 
6170   ins_cost(145); // XXX
6171   format %{ "movlpd  $dst, $mem\t# double" %}
6172   opcode(0x66, 0x0F, 0x12);
6173   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
6174   ins_pipe(pipe_slow); // XXX
6175 %}
6176 
6177 instruct loadD(regD dst, memory mem)
6178 %{
6179   predicate(UseXmmLoadAndClearUpper);
6180   match(Set dst (LoadD mem));
6181 
6182   ins_cost(145); // XXX
6183   format %{ "movsd   $dst, $mem\t# double" %}
6184   opcode(0xF2, 0x0F, 0x10);
6185   ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
6186   ins_pipe(pipe_slow); // XXX
6187 %}
6188 
6189 // Load Aligned Packed Byte to XMM register
6190 instruct loadA8B(regD dst, memory mem) %{
6191   match(Set dst (Load8B mem));
6192   ins_cost(125);
6193   format %{ "MOVQ  $dst,$mem\t! packed8B" %}
6194   ins_encode( movq_ld(dst, mem));
6195   ins_pipe( pipe_slow );
6196 %}
6197 
6198 // Load Aligned Packed Short to XMM register
6199 instruct loadA4S(regD dst, memory mem) %{
6200   match(Set dst (Load4S mem));
6201   ins_cost(125);
6202   format %{ "MOVQ  $dst,$mem\t! packed4S" %}
6203   ins_encode( movq_ld(dst, mem));
6204   ins_pipe( pipe_slow );
6205 %}
6206 
6207 // Load Aligned Packed Char to XMM register
6208 instruct loadA4C(regD dst, memory mem) %{
6209   match(Set dst (Load4C mem));
6210   ins_cost(125);
6211   format %{ "MOVQ  $dst,$mem\t! packed4C" %}
6212   ins_encode( movq_ld(dst, mem));
6213   ins_pipe( pipe_slow );
6214 %}
6215 
6216 // Load Aligned Packed Integer to XMM register
6217 instruct load2IU(regD dst, memory mem) %{
6218   match(Set dst (Load2I mem));
6219   ins_cost(125);
6220   format %{ "MOVQ  $dst,$mem\t! packed2I" %}
6221   ins_encode( movq_ld(dst, mem));
6222   ins_pipe( pipe_slow );
6223 %}
6224 
6225 // Load Aligned Packed Single to XMM
6226 instruct loadA2F(regD dst, memory mem) %{
6227   match(Set dst (Load2F mem));
6228   ins_cost(145);
6229   format %{ "MOVQ  $dst,$mem\t! packed2F" %}
6230   ins_encode( movq_ld(dst, mem));
6231   ins_pipe( pipe_slow );
6232 %}
6233 
6234 // Load Effective Address
6235 instruct leaP8(rRegP dst, indOffset8 mem)
6236 %{
6237   match(Set dst mem);
6238 
6239   ins_cost(110); // XXX
6240   format %{ "leaq    $dst, $mem\t# ptr 8" %}
6241   opcode(0x8D);
6242   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6243   ins_pipe(ialu_reg_reg_fat);
6244 %}
6245 
6246 instruct leaP32(rRegP dst, indOffset32 mem)
6247 %{
6248   match(Set dst mem);
6249 
6250   ins_cost(110);
6251   format %{ "leaq    $dst, $mem\t# ptr 32" %}
6252   opcode(0x8D);
6253   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6254   ins_pipe(ialu_reg_reg_fat);
6255 %}
6256 
6257 // instruct leaPIdx(rRegP dst, indIndex mem)
6258 // %{
6259 //   match(Set dst mem);
6260 
6261 //   ins_cost(110);
6262 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
6263 //   opcode(0x8D);
6264 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6265 //   ins_pipe(ialu_reg_reg_fat);
6266 // %}
6267 
6268 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
6269 %{
6270   match(Set dst mem);
6271 
6272   ins_cost(110);
6273   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
6274   opcode(0x8D);
6275   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6276   ins_pipe(ialu_reg_reg_fat);
6277 %}
6278 
6279 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
6280 %{
6281   match(Set dst mem);
6282 
6283   ins_cost(110);
6284   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
6285   opcode(0x8D);
6286   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6287   ins_pipe(ialu_reg_reg_fat);
6288 %}
6289 
6290 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
6291 %{
6292   match(Set dst mem);
6293 
6294   ins_cost(110);
6295   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
6296   opcode(0x8D);
6297   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6298   ins_pipe(ialu_reg_reg_fat);
6299 %}
6300 
6301 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
6302 %{
6303   match(Set dst mem);
6304 
6305   ins_cost(110);
6306   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
6307   opcode(0x8D);
6308   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6309   ins_pipe(ialu_reg_reg_fat);
6310 %}
6311 
6312 // Load Effective Address which uses Narrow (32-bits) oop
6313 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
6314 %{
6315   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
6316   match(Set dst mem);
6317 
6318   ins_cost(110);
6319   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
6320   opcode(0x8D);
6321   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6322   ins_pipe(ialu_reg_reg_fat);
6323 %}
6324 
6325 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
6326 %{
6327   predicate(Universe::narrow_oop_shift() == 0);
6328   match(Set dst mem);
6329 
6330   ins_cost(110); // XXX
6331   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
6332   opcode(0x8D);
6333   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6334   ins_pipe(ialu_reg_reg_fat);
6335 %}
6336 
6337 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
6338 %{
6339   predicate(Universe::narrow_oop_shift() == 0);
6340   match(Set dst mem);
6341 
6342   ins_cost(110);
6343   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
6344   opcode(0x8D);
6345   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6346   ins_pipe(ialu_reg_reg_fat);
6347 %}
6348 
6349 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
6350 %{
6351   predicate(Universe::narrow_oop_shift() == 0);
6352   match(Set dst mem);
6353 
6354   ins_cost(110);
6355   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
6356   opcode(0x8D);
6357   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6358   ins_pipe(ialu_reg_reg_fat);
6359 %}
6360 
6361 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
6362 %{
6363   predicate(Universe::narrow_oop_shift() == 0);
6364   match(Set dst mem);
6365 
6366   ins_cost(110);
6367   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
6368   opcode(0x8D);
6369   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6370   ins_pipe(ialu_reg_reg_fat);
6371 %}
6372 
6373 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
6374 %{
6375   predicate(Universe::narrow_oop_shift() == 0);
6376   match(Set dst mem);
6377 
6378   ins_cost(110);
6379   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
6380   opcode(0x8D);
6381   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6382   ins_pipe(ialu_reg_reg_fat);
6383 %}
6384 
6385 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
6386 %{
6387   predicate(Universe::narrow_oop_shift() == 0);
6388   match(Set dst mem);
6389 
6390   ins_cost(110);
6391   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
6392   opcode(0x8D);
6393   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
6394   ins_pipe(ialu_reg_reg_fat);
6395 %}
6396 
6397 instruct loadConI(rRegI dst, immI src)
6398 %{
6399   match(Set dst src);
6400 
6401   format %{ "movl    $dst, $src\t# int" %}
6402   ins_encode(load_immI(dst, src));
6403   ins_pipe(ialu_reg_fat); // XXX
6404 %}
6405 
6406 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
6407 %{
6408   match(Set dst src);
6409   effect(KILL cr);
6410 
6411   ins_cost(50);
6412   format %{ "xorl    $dst, $dst\t# int" %}
6413   opcode(0x33); /* + rd */
6414   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
6415   ins_pipe(ialu_reg);
6416 %}
6417 
6418 instruct loadConL(rRegL dst, immL src)
6419 %{
6420   match(Set dst src);
6421 
6422   ins_cost(150);
6423   format %{ "movq    $dst, $src\t# long" %}
6424   ins_encode(load_immL(dst, src));
6425   ins_pipe(ialu_reg);
6426 %}
6427 
6428 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
6429 %{
6430   match(Set dst src);
6431   effect(KILL cr);
6432 
6433   ins_cost(50);
6434   format %{ "xorl    $dst, $dst\t# long" %}
6435   opcode(0x33); /* + rd */
6436   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
6437   ins_pipe(ialu_reg); // XXX
6438 %}
6439 
6440 instruct loadConUL32(rRegL dst, immUL32 src)
6441 %{
6442   match(Set dst src);
6443 
6444   ins_cost(60);
6445   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
6446   ins_encode(load_immUL32(dst, src));
6447   ins_pipe(ialu_reg);
6448 %}
6449 
6450 instruct loadConL32(rRegL dst, immL32 src)
6451 %{
6452   match(Set dst src);
6453 
6454   ins_cost(70);
6455   format %{ "movq    $dst, $src\t# long (32-bit)" %}
6456   ins_encode(load_immL32(dst, src));
6457   ins_pipe(ialu_reg);
6458 %}
6459 
6460 instruct loadConP(rRegP dst, immP con) %{
6461   match(Set dst con);
6462 
6463   format %{ "movq    $dst, $con\t# ptr" %}
6464   ins_encode(load_immP(dst, con));
6465   ins_pipe(ialu_reg_fat); // XXX
6466 %}
6467 
6468 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
6469 %{
6470   match(Set dst src);
6471   effect(KILL cr);
6472 
6473   ins_cost(50);
6474   format %{ "xorl    $dst, $dst\t# ptr" %}
6475   opcode(0x33); /* + rd */
6476   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
6477   ins_pipe(ialu_reg);
6478 %}
6479 
6480 instruct loadConP_poll(rRegP dst, immP_poll src) %{
6481   match(Set dst src);
6482   format %{ "movq    $dst, $src\t!ptr" %}
6483   ins_encode %{
6484     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
6485     __ lea($dst$$Register, polling_page);
6486   %}
6487   ins_pipe(ialu_reg_fat);
6488 %}
6489 
6490 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
6491 %{
6492   match(Set dst src);
6493   effect(KILL cr);
6494 
6495   ins_cost(60);
6496   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
6497   ins_encode(load_immP31(dst, src));
6498   ins_pipe(ialu_reg);
6499 %}
6500 
6501 instruct loadConF(regF dst, immF con) %{
6502   match(Set dst con);
6503   ins_cost(125);
6504   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
6505   ins_encode %{
6506     __ movflt($dst$$XMMRegister, $constantaddress($con));
6507   %}
6508   ins_pipe(pipe_slow);
6509 %}
6510 
6511 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
6512   match(Set dst src);
6513   effect(KILL cr);
6514   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
6515   ins_encode %{
6516     __ xorq($dst$$Register, $dst$$Register);
6517   %}
6518   ins_pipe(ialu_reg);
6519 %}
6520 
6521 instruct loadConN(rRegN dst, immN src) %{
6522   match(Set dst src);
6523 
6524   ins_cost(125);
6525   format %{ "movl    $dst, $src\t# compressed ptr" %}
6526   ins_encode %{
6527     address con = (address)$src$$constant;
6528     if (con == NULL) {
6529       ShouldNotReachHere();
6530     } else {
6531       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
6532     }
6533   %}
6534   ins_pipe(ialu_reg_fat); // XXX
6535 %}
6536 
6537 instruct loadConF0(regF dst, immF0 src)
6538 %{
6539   match(Set dst src);
6540   ins_cost(100);
6541 
6542   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6543   opcode(0x0F, 0x57);
6544   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
6545   ins_pipe(pipe_slow);
6546 %}
6547 
6548 // Use the same format since predicate() can not be used here.
6549 instruct loadConD(regD dst, immD con) %{
6550   match(Set dst con);
6551   ins_cost(125);
6552   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6553   ins_encode %{
6554     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6555   %}
6556   ins_pipe(pipe_slow);
6557 %}
6558 
6559 instruct loadConD0(regD dst, immD0 src)
6560 %{
6561   match(Set dst src);
6562   ins_cost(100);
6563 
6564   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6565   opcode(0x66, 0x0F, 0x57);
6566   ins_encode(OpcP, REX_reg_reg(dst, dst), OpcS, OpcT, reg_reg(dst, dst));
6567   ins_pipe(pipe_slow);
6568 %}
6569 
6570 instruct loadSSI(rRegI dst, stackSlotI src)
6571 %{
6572   match(Set dst src);
6573 
6574   ins_cost(125);
6575   format %{ "movl    $dst, $src\t# int stk" %}
6576   opcode(0x8B);
6577   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6578   ins_pipe(ialu_reg_mem);
6579 %}
6580 
6581 instruct loadSSL(rRegL dst, stackSlotL src)
6582 %{
6583   match(Set dst src);
6584 
6585   ins_cost(125);
6586   format %{ "movq    $dst, $src\t# long stk" %}
6587   opcode(0x8B);
6588   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6589   ins_pipe(ialu_reg_mem);
6590 %}
6591 
6592 instruct loadSSP(rRegP dst, stackSlotP src)
6593 %{
6594   match(Set dst src);
6595 
6596   ins_cost(125);
6597   format %{ "movq    $dst, $src\t# ptr stk" %}
6598   opcode(0x8B);
6599   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6600   ins_pipe(ialu_reg_mem);
6601 %}
6602 
6603 instruct loadSSF(regF dst, stackSlotF src)
6604 %{
6605   match(Set dst src);
6606 
6607   ins_cost(125);
6608   format %{ "movss   $dst, $src\t# float stk" %}
6609   opcode(0xF3, 0x0F, 0x10);
6610   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
6611   ins_pipe(pipe_slow); // XXX
6612 %}
6613 
6614 // Use the same format since predicate() can not be used here.
6615 instruct loadSSD(regD dst, stackSlotD src)
6616 %{
6617   match(Set dst src);
6618 
6619   ins_cost(125);
6620   format %{ "movsd   $dst, $src\t# double stk" %}
6621   ins_encode  %{
6622     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6623   %}
6624   ins_pipe(pipe_slow); // XXX
6625 %}
6626 
6627 // Prefetch instructions.
6628 // Must be safe to execute with invalid address (cannot fault).
6629 
6630 instruct prefetchr( memory mem ) %{
6631   predicate(ReadPrefetchInstr==3);
6632   match(PrefetchRead mem);
6633   ins_cost(125);
6634 
6635   format %{ "PREFETCHR $mem\t# Prefetch into level 1 cache" %}
6636   ins_encode %{
6637     __ prefetchr($mem$$Address);
6638   %}
6639   ins_pipe(ialu_mem);
6640 %}
6641 
6642 instruct prefetchrNTA( memory mem ) %{
6643   predicate(ReadPrefetchInstr==0);
6644   match(PrefetchRead mem);
6645   ins_cost(125);
6646 
6647   format %{ "PREFETCHNTA $mem\t# Prefetch into non-temporal cache for read" %}
6648   ins_encode %{
6649     __ prefetchnta($mem$$Address);
6650   %}
6651   ins_pipe(ialu_mem);
6652 %}
6653 
6654 instruct prefetchrT0( memory mem ) %{
6655   predicate(ReadPrefetchInstr==1);
6656   match(PrefetchRead mem);
6657   ins_cost(125);
6658 
6659   format %{ "PREFETCHT0 $mem\t# prefetch into L1 and L2 caches for read" %}
6660   ins_encode %{
6661     __ prefetcht0($mem$$Address);
6662   %}
6663   ins_pipe(ialu_mem);
6664 %}
6665 
6666 instruct prefetchrT2( memory mem ) %{
6667   predicate(ReadPrefetchInstr==2);
6668   match(PrefetchRead mem);
6669   ins_cost(125);
6670 
6671   format %{ "PREFETCHT2 $mem\t# prefetch into L2 caches for read" %}
6672   ins_encode %{
6673     __ prefetcht2($mem$$Address);
6674   %}
6675   ins_pipe(ialu_mem);
6676 %}
6677 
6678 instruct prefetchwNTA( memory mem ) %{
6679   match(PrefetchWrite mem);
6680   ins_cost(125);
6681 
6682   format %{ "PREFETCHNTA $mem\t# Prefetch to non-temporal cache for write" %}
6683   ins_encode %{
6684     __ prefetchnta($mem$$Address);
6685   %}
6686   ins_pipe(ialu_mem);
6687 %}
6688 
6689 // Prefetch instructions for allocation.
6690 
6691 instruct prefetchAlloc( memory mem ) %{
6692   predicate(AllocatePrefetchInstr==3);
6693   match(PrefetchAllocation mem);
6694   ins_cost(125);
6695 
6696   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6697   ins_encode %{
6698     __ prefetchw($mem$$Address);
6699   %}
6700   ins_pipe(ialu_mem);
6701 %}
6702 
6703 instruct prefetchAllocNTA( memory mem ) %{
6704   predicate(AllocatePrefetchInstr==0);
6705   match(PrefetchAllocation mem);
6706   ins_cost(125);
6707 
6708   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6709   ins_encode %{
6710     __ prefetchnta($mem$$Address);
6711   %}
6712   ins_pipe(ialu_mem);
6713 %}
6714 
6715 instruct prefetchAllocT0( memory mem ) %{
6716   predicate(AllocatePrefetchInstr==1);
6717   match(PrefetchAllocation mem);
6718   ins_cost(125);
6719 
6720   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6721   ins_encode %{
6722     __ prefetcht0($mem$$Address);
6723   %}
6724   ins_pipe(ialu_mem);
6725 %}
6726 
6727 instruct prefetchAllocT2( memory mem ) %{
6728   predicate(AllocatePrefetchInstr==2);
6729   match(PrefetchAllocation mem);
6730   ins_cost(125);
6731 
6732   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6733   ins_encode %{
6734     __ prefetcht2($mem$$Address);
6735   %}
6736   ins_pipe(ialu_mem);
6737 %}
6738 
6739 //----------Store Instructions-------------------------------------------------
6740 
6741 // Store Byte
6742 instruct storeB(memory mem, rRegI src)
6743 %{
6744   match(Set mem (StoreB mem src));
6745 
6746   ins_cost(125); // XXX
6747   format %{ "movb    $mem, $src\t# byte" %}
6748   opcode(0x88);
6749   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6750   ins_pipe(ialu_mem_reg);
6751 %}
6752 
6753 // Store Char/Short
6754 instruct storeC(memory mem, rRegI src)
6755 %{
6756   match(Set mem (StoreC mem src));
6757 
6758   ins_cost(125); // XXX
6759   format %{ "movw    $mem, $src\t# char/short" %}
6760   opcode(0x89);
6761   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6762   ins_pipe(ialu_mem_reg);
6763 %}
6764 
6765 // Store Integer
6766 instruct storeI(memory mem, rRegI src)
6767 %{
6768   match(Set mem (StoreI mem src));
6769 
6770   ins_cost(125); // XXX
6771   format %{ "movl    $mem, $src\t# int" %}
6772   opcode(0x89);
6773   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6774   ins_pipe(ialu_mem_reg);
6775 %}
6776 
6777 // Store Long
6778 instruct storeL(memory mem, rRegL src)
6779 %{
6780   match(Set mem (StoreL mem src));
6781 
6782   ins_cost(125); // XXX
6783   format %{ "movq    $mem, $src\t# long" %}
6784   opcode(0x89);
6785   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6786   ins_pipe(ialu_mem_reg); // XXX
6787 %}
6788 
6789 // Store Pointer
6790 instruct storeP(memory mem, any_RegP src)
6791 %{
6792   match(Set mem (StoreP mem src));
6793 
6794   ins_cost(125); // XXX
6795   format %{ "movq    $mem, $src\t# ptr" %}
6796   opcode(0x89);
6797   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6798   ins_pipe(ialu_mem_reg);
6799 %}
6800 
6801 instruct storeImmP0(memory mem, immP0 zero)
6802 %{
6803   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
6804   match(Set mem (StoreP mem zero));
6805 
6806   ins_cost(125); // XXX
6807   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6808   ins_encode %{
6809     __ movq($mem$$Address, r12);
6810   %}
6811   ins_pipe(ialu_mem_reg);
6812 %}
6813 
6814 // Store NULL Pointer, mark word, or other simple pointer constant.
6815 instruct storeImmP(memory mem, immP31 src)
6816 %{
6817   match(Set mem (StoreP mem src));
6818 
6819   ins_cost(150); // XXX
6820   format %{ "movq    $mem, $src\t# ptr" %}
6821   opcode(0xC7); /* C7 /0 */
6822   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6823   ins_pipe(ialu_mem_imm);
6824 %}
6825 
6826 // Store Compressed Pointer
6827 instruct storeN(memory mem, rRegN src)
6828 %{
6829   match(Set mem (StoreN mem src));
6830 
6831   ins_cost(125); // XXX
6832   format %{ "movl    $mem, $src\t# compressed ptr" %}
6833   ins_encode %{
6834     __ movl($mem$$Address, $src$$Register);
6835   %}
6836   ins_pipe(ialu_mem_reg);
6837 %}
6838 
6839 instruct storeImmN0(memory mem, immN0 zero)
6840 %{
6841   predicate(Universe::narrow_oop_base() == NULL);
6842   match(Set mem (StoreN mem zero));
6843 
6844   ins_cost(125); // XXX
6845   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6846   ins_encode %{
6847     __ movl($mem$$Address, r12);
6848   %}
6849   ins_pipe(ialu_mem_reg);
6850 %}
6851 
6852 instruct storeImmN(memory mem, immN src)
6853 %{
6854   match(Set mem (StoreN mem src));
6855 
6856   ins_cost(150); // XXX
6857   format %{ "movl    $mem, $src\t# compressed ptr" %}
6858   ins_encode %{
6859     address con = (address)$src$$constant;
6860     if (con == NULL) {
6861       __ movl($mem$$Address, (int32_t)0);
6862     } else {
6863       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6864     }
6865   %}
6866   ins_pipe(ialu_mem_imm);
6867 %}
6868 
6869 // Store Integer Immediate
6870 instruct storeImmI0(memory mem, immI0 zero)
6871 %{
6872   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
6873   match(Set mem (StoreI mem zero));
6874 
6875   ins_cost(125); // XXX
6876   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6877   ins_encode %{
6878     __ movl($mem$$Address, r12);
6879   %}
6880   ins_pipe(ialu_mem_reg);
6881 %}
6882 
6883 instruct storeImmI(memory mem, immI src)
6884 %{
6885   match(Set mem (StoreI mem src));
6886 
6887   ins_cost(150);
6888   format %{ "movl    $mem, $src\t# int" %}
6889   opcode(0xC7); /* C7 /0 */
6890   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6891   ins_pipe(ialu_mem_imm);
6892 %}
6893 
6894 // Store Long Immediate
6895 instruct storeImmL0(memory mem, immL0 zero)
6896 %{
6897   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
6898   match(Set mem (StoreL mem zero));
6899 
6900   ins_cost(125); // XXX
6901   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6902   ins_encode %{
6903     __ movq($mem$$Address, r12);
6904   %}
6905   ins_pipe(ialu_mem_reg);
6906 %}
6907 
6908 instruct storeImmL(memory mem, immL32 src)
6909 %{
6910   match(Set mem (StoreL mem src));
6911 
6912   ins_cost(150);
6913   format %{ "movq    $mem, $src\t# long" %}
6914   opcode(0xC7); /* C7 /0 */
6915   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6916   ins_pipe(ialu_mem_imm);
6917 %}
6918 
6919 // Store Short/Char Immediate
6920 instruct storeImmC0(memory mem, immI0 zero)
6921 %{
6922   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
6923   match(Set mem (StoreC mem zero));
6924 
6925   ins_cost(125); // XXX
6926   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6927   ins_encode %{
6928     __ movw($mem$$Address, r12);
6929   %}
6930   ins_pipe(ialu_mem_reg);
6931 %}
6932 
6933 instruct storeImmI16(memory mem, immI16 src)
6934 %{
6935   predicate(UseStoreImmI16);
6936   match(Set mem (StoreC mem src));
6937 
6938   ins_cost(150);
6939   format %{ "movw    $mem, $src\t# short/char" %}
6940   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6941   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6942   ins_pipe(ialu_mem_imm);
6943 %}
6944 
6945 // Store Byte Immediate
6946 instruct storeImmB0(memory mem, immI0 zero)
6947 %{
6948   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
6949   match(Set mem (StoreB mem zero));
6950 
6951   ins_cost(125); // XXX
6952   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6953   ins_encode %{
6954     __ movb($mem$$Address, r12);
6955   %}
6956   ins_pipe(ialu_mem_reg);
6957 %}
6958 
6959 instruct storeImmB(memory mem, immI8 src)
6960 %{
6961   match(Set mem (StoreB mem src));
6962 
6963   ins_cost(150); // XXX
6964   format %{ "movb    $mem, $src\t# byte" %}
6965   opcode(0xC6); /* C6 /0 */
6966   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6967   ins_pipe(ialu_mem_imm);
6968 %}
6969 
6970 // Store Aligned Packed Byte XMM register to memory
6971 instruct storeA8B(memory mem, regD src) %{
6972   match(Set mem (Store8B mem src));
6973   ins_cost(145);
6974   format %{ "MOVQ  $mem,$src\t! packed8B" %}
6975   ins_encode( movq_st(mem, src));
6976   ins_pipe( pipe_slow );
6977 %}
6978 
6979 // Store Aligned Packed Char/Short XMM register to memory
6980 instruct storeA4C(memory mem, regD src) %{
6981   match(Set mem (Store4C mem src));
6982   ins_cost(145);
6983   format %{ "MOVQ  $mem,$src\t! packed4C" %}
6984   ins_encode( movq_st(mem, src));
6985   ins_pipe( pipe_slow );
6986 %}
6987 
6988 // Store Aligned Packed Integer XMM register to memory
6989 instruct storeA2I(memory mem, regD src) %{
6990   match(Set mem (Store2I mem src));
6991   ins_cost(145);
6992   format %{ "MOVQ  $mem,$src\t! packed2I" %}
6993   ins_encode( movq_st(mem, src));
6994   ins_pipe( pipe_slow );
6995 %}
6996 
6997 // Store CMS card-mark Immediate
6998 instruct storeImmCM0_reg(memory mem, immI0 zero)
6999 %{
7000   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
7001   match(Set mem (StoreCM mem zero));
7002 
7003   ins_cost(125); // XXX
7004   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
7005   ins_encode %{
7006     __ movb($mem$$Address, r12);
7007   %}
7008   ins_pipe(ialu_mem_reg);
7009 %}
7010 
7011 instruct storeImmCM0(memory mem, immI0 src)
7012 %{
7013   match(Set mem (StoreCM mem src));
7014 
7015   ins_cost(150); // XXX
7016   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
7017   opcode(0xC6); /* C6 /0 */
7018   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
7019   ins_pipe(ialu_mem_imm);
7020 %}
7021 
7022 // Store Aligned Packed Single Float XMM register to memory
7023 instruct storeA2F(memory mem, regD src) %{
7024   match(Set mem (Store2F mem src));
7025   ins_cost(145);
7026   format %{ "MOVQ  $mem,$src\t! packed2F" %}
7027   ins_encode( movq_st(mem, src));
7028   ins_pipe( pipe_slow );
7029 %}
7030 
7031 // Store Float
7032 instruct storeF(memory mem, regF src)
7033 %{
7034   match(Set mem (StoreF mem src));
7035 
7036   ins_cost(95); // XXX
7037   format %{ "movss   $mem, $src\t# float" %}
7038   opcode(0xF3, 0x0F, 0x11);
7039   ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
7040   ins_pipe(pipe_slow); // XXX
7041 %}
7042 
7043 // Store immediate Float value (it is faster than store from XMM register)
7044 instruct storeF0(memory mem, immF0 zero)
7045 %{
7046   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
7047   match(Set mem (StoreF mem zero));
7048 
7049   ins_cost(25); // XXX
7050   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
7051   ins_encode %{
7052     __ movl($mem$$Address, r12);
7053   %}
7054   ins_pipe(ialu_mem_reg);
7055 %}
7056 
7057 instruct storeF_imm(memory mem, immF src)
7058 %{
7059   match(Set mem (StoreF mem src));
7060 
7061   ins_cost(50);
7062   format %{ "movl    $mem, $src\t# float" %}
7063   opcode(0xC7); /* C7 /0 */
7064   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
7065   ins_pipe(ialu_mem_imm);
7066 %}
7067 
7068 // Store Double
7069 instruct storeD(memory mem, regD src)
7070 %{
7071   match(Set mem (StoreD mem src));
7072 
7073   ins_cost(95); // XXX
7074   format %{ "movsd   $mem, $src\t# double" %}
7075   opcode(0xF2, 0x0F, 0x11);
7076   ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
7077   ins_pipe(pipe_slow); // XXX
7078 %}
7079 
7080 // Store immediate double 0.0 (it is faster than store from XMM register)
7081 instruct storeD0_imm(memory mem, immD0 src)
7082 %{
7083   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
7084   match(Set mem (StoreD mem src));
7085 
7086   ins_cost(50);
7087   format %{ "movq    $mem, $src\t# double 0." %}
7088   opcode(0xC7); /* C7 /0 */
7089   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
7090   ins_pipe(ialu_mem_imm);
7091 %}
7092 
7093 instruct storeD0(memory mem, immD0 zero)
7094 %{
7095   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
7096   match(Set mem (StoreD mem zero));
7097 
7098   ins_cost(25); // XXX
7099   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
7100   ins_encode %{
7101     __ movq($mem$$Address, r12);
7102   %}
7103   ins_pipe(ialu_mem_reg);
7104 %}
7105 
7106 instruct storeSSI(stackSlotI dst, rRegI src)
7107 %{
7108   match(Set dst src);
7109 
7110   ins_cost(100);
7111   format %{ "movl    $dst, $src\t# int stk" %}
7112   opcode(0x89);
7113   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7114   ins_pipe( ialu_mem_reg );
7115 %}
7116 
7117 instruct storeSSL(stackSlotL dst, rRegL src)
7118 %{
7119   match(Set dst src);
7120 
7121   ins_cost(100);
7122   format %{ "movq    $dst, $src\t# long stk" %}
7123   opcode(0x89);
7124   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7125   ins_pipe(ialu_mem_reg);
7126 %}
7127 
7128 instruct storeSSP(stackSlotP dst, rRegP src)
7129 %{
7130   match(Set dst src);
7131 
7132   ins_cost(100);
7133   format %{ "movq    $dst, $src\t# ptr stk" %}
7134   opcode(0x89);
7135   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7136   ins_pipe(ialu_mem_reg);
7137 %}
7138 
7139 instruct storeSSF(stackSlotF dst, regF src)
7140 %{
7141   match(Set dst src);
7142 
7143   ins_cost(95); // XXX
7144   format %{ "movss   $dst, $src\t# float stk" %}
7145   opcode(0xF3, 0x0F, 0x11);
7146   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
7147   ins_pipe(pipe_slow); // XXX
7148 %}
7149 
7150 instruct storeSSD(stackSlotD dst, regD src)
7151 %{
7152   match(Set dst src);
7153 
7154   ins_cost(95); // XXX
7155   format %{ "movsd   $dst, $src\t# double stk" %}
7156   opcode(0xF2, 0x0F, 0x11);
7157   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
7158   ins_pipe(pipe_slow); // XXX
7159 %}
7160 
7161 //----------BSWAP Instructions-------------------------------------------------
7162 instruct bytes_reverse_int(rRegI dst) %{
7163   match(Set dst (ReverseBytesI dst));
7164 
7165   format %{ "bswapl  $dst" %}
7166   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
7167   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
7168   ins_pipe( ialu_reg );
7169 %}
7170 
7171 instruct bytes_reverse_long(rRegL dst) %{
7172   match(Set dst (ReverseBytesL dst));
7173 
7174   format %{ "bswapq  $dst" %}
7175 
7176   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
7177   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
7178   ins_pipe( ialu_reg);
7179 %}
7180 
7181 instruct bytes_reverse_unsigned_short(rRegI dst) %{
7182   match(Set dst (ReverseBytesUS dst));
7183 
7184   format %{ "bswapl  $dst\n\t"
7185             "shrl    $dst,16\n\t" %}
7186   ins_encode %{
7187     __ bswapl($dst$$Register);
7188     __ shrl($dst$$Register, 16);
7189   %}
7190   ins_pipe( ialu_reg );
7191 %}
7192 
7193 instruct bytes_reverse_short(rRegI dst) %{
7194   match(Set dst (ReverseBytesS dst));
7195 
7196   format %{ "bswapl  $dst\n\t"
7197             "sar     $dst,16\n\t" %}
7198   ins_encode %{
7199     __ bswapl($dst$$Register);
7200     __ sarl($dst$$Register, 16);
7201   %}
7202   ins_pipe( ialu_reg );
7203 %}
7204 
7205 //---------- Zeros Count Instructions ------------------------------------------
7206 
7207 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
7208   predicate(UseCountLeadingZerosInstruction);
7209   match(Set dst (CountLeadingZerosI src));
7210   effect(KILL cr);
7211 
7212   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
7213   ins_encode %{
7214     __ lzcntl($dst$$Register, $src$$Register);
7215   %}
7216   ins_pipe(ialu_reg);
7217 %}
7218 
7219 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
7220   predicate(!UseCountLeadingZerosInstruction);
7221   match(Set dst (CountLeadingZerosI src));
7222   effect(KILL cr);
7223 
7224   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
7225             "jnz     skip\n\t"
7226             "movl    $dst, -1\n"
7227       "skip:\n\t"
7228             "negl    $dst\n\t"
7229             "addl    $dst, 31" %}
7230   ins_encode %{
7231     Register Rdst = $dst$$Register;
7232     Register Rsrc = $src$$Register;
7233     Label skip;
7234     __ bsrl(Rdst, Rsrc);
7235     __ jccb(Assembler::notZero, skip);
7236     __ movl(Rdst, -1);
7237     __ bind(skip);
7238     __ negl(Rdst);
7239     __ addl(Rdst, BitsPerInt - 1);
7240   %}
7241   ins_pipe(ialu_reg);
7242 %}
7243 
7244 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
7245   predicate(UseCountLeadingZerosInstruction);
7246   match(Set dst (CountLeadingZerosL src));
7247   effect(KILL cr);
7248 
7249   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
7250   ins_encode %{
7251     __ lzcntq($dst$$Register, $src$$Register);
7252   %}
7253   ins_pipe(ialu_reg);
7254 %}
7255 
7256 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
7257   predicate(!UseCountLeadingZerosInstruction);
7258   match(Set dst (CountLeadingZerosL src));
7259   effect(KILL cr);
7260 
7261   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
7262             "jnz     skip\n\t"
7263             "movl    $dst, -1\n"
7264       "skip:\n\t"
7265             "negl    $dst\n\t"
7266             "addl    $dst, 63" %}
7267   ins_encode %{
7268     Register Rdst = $dst$$Register;
7269     Register Rsrc = $src$$Register;
7270     Label skip;
7271     __ bsrq(Rdst, Rsrc);
7272     __ jccb(Assembler::notZero, skip);
7273     __ movl(Rdst, -1);
7274     __ bind(skip);
7275     __ negl(Rdst);
7276     __ addl(Rdst, BitsPerLong - 1);
7277   %}
7278   ins_pipe(ialu_reg);
7279 %}
7280 
7281 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
7282   match(Set dst (CountTrailingZerosI src));
7283   effect(KILL cr);
7284 
7285   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
7286             "jnz     done\n\t"
7287             "movl    $dst, 32\n"
7288       "done:" %}
7289   ins_encode %{
7290     Register Rdst = $dst$$Register;
7291     Label done;
7292     __ bsfl(Rdst, $src$$Register);
7293     __ jccb(Assembler::notZero, done);
7294     __ movl(Rdst, BitsPerInt);
7295     __ bind(done);
7296   %}
7297   ins_pipe(ialu_reg);
7298 %}
7299 
7300 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
7301   match(Set dst (CountTrailingZerosL src));
7302   effect(KILL cr);
7303 
7304   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
7305             "jnz     done\n\t"
7306             "movl    $dst, 64\n"
7307       "done:" %}
7308   ins_encode %{
7309     Register Rdst = $dst$$Register;
7310     Label done;
7311     __ bsfq(Rdst, $src$$Register);
7312     __ jccb(Assembler::notZero, done);
7313     __ movl(Rdst, BitsPerLong);
7314     __ bind(done);
7315   %}
7316   ins_pipe(ialu_reg);
7317 %}
7318 
7319 
7320 //---------- Population Count Instructions -------------------------------------
7321 
7322 instruct popCountI(rRegI dst, rRegI src) %{
7323   predicate(UsePopCountInstruction);
7324   match(Set dst (PopCountI src));
7325 
7326   format %{ "popcnt  $dst, $src" %}
7327   ins_encode %{
7328     __ popcntl($dst$$Register, $src$$Register);
7329   %}
7330   ins_pipe(ialu_reg);
7331 %}
7332 
7333 instruct popCountI_mem(rRegI dst, memory mem) %{
7334   predicate(UsePopCountInstruction);
7335   match(Set dst (PopCountI (LoadI mem)));
7336 
7337   format %{ "popcnt  $dst, $mem" %}
7338   ins_encode %{
7339     __ popcntl($dst$$Register, $mem$$Address);
7340   %}
7341   ins_pipe(ialu_reg);
7342 %}
7343 
7344 // Note: Long.bitCount(long) returns an int.
7345 instruct popCountL(rRegI dst, rRegL src) %{
7346   predicate(UsePopCountInstruction);
7347   match(Set dst (PopCountL src));
7348 
7349   format %{ "popcnt  $dst, $src" %}
7350   ins_encode %{
7351     __ popcntq($dst$$Register, $src$$Register);
7352   %}
7353   ins_pipe(ialu_reg);
7354 %}
7355 
7356 // Note: Long.bitCount(long) returns an int.
7357 instruct popCountL_mem(rRegI dst, memory mem) %{
7358   predicate(UsePopCountInstruction);
7359   match(Set dst (PopCountL (LoadL mem)));
7360 
7361   format %{ "popcnt  $dst, $mem" %}
7362   ins_encode %{
7363     __ popcntq($dst$$Register, $mem$$Address);
7364   %}
7365   ins_pipe(ialu_reg);
7366 %}
7367 
7368 
7369 //----------MemBar Instructions-----------------------------------------------
7370 // Memory barrier flavors
7371 
7372 instruct membar_acquire()
7373 %{
7374   match(MemBarAcquire);
7375   ins_cost(0);
7376 
7377   size(0);
7378   format %{ "MEMBAR-acquire ! (empty encoding)" %}
7379   ins_encode();
7380   ins_pipe(empty);
7381 %}
7382 
7383 instruct membar_acquire_lock()
7384 %{
7385   match(MemBarAcquireLock);
7386   ins_cost(0);
7387 
7388   size(0);
7389   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
7390   ins_encode();
7391   ins_pipe(empty);
7392 %}
7393 
7394 instruct membar_release()
7395 %{
7396   match(MemBarRelease);
7397   ins_cost(0);
7398 
7399   size(0);
7400   format %{ "MEMBAR-release ! (empty encoding)" %}
7401   ins_encode();
7402   ins_pipe(empty);
7403 %}
7404 
7405 instruct membar_release_lock()
7406 %{
7407   match(MemBarReleaseLock);
7408   ins_cost(0);
7409 
7410   size(0);
7411   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
7412   ins_encode();
7413   ins_pipe(empty);
7414 %}
7415 
7416 instruct membar_volatile(rFlagsReg cr) %{
7417   match(MemBarVolatile);
7418   effect(KILL cr);
7419   ins_cost(400);
7420 
7421   format %{
7422     $$template
7423     if (os::is_MP()) {
7424       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
7425     } else {
7426       $$emit$$"MEMBAR-volatile ! (empty encoding)"
7427     }
7428   %}
7429   ins_encode %{
7430     __ membar(Assembler::StoreLoad);
7431   %}
7432   ins_pipe(pipe_slow);
7433 %}
7434 
7435 instruct unnecessary_membar_volatile()
7436 %{
7437   match(MemBarVolatile);
7438   predicate(Matcher::post_store_load_barrier(n));
7439   ins_cost(0);
7440 
7441   size(0);
7442   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
7443   ins_encode();
7444   ins_pipe(empty);
7445 %}
7446 
7447 //----------Move Instructions--------------------------------------------------
7448 
7449 instruct castX2P(rRegP dst, rRegL src)
7450 %{
7451   match(Set dst (CastX2P src));
7452 
7453   format %{ "movq    $dst, $src\t# long->ptr" %}
7454   ins_encode(enc_copy_wide(dst, src));
7455   ins_pipe(ialu_reg_reg); // XXX
7456 %}
7457 
7458 instruct castP2X(rRegL dst, rRegP src)
7459 %{
7460   match(Set dst (CastP2X src));
7461 
7462   format %{ "movq    $dst, $src\t# ptr -> long" %}
7463   ins_encode(enc_copy_wide(dst, src));
7464   ins_pipe(ialu_reg_reg); // XXX
7465 %}
7466 
7467 
7468 // Convert oop pointer into compressed form
7469 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
7470   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7471   match(Set dst (EncodeP src));
7472   effect(KILL cr);
7473   format %{ "encode_heap_oop $dst,$src" %}
7474   ins_encode %{
7475     Register s = $src$$Register;
7476     Register d = $dst$$Register;
7477     if (s != d) {
7478       __ movq(d, s);
7479     }
7480     __ encode_heap_oop(d);
7481   %}
7482   ins_pipe(ialu_reg_long);
7483 %}
7484 
7485 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7486   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7487   match(Set dst (EncodeP src));
7488   effect(KILL cr);
7489   format %{ "encode_heap_oop_not_null $dst,$src" %}
7490   ins_encode %{
7491     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7492   %}
7493   ins_pipe(ialu_reg_long);
7494 %}
7495 
7496 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
7497   predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull &&
7498             n->bottom_type()->is_oopptr()->ptr() != TypePtr::Constant);
7499   match(Set dst (DecodeN src));
7500   effect(KILL cr);
7501   format %{ "decode_heap_oop $dst,$src" %}
7502   ins_encode %{
7503     Register s = $src$$Register;
7504     Register d = $dst$$Register;
7505     if (s != d) {
7506       __ movq(d, s);
7507     }
7508     __ decode_heap_oop(d);
7509   %}
7510   ins_pipe(ialu_reg_long);
7511 %}
7512 
7513 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7514   predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull ||
7515             n->bottom_type()->is_oopptr()->ptr() == TypePtr::Constant);
7516   match(Set dst (DecodeN src));
7517   effect(KILL cr);
7518   format %{ "decode_heap_oop_not_null $dst,$src" %}
7519   ins_encode %{
7520     Register s = $src$$Register;
7521     Register d = $dst$$Register;
7522     if (s != d) {
7523       __ decode_heap_oop_not_null(d, s);
7524     } else {
7525       __ decode_heap_oop_not_null(d);
7526     }
7527   %}
7528   ins_pipe(ialu_reg_long);
7529 %}
7530 
7531 
7532 //----------Conditional Move---------------------------------------------------
7533 // Jump
7534 // dummy instruction for generating temp registers
7535 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7536   match(Jump (LShiftL switch_val shift));
7537   ins_cost(350);
7538   predicate(false);
7539   effect(TEMP dest);
7540 
7541   format %{ "leaq    $dest, [$constantaddress]\n\t"
7542             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7543   ins_encode %{
7544     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7545     // to do that and the compiler is using that register as one it can allocate.
7546     // So we build it all by hand.
7547     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7548     // ArrayAddress dispatch(table, index);
7549     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7550     __ lea($dest$$Register, $constantaddress);
7551     __ jmp(dispatch);
7552   %}
7553   ins_pipe(pipe_jmp);
7554 %}
7555 
7556 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7557   match(Jump (AddL (LShiftL switch_val shift) offset));
7558   ins_cost(350);
7559   effect(TEMP dest);
7560 
7561   format %{ "leaq    $dest, [$constantaddress]\n\t"
7562             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7563   ins_encode %{
7564     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7565     // to do that and the compiler is using that register as one it can allocate.
7566     // So we build it all by hand.
7567     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7568     // ArrayAddress dispatch(table, index);
7569     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7570     __ lea($dest$$Register, $constantaddress);
7571     __ jmp(dispatch);
7572   %}
7573   ins_pipe(pipe_jmp);
7574 %}
7575 
7576 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7577   match(Jump switch_val);
7578   ins_cost(350);
7579   effect(TEMP dest);
7580 
7581   format %{ "leaq    $dest, [$constantaddress]\n\t"
7582             "jmp     [$dest + $switch_val]\n\t" %}
7583   ins_encode %{
7584     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7585     // to do that and the compiler is using that register as one it can allocate.
7586     // So we build it all by hand.
7587     // Address index(noreg, switch_reg, Address::times_1);
7588     // ArrayAddress dispatch(table, index);
7589     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7590     __ lea($dest$$Register, $constantaddress);
7591     __ jmp(dispatch);
7592   %}
7593   ins_pipe(pipe_jmp);
7594 %}
7595 
7596 // Conditional move
7597 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7598 %{
7599   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7600 
7601   ins_cost(200); // XXX
7602   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7603   opcode(0x0F, 0x40);
7604   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7605   ins_pipe(pipe_cmov_reg);
7606 %}
7607 
7608 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7609   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7610 
7611   ins_cost(200); // XXX
7612   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7613   opcode(0x0F, 0x40);
7614   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7615   ins_pipe(pipe_cmov_reg);
7616 %}
7617 
7618 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7619   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7620   ins_cost(200);
7621   expand %{
7622     cmovI_regU(cop, cr, dst, src);
7623   %}
7624 %}
7625 
7626 // Conditional move
7627 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7628   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7629 
7630   ins_cost(250); // XXX
7631   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7632   opcode(0x0F, 0x40);
7633   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7634   ins_pipe(pipe_cmov_mem);
7635 %}
7636 
7637 // Conditional move
7638 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7639 %{
7640   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7641 
7642   ins_cost(250); // XXX
7643   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7644   opcode(0x0F, 0x40);
7645   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7646   ins_pipe(pipe_cmov_mem);
7647 %}
7648 
7649 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7650   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7651   ins_cost(250);
7652   expand %{
7653     cmovI_memU(cop, cr, dst, src);
7654   %}
7655 %}
7656 
7657 // Conditional move
7658 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7659 %{
7660   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7661 
7662   ins_cost(200); // XXX
7663   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7664   opcode(0x0F, 0x40);
7665   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7666   ins_pipe(pipe_cmov_reg);
7667 %}
7668 
7669 // Conditional move
7670 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7671 %{
7672   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7673 
7674   ins_cost(200); // XXX
7675   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7676   opcode(0x0F, 0x40);
7677   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7678   ins_pipe(pipe_cmov_reg);
7679 %}
7680 
7681 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7682   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7683   ins_cost(200);
7684   expand %{
7685     cmovN_regU(cop, cr, dst, src);
7686   %}
7687 %}
7688 
7689 // Conditional move
7690 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7691 %{
7692   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7693 
7694   ins_cost(200); // XXX
7695   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7696   opcode(0x0F, 0x40);
7697   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7698   ins_pipe(pipe_cmov_reg);  // XXX
7699 %}
7700 
7701 // Conditional move
7702 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7703 %{
7704   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7705 
7706   ins_cost(200); // XXX
7707   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7708   opcode(0x0F, 0x40);
7709   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7710   ins_pipe(pipe_cmov_reg); // XXX
7711 %}
7712 
7713 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7714   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7715   ins_cost(200);
7716   expand %{
7717     cmovP_regU(cop, cr, dst, src);
7718   %}
7719 %}
7720 
7721 // DISABLED: Requires the ADLC to emit a bottom_type call that
7722 // correctly meets the two pointer arguments; one is an incoming
7723 // register but the other is a memory operand.  ALSO appears to
7724 // be buggy with implicit null checks.
7725 //
7726 //// Conditional move
7727 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7728 //%{
7729 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7730 //  ins_cost(250);
7731 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7732 //  opcode(0x0F,0x40);
7733 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7734 //  ins_pipe( pipe_cmov_mem );
7735 //%}
7736 //
7737 //// Conditional move
7738 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7739 //%{
7740 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7741 //  ins_cost(250);
7742 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7743 //  opcode(0x0F,0x40);
7744 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7745 //  ins_pipe( pipe_cmov_mem );
7746 //%}
7747 
7748 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7749 %{
7750   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7751 
7752   ins_cost(200); // XXX
7753   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7754   opcode(0x0F, 0x40);
7755   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7756   ins_pipe(pipe_cmov_reg);  // XXX
7757 %}
7758 
7759 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7760 %{
7761   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7762 
7763   ins_cost(200); // XXX
7764   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7765   opcode(0x0F, 0x40);
7766   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7767   ins_pipe(pipe_cmov_mem);  // XXX
7768 %}
7769 
7770 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7771 %{
7772   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7773 
7774   ins_cost(200); // XXX
7775   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7776   opcode(0x0F, 0x40);
7777   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7778   ins_pipe(pipe_cmov_reg); // XXX
7779 %}
7780 
7781 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7782   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7783   ins_cost(200);
7784   expand %{
7785     cmovL_regU(cop, cr, dst, src);
7786   %}
7787 %}
7788 
7789 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7790 %{
7791   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7792 
7793   ins_cost(200); // XXX
7794   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7795   opcode(0x0F, 0x40);
7796   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7797   ins_pipe(pipe_cmov_mem); // XXX
7798 %}
7799 
7800 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7801   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7802   ins_cost(200);
7803   expand %{
7804     cmovL_memU(cop, cr, dst, src);
7805   %}
7806 %}
7807 
7808 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7809 %{
7810   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7811 
7812   ins_cost(200); // XXX
7813   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7814             "movss     $dst, $src\n"
7815     "skip:" %}
7816   ins_encode(enc_cmovf_branch(cop, dst, src));
7817   ins_pipe(pipe_slow);
7818 %}
7819 
7820 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7821 // %{
7822 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7823 
7824 //   ins_cost(200); // XXX
7825 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7826 //             "movss     $dst, $src\n"
7827 //     "skip:" %}
7828 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7829 //   ins_pipe(pipe_slow);
7830 // %}
7831 
7832 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7833 %{
7834   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7835 
7836   ins_cost(200); // XXX
7837   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7838             "movss     $dst, $src\n"
7839     "skip:" %}
7840   ins_encode(enc_cmovf_branch(cop, dst, src));
7841   ins_pipe(pipe_slow);
7842 %}
7843 
7844 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7845   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7846   ins_cost(200);
7847   expand %{
7848     cmovF_regU(cop, cr, dst, src);
7849   %}
7850 %}
7851 
7852 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7853 %{
7854   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7855 
7856   ins_cost(200); // XXX
7857   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7858             "movsd     $dst, $src\n"
7859     "skip:" %}
7860   ins_encode(enc_cmovd_branch(cop, dst, src));
7861   ins_pipe(pipe_slow);
7862 %}
7863 
7864 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7865 %{
7866   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7867 
7868   ins_cost(200); // XXX
7869   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7870             "movsd     $dst, $src\n"
7871     "skip:" %}
7872   ins_encode(enc_cmovd_branch(cop, dst, src));
7873   ins_pipe(pipe_slow);
7874 %}
7875 
7876 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7877   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7878   ins_cost(200);
7879   expand %{
7880     cmovD_regU(cop, cr, dst, src);
7881   %}
7882 %}
7883 
7884 //----------Arithmetic Instructions--------------------------------------------
7885 //----------Addition Instructions----------------------------------------------
7886 
7887 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7888 %{
7889   match(Set dst (AddI dst src));
7890   effect(KILL cr);
7891 
7892   format %{ "addl    $dst, $src\t# int" %}
7893   opcode(0x03);
7894   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7895   ins_pipe(ialu_reg_reg);
7896 %}
7897 
7898 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7899 %{
7900   match(Set dst (AddI dst src));
7901   effect(KILL cr);
7902 
7903   format %{ "addl    $dst, $src\t# int" %}
7904   opcode(0x81, 0x00); /* /0 id */
7905   ins_encode(OpcSErm(dst, src), Con8or32(src));
7906   ins_pipe( ialu_reg );
7907 %}
7908 
7909 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7910 %{
7911   match(Set dst (AddI dst (LoadI src)));
7912   effect(KILL cr);
7913 
7914   ins_cost(125); // XXX
7915   format %{ "addl    $dst, $src\t# int" %}
7916   opcode(0x03);
7917   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7918   ins_pipe(ialu_reg_mem);
7919 %}
7920 
7921 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7922 %{
7923   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7924   effect(KILL cr);
7925 
7926   ins_cost(150); // XXX
7927   format %{ "addl    $dst, $src\t# int" %}
7928   opcode(0x01); /* Opcode 01 /r */
7929   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7930   ins_pipe(ialu_mem_reg);
7931 %}
7932 
7933 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7934 %{
7935   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7936   effect(KILL cr);
7937 
7938   ins_cost(125); // XXX
7939   format %{ "addl    $dst, $src\t# int" %}
7940   opcode(0x81); /* Opcode 81 /0 id */
7941   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7942   ins_pipe(ialu_mem_imm);
7943 %}
7944 
7945 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7946 %{
7947   predicate(UseIncDec);
7948   match(Set dst (AddI dst src));
7949   effect(KILL cr);
7950 
7951   format %{ "incl    $dst\t# int" %}
7952   opcode(0xFF, 0x00); // FF /0
7953   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7954   ins_pipe(ialu_reg);
7955 %}
7956 
7957 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7958 %{
7959   predicate(UseIncDec);
7960   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7961   effect(KILL cr);
7962 
7963   ins_cost(125); // XXX
7964   format %{ "incl    $dst\t# int" %}
7965   opcode(0xFF); /* Opcode FF /0 */
7966   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7967   ins_pipe(ialu_mem_imm);
7968 %}
7969 
7970 // XXX why does that use AddI
7971 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7972 %{
7973   predicate(UseIncDec);
7974   match(Set dst (AddI dst src));
7975   effect(KILL cr);
7976 
7977   format %{ "decl    $dst\t# int" %}
7978   opcode(0xFF, 0x01); // FF /1
7979   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7980   ins_pipe(ialu_reg);
7981 %}
7982 
7983 // XXX why does that use AddI
7984 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7985 %{
7986   predicate(UseIncDec);
7987   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7988   effect(KILL cr);
7989 
7990   ins_cost(125); // XXX
7991   format %{ "decl    $dst\t# int" %}
7992   opcode(0xFF); /* Opcode FF /1 */
7993   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7994   ins_pipe(ialu_mem_imm);
7995 %}
7996 
7997 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7998 %{
7999   match(Set dst (AddI src0 src1));
8000 
8001   ins_cost(110);
8002   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
8003   opcode(0x8D); /* 0x8D /r */
8004   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
8005   ins_pipe(ialu_reg_reg);
8006 %}
8007 
8008 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8009 %{
8010   match(Set dst (AddL dst src));
8011   effect(KILL cr);
8012 
8013   format %{ "addq    $dst, $src\t# long" %}
8014   opcode(0x03);
8015   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8016   ins_pipe(ialu_reg_reg);
8017 %}
8018 
8019 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
8020 %{
8021   match(Set dst (AddL dst src));
8022   effect(KILL cr);
8023 
8024   format %{ "addq    $dst, $src\t# long" %}
8025   opcode(0x81, 0x00); /* /0 id */
8026   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8027   ins_pipe( ialu_reg );
8028 %}
8029 
8030 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8031 %{
8032   match(Set dst (AddL dst (LoadL src)));
8033   effect(KILL cr);
8034 
8035   ins_cost(125); // XXX
8036   format %{ "addq    $dst, $src\t# long" %}
8037   opcode(0x03);
8038   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8039   ins_pipe(ialu_reg_mem);
8040 %}
8041 
8042 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8043 %{
8044   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
8045   effect(KILL cr);
8046 
8047   ins_cost(150); // XXX
8048   format %{ "addq    $dst, $src\t# long" %}
8049   opcode(0x01); /* Opcode 01 /r */
8050   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8051   ins_pipe(ialu_mem_reg);
8052 %}
8053 
8054 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8055 %{
8056   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
8057   effect(KILL cr);
8058 
8059   ins_cost(125); // XXX
8060   format %{ "addq    $dst, $src\t# long" %}
8061   opcode(0x81); /* Opcode 81 /0 id */
8062   ins_encode(REX_mem_wide(dst),
8063              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
8064   ins_pipe(ialu_mem_imm);
8065 %}
8066 
8067 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
8068 %{
8069   predicate(UseIncDec);
8070   match(Set dst (AddL dst src));
8071   effect(KILL cr);
8072 
8073   format %{ "incq    $dst\t# long" %}
8074   opcode(0xFF, 0x00); // FF /0
8075   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8076   ins_pipe(ialu_reg);
8077 %}
8078 
8079 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
8080 %{
8081   predicate(UseIncDec);
8082   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
8083   effect(KILL cr);
8084 
8085   ins_cost(125); // XXX
8086   format %{ "incq    $dst\t# long" %}
8087   opcode(0xFF); /* Opcode FF /0 */
8088   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
8089   ins_pipe(ialu_mem_imm);
8090 %}
8091 
8092 // XXX why does that use AddL
8093 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
8094 %{
8095   predicate(UseIncDec);
8096   match(Set dst (AddL dst src));
8097   effect(KILL cr);
8098 
8099   format %{ "decq    $dst\t# long" %}
8100   opcode(0xFF, 0x01); // FF /1
8101   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8102   ins_pipe(ialu_reg);
8103 %}
8104 
8105 // XXX why does that use AddL
8106 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
8107 %{
8108   predicate(UseIncDec);
8109   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
8110   effect(KILL cr);
8111 
8112   ins_cost(125); // XXX
8113   format %{ "decq    $dst\t# long" %}
8114   opcode(0xFF); /* Opcode FF /1 */
8115   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
8116   ins_pipe(ialu_mem_imm);
8117 %}
8118 
8119 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
8120 %{
8121   match(Set dst (AddL src0 src1));
8122 
8123   ins_cost(110);
8124   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
8125   opcode(0x8D); /* 0x8D /r */
8126   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
8127   ins_pipe(ialu_reg_reg);
8128 %}
8129 
8130 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
8131 %{
8132   match(Set dst (AddP dst src));
8133   effect(KILL cr);
8134 
8135   format %{ "addq    $dst, $src\t# ptr" %}
8136   opcode(0x03);
8137   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8138   ins_pipe(ialu_reg_reg);
8139 %}
8140 
8141 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
8142 %{
8143   match(Set dst (AddP dst src));
8144   effect(KILL cr);
8145 
8146   format %{ "addq    $dst, $src\t# ptr" %}
8147   opcode(0x81, 0x00); /* /0 id */
8148   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8149   ins_pipe( ialu_reg );
8150 %}
8151 
8152 // XXX addP mem ops ????
8153 
8154 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
8155 %{
8156   match(Set dst (AddP src0 src1));
8157 
8158   ins_cost(110);
8159   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
8160   opcode(0x8D); /* 0x8D /r */
8161   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
8162   ins_pipe(ialu_reg_reg);
8163 %}
8164 
8165 instruct checkCastPP(rRegP dst)
8166 %{
8167   match(Set dst (CheckCastPP dst));
8168 
8169   size(0);
8170   format %{ "# checkcastPP of $dst" %}
8171   ins_encode(/* empty encoding */);
8172   ins_pipe(empty);
8173 %}
8174 
8175 instruct castPP(rRegP dst)
8176 %{
8177   match(Set dst (CastPP dst));
8178 
8179   size(0);
8180   format %{ "# castPP of $dst" %}
8181   ins_encode(/* empty encoding */);
8182   ins_pipe(empty);
8183 %}
8184 
8185 instruct castII(rRegI dst)
8186 %{
8187   match(Set dst (CastII dst));
8188 
8189   size(0);
8190   format %{ "# castII of $dst" %}
8191   ins_encode(/* empty encoding */);
8192   ins_cost(0);
8193   ins_pipe(empty);
8194 %}
8195 
8196 // LoadP-locked same as a regular LoadP when used with compare-swap
8197 instruct loadPLocked(rRegP dst, memory mem)
8198 %{
8199   match(Set dst (LoadPLocked mem));
8200 
8201   ins_cost(125); // XXX
8202   format %{ "movq    $dst, $mem\t# ptr locked" %}
8203   opcode(0x8B);
8204   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
8205   ins_pipe(ialu_reg_mem); // XXX
8206 %}
8207 
8208 // LoadL-locked - same as a regular LoadL when used with compare-swap
8209 instruct loadLLocked(rRegL dst, memory mem)
8210 %{
8211   match(Set dst (LoadLLocked mem));
8212 
8213   ins_cost(125); // XXX
8214   format %{ "movq    $dst, $mem\t# long locked" %}
8215   opcode(0x8B);
8216   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
8217   ins_pipe(ialu_reg_mem); // XXX
8218 %}
8219 
8220 // Conditional-store of the updated heap-top.
8221 // Used during allocation of the shared heap.
8222 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
8223 
8224 instruct storePConditional(memory heap_top_ptr,
8225                            rax_RegP oldval, rRegP newval,
8226                            rFlagsReg cr)
8227 %{
8228   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8229 
8230   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
8231             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
8232   opcode(0x0F, 0xB1);
8233   ins_encode(lock_prefix,
8234              REX_reg_mem_wide(newval, heap_top_ptr),
8235              OpcP, OpcS,
8236              reg_mem(newval, heap_top_ptr));
8237   ins_pipe(pipe_cmpxchg);
8238 %}
8239 
8240 // Conditional-store of an int value.
8241 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
8242 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
8243 %{
8244   match(Set cr (StoreIConditional mem (Binary oldval newval)));
8245   effect(KILL oldval);
8246 
8247   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
8248   opcode(0x0F, 0xB1);
8249   ins_encode(lock_prefix,
8250              REX_reg_mem(newval, mem),
8251              OpcP, OpcS,
8252              reg_mem(newval, mem));
8253   ins_pipe(pipe_cmpxchg);
8254 %}
8255 
8256 // Conditional-store of a long value.
8257 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
8258 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
8259 %{
8260   match(Set cr (StoreLConditional mem (Binary oldval newval)));
8261   effect(KILL oldval);
8262 
8263   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
8264   opcode(0x0F, 0xB1);
8265   ins_encode(lock_prefix,
8266              REX_reg_mem_wide(newval, mem),
8267              OpcP, OpcS,
8268              reg_mem(newval, mem));
8269   ins_pipe(pipe_cmpxchg);
8270 %}
8271 
8272 
8273 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
8274 instruct compareAndSwapP(rRegI res,
8275                          memory mem_ptr,
8276                          rax_RegP oldval, rRegP newval,
8277                          rFlagsReg cr)
8278 %{
8279   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
8280   effect(KILL cr, KILL oldval);
8281 
8282   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8283             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8284             "sete    $res\n\t"
8285             "movzbl  $res, $res" %}
8286   opcode(0x0F, 0xB1);
8287   ins_encode(lock_prefix,
8288              REX_reg_mem_wide(newval, mem_ptr),
8289              OpcP, OpcS,
8290              reg_mem(newval, mem_ptr),
8291              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8292              REX_reg_breg(res, res), // movzbl
8293              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8294   ins_pipe( pipe_cmpxchg );
8295 %}
8296 
8297 instruct compareAndSwapL(rRegI res,
8298                          memory mem_ptr,
8299                          rax_RegL oldval, rRegL newval,
8300                          rFlagsReg cr)
8301 %{
8302   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
8303   effect(KILL cr, KILL oldval);
8304 
8305   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8306             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8307             "sete    $res\n\t"
8308             "movzbl  $res, $res" %}
8309   opcode(0x0F, 0xB1);
8310   ins_encode(lock_prefix,
8311              REX_reg_mem_wide(newval, mem_ptr),
8312              OpcP, OpcS,
8313              reg_mem(newval, mem_ptr),
8314              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8315              REX_reg_breg(res, res), // movzbl
8316              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8317   ins_pipe( pipe_cmpxchg );
8318 %}
8319 
8320 instruct compareAndSwapI(rRegI res,
8321                          memory mem_ptr,
8322                          rax_RegI oldval, rRegI newval,
8323                          rFlagsReg cr)
8324 %{
8325   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
8326   effect(KILL cr, KILL oldval);
8327 
8328   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8329             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8330             "sete    $res\n\t"
8331             "movzbl  $res, $res" %}
8332   opcode(0x0F, 0xB1);
8333   ins_encode(lock_prefix,
8334              REX_reg_mem(newval, mem_ptr),
8335              OpcP, OpcS,
8336              reg_mem(newval, mem_ptr),
8337              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8338              REX_reg_breg(res, res), // movzbl
8339              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8340   ins_pipe( pipe_cmpxchg );
8341 %}
8342 
8343 
8344 instruct compareAndSwapN(rRegI res,
8345                           memory mem_ptr,
8346                           rax_RegN oldval, rRegN newval,
8347                           rFlagsReg cr) %{
8348   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
8349   effect(KILL cr, KILL oldval);
8350 
8351   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8352             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
8353             "sete    $res\n\t"
8354             "movzbl  $res, $res" %}
8355   opcode(0x0F, 0xB1);
8356   ins_encode(lock_prefix,
8357              REX_reg_mem(newval, mem_ptr),
8358              OpcP, OpcS,
8359              reg_mem(newval, mem_ptr),
8360              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
8361              REX_reg_breg(res, res), // movzbl
8362              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
8363   ins_pipe( pipe_cmpxchg );
8364 %}
8365 
8366 //----------Subtraction Instructions-------------------------------------------
8367 
8368 // Integer Subtraction Instructions
8369 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8370 %{
8371   match(Set dst (SubI dst src));
8372   effect(KILL cr);
8373 
8374   format %{ "subl    $dst, $src\t# int" %}
8375   opcode(0x2B);
8376   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8377   ins_pipe(ialu_reg_reg);
8378 %}
8379 
8380 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8381 %{
8382   match(Set dst (SubI dst src));
8383   effect(KILL cr);
8384 
8385   format %{ "subl    $dst, $src\t# int" %}
8386   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8387   ins_encode(OpcSErm(dst, src), Con8or32(src));
8388   ins_pipe(ialu_reg);
8389 %}
8390 
8391 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8392 %{
8393   match(Set dst (SubI dst (LoadI src)));
8394   effect(KILL cr);
8395 
8396   ins_cost(125);
8397   format %{ "subl    $dst, $src\t# int" %}
8398   opcode(0x2B);
8399   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8400   ins_pipe(ialu_reg_mem);
8401 %}
8402 
8403 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8404 %{
8405   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8406   effect(KILL cr);
8407 
8408   ins_cost(150);
8409   format %{ "subl    $dst, $src\t# int" %}
8410   opcode(0x29); /* Opcode 29 /r */
8411   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8412   ins_pipe(ialu_mem_reg);
8413 %}
8414 
8415 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8416 %{
8417   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8418   effect(KILL cr);
8419 
8420   ins_cost(125); // XXX
8421   format %{ "subl    $dst, $src\t# int" %}
8422   opcode(0x81); /* Opcode 81 /5 id */
8423   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8424   ins_pipe(ialu_mem_imm);
8425 %}
8426 
8427 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8428 %{
8429   match(Set dst (SubL dst src));
8430   effect(KILL cr);
8431 
8432   format %{ "subq    $dst, $src\t# long" %}
8433   opcode(0x2B);
8434   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8435   ins_pipe(ialu_reg_reg);
8436 %}
8437 
8438 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8439 %{
8440   match(Set dst (SubL dst src));
8441   effect(KILL cr);
8442 
8443   format %{ "subq    $dst, $src\t# long" %}
8444   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8445   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8446   ins_pipe(ialu_reg);
8447 %}
8448 
8449 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8450 %{
8451   match(Set dst (SubL dst (LoadL src)));
8452   effect(KILL cr);
8453 
8454   ins_cost(125);
8455   format %{ "subq    $dst, $src\t# long" %}
8456   opcode(0x2B);
8457   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8458   ins_pipe(ialu_reg_mem);
8459 %}
8460 
8461 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8462 %{
8463   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8464   effect(KILL cr);
8465 
8466   ins_cost(150);
8467   format %{ "subq    $dst, $src\t# long" %}
8468   opcode(0x29); /* Opcode 29 /r */
8469   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8470   ins_pipe(ialu_mem_reg);
8471 %}
8472 
8473 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8474 %{
8475   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8476   effect(KILL cr);
8477 
8478   ins_cost(125); // XXX
8479   format %{ "subq    $dst, $src\t# long" %}
8480   opcode(0x81); /* Opcode 81 /5 id */
8481   ins_encode(REX_mem_wide(dst),
8482              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8483   ins_pipe(ialu_mem_imm);
8484 %}
8485 
8486 // Subtract from a pointer
8487 // XXX hmpf???
8488 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8489 %{
8490   match(Set dst (AddP dst (SubI zero src)));
8491   effect(KILL cr);
8492 
8493   format %{ "subq    $dst, $src\t# ptr - int" %}
8494   opcode(0x2B);
8495   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8496   ins_pipe(ialu_reg_reg);
8497 %}
8498 
8499 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8500 %{
8501   match(Set dst (SubI zero dst));
8502   effect(KILL cr);
8503 
8504   format %{ "negl    $dst\t# int" %}
8505   opcode(0xF7, 0x03);  // Opcode F7 /3
8506   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8507   ins_pipe(ialu_reg);
8508 %}
8509 
8510 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8511 %{
8512   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8513   effect(KILL cr);
8514 
8515   format %{ "negl    $dst\t# int" %}
8516   opcode(0xF7, 0x03);  // Opcode F7 /3
8517   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8518   ins_pipe(ialu_reg);
8519 %}
8520 
8521 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8522 %{
8523   match(Set dst (SubL zero dst));
8524   effect(KILL cr);
8525 
8526   format %{ "negq    $dst\t# long" %}
8527   opcode(0xF7, 0x03);  // Opcode F7 /3
8528   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8529   ins_pipe(ialu_reg);
8530 %}
8531 
8532 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8533 %{
8534   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8535   effect(KILL cr);
8536 
8537   format %{ "negq    $dst\t# long" %}
8538   opcode(0xF7, 0x03);  // Opcode F7 /3
8539   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8540   ins_pipe(ialu_reg);
8541 %}
8542 
8543 
8544 //----------Multiplication/Division Instructions-------------------------------
8545 // Integer Multiplication Instructions
8546 // Multiply Register
8547 
8548 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8549 %{
8550   match(Set dst (MulI dst src));
8551   effect(KILL cr);
8552 
8553   ins_cost(300);
8554   format %{ "imull   $dst, $src\t# int" %}
8555   opcode(0x0F, 0xAF);
8556   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8557   ins_pipe(ialu_reg_reg_alu0);
8558 %}
8559 
8560 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8561 %{
8562   match(Set dst (MulI src imm));
8563   effect(KILL cr);
8564 
8565   ins_cost(300);
8566   format %{ "imull   $dst, $src, $imm\t# int" %}
8567   opcode(0x69); /* 69 /r id */
8568   ins_encode(REX_reg_reg(dst, src),
8569              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8570   ins_pipe(ialu_reg_reg_alu0);
8571 %}
8572 
8573 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8574 %{
8575   match(Set dst (MulI dst (LoadI src)));
8576   effect(KILL cr);
8577 
8578   ins_cost(350);
8579   format %{ "imull   $dst, $src\t# int" %}
8580   opcode(0x0F, 0xAF);
8581   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8582   ins_pipe(ialu_reg_mem_alu0);
8583 %}
8584 
8585 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8586 %{
8587   match(Set dst (MulI (LoadI src) imm));
8588   effect(KILL cr);
8589 
8590   ins_cost(300);
8591   format %{ "imull   $dst, $src, $imm\t# int" %}
8592   opcode(0x69); /* 69 /r id */
8593   ins_encode(REX_reg_mem(dst, src),
8594              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8595   ins_pipe(ialu_reg_mem_alu0);
8596 %}
8597 
8598 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8599 %{
8600   match(Set dst (MulL dst src));
8601   effect(KILL cr);
8602 
8603   ins_cost(300);
8604   format %{ "imulq   $dst, $src\t# long" %}
8605   opcode(0x0F, 0xAF);
8606   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8607   ins_pipe(ialu_reg_reg_alu0);
8608 %}
8609 
8610 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8611 %{
8612   match(Set dst (MulL src imm));
8613   effect(KILL cr);
8614 
8615   ins_cost(300);
8616   format %{ "imulq   $dst, $src, $imm\t# long" %}
8617   opcode(0x69); /* 69 /r id */
8618   ins_encode(REX_reg_reg_wide(dst, src),
8619              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8620   ins_pipe(ialu_reg_reg_alu0);
8621 %}
8622 
8623 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8624 %{
8625   match(Set dst (MulL dst (LoadL src)));
8626   effect(KILL cr);
8627 
8628   ins_cost(350);
8629   format %{ "imulq   $dst, $src\t# long" %}
8630   opcode(0x0F, 0xAF);
8631   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8632   ins_pipe(ialu_reg_mem_alu0);
8633 %}
8634 
8635 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8636 %{
8637   match(Set dst (MulL (LoadL src) imm));
8638   effect(KILL cr);
8639 
8640   ins_cost(300);
8641   format %{ "imulq   $dst, $src, $imm\t# long" %}
8642   opcode(0x69); /* 69 /r id */
8643   ins_encode(REX_reg_mem_wide(dst, src),
8644              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8645   ins_pipe(ialu_reg_mem_alu0);
8646 %}
8647 
8648 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8649 %{
8650   match(Set dst (MulHiL src rax));
8651   effect(USE_KILL rax, KILL cr);
8652 
8653   ins_cost(300);
8654   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8655   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8656   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8657   ins_pipe(ialu_reg_reg_alu0);
8658 %}
8659 
8660 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8661                    rFlagsReg cr)
8662 %{
8663   match(Set rax (DivI rax div));
8664   effect(KILL rdx, KILL cr);
8665 
8666   ins_cost(30*100+10*100); // XXX
8667   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8668             "jne,s   normal\n\t"
8669             "xorl    rdx, rdx\n\t"
8670             "cmpl    $div, -1\n\t"
8671             "je,s    done\n"
8672     "normal: cdql\n\t"
8673             "idivl   $div\n"
8674     "done:"        %}
8675   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8676   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8677   ins_pipe(ialu_reg_reg_alu0);
8678 %}
8679 
8680 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8681                    rFlagsReg cr)
8682 %{
8683   match(Set rax (DivL rax div));
8684   effect(KILL rdx, KILL cr);
8685 
8686   ins_cost(30*100+10*100); // XXX
8687   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8688             "cmpq    rax, rdx\n\t"
8689             "jne,s   normal\n\t"
8690             "xorl    rdx, rdx\n\t"
8691             "cmpq    $div, -1\n\t"
8692             "je,s    done\n"
8693     "normal: cdqq\n\t"
8694             "idivq   $div\n"
8695     "done:"        %}
8696   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8697   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8698   ins_pipe(ialu_reg_reg_alu0);
8699 %}
8700 
8701 // Integer DIVMOD with Register, both quotient and mod results
8702 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8703                              rFlagsReg cr)
8704 %{
8705   match(DivModI rax div);
8706   effect(KILL cr);
8707 
8708   ins_cost(30*100+10*100); // XXX
8709   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8710             "jne,s   normal\n\t"
8711             "xorl    rdx, rdx\n\t"
8712             "cmpl    $div, -1\n\t"
8713             "je,s    done\n"
8714     "normal: cdql\n\t"
8715             "idivl   $div\n"
8716     "done:"        %}
8717   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8718   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8719   ins_pipe(pipe_slow);
8720 %}
8721 
8722 // Long DIVMOD with Register, both quotient and mod results
8723 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8724                              rFlagsReg cr)
8725 %{
8726   match(DivModL rax div);
8727   effect(KILL cr);
8728 
8729   ins_cost(30*100+10*100); // XXX
8730   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8731             "cmpq    rax, rdx\n\t"
8732             "jne,s   normal\n\t"
8733             "xorl    rdx, rdx\n\t"
8734             "cmpq    $div, -1\n\t"
8735             "je,s    done\n"
8736     "normal: cdqq\n\t"
8737             "idivq   $div\n"
8738     "done:"        %}
8739   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8740   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8741   ins_pipe(pipe_slow);
8742 %}
8743 
8744 //----------- DivL-By-Constant-Expansions--------------------------------------
8745 // DivI cases are handled by the compiler
8746 
8747 // Magic constant, reciprocal of 10
8748 instruct loadConL_0x6666666666666667(rRegL dst)
8749 %{
8750   effect(DEF dst);
8751 
8752   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8753   ins_encode(load_immL(dst, 0x6666666666666667));
8754   ins_pipe(ialu_reg);
8755 %}
8756 
8757 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8758 %{
8759   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8760 
8761   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8762   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8763   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8764   ins_pipe(ialu_reg_reg_alu0);
8765 %}
8766 
8767 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8768 %{
8769   effect(USE_DEF dst, KILL cr);
8770 
8771   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8772   opcode(0xC1, 0x7); /* C1 /7 ib */
8773   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8774   ins_pipe(ialu_reg);
8775 %}
8776 
8777 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8778 %{
8779   effect(USE_DEF dst, KILL cr);
8780 
8781   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8782   opcode(0xC1, 0x7); /* C1 /7 ib */
8783   ins_encode(reg_opc_imm_wide(dst, 0x2));
8784   ins_pipe(ialu_reg);
8785 %}
8786 
8787 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8788 %{
8789   match(Set dst (DivL src div));
8790 
8791   ins_cost((5+8)*100);
8792   expand %{
8793     rax_RegL rax;                     // Killed temp
8794     rFlagsReg cr;                     // Killed
8795     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8796     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8797     sarL_rReg_63(src, cr);            // sarq  src, 63
8798     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8799     subL_rReg(dst, src, cr);          // subl  rdx, src
8800   %}
8801 %}
8802 
8803 //-----------------------------------------------------------------------------
8804 
8805 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8806                    rFlagsReg cr)
8807 %{
8808   match(Set rdx (ModI rax div));
8809   effect(KILL rax, KILL cr);
8810 
8811   ins_cost(300); // XXX
8812   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8813             "jne,s   normal\n\t"
8814             "xorl    rdx, rdx\n\t"
8815             "cmpl    $div, -1\n\t"
8816             "je,s    done\n"
8817     "normal: cdql\n\t"
8818             "idivl   $div\n"
8819     "done:"        %}
8820   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8821   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8822   ins_pipe(ialu_reg_reg_alu0);
8823 %}
8824 
8825 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8826                    rFlagsReg cr)
8827 %{
8828   match(Set rdx (ModL rax div));
8829   effect(KILL rax, KILL cr);
8830 
8831   ins_cost(300); // XXX
8832   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8833             "cmpq    rax, rdx\n\t"
8834             "jne,s   normal\n\t"
8835             "xorl    rdx, rdx\n\t"
8836             "cmpq    $div, -1\n\t"
8837             "je,s    done\n"
8838     "normal: cdqq\n\t"
8839             "idivq   $div\n"
8840     "done:"        %}
8841   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8842   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8843   ins_pipe(ialu_reg_reg_alu0);
8844 %}
8845 
8846 // Integer Shift Instructions
8847 // Shift Left by one
8848 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8849 %{
8850   match(Set dst (LShiftI dst shift));
8851   effect(KILL cr);
8852 
8853   format %{ "sall    $dst, $shift" %}
8854   opcode(0xD1, 0x4); /* D1 /4 */
8855   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8856   ins_pipe(ialu_reg);
8857 %}
8858 
8859 // Shift Left by one
8860 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8861 %{
8862   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8863   effect(KILL cr);
8864 
8865   format %{ "sall    $dst, $shift\t" %}
8866   opcode(0xD1, 0x4); /* D1 /4 */
8867   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8868   ins_pipe(ialu_mem_imm);
8869 %}
8870 
8871 // Shift Left by 8-bit immediate
8872 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8873 %{
8874   match(Set dst (LShiftI dst shift));
8875   effect(KILL cr);
8876 
8877   format %{ "sall    $dst, $shift" %}
8878   opcode(0xC1, 0x4); /* C1 /4 ib */
8879   ins_encode(reg_opc_imm(dst, shift));
8880   ins_pipe(ialu_reg);
8881 %}
8882 
8883 // Shift Left by 8-bit immediate
8884 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8885 %{
8886   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8887   effect(KILL cr);
8888 
8889   format %{ "sall    $dst, $shift" %}
8890   opcode(0xC1, 0x4); /* C1 /4 ib */
8891   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8892   ins_pipe(ialu_mem_imm);
8893 %}
8894 
8895 // Shift Left by variable
8896 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8897 %{
8898   match(Set dst (LShiftI dst shift));
8899   effect(KILL cr);
8900 
8901   format %{ "sall    $dst, $shift" %}
8902   opcode(0xD3, 0x4); /* D3 /4 */
8903   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8904   ins_pipe(ialu_reg_reg);
8905 %}
8906 
8907 // Shift Left by variable
8908 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8909 %{
8910   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8911   effect(KILL cr);
8912 
8913   format %{ "sall    $dst, $shift" %}
8914   opcode(0xD3, 0x4); /* D3 /4 */
8915   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8916   ins_pipe(ialu_mem_reg);
8917 %}
8918 
8919 // Arithmetic shift right by one
8920 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8921 %{
8922   match(Set dst (RShiftI dst shift));
8923   effect(KILL cr);
8924 
8925   format %{ "sarl    $dst, $shift" %}
8926   opcode(0xD1, 0x7); /* D1 /7 */
8927   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8928   ins_pipe(ialu_reg);
8929 %}
8930 
8931 // Arithmetic shift right by one
8932 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8933 %{
8934   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8935   effect(KILL cr);
8936 
8937   format %{ "sarl    $dst, $shift" %}
8938   opcode(0xD1, 0x7); /* D1 /7 */
8939   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8940   ins_pipe(ialu_mem_imm);
8941 %}
8942 
8943 // Arithmetic Shift Right by 8-bit immediate
8944 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8945 %{
8946   match(Set dst (RShiftI dst shift));
8947   effect(KILL cr);
8948 
8949   format %{ "sarl    $dst, $shift" %}
8950   opcode(0xC1, 0x7); /* C1 /7 ib */
8951   ins_encode(reg_opc_imm(dst, shift));
8952   ins_pipe(ialu_mem_imm);
8953 %}
8954 
8955 // Arithmetic Shift Right by 8-bit immediate
8956 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8957 %{
8958   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8959   effect(KILL cr);
8960 
8961   format %{ "sarl    $dst, $shift" %}
8962   opcode(0xC1, 0x7); /* C1 /7 ib */
8963   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8964   ins_pipe(ialu_mem_imm);
8965 %}
8966 
8967 // Arithmetic Shift Right by variable
8968 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8969 %{
8970   match(Set dst (RShiftI dst shift));
8971   effect(KILL cr);
8972 
8973   format %{ "sarl    $dst, $shift" %}
8974   opcode(0xD3, 0x7); /* D3 /7 */
8975   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8976   ins_pipe(ialu_reg_reg);
8977 %}
8978 
8979 // Arithmetic Shift Right by variable
8980 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8981 %{
8982   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8983   effect(KILL cr);
8984 
8985   format %{ "sarl    $dst, $shift" %}
8986   opcode(0xD3, 0x7); /* D3 /7 */
8987   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8988   ins_pipe(ialu_mem_reg);
8989 %}
8990 
8991 // Logical shift right by one
8992 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8993 %{
8994   match(Set dst (URShiftI dst shift));
8995   effect(KILL cr);
8996 
8997   format %{ "shrl    $dst, $shift" %}
8998   opcode(0xD1, 0x5); /* D1 /5 */
8999   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9000   ins_pipe(ialu_reg);
9001 %}
9002 
9003 // Logical shift right by one
9004 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9005 %{
9006   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
9007   effect(KILL cr);
9008 
9009   format %{ "shrl    $dst, $shift" %}
9010   opcode(0xD1, 0x5); /* D1 /5 */
9011   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
9012   ins_pipe(ialu_mem_imm);
9013 %}
9014 
9015 // Logical Shift Right by 8-bit immediate
9016 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
9017 %{
9018   match(Set dst (URShiftI dst shift));
9019   effect(KILL cr);
9020 
9021   format %{ "shrl    $dst, $shift" %}
9022   opcode(0xC1, 0x5); /* C1 /5 ib */
9023   ins_encode(reg_opc_imm(dst, shift));
9024   ins_pipe(ialu_reg);
9025 %}
9026 
9027 // Logical Shift Right by 8-bit immediate
9028 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9029 %{
9030   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
9031   effect(KILL cr);
9032 
9033   format %{ "shrl    $dst, $shift" %}
9034   opcode(0xC1, 0x5); /* C1 /5 ib */
9035   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
9036   ins_pipe(ialu_mem_imm);
9037 %}
9038 
9039 // Logical Shift Right by variable
9040 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
9041 %{
9042   match(Set dst (URShiftI dst shift));
9043   effect(KILL cr);
9044 
9045   format %{ "shrl    $dst, $shift" %}
9046   opcode(0xD3, 0x5); /* D3 /5 */
9047   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9048   ins_pipe(ialu_reg_reg);
9049 %}
9050 
9051 // Logical Shift Right by variable
9052 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9053 %{
9054   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
9055   effect(KILL cr);
9056 
9057   format %{ "shrl    $dst, $shift" %}
9058   opcode(0xD3, 0x5); /* D3 /5 */
9059   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
9060   ins_pipe(ialu_mem_reg);
9061 %}
9062 
9063 // Long Shift Instructions
9064 // Shift Left by one
9065 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9066 %{
9067   match(Set dst (LShiftL dst shift));
9068   effect(KILL cr);
9069 
9070   format %{ "salq    $dst, $shift" %}
9071   opcode(0xD1, 0x4); /* D1 /4 */
9072   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9073   ins_pipe(ialu_reg);
9074 %}
9075 
9076 // Shift Left by one
9077 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9078 %{
9079   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9080   effect(KILL cr);
9081 
9082   format %{ "salq    $dst, $shift" %}
9083   opcode(0xD1, 0x4); /* D1 /4 */
9084   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9085   ins_pipe(ialu_mem_imm);
9086 %}
9087 
9088 // Shift Left by 8-bit immediate
9089 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9090 %{
9091   match(Set dst (LShiftL dst shift));
9092   effect(KILL cr);
9093 
9094   format %{ "salq    $dst, $shift" %}
9095   opcode(0xC1, 0x4); /* C1 /4 ib */
9096   ins_encode(reg_opc_imm_wide(dst, shift));
9097   ins_pipe(ialu_reg);
9098 %}
9099 
9100 // Shift Left by 8-bit immediate
9101 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9102 %{
9103   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9104   effect(KILL cr);
9105 
9106   format %{ "salq    $dst, $shift" %}
9107   opcode(0xC1, 0x4); /* C1 /4 ib */
9108   ins_encode(REX_mem_wide(dst), OpcP,
9109              RM_opc_mem(secondary, dst), Con8or32(shift));
9110   ins_pipe(ialu_mem_imm);
9111 %}
9112 
9113 // Shift Left by variable
9114 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9115 %{
9116   match(Set dst (LShiftL dst shift));
9117   effect(KILL cr);
9118 
9119   format %{ "salq    $dst, $shift" %}
9120   opcode(0xD3, 0x4); /* D3 /4 */
9121   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9122   ins_pipe(ialu_reg_reg);
9123 %}
9124 
9125 // Shift Left by variable
9126 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9127 %{
9128   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
9129   effect(KILL cr);
9130 
9131   format %{ "salq    $dst, $shift" %}
9132   opcode(0xD3, 0x4); /* D3 /4 */
9133   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9134   ins_pipe(ialu_mem_reg);
9135 %}
9136 
9137 // Arithmetic shift right by one
9138 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9139 %{
9140   match(Set dst (RShiftL dst shift));
9141   effect(KILL cr);
9142 
9143   format %{ "sarq    $dst, $shift" %}
9144   opcode(0xD1, 0x7); /* D1 /7 */
9145   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9146   ins_pipe(ialu_reg);
9147 %}
9148 
9149 // Arithmetic shift right by one
9150 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9151 %{
9152   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9153   effect(KILL cr);
9154 
9155   format %{ "sarq    $dst, $shift" %}
9156   opcode(0xD1, 0x7); /* D1 /7 */
9157   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9158   ins_pipe(ialu_mem_imm);
9159 %}
9160 
9161 // Arithmetic Shift Right by 8-bit immediate
9162 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9163 %{
9164   match(Set dst (RShiftL dst shift));
9165   effect(KILL cr);
9166 
9167   format %{ "sarq    $dst, $shift" %}
9168   opcode(0xC1, 0x7); /* C1 /7 ib */
9169   ins_encode(reg_opc_imm_wide(dst, shift));
9170   ins_pipe(ialu_mem_imm);
9171 %}
9172 
9173 // Arithmetic Shift Right by 8-bit immediate
9174 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9175 %{
9176   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9177   effect(KILL cr);
9178 
9179   format %{ "sarq    $dst, $shift" %}
9180   opcode(0xC1, 0x7); /* C1 /7 ib */
9181   ins_encode(REX_mem_wide(dst), OpcP,
9182              RM_opc_mem(secondary, dst), Con8or32(shift));
9183   ins_pipe(ialu_mem_imm);
9184 %}
9185 
9186 // Arithmetic Shift Right by variable
9187 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9188 %{
9189   match(Set dst (RShiftL dst shift));
9190   effect(KILL cr);
9191 
9192   format %{ "sarq    $dst, $shift" %}
9193   opcode(0xD3, 0x7); /* D3 /7 */
9194   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9195   ins_pipe(ialu_reg_reg);
9196 %}
9197 
9198 // Arithmetic Shift Right by variable
9199 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9200 %{
9201   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9202   effect(KILL cr);
9203 
9204   format %{ "sarq    $dst, $shift" %}
9205   opcode(0xD3, 0x7); /* D3 /7 */
9206   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9207   ins_pipe(ialu_mem_reg);
9208 %}
9209 
9210 // Logical shift right by one
9211 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9212 %{
9213   match(Set dst (URShiftL dst shift));
9214   effect(KILL cr);
9215 
9216   format %{ "shrq    $dst, $shift" %}
9217   opcode(0xD1, 0x5); /* D1 /5 */
9218   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9219   ins_pipe(ialu_reg);
9220 %}
9221 
9222 // Logical shift right by one
9223 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9224 %{
9225   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9226   effect(KILL cr);
9227 
9228   format %{ "shrq    $dst, $shift" %}
9229   opcode(0xD1, 0x5); /* D1 /5 */
9230   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9231   ins_pipe(ialu_mem_imm);
9232 %}
9233 
9234 // Logical Shift Right by 8-bit immediate
9235 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9236 %{
9237   match(Set dst (URShiftL dst shift));
9238   effect(KILL cr);
9239 
9240   format %{ "shrq    $dst, $shift" %}
9241   opcode(0xC1, 0x5); /* C1 /5 ib */
9242   ins_encode(reg_opc_imm_wide(dst, shift));
9243   ins_pipe(ialu_reg);
9244 %}
9245 
9246 
9247 // Logical Shift Right by 8-bit immediate
9248 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9249 %{
9250   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9251   effect(KILL cr);
9252 
9253   format %{ "shrq    $dst, $shift" %}
9254   opcode(0xC1, 0x5); /* C1 /5 ib */
9255   ins_encode(REX_mem_wide(dst), OpcP,
9256              RM_opc_mem(secondary, dst), Con8or32(shift));
9257   ins_pipe(ialu_mem_imm);
9258 %}
9259 
9260 // Logical Shift Right by variable
9261 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9262 %{
9263   match(Set dst (URShiftL dst shift));
9264   effect(KILL cr);
9265 
9266   format %{ "shrq    $dst, $shift" %}
9267   opcode(0xD3, 0x5); /* D3 /5 */
9268   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9269   ins_pipe(ialu_reg_reg);
9270 %}
9271 
9272 // Logical Shift Right by variable
9273 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9274 %{
9275   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9276   effect(KILL cr);
9277 
9278   format %{ "shrq    $dst, $shift" %}
9279   opcode(0xD3, 0x5); /* D3 /5 */
9280   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9281   ins_pipe(ialu_mem_reg);
9282 %}
9283 
9284 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9285 // This idiom is used by the compiler for the i2b bytecode.
9286 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9287 %{
9288   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9289 
9290   format %{ "movsbl  $dst, $src\t# i2b" %}
9291   opcode(0x0F, 0xBE);
9292   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9293   ins_pipe(ialu_reg_reg);
9294 %}
9295 
9296 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9297 // This idiom is used by the compiler the i2s bytecode.
9298 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9299 %{
9300   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9301 
9302   format %{ "movswl  $dst, $src\t# i2s" %}
9303   opcode(0x0F, 0xBF);
9304   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9305   ins_pipe(ialu_reg_reg);
9306 %}
9307 
9308 // ROL/ROR instructions
9309 
9310 // ROL expand
9311 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9312   effect(KILL cr, USE_DEF dst);
9313 
9314   format %{ "roll    $dst" %}
9315   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9316   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9317   ins_pipe(ialu_reg);
9318 %}
9319 
9320 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9321   effect(USE_DEF dst, USE shift, KILL cr);
9322 
9323   format %{ "roll    $dst, $shift" %}
9324   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9325   ins_encode( reg_opc_imm(dst, shift) );
9326   ins_pipe(ialu_reg);
9327 %}
9328 
9329 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9330 %{
9331   effect(USE_DEF dst, USE shift, KILL cr);
9332 
9333   format %{ "roll    $dst, $shift" %}
9334   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9335   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9336   ins_pipe(ialu_reg_reg);
9337 %}
9338 // end of ROL expand
9339 
9340 // Rotate Left by one
9341 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9342 %{
9343   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9344 
9345   expand %{
9346     rolI_rReg_imm1(dst, cr);
9347   %}
9348 %}
9349 
9350 // Rotate Left by 8-bit immediate
9351 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9352 %{
9353   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9354   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9355 
9356   expand %{
9357     rolI_rReg_imm8(dst, lshift, cr);
9358   %}
9359 %}
9360 
9361 // Rotate Left by variable
9362 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9363 %{
9364   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9365 
9366   expand %{
9367     rolI_rReg_CL(dst, shift, cr);
9368   %}
9369 %}
9370 
9371 // Rotate Left by variable
9372 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9373 %{
9374   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9375 
9376   expand %{
9377     rolI_rReg_CL(dst, shift, cr);
9378   %}
9379 %}
9380 
9381 // ROR expand
9382 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9383 %{
9384   effect(USE_DEF dst, KILL cr);
9385 
9386   format %{ "rorl    $dst" %}
9387   opcode(0xD1, 0x1); /* D1 /1 */
9388   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9389   ins_pipe(ialu_reg);
9390 %}
9391 
9392 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9393 %{
9394   effect(USE_DEF dst, USE shift, KILL cr);
9395 
9396   format %{ "rorl    $dst, $shift" %}
9397   opcode(0xC1, 0x1); /* C1 /1 ib */
9398   ins_encode(reg_opc_imm(dst, shift));
9399   ins_pipe(ialu_reg);
9400 %}
9401 
9402 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9403 %{
9404   effect(USE_DEF dst, USE shift, KILL cr);
9405 
9406   format %{ "rorl    $dst, $shift" %}
9407   opcode(0xD3, 0x1); /* D3 /1 */
9408   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9409   ins_pipe(ialu_reg_reg);
9410 %}
9411 // end of ROR expand
9412 
9413 // Rotate Right by one
9414 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9415 %{
9416   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9417 
9418   expand %{
9419     rorI_rReg_imm1(dst, cr);
9420   %}
9421 %}
9422 
9423 // Rotate Right by 8-bit immediate
9424 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9425 %{
9426   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9427   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9428 
9429   expand %{
9430     rorI_rReg_imm8(dst, rshift, cr);
9431   %}
9432 %}
9433 
9434 // Rotate Right by variable
9435 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9436 %{
9437   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9438 
9439   expand %{
9440     rorI_rReg_CL(dst, shift, cr);
9441   %}
9442 %}
9443 
9444 // Rotate Right by variable
9445 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9446 %{
9447   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9448 
9449   expand %{
9450     rorI_rReg_CL(dst, shift, cr);
9451   %}
9452 %}
9453 
9454 // for long rotate
9455 // ROL expand
9456 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9457   effect(USE_DEF dst, KILL cr);
9458 
9459   format %{ "rolq    $dst" %}
9460   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9461   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9462   ins_pipe(ialu_reg);
9463 %}
9464 
9465 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9466   effect(USE_DEF dst, USE shift, KILL cr);
9467 
9468   format %{ "rolq    $dst, $shift" %}
9469   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9470   ins_encode( reg_opc_imm_wide(dst, shift) );
9471   ins_pipe(ialu_reg);
9472 %}
9473 
9474 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9475 %{
9476   effect(USE_DEF dst, USE shift, KILL cr);
9477 
9478   format %{ "rolq    $dst, $shift" %}
9479   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9480   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9481   ins_pipe(ialu_reg_reg);
9482 %}
9483 // end of ROL expand
9484 
9485 // Rotate Left by one
9486 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9487 %{
9488   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9489 
9490   expand %{
9491     rolL_rReg_imm1(dst, cr);
9492   %}
9493 %}
9494 
9495 // Rotate Left by 8-bit immediate
9496 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9497 %{
9498   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9499   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9500 
9501   expand %{
9502     rolL_rReg_imm8(dst, lshift, cr);
9503   %}
9504 %}
9505 
9506 // Rotate Left by variable
9507 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9508 %{
9509   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9510 
9511   expand %{
9512     rolL_rReg_CL(dst, shift, cr);
9513   %}
9514 %}
9515 
9516 // Rotate Left by variable
9517 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9518 %{
9519   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9520 
9521   expand %{
9522     rolL_rReg_CL(dst, shift, cr);
9523   %}
9524 %}
9525 
9526 // ROR expand
9527 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9528 %{
9529   effect(USE_DEF dst, KILL cr);
9530 
9531   format %{ "rorq    $dst" %}
9532   opcode(0xD1, 0x1); /* D1 /1 */
9533   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9534   ins_pipe(ialu_reg);
9535 %}
9536 
9537 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9538 %{
9539   effect(USE_DEF dst, USE shift, KILL cr);
9540 
9541   format %{ "rorq    $dst, $shift" %}
9542   opcode(0xC1, 0x1); /* C1 /1 ib */
9543   ins_encode(reg_opc_imm_wide(dst, shift));
9544   ins_pipe(ialu_reg);
9545 %}
9546 
9547 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9548 %{
9549   effect(USE_DEF dst, USE shift, KILL cr);
9550 
9551   format %{ "rorq    $dst, $shift" %}
9552   opcode(0xD3, 0x1); /* D3 /1 */
9553   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9554   ins_pipe(ialu_reg_reg);
9555 %}
9556 // end of ROR expand
9557 
9558 // Rotate Right by one
9559 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9560 %{
9561   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9562 
9563   expand %{
9564     rorL_rReg_imm1(dst, cr);
9565   %}
9566 %}
9567 
9568 // Rotate Right by 8-bit immediate
9569 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9570 %{
9571   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9572   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9573 
9574   expand %{
9575     rorL_rReg_imm8(dst, rshift, cr);
9576   %}
9577 %}
9578 
9579 // Rotate Right by variable
9580 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9581 %{
9582   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9583 
9584   expand %{
9585     rorL_rReg_CL(dst, shift, cr);
9586   %}
9587 %}
9588 
9589 // Rotate Right by variable
9590 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9591 %{
9592   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9593 
9594   expand %{
9595     rorL_rReg_CL(dst, shift, cr);
9596   %}
9597 %}
9598 
9599 // Logical Instructions
9600 
9601 // Integer Logical Instructions
9602 
9603 // And Instructions
9604 // And Register with Register
9605 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9606 %{
9607   match(Set dst (AndI dst src));
9608   effect(KILL cr);
9609 
9610   format %{ "andl    $dst, $src\t# int" %}
9611   opcode(0x23);
9612   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9613   ins_pipe(ialu_reg_reg);
9614 %}
9615 
9616 // And Register with Immediate 255
9617 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9618 %{
9619   match(Set dst (AndI dst src));
9620 
9621   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9622   opcode(0x0F, 0xB6);
9623   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9624   ins_pipe(ialu_reg);
9625 %}
9626 
9627 // And Register with Immediate 255 and promote to long
9628 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9629 %{
9630   match(Set dst (ConvI2L (AndI src mask)));
9631 
9632   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9633   opcode(0x0F, 0xB6);
9634   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9635   ins_pipe(ialu_reg);
9636 %}
9637 
9638 // And Register with Immediate 65535
9639 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9640 %{
9641   match(Set dst (AndI dst src));
9642 
9643   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9644   opcode(0x0F, 0xB7);
9645   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9646   ins_pipe(ialu_reg);
9647 %}
9648 
9649 // And Register with Immediate 65535 and promote to long
9650 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9651 %{
9652   match(Set dst (ConvI2L (AndI src mask)));
9653 
9654   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9655   opcode(0x0F, 0xB7);
9656   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9657   ins_pipe(ialu_reg);
9658 %}
9659 
9660 // And Register with Immediate
9661 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9662 %{
9663   match(Set dst (AndI dst src));
9664   effect(KILL cr);
9665 
9666   format %{ "andl    $dst, $src\t# int" %}
9667   opcode(0x81, 0x04); /* Opcode 81 /4 */
9668   ins_encode(OpcSErm(dst, src), Con8or32(src));
9669   ins_pipe(ialu_reg);
9670 %}
9671 
9672 // And Register with Memory
9673 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9674 %{
9675   match(Set dst (AndI dst (LoadI src)));
9676   effect(KILL cr);
9677 
9678   ins_cost(125);
9679   format %{ "andl    $dst, $src\t# int" %}
9680   opcode(0x23);
9681   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9682   ins_pipe(ialu_reg_mem);
9683 %}
9684 
9685 // And Memory with Register
9686 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9687 %{
9688   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9689   effect(KILL cr);
9690 
9691   ins_cost(150);
9692   format %{ "andl    $dst, $src\t# int" %}
9693   opcode(0x21); /* Opcode 21 /r */
9694   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9695   ins_pipe(ialu_mem_reg);
9696 %}
9697 
9698 // And Memory with Immediate
9699 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9700 %{
9701   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9702   effect(KILL cr);
9703 
9704   ins_cost(125);
9705   format %{ "andl    $dst, $src\t# int" %}
9706   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9707   ins_encode(REX_mem(dst), OpcSE(src),
9708              RM_opc_mem(secondary, dst), Con8or32(src));
9709   ins_pipe(ialu_mem_imm);
9710 %}
9711 
9712 // Or Instructions
9713 // Or Register with Register
9714 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9715 %{
9716   match(Set dst (OrI dst src));
9717   effect(KILL cr);
9718 
9719   format %{ "orl     $dst, $src\t# int" %}
9720   opcode(0x0B);
9721   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9722   ins_pipe(ialu_reg_reg);
9723 %}
9724 
9725 // Or Register with Immediate
9726 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9727 %{
9728   match(Set dst (OrI dst src));
9729   effect(KILL cr);
9730 
9731   format %{ "orl     $dst, $src\t# int" %}
9732   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9733   ins_encode(OpcSErm(dst, src), Con8or32(src));
9734   ins_pipe(ialu_reg);
9735 %}
9736 
9737 // Or Register with Memory
9738 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9739 %{
9740   match(Set dst (OrI dst (LoadI src)));
9741   effect(KILL cr);
9742 
9743   ins_cost(125);
9744   format %{ "orl     $dst, $src\t# int" %}
9745   opcode(0x0B);
9746   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9747   ins_pipe(ialu_reg_mem);
9748 %}
9749 
9750 // Or Memory with Register
9751 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9752 %{
9753   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9754   effect(KILL cr);
9755 
9756   ins_cost(150);
9757   format %{ "orl     $dst, $src\t# int" %}
9758   opcode(0x09); /* Opcode 09 /r */
9759   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9760   ins_pipe(ialu_mem_reg);
9761 %}
9762 
9763 // Or Memory with Immediate
9764 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9765 %{
9766   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9767   effect(KILL cr);
9768 
9769   ins_cost(125);
9770   format %{ "orl     $dst, $src\t# int" %}
9771   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9772   ins_encode(REX_mem(dst), OpcSE(src),
9773              RM_opc_mem(secondary, dst), Con8or32(src));
9774   ins_pipe(ialu_mem_imm);
9775 %}
9776 
9777 // Xor Instructions
9778 // Xor Register with Register
9779 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9780 %{
9781   match(Set dst (XorI dst src));
9782   effect(KILL cr);
9783 
9784   format %{ "xorl    $dst, $src\t# int" %}
9785   opcode(0x33);
9786   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9787   ins_pipe(ialu_reg_reg);
9788 %}
9789 
9790 // Xor Register with Immediate -1
9791 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9792   match(Set dst (XorI dst imm));
9793 
9794   format %{ "not    $dst" %}
9795   ins_encode %{
9796      __ notl($dst$$Register);
9797   %}
9798   ins_pipe(ialu_reg);
9799 %}
9800 
9801 // Xor Register with Immediate
9802 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9803 %{
9804   match(Set dst (XorI dst src));
9805   effect(KILL cr);
9806 
9807   format %{ "xorl    $dst, $src\t# int" %}
9808   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9809   ins_encode(OpcSErm(dst, src), Con8or32(src));
9810   ins_pipe(ialu_reg);
9811 %}
9812 
9813 // Xor Register with Memory
9814 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9815 %{
9816   match(Set dst (XorI dst (LoadI src)));
9817   effect(KILL cr);
9818 
9819   ins_cost(125);
9820   format %{ "xorl    $dst, $src\t# int" %}
9821   opcode(0x33);
9822   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9823   ins_pipe(ialu_reg_mem);
9824 %}
9825 
9826 // Xor Memory with Register
9827 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9828 %{
9829   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9830   effect(KILL cr);
9831 
9832   ins_cost(150);
9833   format %{ "xorl    $dst, $src\t# int" %}
9834   opcode(0x31); /* Opcode 31 /r */
9835   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9836   ins_pipe(ialu_mem_reg);
9837 %}
9838 
9839 // Xor Memory with Immediate
9840 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9841 %{
9842   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9843   effect(KILL cr);
9844 
9845   ins_cost(125);
9846   format %{ "xorl    $dst, $src\t# int" %}
9847   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9848   ins_encode(REX_mem(dst), OpcSE(src),
9849              RM_opc_mem(secondary, dst), Con8or32(src));
9850   ins_pipe(ialu_mem_imm);
9851 %}
9852 
9853 
9854 // Long Logical Instructions
9855 
9856 // And Instructions
9857 // And Register with Register
9858 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9859 %{
9860   match(Set dst (AndL dst src));
9861   effect(KILL cr);
9862 
9863   format %{ "andq    $dst, $src\t# long" %}
9864   opcode(0x23);
9865   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9866   ins_pipe(ialu_reg_reg);
9867 %}
9868 
9869 // And Register with Immediate 255
9870 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9871 %{
9872   match(Set dst (AndL dst src));
9873 
9874   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9875   opcode(0x0F, 0xB6);
9876   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9877   ins_pipe(ialu_reg);
9878 %}
9879 
9880 // And Register with Immediate 65535
9881 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9882 %{
9883   match(Set dst (AndL dst src));
9884 
9885   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9886   opcode(0x0F, 0xB7);
9887   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9888   ins_pipe(ialu_reg);
9889 %}
9890 
9891 // And Register with Immediate
9892 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9893 %{
9894   match(Set dst (AndL dst src));
9895   effect(KILL cr);
9896 
9897   format %{ "andq    $dst, $src\t# long" %}
9898   opcode(0x81, 0x04); /* Opcode 81 /4 */
9899   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9900   ins_pipe(ialu_reg);
9901 %}
9902 
9903 // And Register with Memory
9904 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9905 %{
9906   match(Set dst (AndL dst (LoadL src)));
9907   effect(KILL cr);
9908 
9909   ins_cost(125);
9910   format %{ "andq    $dst, $src\t# long" %}
9911   opcode(0x23);
9912   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9913   ins_pipe(ialu_reg_mem);
9914 %}
9915 
9916 // And Memory with Register
9917 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9918 %{
9919   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9920   effect(KILL cr);
9921 
9922   ins_cost(150);
9923   format %{ "andq    $dst, $src\t# long" %}
9924   opcode(0x21); /* Opcode 21 /r */
9925   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9926   ins_pipe(ialu_mem_reg);
9927 %}
9928 
9929 // And Memory with Immediate
9930 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9931 %{
9932   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9933   effect(KILL cr);
9934 
9935   ins_cost(125);
9936   format %{ "andq    $dst, $src\t# long" %}
9937   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9938   ins_encode(REX_mem_wide(dst), OpcSE(src),
9939              RM_opc_mem(secondary, dst), Con8or32(src));
9940   ins_pipe(ialu_mem_imm);
9941 %}
9942 
9943 // Or Instructions
9944 // Or Register with Register
9945 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9946 %{
9947   match(Set dst (OrL dst src));
9948   effect(KILL cr);
9949 
9950   format %{ "orq     $dst, $src\t# long" %}
9951   opcode(0x0B);
9952   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9953   ins_pipe(ialu_reg_reg);
9954 %}
9955 
9956 // Use any_RegP to match R15 (TLS register) without spilling.
9957 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9958   match(Set dst (OrL dst (CastP2X src)));
9959   effect(KILL cr);
9960 
9961   format %{ "orq     $dst, $src\t# long" %}
9962   opcode(0x0B);
9963   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9964   ins_pipe(ialu_reg_reg);
9965 %}
9966 
9967 
9968 // Or Register with Immediate
9969 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9970 %{
9971   match(Set dst (OrL dst src));
9972   effect(KILL cr);
9973 
9974   format %{ "orq     $dst, $src\t# long" %}
9975   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9976   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9977   ins_pipe(ialu_reg);
9978 %}
9979 
9980 // Or Register with Memory
9981 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9982 %{
9983   match(Set dst (OrL dst (LoadL src)));
9984   effect(KILL cr);
9985 
9986   ins_cost(125);
9987   format %{ "orq     $dst, $src\t# long" %}
9988   opcode(0x0B);
9989   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9990   ins_pipe(ialu_reg_mem);
9991 %}
9992 
9993 // Or Memory with Register
9994 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9995 %{
9996   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9997   effect(KILL cr);
9998 
9999   ins_cost(150);
10000   format %{ "orq     $dst, $src\t# long" %}
10001   opcode(0x09); /* Opcode 09 /r */
10002   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10003   ins_pipe(ialu_mem_reg);
10004 %}
10005 
10006 // Or Memory with Immediate
10007 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10008 %{
10009   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10010   effect(KILL cr);
10011 
10012   ins_cost(125);
10013   format %{ "orq     $dst, $src\t# long" %}
10014   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10015   ins_encode(REX_mem_wide(dst), OpcSE(src),
10016              RM_opc_mem(secondary, dst), Con8or32(src));
10017   ins_pipe(ialu_mem_imm);
10018 %}
10019 
10020 // Xor Instructions
10021 // Xor Register with Register
10022 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10023 %{
10024   match(Set dst (XorL dst src));
10025   effect(KILL cr);
10026 
10027   format %{ "xorq    $dst, $src\t# long" %}
10028   opcode(0x33);
10029   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10030   ins_pipe(ialu_reg_reg);
10031 %}
10032 
10033 // Xor Register with Immediate -1
10034 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10035   match(Set dst (XorL dst imm));
10036 
10037   format %{ "notq   $dst" %}
10038   ins_encode %{
10039      __ notq($dst$$Register);
10040   %}
10041   ins_pipe(ialu_reg);
10042 %}
10043 
10044 // Xor Register with Immediate
10045 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10046 %{
10047   match(Set dst (XorL dst src));
10048   effect(KILL cr);
10049 
10050   format %{ "xorq    $dst, $src\t# long" %}
10051   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10052   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10053   ins_pipe(ialu_reg);
10054 %}
10055 
10056 // Xor Register with Memory
10057 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10058 %{
10059   match(Set dst (XorL dst (LoadL src)));
10060   effect(KILL cr);
10061 
10062   ins_cost(125);
10063   format %{ "xorq    $dst, $src\t# long" %}
10064   opcode(0x33);
10065   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10066   ins_pipe(ialu_reg_mem);
10067 %}
10068 
10069 // Xor Memory with Register
10070 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10071 %{
10072   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10073   effect(KILL cr);
10074 
10075   ins_cost(150);
10076   format %{ "xorq    $dst, $src\t# long" %}
10077   opcode(0x31); /* Opcode 31 /r */
10078   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10079   ins_pipe(ialu_mem_reg);
10080 %}
10081 
10082 // Xor Memory with Immediate
10083 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10084 %{
10085   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10086   effect(KILL cr);
10087 
10088   ins_cost(125);
10089   format %{ "xorq    $dst, $src\t# long" %}
10090   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10091   ins_encode(REX_mem_wide(dst), OpcSE(src),
10092              RM_opc_mem(secondary, dst), Con8or32(src));
10093   ins_pipe(ialu_mem_imm);
10094 %}
10095 
10096 // Convert Int to Boolean
10097 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10098 %{
10099   match(Set dst (Conv2B src));
10100   effect(KILL cr);
10101 
10102   format %{ "testl   $src, $src\t# ci2b\n\t"
10103             "setnz   $dst\n\t"
10104             "movzbl  $dst, $dst" %}
10105   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10106              setNZ_reg(dst),
10107              REX_reg_breg(dst, dst), // movzbl
10108              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10109   ins_pipe(pipe_slow); // XXX
10110 %}
10111 
10112 // Convert Pointer to Boolean
10113 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10114 %{
10115   match(Set dst (Conv2B src));
10116   effect(KILL cr);
10117 
10118   format %{ "testq   $src, $src\t# cp2b\n\t"
10119             "setnz   $dst\n\t"
10120             "movzbl  $dst, $dst" %}
10121   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10122              setNZ_reg(dst),
10123              REX_reg_breg(dst, dst), // movzbl
10124              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10125   ins_pipe(pipe_slow); // XXX
10126 %}
10127 
10128 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10129 %{
10130   match(Set dst (CmpLTMask p q));
10131   effect(KILL cr);
10132 
10133   ins_cost(400); // XXX
10134   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10135             "setlt   $dst\n\t"
10136             "movzbl  $dst, $dst\n\t"
10137             "negl    $dst" %}
10138   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10139              setLT_reg(dst),
10140              REX_reg_breg(dst, dst), // movzbl
10141              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10142              neg_reg(dst));
10143   ins_pipe(pipe_slow);
10144 %}
10145 
10146 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10147 %{
10148   match(Set dst (CmpLTMask dst zero));
10149   effect(KILL cr);
10150 
10151   ins_cost(100); // XXX
10152   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10153   opcode(0xC1, 0x7);  /* C1 /7 ib */
10154   ins_encode(reg_opc_imm(dst, 0x1F));
10155   ins_pipe(ialu_reg);
10156 %}
10157 
10158 
10159 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rRegI tmp, rFlagsReg cr)
10160 %{
10161   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10162   effect(TEMP tmp, KILL cr);
10163 
10164   ins_cost(400); // XXX
10165   format %{ "subl    $p, $q\t# cadd_cmpLTMask1\n\t"
10166             "sbbl    $tmp, $tmp\n\t"
10167             "andl    $tmp, $y\n\t"
10168             "addl    $p, $tmp" %}
10169   ins_encode %{
10170     Register Rp = $p$$Register;
10171     Register Rq = $q$$Register;
10172     Register Ry = $y$$Register;
10173     Register Rt = $tmp$$Register;
10174     __ subl(Rp, Rq);
10175     __ sbbl(Rt, Rt);
10176     __ andl(Rt, Ry);
10177     __ addl(Rp, Rt);
10178   %}
10179   ins_pipe(pipe_cmplt);
10180 %}
10181 
10182 //---------- FP Instructions------------------------------------------------
10183 
10184 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10185 %{
10186   match(Set cr (CmpF src1 src2));
10187 
10188   ins_cost(145);
10189   format %{ "ucomiss $src1, $src2\n\t"
10190             "jnp,s   exit\n\t"
10191             "pushfq\t# saw NaN, set CF\n\t"
10192             "andq    [rsp], #0xffffff2b\n\t"
10193             "popfq\n"
10194     "exit:   nop\t# avoid branch to branch" %}
10195   opcode(0x0F, 0x2E);
10196   ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
10197              cmpfp_fixup);
10198   ins_pipe(pipe_slow);
10199 %}
10200 
10201 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10202   match(Set cr (CmpF src1 src2));
10203 
10204   ins_cost(145);
10205   format %{ "ucomiss $src1, $src2" %}
10206   ins_encode %{
10207     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10208   %}
10209   ins_pipe(pipe_slow);
10210 %}
10211 
10212 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10213 %{
10214   match(Set cr (CmpF src1 (LoadF src2)));
10215 
10216   ins_cost(145);
10217   format %{ "ucomiss $src1, $src2\n\t"
10218             "jnp,s   exit\n\t"
10219             "pushfq\t# saw NaN, set CF\n\t"
10220             "andq    [rsp], #0xffffff2b\n\t"
10221             "popfq\n"
10222     "exit:   nop\t# avoid branch to branch" %}
10223   opcode(0x0F, 0x2E);
10224   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
10225              cmpfp_fixup);
10226   ins_pipe(pipe_slow);
10227 %}
10228 
10229 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10230   match(Set cr (CmpF src1 (LoadF src2)));
10231 
10232   ins_cost(100);
10233   format %{ "ucomiss $src1, $src2" %}
10234   opcode(0x0F, 0x2E);
10235   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2));
10236   ins_pipe(pipe_slow);
10237 %}
10238 
10239 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10240   match(Set cr (CmpF src con));
10241 
10242   ins_cost(145);
10243   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10244             "jnp,s   exit\n\t"
10245             "pushfq\t# saw NaN, set CF\n\t"
10246             "andq    [rsp], #0xffffff2b\n\t"
10247             "popfq\n"
10248     "exit:   nop\t# avoid branch to branch" %}
10249   ins_encode %{
10250     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10251     emit_cmpfp_fixup(_masm);
10252   %}
10253   ins_pipe(pipe_slow);
10254 %}
10255 
10256 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10257   match(Set cr (CmpF src con));
10258   ins_cost(100);
10259   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10260   ins_encode %{
10261     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10262   %}
10263   ins_pipe(pipe_slow);
10264 %}
10265 
10266 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10267 %{
10268   match(Set cr (CmpD src1 src2));
10269 
10270   ins_cost(145);
10271   format %{ "ucomisd $src1, $src2\n\t"
10272             "jnp,s   exit\n\t"
10273             "pushfq\t# saw NaN, set CF\n\t"
10274             "andq    [rsp], #0xffffff2b\n\t"
10275             "popfq\n"
10276     "exit:   nop\t# avoid branch to branch" %}
10277   opcode(0x66, 0x0F, 0x2E);
10278   ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
10279              cmpfp_fixup);
10280   ins_pipe(pipe_slow);
10281 %}
10282 
10283 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10284   match(Set cr (CmpD src1 src2));
10285 
10286   ins_cost(100);
10287   format %{ "ucomisd $src1, $src2 test" %}
10288   ins_encode %{
10289     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10290   %}
10291   ins_pipe(pipe_slow);
10292 %}
10293 
10294 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10295 %{
10296   match(Set cr (CmpD src1 (LoadD src2)));
10297 
10298   ins_cost(145);
10299   format %{ "ucomisd $src1, $src2\n\t"
10300             "jnp,s   exit\n\t"
10301             "pushfq\t# saw NaN, set CF\n\t"
10302             "andq    [rsp], #0xffffff2b\n\t"
10303             "popfq\n"
10304     "exit:   nop\t# avoid branch to branch" %}
10305   opcode(0x66, 0x0F, 0x2E);
10306   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
10307              cmpfp_fixup);
10308   ins_pipe(pipe_slow);
10309 %}
10310 
10311 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10312   match(Set cr (CmpD src1 (LoadD src2)));
10313 
10314   ins_cost(100);
10315   format %{ "ucomisd $src1, $src2" %}
10316   opcode(0x66, 0x0F, 0x2E);
10317   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2));
10318   ins_pipe(pipe_slow);
10319 %}
10320 
10321 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10322   match(Set cr (CmpD src con));
10323 
10324   ins_cost(145);
10325   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10326             "jnp,s   exit\n\t"
10327             "pushfq\t# saw NaN, set CF\n\t"
10328             "andq    [rsp], #0xffffff2b\n\t"
10329             "popfq\n"
10330     "exit:   nop\t# avoid branch to branch" %}
10331   ins_encode %{
10332     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10333     emit_cmpfp_fixup(_masm);
10334   %}
10335   ins_pipe(pipe_slow);
10336 %}
10337 
10338 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10339   match(Set cr (CmpD src con));
10340   ins_cost(100);
10341   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10342   ins_encode %{
10343     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10344   %}
10345   ins_pipe(pipe_slow);
10346 %}
10347 
10348 // Compare into -1,0,1
10349 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10350 %{
10351   match(Set dst (CmpF3 src1 src2));
10352   effect(KILL cr);
10353 
10354   ins_cost(275);
10355   format %{ "ucomiss $src1, $src2\n\t"
10356             "movl    $dst, #-1\n\t"
10357             "jp,s    done\n\t"
10358             "jb,s    done\n\t"
10359             "setne   $dst\n\t"
10360             "movzbl  $dst, $dst\n"
10361     "done:" %}
10362 
10363   opcode(0x0F, 0x2E);
10364   ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
10365              cmpfp3(dst));
10366   ins_pipe(pipe_slow);
10367 %}
10368 
10369 // Compare into -1,0,1
10370 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10371 %{
10372   match(Set dst (CmpF3 src1 (LoadF src2)));
10373   effect(KILL cr);
10374 
10375   ins_cost(275);
10376   format %{ "ucomiss $src1, $src2\n\t"
10377             "movl    $dst, #-1\n\t"
10378             "jp,s    done\n\t"
10379             "jb,s    done\n\t"
10380             "setne   $dst\n\t"
10381             "movzbl  $dst, $dst\n"
10382     "done:" %}
10383 
10384   opcode(0x0F, 0x2E);
10385   ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
10386              cmpfp3(dst));
10387   ins_pipe(pipe_slow);
10388 %}
10389 
10390 // Compare into -1,0,1
10391 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10392   match(Set dst (CmpF3 src con));
10393   effect(KILL cr);
10394 
10395   ins_cost(275);
10396   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10397             "movl    $dst, #-1\n\t"
10398             "jp,s    done\n\t"
10399             "jb,s    done\n\t"
10400             "setne   $dst\n\t"
10401             "movzbl  $dst, $dst\n"
10402     "done:" %}
10403   ins_encode %{
10404     Label L_done;
10405     Register Rdst = $dst$$Register;
10406     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10407     __ movl(Rdst, -1);
10408     __ jcc(Assembler::parity, L_done);
10409     __ jcc(Assembler::below, L_done);
10410     __ setb(Assembler::notEqual, Rdst);
10411     __ movzbl(Rdst, Rdst);
10412     __ bind(L_done);
10413   %}
10414   ins_pipe(pipe_slow);
10415 %}
10416 
10417 // Compare into -1,0,1
10418 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10419 %{
10420   match(Set dst (CmpD3 src1 src2));
10421   effect(KILL cr);
10422 
10423   ins_cost(275);
10424   format %{ "ucomisd $src1, $src2\n\t"
10425             "movl    $dst, #-1\n\t"
10426             "jp,s    done\n\t"
10427             "jb,s    done\n\t"
10428             "setne   $dst\n\t"
10429             "movzbl  $dst, $dst\n"
10430     "done:" %}
10431 
10432   opcode(0x66, 0x0F, 0x2E);
10433   ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
10434              cmpfp3(dst));
10435   ins_pipe(pipe_slow);
10436 %}
10437 
10438 // Compare into -1,0,1
10439 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10440 %{
10441   match(Set dst (CmpD3 src1 (LoadD src2)));
10442   effect(KILL cr);
10443 
10444   ins_cost(275);
10445   format %{ "ucomisd $src1, $src2\n\t"
10446             "movl    $dst, #-1\n\t"
10447             "jp,s    done\n\t"
10448             "jb,s    done\n\t"
10449             "setne   $dst\n\t"
10450             "movzbl  $dst, $dst\n"
10451     "done:" %}
10452 
10453   opcode(0x66, 0x0F, 0x2E);
10454   ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
10455              cmpfp3(dst));
10456   ins_pipe(pipe_slow);
10457 %}
10458 
10459 // Compare into -1,0,1
10460 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10461   match(Set dst (CmpD3 src con));
10462   effect(KILL cr);
10463 
10464   ins_cost(275);
10465   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10466             "movl    $dst, #-1\n\t"
10467             "jp,s    done\n\t"
10468             "jb,s    done\n\t"
10469             "setne   $dst\n\t"
10470             "movzbl  $dst, $dst\n"
10471     "done:" %}
10472   ins_encode %{
10473     Register Rdst = $dst$$Register;
10474     Label L_done;
10475     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10476     __ movl(Rdst, -1);
10477     __ jcc(Assembler::parity, L_done);
10478     __ jcc(Assembler::below, L_done);
10479     __ setb(Assembler::notEqual, Rdst);
10480     __ movzbl(Rdst, Rdst);
10481     __ bind(L_done);
10482   %}
10483   ins_pipe(pipe_slow);
10484 %}
10485 
10486 instruct addF_reg(regF dst, regF src)
10487 %{
10488   match(Set dst (AddF dst src));
10489 
10490   format %{ "addss   $dst, $src" %}
10491   ins_cost(150); // XXX
10492   opcode(0xF3, 0x0F, 0x58);
10493   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10494   ins_pipe(pipe_slow);
10495 %}
10496 
10497 instruct addF_mem(regF dst, memory src)
10498 %{
10499   match(Set dst (AddF dst (LoadF src)));
10500 
10501   format %{ "addss   $dst, $src" %}
10502   ins_cost(150); // XXX
10503   opcode(0xF3, 0x0F, 0x58);
10504   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10505   ins_pipe(pipe_slow);
10506 %}
10507 
10508 instruct addF_imm(regF dst, immF con) %{
10509   match(Set dst (AddF dst con));
10510   format %{ "addss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
10511   ins_cost(150); // XXX
10512   ins_encode %{
10513     __ addss($dst$$XMMRegister, $constantaddress($con));
10514   %}
10515   ins_pipe(pipe_slow);
10516 %}
10517 
10518 instruct addD_reg(regD dst, regD src)
10519 %{
10520   match(Set dst (AddD dst src));
10521 
10522   format %{ "addsd   $dst, $src" %}
10523   ins_cost(150); // XXX
10524   opcode(0xF2, 0x0F, 0x58);
10525   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10526   ins_pipe(pipe_slow);
10527 %}
10528 
10529 instruct addD_mem(regD dst, memory src)
10530 %{
10531   match(Set dst (AddD dst (LoadD src)));
10532 
10533   format %{ "addsd   $dst, $src" %}
10534   ins_cost(150); // XXX
10535   opcode(0xF2, 0x0F, 0x58);
10536   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10537   ins_pipe(pipe_slow);
10538 %}
10539 
10540 instruct addD_imm(regD dst, immD con) %{
10541   match(Set dst (AddD dst con));
10542   format %{ "addsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
10543   ins_cost(150); // XXX
10544   ins_encode %{
10545     __ addsd($dst$$XMMRegister, $constantaddress($con));
10546   %}
10547   ins_pipe(pipe_slow);
10548 %}
10549 
10550 instruct subF_reg(regF dst, regF src)
10551 %{
10552   match(Set dst (SubF dst src));
10553 
10554   format %{ "subss   $dst, $src" %}
10555   ins_cost(150); // XXX
10556   opcode(0xF3, 0x0F, 0x5C);
10557   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10558   ins_pipe(pipe_slow);
10559 %}
10560 
10561 instruct subF_mem(regF dst, memory src)
10562 %{
10563   match(Set dst (SubF dst (LoadF src)));
10564 
10565   format %{ "subss   $dst, $src" %}
10566   ins_cost(150); // XXX
10567   opcode(0xF3, 0x0F, 0x5C);
10568   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10569   ins_pipe(pipe_slow);
10570 %}
10571 
10572 instruct subF_imm(regF dst, immF con) %{
10573   match(Set dst (SubF dst con));
10574   format %{ "subss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
10575   ins_cost(150); // XXX
10576   ins_encode %{
10577     __ subss($dst$$XMMRegister, $constantaddress($con));
10578   %}
10579   ins_pipe(pipe_slow);
10580 %}
10581 
10582 instruct subD_reg(regD dst, regD src)
10583 %{
10584   match(Set dst (SubD dst src));
10585 
10586   format %{ "subsd   $dst, $src" %}
10587   ins_cost(150); // XXX
10588   opcode(0xF2, 0x0F, 0x5C);
10589   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10590   ins_pipe(pipe_slow);
10591 %}
10592 
10593 instruct subD_mem(regD dst, memory src)
10594 %{
10595   match(Set dst (SubD dst (LoadD src)));
10596 
10597   format %{ "subsd   $dst, $src" %}
10598   ins_cost(150); // XXX
10599   opcode(0xF2, 0x0F, 0x5C);
10600   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10601   ins_pipe(pipe_slow);
10602 %}
10603 
10604 instruct subD_imm(regD dst, immD con) %{
10605   match(Set dst (SubD dst con));
10606   format %{ "subsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
10607   ins_cost(150); // XXX
10608   ins_encode %{
10609     __ subsd($dst$$XMMRegister, $constantaddress($con));
10610   %}
10611   ins_pipe(pipe_slow);
10612 %}
10613 
10614 instruct mulF_reg(regF dst, regF src)
10615 %{
10616   match(Set dst (MulF dst src));
10617 
10618   format %{ "mulss   $dst, $src" %}
10619   ins_cost(150); // XXX
10620   opcode(0xF3, 0x0F, 0x59);
10621   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10622   ins_pipe(pipe_slow);
10623 %}
10624 
10625 instruct mulF_mem(regF dst, memory src)
10626 %{
10627   match(Set dst (MulF dst (LoadF src)));
10628 
10629   format %{ "mulss   $dst, $src" %}
10630   ins_cost(150); // XXX
10631   opcode(0xF3, 0x0F, 0x59);
10632   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10633   ins_pipe(pipe_slow);
10634 %}
10635 
10636 instruct mulF_imm(regF dst, immF con) %{
10637   match(Set dst (MulF dst con));
10638   format %{ "mulss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
10639   ins_cost(150); // XXX
10640   ins_encode %{
10641     __ mulss($dst$$XMMRegister, $constantaddress($con));
10642   %}
10643   ins_pipe(pipe_slow);
10644 %}
10645 
10646 instruct mulD_reg(regD dst, regD src)
10647 %{
10648   match(Set dst (MulD dst src));
10649 
10650   format %{ "mulsd   $dst, $src" %}
10651   ins_cost(150); // XXX
10652   opcode(0xF2, 0x0F, 0x59);
10653   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10654   ins_pipe(pipe_slow);
10655 %}
10656 
10657 instruct mulD_mem(regD dst, memory src)
10658 %{
10659   match(Set dst (MulD dst (LoadD src)));
10660 
10661   format %{ "mulsd   $dst, $src" %}
10662   ins_cost(150); // XXX
10663   opcode(0xF2, 0x0F, 0x59);
10664   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10665   ins_pipe(pipe_slow);
10666 %}
10667 
10668 instruct mulD_imm(regD dst, immD con) %{
10669   match(Set dst (MulD dst con));
10670   format %{ "mulsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
10671   ins_cost(150); // XXX
10672   ins_encode %{
10673     __ mulsd($dst$$XMMRegister, $constantaddress($con));
10674   %}
10675   ins_pipe(pipe_slow);
10676 %}
10677 
10678 instruct divF_reg(regF dst, regF src)
10679 %{
10680   match(Set dst (DivF dst src));
10681 
10682   format %{ "divss   $dst, $src" %}
10683   ins_cost(150); // XXX
10684   opcode(0xF3, 0x0F, 0x5E);
10685   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10686   ins_pipe(pipe_slow);
10687 %}
10688 
10689 instruct divF_mem(regF dst, memory src)
10690 %{
10691   match(Set dst (DivF dst (LoadF src)));
10692 
10693   format %{ "divss   $dst, $src" %}
10694   ins_cost(150); // XXX
10695   opcode(0xF3, 0x0F, 0x5E);
10696   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10697   ins_pipe(pipe_slow);
10698 %}
10699 
10700 instruct divF_imm(regF dst, immF con) %{
10701   match(Set dst (DivF dst con));
10702   format %{ "divss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
10703   ins_cost(150); // XXX
10704   ins_encode %{
10705     __ divss($dst$$XMMRegister, $constantaddress($con));
10706   %}
10707   ins_pipe(pipe_slow);
10708 %}
10709 
10710 instruct divD_reg(regD dst, regD src)
10711 %{
10712   match(Set dst (DivD dst src));
10713 
10714   format %{ "divsd   $dst, $src" %}
10715   ins_cost(150); // XXX
10716   opcode(0xF2, 0x0F, 0x5E);
10717   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10718   ins_pipe(pipe_slow);
10719 %}
10720 
10721 instruct divD_mem(regD dst, memory src)
10722 %{
10723   match(Set dst (DivD dst (LoadD src)));
10724 
10725   format %{ "divsd   $dst, $src" %}
10726   ins_cost(150); // XXX
10727   opcode(0xF2, 0x0F, 0x5E);
10728   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10729   ins_pipe(pipe_slow);
10730 %}
10731 
10732 instruct divD_imm(regD dst, immD con) %{
10733   match(Set dst (DivD dst con));
10734   format %{ "divsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
10735   ins_cost(150); // XXX
10736   ins_encode %{
10737     __ divsd($dst$$XMMRegister, $constantaddress($con));
10738   %}
10739   ins_pipe(pipe_slow);
10740 %}
10741 
10742 instruct sqrtF_reg(regF dst, regF src)
10743 %{
10744   match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
10745 
10746   format %{ "sqrtss  $dst, $src" %}
10747   ins_cost(150); // XXX
10748   opcode(0xF3, 0x0F, 0x51);
10749   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10750   ins_pipe(pipe_slow);
10751 %}
10752 
10753 instruct sqrtF_mem(regF dst, memory src)
10754 %{
10755   match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF src)))));
10756 
10757   format %{ "sqrtss  $dst, $src" %}
10758   ins_cost(150); // XXX
10759   opcode(0xF3, 0x0F, 0x51);
10760   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10761   ins_pipe(pipe_slow);
10762 %}
10763 
10764 instruct sqrtF_imm(regF dst, immF con) %{
10765   match(Set dst (ConvD2F (SqrtD (ConvF2D con))));
10766   format %{ "sqrtss  $dst, [$constantaddress]\t# load from constant table: float=$con" %}
10767   ins_cost(150); // XXX
10768   ins_encode %{
10769     __ sqrtss($dst$$XMMRegister, $constantaddress($con));
10770   %}
10771   ins_pipe(pipe_slow);
10772 %}
10773 
10774 instruct sqrtD_reg(regD dst, regD src)
10775 %{
10776   match(Set dst (SqrtD src));
10777 
10778   format %{ "sqrtsd  $dst, $src" %}
10779   ins_cost(150); // XXX
10780   opcode(0xF2, 0x0F, 0x51);
10781   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10782   ins_pipe(pipe_slow);
10783 %}
10784 
10785 instruct sqrtD_mem(regD dst, memory src)
10786 %{
10787   match(Set dst (SqrtD (LoadD src)));
10788 
10789   format %{ "sqrtsd  $dst, $src" %}
10790   ins_cost(150); // XXX
10791   opcode(0xF2, 0x0F, 0x51);
10792   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10793   ins_pipe(pipe_slow);
10794 %}
10795 
10796 instruct sqrtD_imm(regD dst, immD con) %{
10797   match(Set dst (SqrtD con));
10798   format %{ "sqrtsd  $dst, [$constantaddress]\t# load from constant table: double=$con" %}
10799   ins_cost(150); // XXX
10800   ins_encode %{
10801     __ sqrtsd($dst$$XMMRegister, $constantaddress($con));
10802   %}
10803   ins_pipe(pipe_slow);
10804 %}
10805 
10806 instruct absF_reg(regF dst)
10807 %{
10808   match(Set dst (AbsF dst));
10809 
10810   format %{ "andps   $dst, [0x7fffffff]\t# abs float by sign masking" %}
10811   ins_encode(absF_encoding(dst));
10812   ins_pipe(pipe_slow);
10813 %}
10814 
10815 instruct absD_reg(regD dst)
10816 %{
10817   match(Set dst (AbsD dst));
10818 
10819   format %{ "andpd   $dst, [0x7fffffffffffffff]\t"
10820             "# abs double by sign masking" %}
10821   ins_encode(absD_encoding(dst));
10822   ins_pipe(pipe_slow);
10823 %}
10824 
10825 instruct negF_reg(regF dst)
10826 %{
10827   match(Set dst (NegF dst));
10828 
10829   format %{ "xorps   $dst, [0x80000000]\t# neg float by sign flipping" %}
10830   ins_encode(negF_encoding(dst));
10831   ins_pipe(pipe_slow);
10832 %}
10833 
10834 instruct negD_reg(regD dst)
10835 %{
10836   match(Set dst (NegD dst));
10837 
10838   format %{ "xorpd   $dst, [0x8000000000000000]\t"
10839             "# neg double by sign flipping" %}
10840   ins_encode(negD_encoding(dst));
10841   ins_pipe(pipe_slow);
10842 %}
10843 
10844 // -----------Trig and Trancendental Instructions------------------------------
10845 instruct cosD_reg(regD dst) %{
10846   match(Set dst (CosD dst));
10847 
10848   format %{ "dcos   $dst\n\t" %}
10849   opcode(0xD9, 0xFF);
10850   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10851   ins_pipe( pipe_slow );
10852 %}
10853 
10854 instruct sinD_reg(regD dst) %{
10855   match(Set dst (SinD dst));
10856 
10857   format %{ "dsin   $dst\n\t" %}
10858   opcode(0xD9, 0xFE);
10859   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
10860   ins_pipe( pipe_slow );
10861 %}
10862 
10863 instruct tanD_reg(regD dst) %{
10864   match(Set dst (TanD dst));
10865 
10866   format %{ "dtan   $dst\n\t" %}
10867   ins_encode( Push_SrcXD(dst),
10868               Opcode(0xD9), Opcode(0xF2),   //fptan
10869               Opcode(0xDD), Opcode(0xD8),   //fstp st
10870               Push_ResultXD(dst) );
10871   ins_pipe( pipe_slow );
10872 %}
10873 
10874 instruct log10D_reg(regD dst) %{
10875   // The source and result Double operands in XMM registers
10876   match(Set dst (Log10D dst));
10877   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
10878   // fyl2x        ; compute log_10(2) * log_2(x)
10879   format %{ "fldlg2\t\t\t#Log10\n\t"
10880             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
10881          %}
10882    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
10883               Push_SrcXD(dst),
10884               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10885               Push_ResultXD(dst));
10886 
10887   ins_pipe( pipe_slow );
10888 %}
10889 
10890 instruct logD_reg(regD dst) %{
10891   // The source and result Double operands in XMM registers
10892   match(Set dst (LogD dst));
10893   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
10894   // fyl2x        ; compute log_e(2) * log_2(x)
10895   format %{ "fldln2\t\t\t#Log_e\n\t"
10896             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
10897          %}
10898   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
10899               Push_SrcXD(dst),
10900               Opcode(0xD9), Opcode(0xF1),   // fyl2x
10901               Push_ResultXD(dst));
10902   ins_pipe( pipe_slow );
10903 %}
10904 
10905 
10906 
10907 //----------Arithmetic Conversion Instructions---------------------------------
10908 
10909 instruct roundFloat_nop(regF dst)
10910 %{
10911   match(Set dst (RoundFloat dst));
10912 
10913   ins_cost(0);
10914   ins_encode();
10915   ins_pipe(empty);
10916 %}
10917 
10918 instruct roundDouble_nop(regD dst)
10919 %{
10920   match(Set dst (RoundDouble dst));
10921 
10922   ins_cost(0);
10923   ins_encode();
10924   ins_pipe(empty);
10925 %}
10926 
10927 instruct convF2D_reg_reg(regD dst, regF src)
10928 %{
10929   match(Set dst (ConvF2D src));
10930 
10931   format %{ "cvtss2sd $dst, $src" %}
10932   opcode(0xF3, 0x0F, 0x5A);
10933   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10934   ins_pipe(pipe_slow); // XXX
10935 %}
10936 
10937 instruct convF2D_reg_mem(regD dst, memory src)
10938 %{
10939   match(Set dst (ConvF2D (LoadF src)));
10940 
10941   format %{ "cvtss2sd $dst, $src" %}
10942   opcode(0xF3, 0x0F, 0x5A);
10943   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10944   ins_pipe(pipe_slow); // XXX
10945 %}
10946 
10947 instruct convD2F_reg_reg(regF dst, regD src)
10948 %{
10949   match(Set dst (ConvD2F src));
10950 
10951   format %{ "cvtsd2ss $dst, $src" %}
10952   opcode(0xF2, 0x0F, 0x5A);
10953   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
10954   ins_pipe(pipe_slow); // XXX
10955 %}
10956 
10957 instruct convD2F_reg_mem(regF dst, memory src)
10958 %{
10959   match(Set dst (ConvD2F (LoadD src)));
10960 
10961   format %{ "cvtsd2ss $dst, $src" %}
10962   opcode(0xF2, 0x0F, 0x5A);
10963   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
10964   ins_pipe(pipe_slow); // XXX
10965 %}
10966 
10967 // XXX do mem variants
10968 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10969 %{
10970   match(Set dst (ConvF2I src));
10971   effect(KILL cr);
10972 
10973   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10974             "cmpl    $dst, #0x80000000\n\t"
10975             "jne,s   done\n\t"
10976             "subq    rsp, #8\n\t"
10977             "movss   [rsp], $src\n\t"
10978             "call    f2i_fixup\n\t"
10979             "popq    $dst\n"
10980     "done:   "%}
10981   opcode(0xF3, 0x0F, 0x2C);
10982   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
10983              f2i_fixup(dst, src));
10984   ins_pipe(pipe_slow);
10985 %}
10986 
10987 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10988 %{
10989   match(Set dst (ConvF2L src));
10990   effect(KILL cr);
10991 
10992   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10993             "cmpq    $dst, [0x8000000000000000]\n\t"
10994             "jne,s   done\n\t"
10995             "subq    rsp, #8\n\t"
10996             "movss   [rsp], $src\n\t"
10997             "call    f2l_fixup\n\t"
10998             "popq    $dst\n"
10999     "done:   "%}
11000   opcode(0xF3, 0x0F, 0x2C);
11001   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
11002              f2l_fixup(dst, src));
11003   ins_pipe(pipe_slow);
11004 %}
11005 
11006 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
11007 %{
11008   match(Set dst (ConvD2I src));
11009   effect(KILL cr);
11010 
11011   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
11012             "cmpl    $dst, #0x80000000\n\t"
11013             "jne,s   done\n\t"
11014             "subq    rsp, #8\n\t"
11015             "movsd   [rsp], $src\n\t"
11016             "call    d2i_fixup\n\t"
11017             "popq    $dst\n"
11018     "done:   "%}
11019   opcode(0xF2, 0x0F, 0x2C);
11020   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
11021              d2i_fixup(dst, src));
11022   ins_pipe(pipe_slow);
11023 %}
11024 
11025 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
11026 %{
11027   match(Set dst (ConvD2L src));
11028   effect(KILL cr);
11029 
11030   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
11031             "cmpq    $dst, [0x8000000000000000]\n\t"
11032             "jne,s   done\n\t"
11033             "subq    rsp, #8\n\t"
11034             "movsd   [rsp], $src\n\t"
11035             "call    d2l_fixup\n\t"
11036             "popq    $dst\n"
11037     "done:   "%}
11038   opcode(0xF2, 0x0F, 0x2C);
11039   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
11040              d2l_fixup(dst, src));
11041   ins_pipe(pipe_slow);
11042 %}
11043 
11044 instruct convI2F_reg_reg(regF dst, rRegI src)
11045 %{
11046   predicate(!UseXmmI2F);
11047   match(Set dst (ConvI2F src));
11048 
11049   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11050   opcode(0xF3, 0x0F, 0x2A);
11051   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
11052   ins_pipe(pipe_slow); // XXX
11053 %}
11054 
11055 instruct convI2F_reg_mem(regF dst, memory src)
11056 %{
11057   match(Set dst (ConvI2F (LoadI src)));
11058 
11059   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
11060   opcode(0xF3, 0x0F, 0x2A);
11061   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
11062   ins_pipe(pipe_slow); // XXX
11063 %}
11064 
11065 instruct convI2D_reg_reg(regD dst, rRegI src)
11066 %{
11067   predicate(!UseXmmI2D);
11068   match(Set dst (ConvI2D src));
11069 
11070   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11071   opcode(0xF2, 0x0F, 0x2A);
11072   ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
11073   ins_pipe(pipe_slow); // XXX
11074 %}
11075 
11076 instruct convI2D_reg_mem(regD dst, memory src)
11077 %{
11078   match(Set dst (ConvI2D (LoadI src)));
11079 
11080   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
11081   opcode(0xF2, 0x0F, 0x2A);
11082   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
11083   ins_pipe(pipe_slow); // XXX
11084 %}
11085 
11086 instruct convXI2F_reg(regF dst, rRegI src)
11087 %{
11088   predicate(UseXmmI2F);
11089   match(Set dst (ConvI2F src));
11090 
11091   format %{ "movdl $dst, $src\n\t"
11092             "cvtdq2psl $dst, $dst\t# i2f" %}
11093   ins_encode %{
11094     __ movdl($dst$$XMMRegister, $src$$Register);
11095     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
11096   %}
11097   ins_pipe(pipe_slow); // XXX
11098 %}
11099 
11100 instruct convXI2D_reg(regD dst, rRegI src)
11101 %{
11102   predicate(UseXmmI2D);
11103   match(Set dst (ConvI2D src));
11104 
11105   format %{ "movdl $dst, $src\n\t"
11106             "cvtdq2pdl $dst, $dst\t# i2d" %}
11107   ins_encode %{
11108     __ movdl($dst$$XMMRegister, $src$$Register);
11109     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
11110   %}
11111   ins_pipe(pipe_slow); // XXX
11112 %}
11113 
11114 instruct convL2F_reg_reg(regF dst, rRegL src)
11115 %{
11116   match(Set dst (ConvL2F src));
11117 
11118   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11119   opcode(0xF3, 0x0F, 0x2A);
11120   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
11121   ins_pipe(pipe_slow); // XXX
11122 %}
11123 
11124 instruct convL2F_reg_mem(regF dst, memory src)
11125 %{
11126   match(Set dst (ConvL2F (LoadL src)));
11127 
11128   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
11129   opcode(0xF3, 0x0F, 0x2A);
11130   ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
11131   ins_pipe(pipe_slow); // XXX
11132 %}
11133 
11134 instruct convL2D_reg_reg(regD dst, rRegL src)
11135 %{
11136   match(Set dst (ConvL2D src));
11137 
11138   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11139   opcode(0xF2, 0x0F, 0x2A);
11140   ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
11141   ins_pipe(pipe_slow); // XXX
11142 %}
11143 
11144 instruct convL2D_reg_mem(regD dst, memory src)
11145 %{
11146   match(Set dst (ConvL2D (LoadL src)));
11147 
11148   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
11149   opcode(0xF2, 0x0F, 0x2A);
11150   ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
11151   ins_pipe(pipe_slow); // XXX
11152 %}
11153 
11154 instruct convI2L_reg_reg(rRegL dst, rRegI src)
11155 %{
11156   match(Set dst (ConvI2L src));
11157 
11158   ins_cost(125);
11159   format %{ "movslq  $dst, $src\t# i2l" %}
11160   ins_encode %{
11161     __ movslq($dst$$Register, $src$$Register);
11162   %}
11163   ins_pipe(ialu_reg_reg);
11164 %}
11165 
11166 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
11167 // %{
11168 //   match(Set dst (ConvI2L src));
11169 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
11170 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
11171 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
11172 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
11173 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
11174 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
11175 
11176 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
11177 //   ins_encode(enc_copy(dst, src));
11178 // //   opcode(0x63); // needs REX.W
11179 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
11180 //   ins_pipe(ialu_reg_reg);
11181 // %}
11182 
11183 // Zero-extend convert int to long
11184 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
11185 %{
11186   match(Set dst (AndL (ConvI2L src) mask));
11187 
11188   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11189   ins_encode(enc_copy(dst, src));
11190   ins_pipe(ialu_reg_reg);
11191 %}
11192 
11193 // Zero-extend convert int to long
11194 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
11195 %{
11196   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
11197 
11198   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
11199   opcode(0x8B);
11200   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
11201   ins_pipe(ialu_reg_mem);
11202 %}
11203 
11204 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
11205 %{
11206   match(Set dst (AndL src mask));
11207 
11208   format %{ "movl    $dst, $src\t# zero-extend long" %}
11209   ins_encode(enc_copy_always(dst, src));
11210   ins_pipe(ialu_reg_reg);
11211 %}
11212 
11213 instruct convL2I_reg_reg(rRegI dst, rRegL src)
11214 %{
11215   match(Set dst (ConvL2I src));
11216 
11217   format %{ "movl    $dst, $src\t# l2i" %}
11218   ins_encode(enc_copy_always(dst, src));
11219   ins_pipe(ialu_reg_reg);
11220 %}
11221 
11222 
11223 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11224   match(Set dst (MoveF2I src));
11225   effect(DEF dst, USE src);
11226 
11227   ins_cost(125);
11228   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11229   opcode(0x8B);
11230   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
11231   ins_pipe(ialu_reg_mem);
11232 %}
11233 
11234 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11235   match(Set dst (MoveI2F src));
11236   effect(DEF dst, USE src);
11237 
11238   ins_cost(125);
11239   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11240   opcode(0xF3, 0x0F, 0x10);
11241   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
11242   ins_pipe(pipe_slow);
11243 %}
11244 
11245 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11246   match(Set dst (MoveD2L src));
11247   effect(DEF dst, USE src);
11248 
11249   ins_cost(125);
11250   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11251   opcode(0x8B);
11252   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
11253   ins_pipe(ialu_reg_mem);
11254 %}
11255 
11256 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11257   predicate(!UseXmmLoadAndClearUpper);
11258   match(Set dst (MoveL2D src));
11259   effect(DEF dst, USE src);
11260 
11261   ins_cost(125);
11262   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11263   opcode(0x66, 0x0F, 0x12);
11264   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
11265   ins_pipe(pipe_slow);
11266 %}
11267 
11268 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11269   predicate(UseXmmLoadAndClearUpper);
11270   match(Set dst (MoveL2D src));
11271   effect(DEF dst, USE src);
11272 
11273   ins_cost(125);
11274   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11275   opcode(0xF2, 0x0F, 0x10);
11276   ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
11277   ins_pipe(pipe_slow);
11278 %}
11279 
11280 
11281 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11282   match(Set dst (MoveF2I src));
11283   effect(DEF dst, USE src);
11284 
11285   ins_cost(95); // XXX
11286   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11287   opcode(0xF3, 0x0F, 0x11);
11288   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
11289   ins_pipe(pipe_slow);
11290 %}
11291 
11292 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11293   match(Set dst (MoveI2F src));
11294   effect(DEF dst, USE src);
11295 
11296   ins_cost(100);
11297   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11298   opcode(0x89);
11299   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
11300   ins_pipe( ialu_mem_reg );
11301 %}
11302 
11303 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11304   match(Set dst (MoveD2L src));
11305   effect(DEF dst, USE src);
11306 
11307   ins_cost(95); // XXX
11308   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11309   opcode(0xF2, 0x0F, 0x11);
11310   ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
11311   ins_pipe(pipe_slow);
11312 %}
11313 
11314 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11315   match(Set dst (MoveL2D src));
11316   effect(DEF dst, USE src);
11317 
11318   ins_cost(100);
11319   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11320   opcode(0x89);
11321   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
11322   ins_pipe(ialu_mem_reg);
11323 %}
11324 
11325 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11326   match(Set dst (MoveF2I src));
11327   effect(DEF dst, USE src);
11328   ins_cost(85);
11329   format %{ "movd    $dst,$src\t# MoveF2I" %}
11330   ins_encode %{ __ movdl($dst$$Register, $src$$XMMRegister); %}
11331   ins_pipe( pipe_slow );
11332 %}
11333 
11334 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11335   match(Set dst (MoveD2L src));
11336   effect(DEF dst, USE src);
11337   ins_cost(85);
11338   format %{ "movd    $dst,$src\t# MoveD2L" %}
11339   ins_encode %{ __ movdq($dst$$Register, $src$$XMMRegister); %}
11340   ins_pipe( pipe_slow );
11341 %}
11342 
11343 // The next instructions have long latency and use Int unit. Set high cost.
11344 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11345   match(Set dst (MoveI2F src));
11346   effect(DEF dst, USE src);
11347   ins_cost(300);
11348   format %{ "movd    $dst,$src\t# MoveI2F" %}
11349   ins_encode %{ __ movdl($dst$$XMMRegister, $src$$Register); %}
11350   ins_pipe( pipe_slow );
11351 %}
11352 
11353 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11354   match(Set dst (MoveL2D src));
11355   effect(DEF dst, USE src);
11356   ins_cost(300);
11357   format %{ "movd    $dst,$src\t# MoveL2D" %}
11358   ins_encode %{ __ movdq($dst$$XMMRegister, $src$$Register); %}
11359   ins_pipe( pipe_slow );
11360 %}
11361 
11362 // Replicate scalar to packed byte (1 byte) values in xmm
11363 instruct Repl8B_reg(regD dst, regD src) %{
11364   match(Set dst (Replicate8B src));
11365   format %{ "MOVDQA  $dst,$src\n\t"
11366             "PUNPCKLBW $dst,$dst\n\t"
11367             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
11368   ins_encode( pshufd_8x8(dst, src));
11369   ins_pipe( pipe_slow );
11370 %}
11371 
11372 // Replicate scalar to packed byte (1 byte) values in xmm
11373 instruct Repl8B_rRegI(regD dst, rRegI src) %{
11374   match(Set dst (Replicate8B src));
11375   format %{ "MOVD    $dst,$src\n\t"
11376             "PUNPCKLBW $dst,$dst\n\t"
11377             "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
11378   ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
11379   ins_pipe( pipe_slow );
11380 %}
11381 
11382 // Replicate scalar zero to packed byte (1 byte) values in xmm
11383 instruct Repl8B_immI0(regD dst, immI0 zero) %{
11384   match(Set dst (Replicate8B zero));
11385   format %{ "PXOR  $dst,$dst\t! replicate8B" %}
11386   ins_encode( pxor(dst, dst));
11387   ins_pipe( fpu_reg_reg );
11388 %}
11389 
11390 // Replicate scalar to packed shore (2 byte) values in xmm
11391 instruct Repl4S_reg(regD dst, regD src) %{
11392   match(Set dst (Replicate4S src));
11393   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
11394   ins_encode( pshufd_4x16(dst, src));
11395   ins_pipe( fpu_reg_reg );
11396 %}
11397 
11398 // Replicate scalar to packed shore (2 byte) values in xmm
11399 instruct Repl4S_rRegI(regD dst, rRegI src) %{
11400   match(Set dst (Replicate4S src));
11401   format %{ "MOVD    $dst,$src\n\t"
11402             "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
11403   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
11404   ins_pipe( fpu_reg_reg );
11405 %}
11406 
11407 // Replicate scalar zero to packed short (2 byte) values in xmm
11408 instruct Repl4S_immI0(regD dst, immI0 zero) %{
11409   match(Set dst (Replicate4S zero));
11410   format %{ "PXOR  $dst,$dst\t! replicate4S" %}
11411   ins_encode( pxor(dst, dst));
11412   ins_pipe( fpu_reg_reg );
11413 %}
11414 
11415 // Replicate scalar to packed char (2 byte) values in xmm
11416 instruct Repl4C_reg(regD dst, regD src) %{
11417   match(Set dst (Replicate4C src));
11418   format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
11419   ins_encode( pshufd_4x16(dst, src));
11420   ins_pipe( fpu_reg_reg );
11421 %}
11422 
11423 // Replicate scalar to packed char (2 byte) values in xmm
11424 instruct Repl4C_rRegI(regD dst, rRegI src) %{
11425   match(Set dst (Replicate4C src));
11426   format %{ "MOVD    $dst,$src\n\t"
11427             "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
11428   ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
11429   ins_pipe( fpu_reg_reg );
11430 %}
11431 
11432 // Replicate scalar zero to packed char (2 byte) values in xmm
11433 instruct Repl4C_immI0(regD dst, immI0 zero) %{
11434   match(Set dst (Replicate4C zero));
11435   format %{ "PXOR  $dst,$dst\t! replicate4C" %}
11436   ins_encode( pxor(dst, dst));
11437   ins_pipe( fpu_reg_reg );
11438 %}
11439 
11440 // Replicate scalar to packed integer (4 byte) values in xmm
11441 instruct Repl2I_reg(regD dst, regD src) %{
11442   match(Set dst (Replicate2I src));
11443   format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
11444   ins_encode( pshufd(dst, src, 0x00));
11445   ins_pipe( fpu_reg_reg );
11446 %}
11447 
11448 // Replicate scalar to packed integer (4 byte) values in xmm
11449 instruct Repl2I_rRegI(regD dst, rRegI src) %{
11450   match(Set dst (Replicate2I src));
11451   format %{ "MOVD   $dst,$src\n\t"
11452             "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
11453   ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
11454   ins_pipe( fpu_reg_reg );
11455 %}
11456 
11457 // Replicate scalar zero to packed integer (2 byte) values in xmm
11458 instruct Repl2I_immI0(regD dst, immI0 zero) %{
11459   match(Set dst (Replicate2I zero));
11460   format %{ "PXOR  $dst,$dst\t! replicate2I" %}
11461   ins_encode( pxor(dst, dst));
11462   ins_pipe( fpu_reg_reg );
11463 %}
11464 
11465 // Replicate scalar to packed single precision floating point values in xmm
11466 instruct Repl2F_reg(regD dst, regD src) %{
11467   match(Set dst (Replicate2F src));
11468   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
11469   ins_encode( pshufd(dst, src, 0xe0));
11470   ins_pipe( fpu_reg_reg );
11471 %}
11472 
11473 // Replicate scalar to packed single precision floating point values in xmm
11474 instruct Repl2F_regF(regD dst, regF src) %{
11475   match(Set dst (Replicate2F src));
11476   format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
11477   ins_encode( pshufd(dst, src, 0xe0));
11478   ins_pipe( fpu_reg_reg );
11479 %}
11480 
11481 // Replicate scalar to packed single precision floating point values in xmm
11482 instruct Repl2F_immF0(regD dst, immF0 zero) %{
11483   match(Set dst (Replicate2F zero));
11484   format %{ "PXOR  $dst,$dst\t! replicate2F" %}
11485   ins_encode( pxor(dst, dst));
11486   ins_pipe( fpu_reg_reg );
11487 %}
11488 
11489 
11490 // =======================================================================
11491 // fast clearing of an array
11492 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
11493                   rFlagsReg cr)
11494 %{
11495   match(Set dummy (ClearArray cnt base));
11496   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
11497 
11498   format %{ "xorl    rax, rax\t# ClearArray:\n\t"
11499             "rep stosq\t# Store rax to *rdi++ while rcx--" %}
11500   ins_encode(opc_reg_reg(0x33, RAX, RAX), // xorl %eax, %eax
11501              Opcode(0xF3), Opcode(0x48), Opcode(0xAB)); // rep REX_W stos
11502   ins_pipe(pipe_slow);
11503 %}
11504 
11505 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11506                         rax_RegI result, regD tmp1, rFlagsReg cr)
11507 %{
11508   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11509   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11510 
11511   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11512   ins_encode %{
11513     __ string_compare($str1$$Register, $str2$$Register,
11514                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11515                       $tmp1$$XMMRegister);
11516   %}
11517   ins_pipe( pipe_slow );
11518 %}
11519 
11520 // fast search of substring with known size.
11521 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11522                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11523 %{
11524   predicate(UseSSE42Intrinsics);
11525   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11526   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11527 
11528   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11529   ins_encode %{
11530     int icnt2 = (int)$int_cnt2$$constant;
11531     if (icnt2 >= 8) {
11532       // IndexOf for constant substrings with size >= 8 elements
11533       // which don't need to be loaded through stack.
11534       __ string_indexofC8($str1$$Register, $str2$$Register,
11535                           $cnt1$$Register, $cnt2$$Register,
11536                           icnt2, $result$$Register,
11537                           $vec$$XMMRegister, $tmp$$Register);
11538     } else {
11539       // Small strings are loaded through stack if they cross page boundary.
11540       __ string_indexof($str1$$Register, $str2$$Register,
11541                         $cnt1$$Register, $cnt2$$Register,
11542                         icnt2, $result$$Register,
11543                         $vec$$XMMRegister, $tmp$$Register);
11544     }
11545   %}
11546   ins_pipe( pipe_slow );
11547 %}
11548 
11549 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11550                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11551 %{
11552   predicate(UseSSE42Intrinsics);
11553   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11554   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11555 
11556   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11557   ins_encode %{
11558     __ string_indexof($str1$$Register, $str2$$Register,
11559                       $cnt1$$Register, $cnt2$$Register,
11560                       (-1), $result$$Register,
11561                       $vec$$XMMRegister, $tmp$$Register);
11562   %}
11563   ins_pipe( pipe_slow );
11564 %}
11565 
11566 // fast string equals
11567 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11568                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11569 %{
11570   match(Set result (StrEquals (Binary str1 str2) cnt));
11571   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11572 
11573   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11574   ins_encode %{
11575     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
11576                           $cnt$$Register, $result$$Register, $tmp3$$Register,
11577                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11578   %}
11579   ins_pipe( pipe_slow );
11580 %}
11581 
11582 // fast array equals
11583 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11584                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11585 %{
11586   match(Set result (AryEq ary1 ary2));
11587   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11588   //ins_cost(300);
11589 
11590   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11591   ins_encode %{
11592     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
11593                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
11594                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11595   %}
11596   ins_pipe( pipe_slow );
11597 %}
11598 
11599 //----------Control Flow Instructions------------------------------------------
11600 // Signed compare Instructions
11601 
11602 // XXX more variants!!
11603 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11604 %{
11605   match(Set cr (CmpI op1 op2));
11606   effect(DEF cr, USE op1, USE op2);
11607 
11608   format %{ "cmpl    $op1, $op2" %}
11609   opcode(0x3B);  /* Opcode 3B /r */
11610   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11611   ins_pipe(ialu_cr_reg_reg);
11612 %}
11613 
11614 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11615 %{
11616   match(Set cr (CmpI op1 op2));
11617 
11618   format %{ "cmpl    $op1, $op2" %}
11619   opcode(0x81, 0x07); /* Opcode 81 /7 */
11620   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11621   ins_pipe(ialu_cr_reg_imm);
11622 %}
11623 
11624 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11625 %{
11626   match(Set cr (CmpI op1 (LoadI op2)));
11627 
11628   ins_cost(500); // XXX
11629   format %{ "cmpl    $op1, $op2" %}
11630   opcode(0x3B); /* Opcode 3B /r */
11631   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11632   ins_pipe(ialu_cr_reg_mem);
11633 %}
11634 
11635 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11636 %{
11637   match(Set cr (CmpI src zero));
11638 
11639   format %{ "testl   $src, $src" %}
11640   opcode(0x85);
11641   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11642   ins_pipe(ialu_cr_reg_imm);
11643 %}
11644 
11645 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11646 %{
11647   match(Set cr (CmpI (AndI src con) zero));
11648 
11649   format %{ "testl   $src, $con" %}
11650   opcode(0xF7, 0x00);
11651   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11652   ins_pipe(ialu_cr_reg_imm);
11653 %}
11654 
11655 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11656 %{
11657   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11658 
11659   format %{ "testl   $src, $mem" %}
11660   opcode(0x85);
11661   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11662   ins_pipe(ialu_cr_reg_mem);
11663 %}
11664 
11665 // Unsigned compare Instructions; really, same as signed except they
11666 // produce an rFlagsRegU instead of rFlagsReg.
11667 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11668 %{
11669   match(Set cr (CmpU op1 op2));
11670 
11671   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11672   opcode(0x3B); /* Opcode 3B /r */
11673   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11674   ins_pipe(ialu_cr_reg_reg);
11675 %}
11676 
11677 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11678 %{
11679   match(Set cr (CmpU op1 op2));
11680 
11681   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11682   opcode(0x81,0x07); /* Opcode 81 /7 */
11683   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11684   ins_pipe(ialu_cr_reg_imm);
11685 %}
11686 
11687 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11688 %{
11689   match(Set cr (CmpU op1 (LoadI op2)));
11690 
11691   ins_cost(500); // XXX
11692   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11693   opcode(0x3B); /* Opcode 3B /r */
11694   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11695   ins_pipe(ialu_cr_reg_mem);
11696 %}
11697 
11698 // // // Cisc-spilled version of cmpU_rReg
11699 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11700 // //%{
11701 // //  match(Set cr (CmpU (LoadI op1) op2));
11702 // //
11703 // //  format %{ "CMPu   $op1,$op2" %}
11704 // //  ins_cost(500);
11705 // //  opcode(0x39);  /* Opcode 39 /r */
11706 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11707 // //%}
11708 
11709 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11710 %{
11711   match(Set cr (CmpU src zero));
11712 
11713   format %{ "testl  $src, $src\t# unsigned" %}
11714   opcode(0x85);
11715   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11716   ins_pipe(ialu_cr_reg_imm);
11717 %}
11718 
11719 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11720 %{
11721   match(Set cr (CmpP op1 op2));
11722 
11723   format %{ "cmpq    $op1, $op2\t# ptr" %}
11724   opcode(0x3B); /* Opcode 3B /r */
11725   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11726   ins_pipe(ialu_cr_reg_reg);
11727 %}
11728 
11729 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11730 %{
11731   match(Set cr (CmpP op1 (LoadP op2)));
11732 
11733   ins_cost(500); // XXX
11734   format %{ "cmpq    $op1, $op2\t# ptr" %}
11735   opcode(0x3B); /* Opcode 3B /r */
11736   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11737   ins_pipe(ialu_cr_reg_mem);
11738 %}
11739 
11740 // // // Cisc-spilled version of cmpP_rReg
11741 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11742 // //%{
11743 // //  match(Set cr (CmpP (LoadP op1) op2));
11744 // //
11745 // //  format %{ "CMPu   $op1,$op2" %}
11746 // //  ins_cost(500);
11747 // //  opcode(0x39);  /* Opcode 39 /r */
11748 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11749 // //%}
11750 
11751 // XXX this is generalized by compP_rReg_mem???
11752 // Compare raw pointer (used in out-of-heap check).
11753 // Only works because non-oop pointers must be raw pointers
11754 // and raw pointers have no anti-dependencies.
11755 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11756 %{
11757   predicate(!n->in(2)->in(2)->bottom_type()->isa_oop_ptr());
11758   match(Set cr (CmpP op1 (LoadP op2)));
11759 
11760   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11761   opcode(0x3B); /* Opcode 3B /r */
11762   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11763   ins_pipe(ialu_cr_reg_mem);
11764 %}
11765 
11766 // This will generate a signed flags result. This should be OK since
11767 // any compare to a zero should be eq/neq.
11768 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11769 %{
11770   match(Set cr (CmpP src zero));
11771 
11772   format %{ "testq   $src, $src\t# ptr" %}
11773   opcode(0x85);
11774   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11775   ins_pipe(ialu_cr_reg_imm);
11776 %}
11777 
11778 // This will generate a signed flags result. This should be OK since
11779 // any compare to a zero should be eq/neq.
11780 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11781 %{
11782   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11783   match(Set cr (CmpP (LoadP op) zero));
11784 
11785   ins_cost(500); // XXX
11786   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11787   opcode(0xF7); /* Opcode F7 /0 */
11788   ins_encode(REX_mem_wide(op),
11789              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11790   ins_pipe(ialu_cr_reg_imm);
11791 %}
11792 
11793 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11794 %{
11795   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
11796   match(Set cr (CmpP (LoadP mem) zero));
11797 
11798   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11799   ins_encode %{
11800     __ cmpq(r12, $mem$$Address);
11801   %}
11802   ins_pipe(ialu_cr_reg_mem);
11803 %}
11804 
11805 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11806 %{
11807   match(Set cr (CmpN op1 op2));
11808 
11809   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11810   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11811   ins_pipe(ialu_cr_reg_reg);
11812 %}
11813 
11814 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11815 %{
11816   match(Set cr (CmpN src (LoadN mem)));
11817 
11818   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11819   ins_encode %{
11820     __ cmpl($src$$Register, $mem$$Address);
11821   %}
11822   ins_pipe(ialu_cr_reg_mem);
11823 %}
11824 
11825 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11826   match(Set cr (CmpN op1 op2));
11827 
11828   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11829   ins_encode %{
11830     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11831   %}
11832   ins_pipe(ialu_cr_reg_imm);
11833 %}
11834 
11835 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11836 %{
11837   match(Set cr (CmpN src (LoadN mem)));
11838 
11839   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11840   ins_encode %{
11841     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11842   %}
11843   ins_pipe(ialu_cr_reg_mem);
11844 %}
11845 
11846 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11847   match(Set cr (CmpN src zero));
11848 
11849   format %{ "testl   $src, $src\t# compressed ptr" %}
11850   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11851   ins_pipe(ialu_cr_reg_imm);
11852 %}
11853 
11854 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11855 %{
11856   predicate(Universe::narrow_oop_base() != NULL);
11857   match(Set cr (CmpN (LoadN mem) zero));
11858 
11859   ins_cost(500); // XXX
11860   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11861   ins_encode %{
11862     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11863   %}
11864   ins_pipe(ialu_cr_reg_mem);
11865 %}
11866 
11867 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11868 %{
11869   predicate(Universe::narrow_oop_base() == NULL);
11870   match(Set cr (CmpN (LoadN mem) zero));
11871 
11872   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11873   ins_encode %{
11874     __ cmpl(r12, $mem$$Address);
11875   %}
11876   ins_pipe(ialu_cr_reg_mem);
11877 %}
11878 
11879 // Yanked all unsigned pointer compare operations.
11880 // Pointer compares are done with CmpP which is already unsigned.
11881 
11882 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11883 %{
11884   match(Set cr (CmpL op1 op2));
11885 
11886   format %{ "cmpq    $op1, $op2" %}
11887   opcode(0x3B);  /* Opcode 3B /r */
11888   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11889   ins_pipe(ialu_cr_reg_reg);
11890 %}
11891 
11892 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11893 %{
11894   match(Set cr (CmpL op1 op2));
11895 
11896   format %{ "cmpq    $op1, $op2" %}
11897   opcode(0x81, 0x07); /* Opcode 81 /7 */
11898   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11899   ins_pipe(ialu_cr_reg_imm);
11900 %}
11901 
11902 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11903 %{
11904   match(Set cr (CmpL op1 (LoadL op2)));
11905 
11906   format %{ "cmpq    $op1, $op2" %}
11907   opcode(0x3B); /* Opcode 3B /r */
11908   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11909   ins_pipe(ialu_cr_reg_mem);
11910 %}
11911 
11912 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11913 %{
11914   match(Set cr (CmpL src zero));
11915 
11916   format %{ "testq   $src, $src" %}
11917   opcode(0x85);
11918   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11919   ins_pipe(ialu_cr_reg_imm);
11920 %}
11921 
11922 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11923 %{
11924   match(Set cr (CmpL (AndL src con) zero));
11925 
11926   format %{ "testq   $src, $con\t# long" %}
11927   opcode(0xF7, 0x00);
11928   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11929   ins_pipe(ialu_cr_reg_imm);
11930 %}
11931 
11932 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11933 %{
11934   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11935 
11936   format %{ "testq   $src, $mem" %}
11937   opcode(0x85);
11938   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11939   ins_pipe(ialu_cr_reg_mem);
11940 %}
11941 
11942 // Manifest a CmpL result in an integer register.  Very painful.
11943 // This is the test to avoid.
11944 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11945 %{
11946   match(Set dst (CmpL3 src1 src2));
11947   effect(KILL flags);
11948 
11949   ins_cost(275); // XXX
11950   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11951             "movl    $dst, -1\n\t"
11952             "jl,s    done\n\t"
11953             "setne   $dst\n\t"
11954             "movzbl  $dst, $dst\n\t"
11955     "done:" %}
11956   ins_encode(cmpl3_flag(src1, src2, dst));
11957   ins_pipe(pipe_slow);
11958 %}
11959 
11960 //----------Max and Min--------------------------------------------------------
11961 // Min Instructions
11962 
11963 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11964 %{
11965   effect(USE_DEF dst, USE src, USE cr);
11966 
11967   format %{ "cmovlgt $dst, $src\t# min" %}
11968   opcode(0x0F, 0x4F);
11969   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11970   ins_pipe(pipe_cmov_reg);
11971 %}
11972 
11973 
11974 instruct minI_rReg(rRegI dst, rRegI src)
11975 %{
11976   match(Set dst (MinI dst src));
11977 
11978   ins_cost(200);
11979   expand %{
11980     rFlagsReg cr;
11981     compI_rReg(cr, dst, src);
11982     cmovI_reg_g(dst, src, cr);
11983   %}
11984 %}
11985 
11986 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11987 %{
11988   effect(USE_DEF dst, USE src, USE cr);
11989 
11990   format %{ "cmovllt $dst, $src\t# max" %}
11991   opcode(0x0F, 0x4C);
11992   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11993   ins_pipe(pipe_cmov_reg);
11994 %}
11995 
11996 
11997 instruct maxI_rReg(rRegI dst, rRegI src)
11998 %{
11999   match(Set dst (MaxI dst src));
12000 
12001   ins_cost(200);
12002   expand %{
12003     rFlagsReg cr;
12004     compI_rReg(cr, dst, src);
12005     cmovI_reg_l(dst, src, cr);
12006   %}
12007 %}
12008 
12009 // ============================================================================
12010 // Branch Instructions
12011 
12012 // Jump Direct - Label defines a relative address from JMP+1
12013 instruct jmpDir(label labl)
12014 %{
12015   match(Goto);
12016   effect(USE labl);
12017 
12018   ins_cost(300);
12019   format %{ "jmp     $labl" %}
12020   size(5);
12021   ins_encode %{
12022     Label* L = $labl$$label;
12023     __ jmp(*L, false); // Always long jump
12024   %}
12025   ins_pipe(pipe_jmp);
12026 %}
12027 
12028 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12029 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12030 %{
12031   match(If cop cr);
12032   effect(USE labl);
12033 
12034   ins_cost(300);
12035   format %{ "j$cop     $labl" %}
12036   size(6);
12037   ins_encode %{
12038     Label* L = $labl$$label;
12039     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12040   %}
12041   ins_pipe(pipe_jcc);
12042 %}
12043 
12044 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12045 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12046 %{
12047   match(CountedLoopEnd cop cr);
12048   effect(USE labl);
12049 
12050   ins_cost(300);
12051   format %{ "j$cop     $labl\t# loop end" %}
12052   size(6);
12053   ins_encode %{
12054     Label* L = $labl$$label;
12055     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12056   %}
12057   ins_pipe(pipe_jcc);
12058 %}
12059 
12060 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12061 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12062   match(CountedLoopEnd cop cmp);
12063   effect(USE labl);
12064 
12065   ins_cost(300);
12066   format %{ "j$cop,u   $labl\t# loop end" %}
12067   size(6);
12068   ins_encode %{
12069     Label* L = $labl$$label;
12070     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12071   %}
12072   ins_pipe(pipe_jcc);
12073 %}
12074 
12075 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12076   match(CountedLoopEnd cop cmp);
12077   effect(USE labl);
12078 
12079   ins_cost(200);
12080   format %{ "j$cop,u   $labl\t# loop end" %}
12081   size(6);
12082   ins_encode %{
12083     Label* L = $labl$$label;
12084     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12085   %}
12086   ins_pipe(pipe_jcc);
12087 %}
12088 
12089 // Jump Direct Conditional - using unsigned comparison
12090 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12091   match(If cop cmp);
12092   effect(USE labl);
12093 
12094   ins_cost(300);
12095   format %{ "j$cop,u  $labl" %}
12096   size(6);
12097   ins_encode %{
12098     Label* L = $labl$$label;
12099     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12100   %}
12101   ins_pipe(pipe_jcc);
12102 %}
12103 
12104 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12105   match(If cop cmp);
12106   effect(USE labl);
12107 
12108   ins_cost(200);
12109   format %{ "j$cop,u  $labl" %}
12110   size(6);
12111   ins_encode %{
12112     Label* L = $labl$$label;
12113     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12114   %}
12115   ins_pipe(pipe_jcc);
12116 %}
12117 
12118 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12119   match(If cop cmp);
12120   effect(USE labl);
12121 
12122   ins_cost(200);
12123   format %{ $$template
12124     if ($cop$$cmpcode == Assembler::notEqual) {
12125       $$emit$$"jp,u   $labl\n\t"
12126       $$emit$$"j$cop,u   $labl"
12127     } else {
12128       $$emit$$"jp,u   done\n\t"
12129       $$emit$$"j$cop,u   $labl\n\t"
12130       $$emit$$"done:"
12131     }
12132   %}
12133   ins_encode %{
12134     Label* l = $labl$$label;
12135     if ($cop$$cmpcode == Assembler::notEqual) {
12136       __ jcc(Assembler::parity, *l, false);
12137       __ jcc(Assembler::notEqual, *l, false);
12138     } else if ($cop$$cmpcode == Assembler::equal) {
12139       Label done;
12140       __ jccb(Assembler::parity, done);
12141       __ jcc(Assembler::equal, *l, false);
12142       __ bind(done);
12143     } else {
12144        ShouldNotReachHere();
12145     }
12146   %}
12147   ins_pipe(pipe_jcc);
12148 %}
12149 
12150 // ============================================================================
12151 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12152 // superklass array for an instance of the superklass.  Set a hidden
12153 // internal cache on a hit (cache is checked with exposed code in
12154 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12155 // encoding ALSO sets flags.
12156 
12157 instruct partialSubtypeCheck(rdi_RegP result,
12158                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12159                              rFlagsReg cr)
12160 %{
12161   match(Set result (PartialSubtypeCheck sub super));
12162   effect(KILL rcx, KILL cr);
12163 
12164   ins_cost(1100);  // slightly larger than the next version
12165   format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
12166             "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
12167             "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
12168             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12169             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12170             "movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
12171             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12172     "miss:\t" %}
12173 
12174   opcode(0x1); // Force a XOR of RDI
12175   ins_encode(enc_PartialSubtypeCheck());
12176   ins_pipe(pipe_slow);
12177 %}
12178 
12179 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12180                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12181                                      immP0 zero,
12182                                      rdi_RegP result)
12183 %{
12184   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12185   effect(KILL rcx, KILL result);
12186 
12187   ins_cost(1000);
12188   format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
12189             "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
12190             "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
12191             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12192             "jne,s   miss\t\t# Missed: flags nz\n\t"
12193             "movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
12194     "miss:\t" %}
12195 
12196   opcode(0x0); // No need to XOR RDI
12197   ins_encode(enc_PartialSubtypeCheck());
12198   ins_pipe(pipe_slow);
12199 %}
12200 
12201 // ============================================================================
12202 // Branch Instructions -- short offset versions
12203 //
12204 // These instructions are used to replace jumps of a long offset (the default
12205 // match) with jumps of a shorter offset.  These instructions are all tagged
12206 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12207 // match rules in general matching.  Instead, the ADLC generates a conversion
12208 // method in the MachNode which can be used to do in-place replacement of the
12209 // long variant with the shorter variant.  The compiler will determine if a
12210 // branch can be taken by the is_short_branch_offset() predicate in the machine
12211 // specific code section of the file.
12212 
12213 // Jump Direct - Label defines a relative address from JMP+1
12214 instruct jmpDir_short(label labl) %{
12215   match(Goto);
12216   effect(USE labl);
12217 
12218   ins_cost(300);
12219   format %{ "jmp,s   $labl" %}
12220   size(2);
12221   ins_encode %{
12222     Label* L = $labl$$label;
12223     __ jmpb(*L);
12224   %}
12225   ins_pipe(pipe_jmp);
12226   ins_short_branch(1);
12227 %}
12228 
12229 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12230 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12231   match(If cop cr);
12232   effect(USE labl);
12233 
12234   ins_cost(300);
12235   format %{ "j$cop,s   $labl" %}
12236   size(2);
12237   ins_encode %{
12238     Label* L = $labl$$label;
12239     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12240   %}
12241   ins_pipe(pipe_jcc);
12242   ins_short_branch(1);
12243 %}
12244 
12245 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12246 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12247   match(CountedLoopEnd cop cr);
12248   effect(USE labl);
12249 
12250   ins_cost(300);
12251   format %{ "j$cop,s   $labl\t# loop end" %}
12252   size(2);
12253   ins_encode %{
12254     Label* L = $labl$$label;
12255     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12256   %}
12257   ins_pipe(pipe_jcc);
12258   ins_short_branch(1);
12259 %}
12260 
12261 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12262 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12263   match(CountedLoopEnd cop cmp);
12264   effect(USE labl);
12265 
12266   ins_cost(300);
12267   format %{ "j$cop,us  $labl\t# loop end" %}
12268   size(2);
12269   ins_encode %{
12270     Label* L = $labl$$label;
12271     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12272   %}
12273   ins_pipe(pipe_jcc);
12274   ins_short_branch(1);
12275 %}
12276 
12277 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12278   match(CountedLoopEnd cop cmp);
12279   effect(USE labl);
12280 
12281   ins_cost(300);
12282   format %{ "j$cop,us  $labl\t# loop end" %}
12283   size(2);
12284   ins_encode %{
12285     Label* L = $labl$$label;
12286     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12287   %}
12288   ins_pipe(pipe_jcc);
12289   ins_short_branch(1);
12290 %}
12291 
12292 // Jump Direct Conditional - using unsigned comparison
12293 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12294   match(If cop cmp);
12295   effect(USE labl);
12296 
12297   ins_cost(300);
12298   format %{ "j$cop,us  $labl" %}
12299   size(2);
12300   ins_encode %{
12301     Label* L = $labl$$label;
12302     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12303   %}
12304   ins_pipe(pipe_jcc);
12305   ins_short_branch(1);
12306 %}
12307 
12308 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12309   match(If cop cmp);
12310   effect(USE labl);
12311 
12312   ins_cost(300);
12313   format %{ "j$cop,us  $labl" %}
12314   size(2);
12315   ins_encode %{
12316     Label* L = $labl$$label;
12317     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12318   %}
12319   ins_pipe(pipe_jcc);
12320   ins_short_branch(1);
12321 %}
12322 
12323 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12324   match(If cop cmp);
12325   effect(USE labl);
12326 
12327   ins_cost(300);
12328   format %{ $$template
12329     if ($cop$$cmpcode == Assembler::notEqual) {
12330       $$emit$$"jp,u,s   $labl\n\t"
12331       $$emit$$"j$cop,u,s   $labl"
12332     } else {
12333       $$emit$$"jp,u,s   done\n\t"
12334       $$emit$$"j$cop,u,s  $labl\n\t"
12335       $$emit$$"done:"
12336     }
12337   %}
12338   size(4);
12339   ins_encode %{
12340     Label* l = $labl$$label;
12341     if ($cop$$cmpcode == Assembler::notEqual) {
12342       __ jccb(Assembler::parity, *l);
12343       __ jccb(Assembler::notEqual, *l);
12344     } else if ($cop$$cmpcode == Assembler::equal) {
12345       Label done;
12346       __ jccb(Assembler::parity, done);
12347       __ jccb(Assembler::equal, *l);
12348       __ bind(done);
12349     } else {
12350        ShouldNotReachHere();
12351     }
12352   %}
12353   ins_pipe(pipe_jcc);
12354   ins_short_branch(1);
12355 %}
12356 
12357 // ============================================================================
12358 // inlined locking and unlocking
12359 
12360 instruct cmpFastLock(rFlagsReg cr,
12361                      rRegP object, rRegP box, rax_RegI tmp, rRegP scr)
12362 %{
12363   match(Set cr (FastLock object box));
12364   effect(TEMP tmp, TEMP scr);
12365 
12366   ins_cost(300);
12367   format %{ "fastlock $object,$box,$tmp,$scr" %}
12368   ins_encode(Fast_Lock(object, box, tmp, scr));
12369   ins_pipe(pipe_slow);
12370 %}
12371 
12372 instruct cmpFastUnlock(rFlagsReg cr,
12373                        rRegP object, rax_RegP box, rRegP tmp)
12374 %{
12375   match(Set cr (FastUnlock object box));
12376   effect(TEMP tmp);
12377 
12378   ins_cost(300);
12379   format %{ "fastunlock $object, $box, $tmp" %}
12380   ins_encode(Fast_Unlock(object, box, tmp));
12381   ins_pipe(pipe_slow);
12382 %}
12383 
12384 
12385 // ============================================================================
12386 // Safepoint Instructions
12387 instruct safePoint_poll(rFlagsReg cr)
12388 %{
12389   predicate(!Assembler::is_polling_page_far());
12390   match(SafePoint);
12391   effect(KILL cr);
12392 
12393   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12394             "# Safepoint: poll for GC" %}
12395   ins_cost(125);
12396   ins_encode %{
12397     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12398     __ testl(rax, addr);
12399   %}
12400   ins_pipe(ialu_reg_mem);
12401 %}
12402 
12403 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12404 %{
12405   predicate(Assembler::is_polling_page_far());
12406   match(SafePoint poll);
12407   effect(KILL cr, USE poll);
12408 
12409   format %{ "testl  rax, [$poll]\t"
12410             "# Safepoint: poll for GC" %}
12411   ins_cost(125);
12412   ins_encode %{
12413     __ relocate(relocInfo::poll_type);
12414     __ testl(rax, Address($poll$$Register, 0));
12415   %}
12416   ins_pipe(ialu_reg_mem);
12417 %}
12418 
12419 // ============================================================================
12420 // Procedure Call/Return Instructions
12421 // Call Java Static Instruction
12422 // Note: If this code changes, the corresponding ret_addr_offset() and
12423 //       compute_padding() functions will have to be adjusted.
12424 instruct CallStaticJavaDirect(method meth) %{
12425   match(CallStaticJava);
12426   predicate(!((CallStaticJavaNode*) n)->is_method_handle_invoke());
12427   effect(USE meth);
12428 
12429   ins_cost(300);
12430   format %{ "call,static " %}
12431   opcode(0xE8); /* E8 cd */
12432   ins_encode(Java_Static_Call(meth), call_epilog);
12433   ins_pipe(pipe_slow);
12434   ins_alignment(4);
12435 %}
12436 
12437 // Call Java Static Instruction (method handle version)
12438 // Note: If this code changes, the corresponding ret_addr_offset() and
12439 //       compute_padding() functions will have to be adjusted.
12440 instruct CallStaticJavaHandle(method meth, rbp_RegP rbp_mh_SP_save) %{
12441   match(CallStaticJava);
12442   predicate(((CallStaticJavaNode*) n)->is_method_handle_invoke());
12443   effect(USE meth);
12444   // RBP is saved by all callees (for interpreter stack correction).
12445   // We use it here for a similar purpose, in {preserve,restore}_SP.
12446 
12447   ins_cost(300);
12448   format %{ "call,static/MethodHandle " %}
12449   opcode(0xE8); /* E8 cd */
12450   ins_encode(preserve_SP,
12451              Java_Static_Call(meth),
12452              restore_SP,
12453              call_epilog);
12454   ins_pipe(pipe_slow);
12455   ins_alignment(4);
12456 %}
12457 
12458 // Call Java Dynamic Instruction
12459 // Note: If this code changes, the corresponding ret_addr_offset() and
12460 //       compute_padding() functions will have to be adjusted.
12461 instruct CallDynamicJavaDirect(method meth)
12462 %{
12463   match(CallDynamicJava);
12464   effect(USE meth);
12465 
12466   ins_cost(300);
12467   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12468             "call,dynamic " %}
12469   opcode(0xE8); /* E8 cd */
12470   ins_encode(Java_Dynamic_Call(meth), call_epilog);
12471   ins_pipe(pipe_slow);
12472   ins_alignment(4);
12473 %}
12474 
12475 // Call Runtime Instruction
12476 instruct CallRuntimeDirect(method meth)
12477 %{
12478   match(CallRuntime);
12479   effect(USE meth);
12480 
12481   ins_cost(300);
12482   format %{ "call,runtime " %}
12483   opcode(0xE8); /* E8 cd */
12484   ins_encode(Java_To_Runtime(meth));
12485   ins_pipe(pipe_slow);
12486 %}
12487 
12488 // Call runtime without safepoint
12489 instruct CallLeafDirect(method meth)
12490 %{
12491   match(CallLeaf);
12492   effect(USE meth);
12493 
12494   ins_cost(300);
12495   format %{ "call_leaf,runtime " %}
12496   opcode(0xE8); /* E8 cd */
12497   ins_encode(Java_To_Runtime(meth));
12498   ins_pipe(pipe_slow);
12499 %}
12500 
12501 // Call runtime without safepoint
12502 instruct CallLeafNoFPDirect(method meth)
12503 %{
12504   match(CallLeafNoFP);
12505   effect(USE meth);
12506 
12507   ins_cost(300);
12508   format %{ "call_leaf_nofp,runtime " %}
12509   opcode(0xE8); /* E8 cd */
12510   ins_encode(Java_To_Runtime(meth));
12511   ins_pipe(pipe_slow);
12512 %}
12513 
12514 // Return Instruction
12515 // Remove the return address & jump to it.
12516 // Notice: We always emit a nop after a ret to make sure there is room
12517 // for safepoint patching
12518 instruct Ret()
12519 %{
12520   match(Return);
12521 
12522   format %{ "ret" %}
12523   opcode(0xC3);
12524   ins_encode(OpcP);
12525   ins_pipe(pipe_jmp);
12526 %}
12527 
12528 // Tail Call; Jump from runtime stub to Java code.
12529 // Also known as an 'interprocedural jump'.
12530 // Target of jump will eventually return to caller.
12531 // TailJump below removes the return address.
12532 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12533 %{
12534   match(TailCall jump_target method_oop);
12535 
12536   ins_cost(300);
12537   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12538   opcode(0xFF, 0x4); /* Opcode FF /4 */
12539   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12540   ins_pipe(pipe_jmp);
12541 %}
12542 
12543 // Tail Jump; remove the return address; jump to target.
12544 // TailCall above leaves the return address around.
12545 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12546 %{
12547   match(TailJump jump_target ex_oop);
12548 
12549   ins_cost(300);
12550   format %{ "popq    rdx\t# pop return address\n\t"
12551             "jmp     $jump_target" %}
12552   opcode(0xFF, 0x4); /* Opcode FF /4 */
12553   ins_encode(Opcode(0x5a), // popq rdx
12554              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12555   ins_pipe(pipe_jmp);
12556 %}
12557 
12558 // Create exception oop: created by stack-crawling runtime code.
12559 // Created exception is now available to this handler, and is setup
12560 // just prior to jumping to this handler.  No code emitted.
12561 instruct CreateException(rax_RegP ex_oop)
12562 %{
12563   match(Set ex_oop (CreateEx));
12564 
12565   size(0);
12566   // use the following format syntax
12567   format %{ "# exception oop is in rax; no code emitted" %}
12568   ins_encode();
12569   ins_pipe(empty);
12570 %}
12571 
12572 // Rethrow exception:
12573 // The exception oop will come in the first argument position.
12574 // Then JUMP (not call) to the rethrow stub code.
12575 instruct RethrowException()
12576 %{
12577   match(Rethrow);
12578 
12579   // use the following format syntax
12580   format %{ "jmp     rethrow_stub" %}
12581   ins_encode(enc_rethrow);
12582   ins_pipe(pipe_jmp);
12583 %}
12584 
12585 
12586 //----------PEEPHOLE RULES-----------------------------------------------------
12587 // These must follow all instruction definitions as they use the names
12588 // defined in the instructions definitions.
12589 //
12590 // peepmatch ( root_instr_name [preceding_instruction]* );
12591 //
12592 // peepconstraint %{
12593 // (instruction_number.operand_name relational_op instruction_number.operand_name
12594 //  [, ...] );
12595 // // instruction numbers are zero-based using left to right order in peepmatch
12596 //
12597 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12598 // // provide an instruction_number.operand_name for each operand that appears
12599 // // in the replacement instruction's match rule
12600 //
12601 // ---------VM FLAGS---------------------------------------------------------
12602 //
12603 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12604 //
12605 // Each peephole rule is given an identifying number starting with zero and
12606 // increasing by one in the order seen by the parser.  An individual peephole
12607 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12608 // on the command-line.
12609 //
12610 // ---------CURRENT LIMITATIONS----------------------------------------------
12611 //
12612 // Only match adjacent instructions in same basic block
12613 // Only equality constraints
12614 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12615 // Only one replacement instruction
12616 //
12617 // ---------EXAMPLE----------------------------------------------------------
12618 //
12619 // // pertinent parts of existing instructions in architecture description
12620 // instruct movI(rRegI dst, rRegI src)
12621 // %{
12622 //   match(Set dst (CopyI src));
12623 // %}
12624 //
12625 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12626 // %{
12627 //   match(Set dst (AddI dst src));
12628 //   effect(KILL cr);
12629 // %}
12630 //
12631 // // Change (inc mov) to lea
12632 // peephole %{
12633 //   // increment preceeded by register-register move
12634 //   peepmatch ( incI_rReg movI );
12635 //   // require that the destination register of the increment
12636 //   // match the destination register of the move
12637 //   peepconstraint ( 0.dst == 1.dst );
12638 //   // construct a replacement instruction that sets
12639 //   // the destination to ( move's source register + one )
12640 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12641 // %}
12642 //
12643 
12644 // Implementation no longer uses movX instructions since
12645 // machine-independent system no longer uses CopyX nodes.
12646 //
12647 // peephole
12648 // %{
12649 //   peepmatch (incI_rReg movI);
12650 //   peepconstraint (0.dst == 1.dst);
12651 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12652 // %}
12653 
12654 // peephole
12655 // %{
12656 //   peepmatch (decI_rReg movI);
12657 //   peepconstraint (0.dst == 1.dst);
12658 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12659 // %}
12660 
12661 // peephole
12662 // %{
12663 //   peepmatch (addI_rReg_imm movI);
12664 //   peepconstraint (0.dst == 1.dst);
12665 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12666 // %}
12667 
12668 // peephole
12669 // %{
12670 //   peepmatch (incL_rReg movL);
12671 //   peepconstraint (0.dst == 1.dst);
12672 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12673 // %}
12674 
12675 // peephole
12676 // %{
12677 //   peepmatch (decL_rReg movL);
12678 //   peepconstraint (0.dst == 1.dst);
12679 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12680 // %}
12681 
12682 // peephole
12683 // %{
12684 //   peepmatch (addL_rReg_imm movL);
12685 //   peepconstraint (0.dst == 1.dst);
12686 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12687 // %}
12688 
12689 // peephole
12690 // %{
12691 //   peepmatch (addP_rReg_imm movP);
12692 //   peepconstraint (0.dst == 1.dst);
12693 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12694 // %}
12695 
12696 // // Change load of spilled value to only a spill
12697 // instruct storeI(memory mem, rRegI src)
12698 // %{
12699 //   match(Set mem (StoreI mem src));
12700 // %}
12701 //
12702 // instruct loadI(rRegI dst, memory mem)
12703 // %{
12704 //   match(Set dst (LoadI mem));
12705 // %}
12706 //
12707 
12708 peephole
12709 %{
12710   peepmatch (loadI storeI);
12711   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12712   peepreplace (storeI(1.mem 1.mem 1.src));
12713 %}
12714 
12715 peephole
12716 %{
12717   peepmatch (loadL storeL);
12718   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12719   peepreplace (storeL(1.mem 1.mem 1.src));
12720 %}
12721 
12722 //----------SMARTSPILL RULES---------------------------------------------------
12723 // These must follow all instruction definitions as they use the names
12724 // defined in the instructions definitions.