1 //
   2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer/long registers
 173 reg_class all_reg(RAX, RAX_H,
 174                   RDX, RDX_H,
 175                   RBP, RBP_H,
 176                   RDI, RDI_H,
 177                   RSI, RSI_H,
 178                   RCX, RCX_H,
 179                   RBX, RBX_H,
 180                   RSP, RSP_H,
 181                   R8,  R8_H,
 182                   R9,  R9_H,
 183                   R10, R10_H,
 184                   R11, R11_H,
 185                   R12, R12_H,
 186                   R13, R13_H,
 187                   R14, R14_H,
 188                   R15, R15_H);
 189 
 190 // Class for all int registers
 191 reg_class all_int_reg(RAX
 192                       RDX,
 193                       RBP,
 194                       RDI,
 195                       RSI,
 196                       RCX,
 197                       RBX,
 198                       R8,
 199                       R9,
 200                       R10,
 201                       R11,
 202                       R12,
 203                       R13,
 204                       R14);
 205 
 206 // Class for all pointer registers
 207 reg_class any_reg %{
 208   return _ANY_REG_mask;
 209 %}
 210 
 211 // Class for all pointer registers (excluding RSP)
 212 reg_class ptr_reg %{
 213   return _PTR_REG_mask;
 214 %}
 215 
 216 // Class for all pointer registers (excluding RSP and RBP)
 217 reg_class ptr_reg_no_rbp %{
 218   return _PTR_REG_NO_RBP_mask;
 219 %}
 220 
 221 // Class for all pointer registers (excluding RAX and RSP)
 222 reg_class ptr_no_rax_reg %{
 223   return _PTR_NO_RAX_REG_mask;
 224 %}
 225 
 226 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 227 reg_class ptr_no_rax_rbx_reg %{
 228   return _PTR_NO_RAX_RBX_REG_mask;
 229 %}
 230 
 231 // Class for all long registers (excluding RSP)
 232 reg_class long_reg %{
 233   return _LONG_REG_mask;
 234 %}
 235 
 236 // Class for all long registers (excluding RAX, RDX and RSP)
 237 reg_class long_no_rax_rdx_reg %{
 238   return _LONG_NO_RAX_RDX_REG_mask;
 239 %}
 240 
 241 // Class for all long registers (excluding RCX and RSP)
 242 reg_class long_no_rcx_reg %{
 243   return _LONG_NO_RCX_REG_mask;
 244 %}
 245 
 246 // Class for all int registers (excluding RSP)
 247 reg_class int_reg %{
 248   return _INT_REG_mask;
 249 %}
 250 
 251 // Class for all int registers (excluding RAX, RDX, and RSP)
 252 reg_class int_no_rax_rdx_reg %{
 253   return _INT_NO_RAX_RDX_REG_mask;
 254 %}
 255 
 256 // Class for all int registers (excluding RCX and RSP)
 257 reg_class int_no_rcx_reg %{
 258   return _INT_NO_RCX_REG_mask;
 259 %}
 260 
 261 // Class for all pointer registers (including RSP and RBP)
 262 reg_class any_reg0_with_rbp(RAX, RAX_H,
 263                             RDX, RDX_H,
 264                             RBP, RBP_H,
 265                             RDI, RDI_H,
 266                             RSI, RSI_H,
 267                             RCX, RCX_H,
 268                             RBX, RBX_H,
 269                             RSP, RSP_H,
 270                             R8,  R8_H,
 271                             R9,  R9_H,
 272                             R10, R10_H,
 273                             R11, R11_H,
 274                             R12, R12_H,
 275                             R13, R13_H,
 276                             R14, R14_H,
 277                             R15, R15_H);
 278 
 279 // Class for all pointer registers (including RSP, but excluding RBP)
 280 reg_class any_reg0_no_rbp(RAX, RAX_H,
 281                           RDX, RDX_H,
 282                           RDI, RDI_H,
 283                           RSI, RSI_H,
 284                           RCX, RCX_H,
 285                           RBX, RBX_H,
 286                           RSP, RSP_H,
 287                           R8,  R8_H,
 288                           R9,  R9_H,
 289                           R10, R10_H,
 290                           R11, R11_H,
 291                           R12, R12_H,
 292                           R13, R13_H,
 293                           R14, R14_H,
 294                           R15, R15_H);
 295 
 296 // Dynamic register class that selects at runtime between register classes
 297 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 298 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 299 reg_class_dynamic any_reg0(any_reg0_no_rbp, any_reg0_with_rbp, %{ PreserveFramePointer %});
 300 
 301 // Class for all pointer registers
 302 reg_class ptr_reg0_with_rbp(RAX, RAX_H,
 303                             RDX, RDX_H,
 304                             RBP, RBP_H,
 305                             RDI, RDI_H,
 306                             RSI, RSI_H,
 307                             RCX, RCX_H,
 308                             RBX, RBX_H,
 309                             R8,  R8_H,
 310                             R9,  R9_H,
 311                             R10, R10_H,
 312                             R11, R11_H,
 313                             R13, R13_H,
 314                             R14, R14_H);
 315 
 316 reg_class ptr_reg0_no_rbp(RAX, RAX_H,
 317                           RDX, RDX_H,
 318                           RDI, RDI_H,
 319                           RSI, RSI_H,
 320                           RCX, RCX_H,
 321                           RBX, RBX_H,
 322                           R8,  R8_H,
 323                           R9,  R9_H,
 324                           R10, R10_H,
 325                           R11, R11_H,
 326                           R13, R13_H,
 327                           R14, R14_H);
 328 
 329 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 330 reg_class_dynamic ptr_reg0(ptr_reg0_no_rbp, ptr_reg0_with_rbp, %{ PreserveFramePointer %});
 331 
 332 // Class for all pointer registers (excluding RAX and RSP)
 333 reg_class ptr_no_rax_reg0_with_rbp(RDX, RDX_H,
 334                                   RBP, RBP_H,
 335                                   RDI, RDI_H,
 336                                   RSI, RSI_H,
 337                                   RCX, RCX_H,
 338                                   RBX, RBX_H,
 339                                   R8,  R8_H,
 340                                   R9,  R9_H,
 341                                   R10, R10_H,
 342                                   R11, R11_H,
 343                                   R13, R13_H,
 344                                   R14, R14_H);
 345 
 346 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 347 reg_class ptr_no_rax_reg0_no_rbp(RDX, RDX_H,
 348                                 RDI, RDI_H,
 349                                 RSI, RSI_H,
 350                                 RCX, RCX_H,
 351                                 RBX, RBX_H,
 352                                 R8,  R8_H,
 353                                 R9,  R9_H,
 354                                 R10, R10_H,
 355                                 R11, R11_H,
 356                                 R13, R13_H,
 357                                 R14, R14_H);
 358 
 359 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 360 reg_class_dynamic ptr_no_rax_reg0(ptr_no_rax_reg0_no_rbp, ptr_no_rax_reg0_with_rbp, %{ PreserveFramePointer %});
 361 
 362 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 363 reg_class ptr_no_rax_rbx_reg0_with_rbp(RDX, RDX_H,
 364                                       RBP, RBP_H,
 365                                       RDI, RDI_H,
 366                                       RSI, RSI_H,
 367                                       RCX, RCX_H,
 368                                       R8,  R8_H,
 369                                       R9,  R9_H,
 370                                       R10, R10_H,
 371                                       R11, R11_H,
 372                                       R13, R13_H,
 373                                       R14, R14_H);
 374 
 375 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 376 reg_class ptr_no_rax_rbx_reg0_no_rbp(RDX, RDX_H,
 377                                     RDI, RDI_H,
 378                                     RSI, RSI_H,
 379                                     RCX, RCX_H,
 380                                     R8,  R8_H,
 381                                     R9,  R9_H,
 382                                     R10, R10_H,
 383                                     R11, R11_H,
 384                                     R13, R13_H,
 385                                     R14, R14_H);
 386 
 387 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 388 reg_class_dynamic ptr_no_rax_rbx_reg0(ptr_no_rax_rbx_reg0_no_rbp, ptr_no_rax_rbx_reg0_with_rbp, %{ PreserveFramePointer %});
 389 
 390 // Singleton class for RAX pointer register
 391 reg_class ptr_rax_reg(RAX, RAX_H);
 392 
 393 // Singleton class for RBX pointer register
 394 reg_class ptr_rbx_reg(RBX, RBX_H);
 395 
 396 // Singleton class for RSI pointer register
 397 reg_class ptr_rsi_reg(RSI, RSI_H);
 398 
 399 // Singleton class for RDI pointer register
 400 reg_class ptr_rdi_reg(RDI, RDI_H);
 401 
 402 // Singleton class for stack pointer
 403 reg_class ptr_rsp_reg(RSP, RSP_H);
 404 
 405 // Singleton class for TLS pointer
 406 reg_class ptr_r15_reg(R15, R15_H);
 407 
 408 // Class for all long registers (excluding RSP)
 409 reg_class long_reg0_with_rbp(RAX, RAX_H,
 410                             RDX, RDX_H,
 411                             RBP, RBP_H,
 412                             RDI, RDI_H,
 413                             RSI, RSI_H,
 414                             RCX, RCX_H,
 415                             RBX, RBX_H,
 416                             R8,  R8_H,
 417                             R9,  R9_H,
 418                             R10, R10_H,
 419                             R11, R11_H,
 420                             R13, R13_H,
 421                             R14, R14_H);
 422 
 423 // Class for all long registers (excluding RSP and RBP)
 424 reg_class long_reg0_no_rbp(RAX, RAX_H,
 425                           RDX, RDX_H,
 426                           RDI, RDI_H,
 427                           RSI, RSI_H,
 428                           RCX, RCX_H,
 429                           RBX, RBX_H,
 430                           R8,  R8_H,
 431                           R9,  R9_H,
 432                           R10, R10_H,
 433                           R11, R11_H,
 434                           R13, R13_H,
 435                           R14, R14_H);
 436 
 437 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 438 reg_class_dynamic long_reg0(long_reg0_no_rbp, long_reg0_with_rbp, %{ PreserveFramePointer %});
 439 
 440 // Class for all long registers (excluding RAX, RDX and RSP)
 441 reg_class long_no_rax_rdx_reg0_with_rbp(RBP, RBP_H,
 442                                        RDI, RDI_H,
 443                                        RSI, RSI_H,
 444                                        RCX, RCX_H,
 445                                        RBX, RBX_H,
 446                                        R8,  R8_H,
 447                                        R9,  R9_H,
 448                                        R10, R10_H,
 449                                        R11, R11_H,
 450                                        R13, R13_H,
 451                                        R14, R14_H);
 452 
 453 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 454 reg_class long_no_rax_rdx_reg0_no_rbp(RDI, RDI_H,
 455                                      RSI, RSI_H,
 456                                      RCX, RCX_H,
 457                                      RBX, RBX_H,
 458                                      R8,  R8_H,
 459                                      R9,  R9_H,
 460                                      R10, R10_H,
 461                                      R11, R11_H,
 462                                      R13, R13_H,
 463                                      R14, R14_H);
 464 
 465 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 466 reg_class_dynamic long_no_rax_rdx_reg0(long_no_rax_rdx_reg0_no_rbp, long_no_rax_rdx_reg0_with_rbp, %{ PreserveFramePointer %});
 467 
 468 // Class for all long registers (excluding RCX and RSP)
 469 reg_class long_no_rcx_reg0_with_rbp(RBP, RBP_H,
 470                                    RDI, RDI_H,
 471                                    RSI, RSI_H,
 472                                    RAX, RAX_H,
 473                                    RDX, RDX_H,
 474                                    RBX, RBX_H,
 475                                    R8,  R8_H,
 476                                    R9,  R9_H,
 477                                    R10, R10_H,
 478                                    R11, R11_H,
 479                                    R13, R13_H,
 480                                    R14, R14_H);
 481 
 482 // Class for all long registers (excluding RCX, RSP, and RBP)
 483 reg_class long_no_rcx_reg0_no_rbp(RDI, RDI_H,
 484                                  RSI, RSI_H,
 485                                  RAX, RAX_H,
 486                                  RDX, RDX_H,
 487                                  RBX, RBX_H,
 488                                  R8,  R8_H,
 489                                  R9,  R9_H,
 490                                  R10, R10_H,
 491                                  R11, R11_H,
 492                                  R13, R13_H,
 493                                  R14, R14_H);
 494 
 495 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 496 reg_class_dynamic long_no_rcx_reg0(long_no_rcx_reg0_no_rbp, long_no_rcx_reg0_with_rbp, %{ PreserveFramePointer %});
 497 
 498 // Singleton class for RAX long register
 499 reg_class long_rax_reg(RAX, RAX_H);
 500 
 501 // Singleton class for RCX long register
 502 reg_class long_rcx_reg(RCX, RCX_H);
 503 
 504 // Singleton class for RDX long register
 505 reg_class long_rdx_reg(RDX, RDX_H);
 506 
 507 // Class for all int registers (excluding RSP)
 508 reg_class int_reg0_with_rbp(RAX,
 509                            RDX,
 510                            RBP,
 511                            RDI,
 512                            RSI,
 513                            RCX,
 514                            RBX,
 515                            R8,
 516                            R9,
 517                            R10,
 518                            R11,
 519                            R13,
 520                            R14);
 521 
 522 // Class for all int registers (excluding RSP and RBP)
 523 reg_class int_reg0_no_rbp(RAX,
 524                          RDX,
 525                          RDI,
 526                          RSI,
 527                          RCX,
 528                          RBX,
 529                          R8,
 530                          R9,
 531                          R10,
 532                          R11,
 533                          R13,
 534                          R14);
 535 
 536 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 537 reg_class_dynamic int_reg0(int_reg0_no_rbp, int_reg0_with_rbp, %{ PreserveFramePointer %});
 538 
 539 // Class for all int registers (excluding RCX and RSP)
 540 reg_class int_no_rcx_reg0_with_rbp(RAX,
 541                                   RDX,
 542                                   RBP,
 543                                   RDI,
 544                                   RSI,
 545                                   RBX,
 546                                   R8,
 547                                   R9,
 548                                   R10,
 549                                   R11,
 550                                   R13,
 551                                   R14);
 552 
 553 // Class for all int registers (excluding RCX, RSP, and RBP)
 554 reg_class int_no_rcx_reg0_no_rbp(RAX,
 555                                 RDX,
 556                                 RDI,
 557                                 RSI,
 558                                 RBX,
 559                                 R8,
 560                                 R9,
 561                                 R10,
 562                                 R11,
 563                                 R13,
 564                                 R14);
 565 
 566 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 567 reg_class_dynamic int_no_rcx_reg0(int_no_rcx_reg0_no_rbp, int_no_rcx_reg0_with_rbp, %{ PreserveFramePointer %});
 568 
 569 // Class for all int registers (excluding RAX, RDX, and RSP)
 570 reg_class int_no_rax_rdx_reg0_with_rbp(RBP,
 571                                       RDI,
 572                                       RSI,
 573                                       RCX,
 574                                       RBX,
 575                                       R8,
 576                                       R9,
 577                                       R10,
 578                                       R11,
 579                                       R13,
 580                                       R14);
 581 
 582 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 583 reg_class int_no_rax_rdx_reg0_no_rbp(RDI,
 584                                     RSI,
 585                                     RCX,
 586                                     RBX,
 587                                     R8,
 588                                     R9,
 589                                     R10,
 590                                     R11,
 591                                     R13,
 592                                     R14);
 593 
 594 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 595 reg_class_dynamic int_no_rax_rdx_reg0(int_no_rax_rdx_reg0_no_rbp, int_no_rax_rdx_reg0_with_rbp, %{ PreserveFramePointer %});
 596 
 597 // Singleton class for RAX int register
 598 reg_class int_rax_reg(RAX);
 599 
 600 // Singleton class for RBX int register
 601 reg_class int_rbx_reg(RBX);
 602 
 603 // Singleton class for RCX int register
 604 reg_class int_rcx_reg(RCX);
 605 
 606 // Singleton class for RCX int register
 607 reg_class int_rdx_reg(RDX);
 608 
 609 // Singleton class for RCX int register
 610 reg_class int_rdi_reg(RDI);
 611 
 612 // Singleton class for instruction pointer
 613 // reg_class ip_reg(RIP);
 614 
 615 %}
 616 
 617 //----------SOURCE BLOCK-------------------------------------------------------
 618 // This is a block of C++ code which provides values, functions, and
 619 // definitions necessary in the rest of the architecture description
 620 source_hpp %{
 621 
 622 extern RegMask _ANY_REG_mask;
 623 extern RegMask _PTR_REG_mask;
 624 extern RegMask _PTR_REG_NO_RBP_mask;
 625 extern RegMask _PTR_NO_RAX_REG_mask;
 626 extern RegMask _PTR_NO_RAX_RBX_REG_mask;
 627 extern RegMask _LONG_REG_mask;
 628 extern RegMask _LONG_NO_RAX_RDX_REG_mask;
 629 extern RegMask _LONG_NO_RCX_REG_mask;
 630 extern RegMask _INT_REG_mask;
 631 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 632 extern RegMask _INT_NO_RCX_REG_mask;
 633 
 634 extern RegMask _STACK_OR_PTR_REG_mask;
 635 extern RegMask _STACK_OR_LONG_REG_mask;
 636 extern RegMask _STACK_OR_INT_REG_mask;
 637 
 638 inline const RegMask& STACK_OR_PTR_REG_mask()  { return _STACK_OR_PTR_REG_mask;  }
 639 inline const RegMask& STACK_OR_LONG_REG_mask() { return _STACK_OR_LONG_REG_mask; }
 640 inline const RegMask& STACK_OR_INT_REG_mask()  { return _STACK_OR_INT_REG_mask;  }
 641 
 642 %}
 643 
 644 source %{
 645 #define   RELOC_IMM64    Assembler::imm_operand
 646 #define   RELOC_DISP32   Assembler::disp32_operand
 647 
 648 #define __ _masm.
 649 
 650 RegMask _ANY_REG_mask;
 651 RegMask _PTR_REG_mask;
 652 RegMask _PTR_REG_NO_RBP_mask;
 653 RegMask _PTR_NO_RAX_REG_mask;
 654 RegMask _PTR_NO_RAX_RBX_REG_mask;
 655 RegMask _LONG_REG_mask;
 656 RegMask _LONG_NO_RAX_RDX_REG_mask;
 657 RegMask _LONG_NO_RCX_REG_mask;
 658 RegMask _INT_REG_mask;
 659 RegMask _INT_NO_RAX_RDX_REG_mask;
 660 RegMask _INT_NO_RCX_REG_mask;
 661 RegMask _STACK_OR_PTR_REG_mask;
 662 RegMask _STACK_OR_LONG_REG_mask;
 663 RegMask _STACK_OR_INT_REG_mask;
 664 
 665 #ifdef ASSERT
 666 static bool same_mask(const RegMask &a, const RegMask &b) {
 667     RegMask a_sub_b = a; a_sub_b.SUBTRACT(b);
 668     RegMask b_sub_a = b; b_sub_a.SUBTRACT(a);
 669     return a_sub_b.Size() == 0 && b_sub_a.Size() == 0;
 670 }
 671 #endif
 672 
 673 void reg_mask_init() {
 674   // _ALL_REG_mask is generated by adlc from the all_reg register class below.
 675   // We derive a number of subsets from it.
 676   _ANY_REG_mask = _ALL_REG_mask;
 677 
 678   if (PreserveFramePointer) {
 679     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 680     _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 681   }
 682 
 683   _PTR_REG_mask = _ANY_REG_mask;
 684   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 685   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
 686   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
 687   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
 688   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
 689   _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
 690 
 691   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
 692   _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 693 
 694   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
 695   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 696   _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 697 
 698   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
 699   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 700   _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 701 
 702   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
 703   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
 704   _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 705 
 706   _LONG_REG_mask = _PTR_REG_mask;
 707   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
 708   _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 709 
 710   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
 711   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 712   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 713   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 714   _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 715 
 716   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
 717   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 718   _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 719 
 720   _INT_REG_mask = _ALL_INT_REG_mask;
 721   _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
 722   if (PreserveFramePointer) {
 723     _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
 724   }
 725 
 726   _STACK_OR_INT_REG_mask = _INT_REG_mask;
 727   _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
 728 
 729   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
 730   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
 731   _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 732 
 733   _INT_NO_RCX_REG_mask = _INT_REG_mask;
 734   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 735 
 736   assert(same_mask(_ANY_REG_mask, ANY_REG0_mask()), "!");
 737   assert(same_mask(_PTR_REG_mask, PTR_REG0_mask()), "!");
 738   assert(same_mask(_PTR_REG_NO_RBP_mask, PTR_REG0_NO_RBP_mask()), "!");
 739   assert(same_mask(_PTR_NO_RAX_REG_mask, PTR_NO_RAX_REG0_mask()), "!");
 740   assert(same_mask(_PTR_NO_RAX_RBX_REG_mask, PTR_NO_RAX_RBX_REG0_mask()), "!");
 741   assert(same_mask(_LONG_REG_mask, LONG_REG0_mask()), "!");
 742   assert(same_mask(_LONG_NO_RAX_RDX_REG_mask, LONG_NO_RAX_RDX_REG0_mask()), "!");
 743   assert(same_mask(_LONG_NO_RCX_REG_mask, LONG_NO_RCX_REG0_mask()), "!");
 744   assert(same_mask(_INT_REG_mask, INT_REG0_mask()), "!");
 745   assert(same_mask(_INT_NO_RAX_RDX_REG_mask, INT_NO_RAX_RDX_REG0_mask()), "!");
 746   assert(same_mask(_INT_NO_RCX_REG_mask, INT_NO_RCX_REG0_mask()), "!");
 747 }
 748 
 749 static bool generate_vzeroupper(Compile* C) {
 750   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 751 }
 752 
 753 static int clear_avx_size() {
 754   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 755 }
 756 
 757 // !!!!! Special hack to get all types of calls to specify the byte offset
 758 //       from the start of the call to the point where the return address
 759 //       will point.
 760 int MachCallStaticJavaNode::ret_addr_offset()
 761 {
 762   int offset = 5; // 5 bytes from start of call to where return address points
 763   offset += clear_avx_size();
 764   return offset;
 765 }
 766 
 767 int MachCallDynamicJavaNode::ret_addr_offset()
 768 {
 769   int offset = 15; // 15 bytes from start of call to where return address points
 770   offset += clear_avx_size();
 771   return offset;
 772 }
 773 
 774 int MachCallRuntimeNode::ret_addr_offset() {
 775   int offset = 13; // movq r10,#addr; callq (r10)
 776   offset += clear_avx_size();
 777   return offset;
 778 }
 779 
 780 // Indicate if the safepoint node needs the polling page as an input,
 781 // it does if the polling page is more than disp32 away.
 782 bool SafePointNode::needs_polling_address_input()
 783 {
 784   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 785 }
 786 
 787 //
 788 // Compute padding required for nodes which need alignment
 789 //
 790 
 791 // The address of the call instruction needs to be 4-byte aligned to
 792 // ensure that it does not span a cache line so that it can be patched.
 793 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 794 {
 795   current_offset += clear_avx_size(); // skip vzeroupper
 796   current_offset += 1; // skip call opcode byte
 797   return align_up(current_offset, alignment_required()) - current_offset;
 798 }
 799 
 800 // The address of the call instruction needs to be 4-byte aligned to
 801 // ensure that it does not span a cache line so that it can be patched.
 802 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 803 {
 804   current_offset += clear_avx_size(); // skip vzeroupper
 805   current_offset += 11; // skip movq instruction + call opcode byte
 806   return align_up(current_offset, alignment_required()) - current_offset;
 807 }
 808 
 809 // EMIT_RM()
 810 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 811   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 812   cbuf.insts()->emit_int8(c);
 813 }
 814 
 815 // EMIT_CC()
 816 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 817   unsigned char c = (unsigned char) (f1 | f2);
 818   cbuf.insts()->emit_int8(c);
 819 }
 820 
 821 // EMIT_OPCODE()
 822 void emit_opcode(CodeBuffer &cbuf, int code) {
 823   cbuf.insts()->emit_int8((unsigned char) code);
 824 }
 825 
 826 // EMIT_OPCODE() w/ relocation information
 827 void emit_opcode(CodeBuffer &cbuf,
 828                  int code, relocInfo::relocType reloc, int offset, int format)
 829 {
 830   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 831   emit_opcode(cbuf, code);
 832 }
 833 
 834 // EMIT_D8()
 835 void emit_d8(CodeBuffer &cbuf, int d8) {
 836   cbuf.insts()->emit_int8((unsigned char) d8);
 837 }
 838 
 839 // EMIT_D16()
 840 void emit_d16(CodeBuffer &cbuf, int d16) {
 841   cbuf.insts()->emit_int16(d16);
 842 }
 843 
 844 // EMIT_D32()
 845 void emit_d32(CodeBuffer &cbuf, int d32) {
 846   cbuf.insts()->emit_int32(d32);
 847 }
 848 
 849 // EMIT_D64()
 850 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 851   cbuf.insts()->emit_int64(d64);
 852 }
 853 
 854 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 855 void emit_d32_reloc(CodeBuffer& cbuf,
 856                     int d32,
 857                     relocInfo::relocType reloc,
 858                     int format)
 859 {
 860   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 861   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 862   cbuf.insts()->emit_int32(d32);
 863 }
 864 
 865 // emit 32 bit value and construct relocation entry from RelocationHolder
 866 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 867 #ifdef ASSERT
 868   if (rspec.reloc()->type() == relocInfo::oop_type &&
 869       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 870     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 871     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 872   }
 873 #endif
 874   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 875   cbuf.insts()->emit_int32(d32);
 876 }
 877 
 878 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 879   address next_ip = cbuf.insts_end() + 4;
 880   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 881                  external_word_Relocation::spec(addr),
 882                  RELOC_DISP32);
 883 }
 884 
 885 
 886 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 887 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 888   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 889   cbuf.insts()->emit_int64(d64);
 890 }
 891 
 892 // emit 64 bit value and construct relocation entry from RelocationHolder
 893 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 894 #ifdef ASSERT
 895   if (rspec.reloc()->type() == relocInfo::oop_type &&
 896       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 897     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 898     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 899            "cannot embed scavengable oops in code");
 900   }
 901 #endif
 902   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 903   cbuf.insts()->emit_int64(d64);
 904 }
 905 
 906 // Access stack slot for load or store
 907 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 908 {
 909   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 910   if (-0x80 <= disp && disp < 0x80) {
 911     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 912     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 913     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 914   } else {
 915     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 916     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 917     emit_d32(cbuf, disp);     // Displacement // R/M byte
 918   }
 919 }
 920 
 921    // rRegI ereg, memory mem) %{    // emit_reg_mem
 922 void encode_RegMem(CodeBuffer &cbuf,
 923                    int reg,
 924                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 925 {
 926   assert(disp_reloc == relocInfo::none, "cannot have disp");
 927   int regenc = reg & 7;
 928   int baseenc = base & 7;
 929   int indexenc = index & 7;
 930 
 931   // There is no index & no scale, use form without SIB byte
 932   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 933     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 934     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 935       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 936     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 937       // If 8-bit displacement, mode 0x1
 938       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 939       emit_d8(cbuf, disp);
 940     } else {
 941       // If 32-bit displacement
 942       if (base == -1) { // Special flag for absolute address
 943         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 944         if (disp_reloc != relocInfo::none) {
 945           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 946         } else {
 947           emit_d32(cbuf, disp);
 948         }
 949       } else {
 950         // Normal base + offset
 951         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 952         if (disp_reloc != relocInfo::none) {
 953           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 954         } else {
 955           emit_d32(cbuf, disp);
 956         }
 957       }
 958     }
 959   } else {
 960     // Else, encode with the SIB byte
 961     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 962     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 963       // If no displacement
 964       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 965       emit_rm(cbuf, scale, indexenc, baseenc);
 966     } else {
 967       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 968         // If 8-bit displacement, mode 0x1
 969         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 970         emit_rm(cbuf, scale, indexenc, baseenc);
 971         emit_d8(cbuf, disp);
 972       } else {
 973         // If 32-bit displacement
 974         if (base == 0x04 ) {
 975           emit_rm(cbuf, 0x2, regenc, 0x4);
 976           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 977         } else {
 978           emit_rm(cbuf, 0x2, regenc, 0x4);
 979           emit_rm(cbuf, scale, indexenc, baseenc); // *
 980         }
 981         if (disp_reloc != relocInfo::none) {
 982           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 983         } else {
 984           emit_d32(cbuf, disp);
 985         }
 986       }
 987     }
 988   }
 989 }
 990 
 991 // This could be in MacroAssembler but it's fairly C2 specific
 992 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 993   Label exit;
 994   __ jccb(Assembler::noParity, exit);
 995   __ pushf();
 996   //
 997   // comiss/ucomiss instructions set ZF,PF,CF flags and
 998   // zero OF,AF,SF for NaN values.
 999   // Fixup flags by zeroing ZF,PF so that compare of NaN
1000   // values returns 'less than' result (CF is set).
1001   // Leave the rest of flags unchanged.
1002   //
1003   //    7 6 5 4 3 2 1 0
1004   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
1005   //    0 0 1 0 1 0 1 1   (0x2B)
1006   //
1007   __ andq(Address(rsp, 0), 0xffffff2b);
1008   __ popf();
1009   __ bind(exit);
1010 }
1011 
1012 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
1013   Label done;
1014   __ movl(dst, -1);
1015   __ jcc(Assembler::parity, done);
1016   __ jcc(Assembler::below, done);
1017   __ setb(Assembler::notEqual, dst);
1018   __ movzbl(dst, dst);
1019   __ bind(done);
1020 }
1021 
1022 
1023 //=============================================================================
1024 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1025 
1026 int Compile::ConstantTable::calculate_table_base_offset() const {
1027   return 0;  // absolute addressing, no offset
1028 }
1029 
1030 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1031 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1032   ShouldNotReachHere();
1033 }
1034 
1035 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1036   // Empty encoding
1037 }
1038 
1039 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1040   return 0;
1041 }
1042 
1043 #ifndef PRODUCT
1044 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1045   st->print("# MachConstantBaseNode (empty encoding)");
1046 }
1047 #endif
1048 
1049 
1050 //=============================================================================
1051 #ifndef PRODUCT
1052 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1053   Compile* C = ra_->C;
1054 
1055   int framesize = C->frame_size_in_bytes();
1056   int bangsize = C->bang_size_in_bytes();
1057   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1058   // Remove wordSize for return addr which is already pushed.
1059   framesize -= wordSize;
1060 
1061   if (C->need_stack_bang(bangsize)) {
1062     framesize -= wordSize;
1063     st->print("# stack bang (%d bytes)", bangsize);
1064     st->print("\n\t");
1065     st->print("pushq   rbp\t# Save rbp");
1066     if (PreserveFramePointer) {
1067         st->print("\n\t");
1068         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
1069     }
1070     if (framesize) {
1071       st->print("\n\t");
1072       st->print("subq    rsp, #%d\t# Create frame",framesize);
1073     }
1074   } else {
1075     st->print("subq    rsp, #%d\t# Create frame",framesize);
1076     st->print("\n\t");
1077     framesize -= wordSize;
1078     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
1079     if (PreserveFramePointer) {
1080       st->print("\n\t");
1081       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
1082       if (framesize > 0) {
1083         st->print("\n\t");
1084         st->print("addq    rbp, #%d", framesize);
1085       }
1086     }
1087   }
1088 
1089   if (VerifyStackAtCalls) {
1090     st->print("\n\t");
1091     framesize -= wordSize;
1092     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
1093 #ifdef ASSERT
1094     st->print("\n\t");
1095     st->print("# stack alignment check");
1096 #endif
1097   }
1098   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1099     st->print("\n\t");
1100     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
1101     st->print("\n\t");
1102     st->print("je      fast_entry\t");
1103     st->print("\n\t");
1104     st->print("call    #nmethod_entry_barrier_stub\t");
1105     st->print("\n\tfast_entry:");
1106   }
1107   st->cr();
1108 }
1109 #endif
1110 
1111 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1112   Compile* C = ra_->C;
1113   MacroAssembler _masm(&cbuf);
1114 
1115   int framesize = C->frame_size_in_bytes();
1116   int bangsize = C->bang_size_in_bytes();
1117 
1118   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
1119 
1120   C->set_frame_complete(cbuf.insts_size());
1121 
1122   if (C->has_mach_constant_base_node()) {
1123     // NOTE: We set the table base offset here because users might be
1124     // emitted before MachConstantBaseNode.
1125     Compile::ConstantTable& constant_table = C->constant_table();
1126     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1127   }
1128 }
1129 
1130 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1131 {
1132   return MachNode::size(ra_); // too many variables; just compute it
1133                               // the hard way
1134 }
1135 
1136 int MachPrologNode::reloc() const
1137 {
1138   return 0; // a large enough number
1139 }
1140 
1141 //=============================================================================
1142 #ifndef PRODUCT
1143 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1144 {
1145   Compile* C = ra_->C;
1146   if (generate_vzeroupper(C)) {
1147     st->print("vzeroupper");
1148     st->cr(); st->print("\t");
1149   }
1150 
1151   int framesize = C->frame_size_in_bytes();
1152   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1153   // Remove word for return adr already pushed
1154   // and RBP
1155   framesize -= 2*wordSize;
1156 
1157   if (framesize) {
1158     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
1159     st->print("\t");
1160   }
1161 
1162   st->print_cr("popq   rbp");
1163   if (do_polling() && C->is_method_compilation()) {
1164     st->print("\t");
1165     if (SafepointMechanism::uses_thread_local_poll()) {
1166       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
1167                    "testl  rax, [rscratch1]\t"
1168                    "# Safepoint: poll for GC");
1169     } else if (Assembler::is_polling_page_far()) {
1170       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
1171                    "testl  rax, [rscratch1]\t"
1172                    "# Safepoint: poll for GC");
1173     } else {
1174       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
1175                    "# Safepoint: poll for GC");
1176     }
1177   }
1178 }
1179 #endif
1180 
1181 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1182 {
1183   Compile* C = ra_->C;
1184   MacroAssembler _masm(&cbuf);
1185 
1186   if (generate_vzeroupper(C)) {
1187     // Clear upper bits of YMM registers when current compiled code uses
1188     // wide vectors to avoid AVX <-> SSE transition penalty during call.
1189     __ vzeroupper();
1190   }
1191 
1192   int framesize = C->frame_size_in_bytes();
1193   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
1194   // Remove word for return adr already pushed
1195   // and RBP
1196   framesize -= 2*wordSize;
1197 
1198   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
1199 
1200   if (framesize) {
1201     emit_opcode(cbuf, Assembler::REX_W);
1202     if (framesize < 0x80) {
1203       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
1204       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1205       emit_d8(cbuf, framesize);
1206     } else {
1207       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1208       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1209       emit_d32(cbuf, framesize);
1210     }
1211   }
1212 
1213   // popq rbp
1214   emit_opcode(cbuf, 0x58 | RBP_enc);
1215 
1216   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1217     __ reserved_stack_check();
1218   }
1219 
1220   if (do_polling() && C->is_method_compilation()) {
1221     MacroAssembler _masm(&cbuf);
1222     if (SafepointMechanism::uses_thread_local_poll()) {
1223       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1224       __ relocate(relocInfo::poll_return_type);
1225       __ testl(rax, Address(rscratch1, 0));
1226     } else {
1227       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1228       if (Assembler::is_polling_page_far()) {
1229         __ lea(rscratch1, polling_page);
1230         __ relocate(relocInfo::poll_return_type);
1231         __ testl(rax, Address(rscratch1, 0));
1232       } else {
1233         __ testl(rax, polling_page);
1234       }
1235     }
1236   }
1237 }
1238 
1239 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1240 {
1241   return MachNode::size(ra_); // too many variables; just compute it
1242                               // the hard way
1243 }
1244 
1245 int MachEpilogNode::reloc() const
1246 {
1247   return 2; // a large enough number
1248 }
1249 
1250 const Pipeline* MachEpilogNode::pipeline() const
1251 {
1252   return MachNode::pipeline_class();
1253 }
1254 
1255 int MachEpilogNode::safepoint_offset() const
1256 {
1257   return 0;
1258 }
1259 
1260 //=============================================================================
1261 
1262 enum RC {
1263   rc_bad,
1264   rc_int,
1265   rc_float,
1266   rc_stack
1267 };
1268 
1269 static enum RC rc_class(OptoReg::Name reg)
1270 {
1271   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1272 
1273   if (OptoReg::is_stack(reg)) return rc_stack;
1274 
1275   VMReg r = OptoReg::as_VMReg(reg);
1276 
1277   if (r->is_Register()) return rc_int;
1278 
1279   assert(r->is_XMMRegister(), "must be");
1280   return rc_float;
1281 }
1282 
1283 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1284 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1285                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1286 
1287 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1288                             int stack_offset, int reg, uint ireg, outputStream* st);
1289 
1290 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1291                                       int dst_offset, uint ireg, outputStream* st) {
1292   if (cbuf) {
1293     MacroAssembler _masm(cbuf);
1294     switch (ireg) {
1295     case Op_VecS:
1296       __ movq(Address(rsp, -8), rax);
1297       __ movl(rax, Address(rsp, src_offset));
1298       __ movl(Address(rsp, dst_offset), rax);
1299       __ movq(rax, Address(rsp, -8));
1300       break;
1301     case Op_VecD:
1302       __ pushq(Address(rsp, src_offset));
1303       __ popq (Address(rsp, dst_offset));
1304       break;
1305     case Op_VecX:
1306       __ pushq(Address(rsp, src_offset));
1307       __ popq (Address(rsp, dst_offset));
1308       __ pushq(Address(rsp, src_offset+8));
1309       __ popq (Address(rsp, dst_offset+8));
1310       break;
1311     case Op_VecY:
1312       __ vmovdqu(Address(rsp, -32), xmm0);
1313       __ vmovdqu(xmm0, Address(rsp, src_offset));
1314       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1315       __ vmovdqu(xmm0, Address(rsp, -32));
1316       break;
1317     case Op_VecZ:
1318       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1319       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1320       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1321       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1322       break;
1323     default:
1324       ShouldNotReachHere();
1325     }
1326 #ifndef PRODUCT
1327   } else {
1328     switch (ireg) {
1329     case Op_VecS:
1330       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1331                 "movl    rax, [rsp + #%d]\n\t"
1332                 "movl    [rsp + #%d], rax\n\t"
1333                 "movq    rax, [rsp - #8]",
1334                 src_offset, dst_offset);
1335       break;
1336     case Op_VecD:
1337       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1338                 "popq    [rsp + #%d]",
1339                 src_offset, dst_offset);
1340       break;
1341      case Op_VecX:
1342       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1343                 "popq    [rsp + #%d]\n\t"
1344                 "pushq   [rsp + #%d]\n\t"
1345                 "popq    [rsp + #%d]",
1346                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1347       break;
1348     case Op_VecY:
1349       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1350                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1351                 "vmovdqu [rsp + #%d], xmm0\n\t"
1352                 "vmovdqu xmm0, [rsp - #32]",
1353                 src_offset, dst_offset);
1354       break;
1355     case Op_VecZ:
1356       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1357                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1358                 "vmovdqu [rsp + #%d], xmm0\n\t"
1359                 "vmovdqu xmm0, [rsp - #64]",
1360                 src_offset, dst_offset);
1361       break;
1362     default:
1363       ShouldNotReachHere();
1364     }
1365 #endif
1366   }
1367 }
1368 
1369 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1370                                        PhaseRegAlloc* ra_,
1371                                        bool do_size,
1372                                        outputStream* st) const {
1373   assert(cbuf != NULL || st  != NULL, "sanity");
1374   // Get registers to move
1375   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1376   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1377   OptoReg::Name dst_second = ra_->get_reg_second(this);
1378   OptoReg::Name dst_first = ra_->get_reg_first(this);
1379 
1380   enum RC src_second_rc = rc_class(src_second);
1381   enum RC src_first_rc = rc_class(src_first);
1382   enum RC dst_second_rc = rc_class(dst_second);
1383   enum RC dst_first_rc = rc_class(dst_first);
1384 
1385   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1386          "must move at least 1 register" );
1387 
1388   if (src_first == dst_first && src_second == dst_second) {
1389     // Self copy, no move
1390     return 0;
1391   }
1392   if (bottom_type()->isa_vect() != NULL) {
1393     uint ireg = ideal_reg();
1394     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1395     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1396     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1397       // mem -> mem
1398       int src_offset = ra_->reg2offset(src_first);
1399       int dst_offset = ra_->reg2offset(dst_first);
1400       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1401     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1402       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1403     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1404       int stack_offset = ra_->reg2offset(dst_first);
1405       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1406     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1407       int stack_offset = ra_->reg2offset(src_first);
1408       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1409     } else {
1410       ShouldNotReachHere();
1411     }
1412     return 0;
1413   }
1414   if (src_first_rc == rc_stack) {
1415     // mem ->
1416     if (dst_first_rc == rc_stack) {
1417       // mem -> mem
1418       assert(src_second != dst_first, "overlap");
1419       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1420           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1421         // 64-bit
1422         int src_offset = ra_->reg2offset(src_first);
1423         int dst_offset = ra_->reg2offset(dst_first);
1424         if (cbuf) {
1425           MacroAssembler _masm(cbuf);
1426           __ pushq(Address(rsp, src_offset));
1427           __ popq (Address(rsp, dst_offset));
1428 #ifndef PRODUCT
1429         } else {
1430           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1431                     "popq    [rsp + #%d]",
1432                      src_offset, dst_offset);
1433 #endif
1434         }
1435       } else {
1436         // 32-bit
1437         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1438         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1439         // No pushl/popl, so:
1440         int src_offset = ra_->reg2offset(src_first);
1441         int dst_offset = ra_->reg2offset(dst_first);
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movq(Address(rsp, -8), rax);
1445           __ movl(rax, Address(rsp, src_offset));
1446           __ movl(Address(rsp, dst_offset), rax);
1447           __ movq(rax, Address(rsp, -8));
1448 #ifndef PRODUCT
1449         } else {
1450           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1451                     "movl    rax, [rsp + #%d]\n\t"
1452                     "movl    [rsp + #%d], rax\n\t"
1453                     "movq    rax, [rsp - #8]",
1454                      src_offset, dst_offset);
1455 #endif
1456         }
1457       }
1458       return 0;
1459     } else if (dst_first_rc == rc_int) {
1460       // mem -> gpr
1461       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1462           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1463         // 64-bit
1464         int offset = ra_->reg2offset(src_first);
1465         if (cbuf) {
1466           MacroAssembler _masm(cbuf);
1467           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1468 #ifndef PRODUCT
1469         } else {
1470           st->print("movq    %s, [rsp + #%d]\t# spill",
1471                      Matcher::regName[dst_first],
1472                      offset);
1473 #endif
1474         }
1475       } else {
1476         // 32-bit
1477         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1478         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1479         int offset = ra_->reg2offset(src_first);
1480         if (cbuf) {
1481           MacroAssembler _masm(cbuf);
1482           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1483 #ifndef PRODUCT
1484         } else {
1485           st->print("movl    %s, [rsp + #%d]\t# spill",
1486                      Matcher::regName[dst_first],
1487                      offset);
1488 #endif
1489         }
1490       }
1491       return 0;
1492     } else if (dst_first_rc == rc_float) {
1493       // mem-> xmm
1494       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1495           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1496         // 64-bit
1497         int offset = ra_->reg2offset(src_first);
1498         if (cbuf) {
1499           MacroAssembler _masm(cbuf);
1500           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1501 #ifndef PRODUCT
1502         } else {
1503           st->print("%s  %s, [rsp + #%d]\t# spill",
1504                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1505                      Matcher::regName[dst_first],
1506                      offset);
1507 #endif
1508         }
1509       } else {
1510         // 32-bit
1511         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1512         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1513         int offset = ra_->reg2offset(src_first);
1514         if (cbuf) {
1515           MacroAssembler _masm(cbuf);
1516           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1517 #ifndef PRODUCT
1518         } else {
1519           st->print("movss   %s, [rsp + #%d]\t# spill",
1520                      Matcher::regName[dst_first],
1521                      offset);
1522 #endif
1523         }
1524       }
1525       return 0;
1526     }
1527   } else if (src_first_rc == rc_int) {
1528     // gpr ->
1529     if (dst_first_rc == rc_stack) {
1530       // gpr -> mem
1531       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1532           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1533         // 64-bit
1534         int offset = ra_->reg2offset(dst_first);
1535         if (cbuf) {
1536           MacroAssembler _masm(cbuf);
1537           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1538 #ifndef PRODUCT
1539         } else {
1540           st->print("movq    [rsp + #%d], %s\t# spill",
1541                      offset,
1542                      Matcher::regName[src_first]);
1543 #endif
1544         }
1545       } else {
1546         // 32-bit
1547         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1548         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1549         int offset = ra_->reg2offset(dst_first);
1550         if (cbuf) {
1551           MacroAssembler _masm(cbuf);
1552           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1553 #ifndef PRODUCT
1554         } else {
1555           st->print("movl    [rsp + #%d], %s\t# spill",
1556                      offset,
1557                      Matcher::regName[src_first]);
1558 #endif
1559         }
1560       }
1561       return 0;
1562     } else if (dst_first_rc == rc_int) {
1563       // gpr -> gpr
1564       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1565           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1566         // 64-bit
1567         if (cbuf) {
1568           MacroAssembler _masm(cbuf);
1569           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1570                   as_Register(Matcher::_regEncode[src_first]));
1571 #ifndef PRODUCT
1572         } else {
1573           st->print("movq    %s, %s\t# spill",
1574                      Matcher::regName[dst_first],
1575                      Matcher::regName[src_first]);
1576 #endif
1577         }
1578         return 0;
1579       } else {
1580         // 32-bit
1581         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1582         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1583         if (cbuf) {
1584           MacroAssembler _masm(cbuf);
1585           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1586                   as_Register(Matcher::_regEncode[src_first]));
1587 #ifndef PRODUCT
1588         } else {
1589           st->print("movl    %s, %s\t# spill",
1590                      Matcher::regName[dst_first],
1591                      Matcher::regName[src_first]);
1592 #endif
1593         }
1594         return 0;
1595       }
1596     } else if (dst_first_rc == rc_float) {
1597       // gpr -> xmm
1598       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1599           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1600         // 64-bit
1601         if (cbuf) {
1602           MacroAssembler _masm(cbuf);
1603           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1604 #ifndef PRODUCT
1605         } else {
1606           st->print("movdq   %s, %s\t# spill",
1607                      Matcher::regName[dst_first],
1608                      Matcher::regName[src_first]);
1609 #endif
1610         }
1611       } else {
1612         // 32-bit
1613         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1614         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1615         if (cbuf) {
1616           MacroAssembler _masm(cbuf);
1617           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1618 #ifndef PRODUCT
1619         } else {
1620           st->print("movdl   %s, %s\t# spill",
1621                      Matcher::regName[dst_first],
1622                      Matcher::regName[src_first]);
1623 #endif
1624         }
1625       }
1626       return 0;
1627     }
1628   } else if (src_first_rc == rc_float) {
1629     // xmm ->
1630     if (dst_first_rc == rc_stack) {
1631       // xmm -> mem
1632       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1633           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1634         // 64-bit
1635         int offset = ra_->reg2offset(dst_first);
1636         if (cbuf) {
1637           MacroAssembler _masm(cbuf);
1638           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1639 #ifndef PRODUCT
1640         } else {
1641           st->print("movsd   [rsp + #%d], %s\t# spill",
1642                      offset,
1643                      Matcher::regName[src_first]);
1644 #endif
1645         }
1646       } else {
1647         // 32-bit
1648         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1649         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1650         int offset = ra_->reg2offset(dst_first);
1651         if (cbuf) {
1652           MacroAssembler _masm(cbuf);
1653           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1654 #ifndef PRODUCT
1655         } else {
1656           st->print("movss   [rsp + #%d], %s\t# spill",
1657                      offset,
1658                      Matcher::regName[src_first]);
1659 #endif
1660         }
1661       }
1662       return 0;
1663     } else if (dst_first_rc == rc_int) {
1664       // xmm -> gpr
1665       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1666           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1667         // 64-bit
1668         if (cbuf) {
1669           MacroAssembler _masm(cbuf);
1670           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1671 #ifndef PRODUCT
1672         } else {
1673           st->print("movdq   %s, %s\t# spill",
1674                      Matcher::regName[dst_first],
1675                      Matcher::regName[src_first]);
1676 #endif
1677         }
1678       } else {
1679         // 32-bit
1680         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1681         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1682         if (cbuf) {
1683           MacroAssembler _masm(cbuf);
1684           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1685 #ifndef PRODUCT
1686         } else {
1687           st->print("movdl   %s, %s\t# spill",
1688                      Matcher::regName[dst_first],
1689                      Matcher::regName[src_first]);
1690 #endif
1691         }
1692       }
1693       return 0;
1694     } else if (dst_first_rc == rc_float) {
1695       // xmm -> xmm
1696       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1697           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1698         // 64-bit
1699         if (cbuf) {
1700           MacroAssembler _masm(cbuf);
1701           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1702 #ifndef PRODUCT
1703         } else {
1704           st->print("%s  %s, %s\t# spill",
1705                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1706                      Matcher::regName[dst_first],
1707                      Matcher::regName[src_first]);
1708 #endif
1709         }
1710       } else {
1711         // 32-bit
1712         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1713         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1714         if (cbuf) {
1715           MacroAssembler _masm(cbuf);
1716           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1717 #ifndef PRODUCT
1718         } else {
1719           st->print("%s  %s, %s\t# spill",
1720                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1721                      Matcher::regName[dst_first],
1722                      Matcher::regName[src_first]);
1723 #endif
1724         }
1725       }
1726       return 0;
1727     }
1728   }
1729 
1730   assert(0," foo ");
1731   Unimplemented();
1732   return 0;
1733 }
1734 
1735 #ifndef PRODUCT
1736 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1737   implementation(NULL, ra_, false, st);
1738 }
1739 #endif
1740 
1741 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1742   implementation(&cbuf, ra_, false, NULL);
1743 }
1744 
1745 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1746   return MachNode::size(ra_);
1747 }
1748 
1749 //=============================================================================
1750 #ifndef PRODUCT
1751 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1752 {
1753   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1754   int reg = ra_->get_reg_first(this);
1755   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1756             Matcher::regName[reg], offset);
1757 }
1758 #endif
1759 
1760 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1761 {
1762   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1763   int reg = ra_->get_encode(this);
1764   if (offset >= 0x80) {
1765     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1766     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1767     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1768     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1769     emit_d32(cbuf, offset);
1770   } else {
1771     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1772     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1773     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1774     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1775     emit_d8(cbuf, offset);
1776   }
1777 }
1778 
1779 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1780 {
1781   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1782   return (offset < 0x80) ? 5 : 8; // REX
1783 }
1784 
1785 //=============================================================================
1786 #ifndef PRODUCT
1787 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1788 {
1789   if (UseCompressedClassPointers) {
1790     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1791     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1792     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1793   } else {
1794     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1795                  "# Inline cache check");
1796   }
1797   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1798   st->print_cr("\tnop\t# nops to align entry point");
1799 }
1800 #endif
1801 
1802 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1803 {
1804   MacroAssembler masm(&cbuf);
1805   uint insts_size = cbuf.insts_size();
1806   if (UseCompressedClassPointers) {
1807     masm.load_klass(rscratch1, j_rarg0);
1808     masm.cmpptr(rax, rscratch1);
1809   } else {
1810     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1811   }
1812 
1813   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1814 
1815   /* WARNING these NOPs are critical so that verified entry point is properly
1816      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1817   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1818   if (OptoBreakpoint) {
1819     // Leave space for int3
1820     nops_cnt -= 1;
1821   }
1822   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1823   if (nops_cnt > 0)
1824     masm.nop(nops_cnt);
1825 }
1826 
1827 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1828 {
1829   return MachNode::size(ra_); // too many variables; just compute it
1830                               // the hard way
1831 }
1832 
1833 
1834 //=============================================================================
1835 
1836 int Matcher::regnum_to_fpu_offset(int regnum)
1837 {
1838   return regnum - 32; // The FP registers are in the second chunk
1839 }
1840 
1841 // This is UltraSparc specific, true just means we have fast l2f conversion
1842 const bool Matcher::convL2FSupported(void) {
1843   return true;
1844 }
1845 
1846 // Is this branch offset short enough that a short branch can be used?
1847 //
1848 // NOTE: If the platform does not provide any short branch variants, then
1849 //       this method should return false for offset 0.
1850 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1851   // The passed offset is relative to address of the branch.
1852   // On 86 a branch displacement is calculated relative to address
1853   // of a next instruction.
1854   offset -= br_size;
1855 
1856   // the short version of jmpConUCF2 contains multiple branches,
1857   // making the reach slightly less
1858   if (rule == jmpConUCF2_rule)
1859     return (-126 <= offset && offset <= 125);
1860   return (-128 <= offset && offset <= 127);
1861 }
1862 
1863 const bool Matcher::isSimpleConstant64(jlong value) {
1864   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1865   //return value == (int) value;  // Cf. storeImmL and immL32.
1866 
1867   // Probably always true, even if a temp register is required.
1868   return true;
1869 }
1870 
1871 // The ecx parameter to rep stosq for the ClearArray node is in words.
1872 const bool Matcher::init_array_count_is_in_bytes = false;
1873 
1874 // No additional cost for CMOVL.
1875 const int Matcher::long_cmove_cost() { return 0; }
1876 
1877 // No CMOVF/CMOVD with SSE2
1878 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1879 
1880 // Does the CPU require late expand (see block.cpp for description of late expand)?
1881 const bool Matcher::require_postalloc_expand = false;
1882 
1883 // Do we need to mask the count passed to shift instructions or does
1884 // the cpu only look at the lower 5/6 bits anyway?
1885 const bool Matcher::need_masked_shift_count = false;
1886 
1887 bool Matcher::narrow_oop_use_complex_address() {
1888   assert(UseCompressedOops, "only for compressed oops code");
1889   return (LogMinObjAlignmentInBytes <= 3);
1890 }
1891 
1892 bool Matcher::narrow_klass_use_complex_address() {
1893   assert(UseCompressedClassPointers, "only for compressed klass code");
1894   return (LogKlassAlignmentInBytes <= 3);
1895 }
1896 
1897 bool Matcher::const_oop_prefer_decode() {
1898   // Prefer ConN+DecodeN over ConP.
1899   return true;
1900 }
1901 
1902 bool Matcher::const_klass_prefer_decode() {
1903   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1904   //       or condisider the following:
1905   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1906   //return Universe::narrow_klass_base() == NULL;
1907   return true;
1908 }
1909 
1910 // Is it better to copy float constants, or load them directly from
1911 // memory?  Intel can load a float constant from a direct address,
1912 // requiring no extra registers.  Most RISCs will have to materialize
1913 // an address into a register first, so they would do better to copy
1914 // the constant from stack.
1915 const bool Matcher::rematerialize_float_constants = true; // XXX
1916 
1917 // If CPU can load and store mis-aligned doubles directly then no
1918 // fixup is needed.  Else we split the double into 2 integer pieces
1919 // and move it piece-by-piece.  Only happens when passing doubles into
1920 // C code as the Java calling convention forces doubles to be aligned.
1921 const bool Matcher::misaligned_doubles_ok = true;
1922 
1923 // No-op on amd64
1924 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1925 
1926 // Advertise here if the CPU requires explicit rounding operations to
1927 // implement the UseStrictFP mode.
1928 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1929 
1930 // Are floats conerted to double when stored to stack during deoptimization?
1931 // On x64 it is stored without convertion so we can use normal access.
1932 bool Matcher::float_in_double() { return false; }
1933 
1934 // Do ints take an entire long register or just half?
1935 const bool Matcher::int_in_long = true;
1936 
1937 // Return whether or not this register is ever used as an argument.
1938 // This function is used on startup to build the trampoline stubs in
1939 // generateOptoStub.  Registers not mentioned will be killed by the VM
1940 // call in the trampoline, and arguments in those registers not be
1941 // available to the callee.
1942 bool Matcher::can_be_java_arg(int reg)
1943 {
1944   return
1945     reg ==  RDI_num || reg == RDI_H_num ||
1946     reg ==  RSI_num || reg == RSI_H_num ||
1947     reg ==  RDX_num || reg == RDX_H_num ||
1948     reg ==  RCX_num || reg == RCX_H_num ||
1949     reg ==   R8_num || reg ==  R8_H_num ||
1950     reg ==   R9_num || reg ==  R9_H_num ||
1951     reg ==  R12_num || reg == R12_H_num ||
1952     reg == XMM0_num || reg == XMM0b_num ||
1953     reg == XMM1_num || reg == XMM1b_num ||
1954     reg == XMM2_num || reg == XMM2b_num ||
1955     reg == XMM3_num || reg == XMM3b_num ||
1956     reg == XMM4_num || reg == XMM4b_num ||
1957     reg == XMM5_num || reg == XMM5b_num ||
1958     reg == XMM6_num || reg == XMM6b_num ||
1959     reg == XMM7_num || reg == XMM7b_num;
1960 }
1961 
1962 bool Matcher::is_spillable_arg(int reg)
1963 {
1964   return can_be_java_arg(reg);
1965 }
1966 
1967 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1968   // In 64 bit mode a code which use multiply when
1969   // devisor is constant is faster than hardware
1970   // DIV instruction (it uses MulHiL).
1971   return false;
1972 }
1973 
1974 // Register for DIVI projection of divmodI
1975 RegMask Matcher::divI_proj_mask() {
1976   return INT_RAX_REG_mask();
1977 }
1978 
1979 // Register for MODI projection of divmodI
1980 RegMask Matcher::modI_proj_mask() {
1981   return INT_RDX_REG_mask();
1982 }
1983 
1984 // Register for DIVL projection of divmodL
1985 RegMask Matcher::divL_proj_mask() {
1986   return LONG_RAX_REG_mask();
1987 }
1988 
1989 // Register for MODL projection of divmodL
1990 RegMask Matcher::modL_proj_mask() {
1991   return LONG_RDX_REG_mask();
1992 }
1993 
1994 // Register for saving SP into on method handle invokes. Not used on x86_64.
1995 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1996     return NO_REG_mask();
1997 }
1998 
1999 %}
2000 
2001 //----------ENCODING BLOCK-----------------------------------------------------
2002 // This block specifies the encoding classes used by the compiler to
2003 // output byte streams.  Encoding classes are parameterized macros
2004 // used by Machine Instruction Nodes in order to generate the bit
2005 // encoding of the instruction.  Operands specify their base encoding
2006 // interface with the interface keyword.  There are currently
2007 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2008 // COND_INTER.  REG_INTER causes an operand to generate a function
2009 // which returns its register number when queried.  CONST_INTER causes
2010 // an operand to generate a function which returns the value of the
2011 // constant when queried.  MEMORY_INTER causes an operand to generate
2012 // four functions which return the Base Register, the Index Register,
2013 // the Scale Value, and the Offset Value of the operand when queried.
2014 // COND_INTER causes an operand to generate six functions which return
2015 // the encoding code (ie - encoding bits for the instruction)
2016 // associated with each basic boolean condition for a conditional
2017 // instruction.
2018 //
2019 // Instructions specify two basic values for encoding.  Again, a
2020 // function is available to check if the constant displacement is an
2021 // oop. They use the ins_encode keyword to specify their encoding
2022 // classes (which must be a sequence of enc_class names, and their
2023 // parameters, specified in the encoding block), and they use the
2024 // opcode keyword to specify, in order, their primary, secondary, and
2025 // tertiary opcode.  Only the opcode sections which a particular
2026 // instruction needs for encoding need to be specified.
2027 encode %{
2028   // Build emit functions for each basic byte or larger field in the
2029   // intel encoding scheme (opcode, rm, sib, immediate), and call them
2030   // from C++ code in the enc_class source block.  Emit functions will
2031   // live in the main source block for now.  In future, we can
2032   // generalize this by adding a syntax that specifies the sizes of
2033   // fields in an order, so that the adlc can build the emit functions
2034   // automagically
2035 
2036   // Emit primary opcode
2037   enc_class OpcP
2038   %{
2039     emit_opcode(cbuf, $primary);
2040   %}
2041 
2042   // Emit secondary opcode
2043   enc_class OpcS
2044   %{
2045     emit_opcode(cbuf, $secondary);
2046   %}
2047 
2048   // Emit tertiary opcode
2049   enc_class OpcT
2050   %{
2051     emit_opcode(cbuf, $tertiary);
2052   %}
2053 
2054   // Emit opcode directly
2055   enc_class Opcode(immI d8)
2056   %{
2057     emit_opcode(cbuf, $d8$$constant);
2058   %}
2059 
2060   // Emit size prefix
2061   enc_class SizePrefix
2062   %{
2063     emit_opcode(cbuf, 0x66);
2064   %}
2065 
2066   enc_class reg(rRegI reg)
2067   %{
2068     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
2069   %}
2070 
2071   enc_class reg_reg(rRegI dst, rRegI src)
2072   %{
2073     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2074   %}
2075 
2076   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
2077   %{
2078     emit_opcode(cbuf, $opcode$$constant);
2079     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
2080   %}
2081 
2082   enc_class cdql_enc(no_rax_rdx_RegI div)
2083   %{
2084     // Full implementation of Java idiv and irem; checks for
2085     // special case as described in JVM spec., p.243 & p.271.
2086     //
2087     //         normal case                           special case
2088     //
2089     // input : rax: dividend                         min_int
2090     //         reg: divisor                          -1
2091     //
2092     // output: rax: quotient  (= rax idiv reg)       min_int
2093     //         rdx: remainder (= rax irem reg)       0
2094     //
2095     //  Code sequnce:
2096     //
2097     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
2098     //    5:   75 07/08                jne    e <normal>
2099     //    7:   33 d2                   xor    %edx,%edx
2100     //  [div >= 8 -> offset + 1]
2101     //  [REX_B]
2102     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
2103     //    c:   74 03/04                je     11 <done>
2104     // 000000000000000e <normal>:
2105     //    e:   99                      cltd
2106     //  [div >= 8 -> offset + 1]
2107     //  [REX_B]
2108     //    f:   f7 f9                   idiv   $div
2109     // 0000000000000011 <done>:
2110 
2111     // cmp    $0x80000000,%eax
2112     emit_opcode(cbuf, 0x3d);
2113     emit_d8(cbuf, 0x00);
2114     emit_d8(cbuf, 0x00);
2115     emit_d8(cbuf, 0x00);
2116     emit_d8(cbuf, 0x80);
2117 
2118     // jne    e <normal>
2119     emit_opcode(cbuf, 0x75);
2120     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
2121 
2122     // xor    %edx,%edx
2123     emit_opcode(cbuf, 0x33);
2124     emit_d8(cbuf, 0xD2);
2125 
2126     // cmp    $0xffffffffffffffff,%ecx
2127     if ($div$$reg >= 8) {
2128       emit_opcode(cbuf, Assembler::REX_B);
2129     }
2130     emit_opcode(cbuf, 0x83);
2131     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2132     emit_d8(cbuf, 0xFF);
2133 
2134     // je     11 <done>
2135     emit_opcode(cbuf, 0x74);
2136     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
2137 
2138     // <normal>
2139     // cltd
2140     emit_opcode(cbuf, 0x99);
2141 
2142     // idivl (note: must be emitted by the user of this rule)
2143     // <done>
2144   %}
2145 
2146   enc_class cdqq_enc(no_rax_rdx_RegL div)
2147   %{
2148     // Full implementation of Java ldiv and lrem; checks for
2149     // special case as described in JVM spec., p.243 & p.271.
2150     //
2151     //         normal case                           special case
2152     //
2153     // input : rax: dividend                         min_long
2154     //         reg: divisor                          -1
2155     //
2156     // output: rax: quotient  (= rax idiv reg)       min_long
2157     //         rdx: remainder (= rax irem reg)       0
2158     //
2159     //  Code sequnce:
2160     //
2161     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
2162     //    7:   00 00 80
2163     //    a:   48 39 d0                cmp    %rdx,%rax
2164     //    d:   75 08                   jne    17 <normal>
2165     //    f:   33 d2                   xor    %edx,%edx
2166     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
2167     //   15:   74 05                   je     1c <done>
2168     // 0000000000000017 <normal>:
2169     //   17:   48 99                   cqto
2170     //   19:   48 f7 f9                idiv   $div
2171     // 000000000000001c <done>:
2172 
2173     // mov    $0x8000000000000000,%rdx
2174     emit_opcode(cbuf, Assembler::REX_W);
2175     emit_opcode(cbuf, 0xBA);
2176     emit_d8(cbuf, 0x00);
2177     emit_d8(cbuf, 0x00);
2178     emit_d8(cbuf, 0x00);
2179     emit_d8(cbuf, 0x00);
2180     emit_d8(cbuf, 0x00);
2181     emit_d8(cbuf, 0x00);
2182     emit_d8(cbuf, 0x00);
2183     emit_d8(cbuf, 0x80);
2184 
2185     // cmp    %rdx,%rax
2186     emit_opcode(cbuf, Assembler::REX_W);
2187     emit_opcode(cbuf, 0x39);
2188     emit_d8(cbuf, 0xD0);
2189 
2190     // jne    17 <normal>
2191     emit_opcode(cbuf, 0x75);
2192     emit_d8(cbuf, 0x08);
2193 
2194     // xor    %edx,%edx
2195     emit_opcode(cbuf, 0x33);
2196     emit_d8(cbuf, 0xD2);
2197 
2198     // cmp    $0xffffffffffffffff,$div
2199     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
2200     emit_opcode(cbuf, 0x83);
2201     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2202     emit_d8(cbuf, 0xFF);
2203 
2204     // je     1e <done>
2205     emit_opcode(cbuf, 0x74);
2206     emit_d8(cbuf, 0x05);
2207 
2208     // <normal>
2209     // cqto
2210     emit_opcode(cbuf, Assembler::REX_W);
2211     emit_opcode(cbuf, 0x99);
2212 
2213     // idivq (note: must be emitted by the user of this rule)
2214     // <done>
2215   %}
2216 
2217   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2218   enc_class OpcSE(immI imm)
2219   %{
2220     // Emit primary opcode and set sign-extend bit
2221     // Check for 8-bit immediate, and set sign extend bit in opcode
2222     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2223       emit_opcode(cbuf, $primary | 0x02);
2224     } else {
2225       // 32-bit immediate
2226       emit_opcode(cbuf, $primary);
2227     }
2228   %}
2229 
2230   enc_class OpcSErm(rRegI dst, immI imm)
2231   %{
2232     // OpcSEr/m
2233     int dstenc = $dst$$reg;
2234     if (dstenc >= 8) {
2235       emit_opcode(cbuf, Assembler::REX_B);
2236       dstenc -= 8;
2237     }
2238     // Emit primary opcode and set sign-extend bit
2239     // Check for 8-bit immediate, and set sign extend bit in opcode
2240     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2241       emit_opcode(cbuf, $primary | 0x02);
2242     } else {
2243       // 32-bit immediate
2244       emit_opcode(cbuf, $primary);
2245     }
2246     // Emit r/m byte with secondary opcode, after primary opcode.
2247     emit_rm(cbuf, 0x3, $secondary, dstenc);
2248   %}
2249 
2250   enc_class OpcSErm_wide(rRegL dst, immI imm)
2251   %{
2252     // OpcSEr/m
2253     int dstenc = $dst$$reg;
2254     if (dstenc < 8) {
2255       emit_opcode(cbuf, Assembler::REX_W);
2256     } else {
2257       emit_opcode(cbuf, Assembler::REX_WB);
2258       dstenc -= 8;
2259     }
2260     // Emit primary opcode and set sign-extend bit
2261     // Check for 8-bit immediate, and set sign extend bit in opcode
2262     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2263       emit_opcode(cbuf, $primary | 0x02);
2264     } else {
2265       // 32-bit immediate
2266       emit_opcode(cbuf, $primary);
2267     }
2268     // Emit r/m byte with secondary opcode, after primary opcode.
2269     emit_rm(cbuf, 0x3, $secondary, dstenc);
2270   %}
2271 
2272   enc_class Con8or32(immI imm)
2273   %{
2274     // Check for 8-bit immediate, and set sign extend bit in opcode
2275     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2276       $$$emit8$imm$$constant;
2277     } else {
2278       // 32-bit immediate
2279       $$$emit32$imm$$constant;
2280     }
2281   %}
2282 
2283   enc_class opc2_reg(rRegI dst)
2284   %{
2285     // BSWAP
2286     emit_cc(cbuf, $secondary, $dst$$reg);
2287   %}
2288 
2289   enc_class opc3_reg(rRegI dst)
2290   %{
2291     // BSWAP
2292     emit_cc(cbuf, $tertiary, $dst$$reg);
2293   %}
2294 
2295   enc_class reg_opc(rRegI div)
2296   %{
2297     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2298     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2299   %}
2300 
2301   enc_class enc_cmov(cmpOp cop)
2302   %{
2303     // CMOV
2304     $$$emit8$primary;
2305     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2306   %}
2307 
2308   enc_class enc_PartialSubtypeCheck()
2309   %{
2310     Register Rrdi = as_Register(RDI_enc); // result register
2311     Register Rrax = as_Register(RAX_enc); // super class
2312     Register Rrcx = as_Register(RCX_enc); // killed
2313     Register Rrsi = as_Register(RSI_enc); // sub class
2314     Label miss;
2315     const bool set_cond_codes = true;
2316 
2317     MacroAssembler _masm(&cbuf);
2318     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2319                                      NULL, &miss,
2320                                      /*set_cond_codes:*/ true);
2321     if ($primary) {
2322       __ xorptr(Rrdi, Rrdi);
2323     }
2324     __ bind(miss);
2325   %}
2326 
2327   enc_class clear_avx %{
2328     debug_only(int off0 = cbuf.insts_size());
2329     if (generate_vzeroupper(Compile::current())) {
2330       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2331       // Clear upper bits of YMM registers when current compiled code uses
2332       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2333       MacroAssembler _masm(&cbuf);
2334       __ vzeroupper();
2335     }
2336     debug_only(int off1 = cbuf.insts_size());
2337     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2338   %}
2339 
2340   enc_class Java_To_Runtime(method meth) %{
2341     // No relocation needed
2342     MacroAssembler _masm(&cbuf);
2343     __ mov64(r10, (int64_t) $meth$$method);
2344     __ call(r10);
2345   %}
2346 
2347   enc_class Java_To_Interpreter(method meth)
2348   %{
2349     // CALL Java_To_Interpreter
2350     // This is the instruction starting address for relocation info.
2351     cbuf.set_insts_mark();
2352     $$$emit8$primary;
2353     // CALL directly to the runtime
2354     emit_d32_reloc(cbuf,
2355                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2356                    runtime_call_Relocation::spec(),
2357                    RELOC_DISP32);
2358   %}
2359 
2360   enc_class Java_Static_Call(method meth)
2361   %{
2362     // JAVA STATIC CALL
2363     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2364     // determine who we intended to call.
2365     cbuf.set_insts_mark();
2366     $$$emit8$primary;
2367 
2368     if (!_method) {
2369       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2370                      runtime_call_Relocation::spec(),
2371                      RELOC_DISP32);
2372     } else {
2373       int method_index = resolved_method_index(cbuf);
2374       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2375                                                   : static_call_Relocation::spec(method_index);
2376       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2377                      rspec, RELOC_DISP32);
2378       // Emit stubs for static call.
2379       address mark = cbuf.insts_mark();
2380       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2381       if (stub == NULL) {
2382         ciEnv::current()->record_failure("CodeCache is full");
2383         return;
2384       }
2385 #if INCLUDE_AOT
2386       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2387 #endif
2388     }
2389   %}
2390 
2391   enc_class Java_Dynamic_Call(method meth) %{
2392     MacroAssembler _masm(&cbuf);
2393     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2394   %}
2395 
2396   enc_class Java_Compiled_Call(method meth)
2397   %{
2398     // JAVA COMPILED CALL
2399     int disp = in_bytes(Method:: from_compiled_offset());
2400 
2401     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2402     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2403 
2404     // callq *disp(%rax)
2405     cbuf.set_insts_mark();
2406     $$$emit8$primary;
2407     if (disp < 0x80) {
2408       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2409       emit_d8(cbuf, disp); // Displacement
2410     } else {
2411       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2412       emit_d32(cbuf, disp); // Displacement
2413     }
2414   %}
2415 
2416   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2417   %{
2418     // SAL, SAR, SHR
2419     int dstenc = $dst$$reg;
2420     if (dstenc >= 8) {
2421       emit_opcode(cbuf, Assembler::REX_B);
2422       dstenc -= 8;
2423     }
2424     $$$emit8$primary;
2425     emit_rm(cbuf, 0x3, $secondary, dstenc);
2426     $$$emit8$shift$$constant;
2427   %}
2428 
2429   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2430   %{
2431     // SAL, SAR, SHR
2432     int dstenc = $dst$$reg;
2433     if (dstenc < 8) {
2434       emit_opcode(cbuf, Assembler::REX_W);
2435     } else {
2436       emit_opcode(cbuf, Assembler::REX_WB);
2437       dstenc -= 8;
2438     }
2439     $$$emit8$primary;
2440     emit_rm(cbuf, 0x3, $secondary, dstenc);
2441     $$$emit8$shift$$constant;
2442   %}
2443 
2444   enc_class load_immI(rRegI dst, immI src)
2445   %{
2446     int dstenc = $dst$$reg;
2447     if (dstenc >= 8) {
2448       emit_opcode(cbuf, Assembler::REX_B);
2449       dstenc -= 8;
2450     }
2451     emit_opcode(cbuf, 0xB8 | dstenc);
2452     $$$emit32$src$$constant;
2453   %}
2454 
2455   enc_class load_immL(rRegL dst, immL src)
2456   %{
2457     int dstenc = $dst$$reg;
2458     if (dstenc < 8) {
2459       emit_opcode(cbuf, Assembler::REX_W);
2460     } else {
2461       emit_opcode(cbuf, Assembler::REX_WB);
2462       dstenc -= 8;
2463     }
2464     emit_opcode(cbuf, 0xB8 | dstenc);
2465     emit_d64(cbuf, $src$$constant);
2466   %}
2467 
2468   enc_class load_immUL32(rRegL dst, immUL32 src)
2469   %{
2470     // same as load_immI, but this time we care about zeroes in the high word
2471     int dstenc = $dst$$reg;
2472     if (dstenc >= 8) {
2473       emit_opcode(cbuf, Assembler::REX_B);
2474       dstenc -= 8;
2475     }
2476     emit_opcode(cbuf, 0xB8 | dstenc);
2477     $$$emit32$src$$constant;
2478   %}
2479 
2480   enc_class load_immL32(rRegL dst, immL32 src)
2481   %{
2482     int dstenc = $dst$$reg;
2483     if (dstenc < 8) {
2484       emit_opcode(cbuf, Assembler::REX_W);
2485     } else {
2486       emit_opcode(cbuf, Assembler::REX_WB);
2487       dstenc -= 8;
2488     }
2489     emit_opcode(cbuf, 0xC7);
2490     emit_rm(cbuf, 0x03, 0x00, dstenc);
2491     $$$emit32$src$$constant;
2492   %}
2493 
2494   enc_class load_immP31(rRegP dst, immP32 src)
2495   %{
2496     // same as load_immI, but this time we care about zeroes in the high word
2497     int dstenc = $dst$$reg;
2498     if (dstenc >= 8) {
2499       emit_opcode(cbuf, Assembler::REX_B);
2500       dstenc -= 8;
2501     }
2502     emit_opcode(cbuf, 0xB8 | dstenc);
2503     $$$emit32$src$$constant;
2504   %}
2505 
2506   enc_class load_immP(rRegP dst, immP src)
2507   %{
2508     int dstenc = $dst$$reg;
2509     if (dstenc < 8) {
2510       emit_opcode(cbuf, Assembler::REX_W);
2511     } else {
2512       emit_opcode(cbuf, Assembler::REX_WB);
2513       dstenc -= 8;
2514     }
2515     emit_opcode(cbuf, 0xB8 | dstenc);
2516     // This next line should be generated from ADLC
2517     if ($src->constant_reloc() != relocInfo::none) {
2518       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2519     } else {
2520       emit_d64(cbuf, $src$$constant);
2521     }
2522   %}
2523 
2524   enc_class Con32(immI src)
2525   %{
2526     // Output immediate
2527     $$$emit32$src$$constant;
2528   %}
2529 
2530   enc_class Con32F_as_bits(immF src)
2531   %{
2532     // Output Float immediate bits
2533     jfloat jf = $src$$constant;
2534     jint jf_as_bits = jint_cast(jf);
2535     emit_d32(cbuf, jf_as_bits);
2536   %}
2537 
2538   enc_class Con16(immI src)
2539   %{
2540     // Output immediate
2541     $$$emit16$src$$constant;
2542   %}
2543 
2544   // How is this different from Con32??? XXX
2545   enc_class Con_d32(immI src)
2546   %{
2547     emit_d32(cbuf,$src$$constant);
2548   %}
2549 
2550   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2551     // Output immediate memory reference
2552     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2553     emit_d32(cbuf, 0x00);
2554   %}
2555 
2556   enc_class lock_prefix()
2557   %{
2558     emit_opcode(cbuf, 0xF0); // lock
2559   %}
2560 
2561   enc_class REX_mem(memory mem)
2562   %{
2563     if ($mem$$base >= 8) {
2564       if ($mem$$index < 8) {
2565         emit_opcode(cbuf, Assembler::REX_B);
2566       } else {
2567         emit_opcode(cbuf, Assembler::REX_XB);
2568       }
2569     } else {
2570       if ($mem$$index >= 8) {
2571         emit_opcode(cbuf, Assembler::REX_X);
2572       }
2573     }
2574   %}
2575 
2576   enc_class REX_mem_wide(memory mem)
2577   %{
2578     if ($mem$$base >= 8) {
2579       if ($mem$$index < 8) {
2580         emit_opcode(cbuf, Assembler::REX_WB);
2581       } else {
2582         emit_opcode(cbuf, Assembler::REX_WXB);
2583       }
2584     } else {
2585       if ($mem$$index < 8) {
2586         emit_opcode(cbuf, Assembler::REX_W);
2587       } else {
2588         emit_opcode(cbuf, Assembler::REX_WX);
2589       }
2590     }
2591   %}
2592 
2593   // for byte regs
2594   enc_class REX_breg(rRegI reg)
2595   %{
2596     if ($reg$$reg >= 4) {
2597       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2598     }
2599   %}
2600 
2601   // for byte regs
2602   enc_class REX_reg_breg(rRegI dst, rRegI src)
2603   %{
2604     if ($dst$$reg < 8) {
2605       if ($src$$reg >= 4) {
2606         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2607       }
2608     } else {
2609       if ($src$$reg < 8) {
2610         emit_opcode(cbuf, Assembler::REX_R);
2611       } else {
2612         emit_opcode(cbuf, Assembler::REX_RB);
2613       }
2614     }
2615   %}
2616 
2617   // for byte regs
2618   enc_class REX_breg_mem(rRegI reg, memory mem)
2619   %{
2620     if ($reg$$reg < 8) {
2621       if ($mem$$base < 8) {
2622         if ($mem$$index >= 8) {
2623           emit_opcode(cbuf, Assembler::REX_X);
2624         } else if ($reg$$reg >= 4) {
2625           emit_opcode(cbuf, Assembler::REX);
2626         }
2627       } else {
2628         if ($mem$$index < 8) {
2629           emit_opcode(cbuf, Assembler::REX_B);
2630         } else {
2631           emit_opcode(cbuf, Assembler::REX_XB);
2632         }
2633       }
2634     } else {
2635       if ($mem$$base < 8) {
2636         if ($mem$$index < 8) {
2637           emit_opcode(cbuf, Assembler::REX_R);
2638         } else {
2639           emit_opcode(cbuf, Assembler::REX_RX);
2640         }
2641       } else {
2642         if ($mem$$index < 8) {
2643           emit_opcode(cbuf, Assembler::REX_RB);
2644         } else {
2645           emit_opcode(cbuf, Assembler::REX_RXB);
2646         }
2647       }
2648     }
2649   %}
2650 
2651   enc_class REX_reg(rRegI reg)
2652   %{
2653     if ($reg$$reg >= 8) {
2654       emit_opcode(cbuf, Assembler::REX_B);
2655     }
2656   %}
2657 
2658   enc_class REX_reg_wide(rRegI reg)
2659   %{
2660     if ($reg$$reg < 8) {
2661       emit_opcode(cbuf, Assembler::REX_W);
2662     } else {
2663       emit_opcode(cbuf, Assembler::REX_WB);
2664     }
2665   %}
2666 
2667   enc_class REX_reg_reg(rRegI dst, rRegI src)
2668   %{
2669     if ($dst$$reg < 8) {
2670       if ($src$$reg >= 8) {
2671         emit_opcode(cbuf, Assembler::REX_B);
2672       }
2673     } else {
2674       if ($src$$reg < 8) {
2675         emit_opcode(cbuf, Assembler::REX_R);
2676       } else {
2677         emit_opcode(cbuf, Assembler::REX_RB);
2678       }
2679     }
2680   %}
2681 
2682   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2683   %{
2684     if ($dst$$reg < 8) {
2685       if ($src$$reg < 8) {
2686         emit_opcode(cbuf, Assembler::REX_W);
2687       } else {
2688         emit_opcode(cbuf, Assembler::REX_WB);
2689       }
2690     } else {
2691       if ($src$$reg < 8) {
2692         emit_opcode(cbuf, Assembler::REX_WR);
2693       } else {
2694         emit_opcode(cbuf, Assembler::REX_WRB);
2695       }
2696     }
2697   %}
2698 
2699   enc_class REX_reg_mem(rRegI reg, memory mem)
2700   %{
2701     if ($reg$$reg < 8) {
2702       if ($mem$$base < 8) {
2703         if ($mem$$index >= 8) {
2704           emit_opcode(cbuf, Assembler::REX_X);
2705         }
2706       } else {
2707         if ($mem$$index < 8) {
2708           emit_opcode(cbuf, Assembler::REX_B);
2709         } else {
2710           emit_opcode(cbuf, Assembler::REX_XB);
2711         }
2712       }
2713     } else {
2714       if ($mem$$base < 8) {
2715         if ($mem$$index < 8) {
2716           emit_opcode(cbuf, Assembler::REX_R);
2717         } else {
2718           emit_opcode(cbuf, Assembler::REX_RX);
2719         }
2720       } else {
2721         if ($mem$$index < 8) {
2722           emit_opcode(cbuf, Assembler::REX_RB);
2723         } else {
2724           emit_opcode(cbuf, Assembler::REX_RXB);
2725         }
2726       }
2727     }
2728   %}
2729 
2730   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2731   %{
2732     if ($reg$$reg < 8) {
2733       if ($mem$$base < 8) {
2734         if ($mem$$index < 8) {
2735           emit_opcode(cbuf, Assembler::REX_W);
2736         } else {
2737           emit_opcode(cbuf, Assembler::REX_WX);
2738         }
2739       } else {
2740         if ($mem$$index < 8) {
2741           emit_opcode(cbuf, Assembler::REX_WB);
2742         } else {
2743           emit_opcode(cbuf, Assembler::REX_WXB);
2744         }
2745       }
2746     } else {
2747       if ($mem$$base < 8) {
2748         if ($mem$$index < 8) {
2749           emit_opcode(cbuf, Assembler::REX_WR);
2750         } else {
2751           emit_opcode(cbuf, Assembler::REX_WRX);
2752         }
2753       } else {
2754         if ($mem$$index < 8) {
2755           emit_opcode(cbuf, Assembler::REX_WRB);
2756         } else {
2757           emit_opcode(cbuf, Assembler::REX_WRXB);
2758         }
2759       }
2760     }
2761   %}
2762 
2763   enc_class reg_mem(rRegI ereg, memory mem)
2764   %{
2765     // High registers handle in encode_RegMem
2766     int reg = $ereg$$reg;
2767     int base = $mem$$base;
2768     int index = $mem$$index;
2769     int scale = $mem$$scale;
2770     int disp = $mem$$disp;
2771     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2772 
2773     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2774   %}
2775 
2776   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2777   %{
2778     int rm_byte_opcode = $rm_opcode$$constant;
2779 
2780     // High registers handle in encode_RegMem
2781     int base = $mem$$base;
2782     int index = $mem$$index;
2783     int scale = $mem$$scale;
2784     int displace = $mem$$disp;
2785 
2786     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2787                                             // working with static
2788                                             // globals
2789     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2790                   disp_reloc);
2791   %}
2792 
2793   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2794   %{
2795     int reg_encoding = $dst$$reg;
2796     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2797     int index        = 0x04;            // 0x04 indicates no index
2798     int scale        = 0x00;            // 0x00 indicates no scale
2799     int displace     = $src1$$constant; // 0x00 indicates no displacement
2800     relocInfo::relocType disp_reloc = relocInfo::none;
2801     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2802                   disp_reloc);
2803   %}
2804 
2805   enc_class neg_reg(rRegI dst)
2806   %{
2807     int dstenc = $dst$$reg;
2808     if (dstenc >= 8) {
2809       emit_opcode(cbuf, Assembler::REX_B);
2810       dstenc -= 8;
2811     }
2812     // NEG $dst
2813     emit_opcode(cbuf, 0xF7);
2814     emit_rm(cbuf, 0x3, 0x03, dstenc);
2815   %}
2816 
2817   enc_class neg_reg_wide(rRegI dst)
2818   %{
2819     int dstenc = $dst$$reg;
2820     if (dstenc < 8) {
2821       emit_opcode(cbuf, Assembler::REX_W);
2822     } else {
2823       emit_opcode(cbuf, Assembler::REX_WB);
2824       dstenc -= 8;
2825     }
2826     // NEG $dst
2827     emit_opcode(cbuf, 0xF7);
2828     emit_rm(cbuf, 0x3, 0x03, dstenc);
2829   %}
2830 
2831   enc_class setLT_reg(rRegI dst)
2832   %{
2833     int dstenc = $dst$$reg;
2834     if (dstenc >= 8) {
2835       emit_opcode(cbuf, Assembler::REX_B);
2836       dstenc -= 8;
2837     } else if (dstenc >= 4) {
2838       emit_opcode(cbuf, Assembler::REX);
2839     }
2840     // SETLT $dst
2841     emit_opcode(cbuf, 0x0F);
2842     emit_opcode(cbuf, 0x9C);
2843     emit_rm(cbuf, 0x3, 0x0, dstenc);
2844   %}
2845 
2846   enc_class setNZ_reg(rRegI dst)
2847   %{
2848     int dstenc = $dst$$reg;
2849     if (dstenc >= 8) {
2850       emit_opcode(cbuf, Assembler::REX_B);
2851       dstenc -= 8;
2852     } else if (dstenc >= 4) {
2853       emit_opcode(cbuf, Assembler::REX);
2854     }
2855     // SETNZ $dst
2856     emit_opcode(cbuf, 0x0F);
2857     emit_opcode(cbuf, 0x95);
2858     emit_rm(cbuf, 0x3, 0x0, dstenc);
2859   %}
2860 
2861 
2862   // Compare the lonogs and set -1, 0, or 1 into dst
2863   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2864   %{
2865     int src1enc = $src1$$reg;
2866     int src2enc = $src2$$reg;
2867     int dstenc = $dst$$reg;
2868 
2869     // cmpq $src1, $src2
2870     if (src1enc < 8) {
2871       if (src2enc < 8) {
2872         emit_opcode(cbuf, Assembler::REX_W);
2873       } else {
2874         emit_opcode(cbuf, Assembler::REX_WB);
2875       }
2876     } else {
2877       if (src2enc < 8) {
2878         emit_opcode(cbuf, Assembler::REX_WR);
2879       } else {
2880         emit_opcode(cbuf, Assembler::REX_WRB);
2881       }
2882     }
2883     emit_opcode(cbuf, 0x3B);
2884     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2885 
2886     // movl $dst, -1
2887     if (dstenc >= 8) {
2888       emit_opcode(cbuf, Assembler::REX_B);
2889     }
2890     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2891     emit_d32(cbuf, -1);
2892 
2893     // jl,s done
2894     emit_opcode(cbuf, 0x7C);
2895     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2896 
2897     // setne $dst
2898     if (dstenc >= 4) {
2899       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2900     }
2901     emit_opcode(cbuf, 0x0F);
2902     emit_opcode(cbuf, 0x95);
2903     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2904 
2905     // movzbl $dst, $dst
2906     if (dstenc >= 4) {
2907       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2908     }
2909     emit_opcode(cbuf, 0x0F);
2910     emit_opcode(cbuf, 0xB6);
2911     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2912   %}
2913 
2914   enc_class Push_ResultXD(regD dst) %{
2915     MacroAssembler _masm(&cbuf);
2916     __ fstp_d(Address(rsp, 0));
2917     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2918     __ addptr(rsp, 8);
2919   %}
2920 
2921   enc_class Push_SrcXD(regD src) %{
2922     MacroAssembler _masm(&cbuf);
2923     __ subptr(rsp, 8);
2924     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2925     __ fld_d(Address(rsp, 0));
2926   %}
2927 
2928 
2929   enc_class enc_rethrow()
2930   %{
2931     cbuf.set_insts_mark();
2932     emit_opcode(cbuf, 0xE9); // jmp entry
2933     emit_d32_reloc(cbuf,
2934                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2935                    runtime_call_Relocation::spec(),
2936                    RELOC_DISP32);
2937   %}
2938 
2939 %}
2940 
2941 
2942 
2943 //----------FRAME--------------------------------------------------------------
2944 // Definition of frame structure and management information.
2945 //
2946 //  S T A C K   L A Y O U T    Allocators stack-slot number
2947 //                             |   (to get allocators register number
2948 //  G  Owned by    |        |  v    add OptoReg::stack0())
2949 //  r   CALLER     |        |
2950 //  o     |        +--------+      pad to even-align allocators stack-slot
2951 //  w     V        |  pad0  |        numbers; owned by CALLER
2952 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2953 //  h     ^        |   in   |  5
2954 //        |        |  args  |  4   Holes in incoming args owned by SELF
2955 //  |     |        |        |  3
2956 //  |     |        +--------+
2957 //  V     |        | old out|      Empty on Intel, window on Sparc
2958 //        |    old |preserve|      Must be even aligned.
2959 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2960 //        |        |   in   |  3   area for Intel ret address
2961 //     Owned by    |preserve|      Empty on Sparc.
2962 //       SELF      +--------+
2963 //        |        |  pad2  |  2   pad to align old SP
2964 //        |        +--------+  1
2965 //        |        | locks  |  0
2966 //        |        +--------+----> OptoReg::stack0(), even aligned
2967 //        |        |  pad1  | 11   pad to align new SP
2968 //        |        +--------+
2969 //        |        |        | 10
2970 //        |        | spills |  9   spills
2971 //        V        |        |  8   (pad0 slot for callee)
2972 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2973 //        ^        |  out   |  7
2974 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2975 //     Owned by    +--------+
2976 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2977 //        |    new |preserve|      Must be even-aligned.
2978 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2979 //        |        |        |
2980 //
2981 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2982 //         known from SELF's arguments and the Java calling convention.
2983 //         Region 6-7 is determined per call site.
2984 // Note 2: If the calling convention leaves holes in the incoming argument
2985 //         area, those holes are owned by SELF.  Holes in the outgoing area
2986 //         are owned by the CALLEE.  Holes should not be nessecary in the
2987 //         incoming area, as the Java calling convention is completely under
2988 //         the control of the AD file.  Doubles can be sorted and packed to
2989 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2990 //         varargs C calling conventions.
2991 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2992 //         even aligned with pad0 as needed.
2993 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2994 //         region 6-11 is even aligned; it may be padded out more so that
2995 //         the region from SP to FP meets the minimum stack alignment.
2996 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2997 //         alignment.  Region 11, pad1, may be dynamically extended so that
2998 //         SP meets the minimum alignment.
2999 
3000 frame
3001 %{
3002   // What direction does stack grow in (assumed to be same for C & Java)
3003   stack_direction(TOWARDS_LOW);
3004 
3005   // These three registers define part of the calling convention
3006   // between compiled code and the interpreter.
3007   inline_cache_reg(RAX);                // Inline Cache Register
3008   interpreter_method_oop_reg(RBX);      // Method Oop Register when
3009                                         // calling interpreter
3010 
3011   // Optional: name the operand used by cisc-spilling to access
3012   // [stack_pointer + offset]
3013   cisc_spilling_operand_name(indOffset32);
3014 
3015   // Number of stack slots consumed by locking an object
3016   sync_stack_slots(2);
3017 
3018   // Compiled code's Frame Pointer
3019   frame_pointer(RSP);
3020 
3021   // Interpreter stores its frame pointer in a register which is
3022   // stored to the stack by I2CAdaptors.
3023   // I2CAdaptors convert from interpreted java to compiled java.
3024   interpreter_frame_pointer(RBP);
3025 
3026   // Stack alignment requirement
3027   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3028 
3029   // Number of stack slots between incoming argument block and the start of
3030   // a new frame.  The PROLOG must add this many slots to the stack.  The
3031   // EPILOG must remove this many slots.  amd64 needs two slots for
3032   // return address.
3033   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
3034 
3035   // Number of outgoing stack slots killed above the out_preserve_stack_slots
3036   // for calls to C.  Supports the var-args backing area for register parms.
3037   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3038 
3039   // The after-PROLOG location of the return address.  Location of
3040   // return address specifies a type (REG or STACK) and a number
3041   // representing the register number (i.e. - use a register name) or
3042   // stack slot.
3043   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3044   // Otherwise, it is above the locks and verification slot and alignment word
3045   return_addr(STACK - 2 +
3046               align_up((Compile::current()->in_preserve_stack_slots() +
3047                         Compile::current()->fixed_slots()),
3048                        stack_alignment_in_slots()));
3049 
3050   // Body of function which returns an integer array locating
3051   // arguments either in registers or in stack slots.  Passed an array
3052   // of ideal registers called "sig" and a "length" count.  Stack-slot
3053   // offsets are based on outgoing arguments, i.e. a CALLER setting up
3054   // arguments for a CALLEE.  Incoming stack arguments are
3055   // automatically biased by the preserve_stack_slots field above.
3056 
3057   calling_convention
3058   %{
3059     // No difference between ingoing/outgoing just pass false
3060     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3061   %}
3062 
3063   c_calling_convention
3064   %{
3065     // This is obviously always outgoing
3066     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
3067   %}
3068 
3069   // Location of compiled Java return values.  Same as C for now.
3070   return_value
3071   %{
3072     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3073            "only return normal values");
3074 
3075     static const int lo[Op_RegL + 1] = {
3076       0,
3077       0,
3078       RAX_num,  // Op_RegN
3079       RAX_num,  // Op_RegI
3080       RAX_num,  // Op_RegP
3081       XMM0_num, // Op_RegF
3082       XMM0_num, // Op_RegD
3083       RAX_num   // Op_RegL
3084     };
3085     static const int hi[Op_RegL + 1] = {
3086       0,
3087       0,
3088       OptoReg::Bad, // Op_RegN
3089       OptoReg::Bad, // Op_RegI
3090       RAX_H_num,    // Op_RegP
3091       OptoReg::Bad, // Op_RegF
3092       XMM0b_num,    // Op_RegD
3093       RAX_H_num     // Op_RegL
3094     };
3095     // Excluded flags and vector registers.
3096     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
3097     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3098   %}
3099 %}
3100 
3101 //----------ATTRIBUTES---------------------------------------------------------
3102 //----------Operand Attributes-------------------------------------------------
3103 op_attrib op_cost(0);        // Required cost attribute
3104 
3105 //----------Instruction Attributes---------------------------------------------
3106 ins_attrib ins_cost(100);       // Required cost attribute
3107 ins_attrib ins_size(8);         // Required size attribute (in bits)
3108 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3109                                 // a non-matching short branch variant
3110                                 // of some long branch?
3111 ins_attrib ins_alignment(1);    // Required alignment attribute (must
3112                                 // be a power of 2) specifies the
3113                                 // alignment that some part of the
3114                                 // instruction (not necessarily the
3115                                 // start) requires.  If > 1, a
3116                                 // compute_padding() function must be
3117                                 // provided for the instruction
3118 
3119 //----------OPERANDS-----------------------------------------------------------
3120 // Operand definitions must precede instruction definitions for correct parsing
3121 // in the ADLC because operands constitute user defined types which are used in
3122 // instruction definitions.
3123 
3124 //----------Simple Operands----------------------------------------------------
3125 // Immediate Operands
3126 // Integer Immediate
3127 operand immI()
3128 %{
3129   match(ConI);
3130 
3131   op_cost(10);
3132   format %{ %}
3133   interface(CONST_INTER);
3134 %}
3135 
3136 // Constant for test vs zero
3137 operand immI0()
3138 %{
3139   predicate(n->get_int() == 0);
3140   match(ConI);
3141 
3142   op_cost(0);
3143   format %{ %}
3144   interface(CONST_INTER);
3145 %}
3146 
3147 // Constant for increment
3148 operand immI1()
3149 %{
3150   predicate(n->get_int() == 1);
3151   match(ConI);
3152 
3153   op_cost(0);
3154   format %{ %}
3155   interface(CONST_INTER);
3156 %}
3157 
3158 // Constant for decrement
3159 operand immI_M1()
3160 %{
3161   predicate(n->get_int() == -1);
3162   match(ConI);
3163 
3164   op_cost(0);
3165   format %{ %}
3166   interface(CONST_INTER);
3167 %}
3168 
3169 // Valid scale values for addressing modes
3170 operand immI2()
3171 %{
3172   predicate(0 <= n->get_int() && (n->get_int() <= 3));
3173   match(ConI);
3174 
3175   format %{ %}
3176   interface(CONST_INTER);
3177 %}
3178 
3179 operand immI8()
3180 %{
3181   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
3182   match(ConI);
3183 
3184   op_cost(5);
3185   format %{ %}
3186   interface(CONST_INTER);
3187 %}
3188 
3189 operand immU8()
3190 %{
3191   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
3192   match(ConI);
3193 
3194   op_cost(5);
3195   format %{ %}
3196   interface(CONST_INTER);
3197 %}
3198 
3199 operand immI16()
3200 %{
3201   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3202   match(ConI);
3203 
3204   op_cost(10);
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 // Int Immediate non-negative
3210 operand immU31()
3211 %{
3212   predicate(n->get_int() >= 0);
3213   match(ConI);
3214 
3215   op_cost(0);
3216   format %{ %}
3217   interface(CONST_INTER);
3218 %}
3219 
3220 // Constant for long shifts
3221 operand immI_32()
3222 %{
3223   predicate( n->get_int() == 32 );
3224   match(ConI);
3225 
3226   op_cost(0);
3227   format %{ %}
3228   interface(CONST_INTER);
3229 %}
3230 
3231 // Constant for long shifts
3232 operand immI_64()
3233 %{
3234   predicate( n->get_int() == 64 );
3235   match(ConI);
3236 
3237   op_cost(0);
3238   format %{ %}
3239   interface(CONST_INTER);
3240 %}
3241 
3242 // Pointer Immediate
3243 operand immP()
3244 %{
3245   match(ConP);
3246 
3247   op_cost(10);
3248   format %{ %}
3249   interface(CONST_INTER);
3250 %}
3251 
3252 // NULL Pointer Immediate
3253 operand immP0()
3254 %{
3255   predicate(n->get_ptr() == 0);
3256   match(ConP);
3257 
3258   op_cost(5);
3259   format %{ %}
3260   interface(CONST_INTER);
3261 %}
3262 
3263 // Pointer Immediate
3264 operand immN() %{
3265   match(ConN);
3266 
3267   op_cost(10);
3268   format %{ %}
3269   interface(CONST_INTER);
3270 %}
3271 
3272 operand immNKlass() %{
3273   match(ConNKlass);
3274 
3275   op_cost(10);
3276   format %{ %}
3277   interface(CONST_INTER);
3278 %}
3279 
3280 // NULL Pointer Immediate
3281 operand immN0() %{
3282   predicate(n->get_narrowcon() == 0);
3283   match(ConN);
3284 
3285   op_cost(5);
3286   format %{ %}
3287   interface(CONST_INTER);
3288 %}
3289 
3290 operand immP31()
3291 %{
3292   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3293             && (n->get_ptr() >> 31) == 0);
3294   match(ConP);
3295 
3296   op_cost(5);
3297   format %{ %}
3298   interface(CONST_INTER);
3299 %}
3300 
3301 
3302 // Long Immediate
3303 operand immL()
3304 %{
3305   match(ConL);
3306 
3307   op_cost(20);
3308   format %{ %}
3309   interface(CONST_INTER);
3310 %}
3311 
3312 // Long Immediate 8-bit
3313 operand immL8()
3314 %{
3315   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3316   match(ConL);
3317 
3318   op_cost(5);
3319   format %{ %}
3320   interface(CONST_INTER);
3321 %}
3322 
3323 // Long Immediate 32-bit unsigned
3324 operand immUL32()
3325 %{
3326   predicate(n->get_long() == (unsigned int) (n->get_long()));
3327   match(ConL);
3328 
3329   op_cost(10);
3330   format %{ %}
3331   interface(CONST_INTER);
3332 %}
3333 
3334 // Long Immediate 32-bit signed
3335 operand immL32()
3336 %{
3337   predicate(n->get_long() == (int) (n->get_long()));
3338   match(ConL);
3339 
3340   op_cost(15);
3341   format %{ %}
3342   interface(CONST_INTER);
3343 %}
3344 
3345 // Long Immediate zero
3346 operand immL0()
3347 %{
3348   predicate(n->get_long() == 0L);
3349   match(ConL);
3350 
3351   op_cost(10);
3352   format %{ %}
3353   interface(CONST_INTER);
3354 %}
3355 
3356 // Constant for increment
3357 operand immL1()
3358 %{
3359   predicate(n->get_long() == 1);
3360   match(ConL);
3361 
3362   format %{ %}
3363   interface(CONST_INTER);
3364 %}
3365 
3366 // Constant for decrement
3367 operand immL_M1()
3368 %{
3369   predicate(n->get_long() == -1);
3370   match(ConL);
3371 
3372   format %{ %}
3373   interface(CONST_INTER);
3374 %}
3375 
3376 // Long Immediate: the value 10
3377 operand immL10()
3378 %{
3379   predicate(n->get_long() == 10);
3380   match(ConL);
3381 
3382   format %{ %}
3383   interface(CONST_INTER);
3384 %}
3385 
3386 // Long immediate from 0 to 127.
3387 // Used for a shorter form of long mul by 10.
3388 operand immL_127()
3389 %{
3390   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3391   match(ConL);
3392 
3393   op_cost(10);
3394   format %{ %}
3395   interface(CONST_INTER);
3396 %}
3397 
3398 // Long Immediate: low 32-bit mask
3399 operand immL_32bits()
3400 %{
3401   predicate(n->get_long() == 0xFFFFFFFFL);
3402   match(ConL);
3403   op_cost(20);
3404 
3405   format %{ %}
3406   interface(CONST_INTER);
3407 %}
3408 
3409 // Float Immediate zero
3410 operand immF0()
3411 %{
3412   predicate(jint_cast(n->getf()) == 0);
3413   match(ConF);
3414 
3415   op_cost(5);
3416   format %{ %}
3417   interface(CONST_INTER);
3418 %}
3419 
3420 // Float Immediate
3421 operand immF()
3422 %{
3423   match(ConF);
3424 
3425   op_cost(15);
3426   format %{ %}
3427   interface(CONST_INTER);
3428 %}
3429 
3430 // Double Immediate zero
3431 operand immD0()
3432 %{
3433   predicate(jlong_cast(n->getd()) == 0);
3434   match(ConD);
3435 
3436   op_cost(5);
3437   format %{ %}
3438   interface(CONST_INTER);
3439 %}
3440 
3441 // Double Immediate
3442 operand immD()
3443 %{
3444   match(ConD);
3445 
3446   op_cost(15);
3447   format %{ %}
3448   interface(CONST_INTER);
3449 %}
3450 
3451 // Immediates for special shifts (sign extend)
3452 
3453 // Constants for increment
3454 operand immI_16()
3455 %{
3456   predicate(n->get_int() == 16);
3457   match(ConI);
3458 
3459   format %{ %}
3460   interface(CONST_INTER);
3461 %}
3462 
3463 operand immI_24()
3464 %{
3465   predicate(n->get_int() == 24);
3466   match(ConI);
3467 
3468   format %{ %}
3469   interface(CONST_INTER);
3470 %}
3471 
3472 // Constant for byte-wide masking
3473 operand immI_255()
3474 %{
3475   predicate(n->get_int() == 255);
3476   match(ConI);
3477 
3478   format %{ %}
3479   interface(CONST_INTER);
3480 %}
3481 
3482 // Constant for short-wide masking
3483 operand immI_65535()
3484 %{
3485   predicate(n->get_int() == 65535);
3486   match(ConI);
3487 
3488   format %{ %}
3489   interface(CONST_INTER);
3490 %}
3491 
3492 // Constant for byte-wide masking
3493 operand immL_255()
3494 %{
3495   predicate(n->get_long() == 255);
3496   match(ConL);
3497 
3498   format %{ %}
3499   interface(CONST_INTER);
3500 %}
3501 
3502 // Constant for short-wide masking
3503 operand immL_65535()
3504 %{
3505   predicate(n->get_long() == 65535);
3506   match(ConL);
3507 
3508   format %{ %}
3509   interface(CONST_INTER);
3510 %}
3511 
3512 // Register Operands
3513 // Integer Register
3514 operand rRegI()
3515 %{
3516   constraint(ALLOC_IN_RC(int_reg));
3517   match(RegI);
3518 
3519   match(rax_RegI);
3520   match(rbx_RegI);
3521   match(rcx_RegI);
3522   match(rdx_RegI);
3523   match(rdi_RegI);
3524 
3525   format %{ %}
3526   interface(REG_INTER);
3527 %}
3528 
3529 // Special Registers
3530 operand rax_RegI()
3531 %{
3532   constraint(ALLOC_IN_RC(int_rax_reg));
3533   match(RegI);
3534   match(rRegI);
3535 
3536   format %{ "RAX" %}
3537   interface(REG_INTER);
3538 %}
3539 
3540 // Special Registers
3541 operand rbx_RegI()
3542 %{
3543   constraint(ALLOC_IN_RC(int_rbx_reg));
3544   match(RegI);
3545   match(rRegI);
3546 
3547   format %{ "RBX" %}
3548   interface(REG_INTER);
3549 %}
3550 
3551 operand rcx_RegI()
3552 %{
3553   constraint(ALLOC_IN_RC(int_rcx_reg));
3554   match(RegI);
3555   match(rRegI);
3556 
3557   format %{ "RCX" %}
3558   interface(REG_INTER);
3559 %}
3560 
3561 operand rdx_RegI()
3562 %{
3563   constraint(ALLOC_IN_RC(int_rdx_reg));
3564   match(RegI);
3565   match(rRegI);
3566 
3567   format %{ "RDX" %}
3568   interface(REG_INTER);
3569 %}
3570 
3571 operand rdi_RegI()
3572 %{
3573   constraint(ALLOC_IN_RC(int_rdi_reg));
3574   match(RegI);
3575   match(rRegI);
3576 
3577   format %{ "RDI" %}
3578   interface(REG_INTER);
3579 %}
3580 
3581 operand no_rcx_RegI()
3582 %{
3583   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3584   match(RegI);
3585   match(rax_RegI);
3586   match(rbx_RegI);
3587   match(rdx_RegI);
3588   match(rdi_RegI);
3589 
3590   format %{ %}
3591   interface(REG_INTER);
3592 %}
3593 
3594 operand no_rax_rdx_RegI()
3595 %{
3596   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3597   match(RegI);
3598   match(rbx_RegI);
3599   match(rcx_RegI);
3600   match(rdi_RegI);
3601 
3602   format %{ %}
3603   interface(REG_INTER);
3604 %}
3605 
3606 // Pointer Register
3607 operand any_RegP()
3608 %{
3609   constraint(ALLOC_IN_RC(any_reg));
3610   match(RegP);
3611   match(rax_RegP);
3612   match(rbx_RegP);
3613   match(rdi_RegP);
3614   match(rsi_RegP);
3615   match(rbp_RegP);
3616   match(r15_RegP);
3617   match(rRegP);
3618 
3619   format %{ %}
3620   interface(REG_INTER);
3621 %}
3622 
3623 operand rRegP()
3624 %{
3625   constraint(ALLOC_IN_RC(ptr_reg));
3626   match(RegP);
3627   match(rax_RegP);
3628   match(rbx_RegP);
3629   match(rdi_RegP);
3630   match(rsi_RegP);
3631   match(rbp_RegP);  // See Q&A below about
3632   match(r15_RegP);  // r15_RegP and rbp_RegP.
3633 
3634   format %{ %}
3635   interface(REG_INTER);
3636 %}
3637 
3638 operand rRegN() %{
3639   constraint(ALLOC_IN_RC(int_reg));
3640   match(RegN);
3641 
3642   format %{ %}
3643   interface(REG_INTER);
3644 %}
3645 
3646 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3647 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3648 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3649 // The output of an instruction is controlled by the allocator, which respects
3650 // register class masks, not match rules.  Unless an instruction mentions
3651 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3652 // by the allocator as an input.
3653 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3654 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3655 // result, RBP is not included in the output of the instruction either.
3656 
3657 operand no_rax_RegP()
3658 %{
3659   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3660   match(RegP);
3661   match(rbx_RegP);
3662   match(rsi_RegP);
3663   match(rdi_RegP);
3664 
3665   format %{ %}
3666   interface(REG_INTER);
3667 %}
3668 
3669 // This operand is not allowed to use RBP even if
3670 // RBP is not used to hold the frame pointer.
3671 operand no_rbp_RegP()
3672 %{
3673   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3674   match(RegP);
3675   match(rbx_RegP);
3676   match(rsi_RegP);
3677   match(rdi_RegP);
3678 
3679   format %{ %}
3680   interface(REG_INTER);
3681 %}
3682 
3683 operand no_rax_rbx_RegP()
3684 %{
3685   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3686   match(RegP);
3687   match(rsi_RegP);
3688   match(rdi_RegP);
3689 
3690   format %{ %}
3691   interface(REG_INTER);
3692 %}
3693 
3694 // Special Registers
3695 // Return a pointer value
3696 operand rax_RegP()
3697 %{
3698   constraint(ALLOC_IN_RC(ptr_rax_reg));
3699   match(RegP);
3700   match(rRegP);
3701 
3702   format %{ %}
3703   interface(REG_INTER);
3704 %}
3705 
3706 // Special Registers
3707 // Return a compressed pointer value
3708 operand rax_RegN()
3709 %{
3710   constraint(ALLOC_IN_RC(int_rax_reg));
3711   match(RegN);
3712   match(rRegN);
3713 
3714   format %{ %}
3715   interface(REG_INTER);
3716 %}
3717 
3718 // Used in AtomicAdd
3719 operand rbx_RegP()
3720 %{
3721   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3722   match(RegP);
3723   match(rRegP);
3724 
3725   format %{ %}
3726   interface(REG_INTER);
3727 %}
3728 
3729 operand rsi_RegP()
3730 %{
3731   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3732   match(RegP);
3733   match(rRegP);
3734 
3735   format %{ %}
3736   interface(REG_INTER);
3737 %}
3738 
3739 // Used in rep stosq
3740 operand rdi_RegP()
3741 %{
3742   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3743   match(RegP);
3744   match(rRegP);
3745 
3746   format %{ %}
3747   interface(REG_INTER);
3748 %}
3749 
3750 operand r15_RegP()
3751 %{
3752   constraint(ALLOC_IN_RC(ptr_r15_reg));
3753   match(RegP);
3754   match(rRegP);
3755 
3756   format %{ %}
3757   interface(REG_INTER);
3758 %}
3759 
3760 operand rRegL()
3761 %{
3762   constraint(ALLOC_IN_RC(long_reg));
3763   match(RegL);
3764   match(rax_RegL);
3765   match(rdx_RegL);
3766 
3767   format %{ %}
3768   interface(REG_INTER);
3769 %}
3770 
3771 // Special Registers
3772 operand no_rax_rdx_RegL()
3773 %{
3774   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3775   match(RegL);
3776   match(rRegL);
3777 
3778   format %{ %}
3779   interface(REG_INTER);
3780 %}
3781 
3782 operand no_rax_RegL()
3783 %{
3784   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3785   match(RegL);
3786   match(rRegL);
3787   match(rdx_RegL);
3788 
3789   format %{ %}
3790   interface(REG_INTER);
3791 %}
3792 
3793 operand no_rcx_RegL()
3794 %{
3795   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3796   match(RegL);
3797   match(rRegL);
3798 
3799   format %{ %}
3800   interface(REG_INTER);
3801 %}
3802 
3803 operand rax_RegL()
3804 %{
3805   constraint(ALLOC_IN_RC(long_rax_reg));
3806   match(RegL);
3807   match(rRegL);
3808 
3809   format %{ "RAX" %}
3810   interface(REG_INTER);
3811 %}
3812 
3813 operand rcx_RegL()
3814 %{
3815   constraint(ALLOC_IN_RC(long_rcx_reg));
3816   match(RegL);
3817   match(rRegL);
3818 
3819   format %{ %}
3820   interface(REG_INTER);
3821 %}
3822 
3823 operand rdx_RegL()
3824 %{
3825   constraint(ALLOC_IN_RC(long_rdx_reg));
3826   match(RegL);
3827   match(rRegL);
3828 
3829   format %{ %}
3830   interface(REG_INTER);
3831 %}
3832 
3833 // Flags register, used as output of compare instructions
3834 operand rFlagsReg()
3835 %{
3836   constraint(ALLOC_IN_RC(int_flags));
3837   match(RegFlags);
3838 
3839   format %{ "RFLAGS" %}
3840   interface(REG_INTER);
3841 %}
3842 
3843 // Flags register, used as output of FLOATING POINT compare instructions
3844 operand rFlagsRegU()
3845 %{
3846   constraint(ALLOC_IN_RC(int_flags));
3847   match(RegFlags);
3848 
3849   format %{ "RFLAGS_U" %}
3850   interface(REG_INTER);
3851 %}
3852 
3853 operand rFlagsRegUCF() %{
3854   constraint(ALLOC_IN_RC(int_flags));
3855   match(RegFlags);
3856   predicate(false);
3857 
3858   format %{ "RFLAGS_U_CF" %}
3859   interface(REG_INTER);
3860 %}
3861 
3862 // Float register operands
3863 operand regF() %{
3864    constraint(ALLOC_IN_RC(float_reg));
3865    match(RegF);
3866 
3867    format %{ %}
3868    interface(REG_INTER);
3869 %}
3870 
3871 // Float register operands
3872 operand vlRegF() %{
3873    constraint(ALLOC_IN_RC(float_reg_vl));
3874    match(RegF);
3875 
3876    format %{ %}
3877    interface(REG_INTER);
3878 %}
3879 
3880 // Double register operands
3881 operand regD() %{
3882    constraint(ALLOC_IN_RC(double_reg));
3883    match(RegD);
3884 
3885    format %{ %}
3886    interface(REG_INTER);
3887 %}
3888 
3889 // Double register operands
3890 operand vlRegD() %{
3891    constraint(ALLOC_IN_RC(double_reg_vl));
3892    match(RegD);
3893 
3894    format %{ %}
3895    interface(REG_INTER);
3896 %}
3897 
3898 // Vectors
3899 operand vecS() %{
3900   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3901   match(VecS);
3902 
3903   format %{ %}
3904   interface(REG_INTER);
3905 %}
3906 
3907 // Vectors
3908 operand legVecS() %{
3909   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3910   match(VecS);
3911 
3912   format %{ %}
3913   interface(REG_INTER);
3914 %}
3915 
3916 operand vecD() %{
3917   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3918   match(VecD);
3919 
3920   format %{ %}
3921   interface(REG_INTER);
3922 %}
3923 
3924 operand legVecD() %{
3925   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3926   match(VecD);
3927 
3928   format %{ %}
3929   interface(REG_INTER);
3930 %}
3931 
3932 operand vecX() %{
3933   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3934   match(VecX);
3935 
3936   format %{ %}
3937   interface(REG_INTER);
3938 %}
3939 
3940 operand legVecX() %{
3941   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3942   match(VecX);
3943 
3944   format %{ %}
3945   interface(REG_INTER);
3946 %}
3947 
3948 operand vecY() %{
3949   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3950   match(VecY);
3951 
3952   format %{ %}
3953   interface(REG_INTER);
3954 %}
3955 
3956 operand legVecY() %{
3957   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3958   match(VecY);
3959 
3960   format %{ %}
3961   interface(REG_INTER);
3962 %}
3963 
3964 //----------Memory Operands----------------------------------------------------
3965 // Direct Memory Operand
3966 // operand direct(immP addr)
3967 // %{
3968 //   match(addr);
3969 
3970 //   format %{ "[$addr]" %}
3971 //   interface(MEMORY_INTER) %{
3972 //     base(0xFFFFFFFF);
3973 //     index(0x4);
3974 //     scale(0x0);
3975 //     disp($addr);
3976 //   %}
3977 // %}
3978 
3979 // Indirect Memory Operand
3980 operand indirect(any_RegP reg)
3981 %{
3982   constraint(ALLOC_IN_RC(ptr_reg));
3983   match(reg);
3984 
3985   format %{ "[$reg]" %}
3986   interface(MEMORY_INTER) %{
3987     base($reg);
3988     index(0x4);
3989     scale(0x0);
3990     disp(0x0);
3991   %}
3992 %}
3993 
3994 // Indirect Memory Plus Short Offset Operand
3995 operand indOffset8(any_RegP reg, immL8 off)
3996 %{
3997   constraint(ALLOC_IN_RC(ptr_reg));
3998   match(AddP reg off);
3999 
4000   format %{ "[$reg + $off (8-bit)]" %}
4001   interface(MEMORY_INTER) %{
4002     base($reg);
4003     index(0x4);
4004     scale(0x0);
4005     disp($off);
4006   %}
4007 %}
4008 
4009 // Indirect Memory Plus Long Offset Operand
4010 operand indOffset32(any_RegP reg, immL32 off)
4011 %{
4012   constraint(ALLOC_IN_RC(ptr_reg));
4013   match(AddP reg off);
4014 
4015   format %{ "[$reg + $off (32-bit)]" %}
4016   interface(MEMORY_INTER) %{
4017     base($reg);
4018     index(0x4);
4019     scale(0x0);
4020     disp($off);
4021   %}
4022 %}
4023 
4024 // Indirect Memory Plus Index Register Plus Offset Operand
4025 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
4026 %{
4027   constraint(ALLOC_IN_RC(ptr_reg));
4028   match(AddP (AddP reg lreg) off);
4029 
4030   op_cost(10);
4031   format %{"[$reg + $off + $lreg]" %}
4032   interface(MEMORY_INTER) %{
4033     base($reg);
4034     index($lreg);
4035     scale(0x0);
4036     disp($off);
4037   %}
4038 %}
4039 
4040 // Indirect Memory Plus Index Register Plus Offset Operand
4041 operand indIndex(any_RegP reg, rRegL lreg)
4042 %{
4043   constraint(ALLOC_IN_RC(ptr_reg));
4044   match(AddP reg lreg);
4045 
4046   op_cost(10);
4047   format %{"[$reg + $lreg]" %}
4048   interface(MEMORY_INTER) %{
4049     base($reg);
4050     index($lreg);
4051     scale(0x0);
4052     disp(0x0);
4053   %}
4054 %}
4055 
4056 // Indirect Memory Times Scale Plus Index Register
4057 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
4058 %{
4059   constraint(ALLOC_IN_RC(ptr_reg));
4060   match(AddP reg (LShiftL lreg scale));
4061 
4062   op_cost(10);
4063   format %{"[$reg + $lreg << $scale]" %}
4064   interface(MEMORY_INTER) %{
4065     base($reg);
4066     index($lreg);
4067     scale($scale);
4068     disp(0x0);
4069   %}
4070 %}
4071 
4072 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
4073 %{
4074   constraint(ALLOC_IN_RC(ptr_reg));
4075   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4076   match(AddP reg (LShiftL (ConvI2L idx) scale));
4077 
4078   op_cost(10);
4079   format %{"[$reg + pos $idx << $scale]" %}
4080   interface(MEMORY_INTER) %{
4081     base($reg);
4082     index($idx);
4083     scale($scale);
4084     disp(0x0);
4085   %}
4086 %}
4087 
4088 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4089 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
4090 %{
4091   constraint(ALLOC_IN_RC(ptr_reg));
4092   match(AddP (AddP reg (LShiftL lreg scale)) off);
4093 
4094   op_cost(10);
4095   format %{"[$reg + $off + $lreg << $scale]" %}
4096   interface(MEMORY_INTER) %{
4097     base($reg);
4098     index($lreg);
4099     scale($scale);
4100     disp($off);
4101   %}
4102 %}
4103 
4104 // Indirect Memory Plus Positive Index Register Plus Offset Operand
4105 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
4106 %{
4107   constraint(ALLOC_IN_RC(ptr_reg));
4108   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4109   match(AddP (AddP reg (ConvI2L idx)) off);
4110 
4111   op_cost(10);
4112   format %{"[$reg + $off + $idx]" %}
4113   interface(MEMORY_INTER) %{
4114     base($reg);
4115     index($idx);
4116     scale(0x0);
4117     disp($off);
4118   %}
4119 %}
4120 
4121 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4122 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
4123 %{
4124   constraint(ALLOC_IN_RC(ptr_reg));
4125   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4126   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
4127 
4128   op_cost(10);
4129   format %{"[$reg + $off + $idx << $scale]" %}
4130   interface(MEMORY_INTER) %{
4131     base($reg);
4132     index($idx);
4133     scale($scale);
4134     disp($off);
4135   %}
4136 %}
4137 
4138 // Indirect Narrow Oop Plus Offset Operand
4139 // Note: x86 architecture doesn't support "scale * index + offset" without a base
4140 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
4141 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
4142   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
4143   constraint(ALLOC_IN_RC(ptr_reg));
4144   match(AddP (DecodeN reg) off);
4145 
4146   op_cost(10);
4147   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
4148   interface(MEMORY_INTER) %{
4149     base(0xc); // R12
4150     index($reg);
4151     scale(0x3);
4152     disp($off);
4153   %}
4154 %}
4155 
4156 // Indirect Memory Operand
4157 operand indirectNarrow(rRegN reg)
4158 %{
4159   predicate(Universe::narrow_oop_shift() == 0);
4160   constraint(ALLOC_IN_RC(ptr_reg));
4161   match(DecodeN reg);
4162 
4163   format %{ "[$reg]" %}
4164   interface(MEMORY_INTER) %{
4165     base($reg);
4166     index(0x4);
4167     scale(0x0);
4168     disp(0x0);
4169   %}
4170 %}
4171 
4172 // Indirect Memory Plus Short Offset Operand
4173 operand indOffset8Narrow(rRegN reg, immL8 off)
4174 %{
4175   predicate(Universe::narrow_oop_shift() == 0);
4176   constraint(ALLOC_IN_RC(ptr_reg));
4177   match(AddP (DecodeN reg) off);
4178 
4179   format %{ "[$reg + $off (8-bit)]" %}
4180   interface(MEMORY_INTER) %{
4181     base($reg);
4182     index(0x4);
4183     scale(0x0);
4184     disp($off);
4185   %}
4186 %}
4187 
4188 // Indirect Memory Plus Long Offset Operand
4189 operand indOffset32Narrow(rRegN reg, immL32 off)
4190 %{
4191   predicate(Universe::narrow_oop_shift() == 0);
4192   constraint(ALLOC_IN_RC(ptr_reg));
4193   match(AddP (DecodeN reg) off);
4194 
4195   format %{ "[$reg + $off (32-bit)]" %}
4196   interface(MEMORY_INTER) %{
4197     base($reg);
4198     index(0x4);
4199     scale(0x0);
4200     disp($off);
4201   %}
4202 %}
4203 
4204 // Indirect Memory Plus Index Register Plus Offset Operand
4205 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4206 %{
4207   predicate(Universe::narrow_oop_shift() == 0);
4208   constraint(ALLOC_IN_RC(ptr_reg));
4209   match(AddP (AddP (DecodeN reg) lreg) off);
4210 
4211   op_cost(10);
4212   format %{"[$reg + $off + $lreg]" %}
4213   interface(MEMORY_INTER) %{
4214     base($reg);
4215     index($lreg);
4216     scale(0x0);
4217     disp($off);
4218   %}
4219 %}
4220 
4221 // Indirect Memory Plus Index Register Plus Offset Operand
4222 operand indIndexNarrow(rRegN reg, rRegL lreg)
4223 %{
4224   predicate(Universe::narrow_oop_shift() == 0);
4225   constraint(ALLOC_IN_RC(ptr_reg));
4226   match(AddP (DecodeN reg) lreg);
4227 
4228   op_cost(10);
4229   format %{"[$reg + $lreg]" %}
4230   interface(MEMORY_INTER) %{
4231     base($reg);
4232     index($lreg);
4233     scale(0x0);
4234     disp(0x0);
4235   %}
4236 %}
4237 
4238 // Indirect Memory Times Scale Plus Index Register
4239 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4240 %{
4241   predicate(Universe::narrow_oop_shift() == 0);
4242   constraint(ALLOC_IN_RC(ptr_reg));
4243   match(AddP (DecodeN reg) (LShiftL lreg scale));
4244 
4245   op_cost(10);
4246   format %{"[$reg + $lreg << $scale]" %}
4247   interface(MEMORY_INTER) %{
4248     base($reg);
4249     index($lreg);
4250     scale($scale);
4251     disp(0x0);
4252   %}
4253 %}
4254 
4255 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4256 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4257 %{
4258   predicate(Universe::narrow_oop_shift() == 0);
4259   constraint(ALLOC_IN_RC(ptr_reg));
4260   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4261 
4262   op_cost(10);
4263   format %{"[$reg + $off + $lreg << $scale]" %}
4264   interface(MEMORY_INTER) %{
4265     base($reg);
4266     index($lreg);
4267     scale($scale);
4268     disp($off);
4269   %}
4270 %}
4271 
4272 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4273 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4274 %{
4275   constraint(ALLOC_IN_RC(ptr_reg));
4276   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4277   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4278 
4279   op_cost(10);
4280   format %{"[$reg + $off + $idx]" %}
4281   interface(MEMORY_INTER) %{
4282     base($reg);
4283     index($idx);
4284     scale(0x0);
4285     disp($off);
4286   %}
4287 %}
4288 
4289 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4290 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4291 %{
4292   constraint(ALLOC_IN_RC(ptr_reg));
4293   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4294   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4295 
4296   op_cost(10);
4297   format %{"[$reg + $off + $idx << $scale]" %}
4298   interface(MEMORY_INTER) %{
4299     base($reg);
4300     index($idx);
4301     scale($scale);
4302     disp($off);
4303   %}
4304 %}
4305 
4306 //----------Special Memory Operands--------------------------------------------
4307 // Stack Slot Operand - This operand is used for loading and storing temporary
4308 //                      values on the stack where a match requires a value to
4309 //                      flow through memory.
4310 operand stackSlotP(sRegP reg)
4311 %{
4312   constraint(ALLOC_IN_RC(stack_slots));
4313   // No match rule because this operand is only generated in matching
4314 
4315   format %{ "[$reg]" %}
4316   interface(MEMORY_INTER) %{
4317     base(0x4);   // RSP
4318     index(0x4);  // No Index
4319     scale(0x0);  // No Scale
4320     disp($reg);  // Stack Offset
4321   %}
4322 %}
4323 
4324 operand stackSlotI(sRegI reg)
4325 %{
4326   constraint(ALLOC_IN_RC(stack_slots));
4327   // No match rule because this operand is only generated in matching
4328 
4329   format %{ "[$reg]" %}
4330   interface(MEMORY_INTER) %{
4331     base(0x4);   // RSP
4332     index(0x4);  // No Index
4333     scale(0x0);  // No Scale
4334     disp($reg);  // Stack Offset
4335   %}
4336 %}
4337 
4338 operand stackSlotF(sRegF reg)
4339 %{
4340   constraint(ALLOC_IN_RC(stack_slots));
4341   // No match rule because this operand is only generated in matching
4342 
4343   format %{ "[$reg]" %}
4344   interface(MEMORY_INTER) %{
4345     base(0x4);   // RSP
4346     index(0x4);  // No Index
4347     scale(0x0);  // No Scale
4348     disp($reg);  // Stack Offset
4349   %}
4350 %}
4351 
4352 operand stackSlotD(sRegD reg)
4353 %{
4354   constraint(ALLOC_IN_RC(stack_slots));
4355   // No match rule because this operand is only generated in matching
4356 
4357   format %{ "[$reg]" %}
4358   interface(MEMORY_INTER) %{
4359     base(0x4);   // RSP
4360     index(0x4);  // No Index
4361     scale(0x0);  // No Scale
4362     disp($reg);  // Stack Offset
4363   %}
4364 %}
4365 operand stackSlotL(sRegL reg)
4366 %{
4367   constraint(ALLOC_IN_RC(stack_slots));
4368   // No match rule because this operand is only generated in matching
4369 
4370   format %{ "[$reg]" %}
4371   interface(MEMORY_INTER) %{
4372     base(0x4);   // RSP
4373     index(0x4);  // No Index
4374     scale(0x0);  // No Scale
4375     disp($reg);  // Stack Offset
4376   %}
4377 %}
4378 
4379 //----------Conditional Branch Operands----------------------------------------
4380 // Comparison Op  - This is the operation of the comparison, and is limited to
4381 //                  the following set of codes:
4382 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4383 //
4384 // Other attributes of the comparison, such as unsignedness, are specified
4385 // by the comparison instruction that sets a condition code flags register.
4386 // That result is represented by a flags operand whose subtype is appropriate
4387 // to the unsignedness (etc.) of the comparison.
4388 //
4389 // Later, the instruction which matches both the Comparison Op (a Bool) and
4390 // the flags (produced by the Cmp) specifies the coding of the comparison op
4391 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4392 
4393 // Comparision Code
4394 operand cmpOp()
4395 %{
4396   match(Bool);
4397 
4398   format %{ "" %}
4399   interface(COND_INTER) %{
4400     equal(0x4, "e");
4401     not_equal(0x5, "ne");
4402     less(0xC, "l");
4403     greater_equal(0xD, "ge");
4404     less_equal(0xE, "le");
4405     greater(0xF, "g");
4406     overflow(0x0, "o");
4407     no_overflow(0x1, "no");
4408   %}
4409 %}
4410 
4411 // Comparison Code, unsigned compare.  Used by FP also, with
4412 // C2 (unordered) turned into GT or LT already.  The other bits
4413 // C0 and C3 are turned into Carry & Zero flags.
4414 operand cmpOpU()
4415 %{
4416   match(Bool);
4417 
4418   format %{ "" %}
4419   interface(COND_INTER) %{
4420     equal(0x4, "e");
4421     not_equal(0x5, "ne");
4422     less(0x2, "b");
4423     greater_equal(0x3, "nb");
4424     less_equal(0x6, "be");
4425     greater(0x7, "nbe");
4426     overflow(0x0, "o");
4427     no_overflow(0x1, "no");
4428   %}
4429 %}
4430 
4431 
4432 // Floating comparisons that don't require any fixup for the unordered case
4433 operand cmpOpUCF() %{
4434   match(Bool);
4435   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4436             n->as_Bool()->_test._test == BoolTest::ge ||
4437             n->as_Bool()->_test._test == BoolTest::le ||
4438             n->as_Bool()->_test._test == BoolTest::gt);
4439   format %{ "" %}
4440   interface(COND_INTER) %{
4441     equal(0x4, "e");
4442     not_equal(0x5, "ne");
4443     less(0x2, "b");
4444     greater_equal(0x3, "nb");
4445     less_equal(0x6, "be");
4446     greater(0x7, "nbe");
4447     overflow(0x0, "o");
4448     no_overflow(0x1, "no");
4449   %}
4450 %}
4451 
4452 
4453 // Floating comparisons that can be fixed up with extra conditional jumps
4454 operand cmpOpUCF2() %{
4455   match(Bool);
4456   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4457             n->as_Bool()->_test._test == BoolTest::eq);
4458   format %{ "" %}
4459   interface(COND_INTER) %{
4460     equal(0x4, "e");
4461     not_equal(0x5, "ne");
4462     less(0x2, "b");
4463     greater_equal(0x3, "nb");
4464     less_equal(0x6, "be");
4465     greater(0x7, "nbe");
4466     overflow(0x0, "o");
4467     no_overflow(0x1, "no");
4468   %}
4469 %}
4470 
4471 // Operands for bound floating pointer register arguments
4472 operand rxmm0() %{
4473   constraint(ALLOC_IN_RC(xmm0_reg));
4474   match(VecX);
4475   format%{%}
4476   interface(REG_INTER);
4477 %}
4478 operand rxmm1() %{
4479   constraint(ALLOC_IN_RC(xmm1_reg));
4480   match(VecX);
4481   format%{%}
4482   interface(REG_INTER);
4483 %}
4484 operand rxmm2() %{
4485   constraint(ALLOC_IN_RC(xmm2_reg));
4486   match(VecX);
4487   format%{%}
4488   interface(REG_INTER);
4489 %}
4490 operand rxmm3() %{
4491   constraint(ALLOC_IN_RC(xmm3_reg));
4492   match(VecX);
4493   format%{%}
4494   interface(REG_INTER);
4495 %}
4496 operand rxmm4() %{
4497   constraint(ALLOC_IN_RC(xmm4_reg));
4498   match(VecX);
4499   format%{%}
4500   interface(REG_INTER);
4501 %}
4502 operand rxmm5() %{
4503   constraint(ALLOC_IN_RC(xmm5_reg));
4504   match(VecX);
4505   format%{%}
4506   interface(REG_INTER);
4507 %}
4508 operand rxmm6() %{
4509   constraint(ALLOC_IN_RC(xmm6_reg));
4510   match(VecX);
4511   format%{%}
4512   interface(REG_INTER);
4513 %}
4514 operand rxmm7() %{
4515   constraint(ALLOC_IN_RC(xmm7_reg));
4516   match(VecX);
4517   format%{%}
4518   interface(REG_INTER);
4519 %}
4520 operand rxmm8() %{
4521   constraint(ALLOC_IN_RC(xmm8_reg));
4522   match(VecX);
4523   format%{%}
4524   interface(REG_INTER);
4525 %}
4526 operand rxmm9() %{
4527   constraint(ALLOC_IN_RC(xmm9_reg));
4528   match(VecX);
4529   format%{%}
4530   interface(REG_INTER);
4531 %}
4532 operand rxmm10() %{
4533   constraint(ALLOC_IN_RC(xmm10_reg));
4534   match(VecX);
4535   format%{%}
4536   interface(REG_INTER);
4537 %}
4538 operand rxmm11() %{
4539   constraint(ALLOC_IN_RC(xmm11_reg));
4540   match(VecX);
4541   format%{%}
4542   interface(REG_INTER);
4543 %}
4544 operand rxmm12() %{
4545   constraint(ALLOC_IN_RC(xmm12_reg));
4546   match(VecX);
4547   format%{%}
4548   interface(REG_INTER);
4549 %}
4550 operand rxmm13() %{
4551   constraint(ALLOC_IN_RC(xmm13_reg));
4552   match(VecX);
4553   format%{%}
4554   interface(REG_INTER);
4555 %}
4556 operand rxmm14() %{
4557   constraint(ALLOC_IN_RC(xmm14_reg));
4558   match(VecX);
4559   format%{%}
4560   interface(REG_INTER);
4561 %}
4562 operand rxmm15() %{
4563   constraint(ALLOC_IN_RC(xmm15_reg));
4564   match(VecX);
4565   format%{%}
4566   interface(REG_INTER);
4567 %}
4568 operand rxmm16() %{
4569   constraint(ALLOC_IN_RC(xmm16_reg));
4570   match(VecX);
4571   format%{%}
4572   interface(REG_INTER);
4573 %}
4574 operand rxmm17() %{
4575   constraint(ALLOC_IN_RC(xmm17_reg));
4576   match(VecX);
4577   format%{%}
4578   interface(REG_INTER);
4579 %}
4580 operand rxmm18() %{
4581   constraint(ALLOC_IN_RC(xmm18_reg));
4582   match(VecX);
4583   format%{%}
4584   interface(REG_INTER);
4585 %}
4586 operand rxmm19() %{
4587   constraint(ALLOC_IN_RC(xmm19_reg));
4588   match(VecX);
4589   format%{%}
4590   interface(REG_INTER);
4591 %}
4592 operand rxmm20() %{
4593   constraint(ALLOC_IN_RC(xmm20_reg));
4594   match(VecX);
4595   format%{%}
4596   interface(REG_INTER);
4597 %}
4598 operand rxmm21() %{
4599   constraint(ALLOC_IN_RC(xmm21_reg));
4600   match(VecX);
4601   format%{%}
4602   interface(REG_INTER);
4603 %}
4604 operand rxmm22() %{
4605   constraint(ALLOC_IN_RC(xmm22_reg));
4606   match(VecX);
4607   format%{%}
4608   interface(REG_INTER);
4609 %}
4610 operand rxmm23() %{
4611   constraint(ALLOC_IN_RC(xmm23_reg));
4612   match(VecX);
4613   format%{%}
4614   interface(REG_INTER);
4615 %}
4616 operand rxmm24() %{
4617   constraint(ALLOC_IN_RC(xmm24_reg));
4618   match(VecX);
4619   format%{%}
4620   interface(REG_INTER);
4621 %}
4622 operand rxmm25() %{
4623   constraint(ALLOC_IN_RC(xmm25_reg));
4624   match(VecX);
4625   format%{%}
4626   interface(REG_INTER);
4627 %}
4628 operand rxmm26() %{
4629   constraint(ALLOC_IN_RC(xmm26_reg));
4630   match(VecX);
4631   format%{%}
4632   interface(REG_INTER);
4633 %}
4634 operand rxmm27() %{
4635   constraint(ALLOC_IN_RC(xmm27_reg));
4636   match(VecX);
4637   format%{%}
4638   interface(REG_INTER);
4639 %}
4640 operand rxmm28() %{
4641   constraint(ALLOC_IN_RC(xmm28_reg));
4642   match(VecX);
4643   format%{%}
4644   interface(REG_INTER);
4645 %}
4646 operand rxmm29() %{
4647   constraint(ALLOC_IN_RC(xmm29_reg));
4648   match(VecX);
4649   format%{%}
4650   interface(REG_INTER);
4651 %}
4652 operand rxmm30() %{
4653   constraint(ALLOC_IN_RC(xmm30_reg));
4654   match(VecX);
4655   format%{%}
4656   interface(REG_INTER);
4657 %}
4658 operand rxmm31() %{
4659   constraint(ALLOC_IN_RC(xmm31_reg));
4660   match(VecX);
4661   format%{%}
4662   interface(REG_INTER);
4663 %}
4664 
4665 //----------OPERAND CLASSES----------------------------------------------------
4666 // Operand Classes are groups of operands that are used as to simplify
4667 // instruction definitions by not requiring the AD writer to specify separate
4668 // instructions for every form of operand when the instruction accepts
4669 // multiple operand types with the same basic encoding and format.  The classic
4670 // case of this is memory operands.
4671 
4672 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4673                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4674                indCompressedOopOffset,
4675                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4676                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4677                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4678 
4679 //----------PIPELINE-----------------------------------------------------------
4680 // Rules which define the behavior of the target architectures pipeline.
4681 pipeline %{
4682 
4683 //----------ATTRIBUTES---------------------------------------------------------
4684 attributes %{
4685   variable_size_instructions;        // Fixed size instructions
4686   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4687   instruction_unit_size = 1;         // An instruction is 1 bytes long
4688   instruction_fetch_unit_size = 16;  // The processor fetches one line
4689   instruction_fetch_units = 1;       // of 16 bytes
4690 
4691   // List of nop instructions
4692   nops( MachNop );
4693 %}
4694 
4695 //----------RESOURCES----------------------------------------------------------
4696 // Resources are the functional units available to the machine
4697 
4698 // Generic P2/P3 pipeline
4699 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4700 // 3 instructions decoded per cycle.
4701 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4702 // 3 ALU op, only ALU0 handles mul instructions.
4703 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4704            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4705            BR, FPU,
4706            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4707 
4708 //----------PIPELINE DESCRIPTION-----------------------------------------------
4709 // Pipeline Description specifies the stages in the machine's pipeline
4710 
4711 // Generic P2/P3 pipeline
4712 pipe_desc(S0, S1, S2, S3, S4, S5);
4713 
4714 //----------PIPELINE CLASSES---------------------------------------------------
4715 // Pipeline Classes describe the stages in which input and output are
4716 // referenced by the hardware pipeline.
4717 
4718 // Naming convention: ialu or fpu
4719 // Then: _reg
4720 // Then: _reg if there is a 2nd register
4721 // Then: _long if it's a pair of instructions implementing a long
4722 // Then: _fat if it requires the big decoder
4723 //   Or: _mem if it requires the big decoder and a memory unit.
4724 
4725 // Integer ALU reg operation
4726 pipe_class ialu_reg(rRegI dst)
4727 %{
4728     single_instruction;
4729     dst    : S4(write);
4730     dst    : S3(read);
4731     DECODE : S0;        // any decoder
4732     ALU    : S3;        // any alu
4733 %}
4734 
4735 // Long ALU reg operation
4736 pipe_class ialu_reg_long(rRegL dst)
4737 %{
4738     instruction_count(2);
4739     dst    : S4(write);
4740     dst    : S3(read);
4741     DECODE : S0(2);     // any 2 decoders
4742     ALU    : S3(2);     // both alus
4743 %}
4744 
4745 // Integer ALU reg operation using big decoder
4746 pipe_class ialu_reg_fat(rRegI dst)
4747 %{
4748     single_instruction;
4749     dst    : S4(write);
4750     dst    : S3(read);
4751     D0     : S0;        // big decoder only
4752     ALU    : S3;        // any alu
4753 %}
4754 
4755 // Long ALU reg operation using big decoder
4756 pipe_class ialu_reg_long_fat(rRegL dst)
4757 %{
4758     instruction_count(2);
4759     dst    : S4(write);
4760     dst    : S3(read);
4761     D0     : S0(2);     // big decoder only; twice
4762     ALU    : S3(2);     // any 2 alus
4763 %}
4764 
4765 // Integer ALU reg-reg operation
4766 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4767 %{
4768     single_instruction;
4769     dst    : S4(write);
4770     src    : S3(read);
4771     DECODE : S0;        // any decoder
4772     ALU    : S3;        // any alu
4773 %}
4774 
4775 // Long ALU reg-reg operation
4776 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4777 %{
4778     instruction_count(2);
4779     dst    : S4(write);
4780     src    : S3(read);
4781     DECODE : S0(2);     // any 2 decoders
4782     ALU    : S3(2);     // both alus
4783 %}
4784 
4785 // Integer ALU reg-reg operation
4786 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4787 %{
4788     single_instruction;
4789     dst    : S4(write);
4790     src    : S3(read);
4791     D0     : S0;        // big decoder only
4792     ALU    : S3;        // any alu
4793 %}
4794 
4795 // Long ALU reg-reg operation
4796 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4797 %{
4798     instruction_count(2);
4799     dst    : S4(write);
4800     src    : S3(read);
4801     D0     : S0(2);     // big decoder only; twice
4802     ALU    : S3(2);     // both alus
4803 %}
4804 
4805 // Integer ALU reg-mem operation
4806 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4807 %{
4808     single_instruction;
4809     dst    : S5(write);
4810     mem    : S3(read);
4811     D0     : S0;        // big decoder only
4812     ALU    : S4;        // any alu
4813     MEM    : S3;        // any mem
4814 %}
4815 
4816 // Integer mem operation (prefetch)
4817 pipe_class ialu_mem(memory mem)
4818 %{
4819     single_instruction;
4820     mem    : S3(read);
4821     D0     : S0;        // big decoder only
4822     MEM    : S3;        // any mem
4823 %}
4824 
4825 // Integer Store to Memory
4826 pipe_class ialu_mem_reg(memory mem, rRegI src)
4827 %{
4828     single_instruction;
4829     mem    : S3(read);
4830     src    : S5(read);
4831     D0     : S0;        // big decoder only
4832     ALU    : S4;        // any alu
4833     MEM    : S3;
4834 %}
4835 
4836 // // Long Store to Memory
4837 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4838 // %{
4839 //     instruction_count(2);
4840 //     mem    : S3(read);
4841 //     src    : S5(read);
4842 //     D0     : S0(2);          // big decoder only; twice
4843 //     ALU    : S4(2);     // any 2 alus
4844 //     MEM    : S3(2);  // Both mems
4845 // %}
4846 
4847 // Integer Store to Memory
4848 pipe_class ialu_mem_imm(memory mem)
4849 %{
4850     single_instruction;
4851     mem    : S3(read);
4852     D0     : S0;        // big decoder only
4853     ALU    : S4;        // any alu
4854     MEM    : S3;
4855 %}
4856 
4857 // Integer ALU0 reg-reg operation
4858 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4859 %{
4860     single_instruction;
4861     dst    : S4(write);
4862     src    : S3(read);
4863     D0     : S0;        // Big decoder only
4864     ALU0   : S3;        // only alu0
4865 %}
4866 
4867 // Integer ALU0 reg-mem operation
4868 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4869 %{
4870     single_instruction;
4871     dst    : S5(write);
4872     mem    : S3(read);
4873     D0     : S0;        // big decoder only
4874     ALU0   : S4;        // ALU0 only
4875     MEM    : S3;        // any mem
4876 %}
4877 
4878 // Integer ALU reg-reg operation
4879 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4880 %{
4881     single_instruction;
4882     cr     : S4(write);
4883     src1   : S3(read);
4884     src2   : S3(read);
4885     DECODE : S0;        // any decoder
4886     ALU    : S3;        // any alu
4887 %}
4888 
4889 // Integer ALU reg-imm operation
4890 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4891 %{
4892     single_instruction;
4893     cr     : S4(write);
4894     src1   : S3(read);
4895     DECODE : S0;        // any decoder
4896     ALU    : S3;        // any alu
4897 %}
4898 
4899 // Integer ALU reg-mem operation
4900 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4901 %{
4902     single_instruction;
4903     cr     : S4(write);
4904     src1   : S3(read);
4905     src2   : S3(read);
4906     D0     : S0;        // big decoder only
4907     ALU    : S4;        // any alu
4908     MEM    : S3;
4909 %}
4910 
4911 // Conditional move reg-reg
4912 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4913 %{
4914     instruction_count(4);
4915     y      : S4(read);
4916     q      : S3(read);
4917     p      : S3(read);
4918     DECODE : S0(4);     // any decoder
4919 %}
4920 
4921 // Conditional move reg-reg
4922 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4923 %{
4924     single_instruction;
4925     dst    : S4(write);
4926     src    : S3(read);
4927     cr     : S3(read);
4928     DECODE : S0;        // any decoder
4929 %}
4930 
4931 // Conditional move reg-mem
4932 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4933 %{
4934     single_instruction;
4935     dst    : S4(write);
4936     src    : S3(read);
4937     cr     : S3(read);
4938     DECODE : S0;        // any decoder
4939     MEM    : S3;
4940 %}
4941 
4942 // Conditional move reg-reg long
4943 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4944 %{
4945     single_instruction;
4946     dst    : S4(write);
4947     src    : S3(read);
4948     cr     : S3(read);
4949     DECODE : S0(2);     // any 2 decoders
4950 %}
4951 
4952 // XXX
4953 // // Conditional move double reg-reg
4954 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4955 // %{
4956 //     single_instruction;
4957 //     dst    : S4(write);
4958 //     src    : S3(read);
4959 //     cr     : S3(read);
4960 //     DECODE : S0;     // any decoder
4961 // %}
4962 
4963 // Float reg-reg operation
4964 pipe_class fpu_reg(regD dst)
4965 %{
4966     instruction_count(2);
4967     dst    : S3(read);
4968     DECODE : S0(2);     // any 2 decoders
4969     FPU    : S3;
4970 %}
4971 
4972 // Float reg-reg operation
4973 pipe_class fpu_reg_reg(regD dst, regD src)
4974 %{
4975     instruction_count(2);
4976     dst    : S4(write);
4977     src    : S3(read);
4978     DECODE : S0(2);     // any 2 decoders
4979     FPU    : S3;
4980 %}
4981 
4982 // Float reg-reg operation
4983 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4984 %{
4985     instruction_count(3);
4986     dst    : S4(write);
4987     src1   : S3(read);
4988     src2   : S3(read);
4989     DECODE : S0(3);     // any 3 decoders
4990     FPU    : S3(2);
4991 %}
4992 
4993 // Float reg-reg operation
4994 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4995 %{
4996     instruction_count(4);
4997     dst    : S4(write);
4998     src1   : S3(read);
4999     src2   : S3(read);
5000     src3   : S3(read);
5001     DECODE : S0(4);     // any 3 decoders
5002     FPU    : S3(2);
5003 %}
5004 
5005 // Float reg-reg operation
5006 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
5007 %{
5008     instruction_count(4);
5009     dst    : S4(write);
5010     src1   : S3(read);
5011     src2   : S3(read);
5012     src3   : S3(read);
5013     DECODE : S1(3);     // any 3 decoders
5014     D0     : S0;        // Big decoder only
5015     FPU    : S3(2);
5016     MEM    : S3;
5017 %}
5018 
5019 // Float reg-mem operation
5020 pipe_class fpu_reg_mem(regD dst, memory mem)
5021 %{
5022     instruction_count(2);
5023     dst    : S5(write);
5024     mem    : S3(read);
5025     D0     : S0;        // big decoder only
5026     DECODE : S1;        // any decoder for FPU POP
5027     FPU    : S4;
5028     MEM    : S3;        // any mem
5029 %}
5030 
5031 // Float reg-mem operation
5032 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
5033 %{
5034     instruction_count(3);
5035     dst    : S5(write);
5036     src1   : S3(read);
5037     mem    : S3(read);
5038     D0     : S0;        // big decoder only
5039     DECODE : S1(2);     // any decoder for FPU POP
5040     FPU    : S4;
5041     MEM    : S3;        // any mem
5042 %}
5043 
5044 // Float mem-reg operation
5045 pipe_class fpu_mem_reg(memory mem, regD src)
5046 %{
5047     instruction_count(2);
5048     src    : S5(read);
5049     mem    : S3(read);
5050     DECODE : S0;        // any decoder for FPU PUSH
5051     D0     : S1;        // big decoder only
5052     FPU    : S4;
5053     MEM    : S3;        // any mem
5054 %}
5055 
5056 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
5057 %{
5058     instruction_count(3);
5059     src1   : S3(read);
5060     src2   : S3(read);
5061     mem    : S3(read);
5062     DECODE : S0(2);     // any decoder for FPU PUSH
5063     D0     : S1;        // big decoder only
5064     FPU    : S4;
5065     MEM    : S3;        // any mem
5066 %}
5067 
5068 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
5069 %{
5070     instruction_count(3);
5071     src1   : S3(read);
5072     src2   : S3(read);
5073     mem    : S4(read);
5074     DECODE : S0;        // any decoder for FPU PUSH
5075     D0     : S0(2);     // big decoder only
5076     FPU    : S4;
5077     MEM    : S3(2);     // any mem
5078 %}
5079 
5080 pipe_class fpu_mem_mem(memory dst, memory src1)
5081 %{
5082     instruction_count(2);
5083     src1   : S3(read);
5084     dst    : S4(read);
5085     D0     : S0(2);     // big decoder only
5086     MEM    : S3(2);     // any mem
5087 %}
5088 
5089 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
5090 %{
5091     instruction_count(3);
5092     src1   : S3(read);
5093     src2   : S3(read);
5094     dst    : S4(read);
5095     D0     : S0(3);     // big decoder only
5096     FPU    : S4;
5097     MEM    : S3(3);     // any mem
5098 %}
5099 
5100 pipe_class fpu_mem_reg_con(memory mem, regD src1)
5101 %{
5102     instruction_count(3);
5103     src1   : S4(read);
5104     mem    : S4(read);
5105     DECODE : S0;        // any decoder for FPU PUSH
5106     D0     : S0(2);     // big decoder only
5107     FPU    : S4;
5108     MEM    : S3(2);     // any mem
5109 %}
5110 
5111 // Float load constant
5112 pipe_class fpu_reg_con(regD dst)
5113 %{
5114     instruction_count(2);
5115     dst    : S5(write);
5116     D0     : S0;        // big decoder only for the load
5117     DECODE : S1;        // any decoder for FPU POP
5118     FPU    : S4;
5119     MEM    : S3;        // any mem
5120 %}
5121 
5122 // Float load constant
5123 pipe_class fpu_reg_reg_con(regD dst, regD src)
5124 %{
5125     instruction_count(3);
5126     dst    : S5(write);
5127     src    : S3(read);
5128     D0     : S0;        // big decoder only for the load
5129     DECODE : S1(2);     // any decoder for FPU POP
5130     FPU    : S4;
5131     MEM    : S3;        // any mem
5132 %}
5133 
5134 // UnConditional branch
5135 pipe_class pipe_jmp(label labl)
5136 %{
5137     single_instruction;
5138     BR   : S3;
5139 %}
5140 
5141 // Conditional branch
5142 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
5143 %{
5144     single_instruction;
5145     cr    : S1(read);
5146     BR    : S3;
5147 %}
5148 
5149 // Allocation idiom
5150 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
5151 %{
5152     instruction_count(1); force_serialization;
5153     fixed_latency(6);
5154     heap_ptr : S3(read);
5155     DECODE   : S0(3);
5156     D0       : S2;
5157     MEM      : S3;
5158     ALU      : S3(2);
5159     dst      : S5(write);
5160     BR       : S5;
5161 %}
5162 
5163 // Generic big/slow expanded idiom
5164 pipe_class pipe_slow()
5165 %{
5166     instruction_count(10); multiple_bundles; force_serialization;
5167     fixed_latency(100);
5168     D0  : S0(2);
5169     MEM : S3(2);
5170 %}
5171 
5172 // The real do-nothing guy
5173 pipe_class empty()
5174 %{
5175     instruction_count(0);
5176 %}
5177 
5178 // Define the class for the Nop node
5179 define
5180 %{
5181    MachNop = empty;
5182 %}
5183 
5184 %}
5185 
5186 //----------INSTRUCTIONS-------------------------------------------------------
5187 //
5188 // match      -- States which machine-independent subtree may be replaced
5189 //               by this instruction.
5190 // ins_cost   -- The estimated cost of this instruction is used by instruction
5191 //               selection to identify a minimum cost tree of machine
5192 //               instructions that matches a tree of machine-independent
5193 //               instructions.
5194 // format     -- A string providing the disassembly for this instruction.
5195 //               The value of an instruction's operand may be inserted
5196 //               by referring to it with a '$' prefix.
5197 // opcode     -- Three instruction opcodes may be provided.  These are referred
5198 //               to within an encode class as $primary, $secondary, and $tertiary
5199 //               rrspectively.  The primary opcode is commonly used to
5200 //               indicate the type of machine instruction, while secondary
5201 //               and tertiary are often used for prefix options or addressing
5202 //               modes.
5203 // ins_encode -- A list of encode classes with parameters. The encode class
5204 //               name must have been defined in an 'enc_class' specification
5205 //               in the encode section of the architecture description.
5206 
5207 
5208 //----------Load/Store/Move Instructions---------------------------------------
5209 //----------Load Instructions--------------------------------------------------
5210 
5211 // Load Byte (8 bit signed)
5212 instruct loadB(rRegI dst, memory mem)
5213 %{
5214   match(Set dst (LoadB mem));
5215 
5216   ins_cost(125);
5217   format %{ "movsbl  $dst, $mem\t# byte" %}
5218 
5219   ins_encode %{
5220     __ movsbl($dst$$Register, $mem$$Address);
5221   %}
5222 
5223   ins_pipe(ialu_reg_mem);
5224 %}
5225 
5226 // Load Byte (8 bit signed) into Long Register
5227 instruct loadB2L(rRegL dst, memory mem)
5228 %{
5229   match(Set dst (ConvI2L (LoadB mem)));
5230 
5231   ins_cost(125);
5232   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
5233 
5234   ins_encode %{
5235     __ movsbq($dst$$Register, $mem$$Address);
5236   %}
5237 
5238   ins_pipe(ialu_reg_mem);
5239 %}
5240 
5241 // Load Unsigned Byte (8 bit UNsigned)
5242 instruct loadUB(rRegI dst, memory mem)
5243 %{
5244   match(Set dst (LoadUB mem));
5245 
5246   ins_cost(125);
5247   format %{ "movzbl  $dst, $mem\t# ubyte" %}
5248 
5249   ins_encode %{
5250     __ movzbl($dst$$Register, $mem$$Address);
5251   %}
5252 
5253   ins_pipe(ialu_reg_mem);
5254 %}
5255 
5256 // Load Unsigned Byte (8 bit UNsigned) into Long Register
5257 instruct loadUB2L(rRegL dst, memory mem)
5258 %{
5259   match(Set dst (ConvI2L (LoadUB mem)));
5260 
5261   ins_cost(125);
5262   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
5263 
5264   ins_encode %{
5265     __ movzbq($dst$$Register, $mem$$Address);
5266   %}
5267 
5268   ins_pipe(ialu_reg_mem);
5269 %}
5270 
5271 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5272 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5273   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5274   effect(KILL cr);
5275 
5276   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5277             "andl    $dst, right_n_bits($mask, 8)" %}
5278   ins_encode %{
5279     Register Rdst = $dst$$Register;
5280     __ movzbq(Rdst, $mem$$Address);
5281     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5282   %}
5283   ins_pipe(ialu_reg_mem);
5284 %}
5285 
5286 // Load Short (16 bit signed)
5287 instruct loadS(rRegI dst, memory mem)
5288 %{
5289   match(Set dst (LoadS mem));
5290 
5291   ins_cost(125);
5292   format %{ "movswl $dst, $mem\t# short" %}
5293 
5294   ins_encode %{
5295     __ movswl($dst$$Register, $mem$$Address);
5296   %}
5297 
5298   ins_pipe(ialu_reg_mem);
5299 %}
5300 
5301 // Load Short (16 bit signed) to Byte (8 bit signed)
5302 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5303   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5304 
5305   ins_cost(125);
5306   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5307   ins_encode %{
5308     __ movsbl($dst$$Register, $mem$$Address);
5309   %}
5310   ins_pipe(ialu_reg_mem);
5311 %}
5312 
5313 // Load Short (16 bit signed) into Long Register
5314 instruct loadS2L(rRegL dst, memory mem)
5315 %{
5316   match(Set dst (ConvI2L (LoadS mem)));
5317 
5318   ins_cost(125);
5319   format %{ "movswq $dst, $mem\t# short -> long" %}
5320 
5321   ins_encode %{
5322     __ movswq($dst$$Register, $mem$$Address);
5323   %}
5324 
5325   ins_pipe(ialu_reg_mem);
5326 %}
5327 
5328 // Load Unsigned Short/Char (16 bit UNsigned)
5329 instruct loadUS(rRegI dst, memory mem)
5330 %{
5331   match(Set dst (LoadUS mem));
5332 
5333   ins_cost(125);
5334   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5335 
5336   ins_encode %{
5337     __ movzwl($dst$$Register, $mem$$Address);
5338   %}
5339 
5340   ins_pipe(ialu_reg_mem);
5341 %}
5342 
5343 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5344 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5345   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5346 
5347   ins_cost(125);
5348   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5349   ins_encode %{
5350     __ movsbl($dst$$Register, $mem$$Address);
5351   %}
5352   ins_pipe(ialu_reg_mem);
5353 %}
5354 
5355 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5356 instruct loadUS2L(rRegL dst, memory mem)
5357 %{
5358   match(Set dst (ConvI2L (LoadUS mem)));
5359 
5360   ins_cost(125);
5361   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5362 
5363   ins_encode %{
5364     __ movzwq($dst$$Register, $mem$$Address);
5365   %}
5366 
5367   ins_pipe(ialu_reg_mem);
5368 %}
5369 
5370 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5371 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5372   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5373 
5374   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5375   ins_encode %{
5376     __ movzbq($dst$$Register, $mem$$Address);
5377   %}
5378   ins_pipe(ialu_reg_mem);
5379 %}
5380 
5381 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5382 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5383   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5384   effect(KILL cr);
5385 
5386   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5387             "andl    $dst, right_n_bits($mask, 16)" %}
5388   ins_encode %{
5389     Register Rdst = $dst$$Register;
5390     __ movzwq(Rdst, $mem$$Address);
5391     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5392   %}
5393   ins_pipe(ialu_reg_mem);
5394 %}
5395 
5396 // Load Integer
5397 instruct loadI(rRegI dst, memory mem)
5398 %{
5399   match(Set dst (LoadI mem));
5400 
5401   ins_cost(125);
5402   format %{ "movl    $dst, $mem\t# int" %}
5403 
5404   ins_encode %{
5405     __ movl($dst$$Register, $mem$$Address);
5406   %}
5407 
5408   ins_pipe(ialu_reg_mem);
5409 %}
5410 
5411 // Load Integer (32 bit signed) to Byte (8 bit signed)
5412 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5413   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5414 
5415   ins_cost(125);
5416   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5417   ins_encode %{
5418     __ movsbl($dst$$Register, $mem$$Address);
5419   %}
5420   ins_pipe(ialu_reg_mem);
5421 %}
5422 
5423 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5424 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5425   match(Set dst (AndI (LoadI mem) mask));
5426 
5427   ins_cost(125);
5428   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5429   ins_encode %{
5430     __ movzbl($dst$$Register, $mem$$Address);
5431   %}
5432   ins_pipe(ialu_reg_mem);
5433 %}
5434 
5435 // Load Integer (32 bit signed) to Short (16 bit signed)
5436 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5437   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5438 
5439   ins_cost(125);
5440   format %{ "movswl  $dst, $mem\t# int -> short" %}
5441   ins_encode %{
5442     __ movswl($dst$$Register, $mem$$Address);
5443   %}
5444   ins_pipe(ialu_reg_mem);
5445 %}
5446 
5447 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5448 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5449   match(Set dst (AndI (LoadI mem) mask));
5450 
5451   ins_cost(125);
5452   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5453   ins_encode %{
5454     __ movzwl($dst$$Register, $mem$$Address);
5455   %}
5456   ins_pipe(ialu_reg_mem);
5457 %}
5458 
5459 // Load Integer into Long Register
5460 instruct loadI2L(rRegL dst, memory mem)
5461 %{
5462   match(Set dst (ConvI2L (LoadI mem)));
5463 
5464   ins_cost(125);
5465   format %{ "movslq  $dst, $mem\t# int -> long" %}
5466 
5467   ins_encode %{
5468     __ movslq($dst$$Register, $mem$$Address);
5469   %}
5470 
5471   ins_pipe(ialu_reg_mem);
5472 %}
5473 
5474 // Load Integer with mask 0xFF into Long Register
5475 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5476   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5477 
5478   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5479   ins_encode %{
5480     __ movzbq($dst$$Register, $mem$$Address);
5481   %}
5482   ins_pipe(ialu_reg_mem);
5483 %}
5484 
5485 // Load Integer with mask 0xFFFF into Long Register
5486 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5487   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5488 
5489   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5490   ins_encode %{
5491     __ movzwq($dst$$Register, $mem$$Address);
5492   %}
5493   ins_pipe(ialu_reg_mem);
5494 %}
5495 
5496 // Load Integer with a 31-bit mask into Long Register
5497 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5498   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5499   effect(KILL cr);
5500 
5501   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5502             "andl    $dst, $mask" %}
5503   ins_encode %{
5504     Register Rdst = $dst$$Register;
5505     __ movl(Rdst, $mem$$Address);
5506     __ andl(Rdst, $mask$$constant);
5507   %}
5508   ins_pipe(ialu_reg_mem);
5509 %}
5510 
5511 // Load Unsigned Integer into Long Register
5512 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5513 %{
5514   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5515 
5516   ins_cost(125);
5517   format %{ "movl    $dst, $mem\t# uint -> long" %}
5518 
5519   ins_encode %{
5520     __ movl($dst$$Register, $mem$$Address);
5521   %}
5522 
5523   ins_pipe(ialu_reg_mem);
5524 %}
5525 
5526 // Load Long
5527 instruct loadL(rRegL dst, memory mem)
5528 %{
5529   match(Set dst (LoadL mem));
5530 
5531   ins_cost(125);
5532   format %{ "movq    $dst, $mem\t# long" %}
5533 
5534   ins_encode %{
5535     __ movq($dst$$Register, $mem$$Address);
5536   %}
5537 
5538   ins_pipe(ialu_reg_mem); // XXX
5539 %}
5540 
5541 // Load Range
5542 instruct loadRange(rRegI dst, memory mem)
5543 %{
5544   match(Set dst (LoadRange mem));
5545 
5546   ins_cost(125); // XXX
5547   format %{ "movl    $dst, $mem\t# range" %}
5548   opcode(0x8B);
5549   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5550   ins_pipe(ialu_reg_mem);
5551 %}
5552 
5553 // Load Pointer
5554 instruct loadP(rRegP dst, memory mem)
5555 %{
5556   match(Set dst (LoadP mem));
5557 
5558   ins_cost(125); // XXX
5559   format %{ "movq    $dst, $mem\t# ptr" %}
5560   opcode(0x8B);
5561   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5562   ins_pipe(ialu_reg_mem); // XXX
5563 %}
5564 
5565 // Load Compressed Pointer
5566 instruct loadN(rRegN dst, memory mem)
5567 %{
5568    match(Set dst (LoadN mem));
5569 
5570    ins_cost(125); // XXX
5571    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5572    ins_encode %{
5573      __ movl($dst$$Register, $mem$$Address);
5574    %}
5575    ins_pipe(ialu_reg_mem); // XXX
5576 %}
5577 
5578 
5579 // Load Klass Pointer
5580 instruct loadKlass(rRegP dst, memory mem)
5581 %{
5582   match(Set dst (LoadKlass mem));
5583 
5584   ins_cost(125); // XXX
5585   format %{ "movq    $dst, $mem\t# class" %}
5586   opcode(0x8B);
5587   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5588   ins_pipe(ialu_reg_mem); // XXX
5589 %}
5590 
5591 // Load narrow Klass Pointer
5592 instruct loadNKlass(rRegN dst, memory mem)
5593 %{
5594   match(Set dst (LoadNKlass mem));
5595 
5596   ins_cost(125); // XXX
5597   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5598   ins_encode %{
5599     __ movl($dst$$Register, $mem$$Address);
5600   %}
5601   ins_pipe(ialu_reg_mem); // XXX
5602 %}
5603 
5604 // Load Float
5605 instruct loadF(regF dst, memory mem)
5606 %{
5607   match(Set dst (LoadF mem));
5608 
5609   ins_cost(145); // XXX
5610   format %{ "movss   $dst, $mem\t# float" %}
5611   ins_encode %{
5612     __ movflt($dst$$XMMRegister, $mem$$Address);
5613   %}
5614   ins_pipe(pipe_slow); // XXX
5615 %}
5616 
5617 // Load Float
5618 instruct MoveF2VL(vlRegF dst, regF src) %{
5619   match(Set dst src);
5620   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5621   ins_encode %{
5622     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5623   %}
5624   ins_pipe( fpu_reg_reg );
5625 %}
5626 
5627 // Load Float
5628 instruct MoveVL2F(regF dst, vlRegF src) %{
5629   match(Set dst src);
5630   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5631   ins_encode %{
5632     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5633   %}
5634   ins_pipe( fpu_reg_reg );
5635 %}
5636 
5637 // Load Double
5638 instruct loadD_partial(regD dst, memory mem)
5639 %{
5640   predicate(!UseXmmLoadAndClearUpper);
5641   match(Set dst (LoadD mem));
5642 
5643   ins_cost(145); // XXX
5644   format %{ "movlpd  $dst, $mem\t# double" %}
5645   ins_encode %{
5646     __ movdbl($dst$$XMMRegister, $mem$$Address);
5647   %}
5648   ins_pipe(pipe_slow); // XXX
5649 %}
5650 
5651 instruct loadD(regD dst, memory mem)
5652 %{
5653   predicate(UseXmmLoadAndClearUpper);
5654   match(Set dst (LoadD mem));
5655 
5656   ins_cost(145); // XXX
5657   format %{ "movsd   $dst, $mem\t# double" %}
5658   ins_encode %{
5659     __ movdbl($dst$$XMMRegister, $mem$$Address);
5660   %}
5661   ins_pipe(pipe_slow); // XXX
5662 %}
5663 
5664 // Load Double
5665 instruct MoveD2VL(vlRegD dst, regD src) %{
5666   match(Set dst src);
5667   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5668   ins_encode %{
5669     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5670   %}
5671   ins_pipe( fpu_reg_reg );
5672 %}
5673 
5674 // Load Double
5675 instruct MoveVL2D(regD dst, vlRegD src) %{
5676   match(Set dst src);
5677   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5678   ins_encode %{
5679     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5680   %}
5681   ins_pipe( fpu_reg_reg );
5682 %}
5683 
5684 // Load Effective Address
5685 instruct leaP8(rRegP dst, indOffset8 mem)
5686 %{
5687   match(Set dst mem);
5688 
5689   ins_cost(110); // XXX
5690   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5691   opcode(0x8D);
5692   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5693   ins_pipe(ialu_reg_reg_fat);
5694 %}
5695 
5696 instruct leaP32(rRegP dst, indOffset32 mem)
5697 %{
5698   match(Set dst mem);
5699 
5700   ins_cost(110);
5701   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5702   opcode(0x8D);
5703   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5704   ins_pipe(ialu_reg_reg_fat);
5705 %}
5706 
5707 // instruct leaPIdx(rRegP dst, indIndex mem)
5708 // %{
5709 //   match(Set dst mem);
5710 
5711 //   ins_cost(110);
5712 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5713 //   opcode(0x8D);
5714 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5715 //   ins_pipe(ialu_reg_reg_fat);
5716 // %}
5717 
5718 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5719 %{
5720   match(Set dst mem);
5721 
5722   ins_cost(110);
5723   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5724   opcode(0x8D);
5725   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5726   ins_pipe(ialu_reg_reg_fat);
5727 %}
5728 
5729 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5730 %{
5731   match(Set dst mem);
5732 
5733   ins_cost(110);
5734   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5735   opcode(0x8D);
5736   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5737   ins_pipe(ialu_reg_reg_fat);
5738 %}
5739 
5740 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5741 %{
5742   match(Set dst mem);
5743 
5744   ins_cost(110);
5745   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5746   opcode(0x8D);
5747   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5748   ins_pipe(ialu_reg_reg_fat);
5749 %}
5750 
5751 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5752 %{
5753   match(Set dst mem);
5754 
5755   ins_cost(110);
5756   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5757   opcode(0x8D);
5758   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5759   ins_pipe(ialu_reg_reg_fat);
5760 %}
5761 
5762 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5763 %{
5764   match(Set dst mem);
5765 
5766   ins_cost(110);
5767   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5768   opcode(0x8D);
5769   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5770   ins_pipe(ialu_reg_reg_fat);
5771 %}
5772 
5773 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5774 %{
5775   match(Set dst mem);
5776 
5777   ins_cost(110);
5778   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5779   opcode(0x8D);
5780   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5781   ins_pipe(ialu_reg_reg_fat);
5782 %}
5783 
5784 // Load Effective Address which uses Narrow (32-bits) oop
5785 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5786 %{
5787   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5788   match(Set dst mem);
5789 
5790   ins_cost(110);
5791   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5792   opcode(0x8D);
5793   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5794   ins_pipe(ialu_reg_reg_fat);
5795 %}
5796 
5797 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5798 %{
5799   predicate(Universe::narrow_oop_shift() == 0);
5800   match(Set dst mem);
5801 
5802   ins_cost(110); // XXX
5803   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5804   opcode(0x8D);
5805   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5806   ins_pipe(ialu_reg_reg_fat);
5807 %}
5808 
5809 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5810 %{
5811   predicate(Universe::narrow_oop_shift() == 0);
5812   match(Set dst mem);
5813 
5814   ins_cost(110);
5815   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5816   opcode(0x8D);
5817   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5818   ins_pipe(ialu_reg_reg_fat);
5819 %}
5820 
5821 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5822 %{
5823   predicate(Universe::narrow_oop_shift() == 0);
5824   match(Set dst mem);
5825 
5826   ins_cost(110);
5827   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5828   opcode(0x8D);
5829   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5830   ins_pipe(ialu_reg_reg_fat);
5831 %}
5832 
5833 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5834 %{
5835   predicate(Universe::narrow_oop_shift() == 0);
5836   match(Set dst mem);
5837 
5838   ins_cost(110);
5839   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5840   opcode(0x8D);
5841   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5842   ins_pipe(ialu_reg_reg_fat);
5843 %}
5844 
5845 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5846 %{
5847   predicate(Universe::narrow_oop_shift() == 0);
5848   match(Set dst mem);
5849 
5850   ins_cost(110);
5851   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5852   opcode(0x8D);
5853   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5854   ins_pipe(ialu_reg_reg_fat);
5855 %}
5856 
5857 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5858 %{
5859   predicate(Universe::narrow_oop_shift() == 0);
5860   match(Set dst mem);
5861 
5862   ins_cost(110);
5863   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5864   opcode(0x8D);
5865   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5866   ins_pipe(ialu_reg_reg_fat);
5867 %}
5868 
5869 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5870 %{
5871   predicate(Universe::narrow_oop_shift() == 0);
5872   match(Set dst mem);
5873 
5874   ins_cost(110);
5875   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5876   opcode(0x8D);
5877   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5878   ins_pipe(ialu_reg_reg_fat);
5879 %}
5880 
5881 instruct loadConI(rRegI dst, immI src)
5882 %{
5883   match(Set dst src);
5884 
5885   format %{ "movl    $dst, $src\t# int" %}
5886   ins_encode(load_immI(dst, src));
5887   ins_pipe(ialu_reg_fat); // XXX
5888 %}
5889 
5890 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5891 %{
5892   match(Set dst src);
5893   effect(KILL cr);
5894 
5895   ins_cost(50);
5896   format %{ "xorl    $dst, $dst\t# int" %}
5897   opcode(0x33); /* + rd */
5898   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5899   ins_pipe(ialu_reg);
5900 %}
5901 
5902 instruct loadConL(rRegL dst, immL src)
5903 %{
5904   match(Set dst src);
5905 
5906   ins_cost(150);
5907   format %{ "movq    $dst, $src\t# long" %}
5908   ins_encode(load_immL(dst, src));
5909   ins_pipe(ialu_reg);
5910 %}
5911 
5912 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5913 %{
5914   match(Set dst src);
5915   effect(KILL cr);
5916 
5917   ins_cost(50);
5918   format %{ "xorl    $dst, $dst\t# long" %}
5919   opcode(0x33); /* + rd */
5920   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5921   ins_pipe(ialu_reg); // XXX
5922 %}
5923 
5924 instruct loadConUL32(rRegL dst, immUL32 src)
5925 %{
5926   match(Set dst src);
5927 
5928   ins_cost(60);
5929   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5930   ins_encode(load_immUL32(dst, src));
5931   ins_pipe(ialu_reg);
5932 %}
5933 
5934 instruct loadConL32(rRegL dst, immL32 src)
5935 %{
5936   match(Set dst src);
5937 
5938   ins_cost(70);
5939   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5940   ins_encode(load_immL32(dst, src));
5941   ins_pipe(ialu_reg);
5942 %}
5943 
5944 instruct loadConP(rRegP dst, immP con) %{
5945   match(Set dst con);
5946 
5947   format %{ "movq    $dst, $con\t# ptr" %}
5948   ins_encode(load_immP(dst, con));
5949   ins_pipe(ialu_reg_fat); // XXX
5950 %}
5951 
5952 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5953 %{
5954   match(Set dst src);
5955   effect(KILL cr);
5956 
5957   ins_cost(50);
5958   format %{ "xorl    $dst, $dst\t# ptr" %}
5959   opcode(0x33); /* + rd */
5960   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5961   ins_pipe(ialu_reg);
5962 %}
5963 
5964 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5965 %{
5966   match(Set dst src);
5967   effect(KILL cr);
5968 
5969   ins_cost(60);
5970   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5971   ins_encode(load_immP31(dst, src));
5972   ins_pipe(ialu_reg);
5973 %}
5974 
5975 instruct loadConF(regF dst, immF con) %{
5976   match(Set dst con);
5977   ins_cost(125);
5978   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5979   ins_encode %{
5980     __ movflt($dst$$XMMRegister, $constantaddress($con));
5981   %}
5982   ins_pipe(pipe_slow);
5983 %}
5984 
5985 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5986   match(Set dst src);
5987   effect(KILL cr);
5988   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5989   ins_encode %{
5990     __ xorq($dst$$Register, $dst$$Register);
5991   %}
5992   ins_pipe(ialu_reg);
5993 %}
5994 
5995 instruct loadConN(rRegN dst, immN src) %{
5996   match(Set dst src);
5997 
5998   ins_cost(125);
5999   format %{ "movl    $dst, $src\t# compressed ptr" %}
6000   ins_encode %{
6001     address con = (address)$src$$constant;
6002     if (con == NULL) {
6003       ShouldNotReachHere();
6004     } else {
6005       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
6006     }
6007   %}
6008   ins_pipe(ialu_reg_fat); // XXX
6009 %}
6010 
6011 instruct loadConNKlass(rRegN dst, immNKlass src) %{
6012   match(Set dst src);
6013 
6014   ins_cost(125);
6015   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
6016   ins_encode %{
6017     address con = (address)$src$$constant;
6018     if (con == NULL) {
6019       ShouldNotReachHere();
6020     } else {
6021       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
6022     }
6023   %}
6024   ins_pipe(ialu_reg_fat); // XXX
6025 %}
6026 
6027 instruct loadConF0(regF dst, immF0 src)
6028 %{
6029   match(Set dst src);
6030   ins_cost(100);
6031 
6032   format %{ "xorps   $dst, $dst\t# float 0.0" %}
6033   ins_encode %{
6034     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
6035   %}
6036   ins_pipe(pipe_slow);
6037 %}
6038 
6039 // Use the same format since predicate() can not be used here.
6040 instruct loadConD(regD dst, immD con) %{
6041   match(Set dst con);
6042   ins_cost(125);
6043   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
6044   ins_encode %{
6045     __ movdbl($dst$$XMMRegister, $constantaddress($con));
6046   %}
6047   ins_pipe(pipe_slow);
6048 %}
6049 
6050 instruct loadConD0(regD dst, immD0 src)
6051 %{
6052   match(Set dst src);
6053   ins_cost(100);
6054 
6055   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
6056   ins_encode %{
6057     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
6058   %}
6059   ins_pipe(pipe_slow);
6060 %}
6061 
6062 instruct loadSSI(rRegI dst, stackSlotI src)
6063 %{
6064   match(Set dst src);
6065 
6066   ins_cost(125);
6067   format %{ "movl    $dst, $src\t# int stk" %}
6068   opcode(0x8B);
6069   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6070   ins_pipe(ialu_reg_mem);
6071 %}
6072 
6073 instruct loadSSL(rRegL dst, stackSlotL src)
6074 %{
6075   match(Set dst src);
6076 
6077   ins_cost(125);
6078   format %{ "movq    $dst, $src\t# long stk" %}
6079   opcode(0x8B);
6080   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6081   ins_pipe(ialu_reg_mem);
6082 %}
6083 
6084 instruct loadSSP(rRegP dst, stackSlotP src)
6085 %{
6086   match(Set dst src);
6087 
6088   ins_cost(125);
6089   format %{ "movq    $dst, $src\t# ptr stk" %}
6090   opcode(0x8B);
6091   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
6092   ins_pipe(ialu_reg_mem);
6093 %}
6094 
6095 instruct loadSSF(regF dst, stackSlotF src)
6096 %{
6097   match(Set dst src);
6098 
6099   ins_cost(125);
6100   format %{ "movss   $dst, $src\t# float stk" %}
6101   ins_encode %{
6102     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
6103   %}
6104   ins_pipe(pipe_slow); // XXX
6105 %}
6106 
6107 // Use the same format since predicate() can not be used here.
6108 instruct loadSSD(regD dst, stackSlotD src)
6109 %{
6110   match(Set dst src);
6111 
6112   ins_cost(125);
6113   format %{ "movsd   $dst, $src\t# double stk" %}
6114   ins_encode  %{
6115     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
6116   %}
6117   ins_pipe(pipe_slow); // XXX
6118 %}
6119 
6120 // Prefetch instructions for allocation.
6121 // Must be safe to execute with invalid address (cannot fault).
6122 
6123 instruct prefetchAlloc( memory mem ) %{
6124   predicate(AllocatePrefetchInstr==3);
6125   match(PrefetchAllocation mem);
6126   ins_cost(125);
6127 
6128   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
6129   ins_encode %{
6130     __ prefetchw($mem$$Address);
6131   %}
6132   ins_pipe(ialu_mem);
6133 %}
6134 
6135 instruct prefetchAllocNTA( memory mem ) %{
6136   predicate(AllocatePrefetchInstr==0);
6137   match(PrefetchAllocation mem);
6138   ins_cost(125);
6139 
6140   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
6141   ins_encode %{
6142     __ prefetchnta($mem$$Address);
6143   %}
6144   ins_pipe(ialu_mem);
6145 %}
6146 
6147 instruct prefetchAllocT0( memory mem ) %{
6148   predicate(AllocatePrefetchInstr==1);
6149   match(PrefetchAllocation mem);
6150   ins_cost(125);
6151 
6152   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
6153   ins_encode %{
6154     __ prefetcht0($mem$$Address);
6155   %}
6156   ins_pipe(ialu_mem);
6157 %}
6158 
6159 instruct prefetchAllocT2( memory mem ) %{
6160   predicate(AllocatePrefetchInstr==2);
6161   match(PrefetchAllocation mem);
6162   ins_cost(125);
6163 
6164   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
6165   ins_encode %{
6166     __ prefetcht2($mem$$Address);
6167   %}
6168   ins_pipe(ialu_mem);
6169 %}
6170 
6171 //----------Store Instructions-------------------------------------------------
6172 
6173 // Store Byte
6174 instruct storeB(memory mem, rRegI src)
6175 %{
6176   match(Set mem (StoreB mem src));
6177 
6178   ins_cost(125); // XXX
6179   format %{ "movb    $mem, $src\t# byte" %}
6180   opcode(0x88);
6181   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
6182   ins_pipe(ialu_mem_reg);
6183 %}
6184 
6185 // Store Char/Short
6186 instruct storeC(memory mem, rRegI src)
6187 %{
6188   match(Set mem (StoreC mem src));
6189 
6190   ins_cost(125); // XXX
6191   format %{ "movw    $mem, $src\t# char/short" %}
6192   opcode(0x89);
6193   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6194   ins_pipe(ialu_mem_reg);
6195 %}
6196 
6197 // Store Integer
6198 instruct storeI(memory mem, rRegI src)
6199 %{
6200   match(Set mem (StoreI mem src));
6201 
6202   ins_cost(125); // XXX
6203   format %{ "movl    $mem, $src\t# int" %}
6204   opcode(0x89);
6205   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
6206   ins_pipe(ialu_mem_reg);
6207 %}
6208 
6209 // Store Long
6210 instruct storeL(memory mem, rRegL src)
6211 %{
6212   match(Set mem (StoreL mem src));
6213 
6214   ins_cost(125); // XXX
6215   format %{ "movq    $mem, $src\t# long" %}
6216   opcode(0x89);
6217   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6218   ins_pipe(ialu_mem_reg); // XXX
6219 %}
6220 
6221 // Store Pointer
6222 instruct storeP(memory mem, any_RegP src)
6223 %{
6224   match(Set mem (StoreP mem src));
6225 
6226   ins_cost(125); // XXX
6227   format %{ "movq    $mem, $src\t# ptr" %}
6228   opcode(0x89);
6229   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
6230   ins_pipe(ialu_mem_reg);
6231 %}
6232 
6233 instruct storeImmP0(memory mem, immP0 zero)
6234 %{
6235   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6236   match(Set mem (StoreP mem zero));
6237 
6238   ins_cost(125); // XXX
6239   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
6240   ins_encode %{
6241     __ movq($mem$$Address, r12);
6242   %}
6243   ins_pipe(ialu_mem_reg);
6244 %}
6245 
6246 // Store NULL Pointer, mark word, or other simple pointer constant.
6247 instruct storeImmP(memory mem, immP31 src)
6248 %{
6249   match(Set mem (StoreP mem src));
6250 
6251   ins_cost(150); // XXX
6252   format %{ "movq    $mem, $src\t# ptr" %}
6253   opcode(0xC7); /* C7 /0 */
6254   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6255   ins_pipe(ialu_mem_imm);
6256 %}
6257 
6258 // Store Compressed Pointer
6259 instruct storeN(memory mem, rRegN src)
6260 %{
6261   match(Set mem (StoreN mem src));
6262 
6263   ins_cost(125); // XXX
6264   format %{ "movl    $mem, $src\t# compressed ptr" %}
6265   ins_encode %{
6266     __ movl($mem$$Address, $src$$Register);
6267   %}
6268   ins_pipe(ialu_mem_reg);
6269 %}
6270 
6271 instruct storeNKlass(memory mem, rRegN src)
6272 %{
6273   match(Set mem (StoreNKlass mem src));
6274 
6275   ins_cost(125); // XXX
6276   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6277   ins_encode %{
6278     __ movl($mem$$Address, $src$$Register);
6279   %}
6280   ins_pipe(ialu_mem_reg);
6281 %}
6282 
6283 instruct storeImmN0(memory mem, immN0 zero)
6284 %{
6285   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
6286   match(Set mem (StoreN mem zero));
6287 
6288   ins_cost(125); // XXX
6289   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6290   ins_encode %{
6291     __ movl($mem$$Address, r12);
6292   %}
6293   ins_pipe(ialu_mem_reg);
6294 %}
6295 
6296 instruct storeImmN(memory mem, immN src)
6297 %{
6298   match(Set mem (StoreN mem src));
6299 
6300   ins_cost(150); // XXX
6301   format %{ "movl    $mem, $src\t# compressed ptr" %}
6302   ins_encode %{
6303     address con = (address)$src$$constant;
6304     if (con == NULL) {
6305       __ movl($mem$$Address, (int32_t)0);
6306     } else {
6307       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6308     }
6309   %}
6310   ins_pipe(ialu_mem_imm);
6311 %}
6312 
6313 instruct storeImmNKlass(memory mem, immNKlass src)
6314 %{
6315   match(Set mem (StoreNKlass mem src));
6316 
6317   ins_cost(150); // XXX
6318   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6319   ins_encode %{
6320     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6321   %}
6322   ins_pipe(ialu_mem_imm);
6323 %}
6324 
6325 // Store Integer Immediate
6326 instruct storeImmI0(memory mem, immI0 zero)
6327 %{
6328   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6329   match(Set mem (StoreI mem zero));
6330 
6331   ins_cost(125); // XXX
6332   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6333   ins_encode %{
6334     __ movl($mem$$Address, r12);
6335   %}
6336   ins_pipe(ialu_mem_reg);
6337 %}
6338 
6339 instruct storeImmI(memory mem, immI src)
6340 %{
6341   match(Set mem (StoreI mem src));
6342 
6343   ins_cost(150);
6344   format %{ "movl    $mem, $src\t# int" %}
6345   opcode(0xC7); /* C7 /0 */
6346   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6347   ins_pipe(ialu_mem_imm);
6348 %}
6349 
6350 // Store Long Immediate
6351 instruct storeImmL0(memory mem, immL0 zero)
6352 %{
6353   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6354   match(Set mem (StoreL mem zero));
6355 
6356   ins_cost(125); // XXX
6357   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6358   ins_encode %{
6359     __ movq($mem$$Address, r12);
6360   %}
6361   ins_pipe(ialu_mem_reg);
6362 %}
6363 
6364 instruct storeImmL(memory mem, immL32 src)
6365 %{
6366   match(Set mem (StoreL mem src));
6367 
6368   ins_cost(150);
6369   format %{ "movq    $mem, $src\t# long" %}
6370   opcode(0xC7); /* C7 /0 */
6371   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6372   ins_pipe(ialu_mem_imm);
6373 %}
6374 
6375 // Store Short/Char Immediate
6376 instruct storeImmC0(memory mem, immI0 zero)
6377 %{
6378   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6379   match(Set mem (StoreC mem zero));
6380 
6381   ins_cost(125); // XXX
6382   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6383   ins_encode %{
6384     __ movw($mem$$Address, r12);
6385   %}
6386   ins_pipe(ialu_mem_reg);
6387 %}
6388 
6389 instruct storeImmI16(memory mem, immI16 src)
6390 %{
6391   predicate(UseStoreImmI16);
6392   match(Set mem (StoreC mem src));
6393 
6394   ins_cost(150);
6395   format %{ "movw    $mem, $src\t# short/char" %}
6396   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6397   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6398   ins_pipe(ialu_mem_imm);
6399 %}
6400 
6401 // Store Byte Immediate
6402 instruct storeImmB0(memory mem, immI0 zero)
6403 %{
6404   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6405   match(Set mem (StoreB mem zero));
6406 
6407   ins_cost(125); // XXX
6408   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6409   ins_encode %{
6410     __ movb($mem$$Address, r12);
6411   %}
6412   ins_pipe(ialu_mem_reg);
6413 %}
6414 
6415 instruct storeImmB(memory mem, immI8 src)
6416 %{
6417   match(Set mem (StoreB mem src));
6418 
6419   ins_cost(150); // XXX
6420   format %{ "movb    $mem, $src\t# byte" %}
6421   opcode(0xC6); /* C6 /0 */
6422   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6423   ins_pipe(ialu_mem_imm);
6424 %}
6425 
6426 // Store CMS card-mark Immediate
6427 instruct storeImmCM0_reg(memory mem, immI0 zero)
6428 %{
6429   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6430   match(Set mem (StoreCM mem zero));
6431 
6432   ins_cost(125); // XXX
6433   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6434   ins_encode %{
6435     __ movb($mem$$Address, r12);
6436   %}
6437   ins_pipe(ialu_mem_reg);
6438 %}
6439 
6440 instruct storeImmCM0(memory mem, immI0 src)
6441 %{
6442   match(Set mem (StoreCM mem src));
6443 
6444   ins_cost(150); // XXX
6445   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6446   opcode(0xC6); /* C6 /0 */
6447   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6448   ins_pipe(ialu_mem_imm);
6449 %}
6450 
6451 // Store Float
6452 instruct storeF(memory mem, regF src)
6453 %{
6454   match(Set mem (StoreF mem src));
6455 
6456   ins_cost(95); // XXX
6457   format %{ "movss   $mem, $src\t# float" %}
6458   ins_encode %{
6459     __ movflt($mem$$Address, $src$$XMMRegister);
6460   %}
6461   ins_pipe(pipe_slow); // XXX
6462 %}
6463 
6464 // Store immediate Float value (it is faster than store from XMM register)
6465 instruct storeF0(memory mem, immF0 zero)
6466 %{
6467   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6468   match(Set mem (StoreF mem zero));
6469 
6470   ins_cost(25); // XXX
6471   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6472   ins_encode %{
6473     __ movl($mem$$Address, r12);
6474   %}
6475   ins_pipe(ialu_mem_reg);
6476 %}
6477 
6478 instruct storeF_imm(memory mem, immF src)
6479 %{
6480   match(Set mem (StoreF mem src));
6481 
6482   ins_cost(50);
6483   format %{ "movl    $mem, $src\t# float" %}
6484   opcode(0xC7); /* C7 /0 */
6485   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6486   ins_pipe(ialu_mem_imm);
6487 %}
6488 
6489 // Store Double
6490 instruct storeD(memory mem, regD src)
6491 %{
6492   match(Set mem (StoreD mem src));
6493 
6494   ins_cost(95); // XXX
6495   format %{ "movsd   $mem, $src\t# double" %}
6496   ins_encode %{
6497     __ movdbl($mem$$Address, $src$$XMMRegister);
6498   %}
6499   ins_pipe(pipe_slow); // XXX
6500 %}
6501 
6502 // Store immediate double 0.0 (it is faster than store from XMM register)
6503 instruct storeD0_imm(memory mem, immD0 src)
6504 %{
6505   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6506   match(Set mem (StoreD mem src));
6507 
6508   ins_cost(50);
6509   format %{ "movq    $mem, $src\t# double 0." %}
6510   opcode(0xC7); /* C7 /0 */
6511   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6512   ins_pipe(ialu_mem_imm);
6513 %}
6514 
6515 instruct storeD0(memory mem, immD0 zero)
6516 %{
6517   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6518   match(Set mem (StoreD mem zero));
6519 
6520   ins_cost(25); // XXX
6521   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6522   ins_encode %{
6523     __ movq($mem$$Address, r12);
6524   %}
6525   ins_pipe(ialu_mem_reg);
6526 %}
6527 
6528 instruct storeSSI(stackSlotI dst, rRegI src)
6529 %{
6530   match(Set dst src);
6531 
6532   ins_cost(100);
6533   format %{ "movl    $dst, $src\t# int stk" %}
6534   opcode(0x89);
6535   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6536   ins_pipe( ialu_mem_reg );
6537 %}
6538 
6539 instruct storeSSL(stackSlotL dst, rRegL src)
6540 %{
6541   match(Set dst src);
6542 
6543   ins_cost(100);
6544   format %{ "movq    $dst, $src\t# long stk" %}
6545   opcode(0x89);
6546   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6547   ins_pipe(ialu_mem_reg);
6548 %}
6549 
6550 instruct storeSSP(stackSlotP dst, rRegP src)
6551 %{
6552   match(Set dst src);
6553 
6554   ins_cost(100);
6555   format %{ "movq    $dst, $src\t# ptr stk" %}
6556   opcode(0x89);
6557   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6558   ins_pipe(ialu_mem_reg);
6559 %}
6560 
6561 instruct storeSSF(stackSlotF dst, regF src)
6562 %{
6563   match(Set dst src);
6564 
6565   ins_cost(95); // XXX
6566   format %{ "movss   $dst, $src\t# float stk" %}
6567   ins_encode %{
6568     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6569   %}
6570   ins_pipe(pipe_slow); // XXX
6571 %}
6572 
6573 instruct storeSSD(stackSlotD dst, regD src)
6574 %{
6575   match(Set dst src);
6576 
6577   ins_cost(95); // XXX
6578   format %{ "movsd   $dst, $src\t# double stk" %}
6579   ins_encode %{
6580     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6581   %}
6582   ins_pipe(pipe_slow); // XXX
6583 %}
6584 
6585 //----------BSWAP Instructions-------------------------------------------------
6586 instruct bytes_reverse_int(rRegI dst) %{
6587   match(Set dst (ReverseBytesI dst));
6588 
6589   format %{ "bswapl  $dst" %}
6590   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6591   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6592   ins_pipe( ialu_reg );
6593 %}
6594 
6595 instruct bytes_reverse_long(rRegL dst) %{
6596   match(Set dst (ReverseBytesL dst));
6597 
6598   format %{ "bswapq  $dst" %}
6599   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6600   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6601   ins_pipe( ialu_reg);
6602 %}
6603 
6604 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6605   match(Set dst (ReverseBytesUS dst));
6606   effect(KILL cr);
6607 
6608   format %{ "bswapl  $dst\n\t"
6609             "shrl    $dst,16\n\t" %}
6610   ins_encode %{
6611     __ bswapl($dst$$Register);
6612     __ shrl($dst$$Register, 16);
6613   %}
6614   ins_pipe( ialu_reg );
6615 %}
6616 
6617 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6618   match(Set dst (ReverseBytesS dst));
6619   effect(KILL cr);
6620 
6621   format %{ "bswapl  $dst\n\t"
6622             "sar     $dst,16\n\t" %}
6623   ins_encode %{
6624     __ bswapl($dst$$Register);
6625     __ sarl($dst$$Register, 16);
6626   %}
6627   ins_pipe( ialu_reg );
6628 %}
6629 
6630 //---------- Zeros Count Instructions ------------------------------------------
6631 
6632 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6633   predicate(UseCountLeadingZerosInstruction);
6634   match(Set dst (CountLeadingZerosI src));
6635   effect(KILL cr);
6636 
6637   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6638   ins_encode %{
6639     __ lzcntl($dst$$Register, $src$$Register);
6640   %}
6641   ins_pipe(ialu_reg);
6642 %}
6643 
6644 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6645   predicate(!UseCountLeadingZerosInstruction);
6646   match(Set dst (CountLeadingZerosI src));
6647   effect(KILL cr);
6648 
6649   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6650             "jnz     skip\n\t"
6651             "movl    $dst, -1\n"
6652       "skip:\n\t"
6653             "negl    $dst\n\t"
6654             "addl    $dst, 31" %}
6655   ins_encode %{
6656     Register Rdst = $dst$$Register;
6657     Register Rsrc = $src$$Register;
6658     Label skip;
6659     __ bsrl(Rdst, Rsrc);
6660     __ jccb(Assembler::notZero, skip);
6661     __ movl(Rdst, -1);
6662     __ bind(skip);
6663     __ negl(Rdst);
6664     __ addl(Rdst, BitsPerInt - 1);
6665   %}
6666   ins_pipe(ialu_reg);
6667 %}
6668 
6669 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6670   predicate(UseCountLeadingZerosInstruction);
6671   match(Set dst (CountLeadingZerosL src));
6672   effect(KILL cr);
6673 
6674   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6675   ins_encode %{
6676     __ lzcntq($dst$$Register, $src$$Register);
6677   %}
6678   ins_pipe(ialu_reg);
6679 %}
6680 
6681 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6682   predicate(!UseCountLeadingZerosInstruction);
6683   match(Set dst (CountLeadingZerosL src));
6684   effect(KILL cr);
6685 
6686   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6687             "jnz     skip\n\t"
6688             "movl    $dst, -1\n"
6689       "skip:\n\t"
6690             "negl    $dst\n\t"
6691             "addl    $dst, 63" %}
6692   ins_encode %{
6693     Register Rdst = $dst$$Register;
6694     Register Rsrc = $src$$Register;
6695     Label skip;
6696     __ bsrq(Rdst, Rsrc);
6697     __ jccb(Assembler::notZero, skip);
6698     __ movl(Rdst, -1);
6699     __ bind(skip);
6700     __ negl(Rdst);
6701     __ addl(Rdst, BitsPerLong - 1);
6702   %}
6703   ins_pipe(ialu_reg);
6704 %}
6705 
6706 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6707   predicate(UseCountTrailingZerosInstruction);
6708   match(Set dst (CountTrailingZerosI src));
6709   effect(KILL cr);
6710 
6711   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6712   ins_encode %{
6713     __ tzcntl($dst$$Register, $src$$Register);
6714   %}
6715   ins_pipe(ialu_reg);
6716 %}
6717 
6718 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6719   predicate(!UseCountTrailingZerosInstruction);
6720   match(Set dst (CountTrailingZerosI src));
6721   effect(KILL cr);
6722 
6723   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6724             "jnz     done\n\t"
6725             "movl    $dst, 32\n"
6726       "done:" %}
6727   ins_encode %{
6728     Register Rdst = $dst$$Register;
6729     Label done;
6730     __ bsfl(Rdst, $src$$Register);
6731     __ jccb(Assembler::notZero, done);
6732     __ movl(Rdst, BitsPerInt);
6733     __ bind(done);
6734   %}
6735   ins_pipe(ialu_reg);
6736 %}
6737 
6738 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6739   predicate(UseCountTrailingZerosInstruction);
6740   match(Set dst (CountTrailingZerosL src));
6741   effect(KILL cr);
6742 
6743   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6744   ins_encode %{
6745     __ tzcntq($dst$$Register, $src$$Register);
6746   %}
6747   ins_pipe(ialu_reg);
6748 %}
6749 
6750 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6751   predicate(!UseCountTrailingZerosInstruction);
6752   match(Set dst (CountTrailingZerosL src));
6753   effect(KILL cr);
6754 
6755   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6756             "jnz     done\n\t"
6757             "movl    $dst, 64\n"
6758       "done:" %}
6759   ins_encode %{
6760     Register Rdst = $dst$$Register;
6761     Label done;
6762     __ bsfq(Rdst, $src$$Register);
6763     __ jccb(Assembler::notZero, done);
6764     __ movl(Rdst, BitsPerLong);
6765     __ bind(done);
6766   %}
6767   ins_pipe(ialu_reg);
6768 %}
6769 
6770 
6771 //---------- Population Count Instructions -------------------------------------
6772 
6773 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6774   predicate(UsePopCountInstruction);
6775   match(Set dst (PopCountI src));
6776   effect(KILL cr);
6777 
6778   format %{ "popcnt  $dst, $src" %}
6779   ins_encode %{
6780     __ popcntl($dst$$Register, $src$$Register);
6781   %}
6782   ins_pipe(ialu_reg);
6783 %}
6784 
6785 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6786   predicate(UsePopCountInstruction);
6787   match(Set dst (PopCountI (LoadI mem)));
6788   effect(KILL cr);
6789 
6790   format %{ "popcnt  $dst, $mem" %}
6791   ins_encode %{
6792     __ popcntl($dst$$Register, $mem$$Address);
6793   %}
6794   ins_pipe(ialu_reg);
6795 %}
6796 
6797 // Note: Long.bitCount(long) returns an int.
6798 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6799   predicate(UsePopCountInstruction);
6800   match(Set dst (PopCountL src));
6801   effect(KILL cr);
6802 
6803   format %{ "popcnt  $dst, $src" %}
6804   ins_encode %{
6805     __ popcntq($dst$$Register, $src$$Register);
6806   %}
6807   ins_pipe(ialu_reg);
6808 %}
6809 
6810 // Note: Long.bitCount(long) returns an int.
6811 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6812   predicate(UsePopCountInstruction);
6813   match(Set dst (PopCountL (LoadL mem)));
6814   effect(KILL cr);
6815 
6816   format %{ "popcnt  $dst, $mem" %}
6817   ins_encode %{
6818     __ popcntq($dst$$Register, $mem$$Address);
6819   %}
6820   ins_pipe(ialu_reg);
6821 %}
6822 
6823 
6824 //----------MemBar Instructions-----------------------------------------------
6825 // Memory barrier flavors
6826 
6827 instruct membar_acquire()
6828 %{
6829   match(MemBarAcquire);
6830   match(LoadFence);
6831   ins_cost(0);
6832 
6833   size(0);
6834   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6835   ins_encode();
6836   ins_pipe(empty);
6837 %}
6838 
6839 instruct membar_acquire_lock()
6840 %{
6841   match(MemBarAcquireLock);
6842   ins_cost(0);
6843 
6844   size(0);
6845   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6846   ins_encode();
6847   ins_pipe(empty);
6848 %}
6849 
6850 instruct membar_release()
6851 %{
6852   match(MemBarRelease);
6853   match(StoreFence);
6854   ins_cost(0);
6855 
6856   size(0);
6857   format %{ "MEMBAR-release ! (empty encoding)" %}
6858   ins_encode();
6859   ins_pipe(empty);
6860 %}
6861 
6862 instruct membar_release_lock()
6863 %{
6864   match(MemBarReleaseLock);
6865   ins_cost(0);
6866 
6867   size(0);
6868   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6869   ins_encode();
6870   ins_pipe(empty);
6871 %}
6872 
6873 instruct membar_volatile(rFlagsReg cr) %{
6874   match(MemBarVolatile);
6875   effect(KILL cr);
6876   ins_cost(400);
6877 
6878   format %{
6879     $$template
6880     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6881   %}
6882   ins_encode %{
6883     __ membar(Assembler::StoreLoad);
6884   %}
6885   ins_pipe(pipe_slow);
6886 %}
6887 
6888 instruct unnecessary_membar_volatile()
6889 %{
6890   match(MemBarVolatile);
6891   predicate(Matcher::post_store_load_barrier(n));
6892   ins_cost(0);
6893 
6894   size(0);
6895   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6896   ins_encode();
6897   ins_pipe(empty);
6898 %}
6899 
6900 instruct membar_storestore() %{
6901   match(MemBarStoreStore);
6902   ins_cost(0);
6903 
6904   size(0);
6905   format %{ "MEMBAR-storestore (empty encoding)" %}
6906   ins_encode( );
6907   ins_pipe(empty);
6908 %}
6909 
6910 //----------Move Instructions--------------------------------------------------
6911 
6912 instruct castX2P(rRegP dst, rRegL src)
6913 %{
6914   match(Set dst (CastX2P src));
6915 
6916   format %{ "movq    $dst, $src\t# long->ptr" %}
6917   ins_encode %{
6918     if ($dst$$reg != $src$$reg) {
6919       __ movptr($dst$$Register, $src$$Register);
6920     }
6921   %}
6922   ins_pipe(ialu_reg_reg); // XXX
6923 %}
6924 
6925 instruct castP2X(rRegL dst, rRegP src)
6926 %{
6927   match(Set dst (CastP2X src));
6928 
6929   format %{ "movq    $dst, $src\t# ptr -> long" %}
6930   ins_encode %{
6931     if ($dst$$reg != $src$$reg) {
6932       __ movptr($dst$$Register, $src$$Register);
6933     }
6934   %}
6935   ins_pipe(ialu_reg_reg); // XXX
6936 %}
6937 
6938 // Convert oop into int for vectors alignment masking
6939 instruct convP2I(rRegI dst, rRegP src)
6940 %{
6941   match(Set dst (ConvL2I (CastP2X src)));
6942 
6943   format %{ "movl    $dst, $src\t# ptr -> int" %}
6944   ins_encode %{
6945     __ movl($dst$$Register, $src$$Register);
6946   %}
6947   ins_pipe(ialu_reg_reg); // XXX
6948 %}
6949 
6950 // Convert compressed oop into int for vectors alignment masking
6951 // in case of 32bit oops (heap < 4Gb).
6952 instruct convN2I(rRegI dst, rRegN src)
6953 %{
6954   predicate(Universe::narrow_oop_shift() == 0);
6955   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6956 
6957   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6958   ins_encode %{
6959     __ movl($dst$$Register, $src$$Register);
6960   %}
6961   ins_pipe(ialu_reg_reg); // XXX
6962 %}
6963 
6964 // Convert oop pointer into compressed form
6965 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6966   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6967   match(Set dst (EncodeP src));
6968   effect(KILL cr);
6969   format %{ "encode_heap_oop $dst,$src" %}
6970   ins_encode %{
6971     Register s = $src$$Register;
6972     Register d = $dst$$Register;
6973     if (s != d) {
6974       __ movq(d, s);
6975     }
6976     __ encode_heap_oop(d);
6977   %}
6978   ins_pipe(ialu_reg_long);
6979 %}
6980 
6981 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6982   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6983   match(Set dst (EncodeP src));
6984   effect(KILL cr);
6985   format %{ "encode_heap_oop_not_null $dst,$src" %}
6986   ins_encode %{
6987     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6988   %}
6989   ins_pipe(ialu_reg_long);
6990 %}
6991 
6992 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6993   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6994             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6995   match(Set dst (DecodeN src));
6996   effect(KILL cr);
6997   format %{ "decode_heap_oop $dst,$src" %}
6998   ins_encode %{
6999     Register s = $src$$Register;
7000     Register d = $dst$$Register;
7001     if (s != d) {
7002       __ movq(d, s);
7003     }
7004     __ decode_heap_oop(d);
7005   %}
7006   ins_pipe(ialu_reg_long);
7007 %}
7008 
7009 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7010   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7011             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7012   match(Set dst (DecodeN src));
7013   effect(KILL cr);
7014   format %{ "decode_heap_oop_not_null $dst,$src" %}
7015   ins_encode %{
7016     Register s = $src$$Register;
7017     Register d = $dst$$Register;
7018     if (s != d) {
7019       __ decode_heap_oop_not_null(d, s);
7020     } else {
7021       __ decode_heap_oop_not_null(d);
7022     }
7023   %}
7024   ins_pipe(ialu_reg_long);
7025 %}
7026 
7027 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
7028   match(Set dst (EncodePKlass src));
7029   effect(KILL cr);
7030   format %{ "encode_klass_not_null $dst,$src" %}
7031   ins_encode %{
7032     __ encode_klass_not_null($dst$$Register, $src$$Register);
7033   %}
7034   ins_pipe(ialu_reg_long);
7035 %}
7036 
7037 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
7038   match(Set dst (DecodeNKlass src));
7039   effect(KILL cr);
7040   format %{ "decode_klass_not_null $dst,$src" %}
7041   ins_encode %{
7042     Register s = $src$$Register;
7043     Register d = $dst$$Register;
7044     if (s != d) {
7045       __ decode_klass_not_null(d, s);
7046     } else {
7047       __ decode_klass_not_null(d);
7048     }
7049   %}
7050   ins_pipe(ialu_reg_long);
7051 %}
7052 
7053 
7054 //----------Conditional Move---------------------------------------------------
7055 // Jump
7056 // dummy instruction for generating temp registers
7057 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
7058   match(Jump (LShiftL switch_val shift));
7059   ins_cost(350);
7060   predicate(false);
7061   effect(TEMP dest);
7062 
7063   format %{ "leaq    $dest, [$constantaddress]\n\t"
7064             "jmp     [$dest + $switch_val << $shift]\n\t" %}
7065   ins_encode %{
7066     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7067     // to do that and the compiler is using that register as one it can allocate.
7068     // So we build it all by hand.
7069     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
7070     // ArrayAddress dispatch(table, index);
7071     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
7072     __ lea($dest$$Register, $constantaddress);
7073     __ jmp(dispatch);
7074   %}
7075   ins_pipe(pipe_jmp);
7076 %}
7077 
7078 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
7079   match(Jump (AddL (LShiftL switch_val shift) offset));
7080   ins_cost(350);
7081   effect(TEMP dest);
7082 
7083   format %{ "leaq    $dest, [$constantaddress]\n\t"
7084             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
7085   ins_encode %{
7086     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7087     // to do that and the compiler is using that register as one it can allocate.
7088     // So we build it all by hand.
7089     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7090     // ArrayAddress dispatch(table, index);
7091     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
7092     __ lea($dest$$Register, $constantaddress);
7093     __ jmp(dispatch);
7094   %}
7095   ins_pipe(pipe_jmp);
7096 %}
7097 
7098 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
7099   match(Jump switch_val);
7100   ins_cost(350);
7101   effect(TEMP dest);
7102 
7103   format %{ "leaq    $dest, [$constantaddress]\n\t"
7104             "jmp     [$dest + $switch_val]\n\t" %}
7105   ins_encode %{
7106     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
7107     // to do that and the compiler is using that register as one it can allocate.
7108     // So we build it all by hand.
7109     // Address index(noreg, switch_reg, Address::times_1);
7110     // ArrayAddress dispatch(table, index);
7111     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
7112     __ lea($dest$$Register, $constantaddress);
7113     __ jmp(dispatch);
7114   %}
7115   ins_pipe(pipe_jmp);
7116 %}
7117 
7118 // Conditional move
7119 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
7120 %{
7121   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7122 
7123   ins_cost(200); // XXX
7124   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7125   opcode(0x0F, 0x40);
7126   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7127   ins_pipe(pipe_cmov_reg);
7128 %}
7129 
7130 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
7131   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7132 
7133   ins_cost(200); // XXX
7134   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7135   opcode(0x0F, 0x40);
7136   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7137   ins_pipe(pipe_cmov_reg);
7138 %}
7139 
7140 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
7141   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
7142   ins_cost(200);
7143   expand %{
7144     cmovI_regU(cop, cr, dst, src);
7145   %}
7146 %}
7147 
7148 // Conditional move
7149 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
7150   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7151 
7152   ins_cost(250); // XXX
7153   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
7154   opcode(0x0F, 0x40);
7155   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7156   ins_pipe(pipe_cmov_mem);
7157 %}
7158 
7159 // Conditional move
7160 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
7161 %{
7162   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7163 
7164   ins_cost(250); // XXX
7165   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
7166   opcode(0x0F, 0x40);
7167   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
7168   ins_pipe(pipe_cmov_mem);
7169 %}
7170 
7171 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
7172   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
7173   ins_cost(250);
7174   expand %{
7175     cmovI_memU(cop, cr, dst, src);
7176   %}
7177 %}
7178 
7179 // Conditional move
7180 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
7181 %{
7182   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7183 
7184   ins_cost(200); // XXX
7185   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
7186   opcode(0x0F, 0x40);
7187   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7188   ins_pipe(pipe_cmov_reg);
7189 %}
7190 
7191 // Conditional move
7192 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
7193 %{
7194   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7195 
7196   ins_cost(200); // XXX
7197   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
7198   opcode(0x0F, 0x40);
7199   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
7200   ins_pipe(pipe_cmov_reg);
7201 %}
7202 
7203 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
7204   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
7205   ins_cost(200);
7206   expand %{
7207     cmovN_regU(cop, cr, dst, src);
7208   %}
7209 %}
7210 
7211 // Conditional move
7212 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
7213 %{
7214   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7215 
7216   ins_cost(200); // XXX
7217   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
7218   opcode(0x0F, 0x40);
7219   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7220   ins_pipe(pipe_cmov_reg);  // XXX
7221 %}
7222 
7223 // Conditional move
7224 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
7225 %{
7226   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7227 
7228   ins_cost(200); // XXX
7229   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
7230   opcode(0x0F, 0x40);
7231   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7232   ins_pipe(pipe_cmov_reg); // XXX
7233 %}
7234 
7235 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
7236   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
7237   ins_cost(200);
7238   expand %{
7239     cmovP_regU(cop, cr, dst, src);
7240   %}
7241 %}
7242 
7243 // DISABLED: Requires the ADLC to emit a bottom_type call that
7244 // correctly meets the two pointer arguments; one is an incoming
7245 // register but the other is a memory operand.  ALSO appears to
7246 // be buggy with implicit null checks.
7247 //
7248 //// Conditional move
7249 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
7250 //%{
7251 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7252 //  ins_cost(250);
7253 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7254 //  opcode(0x0F,0x40);
7255 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7256 //  ins_pipe( pipe_cmov_mem );
7257 //%}
7258 //
7259 //// Conditional move
7260 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
7261 //%{
7262 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
7263 //  ins_cost(250);
7264 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
7265 //  opcode(0x0F,0x40);
7266 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
7267 //  ins_pipe( pipe_cmov_mem );
7268 //%}
7269 
7270 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7271 %{
7272   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7273 
7274   ins_cost(200); // XXX
7275   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7276   opcode(0x0F, 0x40);
7277   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7278   ins_pipe(pipe_cmov_reg);  // XXX
7279 %}
7280 
7281 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7282 %{
7283   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7284 
7285   ins_cost(200); // XXX
7286   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7287   opcode(0x0F, 0x40);
7288   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7289   ins_pipe(pipe_cmov_mem);  // XXX
7290 %}
7291 
7292 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7293 %{
7294   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7295 
7296   ins_cost(200); // XXX
7297   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7298   opcode(0x0F, 0x40);
7299   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7300   ins_pipe(pipe_cmov_reg); // XXX
7301 %}
7302 
7303 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7304   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7305   ins_cost(200);
7306   expand %{
7307     cmovL_regU(cop, cr, dst, src);
7308   %}
7309 %}
7310 
7311 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7312 %{
7313   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7314 
7315   ins_cost(200); // XXX
7316   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7317   opcode(0x0F, 0x40);
7318   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7319   ins_pipe(pipe_cmov_mem); // XXX
7320 %}
7321 
7322 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7323   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7324   ins_cost(200);
7325   expand %{
7326     cmovL_memU(cop, cr, dst, src);
7327   %}
7328 %}
7329 
7330 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7331 %{
7332   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7333 
7334   ins_cost(200); // XXX
7335   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7336             "movss     $dst, $src\n"
7337     "skip:" %}
7338   ins_encode %{
7339     Label Lskip;
7340     // Invert sense of branch from sense of CMOV
7341     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7342     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7343     __ bind(Lskip);
7344   %}
7345   ins_pipe(pipe_slow);
7346 %}
7347 
7348 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7349 // %{
7350 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7351 
7352 //   ins_cost(200); // XXX
7353 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7354 //             "movss     $dst, $src\n"
7355 //     "skip:" %}
7356 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7357 //   ins_pipe(pipe_slow);
7358 // %}
7359 
7360 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7361 %{
7362   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7363 
7364   ins_cost(200); // XXX
7365   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7366             "movss     $dst, $src\n"
7367     "skip:" %}
7368   ins_encode %{
7369     Label Lskip;
7370     // Invert sense of branch from sense of CMOV
7371     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7372     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7373     __ bind(Lskip);
7374   %}
7375   ins_pipe(pipe_slow);
7376 %}
7377 
7378 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7379   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7380   ins_cost(200);
7381   expand %{
7382     cmovF_regU(cop, cr, dst, src);
7383   %}
7384 %}
7385 
7386 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7387 %{
7388   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7389 
7390   ins_cost(200); // XXX
7391   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7392             "movsd     $dst, $src\n"
7393     "skip:" %}
7394   ins_encode %{
7395     Label Lskip;
7396     // Invert sense of branch from sense of CMOV
7397     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7398     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7399     __ bind(Lskip);
7400   %}
7401   ins_pipe(pipe_slow);
7402 %}
7403 
7404 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7405 %{
7406   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7407 
7408   ins_cost(200); // XXX
7409   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7410             "movsd     $dst, $src\n"
7411     "skip:" %}
7412   ins_encode %{
7413     Label Lskip;
7414     // Invert sense of branch from sense of CMOV
7415     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7416     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7417     __ bind(Lskip);
7418   %}
7419   ins_pipe(pipe_slow);
7420 %}
7421 
7422 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7423   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7424   ins_cost(200);
7425   expand %{
7426     cmovD_regU(cop, cr, dst, src);
7427   %}
7428 %}
7429 
7430 //----------Arithmetic Instructions--------------------------------------------
7431 //----------Addition Instructions----------------------------------------------
7432 
7433 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7434 %{
7435   match(Set dst (AddI dst src));
7436   effect(KILL cr);
7437 
7438   format %{ "addl    $dst, $src\t# int" %}
7439   opcode(0x03);
7440   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7441   ins_pipe(ialu_reg_reg);
7442 %}
7443 
7444 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7445 %{
7446   match(Set dst (AddI dst src));
7447   effect(KILL cr);
7448 
7449   format %{ "addl    $dst, $src\t# int" %}
7450   opcode(0x81, 0x00); /* /0 id */
7451   ins_encode(OpcSErm(dst, src), Con8or32(src));
7452   ins_pipe( ialu_reg );
7453 %}
7454 
7455 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7456 %{
7457   match(Set dst (AddI dst (LoadI src)));
7458   effect(KILL cr);
7459 
7460   ins_cost(125); // XXX
7461   format %{ "addl    $dst, $src\t# int" %}
7462   opcode(0x03);
7463   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7464   ins_pipe(ialu_reg_mem);
7465 %}
7466 
7467 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7468 %{
7469   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7470   effect(KILL cr);
7471 
7472   ins_cost(150); // XXX
7473   format %{ "addl    $dst, $src\t# int" %}
7474   opcode(0x01); /* Opcode 01 /r */
7475   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7476   ins_pipe(ialu_mem_reg);
7477 %}
7478 
7479 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7480 %{
7481   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7482   effect(KILL cr);
7483 
7484   ins_cost(125); // XXX
7485   format %{ "addl    $dst, $src\t# int" %}
7486   opcode(0x81); /* Opcode 81 /0 id */
7487   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7488   ins_pipe(ialu_mem_imm);
7489 %}
7490 
7491 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7492 %{
7493   predicate(UseIncDec);
7494   match(Set dst (AddI dst src));
7495   effect(KILL cr);
7496 
7497   format %{ "incl    $dst\t# int" %}
7498   opcode(0xFF, 0x00); // FF /0
7499   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7500   ins_pipe(ialu_reg);
7501 %}
7502 
7503 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7504 %{
7505   predicate(UseIncDec);
7506   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7507   effect(KILL cr);
7508 
7509   ins_cost(125); // XXX
7510   format %{ "incl    $dst\t# int" %}
7511   opcode(0xFF); /* Opcode FF /0 */
7512   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7513   ins_pipe(ialu_mem_imm);
7514 %}
7515 
7516 // XXX why does that use AddI
7517 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7518 %{
7519   predicate(UseIncDec);
7520   match(Set dst (AddI dst src));
7521   effect(KILL cr);
7522 
7523   format %{ "decl    $dst\t# int" %}
7524   opcode(0xFF, 0x01); // FF /1
7525   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7526   ins_pipe(ialu_reg);
7527 %}
7528 
7529 // XXX why does that use AddI
7530 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7531 %{
7532   predicate(UseIncDec);
7533   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7534   effect(KILL cr);
7535 
7536   ins_cost(125); // XXX
7537   format %{ "decl    $dst\t# int" %}
7538   opcode(0xFF); /* Opcode FF /1 */
7539   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7540   ins_pipe(ialu_mem_imm);
7541 %}
7542 
7543 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7544 %{
7545   match(Set dst (AddI src0 src1));
7546 
7547   ins_cost(110);
7548   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7549   opcode(0x8D); /* 0x8D /r */
7550   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7551   ins_pipe(ialu_reg_reg);
7552 %}
7553 
7554 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7555 %{
7556   match(Set dst (AddL dst src));
7557   effect(KILL cr);
7558 
7559   format %{ "addq    $dst, $src\t# long" %}
7560   opcode(0x03);
7561   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7562   ins_pipe(ialu_reg_reg);
7563 %}
7564 
7565 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7566 %{
7567   match(Set dst (AddL dst src));
7568   effect(KILL cr);
7569 
7570   format %{ "addq    $dst, $src\t# long" %}
7571   opcode(0x81, 0x00); /* /0 id */
7572   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7573   ins_pipe( ialu_reg );
7574 %}
7575 
7576 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7577 %{
7578   match(Set dst (AddL dst (LoadL src)));
7579   effect(KILL cr);
7580 
7581   ins_cost(125); // XXX
7582   format %{ "addq    $dst, $src\t# long" %}
7583   opcode(0x03);
7584   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7585   ins_pipe(ialu_reg_mem);
7586 %}
7587 
7588 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7589 %{
7590   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7591   effect(KILL cr);
7592 
7593   ins_cost(150); // XXX
7594   format %{ "addq    $dst, $src\t# long" %}
7595   opcode(0x01); /* Opcode 01 /r */
7596   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7597   ins_pipe(ialu_mem_reg);
7598 %}
7599 
7600 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7601 %{
7602   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7603   effect(KILL cr);
7604 
7605   ins_cost(125); // XXX
7606   format %{ "addq    $dst, $src\t# long" %}
7607   opcode(0x81); /* Opcode 81 /0 id */
7608   ins_encode(REX_mem_wide(dst),
7609              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7610   ins_pipe(ialu_mem_imm);
7611 %}
7612 
7613 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7614 %{
7615   predicate(UseIncDec);
7616   match(Set dst (AddL dst src));
7617   effect(KILL cr);
7618 
7619   format %{ "incq    $dst\t# long" %}
7620   opcode(0xFF, 0x00); // FF /0
7621   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7622   ins_pipe(ialu_reg);
7623 %}
7624 
7625 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7626 %{
7627   predicate(UseIncDec);
7628   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7629   effect(KILL cr);
7630 
7631   ins_cost(125); // XXX
7632   format %{ "incq    $dst\t# long" %}
7633   opcode(0xFF); /* Opcode FF /0 */
7634   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7635   ins_pipe(ialu_mem_imm);
7636 %}
7637 
7638 // XXX why does that use AddL
7639 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7640 %{
7641   predicate(UseIncDec);
7642   match(Set dst (AddL dst src));
7643   effect(KILL cr);
7644 
7645   format %{ "decq    $dst\t# long" %}
7646   opcode(0xFF, 0x01); // FF /1
7647   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7648   ins_pipe(ialu_reg);
7649 %}
7650 
7651 // XXX why does that use AddL
7652 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7653 %{
7654   predicate(UseIncDec);
7655   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7656   effect(KILL cr);
7657 
7658   ins_cost(125); // XXX
7659   format %{ "decq    $dst\t# long" %}
7660   opcode(0xFF); /* Opcode FF /1 */
7661   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7662   ins_pipe(ialu_mem_imm);
7663 %}
7664 
7665 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7666 %{
7667   match(Set dst (AddL src0 src1));
7668 
7669   ins_cost(110);
7670   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7671   opcode(0x8D); /* 0x8D /r */
7672   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7673   ins_pipe(ialu_reg_reg);
7674 %}
7675 
7676 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7677 %{
7678   match(Set dst (AddP dst src));
7679   effect(KILL cr);
7680 
7681   format %{ "addq    $dst, $src\t# ptr" %}
7682   opcode(0x03);
7683   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7684   ins_pipe(ialu_reg_reg);
7685 %}
7686 
7687 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7688 %{
7689   match(Set dst (AddP dst src));
7690   effect(KILL cr);
7691 
7692   format %{ "addq    $dst, $src\t# ptr" %}
7693   opcode(0x81, 0x00); /* /0 id */
7694   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7695   ins_pipe( ialu_reg );
7696 %}
7697 
7698 // XXX addP mem ops ????
7699 
7700 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7701 %{
7702   match(Set dst (AddP src0 src1));
7703 
7704   ins_cost(110);
7705   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7706   opcode(0x8D); /* 0x8D /r */
7707   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7708   ins_pipe(ialu_reg_reg);
7709 %}
7710 
7711 instruct checkCastPP(rRegP dst)
7712 %{
7713   match(Set dst (CheckCastPP dst));
7714 
7715   size(0);
7716   format %{ "# checkcastPP of $dst" %}
7717   ins_encode(/* empty encoding */);
7718   ins_pipe(empty);
7719 %}
7720 
7721 instruct castPP(rRegP dst)
7722 %{
7723   match(Set dst (CastPP dst));
7724 
7725   size(0);
7726   format %{ "# castPP of $dst" %}
7727   ins_encode(/* empty encoding */);
7728   ins_pipe(empty);
7729 %}
7730 
7731 instruct castII(rRegI dst)
7732 %{
7733   match(Set dst (CastII dst));
7734 
7735   size(0);
7736   format %{ "# castII of $dst" %}
7737   ins_encode(/* empty encoding */);
7738   ins_cost(0);
7739   ins_pipe(empty);
7740 %}
7741 
7742 // LoadP-locked same as a regular LoadP when used with compare-swap
7743 instruct loadPLocked(rRegP dst, memory mem)
7744 %{
7745   match(Set dst (LoadPLocked mem));
7746 
7747   ins_cost(125); // XXX
7748   format %{ "movq    $dst, $mem\t# ptr locked" %}
7749   opcode(0x8B);
7750   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7751   ins_pipe(ialu_reg_mem); // XXX
7752 %}
7753 
7754 // Conditional-store of the updated heap-top.
7755 // Used during allocation of the shared heap.
7756 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7757 
7758 instruct storePConditional(memory heap_top_ptr,
7759                            rax_RegP oldval, rRegP newval,
7760                            rFlagsReg cr)
7761 %{
7762   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7763 
7764   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7765             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7766   opcode(0x0F, 0xB1);
7767   ins_encode(lock_prefix,
7768              REX_reg_mem_wide(newval, heap_top_ptr),
7769              OpcP, OpcS,
7770              reg_mem(newval, heap_top_ptr));
7771   ins_pipe(pipe_cmpxchg);
7772 %}
7773 
7774 // Conditional-store of an int value.
7775 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7776 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7777 %{
7778   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7779   effect(KILL oldval);
7780 
7781   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7782   opcode(0x0F, 0xB1);
7783   ins_encode(lock_prefix,
7784              REX_reg_mem(newval, mem),
7785              OpcP, OpcS,
7786              reg_mem(newval, mem));
7787   ins_pipe(pipe_cmpxchg);
7788 %}
7789 
7790 // Conditional-store of a long value.
7791 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7792 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7793 %{
7794   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7795   effect(KILL oldval);
7796 
7797   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7798   opcode(0x0F, 0xB1);
7799   ins_encode(lock_prefix,
7800              REX_reg_mem_wide(newval, mem),
7801              OpcP, OpcS,
7802              reg_mem(newval, mem));
7803   ins_pipe(pipe_cmpxchg);
7804 %}
7805 
7806 
7807 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7808 instruct compareAndSwapP(rRegI res,
7809                          memory mem_ptr,
7810                          rax_RegP oldval, rRegP newval,
7811                          rFlagsReg cr)
7812 %{
7813   predicate(VM_Version::supports_cx8());
7814   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7815   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7816   effect(KILL cr, KILL oldval);
7817 
7818   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7819             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7820             "sete    $res\n\t"
7821             "movzbl  $res, $res" %}
7822   opcode(0x0F, 0xB1);
7823   ins_encode(lock_prefix,
7824              REX_reg_mem_wide(newval, mem_ptr),
7825              OpcP, OpcS,
7826              reg_mem(newval, mem_ptr),
7827              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7828              REX_reg_breg(res, res), // movzbl
7829              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7830   ins_pipe( pipe_cmpxchg );
7831 %}
7832 
7833 instruct compareAndSwapL(rRegI res,
7834                          memory mem_ptr,
7835                          rax_RegL oldval, rRegL newval,
7836                          rFlagsReg cr)
7837 %{
7838   predicate(VM_Version::supports_cx8());
7839   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7840   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7841   effect(KILL cr, KILL oldval);
7842 
7843   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7844             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7845             "sete    $res\n\t"
7846             "movzbl  $res, $res" %}
7847   opcode(0x0F, 0xB1);
7848   ins_encode(lock_prefix,
7849              REX_reg_mem_wide(newval, mem_ptr),
7850              OpcP, OpcS,
7851              reg_mem(newval, mem_ptr),
7852              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7853              REX_reg_breg(res, res), // movzbl
7854              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7855   ins_pipe( pipe_cmpxchg );
7856 %}
7857 
7858 instruct compareAndSwapI(rRegI res,
7859                          memory mem_ptr,
7860                          rax_RegI oldval, rRegI newval,
7861                          rFlagsReg cr)
7862 %{
7863   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7864   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7865   effect(KILL cr, KILL oldval);
7866 
7867   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7868             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7869             "sete    $res\n\t"
7870             "movzbl  $res, $res" %}
7871   opcode(0x0F, 0xB1);
7872   ins_encode(lock_prefix,
7873              REX_reg_mem(newval, mem_ptr),
7874              OpcP, OpcS,
7875              reg_mem(newval, mem_ptr),
7876              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7877              REX_reg_breg(res, res), // movzbl
7878              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7879   ins_pipe( pipe_cmpxchg );
7880 %}
7881 
7882 instruct compareAndSwapB(rRegI res,
7883                          memory mem_ptr,
7884                          rax_RegI oldval, rRegI newval,
7885                          rFlagsReg cr)
7886 %{
7887   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7888   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7889   effect(KILL cr, KILL oldval);
7890 
7891   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7892             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7893             "sete    $res\n\t"
7894             "movzbl  $res, $res" %}
7895   opcode(0x0F, 0xB0);
7896   ins_encode(lock_prefix,
7897              REX_breg_mem(newval, mem_ptr),
7898              OpcP, OpcS,
7899              reg_mem(newval, mem_ptr),
7900              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7901              REX_reg_breg(res, res), // movzbl
7902              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7903   ins_pipe( pipe_cmpxchg );
7904 %}
7905 
7906 instruct compareAndSwapS(rRegI res,
7907                          memory mem_ptr,
7908                          rax_RegI oldval, rRegI newval,
7909                          rFlagsReg cr)
7910 %{
7911   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7912   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7913   effect(KILL cr, KILL oldval);
7914 
7915   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7916             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7917             "sete    $res\n\t"
7918             "movzbl  $res, $res" %}
7919   opcode(0x0F, 0xB1);
7920   ins_encode(lock_prefix,
7921              SizePrefix,
7922              REX_reg_mem(newval, mem_ptr),
7923              OpcP, OpcS,
7924              reg_mem(newval, mem_ptr),
7925              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7926              REX_reg_breg(res, res), // movzbl
7927              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7928   ins_pipe( pipe_cmpxchg );
7929 %}
7930 
7931 instruct compareAndSwapN(rRegI res,
7932                           memory mem_ptr,
7933                           rax_RegN oldval, rRegN newval,
7934                           rFlagsReg cr) %{
7935   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7936   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7937   effect(KILL cr, KILL oldval);
7938 
7939   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7940             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7941             "sete    $res\n\t"
7942             "movzbl  $res, $res" %}
7943   opcode(0x0F, 0xB1);
7944   ins_encode(lock_prefix,
7945              REX_reg_mem(newval, mem_ptr),
7946              OpcP, OpcS,
7947              reg_mem(newval, mem_ptr),
7948              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7949              REX_reg_breg(res, res), // movzbl
7950              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7951   ins_pipe( pipe_cmpxchg );
7952 %}
7953 
7954 instruct compareAndExchangeB(
7955                          memory mem_ptr,
7956                          rax_RegI oldval, rRegI newval,
7957                          rFlagsReg cr)
7958 %{
7959   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7960   effect(KILL cr);
7961 
7962   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7963             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7964   opcode(0x0F, 0xB0);
7965   ins_encode(lock_prefix,
7966              REX_breg_mem(newval, mem_ptr),
7967              OpcP, OpcS,
7968              reg_mem(newval, mem_ptr) // lock cmpxchg
7969              );
7970   ins_pipe( pipe_cmpxchg );
7971 %}
7972 
7973 instruct compareAndExchangeS(
7974                          memory mem_ptr,
7975                          rax_RegI oldval, rRegI newval,
7976                          rFlagsReg cr)
7977 %{
7978   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7979   effect(KILL cr);
7980 
7981   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7982             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7983   opcode(0x0F, 0xB1);
7984   ins_encode(lock_prefix,
7985              SizePrefix,
7986              REX_reg_mem(newval, mem_ptr),
7987              OpcP, OpcS,
7988              reg_mem(newval, mem_ptr) // lock cmpxchg
7989              );
7990   ins_pipe( pipe_cmpxchg );
7991 %}
7992 
7993 instruct compareAndExchangeI(
7994                          memory mem_ptr,
7995                          rax_RegI oldval, rRegI newval,
7996                          rFlagsReg cr)
7997 %{
7998   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7999   effect(KILL cr);
8000 
8001   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8002             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8003   opcode(0x0F, 0xB1);
8004   ins_encode(lock_prefix,
8005              REX_reg_mem(newval, mem_ptr),
8006              OpcP, OpcS,
8007              reg_mem(newval, mem_ptr) // lock cmpxchg
8008              );
8009   ins_pipe( pipe_cmpxchg );
8010 %}
8011 
8012 instruct compareAndExchangeL(
8013                          memory mem_ptr,
8014                          rax_RegL oldval, rRegL newval,
8015                          rFlagsReg cr)
8016 %{
8017   predicate(VM_Version::supports_cx8());
8018   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
8019   effect(KILL cr);
8020 
8021   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8022             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
8023   opcode(0x0F, 0xB1);
8024   ins_encode(lock_prefix,
8025              REX_reg_mem_wide(newval, mem_ptr),
8026              OpcP, OpcS,
8027              reg_mem(newval, mem_ptr)  // lock cmpxchg
8028             );
8029   ins_pipe( pipe_cmpxchg );
8030 %}
8031 
8032 instruct compareAndExchangeN(
8033                           memory mem_ptr,
8034                           rax_RegN oldval, rRegN newval,
8035                           rFlagsReg cr) %{
8036   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
8037   effect(KILL cr);
8038 
8039   format %{ "cmpxchgl $mem_ptr,$newval\t# "
8040             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8041   opcode(0x0F, 0xB1);
8042   ins_encode(lock_prefix,
8043              REX_reg_mem(newval, mem_ptr),
8044              OpcP, OpcS,
8045              reg_mem(newval, mem_ptr)  // lock cmpxchg
8046           );
8047   ins_pipe( pipe_cmpxchg );
8048 %}
8049 
8050 instruct compareAndExchangeP(
8051                          memory mem_ptr,
8052                          rax_RegP oldval, rRegP newval,
8053                          rFlagsReg cr)
8054 %{
8055   predicate(VM_Version::supports_cx8());
8056   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
8057   effect(KILL cr);
8058 
8059   format %{ "cmpxchgq $mem_ptr,$newval\t# "
8060             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
8061   opcode(0x0F, 0xB1);
8062   ins_encode(lock_prefix,
8063              REX_reg_mem_wide(newval, mem_ptr),
8064              OpcP, OpcS,
8065              reg_mem(newval, mem_ptr)  // lock cmpxchg
8066           );
8067   ins_pipe( pipe_cmpxchg );
8068 %}
8069 
8070 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8071   predicate(n->as_LoadStore()->result_not_used());
8072   match(Set dummy (GetAndAddB mem add));
8073   effect(KILL cr);
8074   format %{ "ADDB  [$mem],$add" %}
8075   ins_encode %{
8076     __ lock();
8077     __ addb($mem$$Address, $add$$constant);
8078   %}
8079   ins_pipe( pipe_cmpxchg );
8080 %}
8081 
8082 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
8083   match(Set newval (GetAndAddB mem newval));
8084   effect(KILL cr);
8085   format %{ "XADDB  [$mem],$newval" %}
8086   ins_encode %{
8087     __ lock();
8088     __ xaddb($mem$$Address, $newval$$Register);
8089   %}
8090   ins_pipe( pipe_cmpxchg );
8091 %}
8092 
8093 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8094   predicate(n->as_LoadStore()->result_not_used());
8095   match(Set dummy (GetAndAddS mem add));
8096   effect(KILL cr);
8097   format %{ "ADDW  [$mem],$add" %}
8098   ins_encode %{
8099     __ lock();
8100     __ addw($mem$$Address, $add$$constant);
8101   %}
8102   ins_pipe( pipe_cmpxchg );
8103 %}
8104 
8105 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
8106   match(Set newval (GetAndAddS mem newval));
8107   effect(KILL cr);
8108   format %{ "XADDW  [$mem],$newval" %}
8109   ins_encode %{
8110     __ lock();
8111     __ xaddw($mem$$Address, $newval$$Register);
8112   %}
8113   ins_pipe( pipe_cmpxchg );
8114 %}
8115 
8116 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
8117   predicate(n->as_LoadStore()->result_not_used());
8118   match(Set dummy (GetAndAddI mem add));
8119   effect(KILL cr);
8120   format %{ "ADDL  [$mem],$add" %}
8121   ins_encode %{
8122     __ lock();
8123     __ addl($mem$$Address, $add$$constant);
8124   %}
8125   ins_pipe( pipe_cmpxchg );
8126 %}
8127 
8128 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
8129   match(Set newval (GetAndAddI mem newval));
8130   effect(KILL cr);
8131   format %{ "XADDL  [$mem],$newval" %}
8132   ins_encode %{
8133     __ lock();
8134     __ xaddl($mem$$Address, $newval$$Register);
8135   %}
8136   ins_pipe( pipe_cmpxchg );
8137 %}
8138 
8139 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
8140   predicate(n->as_LoadStore()->result_not_used());
8141   match(Set dummy (GetAndAddL mem add));
8142   effect(KILL cr);
8143   format %{ "ADDQ  [$mem],$add" %}
8144   ins_encode %{
8145     __ lock();
8146     __ addq($mem$$Address, $add$$constant);
8147   %}
8148   ins_pipe( pipe_cmpxchg );
8149 %}
8150 
8151 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
8152   match(Set newval (GetAndAddL mem newval));
8153   effect(KILL cr);
8154   format %{ "XADDQ  [$mem],$newval" %}
8155   ins_encode %{
8156     __ lock();
8157     __ xaddq($mem$$Address, $newval$$Register);
8158   %}
8159   ins_pipe( pipe_cmpxchg );
8160 %}
8161 
8162 instruct xchgB( memory mem, rRegI newval) %{
8163   match(Set newval (GetAndSetB mem newval));
8164   format %{ "XCHGB  $newval,[$mem]" %}
8165   ins_encode %{
8166     __ xchgb($newval$$Register, $mem$$Address);
8167   %}
8168   ins_pipe( pipe_cmpxchg );
8169 %}
8170 
8171 instruct xchgS( memory mem, rRegI newval) %{
8172   match(Set newval (GetAndSetS mem newval));
8173   format %{ "XCHGW  $newval,[$mem]" %}
8174   ins_encode %{
8175     __ xchgw($newval$$Register, $mem$$Address);
8176   %}
8177   ins_pipe( pipe_cmpxchg );
8178 %}
8179 
8180 instruct xchgI( memory mem, rRegI newval) %{
8181   match(Set newval (GetAndSetI mem newval));
8182   format %{ "XCHGL  $newval,[$mem]" %}
8183   ins_encode %{
8184     __ xchgl($newval$$Register, $mem$$Address);
8185   %}
8186   ins_pipe( pipe_cmpxchg );
8187 %}
8188 
8189 instruct xchgL( memory mem, rRegL newval) %{
8190   match(Set newval (GetAndSetL mem newval));
8191   format %{ "XCHGL  $newval,[$mem]" %}
8192   ins_encode %{
8193     __ xchgq($newval$$Register, $mem$$Address);
8194   %}
8195   ins_pipe( pipe_cmpxchg );
8196 %}
8197 
8198 instruct xchgP( memory mem, rRegP newval) %{
8199   match(Set newval (GetAndSetP mem newval));
8200   format %{ "XCHGQ  $newval,[$mem]" %}
8201   ins_encode %{
8202     __ xchgq($newval$$Register, $mem$$Address);
8203   %}
8204   ins_pipe( pipe_cmpxchg );
8205 %}
8206 
8207 instruct xchgN( memory mem, rRegN newval) %{
8208   match(Set newval (GetAndSetN mem newval));
8209   format %{ "XCHGL  $newval,$mem]" %}
8210   ins_encode %{
8211     __ xchgl($newval$$Register, $mem$$Address);
8212   %}
8213   ins_pipe( pipe_cmpxchg );
8214 %}
8215 
8216 //----------Subtraction Instructions-------------------------------------------
8217 
8218 // Integer Subtraction Instructions
8219 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8220 %{
8221   match(Set dst (SubI dst src));
8222   effect(KILL cr);
8223 
8224   format %{ "subl    $dst, $src\t# int" %}
8225   opcode(0x2B);
8226   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8227   ins_pipe(ialu_reg_reg);
8228 %}
8229 
8230 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8231 %{
8232   match(Set dst (SubI dst src));
8233   effect(KILL cr);
8234 
8235   format %{ "subl    $dst, $src\t# int" %}
8236   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8237   ins_encode(OpcSErm(dst, src), Con8or32(src));
8238   ins_pipe(ialu_reg);
8239 %}
8240 
8241 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8242 %{
8243   match(Set dst (SubI dst (LoadI src)));
8244   effect(KILL cr);
8245 
8246   ins_cost(125);
8247   format %{ "subl    $dst, $src\t# int" %}
8248   opcode(0x2B);
8249   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8250   ins_pipe(ialu_reg_mem);
8251 %}
8252 
8253 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8254 %{
8255   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8256   effect(KILL cr);
8257 
8258   ins_cost(150);
8259   format %{ "subl    $dst, $src\t# int" %}
8260   opcode(0x29); /* Opcode 29 /r */
8261   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8262   ins_pipe(ialu_mem_reg);
8263 %}
8264 
8265 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
8266 %{
8267   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
8268   effect(KILL cr);
8269 
8270   ins_cost(125); // XXX
8271   format %{ "subl    $dst, $src\t# int" %}
8272   opcode(0x81); /* Opcode 81 /5 id */
8273   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8274   ins_pipe(ialu_mem_imm);
8275 %}
8276 
8277 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8278 %{
8279   match(Set dst (SubL dst src));
8280   effect(KILL cr);
8281 
8282   format %{ "subq    $dst, $src\t# long" %}
8283   opcode(0x2B);
8284   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8285   ins_pipe(ialu_reg_reg);
8286 %}
8287 
8288 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8289 %{
8290   match(Set dst (SubL dst src));
8291   effect(KILL cr);
8292 
8293   format %{ "subq    $dst, $src\t# long" %}
8294   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8295   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8296   ins_pipe(ialu_reg);
8297 %}
8298 
8299 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8300 %{
8301   match(Set dst (SubL dst (LoadL src)));
8302   effect(KILL cr);
8303 
8304   ins_cost(125);
8305   format %{ "subq    $dst, $src\t# long" %}
8306   opcode(0x2B);
8307   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8308   ins_pipe(ialu_reg_mem);
8309 %}
8310 
8311 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8312 %{
8313   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8314   effect(KILL cr);
8315 
8316   ins_cost(150);
8317   format %{ "subq    $dst, $src\t# long" %}
8318   opcode(0x29); /* Opcode 29 /r */
8319   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8320   ins_pipe(ialu_mem_reg);
8321 %}
8322 
8323 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8324 %{
8325   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8326   effect(KILL cr);
8327 
8328   ins_cost(125); // XXX
8329   format %{ "subq    $dst, $src\t# long" %}
8330   opcode(0x81); /* Opcode 81 /5 id */
8331   ins_encode(REX_mem_wide(dst),
8332              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8333   ins_pipe(ialu_mem_imm);
8334 %}
8335 
8336 // Subtract from a pointer
8337 // XXX hmpf???
8338 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8339 %{
8340   match(Set dst (AddP dst (SubI zero src)));
8341   effect(KILL cr);
8342 
8343   format %{ "subq    $dst, $src\t# ptr - int" %}
8344   opcode(0x2B);
8345   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8346   ins_pipe(ialu_reg_reg);
8347 %}
8348 
8349 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8350 %{
8351   match(Set dst (SubI zero dst));
8352   effect(KILL cr);
8353 
8354   format %{ "negl    $dst\t# int" %}
8355   opcode(0xF7, 0x03);  // Opcode F7 /3
8356   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8357   ins_pipe(ialu_reg);
8358 %}
8359 
8360 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8361 %{
8362   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8363   effect(KILL cr);
8364 
8365   format %{ "negl    $dst\t# int" %}
8366   opcode(0xF7, 0x03);  // Opcode F7 /3
8367   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8368   ins_pipe(ialu_reg);
8369 %}
8370 
8371 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8372 %{
8373   match(Set dst (SubL zero dst));
8374   effect(KILL cr);
8375 
8376   format %{ "negq    $dst\t# long" %}
8377   opcode(0xF7, 0x03);  // Opcode F7 /3
8378   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8379   ins_pipe(ialu_reg);
8380 %}
8381 
8382 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8383 %{
8384   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8385   effect(KILL cr);
8386 
8387   format %{ "negq    $dst\t# long" %}
8388   opcode(0xF7, 0x03);  // Opcode F7 /3
8389   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8390   ins_pipe(ialu_reg);
8391 %}
8392 
8393 //----------Multiplication/Division Instructions-------------------------------
8394 // Integer Multiplication Instructions
8395 // Multiply Register
8396 
8397 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8398 %{
8399   match(Set dst (MulI dst src));
8400   effect(KILL cr);
8401 
8402   ins_cost(300);
8403   format %{ "imull   $dst, $src\t# int" %}
8404   opcode(0x0F, 0xAF);
8405   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8406   ins_pipe(ialu_reg_reg_alu0);
8407 %}
8408 
8409 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8410 %{
8411   match(Set dst (MulI src imm));
8412   effect(KILL cr);
8413 
8414   ins_cost(300);
8415   format %{ "imull   $dst, $src, $imm\t# int" %}
8416   opcode(0x69); /* 69 /r id */
8417   ins_encode(REX_reg_reg(dst, src),
8418              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8419   ins_pipe(ialu_reg_reg_alu0);
8420 %}
8421 
8422 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8423 %{
8424   match(Set dst (MulI dst (LoadI src)));
8425   effect(KILL cr);
8426 
8427   ins_cost(350);
8428   format %{ "imull   $dst, $src\t# int" %}
8429   opcode(0x0F, 0xAF);
8430   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8431   ins_pipe(ialu_reg_mem_alu0);
8432 %}
8433 
8434 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8435 %{
8436   match(Set dst (MulI (LoadI src) imm));
8437   effect(KILL cr);
8438 
8439   ins_cost(300);
8440   format %{ "imull   $dst, $src, $imm\t# int" %}
8441   opcode(0x69); /* 69 /r id */
8442   ins_encode(REX_reg_mem(dst, src),
8443              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8444   ins_pipe(ialu_reg_mem_alu0);
8445 %}
8446 
8447 instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
8448 %{
8449   match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
8450   effect(KILL cr, KILL src2);
8451 
8452   expand %{ mulI_rReg(dst, src1, cr);
8453            mulI_rReg(src2, src3, cr);
8454            addI_rReg(dst, src2, cr); %}
8455 %}
8456 
8457 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8458 %{
8459   match(Set dst (MulL dst src));
8460   effect(KILL cr);
8461 
8462   ins_cost(300);
8463   format %{ "imulq   $dst, $src\t# long" %}
8464   opcode(0x0F, 0xAF);
8465   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8466   ins_pipe(ialu_reg_reg_alu0);
8467 %}
8468 
8469 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8470 %{
8471   match(Set dst (MulL src imm));
8472   effect(KILL cr);
8473 
8474   ins_cost(300);
8475   format %{ "imulq   $dst, $src, $imm\t# long" %}
8476   opcode(0x69); /* 69 /r id */
8477   ins_encode(REX_reg_reg_wide(dst, src),
8478              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8479   ins_pipe(ialu_reg_reg_alu0);
8480 %}
8481 
8482 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8483 %{
8484   match(Set dst (MulL dst (LoadL src)));
8485   effect(KILL cr);
8486 
8487   ins_cost(350);
8488   format %{ "imulq   $dst, $src\t# long" %}
8489   opcode(0x0F, 0xAF);
8490   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8491   ins_pipe(ialu_reg_mem_alu0);
8492 %}
8493 
8494 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8495 %{
8496   match(Set dst (MulL (LoadL src) imm));
8497   effect(KILL cr);
8498 
8499   ins_cost(300);
8500   format %{ "imulq   $dst, $src, $imm\t# long" %}
8501   opcode(0x69); /* 69 /r id */
8502   ins_encode(REX_reg_mem_wide(dst, src),
8503              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8504   ins_pipe(ialu_reg_mem_alu0);
8505 %}
8506 
8507 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8508 %{
8509   match(Set dst (MulHiL src rax));
8510   effect(USE_KILL rax, KILL cr);
8511 
8512   ins_cost(300);
8513   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8514   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8515   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8516   ins_pipe(ialu_reg_reg_alu0);
8517 %}
8518 
8519 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8520                    rFlagsReg cr)
8521 %{
8522   match(Set rax (DivI rax div));
8523   effect(KILL rdx, KILL cr);
8524 
8525   ins_cost(30*100+10*100); // XXX
8526   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8527             "jne,s   normal\n\t"
8528             "xorl    rdx, rdx\n\t"
8529             "cmpl    $div, -1\n\t"
8530             "je,s    done\n"
8531     "normal: cdql\n\t"
8532             "idivl   $div\n"
8533     "done:"        %}
8534   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8535   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8536   ins_pipe(ialu_reg_reg_alu0);
8537 %}
8538 
8539 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8540                    rFlagsReg cr)
8541 %{
8542   match(Set rax (DivL rax div));
8543   effect(KILL rdx, KILL cr);
8544 
8545   ins_cost(30*100+10*100); // XXX
8546   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8547             "cmpq    rax, rdx\n\t"
8548             "jne,s   normal\n\t"
8549             "xorl    rdx, rdx\n\t"
8550             "cmpq    $div, -1\n\t"
8551             "je,s    done\n"
8552     "normal: cdqq\n\t"
8553             "idivq   $div\n"
8554     "done:"        %}
8555   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8556   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8557   ins_pipe(ialu_reg_reg_alu0);
8558 %}
8559 
8560 // Integer DIVMOD with Register, both quotient and mod results
8561 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8562                              rFlagsReg cr)
8563 %{
8564   match(DivModI rax div);
8565   effect(KILL cr);
8566 
8567   ins_cost(30*100+10*100); // XXX
8568   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8569             "jne,s   normal\n\t"
8570             "xorl    rdx, rdx\n\t"
8571             "cmpl    $div, -1\n\t"
8572             "je,s    done\n"
8573     "normal: cdql\n\t"
8574             "idivl   $div\n"
8575     "done:"        %}
8576   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8577   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8578   ins_pipe(pipe_slow);
8579 %}
8580 
8581 // Long DIVMOD with Register, both quotient and mod results
8582 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8583                              rFlagsReg cr)
8584 %{
8585   match(DivModL rax div);
8586   effect(KILL cr);
8587 
8588   ins_cost(30*100+10*100); // XXX
8589   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8590             "cmpq    rax, rdx\n\t"
8591             "jne,s   normal\n\t"
8592             "xorl    rdx, rdx\n\t"
8593             "cmpq    $div, -1\n\t"
8594             "je,s    done\n"
8595     "normal: cdqq\n\t"
8596             "idivq   $div\n"
8597     "done:"        %}
8598   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8599   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8600   ins_pipe(pipe_slow);
8601 %}
8602 
8603 //----------- DivL-By-Constant-Expansions--------------------------------------
8604 // DivI cases are handled by the compiler
8605 
8606 // Magic constant, reciprocal of 10
8607 instruct loadConL_0x6666666666666667(rRegL dst)
8608 %{
8609   effect(DEF dst);
8610 
8611   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8612   ins_encode(load_immL(dst, 0x6666666666666667));
8613   ins_pipe(ialu_reg);
8614 %}
8615 
8616 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8617 %{
8618   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8619 
8620   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8621   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8622   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8623   ins_pipe(ialu_reg_reg_alu0);
8624 %}
8625 
8626 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8627 %{
8628   effect(USE_DEF dst, KILL cr);
8629 
8630   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8631   opcode(0xC1, 0x7); /* C1 /7 ib */
8632   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8633   ins_pipe(ialu_reg);
8634 %}
8635 
8636 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8637 %{
8638   effect(USE_DEF dst, KILL cr);
8639 
8640   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8641   opcode(0xC1, 0x7); /* C1 /7 ib */
8642   ins_encode(reg_opc_imm_wide(dst, 0x2));
8643   ins_pipe(ialu_reg);
8644 %}
8645 
8646 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8647 %{
8648   match(Set dst (DivL src div));
8649 
8650   ins_cost((5+8)*100);
8651   expand %{
8652     rax_RegL rax;                     // Killed temp
8653     rFlagsReg cr;                     // Killed
8654     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8655     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8656     sarL_rReg_63(src, cr);            // sarq  src, 63
8657     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8658     subL_rReg(dst, src, cr);          // subl  rdx, src
8659   %}
8660 %}
8661 
8662 //-----------------------------------------------------------------------------
8663 
8664 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8665                    rFlagsReg cr)
8666 %{
8667   match(Set rdx (ModI rax div));
8668   effect(KILL rax, KILL cr);
8669 
8670   ins_cost(300); // XXX
8671   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8672             "jne,s   normal\n\t"
8673             "xorl    rdx, rdx\n\t"
8674             "cmpl    $div, -1\n\t"
8675             "je,s    done\n"
8676     "normal: cdql\n\t"
8677             "idivl   $div\n"
8678     "done:"        %}
8679   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8680   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8681   ins_pipe(ialu_reg_reg_alu0);
8682 %}
8683 
8684 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8685                    rFlagsReg cr)
8686 %{
8687   match(Set rdx (ModL rax div));
8688   effect(KILL rax, KILL cr);
8689 
8690   ins_cost(300); // XXX
8691   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8692             "cmpq    rax, rdx\n\t"
8693             "jne,s   normal\n\t"
8694             "xorl    rdx, rdx\n\t"
8695             "cmpq    $div, -1\n\t"
8696             "je,s    done\n"
8697     "normal: cdqq\n\t"
8698             "idivq   $div\n"
8699     "done:"        %}
8700   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8701   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8702   ins_pipe(ialu_reg_reg_alu0);
8703 %}
8704 
8705 // Integer Shift Instructions
8706 // Shift Left by one
8707 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8708 %{
8709   match(Set dst (LShiftI dst shift));
8710   effect(KILL cr);
8711 
8712   format %{ "sall    $dst, $shift" %}
8713   opcode(0xD1, 0x4); /* D1 /4 */
8714   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8715   ins_pipe(ialu_reg);
8716 %}
8717 
8718 // Shift Left by one
8719 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8720 %{
8721   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8722   effect(KILL cr);
8723 
8724   format %{ "sall    $dst, $shift\t" %}
8725   opcode(0xD1, 0x4); /* D1 /4 */
8726   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8727   ins_pipe(ialu_mem_imm);
8728 %}
8729 
8730 // Shift Left by 8-bit immediate
8731 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8732 %{
8733   match(Set dst (LShiftI dst shift));
8734   effect(KILL cr);
8735 
8736   format %{ "sall    $dst, $shift" %}
8737   opcode(0xC1, 0x4); /* C1 /4 ib */
8738   ins_encode(reg_opc_imm(dst, shift));
8739   ins_pipe(ialu_reg);
8740 %}
8741 
8742 // Shift Left by 8-bit immediate
8743 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8744 %{
8745   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8746   effect(KILL cr);
8747 
8748   format %{ "sall    $dst, $shift" %}
8749   opcode(0xC1, 0x4); /* C1 /4 ib */
8750   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8751   ins_pipe(ialu_mem_imm);
8752 %}
8753 
8754 // Shift Left by variable
8755 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8756 %{
8757   match(Set dst (LShiftI dst shift));
8758   effect(KILL cr);
8759 
8760   format %{ "sall    $dst, $shift" %}
8761   opcode(0xD3, 0x4); /* D3 /4 */
8762   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8763   ins_pipe(ialu_reg_reg);
8764 %}
8765 
8766 // Shift Left by variable
8767 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8768 %{
8769   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8770   effect(KILL cr);
8771 
8772   format %{ "sall    $dst, $shift" %}
8773   opcode(0xD3, 0x4); /* D3 /4 */
8774   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8775   ins_pipe(ialu_mem_reg);
8776 %}
8777 
8778 // Arithmetic shift right by one
8779 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8780 %{
8781   match(Set dst (RShiftI dst shift));
8782   effect(KILL cr);
8783 
8784   format %{ "sarl    $dst, $shift" %}
8785   opcode(0xD1, 0x7); /* D1 /7 */
8786   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8787   ins_pipe(ialu_reg);
8788 %}
8789 
8790 // Arithmetic shift right by one
8791 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8792 %{
8793   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8794   effect(KILL cr);
8795 
8796   format %{ "sarl    $dst, $shift" %}
8797   opcode(0xD1, 0x7); /* D1 /7 */
8798   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8799   ins_pipe(ialu_mem_imm);
8800 %}
8801 
8802 // Arithmetic Shift Right by 8-bit immediate
8803 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8804 %{
8805   match(Set dst (RShiftI dst shift));
8806   effect(KILL cr);
8807 
8808   format %{ "sarl    $dst, $shift" %}
8809   opcode(0xC1, 0x7); /* C1 /7 ib */
8810   ins_encode(reg_opc_imm(dst, shift));
8811   ins_pipe(ialu_mem_imm);
8812 %}
8813 
8814 // Arithmetic Shift Right by 8-bit immediate
8815 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8816 %{
8817   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8818   effect(KILL cr);
8819 
8820   format %{ "sarl    $dst, $shift" %}
8821   opcode(0xC1, 0x7); /* C1 /7 ib */
8822   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8823   ins_pipe(ialu_mem_imm);
8824 %}
8825 
8826 // Arithmetic Shift Right by variable
8827 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8828 %{
8829   match(Set dst (RShiftI dst shift));
8830   effect(KILL cr);
8831 
8832   format %{ "sarl    $dst, $shift" %}
8833   opcode(0xD3, 0x7); /* D3 /7 */
8834   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8835   ins_pipe(ialu_reg_reg);
8836 %}
8837 
8838 // Arithmetic Shift Right by variable
8839 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8840 %{
8841   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8842   effect(KILL cr);
8843 
8844   format %{ "sarl    $dst, $shift" %}
8845   opcode(0xD3, 0x7); /* D3 /7 */
8846   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8847   ins_pipe(ialu_mem_reg);
8848 %}
8849 
8850 // Logical shift right by one
8851 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8852 %{
8853   match(Set dst (URShiftI dst shift));
8854   effect(KILL cr);
8855 
8856   format %{ "shrl    $dst, $shift" %}
8857   opcode(0xD1, 0x5); /* D1 /5 */
8858   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8859   ins_pipe(ialu_reg);
8860 %}
8861 
8862 // Logical shift right by one
8863 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8864 %{
8865   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8866   effect(KILL cr);
8867 
8868   format %{ "shrl    $dst, $shift" %}
8869   opcode(0xD1, 0x5); /* D1 /5 */
8870   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8871   ins_pipe(ialu_mem_imm);
8872 %}
8873 
8874 // Logical Shift Right by 8-bit immediate
8875 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8876 %{
8877   match(Set dst (URShiftI dst shift));
8878   effect(KILL cr);
8879 
8880   format %{ "shrl    $dst, $shift" %}
8881   opcode(0xC1, 0x5); /* C1 /5 ib */
8882   ins_encode(reg_opc_imm(dst, shift));
8883   ins_pipe(ialu_reg);
8884 %}
8885 
8886 // Logical Shift Right by 8-bit immediate
8887 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8888 %{
8889   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8890   effect(KILL cr);
8891 
8892   format %{ "shrl    $dst, $shift" %}
8893   opcode(0xC1, 0x5); /* C1 /5 ib */
8894   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8895   ins_pipe(ialu_mem_imm);
8896 %}
8897 
8898 // Logical Shift Right by variable
8899 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8900 %{
8901   match(Set dst (URShiftI dst shift));
8902   effect(KILL cr);
8903 
8904   format %{ "shrl    $dst, $shift" %}
8905   opcode(0xD3, 0x5); /* D3 /5 */
8906   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8907   ins_pipe(ialu_reg_reg);
8908 %}
8909 
8910 // Logical Shift Right by variable
8911 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8912 %{
8913   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8914   effect(KILL cr);
8915 
8916   format %{ "shrl    $dst, $shift" %}
8917   opcode(0xD3, 0x5); /* D3 /5 */
8918   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8919   ins_pipe(ialu_mem_reg);
8920 %}
8921 
8922 // Long Shift Instructions
8923 // Shift Left by one
8924 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8925 %{
8926   match(Set dst (LShiftL dst shift));
8927   effect(KILL cr);
8928 
8929   format %{ "salq    $dst, $shift" %}
8930   opcode(0xD1, 0x4); /* D1 /4 */
8931   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8932   ins_pipe(ialu_reg);
8933 %}
8934 
8935 // Shift Left by one
8936 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8937 %{
8938   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8939   effect(KILL cr);
8940 
8941   format %{ "salq    $dst, $shift" %}
8942   opcode(0xD1, 0x4); /* D1 /4 */
8943   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8944   ins_pipe(ialu_mem_imm);
8945 %}
8946 
8947 // Shift Left by 8-bit immediate
8948 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8949 %{
8950   match(Set dst (LShiftL dst shift));
8951   effect(KILL cr);
8952 
8953   format %{ "salq    $dst, $shift" %}
8954   opcode(0xC1, 0x4); /* C1 /4 ib */
8955   ins_encode(reg_opc_imm_wide(dst, shift));
8956   ins_pipe(ialu_reg);
8957 %}
8958 
8959 // Shift Left by 8-bit immediate
8960 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8961 %{
8962   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8963   effect(KILL cr);
8964 
8965   format %{ "salq    $dst, $shift" %}
8966   opcode(0xC1, 0x4); /* C1 /4 ib */
8967   ins_encode(REX_mem_wide(dst), OpcP,
8968              RM_opc_mem(secondary, dst), Con8or32(shift));
8969   ins_pipe(ialu_mem_imm);
8970 %}
8971 
8972 // Shift Left by variable
8973 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8974 %{
8975   match(Set dst (LShiftL dst shift));
8976   effect(KILL cr);
8977 
8978   format %{ "salq    $dst, $shift" %}
8979   opcode(0xD3, 0x4); /* D3 /4 */
8980   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8981   ins_pipe(ialu_reg_reg);
8982 %}
8983 
8984 // Shift Left by variable
8985 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8986 %{
8987   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8988   effect(KILL cr);
8989 
8990   format %{ "salq    $dst, $shift" %}
8991   opcode(0xD3, 0x4); /* D3 /4 */
8992   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8993   ins_pipe(ialu_mem_reg);
8994 %}
8995 
8996 // Arithmetic shift right by one
8997 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8998 %{
8999   match(Set dst (RShiftL dst shift));
9000   effect(KILL cr);
9001 
9002   format %{ "sarq    $dst, $shift" %}
9003   opcode(0xD1, 0x7); /* D1 /7 */
9004   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9005   ins_pipe(ialu_reg);
9006 %}
9007 
9008 // Arithmetic shift right by one
9009 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9010 %{
9011   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9012   effect(KILL cr);
9013 
9014   format %{ "sarq    $dst, $shift" %}
9015   opcode(0xD1, 0x7); /* D1 /7 */
9016   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9017   ins_pipe(ialu_mem_imm);
9018 %}
9019 
9020 // Arithmetic Shift Right by 8-bit immediate
9021 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9022 %{
9023   match(Set dst (RShiftL dst shift));
9024   effect(KILL cr);
9025 
9026   format %{ "sarq    $dst, $shift" %}
9027   opcode(0xC1, 0x7); /* C1 /7 ib */
9028   ins_encode(reg_opc_imm_wide(dst, shift));
9029   ins_pipe(ialu_mem_imm);
9030 %}
9031 
9032 // Arithmetic Shift Right by 8-bit immediate
9033 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9034 %{
9035   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9036   effect(KILL cr);
9037 
9038   format %{ "sarq    $dst, $shift" %}
9039   opcode(0xC1, 0x7); /* C1 /7 ib */
9040   ins_encode(REX_mem_wide(dst), OpcP,
9041              RM_opc_mem(secondary, dst), Con8or32(shift));
9042   ins_pipe(ialu_mem_imm);
9043 %}
9044 
9045 // Arithmetic Shift Right by variable
9046 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9047 %{
9048   match(Set dst (RShiftL dst shift));
9049   effect(KILL cr);
9050 
9051   format %{ "sarq    $dst, $shift" %}
9052   opcode(0xD3, 0x7); /* D3 /7 */
9053   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9054   ins_pipe(ialu_reg_reg);
9055 %}
9056 
9057 // Arithmetic Shift Right by variable
9058 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9059 %{
9060   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
9061   effect(KILL cr);
9062 
9063   format %{ "sarq    $dst, $shift" %}
9064   opcode(0xD3, 0x7); /* D3 /7 */
9065   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9066   ins_pipe(ialu_mem_reg);
9067 %}
9068 
9069 // Logical shift right by one
9070 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
9071 %{
9072   match(Set dst (URShiftL dst shift));
9073   effect(KILL cr);
9074 
9075   format %{ "shrq    $dst, $shift" %}
9076   opcode(0xD1, 0x5); /* D1 /5 */
9077   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
9078   ins_pipe(ialu_reg);
9079 %}
9080 
9081 // Logical shift right by one
9082 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
9083 %{
9084   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9085   effect(KILL cr);
9086 
9087   format %{ "shrq    $dst, $shift" %}
9088   opcode(0xD1, 0x5); /* D1 /5 */
9089   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9090   ins_pipe(ialu_mem_imm);
9091 %}
9092 
9093 // Logical Shift Right by 8-bit immediate
9094 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
9095 %{
9096   match(Set dst (URShiftL dst shift));
9097   effect(KILL cr);
9098 
9099   format %{ "shrq    $dst, $shift" %}
9100   opcode(0xC1, 0x5); /* C1 /5 ib */
9101   ins_encode(reg_opc_imm_wide(dst, shift));
9102   ins_pipe(ialu_reg);
9103 %}
9104 
9105 
9106 // Logical Shift Right by 8-bit immediate
9107 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
9108 %{
9109   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9110   effect(KILL cr);
9111 
9112   format %{ "shrq    $dst, $shift" %}
9113   opcode(0xC1, 0x5); /* C1 /5 ib */
9114   ins_encode(REX_mem_wide(dst), OpcP,
9115              RM_opc_mem(secondary, dst), Con8or32(shift));
9116   ins_pipe(ialu_mem_imm);
9117 %}
9118 
9119 // Logical Shift Right by variable
9120 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
9121 %{
9122   match(Set dst (URShiftL dst shift));
9123   effect(KILL cr);
9124 
9125   format %{ "shrq    $dst, $shift" %}
9126   opcode(0xD3, 0x5); /* D3 /5 */
9127   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9128   ins_pipe(ialu_reg_reg);
9129 %}
9130 
9131 // Logical Shift Right by variable
9132 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
9133 %{
9134   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
9135   effect(KILL cr);
9136 
9137   format %{ "shrq    $dst, $shift" %}
9138   opcode(0xD3, 0x5); /* D3 /5 */
9139   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
9140   ins_pipe(ialu_mem_reg);
9141 %}
9142 
9143 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
9144 // This idiom is used by the compiler for the i2b bytecode.
9145 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
9146 %{
9147   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
9148 
9149   format %{ "movsbl  $dst, $src\t# i2b" %}
9150   opcode(0x0F, 0xBE);
9151   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9152   ins_pipe(ialu_reg_reg);
9153 %}
9154 
9155 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
9156 // This idiom is used by the compiler the i2s bytecode.
9157 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
9158 %{
9159   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
9160 
9161   format %{ "movswl  $dst, $src\t# i2s" %}
9162   opcode(0x0F, 0xBF);
9163   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9164   ins_pipe(ialu_reg_reg);
9165 %}
9166 
9167 // ROL/ROR instructions
9168 
9169 // ROL expand
9170 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
9171   effect(KILL cr, USE_DEF dst);
9172 
9173   format %{ "roll    $dst" %}
9174   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9175   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9176   ins_pipe(ialu_reg);
9177 %}
9178 
9179 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
9180   effect(USE_DEF dst, USE shift, KILL cr);
9181 
9182   format %{ "roll    $dst, $shift" %}
9183   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9184   ins_encode( reg_opc_imm(dst, shift) );
9185   ins_pipe(ialu_reg);
9186 %}
9187 
9188 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9189 %{
9190   effect(USE_DEF dst, USE shift, KILL cr);
9191 
9192   format %{ "roll    $dst, $shift" %}
9193   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9194   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9195   ins_pipe(ialu_reg_reg);
9196 %}
9197 // end of ROL expand
9198 
9199 // Rotate Left by one
9200 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9201 %{
9202   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9203 
9204   expand %{
9205     rolI_rReg_imm1(dst, cr);
9206   %}
9207 %}
9208 
9209 // Rotate Left by 8-bit immediate
9210 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9211 %{
9212   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9213   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
9214 
9215   expand %{
9216     rolI_rReg_imm8(dst, lshift, cr);
9217   %}
9218 %}
9219 
9220 // Rotate Left by variable
9221 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9222 %{
9223   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
9224 
9225   expand %{
9226     rolI_rReg_CL(dst, shift, cr);
9227   %}
9228 %}
9229 
9230 // Rotate Left by variable
9231 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9232 %{
9233   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
9234 
9235   expand %{
9236     rolI_rReg_CL(dst, shift, cr);
9237   %}
9238 %}
9239 
9240 // ROR expand
9241 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
9242 %{
9243   effect(USE_DEF dst, KILL cr);
9244 
9245   format %{ "rorl    $dst" %}
9246   opcode(0xD1, 0x1); /* D1 /1 */
9247   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9248   ins_pipe(ialu_reg);
9249 %}
9250 
9251 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
9252 %{
9253   effect(USE_DEF dst, USE shift, KILL cr);
9254 
9255   format %{ "rorl    $dst, $shift" %}
9256   opcode(0xC1, 0x1); /* C1 /1 ib */
9257   ins_encode(reg_opc_imm(dst, shift));
9258   ins_pipe(ialu_reg);
9259 %}
9260 
9261 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
9262 %{
9263   effect(USE_DEF dst, USE shift, KILL cr);
9264 
9265   format %{ "rorl    $dst, $shift" %}
9266   opcode(0xD3, 0x1); /* D3 /1 */
9267   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
9268   ins_pipe(ialu_reg_reg);
9269 %}
9270 // end of ROR expand
9271 
9272 // Rotate Right by one
9273 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9274 %{
9275   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9276 
9277   expand %{
9278     rorI_rReg_imm1(dst, cr);
9279   %}
9280 %}
9281 
9282 // Rotate Right by 8-bit immediate
9283 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9284 %{
9285   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9286   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9287 
9288   expand %{
9289     rorI_rReg_imm8(dst, rshift, cr);
9290   %}
9291 %}
9292 
9293 // Rotate Right by variable
9294 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9295 %{
9296   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9297 
9298   expand %{
9299     rorI_rReg_CL(dst, shift, cr);
9300   %}
9301 %}
9302 
9303 // Rotate Right by variable
9304 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9305 %{
9306   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9307 
9308   expand %{
9309     rorI_rReg_CL(dst, shift, cr);
9310   %}
9311 %}
9312 
9313 // for long rotate
9314 // ROL expand
9315 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9316   effect(USE_DEF dst, KILL cr);
9317 
9318   format %{ "rolq    $dst" %}
9319   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9320   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9321   ins_pipe(ialu_reg);
9322 %}
9323 
9324 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9325   effect(USE_DEF dst, USE shift, KILL cr);
9326 
9327   format %{ "rolq    $dst, $shift" %}
9328   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9329   ins_encode( reg_opc_imm_wide(dst, shift) );
9330   ins_pipe(ialu_reg);
9331 %}
9332 
9333 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9334 %{
9335   effect(USE_DEF dst, USE shift, KILL cr);
9336 
9337   format %{ "rolq    $dst, $shift" %}
9338   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9339   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9340   ins_pipe(ialu_reg_reg);
9341 %}
9342 // end of ROL expand
9343 
9344 // Rotate Left by one
9345 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9346 %{
9347   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9348 
9349   expand %{
9350     rolL_rReg_imm1(dst, cr);
9351   %}
9352 %}
9353 
9354 // Rotate Left by 8-bit immediate
9355 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9356 %{
9357   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9358   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9359 
9360   expand %{
9361     rolL_rReg_imm8(dst, lshift, cr);
9362   %}
9363 %}
9364 
9365 // Rotate Left by variable
9366 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9367 %{
9368   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9369 
9370   expand %{
9371     rolL_rReg_CL(dst, shift, cr);
9372   %}
9373 %}
9374 
9375 // Rotate Left by variable
9376 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9377 %{
9378   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9379 
9380   expand %{
9381     rolL_rReg_CL(dst, shift, cr);
9382   %}
9383 %}
9384 
9385 // ROR expand
9386 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9387 %{
9388   effect(USE_DEF dst, KILL cr);
9389 
9390   format %{ "rorq    $dst" %}
9391   opcode(0xD1, 0x1); /* D1 /1 */
9392   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9393   ins_pipe(ialu_reg);
9394 %}
9395 
9396 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9397 %{
9398   effect(USE_DEF dst, USE shift, KILL cr);
9399 
9400   format %{ "rorq    $dst, $shift" %}
9401   opcode(0xC1, 0x1); /* C1 /1 ib */
9402   ins_encode(reg_opc_imm_wide(dst, shift));
9403   ins_pipe(ialu_reg);
9404 %}
9405 
9406 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9407 %{
9408   effect(USE_DEF dst, USE shift, KILL cr);
9409 
9410   format %{ "rorq    $dst, $shift" %}
9411   opcode(0xD3, 0x1); /* D3 /1 */
9412   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9413   ins_pipe(ialu_reg_reg);
9414 %}
9415 // end of ROR expand
9416 
9417 // Rotate Right by one
9418 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9419 %{
9420   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9421 
9422   expand %{
9423     rorL_rReg_imm1(dst, cr);
9424   %}
9425 %}
9426 
9427 // Rotate Right by 8-bit immediate
9428 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9429 %{
9430   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9431   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9432 
9433   expand %{
9434     rorL_rReg_imm8(dst, rshift, cr);
9435   %}
9436 %}
9437 
9438 // Rotate Right by variable
9439 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9440 %{
9441   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9442 
9443   expand %{
9444     rorL_rReg_CL(dst, shift, cr);
9445   %}
9446 %}
9447 
9448 // Rotate Right by variable
9449 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9450 %{
9451   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9452 
9453   expand %{
9454     rorL_rReg_CL(dst, shift, cr);
9455   %}
9456 %}
9457 
9458 // Logical Instructions
9459 
9460 // Integer Logical Instructions
9461 
9462 // And Instructions
9463 // And Register with Register
9464 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9465 %{
9466   match(Set dst (AndI dst src));
9467   effect(KILL cr);
9468 
9469   format %{ "andl    $dst, $src\t# int" %}
9470   opcode(0x23);
9471   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9472   ins_pipe(ialu_reg_reg);
9473 %}
9474 
9475 // And Register with Immediate 255
9476 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9477 %{
9478   match(Set dst (AndI dst src));
9479 
9480   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9481   opcode(0x0F, 0xB6);
9482   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9483   ins_pipe(ialu_reg);
9484 %}
9485 
9486 // And Register with Immediate 255 and promote to long
9487 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9488 %{
9489   match(Set dst (ConvI2L (AndI src mask)));
9490 
9491   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9492   opcode(0x0F, 0xB6);
9493   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9494   ins_pipe(ialu_reg);
9495 %}
9496 
9497 // And Register with Immediate 65535
9498 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9499 %{
9500   match(Set dst (AndI dst src));
9501 
9502   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9503   opcode(0x0F, 0xB7);
9504   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9505   ins_pipe(ialu_reg);
9506 %}
9507 
9508 // And Register with Immediate 65535 and promote to long
9509 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9510 %{
9511   match(Set dst (ConvI2L (AndI src mask)));
9512 
9513   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9514   opcode(0x0F, 0xB7);
9515   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9516   ins_pipe(ialu_reg);
9517 %}
9518 
9519 // And Register with Immediate
9520 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9521 %{
9522   match(Set dst (AndI dst src));
9523   effect(KILL cr);
9524 
9525   format %{ "andl    $dst, $src\t# int" %}
9526   opcode(0x81, 0x04); /* Opcode 81 /4 */
9527   ins_encode(OpcSErm(dst, src), Con8or32(src));
9528   ins_pipe(ialu_reg);
9529 %}
9530 
9531 // And Register with Memory
9532 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9533 %{
9534   match(Set dst (AndI dst (LoadI src)));
9535   effect(KILL cr);
9536 
9537   ins_cost(125);
9538   format %{ "andl    $dst, $src\t# int" %}
9539   opcode(0x23);
9540   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9541   ins_pipe(ialu_reg_mem);
9542 %}
9543 
9544 // And Memory with Register
9545 instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9546 %{
9547   match(Set dst (StoreB dst (AndI (LoadB dst) src)));
9548   effect(KILL cr);
9549 
9550   ins_cost(150);
9551   format %{ "andb    $dst, $src\t# byte" %}
9552   opcode(0x20);
9553   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9554   ins_pipe(ialu_mem_reg);
9555 %}
9556 
9557 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9558 %{
9559   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9560   effect(KILL cr);
9561 
9562   ins_cost(150);
9563   format %{ "andl    $dst, $src\t# int" %}
9564   opcode(0x21); /* Opcode 21 /r */
9565   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9566   ins_pipe(ialu_mem_reg);
9567 %}
9568 
9569 // And Memory with Immediate
9570 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9571 %{
9572   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9573   effect(KILL cr);
9574 
9575   ins_cost(125);
9576   format %{ "andl    $dst, $src\t# int" %}
9577   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9578   ins_encode(REX_mem(dst), OpcSE(src),
9579              RM_opc_mem(secondary, dst), Con8or32(src));
9580   ins_pipe(ialu_mem_imm);
9581 %}
9582 
9583 // BMI1 instructions
9584 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9585   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9586   predicate(UseBMI1Instructions);
9587   effect(KILL cr);
9588 
9589   ins_cost(125);
9590   format %{ "andnl  $dst, $src1, $src2" %}
9591 
9592   ins_encode %{
9593     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9594   %}
9595   ins_pipe(ialu_reg_mem);
9596 %}
9597 
9598 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9599   match(Set dst (AndI (XorI src1 minus_1) src2));
9600   predicate(UseBMI1Instructions);
9601   effect(KILL cr);
9602 
9603   format %{ "andnl  $dst, $src1, $src2" %}
9604 
9605   ins_encode %{
9606     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9607   %}
9608   ins_pipe(ialu_reg);
9609 %}
9610 
9611 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9612   match(Set dst (AndI (SubI imm_zero src) src));
9613   predicate(UseBMI1Instructions);
9614   effect(KILL cr);
9615 
9616   format %{ "blsil  $dst, $src" %}
9617 
9618   ins_encode %{
9619     __ blsil($dst$$Register, $src$$Register);
9620   %}
9621   ins_pipe(ialu_reg);
9622 %}
9623 
9624 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9625   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9626   predicate(UseBMI1Instructions);
9627   effect(KILL cr);
9628 
9629   ins_cost(125);
9630   format %{ "blsil  $dst, $src" %}
9631 
9632   ins_encode %{
9633     __ blsil($dst$$Register, $src$$Address);
9634   %}
9635   ins_pipe(ialu_reg_mem);
9636 %}
9637 
9638 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9639 %{
9640   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9641   predicate(UseBMI1Instructions);
9642   effect(KILL cr);
9643 
9644   ins_cost(125);
9645   format %{ "blsmskl $dst, $src" %}
9646 
9647   ins_encode %{
9648     __ blsmskl($dst$$Register, $src$$Address);
9649   %}
9650   ins_pipe(ialu_reg_mem);
9651 %}
9652 
9653 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9654 %{
9655   match(Set dst (XorI (AddI src minus_1) src));
9656   predicate(UseBMI1Instructions);
9657   effect(KILL cr);
9658 
9659   format %{ "blsmskl $dst, $src" %}
9660 
9661   ins_encode %{
9662     __ blsmskl($dst$$Register, $src$$Register);
9663   %}
9664 
9665   ins_pipe(ialu_reg);
9666 %}
9667 
9668 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9669 %{
9670   match(Set dst (AndI (AddI src minus_1) src) );
9671   predicate(UseBMI1Instructions);
9672   effect(KILL cr);
9673 
9674   format %{ "blsrl  $dst, $src" %}
9675 
9676   ins_encode %{
9677     __ blsrl($dst$$Register, $src$$Register);
9678   %}
9679 
9680   ins_pipe(ialu_reg_mem);
9681 %}
9682 
9683 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9684 %{
9685   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9686   predicate(UseBMI1Instructions);
9687   effect(KILL cr);
9688 
9689   ins_cost(125);
9690   format %{ "blsrl  $dst, $src" %}
9691 
9692   ins_encode %{
9693     __ blsrl($dst$$Register, $src$$Address);
9694   %}
9695 
9696   ins_pipe(ialu_reg);
9697 %}
9698 
9699 // Or Instructions
9700 // Or Register with Register
9701 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9702 %{
9703   match(Set dst (OrI dst src));
9704   effect(KILL cr);
9705 
9706   format %{ "orl     $dst, $src\t# int" %}
9707   opcode(0x0B);
9708   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9709   ins_pipe(ialu_reg_reg);
9710 %}
9711 
9712 // Or Register with Immediate
9713 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9714 %{
9715   match(Set dst (OrI dst src));
9716   effect(KILL cr);
9717 
9718   format %{ "orl     $dst, $src\t# int" %}
9719   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9720   ins_encode(OpcSErm(dst, src), Con8or32(src));
9721   ins_pipe(ialu_reg);
9722 %}
9723 
9724 // Or Register with Memory
9725 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9726 %{
9727   match(Set dst (OrI dst (LoadI src)));
9728   effect(KILL cr);
9729 
9730   ins_cost(125);
9731   format %{ "orl     $dst, $src\t# int" %}
9732   opcode(0x0B);
9733   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9734   ins_pipe(ialu_reg_mem);
9735 %}
9736 
9737 // Or Memory with Register
9738 instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9739 %{
9740   match(Set dst (StoreB dst (OrI (LoadB dst) src)));
9741   effect(KILL cr);
9742 
9743   ins_cost(150);
9744   format %{ "orb    $dst, $src\t# byte" %}
9745   opcode(0x08);
9746   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9747   ins_pipe(ialu_mem_reg);
9748 %}
9749 
9750 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9751 %{
9752   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9753   effect(KILL cr);
9754 
9755   ins_cost(150);
9756   format %{ "orl     $dst, $src\t# int" %}
9757   opcode(0x09); /* Opcode 09 /r */
9758   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9759   ins_pipe(ialu_mem_reg);
9760 %}
9761 
9762 // Or Memory with Immediate
9763 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9764 %{
9765   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9766   effect(KILL cr);
9767 
9768   ins_cost(125);
9769   format %{ "orl     $dst, $src\t# int" %}
9770   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9771   ins_encode(REX_mem(dst), OpcSE(src),
9772              RM_opc_mem(secondary, dst), Con8or32(src));
9773   ins_pipe(ialu_mem_imm);
9774 %}
9775 
9776 // Xor Instructions
9777 // Xor Register with Register
9778 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9779 %{
9780   match(Set dst (XorI dst src));
9781   effect(KILL cr);
9782 
9783   format %{ "xorl    $dst, $src\t# int" %}
9784   opcode(0x33);
9785   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9786   ins_pipe(ialu_reg_reg);
9787 %}
9788 
9789 // Xor Register with Immediate -1
9790 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9791   match(Set dst (XorI dst imm));
9792 
9793   format %{ "not    $dst" %}
9794   ins_encode %{
9795      __ notl($dst$$Register);
9796   %}
9797   ins_pipe(ialu_reg);
9798 %}
9799 
9800 // Xor Register with Immediate
9801 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9802 %{
9803   match(Set dst (XorI dst src));
9804   effect(KILL cr);
9805 
9806   format %{ "xorl    $dst, $src\t# int" %}
9807   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9808   ins_encode(OpcSErm(dst, src), Con8or32(src));
9809   ins_pipe(ialu_reg);
9810 %}
9811 
9812 // Xor Register with Memory
9813 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9814 %{
9815   match(Set dst (XorI dst (LoadI src)));
9816   effect(KILL cr);
9817 
9818   ins_cost(125);
9819   format %{ "xorl    $dst, $src\t# int" %}
9820   opcode(0x33);
9821   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9822   ins_pipe(ialu_reg_mem);
9823 %}
9824 
9825 // Xor Memory with Register
9826 instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9827 %{
9828   match(Set dst (StoreB dst (XorI (LoadB dst) src)));
9829   effect(KILL cr);
9830 
9831   ins_cost(150);
9832   format %{ "xorb    $dst, $src\t# byte" %}
9833   opcode(0x30);
9834   ins_encode(REX_breg_mem(src, dst), OpcP, reg_mem(src, dst));
9835   ins_pipe(ialu_mem_reg);
9836 %}
9837 
9838 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9839 %{
9840   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9841   effect(KILL cr);
9842 
9843   ins_cost(150);
9844   format %{ "xorl    $dst, $src\t# int" %}
9845   opcode(0x31); /* Opcode 31 /r */
9846   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9847   ins_pipe(ialu_mem_reg);
9848 %}
9849 
9850 // Xor Memory with Immediate
9851 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9852 %{
9853   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9854   effect(KILL cr);
9855 
9856   ins_cost(125);
9857   format %{ "xorl    $dst, $src\t# int" %}
9858   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9859   ins_encode(REX_mem(dst), OpcSE(src),
9860              RM_opc_mem(secondary, dst), Con8or32(src));
9861   ins_pipe(ialu_mem_imm);
9862 %}
9863 
9864 
9865 // Long Logical Instructions
9866 
9867 // And Instructions
9868 // And Register with Register
9869 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9870 %{
9871   match(Set dst (AndL dst src));
9872   effect(KILL cr);
9873 
9874   format %{ "andq    $dst, $src\t# long" %}
9875   opcode(0x23);
9876   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9877   ins_pipe(ialu_reg_reg);
9878 %}
9879 
9880 // And Register with Immediate 255
9881 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9882 %{
9883   match(Set dst (AndL dst src));
9884 
9885   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9886   opcode(0x0F, 0xB6);
9887   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9888   ins_pipe(ialu_reg);
9889 %}
9890 
9891 // And Register with Immediate 65535
9892 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9893 %{
9894   match(Set dst (AndL dst src));
9895 
9896   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9897   opcode(0x0F, 0xB7);
9898   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9899   ins_pipe(ialu_reg);
9900 %}
9901 
9902 // And Register with Immediate
9903 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9904 %{
9905   match(Set dst (AndL dst src));
9906   effect(KILL cr);
9907 
9908   format %{ "andq    $dst, $src\t# long" %}
9909   opcode(0x81, 0x04); /* Opcode 81 /4 */
9910   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9911   ins_pipe(ialu_reg);
9912 %}
9913 
9914 // And Register with Memory
9915 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9916 %{
9917   match(Set dst (AndL dst (LoadL src)));
9918   effect(KILL cr);
9919 
9920   ins_cost(125);
9921   format %{ "andq    $dst, $src\t# long" %}
9922   opcode(0x23);
9923   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9924   ins_pipe(ialu_reg_mem);
9925 %}
9926 
9927 // And Memory with Register
9928 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9929 %{
9930   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9931   effect(KILL cr);
9932 
9933   ins_cost(150);
9934   format %{ "andq    $dst, $src\t# long" %}
9935   opcode(0x21); /* Opcode 21 /r */
9936   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9937   ins_pipe(ialu_mem_reg);
9938 %}
9939 
9940 // And Memory with Immediate
9941 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9942 %{
9943   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9944   effect(KILL cr);
9945 
9946   ins_cost(125);
9947   format %{ "andq    $dst, $src\t# long" %}
9948   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9949   ins_encode(REX_mem_wide(dst), OpcSE(src),
9950              RM_opc_mem(secondary, dst), Con8or32(src));
9951   ins_pipe(ialu_mem_imm);
9952 %}
9953 
9954 // BMI1 instructions
9955 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9956   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9957   predicate(UseBMI1Instructions);
9958   effect(KILL cr);
9959 
9960   ins_cost(125);
9961   format %{ "andnq  $dst, $src1, $src2" %}
9962 
9963   ins_encode %{
9964     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9965   %}
9966   ins_pipe(ialu_reg_mem);
9967 %}
9968 
9969 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9970   match(Set dst (AndL (XorL src1 minus_1) src2));
9971   predicate(UseBMI1Instructions);
9972   effect(KILL cr);
9973 
9974   format %{ "andnq  $dst, $src1, $src2" %}
9975 
9976   ins_encode %{
9977   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9978   %}
9979   ins_pipe(ialu_reg_mem);
9980 %}
9981 
9982 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9983   match(Set dst (AndL (SubL imm_zero src) src));
9984   predicate(UseBMI1Instructions);
9985   effect(KILL cr);
9986 
9987   format %{ "blsiq  $dst, $src" %}
9988 
9989   ins_encode %{
9990     __ blsiq($dst$$Register, $src$$Register);
9991   %}
9992   ins_pipe(ialu_reg);
9993 %}
9994 
9995 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9996   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9997   predicate(UseBMI1Instructions);
9998   effect(KILL cr);
9999 
10000   ins_cost(125);
10001   format %{ "blsiq  $dst, $src" %}
10002 
10003   ins_encode %{
10004     __ blsiq($dst$$Register, $src$$Address);
10005   %}
10006   ins_pipe(ialu_reg_mem);
10007 %}
10008 
10009 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10010 %{
10011   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
10012   predicate(UseBMI1Instructions);
10013   effect(KILL cr);
10014 
10015   ins_cost(125);
10016   format %{ "blsmskq $dst, $src" %}
10017 
10018   ins_encode %{
10019     __ blsmskq($dst$$Register, $src$$Address);
10020   %}
10021   ins_pipe(ialu_reg_mem);
10022 %}
10023 
10024 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10025 %{
10026   match(Set dst (XorL (AddL src minus_1) src));
10027   predicate(UseBMI1Instructions);
10028   effect(KILL cr);
10029 
10030   format %{ "blsmskq $dst, $src" %}
10031 
10032   ins_encode %{
10033     __ blsmskq($dst$$Register, $src$$Register);
10034   %}
10035 
10036   ins_pipe(ialu_reg);
10037 %}
10038 
10039 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
10040 %{
10041   match(Set dst (AndL (AddL src minus_1) src) );
10042   predicate(UseBMI1Instructions);
10043   effect(KILL cr);
10044 
10045   format %{ "blsrq  $dst, $src" %}
10046 
10047   ins_encode %{
10048     __ blsrq($dst$$Register, $src$$Register);
10049   %}
10050 
10051   ins_pipe(ialu_reg);
10052 %}
10053 
10054 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
10055 %{
10056   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
10057   predicate(UseBMI1Instructions);
10058   effect(KILL cr);
10059 
10060   ins_cost(125);
10061   format %{ "blsrq  $dst, $src" %}
10062 
10063   ins_encode %{
10064     __ blsrq($dst$$Register, $src$$Address);
10065   %}
10066 
10067   ins_pipe(ialu_reg);
10068 %}
10069 
10070 // Or Instructions
10071 // Or Register with Register
10072 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10073 %{
10074   match(Set dst (OrL dst src));
10075   effect(KILL cr);
10076 
10077   format %{ "orq     $dst, $src\t# long" %}
10078   opcode(0x0B);
10079   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10080   ins_pipe(ialu_reg_reg);
10081 %}
10082 
10083 // Use any_RegP to match R15 (TLS register) without spilling.
10084 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
10085   match(Set dst (OrL dst (CastP2X src)));
10086   effect(KILL cr);
10087 
10088   format %{ "orq     $dst, $src\t# long" %}
10089   opcode(0x0B);
10090   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10091   ins_pipe(ialu_reg_reg);
10092 %}
10093 
10094 
10095 // Or Register with Immediate
10096 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10097 %{
10098   match(Set dst (OrL dst src));
10099   effect(KILL cr);
10100 
10101   format %{ "orq     $dst, $src\t# long" %}
10102   opcode(0x81, 0x01); /* Opcode 81 /1 id */
10103   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10104   ins_pipe(ialu_reg);
10105 %}
10106 
10107 // Or Register with Memory
10108 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10109 %{
10110   match(Set dst (OrL dst (LoadL src)));
10111   effect(KILL cr);
10112 
10113   ins_cost(125);
10114   format %{ "orq     $dst, $src\t# long" %}
10115   opcode(0x0B);
10116   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10117   ins_pipe(ialu_reg_mem);
10118 %}
10119 
10120 // Or Memory with Register
10121 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10122 %{
10123   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10124   effect(KILL cr);
10125 
10126   ins_cost(150);
10127   format %{ "orq     $dst, $src\t# long" %}
10128   opcode(0x09); /* Opcode 09 /r */
10129   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10130   ins_pipe(ialu_mem_reg);
10131 %}
10132 
10133 // Or Memory with Immediate
10134 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10135 %{
10136   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
10137   effect(KILL cr);
10138 
10139   ins_cost(125);
10140   format %{ "orq     $dst, $src\t# long" %}
10141   opcode(0x81, 0x1); /* Opcode 81 /1 id */
10142   ins_encode(REX_mem_wide(dst), OpcSE(src),
10143              RM_opc_mem(secondary, dst), Con8or32(src));
10144   ins_pipe(ialu_mem_imm);
10145 %}
10146 
10147 // Xor Instructions
10148 // Xor Register with Register
10149 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
10150 %{
10151   match(Set dst (XorL dst src));
10152   effect(KILL cr);
10153 
10154   format %{ "xorq    $dst, $src\t# long" %}
10155   opcode(0x33);
10156   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
10157   ins_pipe(ialu_reg_reg);
10158 %}
10159 
10160 // Xor Register with Immediate -1
10161 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
10162   match(Set dst (XorL dst imm));
10163 
10164   format %{ "notq   $dst" %}
10165   ins_encode %{
10166      __ notq($dst$$Register);
10167   %}
10168   ins_pipe(ialu_reg);
10169 %}
10170 
10171 // Xor Register with Immediate
10172 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
10173 %{
10174   match(Set dst (XorL dst src));
10175   effect(KILL cr);
10176 
10177   format %{ "xorq    $dst, $src\t# long" %}
10178   opcode(0x81, 0x06); /* Opcode 81 /6 id */
10179   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
10180   ins_pipe(ialu_reg);
10181 %}
10182 
10183 // Xor Register with Memory
10184 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
10185 %{
10186   match(Set dst (XorL dst (LoadL src)));
10187   effect(KILL cr);
10188 
10189   ins_cost(125);
10190   format %{ "xorq    $dst, $src\t# long" %}
10191   opcode(0x33);
10192   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
10193   ins_pipe(ialu_reg_mem);
10194 %}
10195 
10196 // Xor Memory with Register
10197 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
10198 %{
10199   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10200   effect(KILL cr);
10201 
10202   ins_cost(150);
10203   format %{ "xorq    $dst, $src\t# long" %}
10204   opcode(0x31); /* Opcode 31 /r */
10205   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
10206   ins_pipe(ialu_mem_reg);
10207 %}
10208 
10209 // Xor Memory with Immediate
10210 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
10211 %{
10212   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
10213   effect(KILL cr);
10214 
10215   ins_cost(125);
10216   format %{ "xorq    $dst, $src\t# long" %}
10217   opcode(0x81, 0x6); /* Opcode 81 /6 id */
10218   ins_encode(REX_mem_wide(dst), OpcSE(src),
10219              RM_opc_mem(secondary, dst), Con8or32(src));
10220   ins_pipe(ialu_mem_imm);
10221 %}
10222 
10223 // Convert Int to Boolean
10224 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
10225 %{
10226   match(Set dst (Conv2B src));
10227   effect(KILL cr);
10228 
10229   format %{ "testl   $src, $src\t# ci2b\n\t"
10230             "setnz   $dst\n\t"
10231             "movzbl  $dst, $dst" %}
10232   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
10233              setNZ_reg(dst),
10234              REX_reg_breg(dst, dst), // movzbl
10235              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10236   ins_pipe(pipe_slow); // XXX
10237 %}
10238 
10239 // Convert Pointer to Boolean
10240 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
10241 %{
10242   match(Set dst (Conv2B src));
10243   effect(KILL cr);
10244 
10245   format %{ "testq   $src, $src\t# cp2b\n\t"
10246             "setnz   $dst\n\t"
10247             "movzbl  $dst, $dst" %}
10248   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
10249              setNZ_reg(dst),
10250              REX_reg_breg(dst, dst), // movzbl
10251              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
10252   ins_pipe(pipe_slow); // XXX
10253 %}
10254 
10255 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
10256 %{
10257   match(Set dst (CmpLTMask p q));
10258   effect(KILL cr);
10259 
10260   ins_cost(400);
10261   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
10262             "setlt   $dst\n\t"
10263             "movzbl  $dst, $dst\n\t"
10264             "negl    $dst" %}
10265   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
10266              setLT_reg(dst),
10267              REX_reg_breg(dst, dst), // movzbl
10268              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
10269              neg_reg(dst));
10270   ins_pipe(pipe_slow);
10271 %}
10272 
10273 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
10274 %{
10275   match(Set dst (CmpLTMask dst zero));
10276   effect(KILL cr);
10277 
10278   ins_cost(100);
10279   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
10280   ins_encode %{
10281   __ sarl($dst$$Register, 31);
10282   %}
10283   ins_pipe(ialu_reg);
10284 %}
10285 
10286 /* Better to save a register than avoid a branch */
10287 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10288 %{
10289   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
10290   effect(KILL cr);
10291   ins_cost(300);
10292   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
10293             "jge    done\n\t"
10294             "addl   $p,$y\n"
10295             "done:  " %}
10296   ins_encode %{
10297     Register Rp = $p$$Register;
10298     Register Rq = $q$$Register;
10299     Register Ry = $y$$Register;
10300     Label done;
10301     __ subl(Rp, Rq);
10302     __ jccb(Assembler::greaterEqual, done);
10303     __ addl(Rp, Ry);
10304     __ bind(done);
10305   %}
10306   ins_pipe(pipe_cmplt);
10307 %}
10308 
10309 /* Better to save a register than avoid a branch */
10310 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
10311 %{
10312   match(Set y (AndI (CmpLTMask p q) y));
10313   effect(KILL cr);
10314 
10315   ins_cost(300);
10316 
10317   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
10318             "jlt      done\n\t"
10319             "xorl     $y, $y\n"
10320             "done:  " %}
10321   ins_encode %{
10322     Register Rp = $p$$Register;
10323     Register Rq = $q$$Register;
10324     Register Ry = $y$$Register;
10325     Label done;
10326     __ cmpl(Rp, Rq);
10327     __ jccb(Assembler::less, done);
10328     __ xorl(Ry, Ry);
10329     __ bind(done);
10330   %}
10331   ins_pipe(pipe_cmplt);
10332 %}
10333 
10334 
10335 //---------- FP Instructions------------------------------------------------
10336 
10337 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10338 %{
10339   match(Set cr (CmpF src1 src2));
10340 
10341   ins_cost(145);
10342   format %{ "ucomiss $src1, $src2\n\t"
10343             "jnp,s   exit\n\t"
10344             "pushfq\t# saw NaN, set CF\n\t"
10345             "andq    [rsp], #0xffffff2b\n\t"
10346             "popfq\n"
10347     "exit:" %}
10348   ins_encode %{
10349     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10350     emit_cmpfp_fixup(_masm);
10351   %}
10352   ins_pipe(pipe_slow);
10353 %}
10354 
10355 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10356   match(Set cr (CmpF src1 src2));
10357 
10358   ins_cost(100);
10359   format %{ "ucomiss $src1, $src2" %}
10360   ins_encode %{
10361     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10362   %}
10363   ins_pipe(pipe_slow);
10364 %}
10365 
10366 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10367 %{
10368   match(Set cr (CmpF src1 (LoadF src2)));
10369 
10370   ins_cost(145);
10371   format %{ "ucomiss $src1, $src2\n\t"
10372             "jnp,s   exit\n\t"
10373             "pushfq\t# saw NaN, set CF\n\t"
10374             "andq    [rsp], #0xffffff2b\n\t"
10375             "popfq\n"
10376     "exit:" %}
10377   ins_encode %{
10378     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10379     emit_cmpfp_fixup(_masm);
10380   %}
10381   ins_pipe(pipe_slow);
10382 %}
10383 
10384 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10385   match(Set cr (CmpF src1 (LoadF src2)));
10386 
10387   ins_cost(100);
10388   format %{ "ucomiss $src1, $src2" %}
10389   ins_encode %{
10390     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10391   %}
10392   ins_pipe(pipe_slow);
10393 %}
10394 
10395 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10396   match(Set cr (CmpF src con));
10397 
10398   ins_cost(145);
10399   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10400             "jnp,s   exit\n\t"
10401             "pushfq\t# saw NaN, set CF\n\t"
10402             "andq    [rsp], #0xffffff2b\n\t"
10403             "popfq\n"
10404     "exit:" %}
10405   ins_encode %{
10406     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10407     emit_cmpfp_fixup(_masm);
10408   %}
10409   ins_pipe(pipe_slow);
10410 %}
10411 
10412 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10413   match(Set cr (CmpF src con));
10414   ins_cost(100);
10415   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10416   ins_encode %{
10417     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10418   %}
10419   ins_pipe(pipe_slow);
10420 %}
10421 
10422 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10423 %{
10424   match(Set cr (CmpD src1 src2));
10425 
10426   ins_cost(145);
10427   format %{ "ucomisd $src1, $src2\n\t"
10428             "jnp,s   exit\n\t"
10429             "pushfq\t# saw NaN, set CF\n\t"
10430             "andq    [rsp], #0xffffff2b\n\t"
10431             "popfq\n"
10432     "exit:" %}
10433   ins_encode %{
10434     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10435     emit_cmpfp_fixup(_masm);
10436   %}
10437   ins_pipe(pipe_slow);
10438 %}
10439 
10440 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10441   match(Set cr (CmpD src1 src2));
10442 
10443   ins_cost(100);
10444   format %{ "ucomisd $src1, $src2 test" %}
10445   ins_encode %{
10446     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10447   %}
10448   ins_pipe(pipe_slow);
10449 %}
10450 
10451 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10452 %{
10453   match(Set cr (CmpD src1 (LoadD src2)));
10454 
10455   ins_cost(145);
10456   format %{ "ucomisd $src1, $src2\n\t"
10457             "jnp,s   exit\n\t"
10458             "pushfq\t# saw NaN, set CF\n\t"
10459             "andq    [rsp], #0xffffff2b\n\t"
10460             "popfq\n"
10461     "exit:" %}
10462   ins_encode %{
10463     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10464     emit_cmpfp_fixup(_masm);
10465   %}
10466   ins_pipe(pipe_slow);
10467 %}
10468 
10469 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10470   match(Set cr (CmpD src1 (LoadD src2)));
10471 
10472   ins_cost(100);
10473   format %{ "ucomisd $src1, $src2" %}
10474   ins_encode %{
10475     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10476   %}
10477   ins_pipe(pipe_slow);
10478 %}
10479 
10480 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10481   match(Set cr (CmpD src con));
10482 
10483   ins_cost(145);
10484   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10485             "jnp,s   exit\n\t"
10486             "pushfq\t# saw NaN, set CF\n\t"
10487             "andq    [rsp], #0xffffff2b\n\t"
10488             "popfq\n"
10489     "exit:" %}
10490   ins_encode %{
10491     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10492     emit_cmpfp_fixup(_masm);
10493   %}
10494   ins_pipe(pipe_slow);
10495 %}
10496 
10497 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10498   match(Set cr (CmpD src con));
10499   ins_cost(100);
10500   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10501   ins_encode %{
10502     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10503   %}
10504   ins_pipe(pipe_slow);
10505 %}
10506 
10507 // Compare into -1,0,1
10508 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10509 %{
10510   match(Set dst (CmpF3 src1 src2));
10511   effect(KILL cr);
10512 
10513   ins_cost(275);
10514   format %{ "ucomiss $src1, $src2\n\t"
10515             "movl    $dst, #-1\n\t"
10516             "jp,s    done\n\t"
10517             "jb,s    done\n\t"
10518             "setne   $dst\n\t"
10519             "movzbl  $dst, $dst\n"
10520     "done:" %}
10521   ins_encode %{
10522     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10523     emit_cmpfp3(_masm, $dst$$Register);
10524   %}
10525   ins_pipe(pipe_slow);
10526 %}
10527 
10528 // Compare into -1,0,1
10529 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10530 %{
10531   match(Set dst (CmpF3 src1 (LoadF src2)));
10532   effect(KILL cr);
10533 
10534   ins_cost(275);
10535   format %{ "ucomiss $src1, $src2\n\t"
10536             "movl    $dst, #-1\n\t"
10537             "jp,s    done\n\t"
10538             "jb,s    done\n\t"
10539             "setne   $dst\n\t"
10540             "movzbl  $dst, $dst\n"
10541     "done:" %}
10542   ins_encode %{
10543     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10544     emit_cmpfp3(_masm, $dst$$Register);
10545   %}
10546   ins_pipe(pipe_slow);
10547 %}
10548 
10549 // Compare into -1,0,1
10550 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10551   match(Set dst (CmpF3 src con));
10552   effect(KILL cr);
10553 
10554   ins_cost(275);
10555   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10556             "movl    $dst, #-1\n\t"
10557             "jp,s    done\n\t"
10558             "jb,s    done\n\t"
10559             "setne   $dst\n\t"
10560             "movzbl  $dst, $dst\n"
10561     "done:" %}
10562   ins_encode %{
10563     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10564     emit_cmpfp3(_masm, $dst$$Register);
10565   %}
10566   ins_pipe(pipe_slow);
10567 %}
10568 
10569 // Compare into -1,0,1
10570 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10571 %{
10572   match(Set dst (CmpD3 src1 src2));
10573   effect(KILL cr);
10574 
10575   ins_cost(275);
10576   format %{ "ucomisd $src1, $src2\n\t"
10577             "movl    $dst, #-1\n\t"
10578             "jp,s    done\n\t"
10579             "jb,s    done\n\t"
10580             "setne   $dst\n\t"
10581             "movzbl  $dst, $dst\n"
10582     "done:" %}
10583   ins_encode %{
10584     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10585     emit_cmpfp3(_masm, $dst$$Register);
10586   %}
10587   ins_pipe(pipe_slow);
10588 %}
10589 
10590 // Compare into -1,0,1
10591 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10592 %{
10593   match(Set dst (CmpD3 src1 (LoadD src2)));
10594   effect(KILL cr);
10595 
10596   ins_cost(275);
10597   format %{ "ucomisd $src1, $src2\n\t"
10598             "movl    $dst, #-1\n\t"
10599             "jp,s    done\n\t"
10600             "jb,s    done\n\t"
10601             "setne   $dst\n\t"
10602             "movzbl  $dst, $dst\n"
10603     "done:" %}
10604   ins_encode %{
10605     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10606     emit_cmpfp3(_masm, $dst$$Register);
10607   %}
10608   ins_pipe(pipe_slow);
10609 %}
10610 
10611 // Compare into -1,0,1
10612 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10613   match(Set dst (CmpD3 src con));
10614   effect(KILL cr);
10615 
10616   ins_cost(275);
10617   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10618             "movl    $dst, #-1\n\t"
10619             "jp,s    done\n\t"
10620             "jb,s    done\n\t"
10621             "setne   $dst\n\t"
10622             "movzbl  $dst, $dst\n"
10623     "done:" %}
10624   ins_encode %{
10625     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10626     emit_cmpfp3(_masm, $dst$$Register);
10627   %}
10628   ins_pipe(pipe_slow);
10629 %}
10630 
10631 //----------Arithmetic Conversion Instructions---------------------------------
10632 
10633 instruct roundFloat_nop(regF dst)
10634 %{
10635   match(Set dst (RoundFloat dst));
10636 
10637   ins_cost(0);
10638   ins_encode();
10639   ins_pipe(empty);
10640 %}
10641 
10642 instruct roundDouble_nop(regD dst)
10643 %{
10644   match(Set dst (RoundDouble dst));
10645 
10646   ins_cost(0);
10647   ins_encode();
10648   ins_pipe(empty);
10649 %}
10650 
10651 instruct convF2D_reg_reg(regD dst, regF src)
10652 %{
10653   match(Set dst (ConvF2D src));
10654 
10655   format %{ "cvtss2sd $dst, $src" %}
10656   ins_encode %{
10657     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10658   %}
10659   ins_pipe(pipe_slow); // XXX
10660 %}
10661 
10662 instruct convF2D_reg_mem(regD dst, memory src)
10663 %{
10664   match(Set dst (ConvF2D (LoadF src)));
10665 
10666   format %{ "cvtss2sd $dst, $src" %}
10667   ins_encode %{
10668     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10669   %}
10670   ins_pipe(pipe_slow); // XXX
10671 %}
10672 
10673 instruct convD2F_reg_reg(regF dst, regD src)
10674 %{
10675   match(Set dst (ConvD2F src));
10676 
10677   format %{ "cvtsd2ss $dst, $src" %}
10678   ins_encode %{
10679     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10680   %}
10681   ins_pipe(pipe_slow); // XXX
10682 %}
10683 
10684 instruct convD2F_reg_mem(regF dst, memory src)
10685 %{
10686   match(Set dst (ConvD2F (LoadD src)));
10687 
10688   format %{ "cvtsd2ss $dst, $src" %}
10689   ins_encode %{
10690     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10691   %}
10692   ins_pipe(pipe_slow); // XXX
10693 %}
10694 
10695 // XXX do mem variants
10696 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10697 %{
10698   match(Set dst (ConvF2I src));
10699   effect(KILL cr);
10700 
10701   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10702             "cmpl    $dst, #0x80000000\n\t"
10703             "jne,s   done\n\t"
10704             "subq    rsp, #8\n\t"
10705             "movss   [rsp], $src\n\t"
10706             "call    f2i_fixup\n\t"
10707             "popq    $dst\n"
10708     "done:   "%}
10709   ins_encode %{
10710     Label done;
10711     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10712     __ cmpl($dst$$Register, 0x80000000);
10713     __ jccb(Assembler::notEqual, done);
10714     __ subptr(rsp, 8);
10715     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10716     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10717     __ pop($dst$$Register);
10718     __ bind(done);
10719   %}
10720   ins_pipe(pipe_slow);
10721 %}
10722 
10723 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10724 %{
10725   match(Set dst (ConvF2L src));
10726   effect(KILL cr);
10727 
10728   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10729             "cmpq    $dst, [0x8000000000000000]\n\t"
10730             "jne,s   done\n\t"
10731             "subq    rsp, #8\n\t"
10732             "movss   [rsp], $src\n\t"
10733             "call    f2l_fixup\n\t"
10734             "popq    $dst\n"
10735     "done:   "%}
10736   ins_encode %{
10737     Label done;
10738     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10739     __ cmp64($dst$$Register,
10740              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10741     __ jccb(Assembler::notEqual, done);
10742     __ subptr(rsp, 8);
10743     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10744     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10745     __ pop($dst$$Register);
10746     __ bind(done);
10747   %}
10748   ins_pipe(pipe_slow);
10749 %}
10750 
10751 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10752 %{
10753   match(Set dst (ConvD2I src));
10754   effect(KILL cr);
10755 
10756   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10757             "cmpl    $dst, #0x80000000\n\t"
10758             "jne,s   done\n\t"
10759             "subq    rsp, #8\n\t"
10760             "movsd   [rsp], $src\n\t"
10761             "call    d2i_fixup\n\t"
10762             "popq    $dst\n"
10763     "done:   "%}
10764   ins_encode %{
10765     Label done;
10766     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10767     __ cmpl($dst$$Register, 0x80000000);
10768     __ jccb(Assembler::notEqual, done);
10769     __ subptr(rsp, 8);
10770     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10771     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10772     __ pop($dst$$Register);
10773     __ bind(done);
10774   %}
10775   ins_pipe(pipe_slow);
10776 %}
10777 
10778 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10779 %{
10780   match(Set dst (ConvD2L src));
10781   effect(KILL cr);
10782 
10783   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10784             "cmpq    $dst, [0x8000000000000000]\n\t"
10785             "jne,s   done\n\t"
10786             "subq    rsp, #8\n\t"
10787             "movsd   [rsp], $src\n\t"
10788             "call    d2l_fixup\n\t"
10789             "popq    $dst\n"
10790     "done:   "%}
10791   ins_encode %{
10792     Label done;
10793     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10794     __ cmp64($dst$$Register,
10795              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10796     __ jccb(Assembler::notEqual, done);
10797     __ subptr(rsp, 8);
10798     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10799     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10800     __ pop($dst$$Register);
10801     __ bind(done);
10802   %}
10803   ins_pipe(pipe_slow);
10804 %}
10805 
10806 instruct convI2F_reg_reg(regF dst, rRegI src)
10807 %{
10808   predicate(!UseXmmI2F);
10809   match(Set dst (ConvI2F src));
10810 
10811   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10812   ins_encode %{
10813     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10814   %}
10815   ins_pipe(pipe_slow); // XXX
10816 %}
10817 
10818 instruct convI2F_reg_mem(regF dst, memory src)
10819 %{
10820   match(Set dst (ConvI2F (LoadI src)));
10821 
10822   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10823   ins_encode %{
10824     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10825   %}
10826   ins_pipe(pipe_slow); // XXX
10827 %}
10828 
10829 instruct convI2D_reg_reg(regD dst, rRegI src)
10830 %{
10831   predicate(!UseXmmI2D);
10832   match(Set dst (ConvI2D src));
10833 
10834   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10835   ins_encode %{
10836     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10837   %}
10838   ins_pipe(pipe_slow); // XXX
10839 %}
10840 
10841 instruct convI2D_reg_mem(regD dst, memory src)
10842 %{
10843   match(Set dst (ConvI2D (LoadI src)));
10844 
10845   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10846   ins_encode %{
10847     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10848   %}
10849   ins_pipe(pipe_slow); // XXX
10850 %}
10851 
10852 instruct convXI2F_reg(regF dst, rRegI src)
10853 %{
10854   predicate(UseXmmI2F);
10855   match(Set dst (ConvI2F src));
10856 
10857   format %{ "movdl $dst, $src\n\t"
10858             "cvtdq2psl $dst, $dst\t# i2f" %}
10859   ins_encode %{
10860     __ movdl($dst$$XMMRegister, $src$$Register);
10861     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10862   %}
10863   ins_pipe(pipe_slow); // XXX
10864 %}
10865 
10866 instruct convXI2D_reg(regD dst, rRegI src)
10867 %{
10868   predicate(UseXmmI2D);
10869   match(Set dst (ConvI2D src));
10870 
10871   format %{ "movdl $dst, $src\n\t"
10872             "cvtdq2pdl $dst, $dst\t# i2d" %}
10873   ins_encode %{
10874     __ movdl($dst$$XMMRegister, $src$$Register);
10875     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10876   %}
10877   ins_pipe(pipe_slow); // XXX
10878 %}
10879 
10880 instruct convL2F_reg_reg(regF dst, rRegL src)
10881 %{
10882   match(Set dst (ConvL2F src));
10883 
10884   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10885   ins_encode %{
10886     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10887   %}
10888   ins_pipe(pipe_slow); // XXX
10889 %}
10890 
10891 instruct convL2F_reg_mem(regF dst, memory src)
10892 %{
10893   match(Set dst (ConvL2F (LoadL src)));
10894 
10895   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10896   ins_encode %{
10897     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10898   %}
10899   ins_pipe(pipe_slow); // XXX
10900 %}
10901 
10902 instruct convL2D_reg_reg(regD dst, rRegL src)
10903 %{
10904   match(Set dst (ConvL2D src));
10905 
10906   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10907   ins_encode %{
10908     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10909   %}
10910   ins_pipe(pipe_slow); // XXX
10911 %}
10912 
10913 instruct convL2D_reg_mem(regD dst, memory src)
10914 %{
10915   match(Set dst (ConvL2D (LoadL src)));
10916 
10917   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10918   ins_encode %{
10919     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10920   %}
10921   ins_pipe(pipe_slow); // XXX
10922 %}
10923 
10924 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10925 %{
10926   match(Set dst (ConvI2L src));
10927 
10928   ins_cost(125);
10929   format %{ "movslq  $dst, $src\t# i2l" %}
10930   ins_encode %{
10931     __ movslq($dst$$Register, $src$$Register);
10932   %}
10933   ins_pipe(ialu_reg_reg);
10934 %}
10935 
10936 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10937 // %{
10938 //   match(Set dst (ConvI2L src));
10939 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10940 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10941 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10942 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10943 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10944 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10945 
10946 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10947 //   ins_encode(enc_copy(dst, src));
10948 // //   opcode(0x63); // needs REX.W
10949 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10950 //   ins_pipe(ialu_reg_reg);
10951 // %}
10952 
10953 // Zero-extend convert int to long
10954 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10955 %{
10956   match(Set dst (AndL (ConvI2L src) mask));
10957 
10958   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10959   ins_encode %{
10960     if ($dst$$reg != $src$$reg) {
10961       __ movl($dst$$Register, $src$$Register);
10962     }
10963   %}
10964   ins_pipe(ialu_reg_reg);
10965 %}
10966 
10967 // Zero-extend convert int to long
10968 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10969 %{
10970   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10971 
10972   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10973   ins_encode %{
10974     __ movl($dst$$Register, $src$$Address);
10975   %}
10976   ins_pipe(ialu_reg_mem);
10977 %}
10978 
10979 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10980 %{
10981   match(Set dst (AndL src mask));
10982 
10983   format %{ "movl    $dst, $src\t# zero-extend long" %}
10984   ins_encode %{
10985     __ movl($dst$$Register, $src$$Register);
10986   %}
10987   ins_pipe(ialu_reg_reg);
10988 %}
10989 
10990 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10991 %{
10992   match(Set dst (ConvL2I src));
10993 
10994   format %{ "movl    $dst, $src\t# l2i" %}
10995   ins_encode %{
10996     __ movl($dst$$Register, $src$$Register);
10997   %}
10998   ins_pipe(ialu_reg_reg);
10999 %}
11000 
11001 
11002 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
11003   match(Set dst (MoveF2I src));
11004   effect(DEF dst, USE src);
11005 
11006   ins_cost(125);
11007   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
11008   ins_encode %{
11009     __ movl($dst$$Register, Address(rsp, $src$$disp));
11010   %}
11011   ins_pipe(ialu_reg_mem);
11012 %}
11013 
11014 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
11015   match(Set dst (MoveI2F src));
11016   effect(DEF dst, USE src);
11017 
11018   ins_cost(125);
11019   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
11020   ins_encode %{
11021     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
11022   %}
11023   ins_pipe(pipe_slow);
11024 %}
11025 
11026 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
11027   match(Set dst (MoveD2L src));
11028   effect(DEF dst, USE src);
11029 
11030   ins_cost(125);
11031   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
11032   ins_encode %{
11033     __ movq($dst$$Register, Address(rsp, $src$$disp));
11034   %}
11035   ins_pipe(ialu_reg_mem);
11036 %}
11037 
11038 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
11039   predicate(!UseXmmLoadAndClearUpper);
11040   match(Set dst (MoveL2D src));
11041   effect(DEF dst, USE src);
11042 
11043   ins_cost(125);
11044   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
11045   ins_encode %{
11046     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11047   %}
11048   ins_pipe(pipe_slow);
11049 %}
11050 
11051 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
11052   predicate(UseXmmLoadAndClearUpper);
11053   match(Set dst (MoveL2D src));
11054   effect(DEF dst, USE src);
11055 
11056   ins_cost(125);
11057   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
11058   ins_encode %{
11059     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
11060   %}
11061   ins_pipe(pipe_slow);
11062 %}
11063 
11064 
11065 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
11066   match(Set dst (MoveF2I src));
11067   effect(DEF dst, USE src);
11068 
11069   ins_cost(95); // XXX
11070   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
11071   ins_encode %{
11072     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
11073   %}
11074   ins_pipe(pipe_slow);
11075 %}
11076 
11077 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
11078   match(Set dst (MoveI2F src));
11079   effect(DEF dst, USE src);
11080 
11081   ins_cost(100);
11082   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
11083   ins_encode %{
11084     __ movl(Address(rsp, $dst$$disp), $src$$Register);
11085   %}
11086   ins_pipe( ialu_mem_reg );
11087 %}
11088 
11089 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
11090   match(Set dst (MoveD2L src));
11091   effect(DEF dst, USE src);
11092 
11093   ins_cost(95); // XXX
11094   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
11095   ins_encode %{
11096     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
11097   %}
11098   ins_pipe(pipe_slow);
11099 %}
11100 
11101 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
11102   match(Set dst (MoveL2D src));
11103   effect(DEF dst, USE src);
11104 
11105   ins_cost(100);
11106   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
11107   ins_encode %{
11108     __ movq(Address(rsp, $dst$$disp), $src$$Register);
11109   %}
11110   ins_pipe(ialu_mem_reg);
11111 %}
11112 
11113 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
11114   match(Set dst (MoveF2I src));
11115   effect(DEF dst, USE src);
11116   ins_cost(85);
11117   format %{ "movd    $dst,$src\t# MoveF2I" %}
11118   ins_encode %{
11119     __ movdl($dst$$Register, $src$$XMMRegister);
11120   %}
11121   ins_pipe( pipe_slow );
11122 %}
11123 
11124 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
11125   match(Set dst (MoveD2L src));
11126   effect(DEF dst, USE src);
11127   ins_cost(85);
11128   format %{ "movd    $dst,$src\t# MoveD2L" %}
11129   ins_encode %{
11130     __ movdq($dst$$Register, $src$$XMMRegister);
11131   %}
11132   ins_pipe( pipe_slow );
11133 %}
11134 
11135 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
11136   match(Set dst (MoveI2F src));
11137   effect(DEF dst, USE src);
11138   ins_cost(100);
11139   format %{ "movd    $dst,$src\t# MoveI2F" %}
11140   ins_encode %{
11141     __ movdl($dst$$XMMRegister, $src$$Register);
11142   %}
11143   ins_pipe( pipe_slow );
11144 %}
11145 
11146 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
11147   match(Set dst (MoveL2D src));
11148   effect(DEF dst, USE src);
11149   ins_cost(100);
11150   format %{ "movd    $dst,$src\t# MoveL2D" %}
11151   ins_encode %{
11152      __ movdq($dst$$XMMRegister, $src$$Register);
11153   %}
11154   ins_pipe( pipe_slow );
11155 %}
11156 
11157 
11158 // =======================================================================
11159 // fast clearing of an array
11160 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11161                   Universe dummy, rFlagsReg cr)
11162 %{
11163   predicate(!((ClearArrayNode*)n)->is_large());
11164   match(Set dummy (ClearArray cnt base));
11165   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11166 
11167   format %{ $$template
11168     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11169     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
11170     $$emit$$"jg      LARGE\n\t"
11171     $$emit$$"dec     rcx\n\t"
11172     $$emit$$"js      DONE\t# Zero length\n\t"
11173     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
11174     $$emit$$"dec     rcx\n\t"
11175     $$emit$$"jge     LOOP\n\t"
11176     $$emit$$"jmp     DONE\n\t"
11177     $$emit$$"# LARGE:\n\t"
11178     if (UseFastStosb) {
11179        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11180        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
11181     } else if (UseXMMForObjInit) {
11182        $$emit$$"mov     rdi,rax\n\t"
11183        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11184        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11185        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11186        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11187        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11188        $$emit$$"add     0x40,rax\n\t"
11189        $$emit$$"# L_zero_64_bytes:\n\t"
11190        $$emit$$"sub     0x8,rcx\n\t"
11191        $$emit$$"jge     L_loop\n\t"
11192        $$emit$$"add     0x4,rcx\n\t"
11193        $$emit$$"jl      L_tail\n\t"
11194        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11195        $$emit$$"add     0x20,rax\n\t"
11196        $$emit$$"sub     0x4,rcx\n\t"
11197        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11198        $$emit$$"add     0x4,rcx\n\t"
11199        $$emit$$"jle     L_end\n\t"
11200        $$emit$$"dec     rcx\n\t"
11201        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11202        $$emit$$"vmovq   xmm0,(rax)\n\t"
11203        $$emit$$"add     0x8,rax\n\t"
11204        $$emit$$"dec     rcx\n\t"
11205        $$emit$$"jge     L_sloop\n\t"
11206        $$emit$$"# L_end:\n\t"
11207     } else {
11208        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
11209     }
11210     $$emit$$"# DONE"
11211   %}
11212   ins_encode %{
11213     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11214                  $tmp$$XMMRegister, false);
11215   %}
11216   ins_pipe(pipe_slow);
11217 %}
11218 
11219 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
11220                         Universe dummy, rFlagsReg cr)
11221 %{
11222   predicate(((ClearArrayNode*)n)->is_large());
11223   match(Set dummy (ClearArray cnt base));
11224   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
11225 
11226   format %{ $$template
11227     if (UseFastStosb) {
11228        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11229        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
11230        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
11231     } else if (UseXMMForObjInit) {
11232        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
11233        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
11234        $$emit$$"jmpq    L_zero_64_bytes\n\t"
11235        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
11236        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11237        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
11238        $$emit$$"add     0x40,rax\n\t"
11239        $$emit$$"# L_zero_64_bytes:\n\t"
11240        $$emit$$"sub     0x8,rcx\n\t"
11241        $$emit$$"jge     L_loop\n\t"
11242        $$emit$$"add     0x4,rcx\n\t"
11243        $$emit$$"jl      L_tail\n\t"
11244        $$emit$$"vmovdqu ymm0,(rax)\n\t"
11245        $$emit$$"add     0x20,rax\n\t"
11246        $$emit$$"sub     0x4,rcx\n\t"
11247        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
11248        $$emit$$"add     0x4,rcx\n\t"
11249        $$emit$$"jle     L_end\n\t"
11250        $$emit$$"dec     rcx\n\t"
11251        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
11252        $$emit$$"vmovq   xmm0,(rax)\n\t"
11253        $$emit$$"add     0x8,rax\n\t"
11254        $$emit$$"dec     rcx\n\t"
11255        $$emit$$"jge     L_sloop\n\t"
11256        $$emit$$"# L_end:\n\t"
11257     } else {
11258        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
11259        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
11260     }
11261   %}
11262   ins_encode %{
11263     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
11264                  $tmp$$XMMRegister, true);
11265   %}
11266   ins_pipe(pipe_slow);
11267 %}
11268 
11269 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11270                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11271 %{
11272   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
11273   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11274   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11275 
11276   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11277   ins_encode %{
11278     __ string_compare($str1$$Register, $str2$$Register,
11279                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11280                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
11281   %}
11282   ins_pipe( pipe_slow );
11283 %}
11284 
11285 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11286                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
11287 %{
11288   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
11289   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11290   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11291 
11292   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11293   ins_encode %{
11294     __ string_compare($str1$$Register, $str2$$Register,
11295                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11296                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
11297   %}
11298   ins_pipe( pipe_slow );
11299 %}
11300 
11301 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
11302                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11303 %{
11304   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
11305   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11306   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11307 
11308   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11309   ins_encode %{
11310     __ string_compare($str1$$Register, $str2$$Register,
11311                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
11312                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
11313   %}
11314   ins_pipe( pipe_slow );
11315 %}
11316 
11317 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11318                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11319 %{
11320   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11321   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11322   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11323 
11324   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11325   ins_encode %{
11326     __ string_compare($str2$$Register, $str1$$Register,
11327                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11328                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11329   %}
11330   ins_pipe( pipe_slow );
11331 %}
11332 
11333 // fast search of substring with known size.
11334 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11335                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11336 %{
11337   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11338   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11339   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11340 
11341   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11342   ins_encode %{
11343     int icnt2 = (int)$int_cnt2$$constant;
11344     if (icnt2 >= 16) {
11345       // IndexOf for constant substrings with size >= 16 elements
11346       // which don't need to be loaded through stack.
11347       __ string_indexofC8($str1$$Register, $str2$$Register,
11348                           $cnt1$$Register, $cnt2$$Register,
11349                           icnt2, $result$$Register,
11350                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11351     } else {
11352       // Small strings are loaded through stack if they cross page boundary.
11353       __ string_indexof($str1$$Register, $str2$$Register,
11354                         $cnt1$$Register, $cnt2$$Register,
11355                         icnt2, $result$$Register,
11356                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11357     }
11358   %}
11359   ins_pipe( pipe_slow );
11360 %}
11361 
11362 // fast search of substring with known size.
11363 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11364                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11365 %{
11366   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11367   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11368   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11369 
11370   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11371   ins_encode %{
11372     int icnt2 = (int)$int_cnt2$$constant;
11373     if (icnt2 >= 8) {
11374       // IndexOf for constant substrings with size >= 8 elements
11375       // which don't need to be loaded through stack.
11376       __ string_indexofC8($str1$$Register, $str2$$Register,
11377                           $cnt1$$Register, $cnt2$$Register,
11378                           icnt2, $result$$Register,
11379                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11380     } else {
11381       // Small strings are loaded through stack if they cross page boundary.
11382       __ string_indexof($str1$$Register, $str2$$Register,
11383                         $cnt1$$Register, $cnt2$$Register,
11384                         icnt2, $result$$Register,
11385                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11386     }
11387   %}
11388   ins_pipe( pipe_slow );
11389 %}
11390 
11391 // fast search of substring with known size.
11392 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11393                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11394 %{
11395   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11396   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11397   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11398 
11399   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11400   ins_encode %{
11401     int icnt2 = (int)$int_cnt2$$constant;
11402     if (icnt2 >= 8) {
11403       // IndexOf for constant substrings with size >= 8 elements
11404       // which don't need to be loaded through stack.
11405       __ string_indexofC8($str1$$Register, $str2$$Register,
11406                           $cnt1$$Register, $cnt2$$Register,
11407                           icnt2, $result$$Register,
11408                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11409     } else {
11410       // Small strings are loaded through stack if they cross page boundary.
11411       __ string_indexof($str1$$Register, $str2$$Register,
11412                         $cnt1$$Register, $cnt2$$Register,
11413                         icnt2, $result$$Register,
11414                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11415     }
11416   %}
11417   ins_pipe( pipe_slow );
11418 %}
11419 
11420 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11421                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11422 %{
11423   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11424   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11425   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11426 
11427   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11428   ins_encode %{
11429     __ string_indexof($str1$$Register, $str2$$Register,
11430                       $cnt1$$Register, $cnt2$$Register,
11431                       (-1), $result$$Register,
11432                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11433   %}
11434   ins_pipe( pipe_slow );
11435 %}
11436 
11437 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11438                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11439 %{
11440   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11441   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11442   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11443 
11444   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11445   ins_encode %{
11446     __ string_indexof($str1$$Register, $str2$$Register,
11447                       $cnt1$$Register, $cnt2$$Register,
11448                       (-1), $result$$Register,
11449                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11450   %}
11451   ins_pipe( pipe_slow );
11452 %}
11453 
11454 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11455                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11456 %{
11457   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11458   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11459   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11460 
11461   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11462   ins_encode %{
11463     __ string_indexof($str1$$Register, $str2$$Register,
11464                       $cnt1$$Register, $cnt2$$Register,
11465                       (-1), $result$$Register,
11466                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11467   %}
11468   ins_pipe( pipe_slow );
11469 %}
11470 
11471 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11472                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11473 %{
11474   predicate(UseSSE42Intrinsics);
11475   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11476   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11477   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11478   ins_encode %{
11479     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11480                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11481   %}
11482   ins_pipe( pipe_slow );
11483 %}
11484 
11485 // fast string equals
11486 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11487                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11488 %{
11489   match(Set result (StrEquals (Binary str1 str2) cnt));
11490   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11491 
11492   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11493   ins_encode %{
11494     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11495                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11496                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11497   %}
11498   ins_pipe( pipe_slow );
11499 %}
11500 
11501 // fast array equals
11502 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11503                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11504 %{
11505   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11506   match(Set result (AryEq ary1 ary2));
11507   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11508 
11509   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11510   ins_encode %{
11511     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11512                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11513                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11514   %}
11515   ins_pipe( pipe_slow );
11516 %}
11517 
11518 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11519                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11520 %{
11521   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11522   match(Set result (AryEq ary1 ary2));
11523   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11524 
11525   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11526   ins_encode %{
11527     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11528                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11529                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11530   %}
11531   ins_pipe( pipe_slow );
11532 %}
11533 
11534 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11535                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11536 %{
11537   match(Set result (HasNegatives ary1 len));
11538   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11539 
11540   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11541   ins_encode %{
11542     __ has_negatives($ary1$$Register, $len$$Register,
11543                      $result$$Register, $tmp3$$Register,
11544                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11545   %}
11546   ins_pipe( pipe_slow );
11547 %}
11548 
11549 // fast char[] to byte[] compression
11550 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11551                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11552   match(Set result (StrCompressedCopy src (Binary dst len)));
11553   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11554 
11555   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11556   ins_encode %{
11557     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11558                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11559                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11560   %}
11561   ins_pipe( pipe_slow );
11562 %}
11563 
11564 // fast byte[] to char[] inflation
11565 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11566                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11567   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11568   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11569 
11570   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11571   ins_encode %{
11572     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11573                           $tmp1$$XMMRegister, $tmp2$$Register);
11574   %}
11575   ins_pipe( pipe_slow );
11576 %}
11577 
11578 // encode char[] to byte[] in ISO_8859_1
11579 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11580                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11581                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11582   match(Set result (EncodeISOArray src (Binary dst len)));
11583   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11584 
11585   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11586   ins_encode %{
11587     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11588                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11589                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11590   %}
11591   ins_pipe( pipe_slow );
11592 %}
11593 
11594 //----------Overflow Math Instructions-----------------------------------------
11595 
11596 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11597 %{
11598   match(Set cr (OverflowAddI op1 op2));
11599   effect(DEF cr, USE_KILL op1, USE op2);
11600 
11601   format %{ "addl    $op1, $op2\t# overflow check int" %}
11602 
11603   ins_encode %{
11604     __ addl($op1$$Register, $op2$$Register);
11605   %}
11606   ins_pipe(ialu_reg_reg);
11607 %}
11608 
11609 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11610 %{
11611   match(Set cr (OverflowAddI op1 op2));
11612   effect(DEF cr, USE_KILL op1, USE op2);
11613 
11614   format %{ "addl    $op1, $op2\t# overflow check int" %}
11615 
11616   ins_encode %{
11617     __ addl($op1$$Register, $op2$$constant);
11618   %}
11619   ins_pipe(ialu_reg_reg);
11620 %}
11621 
11622 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11623 %{
11624   match(Set cr (OverflowAddL op1 op2));
11625   effect(DEF cr, USE_KILL op1, USE op2);
11626 
11627   format %{ "addq    $op1, $op2\t# overflow check long" %}
11628   ins_encode %{
11629     __ addq($op1$$Register, $op2$$Register);
11630   %}
11631   ins_pipe(ialu_reg_reg);
11632 %}
11633 
11634 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11635 %{
11636   match(Set cr (OverflowAddL op1 op2));
11637   effect(DEF cr, USE_KILL op1, USE op2);
11638 
11639   format %{ "addq    $op1, $op2\t# overflow check long" %}
11640   ins_encode %{
11641     __ addq($op1$$Register, $op2$$constant);
11642   %}
11643   ins_pipe(ialu_reg_reg);
11644 %}
11645 
11646 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11647 %{
11648   match(Set cr (OverflowSubI op1 op2));
11649 
11650   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11651   ins_encode %{
11652     __ cmpl($op1$$Register, $op2$$Register);
11653   %}
11654   ins_pipe(ialu_reg_reg);
11655 %}
11656 
11657 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11658 %{
11659   match(Set cr (OverflowSubI op1 op2));
11660 
11661   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11662   ins_encode %{
11663     __ cmpl($op1$$Register, $op2$$constant);
11664   %}
11665   ins_pipe(ialu_reg_reg);
11666 %}
11667 
11668 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11669 %{
11670   match(Set cr (OverflowSubL op1 op2));
11671 
11672   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11673   ins_encode %{
11674     __ cmpq($op1$$Register, $op2$$Register);
11675   %}
11676   ins_pipe(ialu_reg_reg);
11677 %}
11678 
11679 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11680 %{
11681   match(Set cr (OverflowSubL op1 op2));
11682 
11683   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11684   ins_encode %{
11685     __ cmpq($op1$$Register, $op2$$constant);
11686   %}
11687   ins_pipe(ialu_reg_reg);
11688 %}
11689 
11690 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11691 %{
11692   match(Set cr (OverflowSubI zero op2));
11693   effect(DEF cr, USE_KILL op2);
11694 
11695   format %{ "negl    $op2\t# overflow check int" %}
11696   ins_encode %{
11697     __ negl($op2$$Register);
11698   %}
11699   ins_pipe(ialu_reg_reg);
11700 %}
11701 
11702 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11703 %{
11704   match(Set cr (OverflowSubL zero op2));
11705   effect(DEF cr, USE_KILL op2);
11706 
11707   format %{ "negq    $op2\t# overflow check long" %}
11708   ins_encode %{
11709     __ negq($op2$$Register);
11710   %}
11711   ins_pipe(ialu_reg_reg);
11712 %}
11713 
11714 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11715 %{
11716   match(Set cr (OverflowMulI op1 op2));
11717   effect(DEF cr, USE_KILL op1, USE op2);
11718 
11719   format %{ "imull    $op1, $op2\t# overflow check int" %}
11720   ins_encode %{
11721     __ imull($op1$$Register, $op2$$Register);
11722   %}
11723   ins_pipe(ialu_reg_reg_alu0);
11724 %}
11725 
11726 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11727 %{
11728   match(Set cr (OverflowMulI op1 op2));
11729   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11730 
11731   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11732   ins_encode %{
11733     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11734   %}
11735   ins_pipe(ialu_reg_reg_alu0);
11736 %}
11737 
11738 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11739 %{
11740   match(Set cr (OverflowMulL op1 op2));
11741   effect(DEF cr, USE_KILL op1, USE op2);
11742 
11743   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11744   ins_encode %{
11745     __ imulq($op1$$Register, $op2$$Register);
11746   %}
11747   ins_pipe(ialu_reg_reg_alu0);
11748 %}
11749 
11750 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11751 %{
11752   match(Set cr (OverflowMulL op1 op2));
11753   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11754 
11755   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11756   ins_encode %{
11757     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11758   %}
11759   ins_pipe(ialu_reg_reg_alu0);
11760 %}
11761 
11762 
11763 //----------Control Flow Instructions------------------------------------------
11764 // Signed compare Instructions
11765 
11766 // XXX more variants!!
11767 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11768 %{
11769   match(Set cr (CmpI op1 op2));
11770   effect(DEF cr, USE op1, USE op2);
11771 
11772   format %{ "cmpl    $op1, $op2" %}
11773   opcode(0x3B);  /* Opcode 3B /r */
11774   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11775   ins_pipe(ialu_cr_reg_reg);
11776 %}
11777 
11778 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11779 %{
11780   match(Set cr (CmpI op1 op2));
11781 
11782   format %{ "cmpl    $op1, $op2" %}
11783   opcode(0x81, 0x07); /* Opcode 81 /7 */
11784   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11785   ins_pipe(ialu_cr_reg_imm);
11786 %}
11787 
11788 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11789 %{
11790   match(Set cr (CmpI op1 (LoadI op2)));
11791 
11792   ins_cost(500); // XXX
11793   format %{ "cmpl    $op1, $op2" %}
11794   opcode(0x3B); /* Opcode 3B /r */
11795   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11796   ins_pipe(ialu_cr_reg_mem);
11797 %}
11798 
11799 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11800 %{
11801   match(Set cr (CmpI src zero));
11802 
11803   format %{ "testl   $src, $src" %}
11804   opcode(0x85);
11805   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11806   ins_pipe(ialu_cr_reg_imm);
11807 %}
11808 
11809 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11810 %{
11811   match(Set cr (CmpI (AndI src con) zero));
11812 
11813   format %{ "testl   $src, $con" %}
11814   opcode(0xF7, 0x00);
11815   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11816   ins_pipe(ialu_cr_reg_imm);
11817 %}
11818 
11819 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11820 %{
11821   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11822 
11823   format %{ "testl   $src, $mem" %}
11824   opcode(0x85);
11825   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11826   ins_pipe(ialu_cr_reg_mem);
11827 %}
11828 
11829 // Unsigned compare Instructions; really, same as signed except they
11830 // produce an rFlagsRegU instead of rFlagsReg.
11831 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11832 %{
11833   match(Set cr (CmpU op1 op2));
11834 
11835   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11836   opcode(0x3B); /* Opcode 3B /r */
11837   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11838   ins_pipe(ialu_cr_reg_reg);
11839 %}
11840 
11841 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11842 %{
11843   match(Set cr (CmpU op1 op2));
11844 
11845   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11846   opcode(0x81,0x07); /* Opcode 81 /7 */
11847   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11848   ins_pipe(ialu_cr_reg_imm);
11849 %}
11850 
11851 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11852 %{
11853   match(Set cr (CmpU op1 (LoadI op2)));
11854 
11855   ins_cost(500); // XXX
11856   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11857   opcode(0x3B); /* Opcode 3B /r */
11858   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11859   ins_pipe(ialu_cr_reg_mem);
11860 %}
11861 
11862 // // // Cisc-spilled version of cmpU_rReg
11863 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11864 // //%{
11865 // //  match(Set cr (CmpU (LoadI op1) op2));
11866 // //
11867 // //  format %{ "CMPu   $op1,$op2" %}
11868 // //  ins_cost(500);
11869 // //  opcode(0x39);  /* Opcode 39 /r */
11870 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11871 // //%}
11872 
11873 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11874 %{
11875   match(Set cr (CmpU src zero));
11876 
11877   format %{ "testl  $src, $src\t# unsigned" %}
11878   opcode(0x85);
11879   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11880   ins_pipe(ialu_cr_reg_imm);
11881 %}
11882 
11883 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11884 %{
11885   match(Set cr (CmpP op1 op2));
11886 
11887   format %{ "cmpq    $op1, $op2\t# ptr" %}
11888   opcode(0x3B); /* Opcode 3B /r */
11889   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11890   ins_pipe(ialu_cr_reg_reg);
11891 %}
11892 
11893 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11894 %{
11895   match(Set cr (CmpP op1 (LoadP op2)));
11896 
11897   ins_cost(500); // XXX
11898   format %{ "cmpq    $op1, $op2\t# ptr" %}
11899   opcode(0x3B); /* Opcode 3B /r */
11900   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11901   ins_pipe(ialu_cr_reg_mem);
11902 %}
11903 
11904 // // // Cisc-spilled version of cmpP_rReg
11905 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11906 // //%{
11907 // //  match(Set cr (CmpP (LoadP op1) op2));
11908 // //
11909 // //  format %{ "CMPu   $op1,$op2" %}
11910 // //  ins_cost(500);
11911 // //  opcode(0x39);  /* Opcode 39 /r */
11912 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11913 // //%}
11914 
11915 // XXX this is generalized by compP_rReg_mem???
11916 // Compare raw pointer (used in out-of-heap check).
11917 // Only works because non-oop pointers must be raw pointers
11918 // and raw pointers have no anti-dependencies.
11919 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11920 %{
11921   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11922   match(Set cr (CmpP op1 (LoadP op2)));
11923 
11924   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11925   opcode(0x3B); /* Opcode 3B /r */
11926   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11927   ins_pipe(ialu_cr_reg_mem);
11928 %}
11929 
11930 // This will generate a signed flags result. This should be OK since
11931 // any compare to a zero should be eq/neq.
11932 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11933 %{
11934   match(Set cr (CmpP src zero));
11935 
11936   format %{ "testq   $src, $src\t# ptr" %}
11937   opcode(0x85);
11938   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11939   ins_pipe(ialu_cr_reg_imm);
11940 %}
11941 
11942 // This will generate a signed flags result. This should be OK since
11943 // any compare to a zero should be eq/neq.
11944 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11945 %{
11946   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11947   match(Set cr (CmpP (LoadP op) zero));
11948 
11949   ins_cost(500); // XXX
11950   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11951   opcode(0xF7); /* Opcode F7 /0 */
11952   ins_encode(REX_mem_wide(op),
11953              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11954   ins_pipe(ialu_cr_reg_imm);
11955 %}
11956 
11957 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11958 %{
11959   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11960   match(Set cr (CmpP (LoadP mem) zero));
11961 
11962   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11963   ins_encode %{
11964     __ cmpq(r12, $mem$$Address);
11965   %}
11966   ins_pipe(ialu_cr_reg_mem);
11967 %}
11968 
11969 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11970 %{
11971   match(Set cr (CmpN op1 op2));
11972 
11973   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11974   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11975   ins_pipe(ialu_cr_reg_reg);
11976 %}
11977 
11978 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11979 %{
11980   match(Set cr (CmpN src (LoadN mem)));
11981 
11982   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11983   ins_encode %{
11984     __ cmpl($src$$Register, $mem$$Address);
11985   %}
11986   ins_pipe(ialu_cr_reg_mem);
11987 %}
11988 
11989 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11990   match(Set cr (CmpN op1 op2));
11991 
11992   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11993   ins_encode %{
11994     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11995   %}
11996   ins_pipe(ialu_cr_reg_imm);
11997 %}
11998 
11999 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
12000 %{
12001   match(Set cr (CmpN src (LoadN mem)));
12002 
12003   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
12004   ins_encode %{
12005     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
12006   %}
12007   ins_pipe(ialu_cr_reg_mem);
12008 %}
12009 
12010 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
12011   match(Set cr (CmpN op1 op2));
12012 
12013   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
12014   ins_encode %{
12015     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
12016   %}
12017   ins_pipe(ialu_cr_reg_imm);
12018 %}
12019 
12020 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
12021 %{
12022   match(Set cr (CmpN src (LoadNKlass mem)));
12023 
12024   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
12025   ins_encode %{
12026     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
12027   %}
12028   ins_pipe(ialu_cr_reg_mem);
12029 %}
12030 
12031 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
12032   match(Set cr (CmpN src zero));
12033 
12034   format %{ "testl   $src, $src\t# compressed ptr" %}
12035   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
12036   ins_pipe(ialu_cr_reg_imm);
12037 %}
12038 
12039 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
12040 %{
12041   predicate(Universe::narrow_oop_base() != NULL);
12042   match(Set cr (CmpN (LoadN mem) zero));
12043 
12044   ins_cost(500); // XXX
12045   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
12046   ins_encode %{
12047     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
12048   %}
12049   ins_pipe(ialu_cr_reg_mem);
12050 %}
12051 
12052 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
12053 %{
12054   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
12055   match(Set cr (CmpN (LoadN mem) zero));
12056 
12057   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
12058   ins_encode %{
12059     __ cmpl(r12, $mem$$Address);
12060   %}
12061   ins_pipe(ialu_cr_reg_mem);
12062 %}
12063 
12064 // Yanked all unsigned pointer compare operations.
12065 // Pointer compares are done with CmpP which is already unsigned.
12066 
12067 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
12068 %{
12069   match(Set cr (CmpL op1 op2));
12070 
12071   format %{ "cmpq    $op1, $op2" %}
12072   opcode(0x3B);  /* Opcode 3B /r */
12073   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12074   ins_pipe(ialu_cr_reg_reg);
12075 %}
12076 
12077 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
12078 %{
12079   match(Set cr (CmpL op1 op2));
12080 
12081   format %{ "cmpq    $op1, $op2" %}
12082   opcode(0x81, 0x07); /* Opcode 81 /7 */
12083   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12084   ins_pipe(ialu_cr_reg_imm);
12085 %}
12086 
12087 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
12088 %{
12089   match(Set cr (CmpL op1 (LoadL op2)));
12090 
12091   format %{ "cmpq    $op1, $op2" %}
12092   opcode(0x3B); /* Opcode 3B /r */
12093   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12094   ins_pipe(ialu_cr_reg_mem);
12095 %}
12096 
12097 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
12098 %{
12099   match(Set cr (CmpL src zero));
12100 
12101   format %{ "testq   $src, $src" %}
12102   opcode(0x85);
12103   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12104   ins_pipe(ialu_cr_reg_imm);
12105 %}
12106 
12107 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
12108 %{
12109   match(Set cr (CmpL (AndL src con) zero));
12110 
12111   format %{ "testq   $src, $con\t# long" %}
12112   opcode(0xF7, 0x00);
12113   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
12114   ins_pipe(ialu_cr_reg_imm);
12115 %}
12116 
12117 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
12118 %{
12119   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
12120 
12121   format %{ "testq   $src, $mem" %}
12122   opcode(0x85);
12123   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12124   ins_pipe(ialu_cr_reg_mem);
12125 %}
12126 
12127 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
12128 %{
12129   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
12130 
12131   format %{ "testq   $src, $mem" %}
12132   opcode(0x85);
12133   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
12134   ins_pipe(ialu_cr_reg_mem);
12135 %}
12136 
12137 // Manifest a CmpL result in an integer register.  Very painful.
12138 // This is the test to avoid.
12139 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
12140 %{
12141   match(Set dst (CmpL3 src1 src2));
12142   effect(KILL flags);
12143 
12144   ins_cost(275); // XXX
12145   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
12146             "movl    $dst, -1\n\t"
12147             "jl,s    done\n\t"
12148             "setne   $dst\n\t"
12149             "movzbl  $dst, $dst\n\t"
12150     "done:" %}
12151   ins_encode(cmpl3_flag(src1, src2, dst));
12152   ins_pipe(pipe_slow);
12153 %}
12154 
12155 // Unsigned long compare Instructions; really, same as signed long except they
12156 // produce an rFlagsRegU instead of rFlagsReg.
12157 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
12158 %{
12159   match(Set cr (CmpUL op1 op2));
12160 
12161   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12162   opcode(0x3B);  /* Opcode 3B /r */
12163   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
12164   ins_pipe(ialu_cr_reg_reg);
12165 %}
12166 
12167 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
12168 %{
12169   match(Set cr (CmpUL op1 op2));
12170 
12171   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12172   opcode(0x81, 0x07); /* Opcode 81 /7 */
12173   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
12174   ins_pipe(ialu_cr_reg_imm);
12175 %}
12176 
12177 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
12178 %{
12179   match(Set cr (CmpUL op1 (LoadL op2)));
12180 
12181   format %{ "cmpq    $op1, $op2\t# unsigned" %}
12182   opcode(0x3B); /* Opcode 3B /r */
12183   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
12184   ins_pipe(ialu_cr_reg_mem);
12185 %}
12186 
12187 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
12188 %{
12189   match(Set cr (CmpUL src zero));
12190 
12191   format %{ "testq   $src, $src\t# unsigned" %}
12192   opcode(0x85);
12193   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
12194   ins_pipe(ialu_cr_reg_imm);
12195 %}
12196 
12197 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
12198 %{
12199   match(Set cr (CmpI (LoadB mem) imm));
12200 
12201   ins_cost(125);
12202   format %{ "cmpb    $mem, $imm" %}
12203   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
12204   ins_pipe(ialu_cr_reg_mem);
12205 %}
12206 
12207 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
12208 %{
12209   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
12210 
12211   ins_cost(125);
12212   format %{ "testb   $mem, $imm\t# ubyte" %}
12213   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12214   ins_pipe(ialu_cr_reg_mem);
12215 %}
12216 
12217 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
12218 %{
12219   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
12220 
12221   ins_cost(125);
12222   format %{ "testb   $mem, $imm\t# byte" %}
12223   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
12224   ins_pipe(ialu_cr_reg_mem);
12225 %}
12226 
12227 //----------Max and Min--------------------------------------------------------
12228 // Min Instructions
12229 
12230 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
12231 %{
12232   effect(USE_DEF dst, USE src, USE cr);
12233 
12234   format %{ "cmovlgt $dst, $src\t# min" %}
12235   opcode(0x0F, 0x4F);
12236   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12237   ins_pipe(pipe_cmov_reg);
12238 %}
12239 
12240 
12241 instruct minI_rReg(rRegI dst, rRegI src)
12242 %{
12243   match(Set dst (MinI dst src));
12244 
12245   ins_cost(200);
12246   expand %{
12247     rFlagsReg cr;
12248     compI_rReg(cr, dst, src);
12249     cmovI_reg_g(dst, src, cr);
12250   %}
12251 %}
12252 
12253 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
12254 %{
12255   effect(USE_DEF dst, USE src, USE cr);
12256 
12257   format %{ "cmovllt $dst, $src\t# max" %}
12258   opcode(0x0F, 0x4C);
12259   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
12260   ins_pipe(pipe_cmov_reg);
12261 %}
12262 
12263 
12264 instruct maxI_rReg(rRegI dst, rRegI src)
12265 %{
12266   match(Set dst (MaxI dst src));
12267 
12268   ins_cost(200);
12269   expand %{
12270     rFlagsReg cr;
12271     compI_rReg(cr, dst, src);
12272     cmovI_reg_l(dst, src, cr);
12273   %}
12274 %}
12275 
12276 // ============================================================================
12277 // Branch Instructions
12278 
12279 // Jump Direct - Label defines a relative address from JMP+1
12280 instruct jmpDir(label labl)
12281 %{
12282   match(Goto);
12283   effect(USE labl);
12284 
12285   ins_cost(300);
12286   format %{ "jmp     $labl" %}
12287   size(5);
12288   ins_encode %{
12289     Label* L = $labl$$label;
12290     __ jmp(*L, false); // Always long jump
12291   %}
12292   ins_pipe(pipe_jmp);
12293 %}
12294 
12295 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12296 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
12297 %{
12298   match(If cop cr);
12299   effect(USE labl);
12300 
12301   ins_cost(300);
12302   format %{ "j$cop     $labl" %}
12303   size(6);
12304   ins_encode %{
12305     Label* L = $labl$$label;
12306     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12307   %}
12308   ins_pipe(pipe_jcc);
12309 %}
12310 
12311 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12312 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
12313 %{
12314   predicate(!n->has_vector_mask_set());
12315   match(CountedLoopEnd cop cr);
12316   effect(USE labl);
12317 
12318   ins_cost(300);
12319   format %{ "j$cop     $labl\t# loop end" %}
12320   size(6);
12321   ins_encode %{
12322     Label* L = $labl$$label;
12323     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12324   %}
12325   ins_pipe(pipe_jcc);
12326 %}
12327 
12328 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12329 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12330   predicate(!n->has_vector_mask_set());
12331   match(CountedLoopEnd cop cmp);
12332   effect(USE labl);
12333 
12334   ins_cost(300);
12335   format %{ "j$cop,u   $labl\t# loop end" %}
12336   size(6);
12337   ins_encode %{
12338     Label* L = $labl$$label;
12339     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12340   %}
12341   ins_pipe(pipe_jcc);
12342 %}
12343 
12344 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12345   predicate(!n->has_vector_mask_set());
12346   match(CountedLoopEnd cop cmp);
12347   effect(USE labl);
12348 
12349   ins_cost(200);
12350   format %{ "j$cop,u   $labl\t# loop end" %}
12351   size(6);
12352   ins_encode %{
12353     Label* L = $labl$$label;
12354     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12355   %}
12356   ins_pipe(pipe_jcc);
12357 %}
12358 
12359 // mask version
12360 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12361 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12362 %{
12363   predicate(n->has_vector_mask_set());
12364   match(CountedLoopEnd cop cr);
12365   effect(USE labl);
12366 
12367   ins_cost(400);
12368   format %{ "j$cop     $labl\t# loop end\n\t"
12369             "restorevectmask \t# vector mask restore for loops" %}
12370   size(10);
12371   ins_encode %{
12372     Label* L = $labl$$label;
12373     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12374     __ restorevectmask();
12375   %}
12376   ins_pipe(pipe_jcc);
12377 %}
12378 
12379 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12380 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12381   predicate(n->has_vector_mask_set());
12382   match(CountedLoopEnd cop cmp);
12383   effect(USE labl);
12384 
12385   ins_cost(400);
12386   format %{ "j$cop,u   $labl\t# loop end\n\t"
12387             "restorevectmask \t# vector mask restore for loops" %}
12388   size(10);
12389   ins_encode %{
12390     Label* L = $labl$$label;
12391     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12392     __ restorevectmask();
12393   %}
12394   ins_pipe(pipe_jcc);
12395 %}
12396 
12397 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12398   predicate(n->has_vector_mask_set());
12399   match(CountedLoopEnd cop cmp);
12400   effect(USE labl);
12401 
12402   ins_cost(300);
12403   format %{ "j$cop,u   $labl\t# loop end\n\t"
12404             "restorevectmask \t# vector mask restore for loops" %}
12405   size(10);
12406   ins_encode %{
12407     Label* L = $labl$$label;
12408     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12409     __ restorevectmask();
12410   %}
12411   ins_pipe(pipe_jcc);
12412 %}
12413 
12414 // Jump Direct Conditional - using unsigned comparison
12415 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12416   match(If cop cmp);
12417   effect(USE labl);
12418 
12419   ins_cost(300);
12420   format %{ "j$cop,u  $labl" %}
12421   size(6);
12422   ins_encode %{
12423     Label* L = $labl$$label;
12424     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12425   %}
12426   ins_pipe(pipe_jcc);
12427 %}
12428 
12429 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12430   match(If cop cmp);
12431   effect(USE labl);
12432 
12433   ins_cost(200);
12434   format %{ "j$cop,u  $labl" %}
12435   size(6);
12436   ins_encode %{
12437     Label* L = $labl$$label;
12438     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12439   %}
12440   ins_pipe(pipe_jcc);
12441 %}
12442 
12443 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12444   match(If cop cmp);
12445   effect(USE labl);
12446 
12447   ins_cost(200);
12448   format %{ $$template
12449     if ($cop$$cmpcode == Assembler::notEqual) {
12450       $$emit$$"jp,u   $labl\n\t"
12451       $$emit$$"j$cop,u   $labl"
12452     } else {
12453       $$emit$$"jp,u   done\n\t"
12454       $$emit$$"j$cop,u   $labl\n\t"
12455       $$emit$$"done:"
12456     }
12457   %}
12458   ins_encode %{
12459     Label* l = $labl$$label;
12460     if ($cop$$cmpcode == Assembler::notEqual) {
12461       __ jcc(Assembler::parity, *l, false);
12462       __ jcc(Assembler::notEqual, *l, false);
12463     } else if ($cop$$cmpcode == Assembler::equal) {
12464       Label done;
12465       __ jccb(Assembler::parity, done);
12466       __ jcc(Assembler::equal, *l, false);
12467       __ bind(done);
12468     } else {
12469        ShouldNotReachHere();
12470     }
12471   %}
12472   ins_pipe(pipe_jcc);
12473 %}
12474 
12475 // ============================================================================
12476 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12477 // superklass array for an instance of the superklass.  Set a hidden
12478 // internal cache on a hit (cache is checked with exposed code in
12479 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12480 // encoding ALSO sets flags.
12481 
12482 instruct partialSubtypeCheck(rdi_RegP result,
12483                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12484                              rFlagsReg cr)
12485 %{
12486   match(Set result (PartialSubtypeCheck sub super));
12487   effect(KILL rcx, KILL cr);
12488 
12489   ins_cost(1100);  // slightly larger than the next version
12490   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12491             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12492             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12493             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12494             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12495             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12496             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12497     "miss:\t" %}
12498 
12499   opcode(0x1); // Force a XOR of RDI
12500   ins_encode(enc_PartialSubtypeCheck());
12501   ins_pipe(pipe_slow);
12502 %}
12503 
12504 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12505                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12506                                      immP0 zero,
12507                                      rdi_RegP result)
12508 %{
12509   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12510   effect(KILL rcx, KILL result);
12511 
12512   ins_cost(1000);
12513   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12514             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12515             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12516             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12517             "jne,s   miss\t\t# Missed: flags nz\n\t"
12518             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12519     "miss:\t" %}
12520 
12521   opcode(0x0); // No need to XOR RDI
12522   ins_encode(enc_PartialSubtypeCheck());
12523   ins_pipe(pipe_slow);
12524 %}
12525 
12526 // ============================================================================
12527 // Branch Instructions -- short offset versions
12528 //
12529 // These instructions are used to replace jumps of a long offset (the default
12530 // match) with jumps of a shorter offset.  These instructions are all tagged
12531 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12532 // match rules in general matching.  Instead, the ADLC generates a conversion
12533 // method in the MachNode which can be used to do in-place replacement of the
12534 // long variant with the shorter variant.  The compiler will determine if a
12535 // branch can be taken by the is_short_branch_offset() predicate in the machine
12536 // specific code section of the file.
12537 
12538 // Jump Direct - Label defines a relative address from JMP+1
12539 instruct jmpDir_short(label labl) %{
12540   match(Goto);
12541   effect(USE labl);
12542 
12543   ins_cost(300);
12544   format %{ "jmp,s   $labl" %}
12545   size(2);
12546   ins_encode %{
12547     Label* L = $labl$$label;
12548     __ jmpb(*L);
12549   %}
12550   ins_pipe(pipe_jmp);
12551   ins_short_branch(1);
12552 %}
12553 
12554 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12555 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12556   match(If cop cr);
12557   effect(USE labl);
12558 
12559   ins_cost(300);
12560   format %{ "j$cop,s   $labl" %}
12561   size(2);
12562   ins_encode %{
12563     Label* L = $labl$$label;
12564     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12565   %}
12566   ins_pipe(pipe_jcc);
12567   ins_short_branch(1);
12568 %}
12569 
12570 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12571 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12572   match(CountedLoopEnd cop cr);
12573   effect(USE labl);
12574 
12575   ins_cost(300);
12576   format %{ "j$cop,s   $labl\t# loop end" %}
12577   size(2);
12578   ins_encode %{
12579     Label* L = $labl$$label;
12580     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12581   %}
12582   ins_pipe(pipe_jcc);
12583   ins_short_branch(1);
12584 %}
12585 
12586 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12587 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12588   match(CountedLoopEnd cop cmp);
12589   effect(USE labl);
12590 
12591   ins_cost(300);
12592   format %{ "j$cop,us  $labl\t# loop end" %}
12593   size(2);
12594   ins_encode %{
12595     Label* L = $labl$$label;
12596     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12597   %}
12598   ins_pipe(pipe_jcc);
12599   ins_short_branch(1);
12600 %}
12601 
12602 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12603   match(CountedLoopEnd cop cmp);
12604   effect(USE labl);
12605 
12606   ins_cost(300);
12607   format %{ "j$cop,us  $labl\t# loop end" %}
12608   size(2);
12609   ins_encode %{
12610     Label* L = $labl$$label;
12611     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12612   %}
12613   ins_pipe(pipe_jcc);
12614   ins_short_branch(1);
12615 %}
12616 
12617 // Jump Direct Conditional - using unsigned comparison
12618 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12619   match(If cop cmp);
12620   effect(USE labl);
12621 
12622   ins_cost(300);
12623   format %{ "j$cop,us  $labl" %}
12624   size(2);
12625   ins_encode %{
12626     Label* L = $labl$$label;
12627     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12628   %}
12629   ins_pipe(pipe_jcc);
12630   ins_short_branch(1);
12631 %}
12632 
12633 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12634   match(If cop cmp);
12635   effect(USE labl);
12636 
12637   ins_cost(300);
12638   format %{ "j$cop,us  $labl" %}
12639   size(2);
12640   ins_encode %{
12641     Label* L = $labl$$label;
12642     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12643   %}
12644   ins_pipe(pipe_jcc);
12645   ins_short_branch(1);
12646 %}
12647 
12648 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12649   match(If cop cmp);
12650   effect(USE labl);
12651 
12652   ins_cost(300);
12653   format %{ $$template
12654     if ($cop$$cmpcode == Assembler::notEqual) {
12655       $$emit$$"jp,u,s   $labl\n\t"
12656       $$emit$$"j$cop,u,s   $labl"
12657     } else {
12658       $$emit$$"jp,u,s   done\n\t"
12659       $$emit$$"j$cop,u,s  $labl\n\t"
12660       $$emit$$"done:"
12661     }
12662   %}
12663   size(4);
12664   ins_encode %{
12665     Label* l = $labl$$label;
12666     if ($cop$$cmpcode == Assembler::notEqual) {
12667       __ jccb(Assembler::parity, *l);
12668       __ jccb(Assembler::notEqual, *l);
12669     } else if ($cop$$cmpcode == Assembler::equal) {
12670       Label done;
12671       __ jccb(Assembler::parity, done);
12672       __ jccb(Assembler::equal, *l);
12673       __ bind(done);
12674     } else {
12675        ShouldNotReachHere();
12676     }
12677   %}
12678   ins_pipe(pipe_jcc);
12679   ins_short_branch(1);
12680 %}
12681 
12682 // ============================================================================
12683 // inlined locking and unlocking
12684 
12685 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12686   predicate(Compile::current()->use_rtm());
12687   match(Set cr (FastLock object box));
12688   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12689   ins_cost(300);
12690   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12691   ins_encode %{
12692     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12693                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12694                  _counters, _rtm_counters, _stack_rtm_counters,
12695                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12696                  true, ra_->C->profile_rtm());
12697   %}
12698   ins_pipe(pipe_slow);
12699 %}
12700 
12701 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12702   predicate(!Compile::current()->use_rtm());
12703   match(Set cr (FastLock object box));
12704   effect(TEMP tmp, TEMP scr, USE_KILL box);
12705   ins_cost(300);
12706   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12707   ins_encode %{
12708     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12709                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12710   %}
12711   ins_pipe(pipe_slow);
12712 %}
12713 
12714 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12715   match(Set cr (FastUnlock object box));
12716   effect(TEMP tmp, USE_KILL box);
12717   ins_cost(300);
12718   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12719   ins_encode %{
12720     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12721   %}
12722   ins_pipe(pipe_slow);
12723 %}
12724 
12725 
12726 // ============================================================================
12727 // Safepoint Instructions
12728 instruct safePoint_poll(rFlagsReg cr)
12729 %{
12730   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12731   match(SafePoint);
12732   effect(KILL cr);
12733 
12734   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12735             "# Safepoint: poll for GC" %}
12736   ins_cost(125);
12737   ins_encode %{
12738     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12739     __ testl(rax, addr);
12740   %}
12741   ins_pipe(ialu_reg_mem);
12742 %}
12743 
12744 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12745 %{
12746   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12747   match(SafePoint poll);
12748   effect(KILL cr, USE poll);
12749 
12750   format %{ "testl  rax, [$poll]\t"
12751             "# Safepoint: poll for GC" %}
12752   ins_cost(125);
12753   ins_encode %{
12754     __ relocate(relocInfo::poll_type);
12755     __ testl(rax, Address($poll$$Register, 0));
12756   %}
12757   ins_pipe(ialu_reg_mem);
12758 %}
12759 
12760 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12761 %{
12762   predicate(SafepointMechanism::uses_thread_local_poll());
12763   match(SafePoint poll);
12764   effect(KILL cr, USE poll);
12765 
12766   format %{ "testl  rax, [$poll]\t"
12767             "# Safepoint: poll for GC" %}
12768   ins_cost(125);
12769   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12770   ins_encode %{
12771     __ relocate(relocInfo::poll_type);
12772     address pre_pc = __ pc();
12773     __ testl(rax, Address($poll$$Register, 0));
12774     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12775   %}
12776   ins_pipe(ialu_reg_mem);
12777 %}
12778 
12779 // ============================================================================
12780 // Procedure Call/Return Instructions
12781 // Call Java Static Instruction
12782 // Note: If this code changes, the corresponding ret_addr_offset() and
12783 //       compute_padding() functions will have to be adjusted.
12784 instruct CallStaticJavaDirect(method meth) %{
12785   match(CallStaticJava);
12786   effect(USE meth);
12787 
12788   ins_cost(300);
12789   format %{ "call,static " %}
12790   opcode(0xE8); /* E8 cd */
12791   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12792   ins_pipe(pipe_slow);
12793   ins_alignment(4);
12794 %}
12795 
12796 // Call Java Dynamic Instruction
12797 // Note: If this code changes, the corresponding ret_addr_offset() and
12798 //       compute_padding() functions will have to be adjusted.
12799 instruct CallDynamicJavaDirect(method meth)
12800 %{
12801   match(CallDynamicJava);
12802   effect(USE meth);
12803 
12804   ins_cost(300);
12805   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12806             "call,dynamic " %}
12807   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12808   ins_pipe(pipe_slow);
12809   ins_alignment(4);
12810 %}
12811 
12812 // Call Runtime Instruction
12813 instruct CallRuntimeDirect(method meth)
12814 %{
12815   match(CallRuntime);
12816   effect(USE meth);
12817 
12818   ins_cost(300);
12819   format %{ "call,runtime " %}
12820   ins_encode(clear_avx, Java_To_Runtime(meth));
12821   ins_pipe(pipe_slow);
12822 %}
12823 
12824 // Call runtime without safepoint
12825 instruct CallLeafDirect(method meth)
12826 %{
12827   match(CallLeaf);
12828   effect(USE meth);
12829 
12830   ins_cost(300);
12831   format %{ "call_leaf,runtime " %}
12832   ins_encode(clear_avx, Java_To_Runtime(meth));
12833   ins_pipe(pipe_slow);
12834 %}
12835 
12836 // Call runtime without safepoint
12837 instruct CallLeafNoFPDirect(method meth)
12838 %{
12839   match(CallLeafNoFP);
12840   effect(USE meth);
12841 
12842   ins_cost(300);
12843   format %{ "call_leaf_nofp,runtime " %}
12844   ins_encode(clear_avx, Java_To_Runtime(meth));
12845   ins_pipe(pipe_slow);
12846 %}
12847 
12848 // Return Instruction
12849 // Remove the return address & jump to it.
12850 // Notice: We always emit a nop after a ret to make sure there is room
12851 // for safepoint patching
12852 instruct Ret()
12853 %{
12854   match(Return);
12855 
12856   format %{ "ret" %}
12857   opcode(0xC3);
12858   ins_encode(OpcP);
12859   ins_pipe(pipe_jmp);
12860 %}
12861 
12862 // Tail Call; Jump from runtime stub to Java code.
12863 // Also known as an 'interprocedural jump'.
12864 // Target of jump will eventually return to caller.
12865 // TailJump below removes the return address.
12866 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12867 %{
12868   match(TailCall jump_target method_oop);
12869 
12870   ins_cost(300);
12871   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12872   opcode(0xFF, 0x4); /* Opcode FF /4 */
12873   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12874   ins_pipe(pipe_jmp);
12875 %}
12876 
12877 // Tail Jump; remove the return address; jump to target.
12878 // TailCall above leaves the return address around.
12879 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12880 %{
12881   match(TailJump jump_target ex_oop);
12882 
12883   ins_cost(300);
12884   format %{ "popq    rdx\t# pop return address\n\t"
12885             "jmp     $jump_target" %}
12886   opcode(0xFF, 0x4); /* Opcode FF /4 */
12887   ins_encode(Opcode(0x5a), // popq rdx
12888              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12889   ins_pipe(pipe_jmp);
12890 %}
12891 
12892 // Create exception oop: created by stack-crawling runtime code.
12893 // Created exception is now available to this handler, and is setup
12894 // just prior to jumping to this handler.  No code emitted.
12895 instruct CreateException(rax_RegP ex_oop)
12896 %{
12897   match(Set ex_oop (CreateEx));
12898 
12899   size(0);
12900   // use the following format syntax
12901   format %{ "# exception oop is in rax; no code emitted" %}
12902   ins_encode();
12903   ins_pipe(empty);
12904 %}
12905 
12906 // Rethrow exception:
12907 // The exception oop will come in the first argument position.
12908 // Then JUMP (not call) to the rethrow stub code.
12909 instruct RethrowException()
12910 %{
12911   match(Rethrow);
12912 
12913   // use the following format syntax
12914   format %{ "jmp     rethrow_stub" %}
12915   ins_encode(enc_rethrow);
12916   ins_pipe(pipe_jmp);
12917 %}
12918 
12919 // ============================================================================
12920 // This name is KNOWN by the ADLC and cannot be changed.
12921 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12922 // for this guy.
12923 instruct tlsLoadP(r15_RegP dst) %{
12924   match(Set dst (ThreadLocal));
12925   effect(DEF dst);
12926 
12927   size(0);
12928   format %{ "# TLS is in R15" %}
12929   ins_encode( /*empty encoding*/ );
12930   ins_pipe(ialu_reg_reg);
12931 %}
12932 
12933 
12934 //----------PEEPHOLE RULES-----------------------------------------------------
12935 // These must follow all instruction definitions as they use the names
12936 // defined in the instructions definitions.
12937 //
12938 // peepmatch ( root_instr_name [preceding_instruction]* );
12939 //
12940 // peepconstraint %{
12941 // (instruction_number.operand_name relational_op instruction_number.operand_name
12942 //  [, ...] );
12943 // // instruction numbers are zero-based using left to right order in peepmatch
12944 //
12945 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12946 // // provide an instruction_number.operand_name for each operand that appears
12947 // // in the replacement instruction's match rule
12948 //
12949 // ---------VM FLAGS---------------------------------------------------------
12950 //
12951 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12952 //
12953 // Each peephole rule is given an identifying number starting with zero and
12954 // increasing by one in the order seen by the parser.  An individual peephole
12955 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12956 // on the command-line.
12957 //
12958 // ---------CURRENT LIMITATIONS----------------------------------------------
12959 //
12960 // Only match adjacent instructions in same basic block
12961 // Only equality constraints
12962 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12963 // Only one replacement instruction
12964 //
12965 // ---------EXAMPLE----------------------------------------------------------
12966 //
12967 // // pertinent parts of existing instructions in architecture description
12968 // instruct movI(rRegI dst, rRegI src)
12969 // %{
12970 //   match(Set dst (CopyI src));
12971 // %}
12972 //
12973 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12974 // %{
12975 //   match(Set dst (AddI dst src));
12976 //   effect(KILL cr);
12977 // %}
12978 //
12979 // // Change (inc mov) to lea
12980 // peephole %{
12981 //   // increment preceeded by register-register move
12982 //   peepmatch ( incI_rReg movI );
12983 //   // require that the destination register of the increment
12984 //   // match the destination register of the move
12985 //   peepconstraint ( 0.dst == 1.dst );
12986 //   // construct a replacement instruction that sets
12987 //   // the destination to ( move's source register + one )
12988 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12989 // %}
12990 //
12991 
12992 // Implementation no longer uses movX instructions since
12993 // machine-independent system no longer uses CopyX nodes.
12994 //
12995 // peephole
12996 // %{
12997 //   peepmatch (incI_rReg movI);
12998 //   peepconstraint (0.dst == 1.dst);
12999 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13000 // %}
13001 
13002 // peephole
13003 // %{
13004 //   peepmatch (decI_rReg movI);
13005 //   peepconstraint (0.dst == 1.dst);
13006 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13007 // %}
13008 
13009 // peephole
13010 // %{
13011 //   peepmatch (addI_rReg_imm movI);
13012 //   peepconstraint (0.dst == 1.dst);
13013 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
13014 // %}
13015 
13016 // peephole
13017 // %{
13018 //   peepmatch (incL_rReg movL);
13019 //   peepconstraint (0.dst == 1.dst);
13020 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13021 // %}
13022 
13023 // peephole
13024 // %{
13025 //   peepmatch (decL_rReg movL);
13026 //   peepconstraint (0.dst == 1.dst);
13027 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13028 // %}
13029 
13030 // peephole
13031 // %{
13032 //   peepmatch (addL_rReg_imm movL);
13033 //   peepconstraint (0.dst == 1.dst);
13034 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
13035 // %}
13036 
13037 // peephole
13038 // %{
13039 //   peepmatch (addP_rReg_imm movP);
13040 //   peepconstraint (0.dst == 1.dst);
13041 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
13042 // %}
13043 
13044 // // Change load of spilled value to only a spill
13045 // instruct storeI(memory mem, rRegI src)
13046 // %{
13047 //   match(Set mem (StoreI mem src));
13048 // %}
13049 //
13050 // instruct loadI(rRegI dst, memory mem)
13051 // %{
13052 //   match(Set dst (LoadI mem));
13053 // %}
13054 //
13055 
13056 peephole
13057 %{
13058   peepmatch (loadI storeI);
13059   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13060   peepreplace (storeI(1.mem 1.mem 1.src));
13061 %}
13062 
13063 peephole
13064 %{
13065   peepmatch (loadL storeL);
13066   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
13067   peepreplace (storeL(1.mem 1.mem 1.src));
13068 %}
13069 
13070 //----------SMARTSPILL RULES---------------------------------------------------
13071 // These must follow all instruction definitions as they use the names
13072 // defined in the instructions definitions.