1 //
   2 // Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // The registers which can be used for
 321 // a thread local safepoint poll
 322 // * R12 is reserved for heap base
 323 // * R13 cannot be encoded for addressing without an offset byte
 324 // * R15 is reserved for the JavaThread
 325 reg_class ptr_rex_reg(R8,  R8_H,
 326                       R9,  R9_H,
 327                       R10, R10_H,
 328                       R11, R11_H,
 329                       R14, R14_H);
 330 
 331 
 332 // Class for all long registers (excluding RSP)
 333 reg_class long_reg_with_rbp(RAX, RAX_H,
 334                             RDX, RDX_H,
 335                             RBP, RBP_H,
 336                             RDI, RDI_H,
 337                             RSI, RSI_H,
 338                             RCX, RCX_H,
 339                             RBX, RBX_H,
 340                             R8,  R8_H,
 341                             R9,  R9_H,
 342                             R10, R10_H,
 343                             R11, R11_H,
 344                             R13, R13_H,
 345                             R14, R14_H);
 346 
 347 // Class for all long registers (excluding RSP and RBP)
 348 reg_class long_reg_no_rbp(RAX, RAX_H,
 349                           RDX, RDX_H,
 350                           RDI, RDI_H,
 351                           RSI, RSI_H,
 352                           RCX, RCX_H,
 353                           RBX, RBX_H,
 354                           R8,  R8_H,
 355                           R9,  R9_H,
 356                           R10, R10_H,
 357                           R11, R11_H,
 358                           R13, R13_H,
 359                           R14, R14_H);
 360 
 361 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 362 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 363 
 364 // Class for all long registers (excluding RAX, RDX and RSP)
 365 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 366                                        RDI, RDI_H,
 367                                        RSI, RSI_H,
 368                                        RCX, RCX_H,
 369                                        RBX, RBX_H,
 370                                        R8,  R8_H,
 371                                        R9,  R9_H,
 372                                        R10, R10_H,
 373                                        R11, R11_H,
 374                                        R13, R13_H,
 375                                        R14, R14_H);
 376 
 377 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 378 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 379                                      RSI, RSI_H,
 380                                      RCX, RCX_H,
 381                                      RBX, RBX_H,
 382                                      R8,  R8_H,
 383                                      R9,  R9_H,
 384                                      R10, R10_H,
 385                                      R11, R11_H,
 386                                      R13, R13_H,
 387                                      R14, R14_H);
 388 
 389 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 390 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 391 
 392 // Class for all long registers (excluding RCX and RSP)
 393 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 394                                    RDI, RDI_H,
 395                                    RSI, RSI_H,
 396                                    RAX, RAX_H,
 397                                    RDX, RDX_H,
 398                                    RBX, RBX_H,
 399                                    R8,  R8_H,
 400                                    R9,  R9_H,
 401                                    R10, R10_H,
 402                                    R11, R11_H,
 403                                    R13, R13_H,
 404                                    R14, R14_H);
 405 
 406 // Class for all long registers (excluding RCX, RSP, and RBP)
 407 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 408                                  RSI, RSI_H,
 409                                  RAX, RAX_H,
 410                                  RDX, RDX_H,
 411                                  RBX, RBX_H,
 412                                  R8,  R8_H,
 413                                  R9,  R9_H,
 414                                  R10, R10_H,
 415                                  R11, R11_H,
 416                                  R13, R13_H,
 417                                  R14, R14_H);
 418 
 419 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 420 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 421 
 422 // Singleton class for RAX long register
 423 reg_class long_rax_reg(RAX, RAX_H);
 424 
 425 // Singleton class for RCX long register
 426 reg_class long_rcx_reg(RCX, RCX_H);
 427 
 428 // Singleton class for RDX long register
 429 reg_class long_rdx_reg(RDX, RDX_H);
 430 
 431 // Class for all int registers (excluding RSP)
 432 reg_class int_reg_with_rbp(RAX,
 433                            RDX,
 434                            RBP,
 435                            RDI,
 436                            RSI,
 437                            RCX,
 438                            RBX,
 439                            R8,
 440                            R9,
 441                            R10,
 442                            R11,
 443                            R13,
 444                            R14);
 445 
 446 // Class for all int registers (excluding RSP and RBP)
 447 reg_class int_reg_no_rbp(RAX,
 448                          RDX,
 449                          RDI,
 450                          RSI,
 451                          RCX,
 452                          RBX,
 453                          R8,
 454                          R9,
 455                          R10,
 456                          R11,
 457                          R13,
 458                          R14);
 459 
 460 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 461 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 462 
 463 // Class for all int registers (excluding RCX and RSP)
 464 reg_class int_no_rcx_reg_with_rbp(RAX,
 465                                   RDX,
 466                                   RBP,
 467                                   RDI,
 468                                   RSI,
 469                                   RBX,
 470                                   R8,
 471                                   R9,
 472                                   R10,
 473                                   R11,
 474                                   R13,
 475                                   R14);
 476 
 477 // Class for all int registers (excluding RCX, RSP, and RBP)
 478 reg_class int_no_rcx_reg_no_rbp(RAX,
 479                                 RDX,
 480                                 RDI,
 481                                 RSI,
 482                                 RBX,
 483                                 R8,
 484                                 R9,
 485                                 R10,
 486                                 R11,
 487                                 R13,
 488                                 R14);
 489 
 490 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 491 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 492 
 493 // Class for all int registers (excluding RAX, RDX, and RSP)
 494 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 495                                       RDI,
 496                                       RSI,
 497                                       RCX,
 498                                       RBX,
 499                                       R8,
 500                                       R9,
 501                                       R10,
 502                                       R11,
 503                                       R13,
 504                                       R14);
 505 
 506 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 507 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 508                                     RSI,
 509                                     RCX,
 510                                     RBX,
 511                                     R8,
 512                                     R9,
 513                                     R10,
 514                                     R11,
 515                                     R13,
 516                                     R14);
 517 
 518 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 519 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 520 
 521 // Singleton class for RAX int register
 522 reg_class int_rax_reg(RAX);
 523 
 524 // Singleton class for RBX int register
 525 reg_class int_rbx_reg(RBX);
 526 
 527 // Singleton class for RCX int register
 528 reg_class int_rcx_reg(RCX);
 529 
 530 // Singleton class for RCX int register
 531 reg_class int_rdx_reg(RDX);
 532 
 533 // Singleton class for RCX int register
 534 reg_class int_rdi_reg(RDI);
 535 
 536 // Singleton class for instruction pointer
 537 // reg_class ip_reg(RIP);
 538 
 539 %}
 540 
 541 //----------SOURCE BLOCK-------------------------------------------------------
 542 // This is a block of C++ code which provides values, functions, and
 543 // definitions necessary in the rest of the architecture description
 544 source %{
 545 #define   RELOC_IMM64    Assembler::imm_operand
 546 #define   RELOC_DISP32   Assembler::disp32_operand
 547 
 548 #define __ _masm.
 549 
 550 static bool generate_vzeroupper(Compile* C) {
 551   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 552 }
 553 
 554 static int clear_avx_size() {
 555   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 556 }
 557 
 558 // !!!!! Special hack to get all types of calls to specify the byte offset
 559 //       from the start of the call to the point where the return address
 560 //       will point.
 561 int MachCallStaticJavaNode::ret_addr_offset()
 562 {
 563   int offset = 5; // 5 bytes from start of call to where return address points
 564   offset += clear_avx_size();
 565   return offset;
 566 }
 567 
 568 int MachCallDynamicJavaNode::ret_addr_offset()
 569 {
 570   int offset = 15; // 15 bytes from start of call to where return address points
 571   offset += clear_avx_size();
 572   return offset;
 573 }
 574 
 575 int MachCallRuntimeNode::ret_addr_offset() {
 576   int offset = 13; // movq r10,#addr; callq (r10)
 577   offset += clear_avx_size();
 578   return offset;
 579 }
 580 
 581 // Indicate if the safepoint node needs the polling page as an input,
 582 // it does if the polling page is more than disp32 away.
 583 bool SafePointNode::needs_polling_address_input()
 584 {
 585   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 586 }
 587 
 588 //
 589 // Compute padding required for nodes which need alignment
 590 //
 591 
 592 // The address of the call instruction needs to be 4-byte aligned to
 593 // ensure that it does not span a cache line so that it can be patched.
 594 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 595 {
 596   current_offset += clear_avx_size(); // skip vzeroupper
 597   current_offset += 1; // skip call opcode byte
 598   return align_up(current_offset, alignment_required()) - current_offset;
 599 }
 600 
 601 // The address of the call instruction needs to be 4-byte aligned to
 602 // ensure that it does not span a cache line so that it can be patched.
 603 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 604 {
 605   current_offset += clear_avx_size(); // skip vzeroupper
 606   current_offset += 11; // skip movq instruction + call opcode byte
 607   return align_up(current_offset, alignment_required()) - current_offset;
 608 }
 609 
 610 // EMIT_RM()
 611 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 612   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 613   cbuf.insts()->emit_int8(c);
 614 }
 615 
 616 // EMIT_CC()
 617 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 618   unsigned char c = (unsigned char) (f1 | f2);
 619   cbuf.insts()->emit_int8(c);
 620 }
 621 
 622 // EMIT_OPCODE()
 623 void emit_opcode(CodeBuffer &cbuf, int code) {
 624   cbuf.insts()->emit_int8((unsigned char) code);
 625 }
 626 
 627 // EMIT_OPCODE() w/ relocation information
 628 void emit_opcode(CodeBuffer &cbuf,
 629                  int code, relocInfo::relocType reloc, int offset, int format)
 630 {
 631   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 632   emit_opcode(cbuf, code);
 633 }
 634 
 635 // EMIT_D8()
 636 void emit_d8(CodeBuffer &cbuf, int d8) {
 637   cbuf.insts()->emit_int8((unsigned char) d8);
 638 }
 639 
 640 // EMIT_D16()
 641 void emit_d16(CodeBuffer &cbuf, int d16) {
 642   cbuf.insts()->emit_int16(d16);
 643 }
 644 
 645 // EMIT_D32()
 646 void emit_d32(CodeBuffer &cbuf, int d32) {
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // EMIT_D64()
 651 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 652   cbuf.insts()->emit_int64(d64);
 653 }
 654 
 655 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 656 void emit_d32_reloc(CodeBuffer& cbuf,
 657                     int d32,
 658                     relocInfo::relocType reloc,
 659                     int format)
 660 {
 661   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 662   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 663   cbuf.insts()->emit_int32(d32);
 664 }
 665 
 666 // emit 32 bit value and construct relocation entry from RelocationHolder
 667 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 668 #ifdef ASSERT
 669   if (rspec.reloc()->type() == relocInfo::oop_type &&
 670       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 671     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 672     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 673   }
 674 #endif
 675   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 676   cbuf.insts()->emit_int32(d32);
 677 }
 678 
 679 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 680   address next_ip = cbuf.insts_end() + 4;
 681   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 682                  external_word_Relocation::spec(addr),
 683                  RELOC_DISP32);
 684 }
 685 
 686 
 687 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 688 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 689   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 690   cbuf.insts()->emit_int64(d64);
 691 }
 692 
 693 // emit 64 bit value and construct relocation entry from RelocationHolder
 694 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 695 #ifdef ASSERT
 696   if (rspec.reloc()->type() == relocInfo::oop_type &&
 697       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 698     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 699     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 700            "cannot embed scavengable oops in code");
 701   }
 702 #endif
 703   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 704   cbuf.insts()->emit_int64(d64);
 705 }
 706 
 707 // Access stack slot for load or store
 708 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 709 {
 710   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 711   if (-0x80 <= disp && disp < 0x80) {
 712     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 713     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 714     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 715   } else {
 716     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 717     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 718     emit_d32(cbuf, disp);     // Displacement // R/M byte
 719   }
 720 }
 721 
 722    // rRegI ereg, memory mem) %{    // emit_reg_mem
 723 void encode_RegMem(CodeBuffer &cbuf,
 724                    int reg,
 725                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 726 {
 727   assert(disp_reloc == relocInfo::none, "cannot have disp");
 728   int regenc = reg & 7;
 729   int baseenc = base & 7;
 730   int indexenc = index & 7;
 731 
 732   // There is no index & no scale, use form without SIB byte
 733   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 734     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 735     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 736       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 737     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 738       // If 8-bit displacement, mode 0x1
 739       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 740       emit_d8(cbuf, disp);
 741     } else {
 742       // If 32-bit displacement
 743       if (base == -1) { // Special flag for absolute address
 744         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 745         if (disp_reloc != relocInfo::none) {
 746           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 747         } else {
 748           emit_d32(cbuf, disp);
 749         }
 750       } else {
 751         // Normal base + offset
 752         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 753         if (disp_reloc != relocInfo::none) {
 754           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 755         } else {
 756           emit_d32(cbuf, disp);
 757         }
 758       }
 759     }
 760   } else {
 761     // Else, encode with the SIB byte
 762     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 763     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 764       // If no displacement
 765       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 766       emit_rm(cbuf, scale, indexenc, baseenc);
 767     } else {
 768       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 769         // If 8-bit displacement, mode 0x1
 770         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 771         emit_rm(cbuf, scale, indexenc, baseenc);
 772         emit_d8(cbuf, disp);
 773       } else {
 774         // If 32-bit displacement
 775         if (base == 0x04 ) {
 776           emit_rm(cbuf, 0x2, regenc, 0x4);
 777           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 778         } else {
 779           emit_rm(cbuf, 0x2, regenc, 0x4);
 780           emit_rm(cbuf, scale, indexenc, baseenc); // *
 781         }
 782         if (disp_reloc != relocInfo::none) {
 783           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 784         } else {
 785           emit_d32(cbuf, disp);
 786         }
 787       }
 788     }
 789   }
 790 }
 791 
 792 // This could be in MacroAssembler but it's fairly C2 specific
 793 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 794   Label exit;
 795   __ jccb(Assembler::noParity, exit);
 796   __ pushf();
 797   //
 798   // comiss/ucomiss instructions set ZF,PF,CF flags and
 799   // zero OF,AF,SF for NaN values.
 800   // Fixup flags by zeroing ZF,PF so that compare of NaN
 801   // values returns 'less than' result (CF is set).
 802   // Leave the rest of flags unchanged.
 803   //
 804   //    7 6 5 4 3 2 1 0
 805   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 806   //    0 0 1 0 1 0 1 1   (0x2B)
 807   //
 808   __ andq(Address(rsp, 0), 0xffffff2b);
 809   __ popf();
 810   __ bind(exit);
 811 }
 812 
 813 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 814   Label done;
 815   __ movl(dst, -1);
 816   __ jcc(Assembler::parity, done);
 817   __ jcc(Assembler::below, done);
 818   __ setb(Assembler::notEqual, dst);
 819   __ movzbl(dst, dst);
 820   __ bind(done);
 821 }
 822 
 823 
 824 //=============================================================================
 825 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 826 
 827 int Compile::ConstantTable::calculate_table_base_offset() const {
 828   return 0;  // absolute addressing, no offset
 829 }
 830 
 831 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 832 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 833   ShouldNotReachHere();
 834 }
 835 
 836 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 837   // Empty encoding
 838 }
 839 
 840 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 841   return 0;
 842 }
 843 
 844 #ifndef PRODUCT
 845 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 846   st->print("# MachConstantBaseNode (empty encoding)");
 847 }
 848 #endif
 849 
 850 
 851 //=============================================================================
 852 #ifndef PRODUCT
 853 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 854   Compile* C = ra_->C;
 855 
 856   int framesize = C->frame_size_in_bytes();
 857   int bangsize = C->bang_size_in_bytes();
 858   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 859   // Remove wordSize for return addr which is already pushed.
 860   framesize -= wordSize;
 861 
 862   if (C->need_stack_bang(bangsize)) {
 863     framesize -= wordSize;
 864     st->print("# stack bang (%d bytes)", bangsize);
 865     st->print("\n\t");
 866     st->print("pushq   rbp\t# Save rbp");
 867     if (PreserveFramePointer) {
 868         st->print("\n\t");
 869         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 870     }
 871     if (framesize) {
 872       st->print("\n\t");
 873       st->print("subq    rsp, #%d\t# Create frame",framesize);
 874     }
 875   } else {
 876     st->print("subq    rsp, #%d\t# Create frame",framesize);
 877     st->print("\n\t");
 878     framesize -= wordSize;
 879     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 880     if (PreserveFramePointer) {
 881       st->print("\n\t");
 882       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 883       if (framesize > 0) {
 884         st->print("\n\t");
 885         st->print("addq    rbp, #%d", framesize);
 886       }
 887     }
 888   }
 889 
 890   if (VerifyStackAtCalls) {
 891     st->print("\n\t");
 892     framesize -= wordSize;
 893     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 894 #ifdef ASSERT
 895     st->print("\n\t");
 896     st->print("# stack alignment check");
 897 #endif
 898   }
 899   st->cr();
 900 }
 901 #endif
 902 
 903 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 904   Compile* C = ra_->C;
 905   MacroAssembler _masm(&cbuf);
 906 
 907   int framesize = C->frame_size_in_bytes();
 908   int bangsize = C->bang_size_in_bytes();
 909 
 910   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 911 
 912   C->set_frame_complete(cbuf.insts_size());
 913 
 914   if (C->has_mach_constant_base_node()) {
 915     // NOTE: We set the table base offset here because users might be
 916     // emitted before MachConstantBaseNode.
 917     Compile::ConstantTable& constant_table = C->constant_table();
 918     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 919   }
 920 }
 921 
 922 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 923 {
 924   return MachNode::size(ra_); // too many variables; just compute it
 925                               // the hard way
 926 }
 927 
 928 int MachPrologNode::reloc() const
 929 {
 930   return 0; // a large enough number
 931 }
 932 
 933 //=============================================================================
 934 #ifndef PRODUCT
 935 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 936 {
 937   Compile* C = ra_->C;
 938   if (generate_vzeroupper(C)) {
 939     st->print("vzeroupper");
 940     st->cr(); st->print("\t");
 941   }
 942 
 943   int framesize = C->frame_size_in_bytes();
 944   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 945   // Remove word for return adr already pushed
 946   // and RBP
 947   framesize -= 2*wordSize;
 948 
 949   if (framesize) {
 950     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 951     st->print("\t");
 952   }
 953 
 954   st->print_cr("popq   rbp");
 955   if (do_polling() && C->is_method_compilation()) {
 956     st->print("\t");
 957     if (SafepointMechanism::uses_thread_local_poll()) {
 958       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 959                    "testl  rax, [rscratch1]\t"
 960                    "# Safepoint: poll for GC");
 961     } else if (Assembler::is_polling_page_far()) {
 962       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 963                    "testl  rax, [rscratch1]\t"
 964                    "# Safepoint: poll for GC");
 965     } else {
 966       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 967                    "# Safepoint: poll for GC");
 968     }
 969   }
 970 }
 971 #endif
 972 
 973 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 974 {
 975   Compile* C = ra_->C;
 976   MacroAssembler _masm(&cbuf);
 977 
 978   if (generate_vzeroupper(C)) {
 979     // Clear upper bits of YMM registers when current compiled code uses
 980     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 981     __ vzeroupper();
 982   }
 983 
 984   int framesize = C->frame_size_in_bytes();
 985   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 986   // Remove word for return adr already pushed
 987   // and RBP
 988   framesize -= 2*wordSize;
 989 
 990   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 991 
 992   if (framesize) {
 993     emit_opcode(cbuf, Assembler::REX_W);
 994     if (framesize < 0x80) {
 995       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 996       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 997       emit_d8(cbuf, framesize);
 998     } else {
 999       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1000       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1001       emit_d32(cbuf, framesize);
1002     }
1003   }
1004 
1005   // popq rbp
1006   emit_opcode(cbuf, 0x58 | RBP_enc);
1007 
1008   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1009     __ reserved_stack_check();
1010   }
1011 
1012   if (do_polling() && C->is_method_compilation()) {
1013     MacroAssembler _masm(&cbuf);
1014     if (SafepointMechanism::uses_thread_local_poll()) {
1015       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1016       __ relocate(relocInfo::poll_return_type);
1017       __ testl(rax, Address(rscratch1, 0));
1018     } else {
1019       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1020       if (Assembler::is_polling_page_far()) {
1021         __ lea(rscratch1, polling_page);
1022         __ relocate(relocInfo::poll_return_type);
1023         __ testl(rax, Address(rscratch1, 0));
1024       } else {
1025         __ testl(rax, polling_page);
1026       }
1027     }
1028   }
1029 }
1030 
1031 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1032 {
1033   return MachNode::size(ra_); // too many variables; just compute it
1034                               // the hard way
1035 }
1036 
1037 int MachEpilogNode::reloc() const
1038 {
1039   return 2; // a large enough number
1040 }
1041 
1042 const Pipeline* MachEpilogNode::pipeline() const
1043 {
1044   return MachNode::pipeline_class();
1045 }
1046 
1047 int MachEpilogNode::safepoint_offset() const
1048 {
1049   return 0;
1050 }
1051 
1052 //=============================================================================
1053 
1054 enum RC {
1055   rc_bad,
1056   rc_int,
1057   rc_float,
1058   rc_stack
1059 };
1060 
1061 static enum RC rc_class(OptoReg::Name reg)
1062 {
1063   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1064 
1065   if (OptoReg::is_stack(reg)) return rc_stack;
1066 
1067   VMReg r = OptoReg::as_VMReg(reg);
1068 
1069   if (r->is_Register()) return rc_int;
1070 
1071   assert(r->is_XMMRegister(), "must be");
1072   return rc_float;
1073 }
1074 
1075 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1076 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1077                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1078 
1079 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1080                             int stack_offset, int reg, uint ireg, outputStream* st);
1081 
1082 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1083                                       int dst_offset, uint ireg, outputStream* st) {
1084   if (cbuf) {
1085     MacroAssembler _masm(cbuf);
1086     switch (ireg) {
1087     case Op_VecS:
1088       __ movq(Address(rsp, -8), rax);
1089       __ movl(rax, Address(rsp, src_offset));
1090       __ movl(Address(rsp, dst_offset), rax);
1091       __ movq(rax, Address(rsp, -8));
1092       break;
1093     case Op_VecD:
1094       __ pushq(Address(rsp, src_offset));
1095       __ popq (Address(rsp, dst_offset));
1096       break;
1097     case Op_VecX:
1098       __ pushq(Address(rsp, src_offset));
1099       __ popq (Address(rsp, dst_offset));
1100       __ pushq(Address(rsp, src_offset+8));
1101       __ popq (Address(rsp, dst_offset+8));
1102       break;
1103     case Op_VecY:
1104       __ vmovdqu(Address(rsp, -32), xmm0);
1105       __ vmovdqu(xmm0, Address(rsp, src_offset));
1106       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1107       __ vmovdqu(xmm0, Address(rsp, -32));
1108       break;
1109     case Op_VecZ:
1110       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1111       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1112       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1113       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1114       break;
1115     default:
1116       ShouldNotReachHere();
1117     }
1118 #ifndef PRODUCT
1119   } else {
1120     switch (ireg) {
1121     case Op_VecS:
1122       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1123                 "movl    rax, [rsp + #%d]\n\t"
1124                 "movl    [rsp + #%d], rax\n\t"
1125                 "movq    rax, [rsp - #8]",
1126                 src_offset, dst_offset);
1127       break;
1128     case Op_VecD:
1129       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1130                 "popq    [rsp + #%d]",
1131                 src_offset, dst_offset);
1132       break;
1133      case Op_VecX:
1134       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1135                 "popq    [rsp + #%d]\n\t"
1136                 "pushq   [rsp + #%d]\n\t"
1137                 "popq    [rsp + #%d]",
1138                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1139       break;
1140     case Op_VecY:
1141       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1142                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1143                 "vmovdqu [rsp + #%d], xmm0\n\t"
1144                 "vmovdqu xmm0, [rsp - #32]",
1145                 src_offset, dst_offset);
1146       break;
1147     case Op_VecZ:
1148       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1149                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1150                 "vmovdqu [rsp + #%d], xmm0\n\t"
1151                 "vmovdqu xmm0, [rsp - #64]",
1152                 src_offset, dst_offset);
1153       break;
1154     default:
1155       ShouldNotReachHere();
1156     }
1157 #endif
1158   }
1159 }
1160 
1161 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1162                                        PhaseRegAlloc* ra_,
1163                                        bool do_size,
1164                                        outputStream* st) const {
1165   assert(cbuf != NULL || st  != NULL, "sanity");
1166   // Get registers to move
1167   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1168   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1169   OptoReg::Name dst_second = ra_->get_reg_second(this);
1170   OptoReg::Name dst_first = ra_->get_reg_first(this);
1171 
1172   enum RC src_second_rc = rc_class(src_second);
1173   enum RC src_first_rc = rc_class(src_first);
1174   enum RC dst_second_rc = rc_class(dst_second);
1175   enum RC dst_first_rc = rc_class(dst_first);
1176 
1177   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1178          "must move at least 1 register" );
1179 
1180   if (src_first == dst_first && src_second == dst_second) {
1181     // Self copy, no move
1182     return 0;
1183   }
1184   if (bottom_type()->isa_vect() != NULL) {
1185     uint ireg = ideal_reg();
1186     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1187     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1188     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1189       // mem -> mem
1190       int src_offset = ra_->reg2offset(src_first);
1191       int dst_offset = ra_->reg2offset(dst_first);
1192       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1193     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1194       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1195     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1196       int stack_offset = ra_->reg2offset(dst_first);
1197       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1198     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1199       int stack_offset = ra_->reg2offset(src_first);
1200       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1201     } else {
1202       ShouldNotReachHere();
1203     }
1204     return 0;
1205   }
1206   if (src_first_rc == rc_stack) {
1207     // mem ->
1208     if (dst_first_rc == rc_stack) {
1209       // mem -> mem
1210       assert(src_second != dst_first, "overlap");
1211       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1212           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1213         // 64-bit
1214         int src_offset = ra_->reg2offset(src_first);
1215         int dst_offset = ra_->reg2offset(dst_first);
1216         if (cbuf) {
1217           MacroAssembler _masm(cbuf);
1218           __ pushq(Address(rsp, src_offset));
1219           __ popq (Address(rsp, dst_offset));
1220 #ifndef PRODUCT
1221         } else {
1222           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1223                     "popq    [rsp + #%d]",
1224                      src_offset, dst_offset);
1225 #endif
1226         }
1227       } else {
1228         // 32-bit
1229         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1230         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1231         // No pushl/popl, so:
1232         int src_offset = ra_->reg2offset(src_first);
1233         int dst_offset = ra_->reg2offset(dst_first);
1234         if (cbuf) {
1235           MacroAssembler _masm(cbuf);
1236           __ movq(Address(rsp, -8), rax);
1237           __ movl(rax, Address(rsp, src_offset));
1238           __ movl(Address(rsp, dst_offset), rax);
1239           __ movq(rax, Address(rsp, -8));
1240 #ifndef PRODUCT
1241         } else {
1242           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1243                     "movl    rax, [rsp + #%d]\n\t"
1244                     "movl    [rsp + #%d], rax\n\t"
1245                     "movq    rax, [rsp - #8]",
1246                      src_offset, dst_offset);
1247 #endif
1248         }
1249       }
1250       return 0;
1251     } else if (dst_first_rc == rc_int) {
1252       // mem -> gpr
1253       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1254           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1255         // 64-bit
1256         int offset = ra_->reg2offset(src_first);
1257         if (cbuf) {
1258           MacroAssembler _masm(cbuf);
1259           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1260 #ifndef PRODUCT
1261         } else {
1262           st->print("movq    %s, [rsp + #%d]\t# spill",
1263                      Matcher::regName[dst_first],
1264                      offset);
1265 #endif
1266         }
1267       } else {
1268         // 32-bit
1269         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1270         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1271         int offset = ra_->reg2offset(src_first);
1272         if (cbuf) {
1273           MacroAssembler _masm(cbuf);
1274           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1275 #ifndef PRODUCT
1276         } else {
1277           st->print("movl    %s, [rsp + #%d]\t# spill",
1278                      Matcher::regName[dst_first],
1279                      offset);
1280 #endif
1281         }
1282       }
1283       return 0;
1284     } else if (dst_first_rc == rc_float) {
1285       // mem-> xmm
1286       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1287           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1288         // 64-bit
1289         int offset = ra_->reg2offset(src_first);
1290         if (cbuf) {
1291           MacroAssembler _masm(cbuf);
1292           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1293 #ifndef PRODUCT
1294         } else {
1295           st->print("%s  %s, [rsp + #%d]\t# spill",
1296                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1297                      Matcher::regName[dst_first],
1298                      offset);
1299 #endif
1300         }
1301       } else {
1302         // 32-bit
1303         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1304         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1305         int offset = ra_->reg2offset(src_first);
1306         if (cbuf) {
1307           MacroAssembler _masm(cbuf);
1308           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1309 #ifndef PRODUCT
1310         } else {
1311           st->print("movss   %s, [rsp + #%d]\t# spill",
1312                      Matcher::regName[dst_first],
1313                      offset);
1314 #endif
1315         }
1316       }
1317       return 0;
1318     }
1319   } else if (src_first_rc == rc_int) {
1320     // gpr ->
1321     if (dst_first_rc == rc_stack) {
1322       // gpr -> mem
1323       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1324           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1325         // 64-bit
1326         int offset = ra_->reg2offset(dst_first);
1327         if (cbuf) {
1328           MacroAssembler _masm(cbuf);
1329           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1330 #ifndef PRODUCT
1331         } else {
1332           st->print("movq    [rsp + #%d], %s\t# spill",
1333                      offset,
1334                      Matcher::regName[src_first]);
1335 #endif
1336         }
1337       } else {
1338         // 32-bit
1339         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1340         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1341         int offset = ra_->reg2offset(dst_first);
1342         if (cbuf) {
1343           MacroAssembler _masm(cbuf);
1344           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1345 #ifndef PRODUCT
1346         } else {
1347           st->print("movl    [rsp + #%d], %s\t# spill",
1348                      offset,
1349                      Matcher::regName[src_first]);
1350 #endif
1351         }
1352       }
1353       return 0;
1354     } else if (dst_first_rc == rc_int) {
1355       // gpr -> gpr
1356       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1357           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1358         // 64-bit
1359         if (cbuf) {
1360           MacroAssembler _masm(cbuf);
1361           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1362                   as_Register(Matcher::_regEncode[src_first]));
1363 #ifndef PRODUCT
1364         } else {
1365           st->print("movq    %s, %s\t# spill",
1366                      Matcher::regName[dst_first],
1367                      Matcher::regName[src_first]);
1368 #endif
1369         }
1370         return 0;
1371       } else {
1372         // 32-bit
1373         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1374         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1378                   as_Register(Matcher::_regEncode[src_first]));
1379 #ifndef PRODUCT
1380         } else {
1381           st->print("movl    %s, %s\t# spill",
1382                      Matcher::regName[dst_first],
1383                      Matcher::regName[src_first]);
1384 #endif
1385         }
1386         return 0;
1387       }
1388     } else if (dst_first_rc == rc_float) {
1389       // gpr -> xmm
1390       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1391           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1392         // 64-bit
1393         if (cbuf) {
1394           MacroAssembler _masm(cbuf);
1395           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1396 #ifndef PRODUCT
1397         } else {
1398           st->print("movdq   %s, %s\t# spill",
1399                      Matcher::regName[dst_first],
1400                      Matcher::regName[src_first]);
1401 #endif
1402         }
1403       } else {
1404         // 32-bit
1405         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1406         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1407         if (cbuf) {
1408           MacroAssembler _masm(cbuf);
1409           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1410 #ifndef PRODUCT
1411         } else {
1412           st->print("movdl   %s, %s\t# spill",
1413                      Matcher::regName[dst_first],
1414                      Matcher::regName[src_first]);
1415 #endif
1416         }
1417       }
1418       return 0;
1419     }
1420   } else if (src_first_rc == rc_float) {
1421     // xmm ->
1422     if (dst_first_rc == rc_stack) {
1423       // xmm -> mem
1424       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1425           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1426         // 64-bit
1427         int offset = ra_->reg2offset(dst_first);
1428         if (cbuf) {
1429           MacroAssembler _masm(cbuf);
1430           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1431 #ifndef PRODUCT
1432         } else {
1433           st->print("movsd   [rsp + #%d], %s\t# spill",
1434                      offset,
1435                      Matcher::regName[src_first]);
1436 #endif
1437         }
1438       } else {
1439         // 32-bit
1440         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1441         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1442         int offset = ra_->reg2offset(dst_first);
1443         if (cbuf) {
1444           MacroAssembler _masm(cbuf);
1445           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1446 #ifndef PRODUCT
1447         } else {
1448           st->print("movss   [rsp + #%d], %s\t# spill",
1449                      offset,
1450                      Matcher::regName[src_first]);
1451 #endif
1452         }
1453       }
1454       return 0;
1455     } else if (dst_first_rc == rc_int) {
1456       // xmm -> gpr
1457       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1458           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1459         // 64-bit
1460         if (cbuf) {
1461           MacroAssembler _masm(cbuf);
1462           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1463 #ifndef PRODUCT
1464         } else {
1465           st->print("movdq   %s, %s\t# spill",
1466                      Matcher::regName[dst_first],
1467                      Matcher::regName[src_first]);
1468 #endif
1469         }
1470       } else {
1471         // 32-bit
1472         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1473         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1474         if (cbuf) {
1475           MacroAssembler _masm(cbuf);
1476           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1477 #ifndef PRODUCT
1478         } else {
1479           st->print("movdl   %s, %s\t# spill",
1480                      Matcher::regName[dst_first],
1481                      Matcher::regName[src_first]);
1482 #endif
1483         }
1484       }
1485       return 0;
1486     } else if (dst_first_rc == rc_float) {
1487       // xmm -> xmm
1488       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1489           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1490         // 64-bit
1491         if (cbuf) {
1492           MacroAssembler _masm(cbuf);
1493           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1494 #ifndef PRODUCT
1495         } else {
1496           st->print("%s  %s, %s\t# spill",
1497                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1498                      Matcher::regName[dst_first],
1499                      Matcher::regName[src_first]);
1500 #endif
1501         }
1502       } else {
1503         // 32-bit
1504         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1505         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1506         if (cbuf) {
1507           MacroAssembler _masm(cbuf);
1508           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1509 #ifndef PRODUCT
1510         } else {
1511           st->print("%s  %s, %s\t# spill",
1512                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1513                      Matcher::regName[dst_first],
1514                      Matcher::regName[src_first]);
1515 #endif
1516         }
1517       }
1518       return 0;
1519     }
1520   }
1521 
1522   assert(0," foo ");
1523   Unimplemented();
1524   return 0;
1525 }
1526 
1527 #ifndef PRODUCT
1528 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1529   implementation(NULL, ra_, false, st);
1530 }
1531 #endif
1532 
1533 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1534   implementation(&cbuf, ra_, false, NULL);
1535 }
1536 
1537 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1538   return MachNode::size(ra_);
1539 }
1540 
1541 //=============================================================================
1542 #ifndef PRODUCT
1543 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1544 {
1545   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1546   int reg = ra_->get_reg_first(this);
1547   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1548             Matcher::regName[reg], offset);
1549 }
1550 #endif
1551 
1552 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1553 {
1554   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1555   int reg = ra_->get_encode(this);
1556   if (offset >= 0x80) {
1557     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1558     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1559     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1560     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1561     emit_d32(cbuf, offset);
1562   } else {
1563     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1564     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1565     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1566     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1567     emit_d8(cbuf, offset);
1568   }
1569 }
1570 
1571 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1572 {
1573   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1574   return (offset < 0x80) ? 5 : 8; // REX
1575 }
1576 
1577 //=============================================================================
1578 #ifndef PRODUCT
1579 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1580 {
1581   if (UseCompressedClassPointers) {
1582     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1583     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1584     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1585   } else {
1586     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1587                  "# Inline cache check");
1588   }
1589   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1590   st->print_cr("\tnop\t# nops to align entry point");
1591 }
1592 #endif
1593 
1594 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1595 {
1596   MacroAssembler masm(&cbuf);
1597   uint insts_size = cbuf.insts_size();
1598   if (UseCompressedClassPointers) {
1599     masm.load_klass(rscratch1, j_rarg0);
1600     masm.cmpptr(rax, rscratch1);
1601   } else {
1602     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1603   }
1604 
1605   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1606 
1607   /* WARNING these NOPs are critical so that verified entry point is properly
1608      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1609   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1610   if (OptoBreakpoint) {
1611     // Leave space for int3
1612     nops_cnt -= 1;
1613   }
1614   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1615   if (nops_cnt > 0)
1616     masm.nop(nops_cnt);
1617 }
1618 
1619 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1620 {
1621   return MachNode::size(ra_); // too many variables; just compute it
1622                               // the hard way
1623 }
1624 
1625 
1626 //=============================================================================
1627 
1628 int Matcher::regnum_to_fpu_offset(int regnum)
1629 {
1630   return regnum - 32; // The FP registers are in the second chunk
1631 }
1632 
1633 // This is UltraSparc specific, true just means we have fast l2f conversion
1634 const bool Matcher::convL2FSupported(void) {
1635   return true;
1636 }
1637 
1638 // Is this branch offset short enough that a short branch can be used?
1639 //
1640 // NOTE: If the platform does not provide any short branch variants, then
1641 //       this method should return false for offset 0.
1642 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1643   // The passed offset is relative to address of the branch.
1644   // On 86 a branch displacement is calculated relative to address
1645   // of a next instruction.
1646   offset -= br_size;
1647 
1648   // the short version of jmpConUCF2 contains multiple branches,
1649   // making the reach slightly less
1650   if (rule == jmpConUCF2_rule)
1651     return (-126 <= offset && offset <= 125);
1652   return (-128 <= offset && offset <= 127);
1653 }
1654 
1655 const bool Matcher::isSimpleConstant64(jlong value) {
1656   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1657   //return value == (int) value;  // Cf. storeImmL and immL32.
1658 
1659   // Probably always true, even if a temp register is required.
1660   return true;
1661 }
1662 
1663 // The ecx parameter to rep stosq for the ClearArray node is in words.
1664 const bool Matcher::init_array_count_is_in_bytes = false;
1665 
1666 // No additional cost for CMOVL.
1667 const int Matcher::long_cmove_cost() { return 0; }
1668 
1669 // No CMOVF/CMOVD with SSE2
1670 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1671 
1672 // Does the CPU require late expand (see block.cpp for description of late expand)?
1673 const bool Matcher::require_postalloc_expand = false;
1674 
1675 // Do we need to mask the count passed to shift instructions or does
1676 // the cpu only look at the lower 5/6 bits anyway?
1677 const bool Matcher::need_masked_shift_count = false;
1678 
1679 bool Matcher::narrow_oop_use_complex_address() {
1680   assert(UseCompressedOops, "only for compressed oops code");
1681   return (LogMinObjAlignmentInBytes <= 3);
1682 }
1683 
1684 bool Matcher::narrow_klass_use_complex_address() {
1685   assert(UseCompressedClassPointers, "only for compressed klass code");
1686   return (LogKlassAlignmentInBytes <= 3);
1687 }
1688 
1689 bool Matcher::const_oop_prefer_decode() {
1690   // Prefer ConN+DecodeN over ConP.
1691   return true;
1692 }
1693 
1694 bool Matcher::const_klass_prefer_decode() {
1695   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1696   //       or condisider the following:
1697   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1698   //return Universe::narrow_klass_base() == NULL;
1699   return true;
1700 }
1701 
1702 // Is it better to copy float constants, or load them directly from
1703 // memory?  Intel can load a float constant from a direct address,
1704 // requiring no extra registers.  Most RISCs will have to materialize
1705 // an address into a register first, so they would do better to copy
1706 // the constant from stack.
1707 const bool Matcher::rematerialize_float_constants = true; // XXX
1708 
1709 // If CPU can load and store mis-aligned doubles directly then no
1710 // fixup is needed.  Else we split the double into 2 integer pieces
1711 // and move it piece-by-piece.  Only happens when passing doubles into
1712 // C code as the Java calling convention forces doubles to be aligned.
1713 const bool Matcher::misaligned_doubles_ok = true;
1714 
1715 // No-op on amd64
1716 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1717 
1718 // Advertise here if the CPU requires explicit rounding operations to
1719 // implement the UseStrictFP mode.
1720 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1721 
1722 // Are floats conerted to double when stored to stack during deoptimization?
1723 // On x64 it is stored without convertion so we can use normal access.
1724 bool Matcher::float_in_double() { return false; }
1725 
1726 // Do ints take an entire long register or just half?
1727 const bool Matcher::int_in_long = true;
1728 
1729 // Return whether or not this register is ever used as an argument.
1730 // This function is used on startup to build the trampoline stubs in
1731 // generateOptoStub.  Registers not mentioned will be killed by the VM
1732 // call in the trampoline, and arguments in those registers not be
1733 // available to the callee.
1734 bool Matcher::can_be_java_arg(int reg)
1735 {
1736   return
1737     reg ==  RDI_num || reg == RDI_H_num ||
1738     reg ==  RSI_num || reg == RSI_H_num ||
1739     reg ==  RDX_num || reg == RDX_H_num ||
1740     reg ==  RCX_num || reg == RCX_H_num ||
1741     reg ==   R8_num || reg ==  R8_H_num ||
1742     reg ==   R9_num || reg ==  R9_H_num ||
1743     reg ==  R12_num || reg == R12_H_num ||
1744     reg == XMM0_num || reg == XMM0b_num ||
1745     reg == XMM1_num || reg == XMM1b_num ||
1746     reg == XMM2_num || reg == XMM2b_num ||
1747     reg == XMM3_num || reg == XMM3b_num ||
1748     reg == XMM4_num || reg == XMM4b_num ||
1749     reg == XMM5_num || reg == XMM5b_num ||
1750     reg == XMM6_num || reg == XMM6b_num ||
1751     reg == XMM7_num || reg == XMM7b_num;
1752 }
1753 
1754 bool Matcher::is_spillable_arg(int reg)
1755 {
1756   return can_be_java_arg(reg);
1757 }
1758 
1759 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1760   // In 64 bit mode a code which use multiply when
1761   // devisor is constant is faster than hardware
1762   // DIV instruction (it uses MulHiL).
1763   return false;
1764 }
1765 
1766 // Register for DIVI projection of divmodI
1767 RegMask Matcher::divI_proj_mask() {
1768   return INT_RAX_REG_mask();
1769 }
1770 
1771 // Register for MODI projection of divmodI
1772 RegMask Matcher::modI_proj_mask() {
1773   return INT_RDX_REG_mask();
1774 }
1775 
1776 // Register for DIVL projection of divmodL
1777 RegMask Matcher::divL_proj_mask() {
1778   return LONG_RAX_REG_mask();
1779 }
1780 
1781 // Register for MODL projection of divmodL
1782 RegMask Matcher::modL_proj_mask() {
1783   return LONG_RDX_REG_mask();
1784 }
1785 
1786 // Register for saving SP into on method handle invokes. Not used on x86_64.
1787 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1788     return NO_REG_mask();
1789 }
1790 
1791 %}
1792 
1793 //----------ENCODING BLOCK-----------------------------------------------------
1794 // This block specifies the encoding classes used by the compiler to
1795 // output byte streams.  Encoding classes are parameterized macros
1796 // used by Machine Instruction Nodes in order to generate the bit
1797 // encoding of the instruction.  Operands specify their base encoding
1798 // interface with the interface keyword.  There are currently
1799 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1800 // COND_INTER.  REG_INTER causes an operand to generate a function
1801 // which returns its register number when queried.  CONST_INTER causes
1802 // an operand to generate a function which returns the value of the
1803 // constant when queried.  MEMORY_INTER causes an operand to generate
1804 // four functions which return the Base Register, the Index Register,
1805 // the Scale Value, and the Offset Value of the operand when queried.
1806 // COND_INTER causes an operand to generate six functions which return
1807 // the encoding code (ie - encoding bits for the instruction)
1808 // associated with each basic boolean condition for a conditional
1809 // instruction.
1810 //
1811 // Instructions specify two basic values for encoding.  Again, a
1812 // function is available to check if the constant displacement is an
1813 // oop. They use the ins_encode keyword to specify their encoding
1814 // classes (which must be a sequence of enc_class names, and their
1815 // parameters, specified in the encoding block), and they use the
1816 // opcode keyword to specify, in order, their primary, secondary, and
1817 // tertiary opcode.  Only the opcode sections which a particular
1818 // instruction needs for encoding need to be specified.
1819 encode %{
1820   // Build emit functions for each basic byte or larger field in the
1821   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1822   // from C++ code in the enc_class source block.  Emit functions will
1823   // live in the main source block for now.  In future, we can
1824   // generalize this by adding a syntax that specifies the sizes of
1825   // fields in an order, so that the adlc can build the emit functions
1826   // automagically
1827 
1828   // Emit primary opcode
1829   enc_class OpcP
1830   %{
1831     emit_opcode(cbuf, $primary);
1832   %}
1833 
1834   // Emit secondary opcode
1835   enc_class OpcS
1836   %{
1837     emit_opcode(cbuf, $secondary);
1838   %}
1839 
1840   // Emit tertiary opcode
1841   enc_class OpcT
1842   %{
1843     emit_opcode(cbuf, $tertiary);
1844   %}
1845 
1846   // Emit opcode directly
1847   enc_class Opcode(immI d8)
1848   %{
1849     emit_opcode(cbuf, $d8$$constant);
1850   %}
1851 
1852   // Emit size prefix
1853   enc_class SizePrefix
1854   %{
1855     emit_opcode(cbuf, 0x66);
1856   %}
1857 
1858   enc_class reg(rRegI reg)
1859   %{
1860     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1861   %}
1862 
1863   enc_class reg_reg(rRegI dst, rRegI src)
1864   %{
1865     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1866   %}
1867 
1868   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1869   %{
1870     emit_opcode(cbuf, $opcode$$constant);
1871     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1872   %}
1873 
1874   enc_class cdql_enc(no_rax_rdx_RegI div)
1875   %{
1876     // Full implementation of Java idiv and irem; checks for
1877     // special case as described in JVM spec., p.243 & p.271.
1878     //
1879     //         normal case                           special case
1880     //
1881     // input : rax: dividend                         min_int
1882     //         reg: divisor                          -1
1883     //
1884     // output: rax: quotient  (= rax idiv reg)       min_int
1885     //         rdx: remainder (= rax irem reg)       0
1886     //
1887     //  Code sequnce:
1888     //
1889     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1890     //    5:   75 07/08                jne    e <normal>
1891     //    7:   33 d2                   xor    %edx,%edx
1892     //  [div >= 8 -> offset + 1]
1893     //  [REX_B]
1894     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1895     //    c:   74 03/04                je     11 <done>
1896     // 000000000000000e <normal>:
1897     //    e:   99                      cltd
1898     //  [div >= 8 -> offset + 1]
1899     //  [REX_B]
1900     //    f:   f7 f9                   idiv   $div
1901     // 0000000000000011 <done>:
1902 
1903     // cmp    $0x80000000,%eax
1904     emit_opcode(cbuf, 0x3d);
1905     emit_d8(cbuf, 0x00);
1906     emit_d8(cbuf, 0x00);
1907     emit_d8(cbuf, 0x00);
1908     emit_d8(cbuf, 0x80);
1909 
1910     // jne    e <normal>
1911     emit_opcode(cbuf, 0x75);
1912     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1913 
1914     // xor    %edx,%edx
1915     emit_opcode(cbuf, 0x33);
1916     emit_d8(cbuf, 0xD2);
1917 
1918     // cmp    $0xffffffffffffffff,%ecx
1919     if ($div$$reg >= 8) {
1920       emit_opcode(cbuf, Assembler::REX_B);
1921     }
1922     emit_opcode(cbuf, 0x83);
1923     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1924     emit_d8(cbuf, 0xFF);
1925 
1926     // je     11 <done>
1927     emit_opcode(cbuf, 0x74);
1928     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1929 
1930     // <normal>
1931     // cltd
1932     emit_opcode(cbuf, 0x99);
1933 
1934     // idivl (note: must be emitted by the user of this rule)
1935     // <done>
1936   %}
1937 
1938   enc_class cdqq_enc(no_rax_rdx_RegL div)
1939   %{
1940     // Full implementation of Java ldiv and lrem; checks for
1941     // special case as described in JVM spec., p.243 & p.271.
1942     //
1943     //         normal case                           special case
1944     //
1945     // input : rax: dividend                         min_long
1946     //         reg: divisor                          -1
1947     //
1948     // output: rax: quotient  (= rax idiv reg)       min_long
1949     //         rdx: remainder (= rax irem reg)       0
1950     //
1951     //  Code sequnce:
1952     //
1953     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1954     //    7:   00 00 80
1955     //    a:   48 39 d0                cmp    %rdx,%rax
1956     //    d:   75 08                   jne    17 <normal>
1957     //    f:   33 d2                   xor    %edx,%edx
1958     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1959     //   15:   74 05                   je     1c <done>
1960     // 0000000000000017 <normal>:
1961     //   17:   48 99                   cqto
1962     //   19:   48 f7 f9                idiv   $div
1963     // 000000000000001c <done>:
1964 
1965     // mov    $0x8000000000000000,%rdx
1966     emit_opcode(cbuf, Assembler::REX_W);
1967     emit_opcode(cbuf, 0xBA);
1968     emit_d8(cbuf, 0x00);
1969     emit_d8(cbuf, 0x00);
1970     emit_d8(cbuf, 0x00);
1971     emit_d8(cbuf, 0x00);
1972     emit_d8(cbuf, 0x00);
1973     emit_d8(cbuf, 0x00);
1974     emit_d8(cbuf, 0x00);
1975     emit_d8(cbuf, 0x80);
1976 
1977     // cmp    %rdx,%rax
1978     emit_opcode(cbuf, Assembler::REX_W);
1979     emit_opcode(cbuf, 0x39);
1980     emit_d8(cbuf, 0xD0);
1981 
1982     // jne    17 <normal>
1983     emit_opcode(cbuf, 0x75);
1984     emit_d8(cbuf, 0x08);
1985 
1986     // xor    %edx,%edx
1987     emit_opcode(cbuf, 0x33);
1988     emit_d8(cbuf, 0xD2);
1989 
1990     // cmp    $0xffffffffffffffff,$div
1991     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1992     emit_opcode(cbuf, 0x83);
1993     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1994     emit_d8(cbuf, 0xFF);
1995 
1996     // je     1e <done>
1997     emit_opcode(cbuf, 0x74);
1998     emit_d8(cbuf, 0x05);
1999 
2000     // <normal>
2001     // cqto
2002     emit_opcode(cbuf, Assembler::REX_W);
2003     emit_opcode(cbuf, 0x99);
2004 
2005     // idivq (note: must be emitted by the user of this rule)
2006     // <done>
2007   %}
2008 
2009   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2010   enc_class OpcSE(immI imm)
2011   %{
2012     // Emit primary opcode and set sign-extend bit
2013     // Check for 8-bit immediate, and set sign extend bit in opcode
2014     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2015       emit_opcode(cbuf, $primary | 0x02);
2016     } else {
2017       // 32-bit immediate
2018       emit_opcode(cbuf, $primary);
2019     }
2020   %}
2021 
2022   enc_class OpcSErm(rRegI dst, immI imm)
2023   %{
2024     // OpcSEr/m
2025     int dstenc = $dst$$reg;
2026     if (dstenc >= 8) {
2027       emit_opcode(cbuf, Assembler::REX_B);
2028       dstenc -= 8;
2029     }
2030     // Emit primary opcode and set sign-extend bit
2031     // Check for 8-bit immediate, and set sign extend bit in opcode
2032     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2033       emit_opcode(cbuf, $primary | 0x02);
2034     } else {
2035       // 32-bit immediate
2036       emit_opcode(cbuf, $primary);
2037     }
2038     // Emit r/m byte with secondary opcode, after primary opcode.
2039     emit_rm(cbuf, 0x3, $secondary, dstenc);
2040   %}
2041 
2042   enc_class OpcSErm_wide(rRegL dst, immI imm)
2043   %{
2044     // OpcSEr/m
2045     int dstenc = $dst$$reg;
2046     if (dstenc < 8) {
2047       emit_opcode(cbuf, Assembler::REX_W);
2048     } else {
2049       emit_opcode(cbuf, Assembler::REX_WB);
2050       dstenc -= 8;
2051     }
2052     // Emit primary opcode and set sign-extend bit
2053     // Check for 8-bit immediate, and set sign extend bit in opcode
2054     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2055       emit_opcode(cbuf, $primary | 0x02);
2056     } else {
2057       // 32-bit immediate
2058       emit_opcode(cbuf, $primary);
2059     }
2060     // Emit r/m byte with secondary opcode, after primary opcode.
2061     emit_rm(cbuf, 0x3, $secondary, dstenc);
2062   %}
2063 
2064   enc_class Con8or32(immI imm)
2065   %{
2066     // Check for 8-bit immediate, and set sign extend bit in opcode
2067     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2068       $$$emit8$imm$$constant;
2069     } else {
2070       // 32-bit immediate
2071       $$$emit32$imm$$constant;
2072     }
2073   %}
2074 
2075   enc_class opc2_reg(rRegI dst)
2076   %{
2077     // BSWAP
2078     emit_cc(cbuf, $secondary, $dst$$reg);
2079   %}
2080 
2081   enc_class opc3_reg(rRegI dst)
2082   %{
2083     // BSWAP
2084     emit_cc(cbuf, $tertiary, $dst$$reg);
2085   %}
2086 
2087   enc_class reg_opc(rRegI div)
2088   %{
2089     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2090     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2091   %}
2092 
2093   enc_class enc_cmov(cmpOp cop)
2094   %{
2095     // CMOV
2096     $$$emit8$primary;
2097     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2098   %}
2099 
2100   enc_class enc_PartialSubtypeCheck()
2101   %{
2102     Register Rrdi = as_Register(RDI_enc); // result register
2103     Register Rrax = as_Register(RAX_enc); // super class
2104     Register Rrcx = as_Register(RCX_enc); // killed
2105     Register Rrsi = as_Register(RSI_enc); // sub class
2106     Label miss;
2107     const bool set_cond_codes = true;
2108 
2109     MacroAssembler _masm(&cbuf);
2110     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2111                                      NULL, &miss,
2112                                      /*set_cond_codes:*/ true);
2113     if ($primary) {
2114       __ xorptr(Rrdi, Rrdi);
2115     }
2116     __ bind(miss);
2117   %}
2118 
2119   enc_class clear_avx %{
2120     debug_only(int off0 = cbuf.insts_size());
2121     if (generate_vzeroupper(Compile::current())) {
2122       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2123       // Clear upper bits of YMM registers when current compiled code uses
2124       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2125       MacroAssembler _masm(&cbuf);
2126       __ vzeroupper();
2127     }
2128     debug_only(int off1 = cbuf.insts_size());
2129     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2130   %}
2131 
2132   enc_class Java_To_Runtime(method meth) %{
2133     // No relocation needed
2134     MacroAssembler _masm(&cbuf);
2135     __ mov64(r10, (int64_t) $meth$$method);
2136     __ call(r10);
2137   %}
2138 
2139   enc_class Java_To_Interpreter(method meth)
2140   %{
2141     // CALL Java_To_Interpreter
2142     // This is the instruction starting address for relocation info.
2143     cbuf.set_insts_mark();
2144     $$$emit8$primary;
2145     // CALL directly to the runtime
2146     emit_d32_reloc(cbuf,
2147                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2148                    runtime_call_Relocation::spec(),
2149                    RELOC_DISP32);
2150   %}
2151 
2152   enc_class Java_Static_Call(method meth)
2153   %{
2154     // JAVA STATIC CALL
2155     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2156     // determine who we intended to call.
2157     cbuf.set_insts_mark();
2158     $$$emit8$primary;
2159 
2160     if (!_method) {
2161       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2162                      runtime_call_Relocation::spec(),
2163                      RELOC_DISP32);
2164     } else {
2165       int method_index = resolved_method_index(cbuf);
2166       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2167                                                   : static_call_Relocation::spec(method_index);
2168       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2169                      rspec, RELOC_DISP32);
2170       // Emit stubs for static call.
2171       address mark = cbuf.insts_mark();
2172       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2173       if (stub == NULL) {
2174         ciEnv::current()->record_failure("CodeCache is full");
2175         return;
2176       }
2177 #if INCLUDE_AOT
2178       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2179 #endif
2180     }
2181   %}
2182 
2183   enc_class Java_Dynamic_Call(method meth) %{
2184     MacroAssembler _masm(&cbuf);
2185     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2186   %}
2187 
2188   enc_class Java_Compiled_Call(method meth)
2189   %{
2190     // JAVA COMPILED CALL
2191     int disp = in_bytes(Method:: from_compiled_offset());
2192 
2193     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2194     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2195 
2196     // callq *disp(%rax)
2197     cbuf.set_insts_mark();
2198     $$$emit8$primary;
2199     if (disp < 0x80) {
2200       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2201       emit_d8(cbuf, disp); // Displacement
2202     } else {
2203       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2204       emit_d32(cbuf, disp); // Displacement
2205     }
2206   %}
2207 
2208   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2209   %{
2210     // SAL, SAR, SHR
2211     int dstenc = $dst$$reg;
2212     if (dstenc >= 8) {
2213       emit_opcode(cbuf, Assembler::REX_B);
2214       dstenc -= 8;
2215     }
2216     $$$emit8$primary;
2217     emit_rm(cbuf, 0x3, $secondary, dstenc);
2218     $$$emit8$shift$$constant;
2219   %}
2220 
2221   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2222   %{
2223     // SAL, SAR, SHR
2224     int dstenc = $dst$$reg;
2225     if (dstenc < 8) {
2226       emit_opcode(cbuf, Assembler::REX_W);
2227     } else {
2228       emit_opcode(cbuf, Assembler::REX_WB);
2229       dstenc -= 8;
2230     }
2231     $$$emit8$primary;
2232     emit_rm(cbuf, 0x3, $secondary, dstenc);
2233     $$$emit8$shift$$constant;
2234   %}
2235 
2236   enc_class load_immI(rRegI dst, immI src)
2237   %{
2238     int dstenc = $dst$$reg;
2239     if (dstenc >= 8) {
2240       emit_opcode(cbuf, Assembler::REX_B);
2241       dstenc -= 8;
2242     }
2243     emit_opcode(cbuf, 0xB8 | dstenc);
2244     $$$emit32$src$$constant;
2245   %}
2246 
2247   enc_class load_immL(rRegL dst, immL src)
2248   %{
2249     int dstenc = $dst$$reg;
2250     if (dstenc < 8) {
2251       emit_opcode(cbuf, Assembler::REX_W);
2252     } else {
2253       emit_opcode(cbuf, Assembler::REX_WB);
2254       dstenc -= 8;
2255     }
2256     emit_opcode(cbuf, 0xB8 | dstenc);
2257     emit_d64(cbuf, $src$$constant);
2258   %}
2259 
2260   enc_class load_immUL32(rRegL dst, immUL32 src)
2261   %{
2262     // same as load_immI, but this time we care about zeroes in the high word
2263     int dstenc = $dst$$reg;
2264     if (dstenc >= 8) {
2265       emit_opcode(cbuf, Assembler::REX_B);
2266       dstenc -= 8;
2267     }
2268     emit_opcode(cbuf, 0xB8 | dstenc);
2269     $$$emit32$src$$constant;
2270   %}
2271 
2272   enc_class load_immL32(rRegL dst, immL32 src)
2273   %{
2274     int dstenc = $dst$$reg;
2275     if (dstenc < 8) {
2276       emit_opcode(cbuf, Assembler::REX_W);
2277     } else {
2278       emit_opcode(cbuf, Assembler::REX_WB);
2279       dstenc -= 8;
2280     }
2281     emit_opcode(cbuf, 0xC7);
2282     emit_rm(cbuf, 0x03, 0x00, dstenc);
2283     $$$emit32$src$$constant;
2284   %}
2285 
2286   enc_class load_immP31(rRegP dst, immP32 src)
2287   %{
2288     // same as load_immI, but this time we care about zeroes in the high word
2289     int dstenc = $dst$$reg;
2290     if (dstenc >= 8) {
2291       emit_opcode(cbuf, Assembler::REX_B);
2292       dstenc -= 8;
2293     }
2294     emit_opcode(cbuf, 0xB8 | dstenc);
2295     $$$emit32$src$$constant;
2296   %}
2297 
2298   enc_class load_immP(rRegP dst, immP src)
2299   %{
2300     int dstenc = $dst$$reg;
2301     if (dstenc < 8) {
2302       emit_opcode(cbuf, Assembler::REX_W);
2303     } else {
2304       emit_opcode(cbuf, Assembler::REX_WB);
2305       dstenc -= 8;
2306     }
2307     emit_opcode(cbuf, 0xB8 | dstenc);
2308     // This next line should be generated from ADLC
2309     if ($src->constant_reloc() != relocInfo::none) {
2310       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2311     } else {
2312       emit_d64(cbuf, $src$$constant);
2313     }
2314   %}
2315 
2316   enc_class Con32(immI src)
2317   %{
2318     // Output immediate
2319     $$$emit32$src$$constant;
2320   %}
2321 
2322   enc_class Con32F_as_bits(immF src)
2323   %{
2324     // Output Float immediate bits
2325     jfloat jf = $src$$constant;
2326     jint jf_as_bits = jint_cast(jf);
2327     emit_d32(cbuf, jf_as_bits);
2328   %}
2329 
2330   enc_class Con16(immI src)
2331   %{
2332     // Output immediate
2333     $$$emit16$src$$constant;
2334   %}
2335 
2336   // How is this different from Con32??? XXX
2337   enc_class Con_d32(immI src)
2338   %{
2339     emit_d32(cbuf,$src$$constant);
2340   %}
2341 
2342   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2343     // Output immediate memory reference
2344     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2345     emit_d32(cbuf, 0x00);
2346   %}
2347 
2348   enc_class lock_prefix()
2349   %{
2350     if (os::is_MP()) {
2351       emit_opcode(cbuf, 0xF0); // lock
2352     }
2353   %}
2354 
2355   enc_class REX_mem(memory mem)
2356   %{
2357     if ($mem$$base >= 8) {
2358       if ($mem$$index < 8) {
2359         emit_opcode(cbuf, Assembler::REX_B);
2360       } else {
2361         emit_opcode(cbuf, Assembler::REX_XB);
2362       }
2363     } else {
2364       if ($mem$$index >= 8) {
2365         emit_opcode(cbuf, Assembler::REX_X);
2366       }
2367     }
2368   %}
2369 
2370   enc_class REX_mem_wide(memory mem)
2371   %{
2372     if ($mem$$base >= 8) {
2373       if ($mem$$index < 8) {
2374         emit_opcode(cbuf, Assembler::REX_WB);
2375       } else {
2376         emit_opcode(cbuf, Assembler::REX_WXB);
2377       }
2378     } else {
2379       if ($mem$$index < 8) {
2380         emit_opcode(cbuf, Assembler::REX_W);
2381       } else {
2382         emit_opcode(cbuf, Assembler::REX_WX);
2383       }
2384     }
2385   %}
2386 
2387   // for byte regs
2388   enc_class REX_breg(rRegI reg)
2389   %{
2390     if ($reg$$reg >= 4) {
2391       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2392     }
2393   %}
2394 
2395   // for byte regs
2396   enc_class REX_reg_breg(rRegI dst, rRegI src)
2397   %{
2398     if ($dst$$reg < 8) {
2399       if ($src$$reg >= 4) {
2400         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2401       }
2402     } else {
2403       if ($src$$reg < 8) {
2404         emit_opcode(cbuf, Assembler::REX_R);
2405       } else {
2406         emit_opcode(cbuf, Assembler::REX_RB);
2407       }
2408     }
2409   %}
2410 
2411   // for byte regs
2412   enc_class REX_breg_mem(rRegI reg, memory mem)
2413   %{
2414     if ($reg$$reg < 8) {
2415       if ($mem$$base < 8) {
2416         if ($mem$$index >= 8) {
2417           emit_opcode(cbuf, Assembler::REX_X);
2418         } else if ($reg$$reg >= 4) {
2419           emit_opcode(cbuf, Assembler::REX);
2420         }
2421       } else {
2422         if ($mem$$index < 8) {
2423           emit_opcode(cbuf, Assembler::REX_B);
2424         } else {
2425           emit_opcode(cbuf, Assembler::REX_XB);
2426         }
2427       }
2428     } else {
2429       if ($mem$$base < 8) {
2430         if ($mem$$index < 8) {
2431           emit_opcode(cbuf, Assembler::REX_R);
2432         } else {
2433           emit_opcode(cbuf, Assembler::REX_RX);
2434         }
2435       } else {
2436         if ($mem$$index < 8) {
2437           emit_opcode(cbuf, Assembler::REX_RB);
2438         } else {
2439           emit_opcode(cbuf, Assembler::REX_RXB);
2440         }
2441       }
2442     }
2443   %}
2444 
2445   enc_class REX_reg(rRegI reg)
2446   %{
2447     if ($reg$$reg >= 8) {
2448       emit_opcode(cbuf, Assembler::REX_B);
2449     }
2450   %}
2451 
2452   enc_class REX_reg_wide(rRegI reg)
2453   %{
2454     if ($reg$$reg < 8) {
2455       emit_opcode(cbuf, Assembler::REX_W);
2456     } else {
2457       emit_opcode(cbuf, Assembler::REX_WB);
2458     }
2459   %}
2460 
2461   enc_class REX_reg_reg(rRegI dst, rRegI src)
2462   %{
2463     if ($dst$$reg < 8) {
2464       if ($src$$reg >= 8) {
2465         emit_opcode(cbuf, Assembler::REX_B);
2466       }
2467     } else {
2468       if ($src$$reg < 8) {
2469         emit_opcode(cbuf, Assembler::REX_R);
2470       } else {
2471         emit_opcode(cbuf, Assembler::REX_RB);
2472       }
2473     }
2474   %}
2475 
2476   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2477   %{
2478     if ($dst$$reg < 8) {
2479       if ($src$$reg < 8) {
2480         emit_opcode(cbuf, Assembler::REX_W);
2481       } else {
2482         emit_opcode(cbuf, Assembler::REX_WB);
2483       }
2484     } else {
2485       if ($src$$reg < 8) {
2486         emit_opcode(cbuf, Assembler::REX_WR);
2487       } else {
2488         emit_opcode(cbuf, Assembler::REX_WRB);
2489       }
2490     }
2491   %}
2492 
2493   enc_class REX_reg_mem(rRegI reg, memory mem)
2494   %{
2495     if ($reg$$reg < 8) {
2496       if ($mem$$base < 8) {
2497         if ($mem$$index >= 8) {
2498           emit_opcode(cbuf, Assembler::REX_X);
2499         }
2500       } else {
2501         if ($mem$$index < 8) {
2502           emit_opcode(cbuf, Assembler::REX_B);
2503         } else {
2504           emit_opcode(cbuf, Assembler::REX_XB);
2505         }
2506       }
2507     } else {
2508       if ($mem$$base < 8) {
2509         if ($mem$$index < 8) {
2510           emit_opcode(cbuf, Assembler::REX_R);
2511         } else {
2512           emit_opcode(cbuf, Assembler::REX_RX);
2513         }
2514       } else {
2515         if ($mem$$index < 8) {
2516           emit_opcode(cbuf, Assembler::REX_RB);
2517         } else {
2518           emit_opcode(cbuf, Assembler::REX_RXB);
2519         }
2520       }
2521     }
2522   %}
2523 
2524   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2525   %{
2526     if ($reg$$reg < 8) {
2527       if ($mem$$base < 8) {
2528         if ($mem$$index < 8) {
2529           emit_opcode(cbuf, Assembler::REX_W);
2530         } else {
2531           emit_opcode(cbuf, Assembler::REX_WX);
2532         }
2533       } else {
2534         if ($mem$$index < 8) {
2535           emit_opcode(cbuf, Assembler::REX_WB);
2536         } else {
2537           emit_opcode(cbuf, Assembler::REX_WXB);
2538         }
2539       }
2540     } else {
2541       if ($mem$$base < 8) {
2542         if ($mem$$index < 8) {
2543           emit_opcode(cbuf, Assembler::REX_WR);
2544         } else {
2545           emit_opcode(cbuf, Assembler::REX_WRX);
2546         }
2547       } else {
2548         if ($mem$$index < 8) {
2549           emit_opcode(cbuf, Assembler::REX_WRB);
2550         } else {
2551           emit_opcode(cbuf, Assembler::REX_WRXB);
2552         }
2553       }
2554     }
2555   %}
2556 
2557   enc_class reg_mem(rRegI ereg, memory mem)
2558   %{
2559     // High registers handle in encode_RegMem
2560     int reg = $ereg$$reg;
2561     int base = $mem$$base;
2562     int index = $mem$$index;
2563     int scale = $mem$$scale;
2564     int disp = $mem$$disp;
2565     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2566 
2567     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2568   %}
2569 
2570   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2571   %{
2572     int rm_byte_opcode = $rm_opcode$$constant;
2573 
2574     // High registers handle in encode_RegMem
2575     int base = $mem$$base;
2576     int index = $mem$$index;
2577     int scale = $mem$$scale;
2578     int displace = $mem$$disp;
2579 
2580     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2581                                             // working with static
2582                                             // globals
2583     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2584                   disp_reloc);
2585   %}
2586 
2587   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2588   %{
2589     int reg_encoding = $dst$$reg;
2590     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2591     int index        = 0x04;            // 0x04 indicates no index
2592     int scale        = 0x00;            // 0x00 indicates no scale
2593     int displace     = $src1$$constant; // 0x00 indicates no displacement
2594     relocInfo::relocType disp_reloc = relocInfo::none;
2595     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2596                   disp_reloc);
2597   %}
2598 
2599   enc_class neg_reg(rRegI dst)
2600   %{
2601     int dstenc = $dst$$reg;
2602     if (dstenc >= 8) {
2603       emit_opcode(cbuf, Assembler::REX_B);
2604       dstenc -= 8;
2605     }
2606     // NEG $dst
2607     emit_opcode(cbuf, 0xF7);
2608     emit_rm(cbuf, 0x3, 0x03, dstenc);
2609   %}
2610 
2611   enc_class neg_reg_wide(rRegI dst)
2612   %{
2613     int dstenc = $dst$$reg;
2614     if (dstenc < 8) {
2615       emit_opcode(cbuf, Assembler::REX_W);
2616     } else {
2617       emit_opcode(cbuf, Assembler::REX_WB);
2618       dstenc -= 8;
2619     }
2620     // NEG $dst
2621     emit_opcode(cbuf, 0xF7);
2622     emit_rm(cbuf, 0x3, 0x03, dstenc);
2623   %}
2624 
2625   enc_class setLT_reg(rRegI dst)
2626   %{
2627     int dstenc = $dst$$reg;
2628     if (dstenc >= 8) {
2629       emit_opcode(cbuf, Assembler::REX_B);
2630       dstenc -= 8;
2631     } else if (dstenc >= 4) {
2632       emit_opcode(cbuf, Assembler::REX);
2633     }
2634     // SETLT $dst
2635     emit_opcode(cbuf, 0x0F);
2636     emit_opcode(cbuf, 0x9C);
2637     emit_rm(cbuf, 0x3, 0x0, dstenc);
2638   %}
2639 
2640   enc_class setNZ_reg(rRegI dst)
2641   %{
2642     int dstenc = $dst$$reg;
2643     if (dstenc >= 8) {
2644       emit_opcode(cbuf, Assembler::REX_B);
2645       dstenc -= 8;
2646     } else if (dstenc >= 4) {
2647       emit_opcode(cbuf, Assembler::REX);
2648     }
2649     // SETNZ $dst
2650     emit_opcode(cbuf, 0x0F);
2651     emit_opcode(cbuf, 0x95);
2652     emit_rm(cbuf, 0x3, 0x0, dstenc);
2653   %}
2654 
2655 
2656   // Compare the lonogs and set -1, 0, or 1 into dst
2657   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2658   %{
2659     int src1enc = $src1$$reg;
2660     int src2enc = $src2$$reg;
2661     int dstenc = $dst$$reg;
2662 
2663     // cmpq $src1, $src2
2664     if (src1enc < 8) {
2665       if (src2enc < 8) {
2666         emit_opcode(cbuf, Assembler::REX_W);
2667       } else {
2668         emit_opcode(cbuf, Assembler::REX_WB);
2669       }
2670     } else {
2671       if (src2enc < 8) {
2672         emit_opcode(cbuf, Assembler::REX_WR);
2673       } else {
2674         emit_opcode(cbuf, Assembler::REX_WRB);
2675       }
2676     }
2677     emit_opcode(cbuf, 0x3B);
2678     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2679 
2680     // movl $dst, -1
2681     if (dstenc >= 8) {
2682       emit_opcode(cbuf, Assembler::REX_B);
2683     }
2684     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2685     emit_d32(cbuf, -1);
2686 
2687     // jl,s done
2688     emit_opcode(cbuf, 0x7C);
2689     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2690 
2691     // setne $dst
2692     if (dstenc >= 4) {
2693       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2694     }
2695     emit_opcode(cbuf, 0x0F);
2696     emit_opcode(cbuf, 0x95);
2697     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2698 
2699     // movzbl $dst, $dst
2700     if (dstenc >= 4) {
2701       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2702     }
2703     emit_opcode(cbuf, 0x0F);
2704     emit_opcode(cbuf, 0xB6);
2705     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2706   %}
2707 
2708   enc_class Push_ResultXD(regD dst) %{
2709     MacroAssembler _masm(&cbuf);
2710     __ fstp_d(Address(rsp, 0));
2711     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2712     __ addptr(rsp, 8);
2713   %}
2714 
2715   enc_class Push_SrcXD(regD src) %{
2716     MacroAssembler _masm(&cbuf);
2717     __ subptr(rsp, 8);
2718     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2719     __ fld_d(Address(rsp, 0));
2720   %}
2721 
2722 
2723   enc_class enc_rethrow()
2724   %{
2725     cbuf.set_insts_mark();
2726     emit_opcode(cbuf, 0xE9); // jmp entry
2727     emit_d32_reloc(cbuf,
2728                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2729                    runtime_call_Relocation::spec(),
2730                    RELOC_DISP32);
2731   %}
2732 
2733 %}
2734 
2735 
2736 
2737 //----------FRAME--------------------------------------------------------------
2738 // Definition of frame structure and management information.
2739 //
2740 //  S T A C K   L A Y O U T    Allocators stack-slot number
2741 //                             |   (to get allocators register number
2742 //  G  Owned by    |        |  v    add OptoReg::stack0())
2743 //  r   CALLER     |        |
2744 //  o     |        +--------+      pad to even-align allocators stack-slot
2745 //  w     V        |  pad0  |        numbers; owned by CALLER
2746 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2747 //  h     ^        |   in   |  5
2748 //        |        |  args  |  4   Holes in incoming args owned by SELF
2749 //  |     |        |        |  3
2750 //  |     |        +--------+
2751 //  V     |        | old out|      Empty on Intel, window on Sparc
2752 //        |    old |preserve|      Must be even aligned.
2753 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2754 //        |        |   in   |  3   area for Intel ret address
2755 //     Owned by    |preserve|      Empty on Sparc.
2756 //       SELF      +--------+
2757 //        |        |  pad2  |  2   pad to align old SP
2758 //        |        +--------+  1
2759 //        |        | locks  |  0
2760 //        |        +--------+----> OptoReg::stack0(), even aligned
2761 //        |        |  pad1  | 11   pad to align new SP
2762 //        |        +--------+
2763 //        |        |        | 10
2764 //        |        | spills |  9   spills
2765 //        V        |        |  8   (pad0 slot for callee)
2766 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2767 //        ^        |  out   |  7
2768 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2769 //     Owned by    +--------+
2770 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2771 //        |    new |preserve|      Must be even-aligned.
2772 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2773 //        |        |        |
2774 //
2775 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2776 //         known from SELF's arguments and the Java calling convention.
2777 //         Region 6-7 is determined per call site.
2778 // Note 2: If the calling convention leaves holes in the incoming argument
2779 //         area, those holes are owned by SELF.  Holes in the outgoing area
2780 //         are owned by the CALLEE.  Holes should not be nessecary in the
2781 //         incoming area, as the Java calling convention is completely under
2782 //         the control of the AD file.  Doubles can be sorted and packed to
2783 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2784 //         varargs C calling conventions.
2785 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2786 //         even aligned with pad0 as needed.
2787 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2788 //         region 6-11 is even aligned; it may be padded out more so that
2789 //         the region from SP to FP meets the minimum stack alignment.
2790 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2791 //         alignment.  Region 11, pad1, may be dynamically extended so that
2792 //         SP meets the minimum alignment.
2793 
2794 frame
2795 %{
2796   // What direction does stack grow in (assumed to be same for C & Java)
2797   stack_direction(TOWARDS_LOW);
2798 
2799   // These three registers define part of the calling convention
2800   // between compiled code and the interpreter.
2801   inline_cache_reg(RAX);                // Inline Cache Register
2802   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2803                                         // calling interpreter
2804 
2805   // Optional: name the operand used by cisc-spilling to access
2806   // [stack_pointer + offset]
2807   cisc_spilling_operand_name(indOffset32);
2808 
2809   // Number of stack slots consumed by locking an object
2810   sync_stack_slots(2);
2811 
2812   // Compiled code's Frame Pointer
2813   frame_pointer(RSP);
2814 
2815   // Interpreter stores its frame pointer in a register which is
2816   // stored to the stack by I2CAdaptors.
2817   // I2CAdaptors convert from interpreted java to compiled java.
2818   interpreter_frame_pointer(RBP);
2819 
2820   // Stack alignment requirement
2821   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2822 
2823   // Number of stack slots between incoming argument block and the start of
2824   // a new frame.  The PROLOG must add this many slots to the stack.  The
2825   // EPILOG must remove this many slots.  amd64 needs two slots for
2826   // return address.
2827   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2828 
2829   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2830   // for calls to C.  Supports the var-args backing area for register parms.
2831   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2832 
2833   // The after-PROLOG location of the return address.  Location of
2834   // return address specifies a type (REG or STACK) and a number
2835   // representing the register number (i.e. - use a register name) or
2836   // stack slot.
2837   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2838   // Otherwise, it is above the locks and verification slot and alignment word
2839   return_addr(STACK - 2 +
2840               align_up((Compile::current()->in_preserve_stack_slots() +
2841                         Compile::current()->fixed_slots()),
2842                        stack_alignment_in_slots()));
2843 
2844   // Body of function which returns an integer array locating
2845   // arguments either in registers or in stack slots.  Passed an array
2846   // of ideal registers called "sig" and a "length" count.  Stack-slot
2847   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2848   // arguments for a CALLEE.  Incoming stack arguments are
2849   // automatically biased by the preserve_stack_slots field above.
2850 
2851   calling_convention
2852   %{
2853     // No difference between ingoing/outgoing just pass false
2854     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2855   %}
2856 
2857   c_calling_convention
2858   %{
2859     // This is obviously always outgoing
2860     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2861   %}
2862 
2863   // Location of compiled Java return values.  Same as C for now.
2864   return_value
2865   %{
2866     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2867            "only return normal values");
2868 
2869     static const int lo[Op_RegL + 1] = {
2870       0,
2871       0,
2872       RAX_num,  // Op_RegN
2873       RAX_num,  // Op_RegI
2874       RAX_num,  // Op_RegP
2875       XMM0_num, // Op_RegF
2876       XMM0_num, // Op_RegD
2877       RAX_num   // Op_RegL
2878     };
2879     static const int hi[Op_RegL + 1] = {
2880       0,
2881       0,
2882       OptoReg::Bad, // Op_RegN
2883       OptoReg::Bad, // Op_RegI
2884       RAX_H_num,    // Op_RegP
2885       OptoReg::Bad, // Op_RegF
2886       XMM0b_num,    // Op_RegD
2887       RAX_H_num     // Op_RegL
2888     };
2889     // Excluded flags and vector registers.
2890     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2891     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2892   %}
2893 %}
2894 
2895 //----------ATTRIBUTES---------------------------------------------------------
2896 //----------Operand Attributes-------------------------------------------------
2897 op_attrib op_cost(0);        // Required cost attribute
2898 
2899 //----------Instruction Attributes---------------------------------------------
2900 ins_attrib ins_cost(100);       // Required cost attribute
2901 ins_attrib ins_size(8);         // Required size attribute (in bits)
2902 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2903                                 // a non-matching short branch variant
2904                                 // of some long branch?
2905 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2906                                 // be a power of 2) specifies the
2907                                 // alignment that some part of the
2908                                 // instruction (not necessarily the
2909                                 // start) requires.  If > 1, a
2910                                 // compute_padding() function must be
2911                                 // provided for the instruction
2912 
2913 //----------OPERANDS-----------------------------------------------------------
2914 // Operand definitions must precede instruction definitions for correct parsing
2915 // in the ADLC because operands constitute user defined types which are used in
2916 // instruction definitions.
2917 
2918 //----------Simple Operands----------------------------------------------------
2919 // Immediate Operands
2920 // Integer Immediate
2921 operand immI()
2922 %{
2923   match(ConI);
2924 
2925   op_cost(10);
2926   format %{ %}
2927   interface(CONST_INTER);
2928 %}
2929 
2930 // Constant for test vs zero
2931 operand immI0()
2932 %{
2933   predicate(n->get_int() == 0);
2934   match(ConI);
2935 
2936   op_cost(0);
2937   format %{ %}
2938   interface(CONST_INTER);
2939 %}
2940 
2941 // Constant for increment
2942 operand immI1()
2943 %{
2944   predicate(n->get_int() == 1);
2945   match(ConI);
2946 
2947   op_cost(0);
2948   format %{ %}
2949   interface(CONST_INTER);
2950 %}
2951 
2952 // Constant for decrement
2953 operand immI_M1()
2954 %{
2955   predicate(n->get_int() == -1);
2956   match(ConI);
2957 
2958   op_cost(0);
2959   format %{ %}
2960   interface(CONST_INTER);
2961 %}
2962 
2963 // Valid scale values for addressing modes
2964 operand immI2()
2965 %{
2966   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2967   match(ConI);
2968 
2969   format %{ %}
2970   interface(CONST_INTER);
2971 %}
2972 
2973 operand immI8()
2974 %{
2975   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2976   match(ConI);
2977 
2978   op_cost(5);
2979   format %{ %}
2980   interface(CONST_INTER);
2981 %}
2982 
2983 operand immI16()
2984 %{
2985   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2986   match(ConI);
2987 
2988   op_cost(10);
2989   format %{ %}
2990   interface(CONST_INTER);
2991 %}
2992 
2993 // Int Immediate non-negative
2994 operand immU31()
2995 %{
2996   predicate(n->get_int() >= 0);
2997   match(ConI);
2998 
2999   op_cost(0);
3000   format %{ %}
3001   interface(CONST_INTER);
3002 %}
3003 
3004 // Constant for long shifts
3005 operand immI_32()
3006 %{
3007   predicate( n->get_int() == 32 );
3008   match(ConI);
3009 
3010   op_cost(0);
3011   format %{ %}
3012   interface(CONST_INTER);
3013 %}
3014 
3015 // Constant for long shifts
3016 operand immI_64()
3017 %{
3018   predicate( n->get_int() == 64 );
3019   match(ConI);
3020 
3021   op_cost(0);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // Pointer Immediate
3027 operand immP()
3028 %{
3029   match(ConP);
3030 
3031   op_cost(10);
3032   format %{ %}
3033   interface(CONST_INTER);
3034 %}
3035 
3036 // NULL Pointer Immediate
3037 operand immP0()
3038 %{
3039   predicate(n->get_ptr() == 0);
3040   match(ConP);
3041 
3042   op_cost(5);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 // Pointer Immediate
3048 operand immN() %{
3049   match(ConN);
3050 
3051   op_cost(10);
3052   format %{ %}
3053   interface(CONST_INTER);
3054 %}
3055 
3056 operand immNKlass() %{
3057   match(ConNKlass);
3058 
3059   op_cost(10);
3060   format %{ %}
3061   interface(CONST_INTER);
3062 %}
3063 
3064 // NULL Pointer Immediate
3065 operand immN0() %{
3066   predicate(n->get_narrowcon() == 0);
3067   match(ConN);
3068 
3069   op_cost(5);
3070   format %{ %}
3071   interface(CONST_INTER);
3072 %}
3073 
3074 operand immP31()
3075 %{
3076   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3077             && (n->get_ptr() >> 31) == 0);
3078   match(ConP);
3079 
3080   op_cost(5);
3081   format %{ %}
3082   interface(CONST_INTER);
3083 %}
3084 
3085 
3086 // Long Immediate
3087 operand immL()
3088 %{
3089   match(ConL);
3090 
3091   op_cost(20);
3092   format %{ %}
3093   interface(CONST_INTER);
3094 %}
3095 
3096 // Long Immediate 8-bit
3097 operand immL8()
3098 %{
3099   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3100   match(ConL);
3101 
3102   op_cost(5);
3103   format %{ %}
3104   interface(CONST_INTER);
3105 %}
3106 
3107 // Long Immediate 32-bit unsigned
3108 operand immUL32()
3109 %{
3110   predicate(n->get_long() == (unsigned int) (n->get_long()));
3111   match(ConL);
3112 
3113   op_cost(10);
3114   format %{ %}
3115   interface(CONST_INTER);
3116 %}
3117 
3118 // Long Immediate 32-bit signed
3119 operand immL32()
3120 %{
3121   predicate(n->get_long() == (int) (n->get_long()));
3122   match(ConL);
3123 
3124   op_cost(15);
3125   format %{ %}
3126   interface(CONST_INTER);
3127 %}
3128 
3129 // Long Immediate zero
3130 operand immL0()
3131 %{
3132   predicate(n->get_long() == 0L);
3133   match(ConL);
3134 
3135   op_cost(10);
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // Constant for increment
3141 operand immL1()
3142 %{
3143   predicate(n->get_long() == 1);
3144   match(ConL);
3145 
3146   format %{ %}
3147   interface(CONST_INTER);
3148 %}
3149 
3150 // Constant for decrement
3151 operand immL_M1()
3152 %{
3153   predicate(n->get_long() == -1);
3154   match(ConL);
3155 
3156   format %{ %}
3157   interface(CONST_INTER);
3158 %}
3159 
3160 // Long Immediate: the value 10
3161 operand immL10()
3162 %{
3163   predicate(n->get_long() == 10);
3164   match(ConL);
3165 
3166   format %{ %}
3167   interface(CONST_INTER);
3168 %}
3169 
3170 // Long immediate from 0 to 127.
3171 // Used for a shorter form of long mul by 10.
3172 operand immL_127()
3173 %{
3174   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3175   match(ConL);
3176 
3177   op_cost(10);
3178   format %{ %}
3179   interface(CONST_INTER);
3180 %}
3181 
3182 // Long Immediate: low 32-bit mask
3183 operand immL_32bits()
3184 %{
3185   predicate(n->get_long() == 0xFFFFFFFFL);
3186   match(ConL);
3187   op_cost(20);
3188 
3189   format %{ %}
3190   interface(CONST_INTER);
3191 %}
3192 
3193 // Float Immediate zero
3194 operand immF0()
3195 %{
3196   predicate(jint_cast(n->getf()) == 0);
3197   match(ConF);
3198 
3199   op_cost(5);
3200   format %{ %}
3201   interface(CONST_INTER);
3202 %}
3203 
3204 // Float Immediate
3205 operand immF()
3206 %{
3207   match(ConF);
3208 
3209   op_cost(15);
3210   format %{ %}
3211   interface(CONST_INTER);
3212 %}
3213 
3214 // Double Immediate zero
3215 operand immD0()
3216 %{
3217   predicate(jlong_cast(n->getd()) == 0);
3218   match(ConD);
3219 
3220   op_cost(5);
3221   format %{ %}
3222   interface(CONST_INTER);
3223 %}
3224 
3225 // Double Immediate
3226 operand immD()
3227 %{
3228   match(ConD);
3229 
3230   op_cost(15);
3231   format %{ %}
3232   interface(CONST_INTER);
3233 %}
3234 
3235 // Immediates for special shifts (sign extend)
3236 
3237 // Constants for increment
3238 operand immI_16()
3239 %{
3240   predicate(n->get_int() == 16);
3241   match(ConI);
3242 
3243   format %{ %}
3244   interface(CONST_INTER);
3245 %}
3246 
3247 operand immI_24()
3248 %{
3249   predicate(n->get_int() == 24);
3250   match(ConI);
3251 
3252   format %{ %}
3253   interface(CONST_INTER);
3254 %}
3255 
3256 // Constant for byte-wide masking
3257 operand immI_255()
3258 %{
3259   predicate(n->get_int() == 255);
3260   match(ConI);
3261 
3262   format %{ %}
3263   interface(CONST_INTER);
3264 %}
3265 
3266 // Constant for short-wide masking
3267 operand immI_65535()
3268 %{
3269   predicate(n->get_int() == 65535);
3270   match(ConI);
3271 
3272   format %{ %}
3273   interface(CONST_INTER);
3274 %}
3275 
3276 // Constant for byte-wide masking
3277 operand immL_255()
3278 %{
3279   predicate(n->get_long() == 255);
3280   match(ConL);
3281 
3282   format %{ %}
3283   interface(CONST_INTER);
3284 %}
3285 
3286 // Constant for short-wide masking
3287 operand immL_65535()
3288 %{
3289   predicate(n->get_long() == 65535);
3290   match(ConL);
3291 
3292   format %{ %}
3293   interface(CONST_INTER);
3294 %}
3295 
3296 // Register Operands
3297 // Integer Register
3298 operand rRegI()
3299 %{
3300   constraint(ALLOC_IN_RC(int_reg));
3301   match(RegI);
3302 
3303   match(rax_RegI);
3304   match(rbx_RegI);
3305   match(rcx_RegI);
3306   match(rdx_RegI);
3307   match(rdi_RegI);
3308 
3309   format %{ %}
3310   interface(REG_INTER);
3311 %}
3312 
3313 // Special Registers
3314 operand rax_RegI()
3315 %{
3316   constraint(ALLOC_IN_RC(int_rax_reg));
3317   match(RegI);
3318   match(rRegI);
3319 
3320   format %{ "RAX" %}
3321   interface(REG_INTER);
3322 %}
3323 
3324 // Special Registers
3325 operand rbx_RegI()
3326 %{
3327   constraint(ALLOC_IN_RC(int_rbx_reg));
3328   match(RegI);
3329   match(rRegI);
3330 
3331   format %{ "RBX" %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 operand rcx_RegI()
3336 %{
3337   constraint(ALLOC_IN_RC(int_rcx_reg));
3338   match(RegI);
3339   match(rRegI);
3340 
3341   format %{ "RCX" %}
3342   interface(REG_INTER);
3343 %}
3344 
3345 operand rdx_RegI()
3346 %{
3347   constraint(ALLOC_IN_RC(int_rdx_reg));
3348   match(RegI);
3349   match(rRegI);
3350 
3351   format %{ "RDX" %}
3352   interface(REG_INTER);
3353 %}
3354 
3355 operand rdi_RegI()
3356 %{
3357   constraint(ALLOC_IN_RC(int_rdi_reg));
3358   match(RegI);
3359   match(rRegI);
3360 
3361   format %{ "RDI" %}
3362   interface(REG_INTER);
3363 %}
3364 
3365 operand no_rcx_RegI()
3366 %{
3367   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3368   match(RegI);
3369   match(rax_RegI);
3370   match(rbx_RegI);
3371   match(rdx_RegI);
3372   match(rdi_RegI);
3373 
3374   format %{ %}
3375   interface(REG_INTER);
3376 %}
3377 
3378 operand no_rax_rdx_RegI()
3379 %{
3380   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3381   match(RegI);
3382   match(rbx_RegI);
3383   match(rcx_RegI);
3384   match(rdi_RegI);
3385 
3386   format %{ %}
3387   interface(REG_INTER);
3388 %}
3389 
3390 // Pointer Register
3391 operand any_RegP()
3392 %{
3393   constraint(ALLOC_IN_RC(any_reg));
3394   match(RegP);
3395   match(rax_RegP);
3396   match(rbx_RegP);
3397   match(rdi_RegP);
3398   match(rsi_RegP);
3399   match(rbp_RegP);
3400   match(r15_RegP);
3401   match(rRegP);
3402 
3403   format %{ %}
3404   interface(REG_INTER);
3405 %}
3406 
3407 operand rRegP()
3408 %{
3409   constraint(ALLOC_IN_RC(ptr_reg));
3410   match(RegP);
3411   match(rax_RegP);
3412   match(rbx_RegP);
3413   match(rdi_RegP);
3414   match(rsi_RegP);
3415   match(rbp_RegP);  // See Q&A below about
3416   match(r15_RegP);  // r15_RegP and rbp_RegP.
3417 
3418   format %{ %}
3419   interface(REG_INTER);
3420 %}
3421 
3422 operand rRegN() %{
3423   constraint(ALLOC_IN_RC(int_reg));
3424   match(RegN);
3425 
3426   format %{ %}
3427   interface(REG_INTER);
3428 %}
3429 
3430 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3431 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3432 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3433 // The output of an instruction is controlled by the allocator, which respects
3434 // register class masks, not match rules.  Unless an instruction mentions
3435 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3436 // by the allocator as an input.
3437 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3438 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3439 // result, RBP is not included in the output of the instruction either.
3440 
3441 operand no_rax_RegP()
3442 %{
3443   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3444   match(RegP);
3445   match(rbx_RegP);
3446   match(rsi_RegP);
3447   match(rdi_RegP);
3448 
3449   format %{ %}
3450   interface(REG_INTER);
3451 %}
3452 
3453 // This operand is not allowed to use RBP even if
3454 // RBP is not used to hold the frame pointer.
3455 operand no_rbp_RegP()
3456 %{
3457   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3458   match(RegP);
3459   match(rbx_RegP);
3460   match(rsi_RegP);
3461   match(rdi_RegP);
3462 
3463   format %{ %}
3464   interface(REG_INTER);
3465 %}
3466 
3467 operand no_rax_rbx_RegP()
3468 %{
3469   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3470   match(RegP);
3471   match(rsi_RegP);
3472   match(rdi_RegP);
3473 
3474   format %{ %}
3475   interface(REG_INTER);
3476 %}
3477 
3478 // Special Registers
3479 // Return a pointer value
3480 operand rax_RegP()
3481 %{
3482   constraint(ALLOC_IN_RC(ptr_rax_reg));
3483   match(RegP);
3484   match(rRegP);
3485 
3486   format %{ %}
3487   interface(REG_INTER);
3488 %}
3489 
3490 // Special Registers
3491 // Return a compressed pointer value
3492 operand rax_RegN()
3493 %{
3494   constraint(ALLOC_IN_RC(int_rax_reg));
3495   match(RegN);
3496   match(rRegN);
3497 
3498   format %{ %}
3499   interface(REG_INTER);
3500 %}
3501 
3502 // Used in AtomicAdd
3503 operand rbx_RegP()
3504 %{
3505   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3506   match(RegP);
3507   match(rRegP);
3508 
3509   format %{ %}
3510   interface(REG_INTER);
3511 %}
3512 
3513 operand rsi_RegP()
3514 %{
3515   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3516   match(RegP);
3517   match(rRegP);
3518 
3519   format %{ %}
3520   interface(REG_INTER);
3521 %}
3522 
3523 // Used in rep stosq
3524 operand rdi_RegP()
3525 %{
3526   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3527   match(RegP);
3528   match(rRegP);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 operand r15_RegP()
3535 %{
3536   constraint(ALLOC_IN_RC(ptr_r15_reg));
3537   match(RegP);
3538   match(rRegP);
3539 
3540   format %{ %}
3541   interface(REG_INTER);
3542 %}
3543 
3544 operand rex_RegP()
3545 %{
3546   constraint(ALLOC_IN_RC(ptr_rex_reg));
3547   match(RegP);
3548   match(rRegP);
3549 
3550   format %{ %}
3551   interface(REG_INTER);
3552 %}
3553 
3554 operand rRegL()
3555 %{
3556   constraint(ALLOC_IN_RC(long_reg));
3557   match(RegL);
3558   match(rax_RegL);
3559   match(rdx_RegL);
3560 
3561   format %{ %}
3562   interface(REG_INTER);
3563 %}
3564 
3565 // Special Registers
3566 operand no_rax_rdx_RegL()
3567 %{
3568   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3569   match(RegL);
3570   match(rRegL);
3571 
3572   format %{ %}
3573   interface(REG_INTER);
3574 %}
3575 
3576 operand no_rax_RegL()
3577 %{
3578   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3579   match(RegL);
3580   match(rRegL);
3581   match(rdx_RegL);
3582 
3583   format %{ %}
3584   interface(REG_INTER);
3585 %}
3586 
3587 operand no_rcx_RegL()
3588 %{
3589   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3590   match(RegL);
3591   match(rRegL);
3592 
3593   format %{ %}
3594   interface(REG_INTER);
3595 %}
3596 
3597 operand rax_RegL()
3598 %{
3599   constraint(ALLOC_IN_RC(long_rax_reg));
3600   match(RegL);
3601   match(rRegL);
3602 
3603   format %{ "RAX" %}
3604   interface(REG_INTER);
3605 %}
3606 
3607 operand rcx_RegL()
3608 %{
3609   constraint(ALLOC_IN_RC(long_rcx_reg));
3610   match(RegL);
3611   match(rRegL);
3612 
3613   format %{ %}
3614   interface(REG_INTER);
3615 %}
3616 
3617 operand rdx_RegL()
3618 %{
3619   constraint(ALLOC_IN_RC(long_rdx_reg));
3620   match(RegL);
3621   match(rRegL);
3622 
3623   format %{ %}
3624   interface(REG_INTER);
3625 %}
3626 
3627 // Flags register, used as output of compare instructions
3628 operand rFlagsReg()
3629 %{
3630   constraint(ALLOC_IN_RC(int_flags));
3631   match(RegFlags);
3632 
3633   format %{ "RFLAGS" %}
3634   interface(REG_INTER);
3635 %}
3636 
3637 // Flags register, used as output of FLOATING POINT compare instructions
3638 operand rFlagsRegU()
3639 %{
3640   constraint(ALLOC_IN_RC(int_flags));
3641   match(RegFlags);
3642 
3643   format %{ "RFLAGS_U" %}
3644   interface(REG_INTER);
3645 %}
3646 
3647 operand rFlagsRegUCF() %{
3648   constraint(ALLOC_IN_RC(int_flags));
3649   match(RegFlags);
3650   predicate(false);
3651 
3652   format %{ "RFLAGS_U_CF" %}
3653   interface(REG_INTER);
3654 %}
3655 
3656 // Float register operands
3657 operand regF() %{
3658    constraint(ALLOC_IN_RC(float_reg));
3659    match(RegF);
3660 
3661    format %{ %}
3662    interface(REG_INTER);
3663 %}
3664 
3665 // Double register operands
3666 operand regD() %{
3667    constraint(ALLOC_IN_RC(double_reg));
3668    match(RegD);
3669 
3670    format %{ %}
3671    interface(REG_INTER);
3672 %}
3673 
3674 // Vectors
3675 operand vecS() %{
3676   constraint(ALLOC_IN_RC(vectors_reg));
3677   match(VecS);
3678 
3679   format %{ %}
3680   interface(REG_INTER);
3681 %}
3682 
3683 operand vecD() %{
3684   constraint(ALLOC_IN_RC(vectord_reg));
3685   match(VecD);
3686 
3687   format %{ %}
3688   interface(REG_INTER);
3689 %}
3690 
3691 operand vecX() %{
3692   constraint(ALLOC_IN_RC(vectorx_reg));
3693   match(VecX);
3694 
3695   format %{ %}
3696   interface(REG_INTER);
3697 %}
3698 
3699 operand vecY() %{
3700   constraint(ALLOC_IN_RC(vectory_reg));
3701   match(VecY);
3702 
3703   format %{ %}
3704   interface(REG_INTER);
3705 %}
3706 
3707 //----------Memory Operands----------------------------------------------------
3708 // Direct Memory Operand
3709 // operand direct(immP addr)
3710 // %{
3711 //   match(addr);
3712 
3713 //   format %{ "[$addr]" %}
3714 //   interface(MEMORY_INTER) %{
3715 //     base(0xFFFFFFFF);
3716 //     index(0x4);
3717 //     scale(0x0);
3718 //     disp($addr);
3719 //   %}
3720 // %}
3721 
3722 // Indirect Memory Operand
3723 operand indirect(any_RegP reg)
3724 %{
3725   constraint(ALLOC_IN_RC(ptr_reg));
3726   match(reg);
3727 
3728   format %{ "[$reg]" %}
3729   interface(MEMORY_INTER) %{
3730     base($reg);
3731     index(0x4);
3732     scale(0x0);
3733     disp(0x0);
3734   %}
3735 %}
3736 
3737 // Indirect Memory Plus Short Offset Operand
3738 operand indOffset8(any_RegP reg, immL8 off)
3739 %{
3740   constraint(ALLOC_IN_RC(ptr_reg));
3741   match(AddP reg off);
3742 
3743   format %{ "[$reg + $off (8-bit)]" %}
3744   interface(MEMORY_INTER) %{
3745     base($reg);
3746     index(0x4);
3747     scale(0x0);
3748     disp($off);
3749   %}
3750 %}
3751 
3752 // Indirect Memory Plus Long Offset Operand
3753 operand indOffset32(any_RegP reg, immL32 off)
3754 %{
3755   constraint(ALLOC_IN_RC(ptr_reg));
3756   match(AddP reg off);
3757 
3758   format %{ "[$reg + $off (32-bit)]" %}
3759   interface(MEMORY_INTER) %{
3760     base($reg);
3761     index(0x4);
3762     scale(0x0);
3763     disp($off);
3764   %}
3765 %}
3766 
3767 // Indirect Memory Plus Index Register Plus Offset Operand
3768 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3769 %{
3770   constraint(ALLOC_IN_RC(ptr_reg));
3771   match(AddP (AddP reg lreg) off);
3772 
3773   op_cost(10);
3774   format %{"[$reg + $off + $lreg]" %}
3775   interface(MEMORY_INTER) %{
3776     base($reg);
3777     index($lreg);
3778     scale(0x0);
3779     disp($off);
3780   %}
3781 %}
3782 
3783 // Indirect Memory Plus Index Register Plus Offset Operand
3784 operand indIndex(any_RegP reg, rRegL lreg)
3785 %{
3786   constraint(ALLOC_IN_RC(ptr_reg));
3787   match(AddP reg lreg);
3788 
3789   op_cost(10);
3790   format %{"[$reg + $lreg]" %}
3791   interface(MEMORY_INTER) %{
3792     base($reg);
3793     index($lreg);
3794     scale(0x0);
3795     disp(0x0);
3796   %}
3797 %}
3798 
3799 // Indirect Memory Times Scale Plus Index Register
3800 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3801 %{
3802   constraint(ALLOC_IN_RC(ptr_reg));
3803   match(AddP reg (LShiftL lreg scale));
3804 
3805   op_cost(10);
3806   format %{"[$reg + $lreg << $scale]" %}
3807   interface(MEMORY_INTER) %{
3808     base($reg);
3809     index($lreg);
3810     scale($scale);
3811     disp(0x0);
3812   %}
3813 %}
3814 
3815 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3816 %{
3817   constraint(ALLOC_IN_RC(ptr_reg));
3818   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3819   match(AddP reg (LShiftL (ConvI2L idx) scale));
3820 
3821   op_cost(10);
3822   format %{"[$reg + pos $idx << $scale]" %}
3823   interface(MEMORY_INTER) %{
3824     base($reg);
3825     index($idx);
3826     scale($scale);
3827     disp(0x0);
3828   %}
3829 %}
3830 
3831 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3832 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3833 %{
3834   constraint(ALLOC_IN_RC(ptr_reg));
3835   match(AddP (AddP reg (LShiftL lreg scale)) off);
3836 
3837   op_cost(10);
3838   format %{"[$reg + $off + $lreg << $scale]" %}
3839   interface(MEMORY_INTER) %{
3840     base($reg);
3841     index($lreg);
3842     scale($scale);
3843     disp($off);
3844   %}
3845 %}
3846 
3847 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3848 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3849 %{
3850   constraint(ALLOC_IN_RC(ptr_reg));
3851   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3852   match(AddP (AddP reg (ConvI2L idx)) off);
3853 
3854   op_cost(10);
3855   format %{"[$reg + $off + $idx]" %}
3856   interface(MEMORY_INTER) %{
3857     base($reg);
3858     index($idx);
3859     scale(0x0);
3860     disp($off);
3861   %}
3862 %}
3863 
3864 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3865 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3866 %{
3867   constraint(ALLOC_IN_RC(ptr_reg));
3868   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3869   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3870 
3871   op_cost(10);
3872   format %{"[$reg + $off + $idx << $scale]" %}
3873   interface(MEMORY_INTER) %{
3874     base($reg);
3875     index($idx);
3876     scale($scale);
3877     disp($off);
3878   %}
3879 %}
3880 
3881 // Indirect Narrow Oop Plus Offset Operand
3882 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3883 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3884 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3885   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3886   constraint(ALLOC_IN_RC(ptr_reg));
3887   match(AddP (DecodeN reg) off);
3888 
3889   op_cost(10);
3890   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3891   interface(MEMORY_INTER) %{
3892     base(0xc); // R12
3893     index($reg);
3894     scale(0x3);
3895     disp($off);
3896   %}
3897 %}
3898 
3899 // Indirect Memory Operand
3900 operand indirectNarrow(rRegN reg)
3901 %{
3902   predicate(Universe::narrow_oop_shift() == 0);
3903   constraint(ALLOC_IN_RC(ptr_reg));
3904   match(DecodeN reg);
3905 
3906   format %{ "[$reg]" %}
3907   interface(MEMORY_INTER) %{
3908     base($reg);
3909     index(0x4);
3910     scale(0x0);
3911     disp(0x0);
3912   %}
3913 %}
3914 
3915 // Indirect Memory Plus Short Offset Operand
3916 operand indOffset8Narrow(rRegN reg, immL8 off)
3917 %{
3918   predicate(Universe::narrow_oop_shift() == 0);
3919   constraint(ALLOC_IN_RC(ptr_reg));
3920   match(AddP (DecodeN reg) off);
3921 
3922   format %{ "[$reg + $off (8-bit)]" %}
3923   interface(MEMORY_INTER) %{
3924     base($reg);
3925     index(0x4);
3926     scale(0x0);
3927     disp($off);
3928   %}
3929 %}
3930 
3931 // Indirect Memory Plus Long Offset Operand
3932 operand indOffset32Narrow(rRegN reg, immL32 off)
3933 %{
3934   predicate(Universe::narrow_oop_shift() == 0);
3935   constraint(ALLOC_IN_RC(ptr_reg));
3936   match(AddP (DecodeN reg) off);
3937 
3938   format %{ "[$reg + $off (32-bit)]" %}
3939   interface(MEMORY_INTER) %{
3940     base($reg);
3941     index(0x4);
3942     scale(0x0);
3943     disp($off);
3944   %}
3945 %}
3946 
3947 // Indirect Memory Plus Index Register Plus Offset Operand
3948 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3949 %{
3950   predicate(Universe::narrow_oop_shift() == 0);
3951   constraint(ALLOC_IN_RC(ptr_reg));
3952   match(AddP (AddP (DecodeN reg) lreg) off);
3953 
3954   op_cost(10);
3955   format %{"[$reg + $off + $lreg]" %}
3956   interface(MEMORY_INTER) %{
3957     base($reg);
3958     index($lreg);
3959     scale(0x0);
3960     disp($off);
3961   %}
3962 %}
3963 
3964 // Indirect Memory Plus Index Register Plus Offset Operand
3965 operand indIndexNarrow(rRegN reg, rRegL lreg)
3966 %{
3967   predicate(Universe::narrow_oop_shift() == 0);
3968   constraint(ALLOC_IN_RC(ptr_reg));
3969   match(AddP (DecodeN reg) lreg);
3970 
3971   op_cost(10);
3972   format %{"[$reg + $lreg]" %}
3973   interface(MEMORY_INTER) %{
3974     base($reg);
3975     index($lreg);
3976     scale(0x0);
3977     disp(0x0);
3978   %}
3979 %}
3980 
3981 // Indirect Memory Times Scale Plus Index Register
3982 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3983 %{
3984   predicate(Universe::narrow_oop_shift() == 0);
3985   constraint(ALLOC_IN_RC(ptr_reg));
3986   match(AddP (DecodeN reg) (LShiftL lreg scale));
3987 
3988   op_cost(10);
3989   format %{"[$reg + $lreg << $scale]" %}
3990   interface(MEMORY_INTER) %{
3991     base($reg);
3992     index($lreg);
3993     scale($scale);
3994     disp(0x0);
3995   %}
3996 %}
3997 
3998 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3999 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4000 %{
4001   predicate(Universe::narrow_oop_shift() == 0);
4002   constraint(ALLOC_IN_RC(ptr_reg));
4003   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4004 
4005   op_cost(10);
4006   format %{"[$reg + $off + $lreg << $scale]" %}
4007   interface(MEMORY_INTER) %{
4008     base($reg);
4009     index($lreg);
4010     scale($scale);
4011     disp($off);
4012   %}
4013 %}
4014 
4015 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4016 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4017 %{
4018   constraint(ALLOC_IN_RC(ptr_reg));
4019   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4020   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4021 
4022   op_cost(10);
4023   format %{"[$reg + $off + $idx]" %}
4024   interface(MEMORY_INTER) %{
4025     base($reg);
4026     index($idx);
4027     scale(0x0);
4028     disp($off);
4029   %}
4030 %}
4031 
4032 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4033 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4034 %{
4035   constraint(ALLOC_IN_RC(ptr_reg));
4036   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4037   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4038 
4039   op_cost(10);
4040   format %{"[$reg + $off + $idx << $scale]" %}
4041   interface(MEMORY_INTER) %{
4042     base($reg);
4043     index($idx);
4044     scale($scale);
4045     disp($off);
4046   %}
4047 %}
4048 
4049 //----------Special Memory Operands--------------------------------------------
4050 // Stack Slot Operand - This operand is used for loading and storing temporary
4051 //                      values on the stack where a match requires a value to
4052 //                      flow through memory.
4053 operand stackSlotP(sRegP reg)
4054 %{
4055   constraint(ALLOC_IN_RC(stack_slots));
4056   // No match rule because this operand is only generated in matching
4057 
4058   format %{ "[$reg]" %}
4059   interface(MEMORY_INTER) %{
4060     base(0x4);   // RSP
4061     index(0x4);  // No Index
4062     scale(0x0);  // No Scale
4063     disp($reg);  // Stack Offset
4064   %}
4065 %}
4066 
4067 operand stackSlotI(sRegI reg)
4068 %{
4069   constraint(ALLOC_IN_RC(stack_slots));
4070   // No match rule because this operand is only generated in matching
4071 
4072   format %{ "[$reg]" %}
4073   interface(MEMORY_INTER) %{
4074     base(0x4);   // RSP
4075     index(0x4);  // No Index
4076     scale(0x0);  // No Scale
4077     disp($reg);  // Stack Offset
4078   %}
4079 %}
4080 
4081 operand stackSlotF(sRegF reg)
4082 %{
4083   constraint(ALLOC_IN_RC(stack_slots));
4084   // No match rule because this operand is only generated in matching
4085 
4086   format %{ "[$reg]" %}
4087   interface(MEMORY_INTER) %{
4088     base(0x4);   // RSP
4089     index(0x4);  // No Index
4090     scale(0x0);  // No Scale
4091     disp($reg);  // Stack Offset
4092   %}
4093 %}
4094 
4095 operand stackSlotD(sRegD reg)
4096 %{
4097   constraint(ALLOC_IN_RC(stack_slots));
4098   // No match rule because this operand is only generated in matching
4099 
4100   format %{ "[$reg]" %}
4101   interface(MEMORY_INTER) %{
4102     base(0x4);   // RSP
4103     index(0x4);  // No Index
4104     scale(0x0);  // No Scale
4105     disp($reg);  // Stack Offset
4106   %}
4107 %}
4108 operand stackSlotL(sRegL reg)
4109 %{
4110   constraint(ALLOC_IN_RC(stack_slots));
4111   // No match rule because this operand is only generated in matching
4112 
4113   format %{ "[$reg]" %}
4114   interface(MEMORY_INTER) %{
4115     base(0x4);   // RSP
4116     index(0x4);  // No Index
4117     scale(0x0);  // No Scale
4118     disp($reg);  // Stack Offset
4119   %}
4120 %}
4121 
4122 //----------Conditional Branch Operands----------------------------------------
4123 // Comparison Op  - This is the operation of the comparison, and is limited to
4124 //                  the following set of codes:
4125 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4126 //
4127 // Other attributes of the comparison, such as unsignedness, are specified
4128 // by the comparison instruction that sets a condition code flags register.
4129 // That result is represented by a flags operand whose subtype is appropriate
4130 // to the unsignedness (etc.) of the comparison.
4131 //
4132 // Later, the instruction which matches both the Comparison Op (a Bool) and
4133 // the flags (produced by the Cmp) specifies the coding of the comparison op
4134 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4135 
4136 // Comparision Code
4137 operand cmpOp()
4138 %{
4139   match(Bool);
4140 
4141   format %{ "" %}
4142   interface(COND_INTER) %{
4143     equal(0x4, "e");
4144     not_equal(0x5, "ne");
4145     less(0xC, "l");
4146     greater_equal(0xD, "ge");
4147     less_equal(0xE, "le");
4148     greater(0xF, "g");
4149     overflow(0x0, "o");
4150     no_overflow(0x1, "no");
4151   %}
4152 %}
4153 
4154 // Comparison Code, unsigned compare.  Used by FP also, with
4155 // C2 (unordered) turned into GT or LT already.  The other bits
4156 // C0 and C3 are turned into Carry & Zero flags.
4157 operand cmpOpU()
4158 %{
4159   match(Bool);
4160 
4161   format %{ "" %}
4162   interface(COND_INTER) %{
4163     equal(0x4, "e");
4164     not_equal(0x5, "ne");
4165     less(0x2, "b");
4166     greater_equal(0x3, "nb");
4167     less_equal(0x6, "be");
4168     greater(0x7, "nbe");
4169     overflow(0x0, "o");
4170     no_overflow(0x1, "no");
4171   %}
4172 %}
4173 
4174 
4175 // Floating comparisons that don't require any fixup for the unordered case
4176 operand cmpOpUCF() %{
4177   match(Bool);
4178   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4179             n->as_Bool()->_test._test == BoolTest::ge ||
4180             n->as_Bool()->_test._test == BoolTest::le ||
4181             n->as_Bool()->_test._test == BoolTest::gt);
4182   format %{ "" %}
4183   interface(COND_INTER) %{
4184     equal(0x4, "e");
4185     not_equal(0x5, "ne");
4186     less(0x2, "b");
4187     greater_equal(0x3, "nb");
4188     less_equal(0x6, "be");
4189     greater(0x7, "nbe");
4190     overflow(0x0, "o");
4191     no_overflow(0x1, "no");
4192   %}
4193 %}
4194 
4195 
4196 // Floating comparisons that can be fixed up with extra conditional jumps
4197 operand cmpOpUCF2() %{
4198   match(Bool);
4199   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4200             n->as_Bool()->_test._test == BoolTest::eq);
4201   format %{ "" %}
4202   interface(COND_INTER) %{
4203     equal(0x4, "e");
4204     not_equal(0x5, "ne");
4205     less(0x2, "b");
4206     greater_equal(0x3, "nb");
4207     less_equal(0x6, "be");
4208     greater(0x7, "nbe");
4209     overflow(0x0, "o");
4210     no_overflow(0x1, "no");
4211   %}
4212 %}
4213 
4214 
4215 //----------OPERAND CLASSES----------------------------------------------------
4216 // Operand Classes are groups of operands that are used as to simplify
4217 // instruction definitions by not requiring the AD writer to specify separate
4218 // instructions for every form of operand when the instruction accepts
4219 // multiple operand types with the same basic encoding and format.  The classic
4220 // case of this is memory operands.
4221 
4222 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4223                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4224                indCompressedOopOffset,
4225                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4226                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4227                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4228 
4229 //----------PIPELINE-----------------------------------------------------------
4230 // Rules which define the behavior of the target architectures pipeline.
4231 pipeline %{
4232 
4233 //----------ATTRIBUTES---------------------------------------------------------
4234 attributes %{
4235   variable_size_instructions;        // Fixed size instructions
4236   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4237   instruction_unit_size = 1;         // An instruction is 1 bytes long
4238   instruction_fetch_unit_size = 16;  // The processor fetches one line
4239   instruction_fetch_units = 1;       // of 16 bytes
4240 
4241   // List of nop instructions
4242   nops( MachNop );
4243 %}
4244 
4245 //----------RESOURCES----------------------------------------------------------
4246 // Resources are the functional units available to the machine
4247 
4248 // Generic P2/P3 pipeline
4249 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4250 // 3 instructions decoded per cycle.
4251 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4252 // 3 ALU op, only ALU0 handles mul instructions.
4253 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4254            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4255            BR, FPU,
4256            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4257 
4258 //----------PIPELINE DESCRIPTION-----------------------------------------------
4259 // Pipeline Description specifies the stages in the machine's pipeline
4260 
4261 // Generic P2/P3 pipeline
4262 pipe_desc(S0, S1, S2, S3, S4, S5);
4263 
4264 //----------PIPELINE CLASSES---------------------------------------------------
4265 // Pipeline Classes describe the stages in which input and output are
4266 // referenced by the hardware pipeline.
4267 
4268 // Naming convention: ialu or fpu
4269 // Then: _reg
4270 // Then: _reg if there is a 2nd register
4271 // Then: _long if it's a pair of instructions implementing a long
4272 // Then: _fat if it requires the big decoder
4273 //   Or: _mem if it requires the big decoder and a memory unit.
4274 
4275 // Integer ALU reg operation
4276 pipe_class ialu_reg(rRegI dst)
4277 %{
4278     single_instruction;
4279     dst    : S4(write);
4280     dst    : S3(read);
4281     DECODE : S0;        // any decoder
4282     ALU    : S3;        // any alu
4283 %}
4284 
4285 // Long ALU reg operation
4286 pipe_class ialu_reg_long(rRegL dst)
4287 %{
4288     instruction_count(2);
4289     dst    : S4(write);
4290     dst    : S3(read);
4291     DECODE : S0(2);     // any 2 decoders
4292     ALU    : S3(2);     // both alus
4293 %}
4294 
4295 // Integer ALU reg operation using big decoder
4296 pipe_class ialu_reg_fat(rRegI dst)
4297 %{
4298     single_instruction;
4299     dst    : S4(write);
4300     dst    : S3(read);
4301     D0     : S0;        // big decoder only
4302     ALU    : S3;        // any alu
4303 %}
4304 
4305 // Long ALU reg operation using big decoder
4306 pipe_class ialu_reg_long_fat(rRegL dst)
4307 %{
4308     instruction_count(2);
4309     dst    : S4(write);
4310     dst    : S3(read);
4311     D0     : S0(2);     // big decoder only; twice
4312     ALU    : S3(2);     // any 2 alus
4313 %}
4314 
4315 // Integer ALU reg-reg operation
4316 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4317 %{
4318     single_instruction;
4319     dst    : S4(write);
4320     src    : S3(read);
4321     DECODE : S0;        // any decoder
4322     ALU    : S3;        // any alu
4323 %}
4324 
4325 // Long ALU reg-reg operation
4326 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4327 %{
4328     instruction_count(2);
4329     dst    : S4(write);
4330     src    : S3(read);
4331     DECODE : S0(2);     // any 2 decoders
4332     ALU    : S3(2);     // both alus
4333 %}
4334 
4335 // Integer ALU reg-reg operation
4336 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4337 %{
4338     single_instruction;
4339     dst    : S4(write);
4340     src    : S3(read);
4341     D0     : S0;        // big decoder only
4342     ALU    : S3;        // any alu
4343 %}
4344 
4345 // Long ALU reg-reg operation
4346 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4347 %{
4348     instruction_count(2);
4349     dst    : S4(write);
4350     src    : S3(read);
4351     D0     : S0(2);     // big decoder only; twice
4352     ALU    : S3(2);     // both alus
4353 %}
4354 
4355 // Integer ALU reg-mem operation
4356 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4357 %{
4358     single_instruction;
4359     dst    : S5(write);
4360     mem    : S3(read);
4361     D0     : S0;        // big decoder only
4362     ALU    : S4;        // any alu
4363     MEM    : S3;        // any mem
4364 %}
4365 
4366 // Integer mem operation (prefetch)
4367 pipe_class ialu_mem(memory mem)
4368 %{
4369     single_instruction;
4370     mem    : S3(read);
4371     D0     : S0;        // big decoder only
4372     MEM    : S3;        // any mem
4373 %}
4374 
4375 // Integer Store to Memory
4376 pipe_class ialu_mem_reg(memory mem, rRegI src)
4377 %{
4378     single_instruction;
4379     mem    : S3(read);
4380     src    : S5(read);
4381     D0     : S0;        // big decoder only
4382     ALU    : S4;        // any alu
4383     MEM    : S3;
4384 %}
4385 
4386 // // Long Store to Memory
4387 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4388 // %{
4389 //     instruction_count(2);
4390 //     mem    : S3(read);
4391 //     src    : S5(read);
4392 //     D0     : S0(2);          // big decoder only; twice
4393 //     ALU    : S4(2);     // any 2 alus
4394 //     MEM    : S3(2);  // Both mems
4395 // %}
4396 
4397 // Integer Store to Memory
4398 pipe_class ialu_mem_imm(memory mem)
4399 %{
4400     single_instruction;
4401     mem    : S3(read);
4402     D0     : S0;        // big decoder only
4403     ALU    : S4;        // any alu
4404     MEM    : S3;
4405 %}
4406 
4407 // Integer ALU0 reg-reg operation
4408 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4409 %{
4410     single_instruction;
4411     dst    : S4(write);
4412     src    : S3(read);
4413     D0     : S0;        // Big decoder only
4414     ALU0   : S3;        // only alu0
4415 %}
4416 
4417 // Integer ALU0 reg-mem operation
4418 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4419 %{
4420     single_instruction;
4421     dst    : S5(write);
4422     mem    : S3(read);
4423     D0     : S0;        // big decoder only
4424     ALU0   : S4;        // ALU0 only
4425     MEM    : S3;        // any mem
4426 %}
4427 
4428 // Integer ALU reg-reg operation
4429 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4430 %{
4431     single_instruction;
4432     cr     : S4(write);
4433     src1   : S3(read);
4434     src2   : S3(read);
4435     DECODE : S0;        // any decoder
4436     ALU    : S3;        // any alu
4437 %}
4438 
4439 // Integer ALU reg-imm operation
4440 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4441 %{
4442     single_instruction;
4443     cr     : S4(write);
4444     src1   : S3(read);
4445     DECODE : S0;        // any decoder
4446     ALU    : S3;        // any alu
4447 %}
4448 
4449 // Integer ALU reg-mem operation
4450 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4451 %{
4452     single_instruction;
4453     cr     : S4(write);
4454     src1   : S3(read);
4455     src2   : S3(read);
4456     D0     : S0;        // big decoder only
4457     ALU    : S4;        // any alu
4458     MEM    : S3;
4459 %}
4460 
4461 // Conditional move reg-reg
4462 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4463 %{
4464     instruction_count(4);
4465     y      : S4(read);
4466     q      : S3(read);
4467     p      : S3(read);
4468     DECODE : S0(4);     // any decoder
4469 %}
4470 
4471 // Conditional move reg-reg
4472 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4473 %{
4474     single_instruction;
4475     dst    : S4(write);
4476     src    : S3(read);
4477     cr     : S3(read);
4478     DECODE : S0;        // any decoder
4479 %}
4480 
4481 // Conditional move reg-mem
4482 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4483 %{
4484     single_instruction;
4485     dst    : S4(write);
4486     src    : S3(read);
4487     cr     : S3(read);
4488     DECODE : S0;        // any decoder
4489     MEM    : S3;
4490 %}
4491 
4492 // Conditional move reg-reg long
4493 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4494 %{
4495     single_instruction;
4496     dst    : S4(write);
4497     src    : S3(read);
4498     cr     : S3(read);
4499     DECODE : S0(2);     // any 2 decoders
4500 %}
4501 
4502 // XXX
4503 // // Conditional move double reg-reg
4504 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4505 // %{
4506 //     single_instruction;
4507 //     dst    : S4(write);
4508 //     src    : S3(read);
4509 //     cr     : S3(read);
4510 //     DECODE : S0;     // any decoder
4511 // %}
4512 
4513 // Float reg-reg operation
4514 pipe_class fpu_reg(regD dst)
4515 %{
4516     instruction_count(2);
4517     dst    : S3(read);
4518     DECODE : S0(2);     // any 2 decoders
4519     FPU    : S3;
4520 %}
4521 
4522 // Float reg-reg operation
4523 pipe_class fpu_reg_reg(regD dst, regD src)
4524 %{
4525     instruction_count(2);
4526     dst    : S4(write);
4527     src    : S3(read);
4528     DECODE : S0(2);     // any 2 decoders
4529     FPU    : S3;
4530 %}
4531 
4532 // Float reg-reg operation
4533 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4534 %{
4535     instruction_count(3);
4536     dst    : S4(write);
4537     src1   : S3(read);
4538     src2   : S3(read);
4539     DECODE : S0(3);     // any 3 decoders
4540     FPU    : S3(2);
4541 %}
4542 
4543 // Float reg-reg operation
4544 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4545 %{
4546     instruction_count(4);
4547     dst    : S4(write);
4548     src1   : S3(read);
4549     src2   : S3(read);
4550     src3   : S3(read);
4551     DECODE : S0(4);     // any 3 decoders
4552     FPU    : S3(2);
4553 %}
4554 
4555 // Float reg-reg operation
4556 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4557 %{
4558     instruction_count(4);
4559     dst    : S4(write);
4560     src1   : S3(read);
4561     src2   : S3(read);
4562     src3   : S3(read);
4563     DECODE : S1(3);     // any 3 decoders
4564     D0     : S0;        // Big decoder only
4565     FPU    : S3(2);
4566     MEM    : S3;
4567 %}
4568 
4569 // Float reg-mem operation
4570 pipe_class fpu_reg_mem(regD dst, memory mem)
4571 %{
4572     instruction_count(2);
4573     dst    : S5(write);
4574     mem    : S3(read);
4575     D0     : S0;        // big decoder only
4576     DECODE : S1;        // any decoder for FPU POP
4577     FPU    : S4;
4578     MEM    : S3;        // any mem
4579 %}
4580 
4581 // Float reg-mem operation
4582 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4583 %{
4584     instruction_count(3);
4585     dst    : S5(write);
4586     src1   : S3(read);
4587     mem    : S3(read);
4588     D0     : S0;        // big decoder only
4589     DECODE : S1(2);     // any decoder for FPU POP
4590     FPU    : S4;
4591     MEM    : S3;        // any mem
4592 %}
4593 
4594 // Float mem-reg operation
4595 pipe_class fpu_mem_reg(memory mem, regD src)
4596 %{
4597     instruction_count(2);
4598     src    : S5(read);
4599     mem    : S3(read);
4600     DECODE : S0;        // any decoder for FPU PUSH
4601     D0     : S1;        // big decoder only
4602     FPU    : S4;
4603     MEM    : S3;        // any mem
4604 %}
4605 
4606 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4607 %{
4608     instruction_count(3);
4609     src1   : S3(read);
4610     src2   : S3(read);
4611     mem    : S3(read);
4612     DECODE : S0(2);     // any decoder for FPU PUSH
4613     D0     : S1;        // big decoder only
4614     FPU    : S4;
4615     MEM    : S3;        // any mem
4616 %}
4617 
4618 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4619 %{
4620     instruction_count(3);
4621     src1   : S3(read);
4622     src2   : S3(read);
4623     mem    : S4(read);
4624     DECODE : S0;        // any decoder for FPU PUSH
4625     D0     : S0(2);     // big decoder only
4626     FPU    : S4;
4627     MEM    : S3(2);     // any mem
4628 %}
4629 
4630 pipe_class fpu_mem_mem(memory dst, memory src1)
4631 %{
4632     instruction_count(2);
4633     src1   : S3(read);
4634     dst    : S4(read);
4635     D0     : S0(2);     // big decoder only
4636     MEM    : S3(2);     // any mem
4637 %}
4638 
4639 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4640 %{
4641     instruction_count(3);
4642     src1   : S3(read);
4643     src2   : S3(read);
4644     dst    : S4(read);
4645     D0     : S0(3);     // big decoder only
4646     FPU    : S4;
4647     MEM    : S3(3);     // any mem
4648 %}
4649 
4650 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4651 %{
4652     instruction_count(3);
4653     src1   : S4(read);
4654     mem    : S4(read);
4655     DECODE : S0;        // any decoder for FPU PUSH
4656     D0     : S0(2);     // big decoder only
4657     FPU    : S4;
4658     MEM    : S3(2);     // any mem
4659 %}
4660 
4661 // Float load constant
4662 pipe_class fpu_reg_con(regD dst)
4663 %{
4664     instruction_count(2);
4665     dst    : S5(write);
4666     D0     : S0;        // big decoder only for the load
4667     DECODE : S1;        // any decoder for FPU POP
4668     FPU    : S4;
4669     MEM    : S3;        // any mem
4670 %}
4671 
4672 // Float load constant
4673 pipe_class fpu_reg_reg_con(regD dst, regD src)
4674 %{
4675     instruction_count(3);
4676     dst    : S5(write);
4677     src    : S3(read);
4678     D0     : S0;        // big decoder only for the load
4679     DECODE : S1(2);     // any decoder for FPU POP
4680     FPU    : S4;
4681     MEM    : S3;        // any mem
4682 %}
4683 
4684 // UnConditional branch
4685 pipe_class pipe_jmp(label labl)
4686 %{
4687     single_instruction;
4688     BR   : S3;
4689 %}
4690 
4691 // Conditional branch
4692 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4693 %{
4694     single_instruction;
4695     cr    : S1(read);
4696     BR    : S3;
4697 %}
4698 
4699 // Allocation idiom
4700 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4701 %{
4702     instruction_count(1); force_serialization;
4703     fixed_latency(6);
4704     heap_ptr : S3(read);
4705     DECODE   : S0(3);
4706     D0       : S2;
4707     MEM      : S3;
4708     ALU      : S3(2);
4709     dst      : S5(write);
4710     BR       : S5;
4711 %}
4712 
4713 // Generic big/slow expanded idiom
4714 pipe_class pipe_slow()
4715 %{
4716     instruction_count(10); multiple_bundles; force_serialization;
4717     fixed_latency(100);
4718     D0  : S0(2);
4719     MEM : S3(2);
4720 %}
4721 
4722 // The real do-nothing guy
4723 pipe_class empty()
4724 %{
4725     instruction_count(0);
4726 %}
4727 
4728 // Define the class for the Nop node
4729 define
4730 %{
4731    MachNop = empty;
4732 %}
4733 
4734 %}
4735 
4736 //----------INSTRUCTIONS-------------------------------------------------------
4737 //
4738 // match      -- States which machine-independent subtree may be replaced
4739 //               by this instruction.
4740 // ins_cost   -- The estimated cost of this instruction is used by instruction
4741 //               selection to identify a minimum cost tree of machine
4742 //               instructions that matches a tree of machine-independent
4743 //               instructions.
4744 // format     -- A string providing the disassembly for this instruction.
4745 //               The value of an instruction's operand may be inserted
4746 //               by referring to it with a '$' prefix.
4747 // opcode     -- Three instruction opcodes may be provided.  These are referred
4748 //               to within an encode class as $primary, $secondary, and $tertiary
4749 //               rrspectively.  The primary opcode is commonly used to
4750 //               indicate the type of machine instruction, while secondary
4751 //               and tertiary are often used for prefix options or addressing
4752 //               modes.
4753 // ins_encode -- A list of encode classes with parameters. The encode class
4754 //               name must have been defined in an 'enc_class' specification
4755 //               in the encode section of the architecture description.
4756 
4757 
4758 //----------Load/Store/Move Instructions---------------------------------------
4759 //----------Load Instructions--------------------------------------------------
4760 
4761 // Load Byte (8 bit signed)
4762 instruct loadB(rRegI dst, memory mem)
4763 %{
4764   match(Set dst (LoadB mem));
4765 
4766   ins_cost(125);
4767   format %{ "movsbl  $dst, $mem\t# byte" %}
4768 
4769   ins_encode %{
4770     __ movsbl($dst$$Register, $mem$$Address);
4771   %}
4772 
4773   ins_pipe(ialu_reg_mem);
4774 %}
4775 
4776 // Load Byte (8 bit signed) into Long Register
4777 instruct loadB2L(rRegL dst, memory mem)
4778 %{
4779   match(Set dst (ConvI2L (LoadB mem)));
4780 
4781   ins_cost(125);
4782   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4783 
4784   ins_encode %{
4785     __ movsbq($dst$$Register, $mem$$Address);
4786   %}
4787 
4788   ins_pipe(ialu_reg_mem);
4789 %}
4790 
4791 // Load Unsigned Byte (8 bit UNsigned)
4792 instruct loadUB(rRegI dst, memory mem)
4793 %{
4794   match(Set dst (LoadUB mem));
4795 
4796   ins_cost(125);
4797   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4798 
4799   ins_encode %{
4800     __ movzbl($dst$$Register, $mem$$Address);
4801   %}
4802 
4803   ins_pipe(ialu_reg_mem);
4804 %}
4805 
4806 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4807 instruct loadUB2L(rRegL dst, memory mem)
4808 %{
4809   match(Set dst (ConvI2L (LoadUB mem)));
4810 
4811   ins_cost(125);
4812   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4813 
4814   ins_encode %{
4815     __ movzbq($dst$$Register, $mem$$Address);
4816   %}
4817 
4818   ins_pipe(ialu_reg_mem);
4819 %}
4820 
4821 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4822 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4823   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4824   effect(KILL cr);
4825 
4826   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4827             "andl    $dst, right_n_bits($mask, 8)" %}
4828   ins_encode %{
4829     Register Rdst = $dst$$Register;
4830     __ movzbq(Rdst, $mem$$Address);
4831     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4832   %}
4833   ins_pipe(ialu_reg_mem);
4834 %}
4835 
4836 // Load Short (16 bit signed)
4837 instruct loadS(rRegI dst, memory mem)
4838 %{
4839   match(Set dst (LoadS mem));
4840 
4841   ins_cost(125);
4842   format %{ "movswl $dst, $mem\t# short" %}
4843 
4844   ins_encode %{
4845     __ movswl($dst$$Register, $mem$$Address);
4846   %}
4847 
4848   ins_pipe(ialu_reg_mem);
4849 %}
4850 
4851 // Load Short (16 bit signed) to Byte (8 bit signed)
4852 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4853   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4854 
4855   ins_cost(125);
4856   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4857   ins_encode %{
4858     __ movsbl($dst$$Register, $mem$$Address);
4859   %}
4860   ins_pipe(ialu_reg_mem);
4861 %}
4862 
4863 // Load Short (16 bit signed) into Long Register
4864 instruct loadS2L(rRegL dst, memory mem)
4865 %{
4866   match(Set dst (ConvI2L (LoadS mem)));
4867 
4868   ins_cost(125);
4869   format %{ "movswq $dst, $mem\t# short -> long" %}
4870 
4871   ins_encode %{
4872     __ movswq($dst$$Register, $mem$$Address);
4873   %}
4874 
4875   ins_pipe(ialu_reg_mem);
4876 %}
4877 
4878 // Load Unsigned Short/Char (16 bit UNsigned)
4879 instruct loadUS(rRegI dst, memory mem)
4880 %{
4881   match(Set dst (LoadUS mem));
4882 
4883   ins_cost(125);
4884   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4885 
4886   ins_encode %{
4887     __ movzwl($dst$$Register, $mem$$Address);
4888   %}
4889 
4890   ins_pipe(ialu_reg_mem);
4891 %}
4892 
4893 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4894 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4895   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4896 
4897   ins_cost(125);
4898   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4899   ins_encode %{
4900     __ movsbl($dst$$Register, $mem$$Address);
4901   %}
4902   ins_pipe(ialu_reg_mem);
4903 %}
4904 
4905 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4906 instruct loadUS2L(rRegL dst, memory mem)
4907 %{
4908   match(Set dst (ConvI2L (LoadUS mem)));
4909 
4910   ins_cost(125);
4911   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4912 
4913   ins_encode %{
4914     __ movzwq($dst$$Register, $mem$$Address);
4915   %}
4916 
4917   ins_pipe(ialu_reg_mem);
4918 %}
4919 
4920 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4921 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4922   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4923 
4924   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4925   ins_encode %{
4926     __ movzbq($dst$$Register, $mem$$Address);
4927   %}
4928   ins_pipe(ialu_reg_mem);
4929 %}
4930 
4931 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4932 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4933   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4934   effect(KILL cr);
4935 
4936   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4937             "andl    $dst, right_n_bits($mask, 16)" %}
4938   ins_encode %{
4939     Register Rdst = $dst$$Register;
4940     __ movzwq(Rdst, $mem$$Address);
4941     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4942   %}
4943   ins_pipe(ialu_reg_mem);
4944 %}
4945 
4946 // Load Integer
4947 instruct loadI(rRegI dst, memory mem)
4948 %{
4949   match(Set dst (LoadI mem));
4950 
4951   ins_cost(125);
4952   format %{ "movl    $dst, $mem\t# int" %}
4953 
4954   ins_encode %{
4955     __ movl($dst$$Register, $mem$$Address);
4956   %}
4957 
4958   ins_pipe(ialu_reg_mem);
4959 %}
4960 
4961 // Load Integer (32 bit signed) to Byte (8 bit signed)
4962 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4963   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4964 
4965   ins_cost(125);
4966   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4967   ins_encode %{
4968     __ movsbl($dst$$Register, $mem$$Address);
4969   %}
4970   ins_pipe(ialu_reg_mem);
4971 %}
4972 
4973 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4974 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4975   match(Set dst (AndI (LoadI mem) mask));
4976 
4977   ins_cost(125);
4978   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4979   ins_encode %{
4980     __ movzbl($dst$$Register, $mem$$Address);
4981   %}
4982   ins_pipe(ialu_reg_mem);
4983 %}
4984 
4985 // Load Integer (32 bit signed) to Short (16 bit signed)
4986 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4987   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4988 
4989   ins_cost(125);
4990   format %{ "movswl  $dst, $mem\t# int -> short" %}
4991   ins_encode %{
4992     __ movswl($dst$$Register, $mem$$Address);
4993   %}
4994   ins_pipe(ialu_reg_mem);
4995 %}
4996 
4997 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4998 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4999   match(Set dst (AndI (LoadI mem) mask));
5000 
5001   ins_cost(125);
5002   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5003   ins_encode %{
5004     __ movzwl($dst$$Register, $mem$$Address);
5005   %}
5006   ins_pipe(ialu_reg_mem);
5007 %}
5008 
5009 // Load Integer into Long Register
5010 instruct loadI2L(rRegL dst, memory mem)
5011 %{
5012   match(Set dst (ConvI2L (LoadI mem)));
5013 
5014   ins_cost(125);
5015   format %{ "movslq  $dst, $mem\t# int -> long" %}
5016 
5017   ins_encode %{
5018     __ movslq($dst$$Register, $mem$$Address);
5019   %}
5020 
5021   ins_pipe(ialu_reg_mem);
5022 %}
5023 
5024 // Load Integer with mask 0xFF into Long Register
5025 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5026   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5027 
5028   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5029   ins_encode %{
5030     __ movzbq($dst$$Register, $mem$$Address);
5031   %}
5032   ins_pipe(ialu_reg_mem);
5033 %}
5034 
5035 // Load Integer with mask 0xFFFF into Long Register
5036 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5037   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5038 
5039   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5040   ins_encode %{
5041     __ movzwq($dst$$Register, $mem$$Address);
5042   %}
5043   ins_pipe(ialu_reg_mem);
5044 %}
5045 
5046 // Load Integer with a 31-bit mask into Long Register
5047 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5048   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5049   effect(KILL cr);
5050 
5051   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5052             "andl    $dst, $mask" %}
5053   ins_encode %{
5054     Register Rdst = $dst$$Register;
5055     __ movl(Rdst, $mem$$Address);
5056     __ andl(Rdst, $mask$$constant);
5057   %}
5058   ins_pipe(ialu_reg_mem);
5059 %}
5060 
5061 // Load Unsigned Integer into Long Register
5062 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5063 %{
5064   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5065 
5066   ins_cost(125);
5067   format %{ "movl    $dst, $mem\t# uint -> long" %}
5068 
5069   ins_encode %{
5070     __ movl($dst$$Register, $mem$$Address);
5071   %}
5072 
5073   ins_pipe(ialu_reg_mem);
5074 %}
5075 
5076 // Load Long
5077 instruct loadL(rRegL dst, memory mem)
5078 %{
5079   match(Set dst (LoadL mem));
5080 
5081   ins_cost(125);
5082   format %{ "movq    $dst, $mem\t# long" %}
5083 
5084   ins_encode %{
5085     __ movq($dst$$Register, $mem$$Address);
5086   %}
5087 
5088   ins_pipe(ialu_reg_mem); // XXX
5089 %}
5090 
5091 // Load Range
5092 instruct loadRange(rRegI dst, memory mem)
5093 %{
5094   match(Set dst (LoadRange mem));
5095 
5096   ins_cost(125); // XXX
5097   format %{ "movl    $dst, $mem\t# range" %}
5098   opcode(0x8B);
5099   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5100   ins_pipe(ialu_reg_mem);
5101 %}
5102 
5103 // Load Pointer
5104 instruct loadP(rRegP dst, memory mem)
5105 %{
5106   match(Set dst (LoadP mem));
5107 
5108   ins_cost(125); // XXX
5109   format %{ "movq    $dst, $mem\t# ptr" %}
5110   opcode(0x8B);
5111   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5112   ins_pipe(ialu_reg_mem); // XXX
5113 %}
5114 
5115 // Load Compressed Pointer
5116 instruct loadN(rRegN dst, memory mem)
5117 %{
5118    match(Set dst (LoadN mem));
5119 
5120    ins_cost(125); // XXX
5121    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5122    ins_encode %{
5123      __ movl($dst$$Register, $mem$$Address);
5124    %}
5125    ins_pipe(ialu_reg_mem); // XXX
5126 %}
5127 
5128 
5129 // Load Klass Pointer
5130 instruct loadKlass(rRegP dst, memory mem)
5131 %{
5132   match(Set dst (LoadKlass mem));
5133 
5134   ins_cost(125); // XXX
5135   format %{ "movq    $dst, $mem\t# class" %}
5136   opcode(0x8B);
5137   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5138   ins_pipe(ialu_reg_mem); // XXX
5139 %}
5140 
5141 // Load narrow Klass Pointer
5142 instruct loadNKlass(rRegN dst, memory mem)
5143 %{
5144   match(Set dst (LoadNKlass mem));
5145 
5146   ins_cost(125); // XXX
5147   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5148   ins_encode %{
5149     __ movl($dst$$Register, $mem$$Address);
5150   %}
5151   ins_pipe(ialu_reg_mem); // XXX
5152 %}
5153 
5154 // Load Float
5155 instruct loadF(regF dst, memory mem)
5156 %{
5157   match(Set dst (LoadF mem));
5158 
5159   ins_cost(145); // XXX
5160   format %{ "movss   $dst, $mem\t# float" %}
5161   ins_encode %{
5162     __ movflt($dst$$XMMRegister, $mem$$Address);
5163   %}
5164   ins_pipe(pipe_slow); // XXX
5165 %}
5166 
5167 // Load Double
5168 instruct loadD_partial(regD dst, memory mem)
5169 %{
5170   predicate(!UseXmmLoadAndClearUpper);
5171   match(Set dst (LoadD mem));
5172 
5173   ins_cost(145); // XXX
5174   format %{ "movlpd  $dst, $mem\t# double" %}
5175   ins_encode %{
5176     __ movdbl($dst$$XMMRegister, $mem$$Address);
5177   %}
5178   ins_pipe(pipe_slow); // XXX
5179 %}
5180 
5181 instruct loadD(regD dst, memory mem)
5182 %{
5183   predicate(UseXmmLoadAndClearUpper);
5184   match(Set dst (LoadD mem));
5185 
5186   ins_cost(145); // XXX
5187   format %{ "movsd   $dst, $mem\t# double" %}
5188   ins_encode %{
5189     __ movdbl($dst$$XMMRegister, $mem$$Address);
5190   %}
5191   ins_pipe(pipe_slow); // XXX
5192 %}
5193 
5194 // Load Effective Address
5195 instruct leaP8(rRegP dst, indOffset8 mem)
5196 %{
5197   match(Set dst mem);
5198 
5199   ins_cost(110); // XXX
5200   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5201   opcode(0x8D);
5202   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5203   ins_pipe(ialu_reg_reg_fat);
5204 %}
5205 
5206 instruct leaP32(rRegP dst, indOffset32 mem)
5207 %{
5208   match(Set dst mem);
5209 
5210   ins_cost(110);
5211   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5212   opcode(0x8D);
5213   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5214   ins_pipe(ialu_reg_reg_fat);
5215 %}
5216 
5217 // instruct leaPIdx(rRegP dst, indIndex mem)
5218 // %{
5219 //   match(Set dst mem);
5220 
5221 //   ins_cost(110);
5222 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5223 //   opcode(0x8D);
5224 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5225 //   ins_pipe(ialu_reg_reg_fat);
5226 // %}
5227 
5228 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5229 %{
5230   match(Set dst mem);
5231 
5232   ins_cost(110);
5233   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5234   opcode(0x8D);
5235   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5236   ins_pipe(ialu_reg_reg_fat);
5237 %}
5238 
5239 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5240 %{
5241   match(Set dst mem);
5242 
5243   ins_cost(110);
5244   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5245   opcode(0x8D);
5246   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5247   ins_pipe(ialu_reg_reg_fat);
5248 %}
5249 
5250 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5251 %{
5252   match(Set dst mem);
5253 
5254   ins_cost(110);
5255   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5256   opcode(0x8D);
5257   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5258   ins_pipe(ialu_reg_reg_fat);
5259 %}
5260 
5261 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5262 %{
5263   match(Set dst mem);
5264 
5265   ins_cost(110);
5266   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5267   opcode(0x8D);
5268   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5269   ins_pipe(ialu_reg_reg_fat);
5270 %}
5271 
5272 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5273 %{
5274   match(Set dst mem);
5275 
5276   ins_cost(110);
5277   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5278   opcode(0x8D);
5279   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5280   ins_pipe(ialu_reg_reg_fat);
5281 %}
5282 
5283 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5284 %{
5285   match(Set dst mem);
5286 
5287   ins_cost(110);
5288   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5289   opcode(0x8D);
5290   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5291   ins_pipe(ialu_reg_reg_fat);
5292 %}
5293 
5294 // Load Effective Address which uses Narrow (32-bits) oop
5295 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5296 %{
5297   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5298   match(Set dst mem);
5299 
5300   ins_cost(110);
5301   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5302   opcode(0x8D);
5303   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5304   ins_pipe(ialu_reg_reg_fat);
5305 %}
5306 
5307 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5308 %{
5309   predicate(Universe::narrow_oop_shift() == 0);
5310   match(Set dst mem);
5311 
5312   ins_cost(110); // XXX
5313   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5314   opcode(0x8D);
5315   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5316   ins_pipe(ialu_reg_reg_fat);
5317 %}
5318 
5319 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5320 %{
5321   predicate(Universe::narrow_oop_shift() == 0);
5322   match(Set dst mem);
5323 
5324   ins_cost(110);
5325   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5326   opcode(0x8D);
5327   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5328   ins_pipe(ialu_reg_reg_fat);
5329 %}
5330 
5331 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5332 %{
5333   predicate(Universe::narrow_oop_shift() == 0);
5334   match(Set dst mem);
5335 
5336   ins_cost(110);
5337   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5338   opcode(0x8D);
5339   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5340   ins_pipe(ialu_reg_reg_fat);
5341 %}
5342 
5343 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5344 %{
5345   predicate(Universe::narrow_oop_shift() == 0);
5346   match(Set dst mem);
5347 
5348   ins_cost(110);
5349   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5350   opcode(0x8D);
5351   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5352   ins_pipe(ialu_reg_reg_fat);
5353 %}
5354 
5355 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5356 %{
5357   predicate(Universe::narrow_oop_shift() == 0);
5358   match(Set dst mem);
5359 
5360   ins_cost(110);
5361   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5362   opcode(0x8D);
5363   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5364   ins_pipe(ialu_reg_reg_fat);
5365 %}
5366 
5367 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5368 %{
5369   predicate(Universe::narrow_oop_shift() == 0);
5370   match(Set dst mem);
5371 
5372   ins_cost(110);
5373   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5374   opcode(0x8D);
5375   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5376   ins_pipe(ialu_reg_reg_fat);
5377 %}
5378 
5379 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5380 %{
5381   predicate(Universe::narrow_oop_shift() == 0);
5382   match(Set dst mem);
5383 
5384   ins_cost(110);
5385   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5386   opcode(0x8D);
5387   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5388   ins_pipe(ialu_reg_reg_fat);
5389 %}
5390 
5391 instruct loadConI(rRegI dst, immI src)
5392 %{
5393   match(Set dst src);
5394 
5395   format %{ "movl    $dst, $src\t# int" %}
5396   ins_encode(load_immI(dst, src));
5397   ins_pipe(ialu_reg_fat); // XXX
5398 %}
5399 
5400 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5401 %{
5402   match(Set dst src);
5403   effect(KILL cr);
5404 
5405   ins_cost(50);
5406   format %{ "xorl    $dst, $dst\t# int" %}
5407   opcode(0x33); /* + rd */
5408   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5409   ins_pipe(ialu_reg);
5410 %}
5411 
5412 instruct loadConL(rRegL dst, immL src)
5413 %{
5414   match(Set dst src);
5415 
5416   ins_cost(150);
5417   format %{ "movq    $dst, $src\t# long" %}
5418   ins_encode(load_immL(dst, src));
5419   ins_pipe(ialu_reg);
5420 %}
5421 
5422 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5423 %{
5424   match(Set dst src);
5425   effect(KILL cr);
5426 
5427   ins_cost(50);
5428   format %{ "xorl    $dst, $dst\t# long" %}
5429   opcode(0x33); /* + rd */
5430   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5431   ins_pipe(ialu_reg); // XXX
5432 %}
5433 
5434 instruct loadConUL32(rRegL dst, immUL32 src)
5435 %{
5436   match(Set dst src);
5437 
5438   ins_cost(60);
5439   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5440   ins_encode(load_immUL32(dst, src));
5441   ins_pipe(ialu_reg);
5442 %}
5443 
5444 instruct loadConL32(rRegL dst, immL32 src)
5445 %{
5446   match(Set dst src);
5447 
5448   ins_cost(70);
5449   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5450   ins_encode(load_immL32(dst, src));
5451   ins_pipe(ialu_reg);
5452 %}
5453 
5454 instruct loadConP(rRegP dst, immP con) %{
5455   match(Set dst con);
5456 
5457   format %{ "movq    $dst, $con\t# ptr" %}
5458   ins_encode(load_immP(dst, con));
5459   ins_pipe(ialu_reg_fat); // XXX
5460 %}
5461 
5462 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5463 %{
5464   match(Set dst src);
5465   effect(KILL cr);
5466 
5467   ins_cost(50);
5468   format %{ "xorl    $dst, $dst\t# ptr" %}
5469   opcode(0x33); /* + rd */
5470   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5471   ins_pipe(ialu_reg);
5472 %}
5473 
5474 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5475 %{
5476   match(Set dst src);
5477   effect(KILL cr);
5478 
5479   ins_cost(60);
5480   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5481   ins_encode(load_immP31(dst, src));
5482   ins_pipe(ialu_reg);
5483 %}
5484 
5485 instruct loadConF(regF dst, immF con) %{
5486   match(Set dst con);
5487   ins_cost(125);
5488   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5489   ins_encode %{
5490     __ movflt($dst$$XMMRegister, $constantaddress($con));
5491   %}
5492   ins_pipe(pipe_slow);
5493 %}
5494 
5495 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5496   match(Set dst src);
5497   effect(KILL cr);
5498   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5499   ins_encode %{
5500     __ xorq($dst$$Register, $dst$$Register);
5501   %}
5502   ins_pipe(ialu_reg);
5503 %}
5504 
5505 instruct loadConN(rRegN dst, immN src) %{
5506   match(Set dst src);
5507 
5508   ins_cost(125);
5509   format %{ "movl    $dst, $src\t# compressed ptr" %}
5510   ins_encode %{
5511     address con = (address)$src$$constant;
5512     if (con == NULL) {
5513       ShouldNotReachHere();
5514     } else {
5515       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5516     }
5517   %}
5518   ins_pipe(ialu_reg_fat); // XXX
5519 %}
5520 
5521 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5522   match(Set dst src);
5523 
5524   ins_cost(125);
5525   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5526   ins_encode %{
5527     address con = (address)$src$$constant;
5528     if (con == NULL) {
5529       ShouldNotReachHere();
5530     } else {
5531       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5532     }
5533   %}
5534   ins_pipe(ialu_reg_fat); // XXX
5535 %}
5536 
5537 instruct loadConF0(regF dst, immF0 src)
5538 %{
5539   match(Set dst src);
5540   ins_cost(100);
5541 
5542   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5543   ins_encode %{
5544     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5545   %}
5546   ins_pipe(pipe_slow);
5547 %}
5548 
5549 // Use the same format since predicate() can not be used here.
5550 instruct loadConD(regD dst, immD con) %{
5551   match(Set dst con);
5552   ins_cost(125);
5553   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5554   ins_encode %{
5555     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5556   %}
5557   ins_pipe(pipe_slow);
5558 %}
5559 
5560 instruct loadConD0(regD dst, immD0 src)
5561 %{
5562   match(Set dst src);
5563   ins_cost(100);
5564 
5565   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5566   ins_encode %{
5567     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5568   %}
5569   ins_pipe(pipe_slow);
5570 %}
5571 
5572 instruct loadSSI(rRegI dst, stackSlotI src)
5573 %{
5574   match(Set dst src);
5575 
5576   ins_cost(125);
5577   format %{ "movl    $dst, $src\t# int stk" %}
5578   opcode(0x8B);
5579   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5580   ins_pipe(ialu_reg_mem);
5581 %}
5582 
5583 instruct loadSSL(rRegL dst, stackSlotL src)
5584 %{
5585   match(Set dst src);
5586 
5587   ins_cost(125);
5588   format %{ "movq    $dst, $src\t# long stk" %}
5589   opcode(0x8B);
5590   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5591   ins_pipe(ialu_reg_mem);
5592 %}
5593 
5594 instruct loadSSP(rRegP dst, stackSlotP src)
5595 %{
5596   match(Set dst src);
5597 
5598   ins_cost(125);
5599   format %{ "movq    $dst, $src\t# ptr stk" %}
5600   opcode(0x8B);
5601   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5602   ins_pipe(ialu_reg_mem);
5603 %}
5604 
5605 instruct loadSSF(regF dst, stackSlotF src)
5606 %{
5607   match(Set dst src);
5608 
5609   ins_cost(125);
5610   format %{ "movss   $dst, $src\t# float stk" %}
5611   ins_encode %{
5612     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5613   %}
5614   ins_pipe(pipe_slow); // XXX
5615 %}
5616 
5617 // Use the same format since predicate() can not be used here.
5618 instruct loadSSD(regD dst, stackSlotD src)
5619 %{
5620   match(Set dst src);
5621 
5622   ins_cost(125);
5623   format %{ "movsd   $dst, $src\t# double stk" %}
5624   ins_encode  %{
5625     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5626   %}
5627   ins_pipe(pipe_slow); // XXX
5628 %}
5629 
5630 // Prefetch instructions for allocation.
5631 // Must be safe to execute with invalid address (cannot fault).
5632 
5633 instruct prefetchAlloc( memory mem ) %{
5634   predicate(AllocatePrefetchInstr==3);
5635   match(PrefetchAllocation mem);
5636   ins_cost(125);
5637 
5638   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5639   ins_encode %{
5640     __ prefetchw($mem$$Address);
5641   %}
5642   ins_pipe(ialu_mem);
5643 %}
5644 
5645 instruct prefetchAllocNTA( memory mem ) %{
5646   predicate(AllocatePrefetchInstr==0);
5647   match(PrefetchAllocation mem);
5648   ins_cost(125);
5649 
5650   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5651   ins_encode %{
5652     __ prefetchnta($mem$$Address);
5653   %}
5654   ins_pipe(ialu_mem);
5655 %}
5656 
5657 instruct prefetchAllocT0( memory mem ) %{
5658   predicate(AllocatePrefetchInstr==1);
5659   match(PrefetchAllocation mem);
5660   ins_cost(125);
5661 
5662   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5663   ins_encode %{
5664     __ prefetcht0($mem$$Address);
5665   %}
5666   ins_pipe(ialu_mem);
5667 %}
5668 
5669 instruct prefetchAllocT2( memory mem ) %{
5670   predicate(AllocatePrefetchInstr==2);
5671   match(PrefetchAllocation mem);
5672   ins_cost(125);
5673 
5674   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5675   ins_encode %{
5676     __ prefetcht2($mem$$Address);
5677   %}
5678   ins_pipe(ialu_mem);
5679 %}
5680 
5681 //----------Store Instructions-------------------------------------------------
5682 
5683 // Store Byte
5684 instruct storeB(memory mem, rRegI src)
5685 %{
5686   match(Set mem (StoreB mem src));
5687 
5688   ins_cost(125); // XXX
5689   format %{ "movb    $mem, $src\t# byte" %}
5690   opcode(0x88);
5691   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5692   ins_pipe(ialu_mem_reg);
5693 %}
5694 
5695 // Store Char/Short
5696 instruct storeC(memory mem, rRegI src)
5697 %{
5698   match(Set mem (StoreC mem src));
5699 
5700   ins_cost(125); // XXX
5701   format %{ "movw    $mem, $src\t# char/short" %}
5702   opcode(0x89);
5703   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5704   ins_pipe(ialu_mem_reg);
5705 %}
5706 
5707 // Store Integer
5708 instruct storeI(memory mem, rRegI src)
5709 %{
5710   match(Set mem (StoreI mem src));
5711 
5712   ins_cost(125); // XXX
5713   format %{ "movl    $mem, $src\t# int" %}
5714   opcode(0x89);
5715   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5716   ins_pipe(ialu_mem_reg);
5717 %}
5718 
5719 // Store Long
5720 instruct storeL(memory mem, rRegL src)
5721 %{
5722   match(Set mem (StoreL mem src));
5723 
5724   ins_cost(125); // XXX
5725   format %{ "movq    $mem, $src\t# long" %}
5726   opcode(0x89);
5727   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5728   ins_pipe(ialu_mem_reg); // XXX
5729 %}
5730 
5731 // Store Pointer
5732 instruct storeP(memory mem, any_RegP src)
5733 %{
5734   match(Set mem (StoreP mem src));
5735 
5736   ins_cost(125); // XXX
5737   format %{ "movq    $mem, $src\t# ptr" %}
5738   opcode(0x89);
5739   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5740   ins_pipe(ialu_mem_reg);
5741 %}
5742 
5743 instruct storeImmP0(memory mem, immP0 zero)
5744 %{
5745   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5746   match(Set mem (StoreP mem zero));
5747 
5748   ins_cost(125); // XXX
5749   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5750   ins_encode %{
5751     __ movq($mem$$Address, r12);
5752   %}
5753   ins_pipe(ialu_mem_reg);
5754 %}
5755 
5756 // Store NULL Pointer, mark word, or other simple pointer constant.
5757 instruct storeImmP(memory mem, immP31 src)
5758 %{
5759   match(Set mem (StoreP mem src));
5760 
5761   ins_cost(150); // XXX
5762   format %{ "movq    $mem, $src\t# ptr" %}
5763   opcode(0xC7); /* C7 /0 */
5764   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5765   ins_pipe(ialu_mem_imm);
5766 %}
5767 
5768 // Store Compressed Pointer
5769 instruct storeN(memory mem, rRegN src)
5770 %{
5771   match(Set mem (StoreN mem src));
5772 
5773   ins_cost(125); // XXX
5774   format %{ "movl    $mem, $src\t# compressed ptr" %}
5775   ins_encode %{
5776     __ movl($mem$$Address, $src$$Register);
5777   %}
5778   ins_pipe(ialu_mem_reg);
5779 %}
5780 
5781 instruct storeNKlass(memory mem, rRegN src)
5782 %{
5783   match(Set mem (StoreNKlass mem src));
5784 
5785   ins_cost(125); // XXX
5786   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5787   ins_encode %{
5788     __ movl($mem$$Address, $src$$Register);
5789   %}
5790   ins_pipe(ialu_mem_reg);
5791 %}
5792 
5793 instruct storeImmN0(memory mem, immN0 zero)
5794 %{
5795   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5796   match(Set mem (StoreN mem zero));
5797 
5798   ins_cost(125); // XXX
5799   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5800   ins_encode %{
5801     __ movl($mem$$Address, r12);
5802   %}
5803   ins_pipe(ialu_mem_reg);
5804 %}
5805 
5806 instruct storeImmN(memory mem, immN src)
5807 %{
5808   match(Set mem (StoreN mem src));
5809 
5810   ins_cost(150); // XXX
5811   format %{ "movl    $mem, $src\t# compressed ptr" %}
5812   ins_encode %{
5813     address con = (address)$src$$constant;
5814     if (con == NULL) {
5815       __ movl($mem$$Address, (int32_t)0);
5816     } else {
5817       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5818     }
5819   %}
5820   ins_pipe(ialu_mem_imm);
5821 %}
5822 
5823 instruct storeImmNKlass(memory mem, immNKlass src)
5824 %{
5825   match(Set mem (StoreNKlass mem src));
5826 
5827   ins_cost(150); // XXX
5828   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5829   ins_encode %{
5830     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5831   %}
5832   ins_pipe(ialu_mem_imm);
5833 %}
5834 
5835 // Store Integer Immediate
5836 instruct storeImmI0(memory mem, immI0 zero)
5837 %{
5838   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5839   match(Set mem (StoreI mem zero));
5840 
5841   ins_cost(125); // XXX
5842   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5843   ins_encode %{
5844     __ movl($mem$$Address, r12);
5845   %}
5846   ins_pipe(ialu_mem_reg);
5847 %}
5848 
5849 instruct storeImmI(memory mem, immI src)
5850 %{
5851   match(Set mem (StoreI mem src));
5852 
5853   ins_cost(150);
5854   format %{ "movl    $mem, $src\t# int" %}
5855   opcode(0xC7); /* C7 /0 */
5856   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5857   ins_pipe(ialu_mem_imm);
5858 %}
5859 
5860 // Store Long Immediate
5861 instruct storeImmL0(memory mem, immL0 zero)
5862 %{
5863   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5864   match(Set mem (StoreL mem zero));
5865 
5866   ins_cost(125); // XXX
5867   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5868   ins_encode %{
5869     __ movq($mem$$Address, r12);
5870   %}
5871   ins_pipe(ialu_mem_reg);
5872 %}
5873 
5874 instruct storeImmL(memory mem, immL32 src)
5875 %{
5876   match(Set mem (StoreL mem src));
5877 
5878   ins_cost(150);
5879   format %{ "movq    $mem, $src\t# long" %}
5880   opcode(0xC7); /* C7 /0 */
5881   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5882   ins_pipe(ialu_mem_imm);
5883 %}
5884 
5885 // Store Short/Char Immediate
5886 instruct storeImmC0(memory mem, immI0 zero)
5887 %{
5888   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5889   match(Set mem (StoreC mem zero));
5890 
5891   ins_cost(125); // XXX
5892   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5893   ins_encode %{
5894     __ movw($mem$$Address, r12);
5895   %}
5896   ins_pipe(ialu_mem_reg);
5897 %}
5898 
5899 instruct storeImmI16(memory mem, immI16 src)
5900 %{
5901   predicate(UseStoreImmI16);
5902   match(Set mem (StoreC mem src));
5903 
5904   ins_cost(150);
5905   format %{ "movw    $mem, $src\t# short/char" %}
5906   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5907   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5908   ins_pipe(ialu_mem_imm);
5909 %}
5910 
5911 // Store Byte Immediate
5912 instruct storeImmB0(memory mem, immI0 zero)
5913 %{
5914   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5915   match(Set mem (StoreB mem zero));
5916 
5917   ins_cost(125); // XXX
5918   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5919   ins_encode %{
5920     __ movb($mem$$Address, r12);
5921   %}
5922   ins_pipe(ialu_mem_reg);
5923 %}
5924 
5925 instruct storeImmB(memory mem, immI8 src)
5926 %{
5927   match(Set mem (StoreB mem src));
5928 
5929   ins_cost(150); // XXX
5930   format %{ "movb    $mem, $src\t# byte" %}
5931   opcode(0xC6); /* C6 /0 */
5932   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5933   ins_pipe(ialu_mem_imm);
5934 %}
5935 
5936 // Store CMS card-mark Immediate
5937 instruct storeImmCM0_reg(memory mem, immI0 zero)
5938 %{
5939   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5940   match(Set mem (StoreCM mem zero));
5941 
5942   ins_cost(125); // XXX
5943   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5944   ins_encode %{
5945     __ movb($mem$$Address, r12);
5946   %}
5947   ins_pipe(ialu_mem_reg);
5948 %}
5949 
5950 instruct storeImmCM0(memory mem, immI0 src)
5951 %{
5952   match(Set mem (StoreCM mem src));
5953 
5954   ins_cost(150); // XXX
5955   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5956   opcode(0xC6); /* C6 /0 */
5957   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5958   ins_pipe(ialu_mem_imm);
5959 %}
5960 
5961 // Store Float
5962 instruct storeF(memory mem, regF src)
5963 %{
5964   match(Set mem (StoreF mem src));
5965 
5966   ins_cost(95); // XXX
5967   format %{ "movss   $mem, $src\t# float" %}
5968   ins_encode %{
5969     __ movflt($mem$$Address, $src$$XMMRegister);
5970   %}
5971   ins_pipe(pipe_slow); // XXX
5972 %}
5973 
5974 // Store immediate Float value (it is faster than store from XMM register)
5975 instruct storeF0(memory mem, immF0 zero)
5976 %{
5977   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5978   match(Set mem (StoreF mem zero));
5979 
5980   ins_cost(25); // XXX
5981   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5982   ins_encode %{
5983     __ movl($mem$$Address, r12);
5984   %}
5985   ins_pipe(ialu_mem_reg);
5986 %}
5987 
5988 instruct storeF_imm(memory mem, immF src)
5989 %{
5990   match(Set mem (StoreF mem src));
5991 
5992   ins_cost(50);
5993   format %{ "movl    $mem, $src\t# float" %}
5994   opcode(0xC7); /* C7 /0 */
5995   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5996   ins_pipe(ialu_mem_imm);
5997 %}
5998 
5999 // Store Double
6000 instruct storeD(memory mem, regD src)
6001 %{
6002   match(Set mem (StoreD mem src));
6003 
6004   ins_cost(95); // XXX
6005   format %{ "movsd   $mem, $src\t# double" %}
6006   ins_encode %{
6007     __ movdbl($mem$$Address, $src$$XMMRegister);
6008   %}
6009   ins_pipe(pipe_slow); // XXX
6010 %}
6011 
6012 // Store immediate double 0.0 (it is faster than store from XMM register)
6013 instruct storeD0_imm(memory mem, immD0 src)
6014 %{
6015   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6016   match(Set mem (StoreD mem src));
6017 
6018   ins_cost(50);
6019   format %{ "movq    $mem, $src\t# double 0." %}
6020   opcode(0xC7); /* C7 /0 */
6021   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6022   ins_pipe(ialu_mem_imm);
6023 %}
6024 
6025 instruct storeD0(memory mem, immD0 zero)
6026 %{
6027   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6028   match(Set mem (StoreD mem zero));
6029 
6030   ins_cost(25); // XXX
6031   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6032   ins_encode %{
6033     __ movq($mem$$Address, r12);
6034   %}
6035   ins_pipe(ialu_mem_reg);
6036 %}
6037 
6038 instruct storeSSI(stackSlotI dst, rRegI src)
6039 %{
6040   match(Set dst src);
6041 
6042   ins_cost(100);
6043   format %{ "movl    $dst, $src\t# int stk" %}
6044   opcode(0x89);
6045   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6046   ins_pipe( ialu_mem_reg );
6047 %}
6048 
6049 instruct storeSSL(stackSlotL dst, rRegL src)
6050 %{
6051   match(Set dst src);
6052 
6053   ins_cost(100);
6054   format %{ "movq    $dst, $src\t# long stk" %}
6055   opcode(0x89);
6056   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6057   ins_pipe(ialu_mem_reg);
6058 %}
6059 
6060 instruct storeSSP(stackSlotP dst, rRegP src)
6061 %{
6062   match(Set dst src);
6063 
6064   ins_cost(100);
6065   format %{ "movq    $dst, $src\t# ptr stk" %}
6066   opcode(0x89);
6067   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6068   ins_pipe(ialu_mem_reg);
6069 %}
6070 
6071 instruct storeSSF(stackSlotF dst, regF src)
6072 %{
6073   match(Set dst src);
6074 
6075   ins_cost(95); // XXX
6076   format %{ "movss   $dst, $src\t# float stk" %}
6077   ins_encode %{
6078     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6079   %}
6080   ins_pipe(pipe_slow); // XXX
6081 %}
6082 
6083 instruct storeSSD(stackSlotD dst, regD src)
6084 %{
6085   match(Set dst src);
6086 
6087   ins_cost(95); // XXX
6088   format %{ "movsd   $dst, $src\t# double stk" %}
6089   ins_encode %{
6090     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6091   %}
6092   ins_pipe(pipe_slow); // XXX
6093 %}
6094 
6095 //----------BSWAP Instructions-------------------------------------------------
6096 instruct bytes_reverse_int(rRegI dst) %{
6097   match(Set dst (ReverseBytesI dst));
6098 
6099   format %{ "bswapl  $dst" %}
6100   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6101   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6102   ins_pipe( ialu_reg );
6103 %}
6104 
6105 instruct bytes_reverse_long(rRegL dst) %{
6106   match(Set dst (ReverseBytesL dst));
6107 
6108   format %{ "bswapq  $dst" %}
6109   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6110   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6111   ins_pipe( ialu_reg);
6112 %}
6113 
6114 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6115   match(Set dst (ReverseBytesUS dst));
6116   effect(KILL cr);
6117 
6118   format %{ "bswapl  $dst\n\t"
6119             "shrl    $dst,16\n\t" %}
6120   ins_encode %{
6121     __ bswapl($dst$$Register);
6122     __ shrl($dst$$Register, 16);
6123   %}
6124   ins_pipe( ialu_reg );
6125 %}
6126 
6127 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6128   match(Set dst (ReverseBytesS dst));
6129   effect(KILL cr);
6130 
6131   format %{ "bswapl  $dst\n\t"
6132             "sar     $dst,16\n\t" %}
6133   ins_encode %{
6134     __ bswapl($dst$$Register);
6135     __ sarl($dst$$Register, 16);
6136   %}
6137   ins_pipe( ialu_reg );
6138 %}
6139 
6140 //---------- Zeros Count Instructions ------------------------------------------
6141 
6142 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6143   predicate(UseCountLeadingZerosInstruction);
6144   match(Set dst (CountLeadingZerosI src));
6145   effect(KILL cr);
6146 
6147   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6148   ins_encode %{
6149     __ lzcntl($dst$$Register, $src$$Register);
6150   %}
6151   ins_pipe(ialu_reg);
6152 %}
6153 
6154 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6155   predicate(!UseCountLeadingZerosInstruction);
6156   match(Set dst (CountLeadingZerosI src));
6157   effect(KILL cr);
6158 
6159   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6160             "jnz     skip\n\t"
6161             "movl    $dst, -1\n"
6162       "skip:\n\t"
6163             "negl    $dst\n\t"
6164             "addl    $dst, 31" %}
6165   ins_encode %{
6166     Register Rdst = $dst$$Register;
6167     Register Rsrc = $src$$Register;
6168     Label skip;
6169     __ bsrl(Rdst, Rsrc);
6170     __ jccb(Assembler::notZero, skip);
6171     __ movl(Rdst, -1);
6172     __ bind(skip);
6173     __ negl(Rdst);
6174     __ addl(Rdst, BitsPerInt - 1);
6175   %}
6176   ins_pipe(ialu_reg);
6177 %}
6178 
6179 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6180   predicate(UseCountLeadingZerosInstruction);
6181   match(Set dst (CountLeadingZerosL src));
6182   effect(KILL cr);
6183 
6184   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6185   ins_encode %{
6186     __ lzcntq($dst$$Register, $src$$Register);
6187   %}
6188   ins_pipe(ialu_reg);
6189 %}
6190 
6191 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6192   predicate(!UseCountLeadingZerosInstruction);
6193   match(Set dst (CountLeadingZerosL src));
6194   effect(KILL cr);
6195 
6196   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6197             "jnz     skip\n\t"
6198             "movl    $dst, -1\n"
6199       "skip:\n\t"
6200             "negl    $dst\n\t"
6201             "addl    $dst, 63" %}
6202   ins_encode %{
6203     Register Rdst = $dst$$Register;
6204     Register Rsrc = $src$$Register;
6205     Label skip;
6206     __ bsrq(Rdst, Rsrc);
6207     __ jccb(Assembler::notZero, skip);
6208     __ movl(Rdst, -1);
6209     __ bind(skip);
6210     __ negl(Rdst);
6211     __ addl(Rdst, BitsPerLong - 1);
6212   %}
6213   ins_pipe(ialu_reg);
6214 %}
6215 
6216 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6217   predicate(UseCountTrailingZerosInstruction);
6218   match(Set dst (CountTrailingZerosI src));
6219   effect(KILL cr);
6220 
6221   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6222   ins_encode %{
6223     __ tzcntl($dst$$Register, $src$$Register);
6224   %}
6225   ins_pipe(ialu_reg);
6226 %}
6227 
6228 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6229   predicate(!UseCountTrailingZerosInstruction);
6230   match(Set dst (CountTrailingZerosI src));
6231   effect(KILL cr);
6232 
6233   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6234             "jnz     done\n\t"
6235             "movl    $dst, 32\n"
6236       "done:" %}
6237   ins_encode %{
6238     Register Rdst = $dst$$Register;
6239     Label done;
6240     __ bsfl(Rdst, $src$$Register);
6241     __ jccb(Assembler::notZero, done);
6242     __ movl(Rdst, BitsPerInt);
6243     __ bind(done);
6244   %}
6245   ins_pipe(ialu_reg);
6246 %}
6247 
6248 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6249   predicate(UseCountTrailingZerosInstruction);
6250   match(Set dst (CountTrailingZerosL src));
6251   effect(KILL cr);
6252 
6253   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6254   ins_encode %{
6255     __ tzcntq($dst$$Register, $src$$Register);
6256   %}
6257   ins_pipe(ialu_reg);
6258 %}
6259 
6260 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6261   predicate(!UseCountTrailingZerosInstruction);
6262   match(Set dst (CountTrailingZerosL src));
6263   effect(KILL cr);
6264 
6265   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6266             "jnz     done\n\t"
6267             "movl    $dst, 64\n"
6268       "done:" %}
6269   ins_encode %{
6270     Register Rdst = $dst$$Register;
6271     Label done;
6272     __ bsfq(Rdst, $src$$Register);
6273     __ jccb(Assembler::notZero, done);
6274     __ movl(Rdst, BitsPerLong);
6275     __ bind(done);
6276   %}
6277   ins_pipe(ialu_reg);
6278 %}
6279 
6280 
6281 //---------- Population Count Instructions -------------------------------------
6282 
6283 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6284   predicate(UsePopCountInstruction);
6285   match(Set dst (PopCountI src));
6286   effect(KILL cr);
6287 
6288   format %{ "popcnt  $dst, $src" %}
6289   ins_encode %{
6290     __ popcntl($dst$$Register, $src$$Register);
6291   %}
6292   ins_pipe(ialu_reg);
6293 %}
6294 
6295 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6296   predicate(UsePopCountInstruction);
6297   match(Set dst (PopCountI (LoadI mem)));
6298   effect(KILL cr);
6299 
6300   format %{ "popcnt  $dst, $mem" %}
6301   ins_encode %{
6302     __ popcntl($dst$$Register, $mem$$Address);
6303   %}
6304   ins_pipe(ialu_reg);
6305 %}
6306 
6307 // Note: Long.bitCount(long) returns an int.
6308 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6309   predicate(UsePopCountInstruction);
6310   match(Set dst (PopCountL src));
6311   effect(KILL cr);
6312 
6313   format %{ "popcnt  $dst, $src" %}
6314   ins_encode %{
6315     __ popcntq($dst$$Register, $src$$Register);
6316   %}
6317   ins_pipe(ialu_reg);
6318 %}
6319 
6320 // Note: Long.bitCount(long) returns an int.
6321 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6322   predicate(UsePopCountInstruction);
6323   match(Set dst (PopCountL (LoadL mem)));
6324   effect(KILL cr);
6325 
6326   format %{ "popcnt  $dst, $mem" %}
6327   ins_encode %{
6328     __ popcntq($dst$$Register, $mem$$Address);
6329   %}
6330   ins_pipe(ialu_reg);
6331 %}
6332 
6333 
6334 //----------MemBar Instructions-----------------------------------------------
6335 // Memory barrier flavors
6336 
6337 instruct membar_acquire()
6338 %{
6339   match(MemBarAcquire);
6340   match(LoadFence);
6341   ins_cost(0);
6342 
6343   size(0);
6344   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6345   ins_encode();
6346   ins_pipe(empty);
6347 %}
6348 
6349 instruct membar_acquire_lock()
6350 %{
6351   match(MemBarAcquireLock);
6352   ins_cost(0);
6353 
6354   size(0);
6355   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6356   ins_encode();
6357   ins_pipe(empty);
6358 %}
6359 
6360 instruct membar_release()
6361 %{
6362   match(MemBarRelease);
6363   match(StoreFence);
6364   ins_cost(0);
6365 
6366   size(0);
6367   format %{ "MEMBAR-release ! (empty encoding)" %}
6368   ins_encode();
6369   ins_pipe(empty);
6370 %}
6371 
6372 instruct membar_release_lock()
6373 %{
6374   match(MemBarReleaseLock);
6375   ins_cost(0);
6376 
6377   size(0);
6378   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6379   ins_encode();
6380   ins_pipe(empty);
6381 %}
6382 
6383 instruct membar_volatile(rFlagsReg cr) %{
6384   match(MemBarVolatile);
6385   effect(KILL cr);
6386   ins_cost(400);
6387 
6388   format %{
6389     $$template
6390     if (os::is_MP()) {
6391       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6392     } else {
6393       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6394     }
6395   %}
6396   ins_encode %{
6397     __ membar(Assembler::StoreLoad);
6398   %}
6399   ins_pipe(pipe_slow);
6400 %}
6401 
6402 instruct unnecessary_membar_volatile()
6403 %{
6404   match(MemBarVolatile);
6405   predicate(Matcher::post_store_load_barrier(n));
6406   ins_cost(0);
6407 
6408   size(0);
6409   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6410   ins_encode();
6411   ins_pipe(empty);
6412 %}
6413 
6414 instruct membar_storestore() %{
6415   match(MemBarStoreStore);
6416   ins_cost(0);
6417 
6418   size(0);
6419   format %{ "MEMBAR-storestore (empty encoding)" %}
6420   ins_encode( );
6421   ins_pipe(empty);
6422 %}
6423 
6424 //----------Move Instructions--------------------------------------------------
6425 
6426 instruct castX2P(rRegP dst, rRegL src)
6427 %{
6428   match(Set dst (CastX2P src));
6429 
6430   format %{ "movq    $dst, $src\t# long->ptr" %}
6431   ins_encode %{
6432     if ($dst$$reg != $src$$reg) {
6433       __ movptr($dst$$Register, $src$$Register);
6434     }
6435   %}
6436   ins_pipe(ialu_reg_reg); // XXX
6437 %}
6438 
6439 instruct castP2X(rRegL dst, rRegP src)
6440 %{
6441   match(Set dst (CastP2X src));
6442 
6443   format %{ "movq    $dst, $src\t# ptr -> long" %}
6444   ins_encode %{
6445     if ($dst$$reg != $src$$reg) {
6446       __ movptr($dst$$Register, $src$$Register);
6447     }
6448   %}
6449   ins_pipe(ialu_reg_reg); // XXX
6450 %}
6451 
6452 // Convert oop into int for vectors alignment masking
6453 instruct convP2I(rRegI dst, rRegP src)
6454 %{
6455   match(Set dst (ConvL2I (CastP2X src)));
6456 
6457   format %{ "movl    $dst, $src\t# ptr -> int" %}
6458   ins_encode %{
6459     __ movl($dst$$Register, $src$$Register);
6460   %}
6461   ins_pipe(ialu_reg_reg); // XXX
6462 %}
6463 
6464 // Convert compressed oop into int for vectors alignment masking
6465 // in case of 32bit oops (heap < 4Gb).
6466 instruct convN2I(rRegI dst, rRegN src)
6467 %{
6468   predicate(Universe::narrow_oop_shift() == 0);
6469   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6470 
6471   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6472   ins_encode %{
6473     __ movl($dst$$Register, $src$$Register);
6474   %}
6475   ins_pipe(ialu_reg_reg); // XXX
6476 %}
6477 
6478 // Convert oop pointer into compressed form
6479 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6480   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6481   match(Set dst (EncodeP src));
6482   effect(KILL cr);
6483   format %{ "encode_heap_oop $dst,$src" %}
6484   ins_encode %{
6485     Register s = $src$$Register;
6486     Register d = $dst$$Register;
6487     if (s != d) {
6488       __ movq(d, s);
6489     }
6490     __ encode_heap_oop(d);
6491   %}
6492   ins_pipe(ialu_reg_long);
6493 %}
6494 
6495 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6496   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6497   match(Set dst (EncodeP src));
6498   effect(KILL cr);
6499   format %{ "encode_heap_oop_not_null $dst,$src" %}
6500   ins_encode %{
6501     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6502   %}
6503   ins_pipe(ialu_reg_long);
6504 %}
6505 
6506 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6507   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6508             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6509   match(Set dst (DecodeN src));
6510   effect(KILL cr);
6511   format %{ "decode_heap_oop $dst,$src" %}
6512   ins_encode %{
6513     Register s = $src$$Register;
6514     Register d = $dst$$Register;
6515     if (s != d) {
6516       __ movq(d, s);
6517     }
6518     __ decode_heap_oop(d);
6519   %}
6520   ins_pipe(ialu_reg_long);
6521 %}
6522 
6523 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6524   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6525             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6526   match(Set dst (DecodeN src));
6527   effect(KILL cr);
6528   format %{ "decode_heap_oop_not_null $dst,$src" %}
6529   ins_encode %{
6530     Register s = $src$$Register;
6531     Register d = $dst$$Register;
6532     if (s != d) {
6533       __ decode_heap_oop_not_null(d, s);
6534     } else {
6535       __ decode_heap_oop_not_null(d);
6536     }
6537   %}
6538   ins_pipe(ialu_reg_long);
6539 %}
6540 
6541 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6542   match(Set dst (EncodePKlass src));
6543   effect(KILL cr);
6544   format %{ "encode_klass_not_null $dst,$src" %}
6545   ins_encode %{
6546     __ encode_klass_not_null($dst$$Register, $src$$Register);
6547   %}
6548   ins_pipe(ialu_reg_long);
6549 %}
6550 
6551 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6552   match(Set dst (DecodeNKlass src));
6553   effect(KILL cr);
6554   format %{ "decode_klass_not_null $dst,$src" %}
6555   ins_encode %{
6556     Register s = $src$$Register;
6557     Register d = $dst$$Register;
6558     if (s != d) {
6559       __ decode_klass_not_null(d, s);
6560     } else {
6561       __ decode_klass_not_null(d);
6562     }
6563   %}
6564   ins_pipe(ialu_reg_long);
6565 %}
6566 
6567 
6568 //----------Conditional Move---------------------------------------------------
6569 // Jump
6570 // dummy instruction for generating temp registers
6571 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6572   match(Jump (LShiftL switch_val shift));
6573   ins_cost(350);
6574   predicate(false);
6575   effect(TEMP dest);
6576 
6577   format %{ "leaq    $dest, [$constantaddress]\n\t"
6578             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6579   ins_encode %{
6580     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6581     // to do that and the compiler is using that register as one it can allocate.
6582     // So we build it all by hand.
6583     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6584     // ArrayAddress dispatch(table, index);
6585     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6586     __ lea($dest$$Register, $constantaddress);
6587     __ jmp(dispatch);
6588   %}
6589   ins_pipe(pipe_jmp);
6590 %}
6591 
6592 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6593   match(Jump (AddL (LShiftL switch_val shift) offset));
6594   ins_cost(350);
6595   effect(TEMP dest);
6596 
6597   format %{ "leaq    $dest, [$constantaddress]\n\t"
6598             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6599   ins_encode %{
6600     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6601     // to do that and the compiler is using that register as one it can allocate.
6602     // So we build it all by hand.
6603     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6604     // ArrayAddress dispatch(table, index);
6605     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6606     __ lea($dest$$Register, $constantaddress);
6607     __ jmp(dispatch);
6608   %}
6609   ins_pipe(pipe_jmp);
6610 %}
6611 
6612 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6613   match(Jump switch_val);
6614   ins_cost(350);
6615   effect(TEMP dest);
6616 
6617   format %{ "leaq    $dest, [$constantaddress]\n\t"
6618             "jmp     [$dest + $switch_val]\n\t" %}
6619   ins_encode %{
6620     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6621     // to do that and the compiler is using that register as one it can allocate.
6622     // So we build it all by hand.
6623     // Address index(noreg, switch_reg, Address::times_1);
6624     // ArrayAddress dispatch(table, index);
6625     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6626     __ lea($dest$$Register, $constantaddress);
6627     __ jmp(dispatch);
6628   %}
6629   ins_pipe(pipe_jmp);
6630 %}
6631 
6632 // Conditional move
6633 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6634 %{
6635   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6636 
6637   ins_cost(200); // XXX
6638   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6639   opcode(0x0F, 0x40);
6640   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6641   ins_pipe(pipe_cmov_reg);
6642 %}
6643 
6644 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6645   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6646 
6647   ins_cost(200); // XXX
6648   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6649   opcode(0x0F, 0x40);
6650   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6651   ins_pipe(pipe_cmov_reg);
6652 %}
6653 
6654 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6655   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6656   ins_cost(200);
6657   expand %{
6658     cmovI_regU(cop, cr, dst, src);
6659   %}
6660 %}
6661 
6662 // Conditional move
6663 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6664   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6665 
6666   ins_cost(250); // XXX
6667   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6668   opcode(0x0F, 0x40);
6669   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6670   ins_pipe(pipe_cmov_mem);
6671 %}
6672 
6673 // Conditional move
6674 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6675 %{
6676   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6677 
6678   ins_cost(250); // XXX
6679   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6680   opcode(0x0F, 0x40);
6681   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6682   ins_pipe(pipe_cmov_mem);
6683 %}
6684 
6685 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6686   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6687   ins_cost(250);
6688   expand %{
6689     cmovI_memU(cop, cr, dst, src);
6690   %}
6691 %}
6692 
6693 // Conditional move
6694 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6695 %{
6696   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6697 
6698   ins_cost(200); // XXX
6699   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6700   opcode(0x0F, 0x40);
6701   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6702   ins_pipe(pipe_cmov_reg);
6703 %}
6704 
6705 // Conditional move
6706 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6707 %{
6708   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6709 
6710   ins_cost(200); // XXX
6711   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6712   opcode(0x0F, 0x40);
6713   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6714   ins_pipe(pipe_cmov_reg);
6715 %}
6716 
6717 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6718   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6719   ins_cost(200);
6720   expand %{
6721     cmovN_regU(cop, cr, dst, src);
6722   %}
6723 %}
6724 
6725 // Conditional move
6726 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6727 %{
6728   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6729 
6730   ins_cost(200); // XXX
6731   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6732   opcode(0x0F, 0x40);
6733   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6734   ins_pipe(pipe_cmov_reg);  // XXX
6735 %}
6736 
6737 // Conditional move
6738 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6739 %{
6740   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6741 
6742   ins_cost(200); // XXX
6743   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6744   opcode(0x0F, 0x40);
6745   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6746   ins_pipe(pipe_cmov_reg); // XXX
6747 %}
6748 
6749 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6750   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6751   ins_cost(200);
6752   expand %{
6753     cmovP_regU(cop, cr, dst, src);
6754   %}
6755 %}
6756 
6757 // DISABLED: Requires the ADLC to emit a bottom_type call that
6758 // correctly meets the two pointer arguments; one is an incoming
6759 // register but the other is a memory operand.  ALSO appears to
6760 // be buggy with implicit null checks.
6761 //
6762 //// Conditional move
6763 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6764 //%{
6765 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6766 //  ins_cost(250);
6767 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6768 //  opcode(0x0F,0x40);
6769 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6770 //  ins_pipe( pipe_cmov_mem );
6771 //%}
6772 //
6773 //// Conditional move
6774 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6775 //%{
6776 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6777 //  ins_cost(250);
6778 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6779 //  opcode(0x0F,0x40);
6780 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6781 //  ins_pipe( pipe_cmov_mem );
6782 //%}
6783 
6784 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6785 %{
6786   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6787 
6788   ins_cost(200); // XXX
6789   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6790   opcode(0x0F, 0x40);
6791   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6792   ins_pipe(pipe_cmov_reg);  // XXX
6793 %}
6794 
6795 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6796 %{
6797   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6798 
6799   ins_cost(200); // XXX
6800   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6801   opcode(0x0F, 0x40);
6802   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6803   ins_pipe(pipe_cmov_mem);  // XXX
6804 %}
6805 
6806 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6807 %{
6808   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6809 
6810   ins_cost(200); // XXX
6811   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6812   opcode(0x0F, 0x40);
6813   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6814   ins_pipe(pipe_cmov_reg); // XXX
6815 %}
6816 
6817 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6818   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6819   ins_cost(200);
6820   expand %{
6821     cmovL_regU(cop, cr, dst, src);
6822   %}
6823 %}
6824 
6825 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6826 %{
6827   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6828 
6829   ins_cost(200); // XXX
6830   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6831   opcode(0x0F, 0x40);
6832   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6833   ins_pipe(pipe_cmov_mem); // XXX
6834 %}
6835 
6836 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6837   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6838   ins_cost(200);
6839   expand %{
6840     cmovL_memU(cop, cr, dst, src);
6841   %}
6842 %}
6843 
6844 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6845 %{
6846   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6847 
6848   ins_cost(200); // XXX
6849   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6850             "movss     $dst, $src\n"
6851     "skip:" %}
6852   ins_encode %{
6853     Label Lskip;
6854     // Invert sense of branch from sense of CMOV
6855     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6856     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6857     __ bind(Lskip);
6858   %}
6859   ins_pipe(pipe_slow);
6860 %}
6861 
6862 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6863 // %{
6864 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6865 
6866 //   ins_cost(200); // XXX
6867 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6868 //             "movss     $dst, $src\n"
6869 //     "skip:" %}
6870 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6871 //   ins_pipe(pipe_slow);
6872 // %}
6873 
6874 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6875 %{
6876   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6877 
6878   ins_cost(200); // XXX
6879   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6880             "movss     $dst, $src\n"
6881     "skip:" %}
6882   ins_encode %{
6883     Label Lskip;
6884     // Invert sense of branch from sense of CMOV
6885     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6886     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6887     __ bind(Lskip);
6888   %}
6889   ins_pipe(pipe_slow);
6890 %}
6891 
6892 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6893   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6894   ins_cost(200);
6895   expand %{
6896     cmovF_regU(cop, cr, dst, src);
6897   %}
6898 %}
6899 
6900 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6901 %{
6902   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6903 
6904   ins_cost(200); // XXX
6905   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6906             "movsd     $dst, $src\n"
6907     "skip:" %}
6908   ins_encode %{
6909     Label Lskip;
6910     // Invert sense of branch from sense of CMOV
6911     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6912     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6913     __ bind(Lskip);
6914   %}
6915   ins_pipe(pipe_slow);
6916 %}
6917 
6918 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6919 %{
6920   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6921 
6922   ins_cost(200); // XXX
6923   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6924             "movsd     $dst, $src\n"
6925     "skip:" %}
6926   ins_encode %{
6927     Label Lskip;
6928     // Invert sense of branch from sense of CMOV
6929     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6930     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6931     __ bind(Lskip);
6932   %}
6933   ins_pipe(pipe_slow);
6934 %}
6935 
6936 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6937   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6938   ins_cost(200);
6939   expand %{
6940     cmovD_regU(cop, cr, dst, src);
6941   %}
6942 %}
6943 
6944 //----------Arithmetic Instructions--------------------------------------------
6945 //----------Addition Instructions----------------------------------------------
6946 
6947 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6948 %{
6949   match(Set dst (AddI dst src));
6950   effect(KILL cr);
6951 
6952   format %{ "addl    $dst, $src\t# int" %}
6953   opcode(0x03);
6954   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6955   ins_pipe(ialu_reg_reg);
6956 %}
6957 
6958 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6959 %{
6960   match(Set dst (AddI dst src));
6961   effect(KILL cr);
6962 
6963   format %{ "addl    $dst, $src\t# int" %}
6964   opcode(0x81, 0x00); /* /0 id */
6965   ins_encode(OpcSErm(dst, src), Con8or32(src));
6966   ins_pipe( ialu_reg );
6967 %}
6968 
6969 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6970 %{
6971   match(Set dst (AddI dst (LoadI src)));
6972   effect(KILL cr);
6973 
6974   ins_cost(125); // XXX
6975   format %{ "addl    $dst, $src\t# int" %}
6976   opcode(0x03);
6977   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6978   ins_pipe(ialu_reg_mem);
6979 %}
6980 
6981 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6982 %{
6983   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6984   effect(KILL cr);
6985 
6986   ins_cost(150); // XXX
6987   format %{ "addl    $dst, $src\t# int" %}
6988   opcode(0x01); /* Opcode 01 /r */
6989   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6990   ins_pipe(ialu_mem_reg);
6991 %}
6992 
6993 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6994 %{
6995   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6996   effect(KILL cr);
6997 
6998   ins_cost(125); // XXX
6999   format %{ "addl    $dst, $src\t# int" %}
7000   opcode(0x81); /* Opcode 81 /0 id */
7001   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7002   ins_pipe(ialu_mem_imm);
7003 %}
7004 
7005 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7006 %{
7007   predicate(UseIncDec);
7008   match(Set dst (AddI dst src));
7009   effect(KILL cr);
7010 
7011   format %{ "incl    $dst\t# int" %}
7012   opcode(0xFF, 0x00); // FF /0
7013   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7014   ins_pipe(ialu_reg);
7015 %}
7016 
7017 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7018 %{
7019   predicate(UseIncDec);
7020   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7021   effect(KILL cr);
7022 
7023   ins_cost(125); // XXX
7024   format %{ "incl    $dst\t# int" %}
7025   opcode(0xFF); /* Opcode FF /0 */
7026   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7027   ins_pipe(ialu_mem_imm);
7028 %}
7029 
7030 // XXX why does that use AddI
7031 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7032 %{
7033   predicate(UseIncDec);
7034   match(Set dst (AddI dst src));
7035   effect(KILL cr);
7036 
7037   format %{ "decl    $dst\t# int" %}
7038   opcode(0xFF, 0x01); // FF /1
7039   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7040   ins_pipe(ialu_reg);
7041 %}
7042 
7043 // XXX why does that use AddI
7044 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7045 %{
7046   predicate(UseIncDec);
7047   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7048   effect(KILL cr);
7049 
7050   ins_cost(125); // XXX
7051   format %{ "decl    $dst\t# int" %}
7052   opcode(0xFF); /* Opcode FF /1 */
7053   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7054   ins_pipe(ialu_mem_imm);
7055 %}
7056 
7057 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7058 %{
7059   match(Set dst (AddI src0 src1));
7060 
7061   ins_cost(110);
7062   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7063   opcode(0x8D); /* 0x8D /r */
7064   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7065   ins_pipe(ialu_reg_reg);
7066 %}
7067 
7068 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7069 %{
7070   match(Set dst (AddL dst src));
7071   effect(KILL cr);
7072 
7073   format %{ "addq    $dst, $src\t# long" %}
7074   opcode(0x03);
7075   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7076   ins_pipe(ialu_reg_reg);
7077 %}
7078 
7079 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7080 %{
7081   match(Set dst (AddL dst src));
7082   effect(KILL cr);
7083 
7084   format %{ "addq    $dst, $src\t# long" %}
7085   opcode(0x81, 0x00); /* /0 id */
7086   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7087   ins_pipe( ialu_reg );
7088 %}
7089 
7090 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7091 %{
7092   match(Set dst (AddL dst (LoadL src)));
7093   effect(KILL cr);
7094 
7095   ins_cost(125); // XXX
7096   format %{ "addq    $dst, $src\t# long" %}
7097   opcode(0x03);
7098   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7099   ins_pipe(ialu_reg_mem);
7100 %}
7101 
7102 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7103 %{
7104   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7105   effect(KILL cr);
7106 
7107   ins_cost(150); // XXX
7108   format %{ "addq    $dst, $src\t# long" %}
7109   opcode(0x01); /* Opcode 01 /r */
7110   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7111   ins_pipe(ialu_mem_reg);
7112 %}
7113 
7114 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7115 %{
7116   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7117   effect(KILL cr);
7118 
7119   ins_cost(125); // XXX
7120   format %{ "addq    $dst, $src\t# long" %}
7121   opcode(0x81); /* Opcode 81 /0 id */
7122   ins_encode(REX_mem_wide(dst),
7123              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7124   ins_pipe(ialu_mem_imm);
7125 %}
7126 
7127 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7128 %{
7129   predicate(UseIncDec);
7130   match(Set dst (AddL dst src));
7131   effect(KILL cr);
7132 
7133   format %{ "incq    $dst\t# long" %}
7134   opcode(0xFF, 0x00); // FF /0
7135   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7136   ins_pipe(ialu_reg);
7137 %}
7138 
7139 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7140 %{
7141   predicate(UseIncDec);
7142   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7143   effect(KILL cr);
7144 
7145   ins_cost(125); // XXX
7146   format %{ "incq    $dst\t# long" %}
7147   opcode(0xFF); /* Opcode FF /0 */
7148   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7149   ins_pipe(ialu_mem_imm);
7150 %}
7151 
7152 // XXX why does that use AddL
7153 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7154 %{
7155   predicate(UseIncDec);
7156   match(Set dst (AddL dst src));
7157   effect(KILL cr);
7158 
7159   format %{ "decq    $dst\t# long" %}
7160   opcode(0xFF, 0x01); // FF /1
7161   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7162   ins_pipe(ialu_reg);
7163 %}
7164 
7165 // XXX why does that use AddL
7166 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7167 %{
7168   predicate(UseIncDec);
7169   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7170   effect(KILL cr);
7171 
7172   ins_cost(125); // XXX
7173   format %{ "decq    $dst\t# long" %}
7174   opcode(0xFF); /* Opcode FF /1 */
7175   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7176   ins_pipe(ialu_mem_imm);
7177 %}
7178 
7179 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7180 %{
7181   match(Set dst (AddL src0 src1));
7182 
7183   ins_cost(110);
7184   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7185   opcode(0x8D); /* 0x8D /r */
7186   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7187   ins_pipe(ialu_reg_reg);
7188 %}
7189 
7190 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7191 %{
7192   match(Set dst (AddP dst src));
7193   effect(KILL cr);
7194 
7195   format %{ "addq    $dst, $src\t# ptr" %}
7196   opcode(0x03);
7197   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7198   ins_pipe(ialu_reg_reg);
7199 %}
7200 
7201 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7202 %{
7203   match(Set dst (AddP dst src));
7204   effect(KILL cr);
7205 
7206   format %{ "addq    $dst, $src\t# ptr" %}
7207   opcode(0x81, 0x00); /* /0 id */
7208   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7209   ins_pipe( ialu_reg );
7210 %}
7211 
7212 // XXX addP mem ops ????
7213 
7214 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7215 %{
7216   match(Set dst (AddP src0 src1));
7217 
7218   ins_cost(110);
7219   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7220   opcode(0x8D); /* 0x8D /r */
7221   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7222   ins_pipe(ialu_reg_reg);
7223 %}
7224 
7225 instruct checkCastPP(rRegP dst)
7226 %{
7227   match(Set dst (CheckCastPP dst));
7228 
7229   size(0);
7230   format %{ "# checkcastPP of $dst" %}
7231   ins_encode(/* empty encoding */);
7232   ins_pipe(empty);
7233 %}
7234 
7235 instruct castPP(rRegP dst)
7236 %{
7237   match(Set dst (CastPP dst));
7238 
7239   size(0);
7240   format %{ "# castPP of $dst" %}
7241   ins_encode(/* empty encoding */);
7242   ins_pipe(empty);
7243 %}
7244 
7245 instruct castII(rRegI dst)
7246 %{
7247   match(Set dst (CastII dst));
7248 
7249   size(0);
7250   format %{ "# castII of $dst" %}
7251   ins_encode(/* empty encoding */);
7252   ins_cost(0);
7253   ins_pipe(empty);
7254 %}
7255 
7256 // LoadP-locked same as a regular LoadP when used with compare-swap
7257 instruct loadPLocked(rRegP dst, memory mem)
7258 %{
7259   match(Set dst (LoadPLocked mem));
7260 
7261   ins_cost(125); // XXX
7262   format %{ "movq    $dst, $mem\t# ptr locked" %}
7263   opcode(0x8B);
7264   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7265   ins_pipe(ialu_reg_mem); // XXX
7266 %}
7267 
7268 // Conditional-store of the updated heap-top.
7269 // Used during allocation of the shared heap.
7270 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7271 
7272 instruct storePConditional(memory heap_top_ptr,
7273                            rax_RegP oldval, rRegP newval,
7274                            rFlagsReg cr)
7275 %{
7276   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7277 
7278   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7279             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7280   opcode(0x0F, 0xB1);
7281   ins_encode(lock_prefix,
7282              REX_reg_mem_wide(newval, heap_top_ptr),
7283              OpcP, OpcS,
7284              reg_mem(newval, heap_top_ptr));
7285   ins_pipe(pipe_cmpxchg);
7286 %}
7287 
7288 // Conditional-store of an int value.
7289 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7290 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7291 %{
7292   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7293   effect(KILL oldval);
7294 
7295   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7296   opcode(0x0F, 0xB1);
7297   ins_encode(lock_prefix,
7298              REX_reg_mem(newval, mem),
7299              OpcP, OpcS,
7300              reg_mem(newval, mem));
7301   ins_pipe(pipe_cmpxchg);
7302 %}
7303 
7304 // Conditional-store of a long value.
7305 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7306 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7307 %{
7308   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7309   effect(KILL oldval);
7310 
7311   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7312   opcode(0x0F, 0xB1);
7313   ins_encode(lock_prefix,
7314              REX_reg_mem_wide(newval, mem),
7315              OpcP, OpcS,
7316              reg_mem(newval, mem));
7317   ins_pipe(pipe_cmpxchg);
7318 %}
7319 
7320 
7321 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7322 instruct compareAndSwapP(rRegI res,
7323                          memory mem_ptr,
7324                          rax_RegP oldval, rRegP newval,
7325                          rFlagsReg cr)
7326 %{
7327   predicate(VM_Version::supports_cx8());
7328   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7329   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7330   effect(KILL cr, KILL oldval);
7331 
7332   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7333             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7334             "sete    $res\n\t"
7335             "movzbl  $res, $res" %}
7336   opcode(0x0F, 0xB1);
7337   ins_encode(lock_prefix,
7338              REX_reg_mem_wide(newval, mem_ptr),
7339              OpcP, OpcS,
7340              reg_mem(newval, mem_ptr),
7341              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7342              REX_reg_breg(res, res), // movzbl
7343              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7344   ins_pipe( pipe_cmpxchg );
7345 %}
7346 
7347 instruct compareAndSwapL(rRegI res,
7348                          memory mem_ptr,
7349                          rax_RegL oldval, rRegL newval,
7350                          rFlagsReg cr)
7351 %{
7352   predicate(VM_Version::supports_cx8());
7353   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7354   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7355   effect(KILL cr, KILL oldval);
7356 
7357   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7358             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7359             "sete    $res\n\t"
7360             "movzbl  $res, $res" %}
7361   opcode(0x0F, 0xB1);
7362   ins_encode(lock_prefix,
7363              REX_reg_mem_wide(newval, mem_ptr),
7364              OpcP, OpcS,
7365              reg_mem(newval, mem_ptr),
7366              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7367              REX_reg_breg(res, res), // movzbl
7368              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7369   ins_pipe( pipe_cmpxchg );
7370 %}
7371 
7372 instruct compareAndSwapI(rRegI res,
7373                          memory mem_ptr,
7374                          rax_RegI oldval, rRegI newval,
7375                          rFlagsReg cr)
7376 %{
7377   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7378   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7379   effect(KILL cr, KILL oldval);
7380 
7381   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7382             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7383             "sete    $res\n\t"
7384             "movzbl  $res, $res" %}
7385   opcode(0x0F, 0xB1);
7386   ins_encode(lock_prefix,
7387              REX_reg_mem(newval, mem_ptr),
7388              OpcP, OpcS,
7389              reg_mem(newval, mem_ptr),
7390              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7391              REX_reg_breg(res, res), // movzbl
7392              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7393   ins_pipe( pipe_cmpxchg );
7394 %}
7395 
7396 instruct compareAndSwapB(rRegI res,
7397                          memory mem_ptr,
7398                          rax_RegI oldval, rRegI newval,
7399                          rFlagsReg cr)
7400 %{
7401   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7402   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7403   effect(KILL cr, KILL oldval);
7404 
7405   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7406             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7407             "sete    $res\n\t"
7408             "movzbl  $res, $res" %}
7409   opcode(0x0F, 0xB0);
7410   ins_encode(lock_prefix,
7411              REX_breg_mem(newval, mem_ptr),
7412              OpcP, OpcS,
7413              reg_mem(newval, mem_ptr),
7414              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7415              REX_reg_breg(res, res), // movzbl
7416              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7417   ins_pipe( pipe_cmpxchg );
7418 %}
7419 
7420 instruct compareAndSwapS(rRegI res,
7421                          memory mem_ptr,
7422                          rax_RegI oldval, rRegI newval,
7423                          rFlagsReg cr)
7424 %{
7425   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7426   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7427   effect(KILL cr, KILL oldval);
7428 
7429   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7430             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7431             "sete    $res\n\t"
7432             "movzbl  $res, $res" %}
7433   opcode(0x0F, 0xB1);
7434   ins_encode(lock_prefix,
7435              SizePrefix,
7436              REX_reg_mem(newval, mem_ptr),
7437              OpcP, OpcS,
7438              reg_mem(newval, mem_ptr),
7439              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7440              REX_reg_breg(res, res), // movzbl
7441              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7442   ins_pipe( pipe_cmpxchg );
7443 %}
7444 
7445 instruct compareAndSwapN(rRegI res,
7446                           memory mem_ptr,
7447                           rax_RegN oldval, rRegN newval,
7448                           rFlagsReg cr) %{
7449   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7450   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7451   effect(KILL cr, KILL oldval);
7452 
7453   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7454             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7455             "sete    $res\n\t"
7456             "movzbl  $res, $res" %}
7457   opcode(0x0F, 0xB1);
7458   ins_encode(lock_prefix,
7459              REX_reg_mem(newval, mem_ptr),
7460              OpcP, OpcS,
7461              reg_mem(newval, mem_ptr),
7462              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7463              REX_reg_breg(res, res), // movzbl
7464              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7465   ins_pipe( pipe_cmpxchg );
7466 %}
7467 
7468 instruct compareAndExchangeB(
7469                          memory mem_ptr,
7470                          rax_RegI oldval, rRegI newval,
7471                          rFlagsReg cr)
7472 %{
7473   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7474   effect(KILL cr);
7475 
7476   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7477             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7478   opcode(0x0F, 0xB0);
7479   ins_encode(lock_prefix,
7480              REX_breg_mem(newval, mem_ptr),
7481              OpcP, OpcS,
7482              reg_mem(newval, mem_ptr) // lock cmpxchg
7483              );
7484   ins_pipe( pipe_cmpxchg );
7485 %}
7486 
7487 instruct compareAndExchangeS(
7488                          memory mem_ptr,
7489                          rax_RegI oldval, rRegI newval,
7490                          rFlagsReg cr)
7491 %{
7492   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7493   effect(KILL cr);
7494 
7495   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7496             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7497   opcode(0x0F, 0xB1);
7498   ins_encode(lock_prefix,
7499              SizePrefix,
7500              REX_reg_mem(newval, mem_ptr),
7501              OpcP, OpcS,
7502              reg_mem(newval, mem_ptr) // lock cmpxchg
7503              );
7504   ins_pipe( pipe_cmpxchg );
7505 %}
7506 
7507 instruct compareAndExchangeI(
7508                          memory mem_ptr,
7509                          rax_RegI oldval, rRegI newval,
7510                          rFlagsReg cr)
7511 %{
7512   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7513   effect(KILL cr);
7514 
7515   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7516             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7517   opcode(0x0F, 0xB1);
7518   ins_encode(lock_prefix,
7519              REX_reg_mem(newval, mem_ptr),
7520              OpcP, OpcS,
7521              reg_mem(newval, mem_ptr) // lock cmpxchg
7522              );
7523   ins_pipe( pipe_cmpxchg );
7524 %}
7525 
7526 instruct compareAndExchangeL(
7527                          memory mem_ptr,
7528                          rax_RegL oldval, rRegL newval,
7529                          rFlagsReg cr)
7530 %{
7531   predicate(VM_Version::supports_cx8());
7532   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7533   effect(KILL cr);
7534 
7535   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7536             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7537   opcode(0x0F, 0xB1);
7538   ins_encode(lock_prefix,
7539              REX_reg_mem_wide(newval, mem_ptr),
7540              OpcP, OpcS,
7541              reg_mem(newval, mem_ptr)  // lock cmpxchg
7542             );
7543   ins_pipe( pipe_cmpxchg );
7544 %}
7545 
7546 instruct compareAndExchangeN(
7547                           memory mem_ptr,
7548                           rax_RegN oldval, rRegN newval,
7549                           rFlagsReg cr) %{
7550   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7551   effect(KILL cr);
7552 
7553   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7554             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7555   opcode(0x0F, 0xB1);
7556   ins_encode(lock_prefix,
7557              REX_reg_mem(newval, mem_ptr),
7558              OpcP, OpcS,
7559              reg_mem(newval, mem_ptr)  // lock cmpxchg
7560           );
7561   ins_pipe( pipe_cmpxchg );
7562 %}
7563 
7564 instruct compareAndExchangeP(
7565                          memory mem_ptr,
7566                          rax_RegP oldval, rRegP newval,
7567                          rFlagsReg cr)
7568 %{
7569   predicate(VM_Version::supports_cx8());
7570   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7571   effect(KILL cr);
7572 
7573   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7574             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7575   opcode(0x0F, 0xB1);
7576   ins_encode(lock_prefix,
7577              REX_reg_mem_wide(newval, mem_ptr),
7578              OpcP, OpcS,
7579              reg_mem(newval, mem_ptr)  // lock cmpxchg
7580           );
7581   ins_pipe( pipe_cmpxchg );
7582 %}
7583 
7584 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7585   predicate(n->as_LoadStore()->result_not_used());
7586   match(Set dummy (GetAndAddB mem add));
7587   effect(KILL cr);
7588   format %{ "ADDB  [$mem],$add" %}
7589   ins_encode %{
7590     if (os::is_MP()) { __ lock(); }
7591     __ addb($mem$$Address, $add$$constant);
7592   %}
7593   ins_pipe( pipe_cmpxchg );
7594 %}
7595 
7596 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7597   match(Set newval (GetAndAddB mem newval));
7598   effect(KILL cr);
7599   format %{ "XADDB  [$mem],$newval" %}
7600   ins_encode %{
7601     if (os::is_MP()) { __ lock(); }
7602     __ xaddb($mem$$Address, $newval$$Register);
7603   %}
7604   ins_pipe( pipe_cmpxchg );
7605 %}
7606 
7607 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7608   predicate(n->as_LoadStore()->result_not_used());
7609   match(Set dummy (GetAndAddS mem add));
7610   effect(KILL cr);
7611   format %{ "ADDW  [$mem],$add" %}
7612   ins_encode %{
7613     if (os::is_MP()) { __ lock(); }
7614     __ addw($mem$$Address, $add$$constant);
7615   %}
7616   ins_pipe( pipe_cmpxchg );
7617 %}
7618 
7619 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7620   match(Set newval (GetAndAddS mem newval));
7621   effect(KILL cr);
7622   format %{ "XADDW  [$mem],$newval" %}
7623   ins_encode %{
7624     if (os::is_MP()) { __ lock(); }
7625     __ xaddw($mem$$Address, $newval$$Register);
7626   %}
7627   ins_pipe( pipe_cmpxchg );
7628 %}
7629 
7630 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7631   predicate(n->as_LoadStore()->result_not_used());
7632   match(Set dummy (GetAndAddI mem add));
7633   effect(KILL cr);
7634   format %{ "ADDL  [$mem],$add" %}
7635   ins_encode %{
7636     if (os::is_MP()) { __ lock(); }
7637     __ addl($mem$$Address, $add$$constant);
7638   %}
7639   ins_pipe( pipe_cmpxchg );
7640 %}
7641 
7642 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7643   match(Set newval (GetAndAddI mem newval));
7644   effect(KILL cr);
7645   format %{ "XADDL  [$mem],$newval" %}
7646   ins_encode %{
7647     if (os::is_MP()) { __ lock(); }
7648     __ xaddl($mem$$Address, $newval$$Register);
7649   %}
7650   ins_pipe( pipe_cmpxchg );
7651 %}
7652 
7653 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7654   predicate(n->as_LoadStore()->result_not_used());
7655   match(Set dummy (GetAndAddL mem add));
7656   effect(KILL cr);
7657   format %{ "ADDQ  [$mem],$add" %}
7658   ins_encode %{
7659     if (os::is_MP()) { __ lock(); }
7660     __ addq($mem$$Address, $add$$constant);
7661   %}
7662   ins_pipe( pipe_cmpxchg );
7663 %}
7664 
7665 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7666   match(Set newval (GetAndAddL mem newval));
7667   effect(KILL cr);
7668   format %{ "XADDQ  [$mem],$newval" %}
7669   ins_encode %{
7670     if (os::is_MP()) { __ lock(); }
7671     __ xaddq($mem$$Address, $newval$$Register);
7672   %}
7673   ins_pipe( pipe_cmpxchg );
7674 %}
7675 
7676 instruct xchgB( memory mem, rRegI newval) %{
7677   match(Set newval (GetAndSetB mem newval));
7678   format %{ "XCHGB  $newval,[$mem]" %}
7679   ins_encode %{
7680     __ xchgb($newval$$Register, $mem$$Address);
7681   %}
7682   ins_pipe( pipe_cmpxchg );
7683 %}
7684 
7685 instruct xchgS( memory mem, rRegI newval) %{
7686   match(Set newval (GetAndSetS mem newval));
7687   format %{ "XCHGW  $newval,[$mem]" %}
7688   ins_encode %{
7689     __ xchgw($newval$$Register, $mem$$Address);
7690   %}
7691   ins_pipe( pipe_cmpxchg );
7692 %}
7693 
7694 instruct xchgI( memory mem, rRegI newval) %{
7695   match(Set newval (GetAndSetI mem newval));
7696   format %{ "XCHGL  $newval,[$mem]" %}
7697   ins_encode %{
7698     __ xchgl($newval$$Register, $mem$$Address);
7699   %}
7700   ins_pipe( pipe_cmpxchg );
7701 %}
7702 
7703 instruct xchgL( memory mem, rRegL newval) %{
7704   match(Set newval (GetAndSetL mem newval));
7705   format %{ "XCHGL  $newval,[$mem]" %}
7706   ins_encode %{
7707     __ xchgq($newval$$Register, $mem$$Address);
7708   %}
7709   ins_pipe( pipe_cmpxchg );
7710 %}
7711 
7712 instruct xchgP( memory mem, rRegP newval) %{
7713   match(Set newval (GetAndSetP mem newval));
7714   format %{ "XCHGQ  $newval,[$mem]" %}
7715   ins_encode %{
7716     __ xchgq($newval$$Register, $mem$$Address);
7717   %}
7718   ins_pipe( pipe_cmpxchg );
7719 %}
7720 
7721 instruct xchgN( memory mem, rRegN newval) %{
7722   match(Set newval (GetAndSetN mem newval));
7723   format %{ "XCHGL  $newval,$mem]" %}
7724   ins_encode %{
7725     __ xchgl($newval$$Register, $mem$$Address);
7726   %}
7727   ins_pipe( pipe_cmpxchg );
7728 %}
7729 
7730 //----------Subtraction Instructions-------------------------------------------
7731 
7732 // Integer Subtraction Instructions
7733 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7734 %{
7735   match(Set dst (SubI dst src));
7736   effect(KILL cr);
7737 
7738   format %{ "subl    $dst, $src\t# int" %}
7739   opcode(0x2B);
7740   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7741   ins_pipe(ialu_reg_reg);
7742 %}
7743 
7744 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7745 %{
7746   match(Set dst (SubI dst src));
7747   effect(KILL cr);
7748 
7749   format %{ "subl    $dst, $src\t# int" %}
7750   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7751   ins_encode(OpcSErm(dst, src), Con8or32(src));
7752   ins_pipe(ialu_reg);
7753 %}
7754 
7755 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7756 %{
7757   match(Set dst (SubI dst (LoadI src)));
7758   effect(KILL cr);
7759 
7760   ins_cost(125);
7761   format %{ "subl    $dst, $src\t# int" %}
7762   opcode(0x2B);
7763   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7764   ins_pipe(ialu_reg_mem);
7765 %}
7766 
7767 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7768 %{
7769   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7770   effect(KILL cr);
7771 
7772   ins_cost(150);
7773   format %{ "subl    $dst, $src\t# int" %}
7774   opcode(0x29); /* Opcode 29 /r */
7775   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7776   ins_pipe(ialu_mem_reg);
7777 %}
7778 
7779 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7780 %{
7781   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7782   effect(KILL cr);
7783 
7784   ins_cost(125); // XXX
7785   format %{ "subl    $dst, $src\t# int" %}
7786   opcode(0x81); /* Opcode 81 /5 id */
7787   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7788   ins_pipe(ialu_mem_imm);
7789 %}
7790 
7791 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7792 %{
7793   match(Set dst (SubL dst src));
7794   effect(KILL cr);
7795 
7796   format %{ "subq    $dst, $src\t# long" %}
7797   opcode(0x2B);
7798   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7799   ins_pipe(ialu_reg_reg);
7800 %}
7801 
7802 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7803 %{
7804   match(Set dst (SubL dst src));
7805   effect(KILL cr);
7806 
7807   format %{ "subq    $dst, $src\t# long" %}
7808   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7809   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7810   ins_pipe(ialu_reg);
7811 %}
7812 
7813 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7814 %{
7815   match(Set dst (SubL dst (LoadL src)));
7816   effect(KILL cr);
7817 
7818   ins_cost(125);
7819   format %{ "subq    $dst, $src\t# long" %}
7820   opcode(0x2B);
7821   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7822   ins_pipe(ialu_reg_mem);
7823 %}
7824 
7825 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7826 %{
7827   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7828   effect(KILL cr);
7829 
7830   ins_cost(150);
7831   format %{ "subq    $dst, $src\t# long" %}
7832   opcode(0x29); /* Opcode 29 /r */
7833   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7834   ins_pipe(ialu_mem_reg);
7835 %}
7836 
7837 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7838 %{
7839   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7840   effect(KILL cr);
7841 
7842   ins_cost(125); // XXX
7843   format %{ "subq    $dst, $src\t# long" %}
7844   opcode(0x81); /* Opcode 81 /5 id */
7845   ins_encode(REX_mem_wide(dst),
7846              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7847   ins_pipe(ialu_mem_imm);
7848 %}
7849 
7850 // Subtract from a pointer
7851 // XXX hmpf???
7852 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7853 %{
7854   match(Set dst (AddP dst (SubI zero src)));
7855   effect(KILL cr);
7856 
7857   format %{ "subq    $dst, $src\t# ptr - int" %}
7858   opcode(0x2B);
7859   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7860   ins_pipe(ialu_reg_reg);
7861 %}
7862 
7863 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7864 %{
7865   match(Set dst (SubI zero dst));
7866   effect(KILL cr);
7867 
7868   format %{ "negl    $dst\t# int" %}
7869   opcode(0xF7, 0x03);  // Opcode F7 /3
7870   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7871   ins_pipe(ialu_reg);
7872 %}
7873 
7874 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7875 %{
7876   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7877   effect(KILL cr);
7878 
7879   format %{ "negl    $dst\t# int" %}
7880   opcode(0xF7, 0x03);  // Opcode F7 /3
7881   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7882   ins_pipe(ialu_reg);
7883 %}
7884 
7885 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7886 %{
7887   match(Set dst (SubL zero dst));
7888   effect(KILL cr);
7889 
7890   format %{ "negq    $dst\t# long" %}
7891   opcode(0xF7, 0x03);  // Opcode F7 /3
7892   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7893   ins_pipe(ialu_reg);
7894 %}
7895 
7896 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7897 %{
7898   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7899   effect(KILL cr);
7900 
7901   format %{ "negq    $dst\t# long" %}
7902   opcode(0xF7, 0x03);  // Opcode F7 /3
7903   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7904   ins_pipe(ialu_reg);
7905 %}
7906 
7907 //----------Multiplication/Division Instructions-------------------------------
7908 // Integer Multiplication Instructions
7909 // Multiply Register
7910 
7911 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7912 %{
7913   match(Set dst (MulI dst src));
7914   effect(KILL cr);
7915 
7916   ins_cost(300);
7917   format %{ "imull   $dst, $src\t# int" %}
7918   opcode(0x0F, 0xAF);
7919   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7920   ins_pipe(ialu_reg_reg_alu0);
7921 %}
7922 
7923 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7924 %{
7925   match(Set dst (MulI src imm));
7926   effect(KILL cr);
7927 
7928   ins_cost(300);
7929   format %{ "imull   $dst, $src, $imm\t# int" %}
7930   opcode(0x69); /* 69 /r id */
7931   ins_encode(REX_reg_reg(dst, src),
7932              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7933   ins_pipe(ialu_reg_reg_alu0);
7934 %}
7935 
7936 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7937 %{
7938   match(Set dst (MulI dst (LoadI src)));
7939   effect(KILL cr);
7940 
7941   ins_cost(350);
7942   format %{ "imull   $dst, $src\t# int" %}
7943   opcode(0x0F, 0xAF);
7944   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7945   ins_pipe(ialu_reg_mem_alu0);
7946 %}
7947 
7948 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7949 %{
7950   match(Set dst (MulI (LoadI src) imm));
7951   effect(KILL cr);
7952 
7953   ins_cost(300);
7954   format %{ "imull   $dst, $src, $imm\t# int" %}
7955   opcode(0x69); /* 69 /r id */
7956   ins_encode(REX_reg_mem(dst, src),
7957              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7958   ins_pipe(ialu_reg_mem_alu0);
7959 %}
7960 
7961 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7962 %{
7963   match(Set dst (MulL dst src));
7964   effect(KILL cr);
7965 
7966   ins_cost(300);
7967   format %{ "imulq   $dst, $src\t# long" %}
7968   opcode(0x0F, 0xAF);
7969   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7970   ins_pipe(ialu_reg_reg_alu0);
7971 %}
7972 
7973 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7974 %{
7975   match(Set dst (MulL src imm));
7976   effect(KILL cr);
7977 
7978   ins_cost(300);
7979   format %{ "imulq   $dst, $src, $imm\t# long" %}
7980   opcode(0x69); /* 69 /r id */
7981   ins_encode(REX_reg_reg_wide(dst, src),
7982              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7983   ins_pipe(ialu_reg_reg_alu0);
7984 %}
7985 
7986 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7987 %{
7988   match(Set dst (MulL dst (LoadL src)));
7989   effect(KILL cr);
7990 
7991   ins_cost(350);
7992   format %{ "imulq   $dst, $src\t# long" %}
7993   opcode(0x0F, 0xAF);
7994   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7995   ins_pipe(ialu_reg_mem_alu0);
7996 %}
7997 
7998 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7999 %{
8000   match(Set dst (MulL (LoadL src) imm));
8001   effect(KILL cr);
8002 
8003   ins_cost(300);
8004   format %{ "imulq   $dst, $src, $imm\t# long" %}
8005   opcode(0x69); /* 69 /r id */
8006   ins_encode(REX_reg_mem_wide(dst, src),
8007              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8008   ins_pipe(ialu_reg_mem_alu0);
8009 %}
8010 
8011 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8012 %{
8013   match(Set dst (MulHiL src rax));
8014   effect(USE_KILL rax, KILL cr);
8015 
8016   ins_cost(300);
8017   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8018   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8019   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8020   ins_pipe(ialu_reg_reg_alu0);
8021 %}
8022 
8023 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8024                    rFlagsReg cr)
8025 %{
8026   match(Set rax (DivI rax div));
8027   effect(KILL rdx, KILL cr);
8028 
8029   ins_cost(30*100+10*100); // XXX
8030   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8031             "jne,s   normal\n\t"
8032             "xorl    rdx, rdx\n\t"
8033             "cmpl    $div, -1\n\t"
8034             "je,s    done\n"
8035     "normal: cdql\n\t"
8036             "idivl   $div\n"
8037     "done:"        %}
8038   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8039   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8040   ins_pipe(ialu_reg_reg_alu0);
8041 %}
8042 
8043 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8044                    rFlagsReg cr)
8045 %{
8046   match(Set rax (DivL rax div));
8047   effect(KILL rdx, KILL cr);
8048 
8049   ins_cost(30*100+10*100); // XXX
8050   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8051             "cmpq    rax, rdx\n\t"
8052             "jne,s   normal\n\t"
8053             "xorl    rdx, rdx\n\t"
8054             "cmpq    $div, -1\n\t"
8055             "je,s    done\n"
8056     "normal: cdqq\n\t"
8057             "idivq   $div\n"
8058     "done:"        %}
8059   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8060   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8061   ins_pipe(ialu_reg_reg_alu0);
8062 %}
8063 
8064 // Integer DIVMOD with Register, both quotient and mod results
8065 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8066                              rFlagsReg cr)
8067 %{
8068   match(DivModI rax div);
8069   effect(KILL cr);
8070 
8071   ins_cost(30*100+10*100); // XXX
8072   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8073             "jne,s   normal\n\t"
8074             "xorl    rdx, rdx\n\t"
8075             "cmpl    $div, -1\n\t"
8076             "je,s    done\n"
8077     "normal: cdql\n\t"
8078             "idivl   $div\n"
8079     "done:"        %}
8080   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8081   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8082   ins_pipe(pipe_slow);
8083 %}
8084 
8085 // Long DIVMOD with Register, both quotient and mod results
8086 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8087                              rFlagsReg cr)
8088 %{
8089   match(DivModL rax div);
8090   effect(KILL cr);
8091 
8092   ins_cost(30*100+10*100); // XXX
8093   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8094             "cmpq    rax, rdx\n\t"
8095             "jne,s   normal\n\t"
8096             "xorl    rdx, rdx\n\t"
8097             "cmpq    $div, -1\n\t"
8098             "je,s    done\n"
8099     "normal: cdqq\n\t"
8100             "idivq   $div\n"
8101     "done:"        %}
8102   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8103   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8104   ins_pipe(pipe_slow);
8105 %}
8106 
8107 //----------- DivL-By-Constant-Expansions--------------------------------------
8108 // DivI cases are handled by the compiler
8109 
8110 // Magic constant, reciprocal of 10
8111 instruct loadConL_0x6666666666666667(rRegL dst)
8112 %{
8113   effect(DEF dst);
8114 
8115   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8116   ins_encode(load_immL(dst, 0x6666666666666667));
8117   ins_pipe(ialu_reg);
8118 %}
8119 
8120 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8121 %{
8122   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8123 
8124   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8125   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8126   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8127   ins_pipe(ialu_reg_reg_alu0);
8128 %}
8129 
8130 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8131 %{
8132   effect(USE_DEF dst, KILL cr);
8133 
8134   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8135   opcode(0xC1, 0x7); /* C1 /7 ib */
8136   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8137   ins_pipe(ialu_reg);
8138 %}
8139 
8140 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8141 %{
8142   effect(USE_DEF dst, KILL cr);
8143 
8144   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8145   opcode(0xC1, 0x7); /* C1 /7 ib */
8146   ins_encode(reg_opc_imm_wide(dst, 0x2));
8147   ins_pipe(ialu_reg);
8148 %}
8149 
8150 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8151 %{
8152   match(Set dst (DivL src div));
8153 
8154   ins_cost((5+8)*100);
8155   expand %{
8156     rax_RegL rax;                     // Killed temp
8157     rFlagsReg cr;                     // Killed
8158     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8159     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8160     sarL_rReg_63(src, cr);            // sarq  src, 63
8161     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8162     subL_rReg(dst, src, cr);          // subl  rdx, src
8163   %}
8164 %}
8165 
8166 //-----------------------------------------------------------------------------
8167 
8168 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8169                    rFlagsReg cr)
8170 %{
8171   match(Set rdx (ModI rax div));
8172   effect(KILL rax, KILL cr);
8173 
8174   ins_cost(300); // XXX
8175   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8176             "jne,s   normal\n\t"
8177             "xorl    rdx, rdx\n\t"
8178             "cmpl    $div, -1\n\t"
8179             "je,s    done\n"
8180     "normal: cdql\n\t"
8181             "idivl   $div\n"
8182     "done:"        %}
8183   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8184   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8185   ins_pipe(ialu_reg_reg_alu0);
8186 %}
8187 
8188 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8189                    rFlagsReg cr)
8190 %{
8191   match(Set rdx (ModL rax div));
8192   effect(KILL rax, KILL cr);
8193 
8194   ins_cost(300); // XXX
8195   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8196             "cmpq    rax, rdx\n\t"
8197             "jne,s   normal\n\t"
8198             "xorl    rdx, rdx\n\t"
8199             "cmpq    $div, -1\n\t"
8200             "je,s    done\n"
8201     "normal: cdqq\n\t"
8202             "idivq   $div\n"
8203     "done:"        %}
8204   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8205   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8206   ins_pipe(ialu_reg_reg_alu0);
8207 %}
8208 
8209 // Integer Shift Instructions
8210 // Shift Left by one
8211 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8212 %{
8213   match(Set dst (LShiftI dst shift));
8214   effect(KILL cr);
8215 
8216   format %{ "sall    $dst, $shift" %}
8217   opcode(0xD1, 0x4); /* D1 /4 */
8218   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8219   ins_pipe(ialu_reg);
8220 %}
8221 
8222 // Shift Left by one
8223 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8224 %{
8225   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8226   effect(KILL cr);
8227 
8228   format %{ "sall    $dst, $shift\t" %}
8229   opcode(0xD1, 0x4); /* D1 /4 */
8230   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8231   ins_pipe(ialu_mem_imm);
8232 %}
8233 
8234 // Shift Left by 8-bit immediate
8235 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8236 %{
8237   match(Set dst (LShiftI dst shift));
8238   effect(KILL cr);
8239 
8240   format %{ "sall    $dst, $shift" %}
8241   opcode(0xC1, 0x4); /* C1 /4 ib */
8242   ins_encode(reg_opc_imm(dst, shift));
8243   ins_pipe(ialu_reg);
8244 %}
8245 
8246 // Shift Left by 8-bit immediate
8247 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8248 %{
8249   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8250   effect(KILL cr);
8251 
8252   format %{ "sall    $dst, $shift" %}
8253   opcode(0xC1, 0x4); /* C1 /4 ib */
8254   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8255   ins_pipe(ialu_mem_imm);
8256 %}
8257 
8258 // Shift Left by variable
8259 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8260 %{
8261   match(Set dst (LShiftI dst shift));
8262   effect(KILL cr);
8263 
8264   format %{ "sall    $dst, $shift" %}
8265   opcode(0xD3, 0x4); /* D3 /4 */
8266   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8267   ins_pipe(ialu_reg_reg);
8268 %}
8269 
8270 // Shift Left by variable
8271 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8272 %{
8273   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8274   effect(KILL cr);
8275 
8276   format %{ "sall    $dst, $shift" %}
8277   opcode(0xD3, 0x4); /* D3 /4 */
8278   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8279   ins_pipe(ialu_mem_reg);
8280 %}
8281 
8282 // Arithmetic shift right by one
8283 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8284 %{
8285   match(Set dst (RShiftI dst shift));
8286   effect(KILL cr);
8287 
8288   format %{ "sarl    $dst, $shift" %}
8289   opcode(0xD1, 0x7); /* D1 /7 */
8290   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8291   ins_pipe(ialu_reg);
8292 %}
8293 
8294 // Arithmetic shift right by one
8295 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8296 %{
8297   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8298   effect(KILL cr);
8299 
8300   format %{ "sarl    $dst, $shift" %}
8301   opcode(0xD1, 0x7); /* D1 /7 */
8302   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8303   ins_pipe(ialu_mem_imm);
8304 %}
8305 
8306 // Arithmetic Shift Right by 8-bit immediate
8307 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8308 %{
8309   match(Set dst (RShiftI dst shift));
8310   effect(KILL cr);
8311 
8312   format %{ "sarl    $dst, $shift" %}
8313   opcode(0xC1, 0x7); /* C1 /7 ib */
8314   ins_encode(reg_opc_imm(dst, shift));
8315   ins_pipe(ialu_mem_imm);
8316 %}
8317 
8318 // Arithmetic Shift Right by 8-bit immediate
8319 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8320 %{
8321   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8322   effect(KILL cr);
8323 
8324   format %{ "sarl    $dst, $shift" %}
8325   opcode(0xC1, 0x7); /* C1 /7 ib */
8326   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8327   ins_pipe(ialu_mem_imm);
8328 %}
8329 
8330 // Arithmetic Shift Right by variable
8331 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8332 %{
8333   match(Set dst (RShiftI dst shift));
8334   effect(KILL cr);
8335 
8336   format %{ "sarl    $dst, $shift" %}
8337   opcode(0xD3, 0x7); /* D3 /7 */
8338   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8339   ins_pipe(ialu_reg_reg);
8340 %}
8341 
8342 // Arithmetic Shift Right by variable
8343 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8344 %{
8345   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8346   effect(KILL cr);
8347 
8348   format %{ "sarl    $dst, $shift" %}
8349   opcode(0xD3, 0x7); /* D3 /7 */
8350   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8351   ins_pipe(ialu_mem_reg);
8352 %}
8353 
8354 // Logical shift right by one
8355 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8356 %{
8357   match(Set dst (URShiftI dst shift));
8358   effect(KILL cr);
8359 
8360   format %{ "shrl    $dst, $shift" %}
8361   opcode(0xD1, 0x5); /* D1 /5 */
8362   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8363   ins_pipe(ialu_reg);
8364 %}
8365 
8366 // Logical shift right by one
8367 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8368 %{
8369   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8370   effect(KILL cr);
8371 
8372   format %{ "shrl    $dst, $shift" %}
8373   opcode(0xD1, 0x5); /* D1 /5 */
8374   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8375   ins_pipe(ialu_mem_imm);
8376 %}
8377 
8378 // Logical Shift Right by 8-bit immediate
8379 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8380 %{
8381   match(Set dst (URShiftI dst shift));
8382   effect(KILL cr);
8383 
8384   format %{ "shrl    $dst, $shift" %}
8385   opcode(0xC1, 0x5); /* C1 /5 ib */
8386   ins_encode(reg_opc_imm(dst, shift));
8387   ins_pipe(ialu_reg);
8388 %}
8389 
8390 // Logical Shift Right by 8-bit immediate
8391 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8392 %{
8393   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8394   effect(KILL cr);
8395 
8396   format %{ "shrl    $dst, $shift" %}
8397   opcode(0xC1, 0x5); /* C1 /5 ib */
8398   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8399   ins_pipe(ialu_mem_imm);
8400 %}
8401 
8402 // Logical Shift Right by variable
8403 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8404 %{
8405   match(Set dst (URShiftI dst shift));
8406   effect(KILL cr);
8407 
8408   format %{ "shrl    $dst, $shift" %}
8409   opcode(0xD3, 0x5); /* D3 /5 */
8410   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8411   ins_pipe(ialu_reg_reg);
8412 %}
8413 
8414 // Logical Shift Right by variable
8415 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8416 %{
8417   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8418   effect(KILL cr);
8419 
8420   format %{ "shrl    $dst, $shift" %}
8421   opcode(0xD3, 0x5); /* D3 /5 */
8422   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8423   ins_pipe(ialu_mem_reg);
8424 %}
8425 
8426 // Long Shift Instructions
8427 // Shift Left by one
8428 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8429 %{
8430   match(Set dst (LShiftL dst shift));
8431   effect(KILL cr);
8432 
8433   format %{ "salq    $dst, $shift" %}
8434   opcode(0xD1, 0x4); /* D1 /4 */
8435   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8436   ins_pipe(ialu_reg);
8437 %}
8438 
8439 // Shift Left by one
8440 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8441 %{
8442   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8443   effect(KILL cr);
8444 
8445   format %{ "salq    $dst, $shift" %}
8446   opcode(0xD1, 0x4); /* D1 /4 */
8447   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8448   ins_pipe(ialu_mem_imm);
8449 %}
8450 
8451 // Shift Left by 8-bit immediate
8452 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8453 %{
8454   match(Set dst (LShiftL dst shift));
8455   effect(KILL cr);
8456 
8457   format %{ "salq    $dst, $shift" %}
8458   opcode(0xC1, 0x4); /* C1 /4 ib */
8459   ins_encode(reg_opc_imm_wide(dst, shift));
8460   ins_pipe(ialu_reg);
8461 %}
8462 
8463 // Shift Left by 8-bit immediate
8464 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8465 %{
8466   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8467   effect(KILL cr);
8468 
8469   format %{ "salq    $dst, $shift" %}
8470   opcode(0xC1, 0x4); /* C1 /4 ib */
8471   ins_encode(REX_mem_wide(dst), OpcP,
8472              RM_opc_mem(secondary, dst), Con8or32(shift));
8473   ins_pipe(ialu_mem_imm);
8474 %}
8475 
8476 // Shift Left by variable
8477 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8478 %{
8479   match(Set dst (LShiftL dst shift));
8480   effect(KILL cr);
8481 
8482   format %{ "salq    $dst, $shift" %}
8483   opcode(0xD3, 0x4); /* D3 /4 */
8484   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8485   ins_pipe(ialu_reg_reg);
8486 %}
8487 
8488 // Shift Left by variable
8489 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8490 %{
8491   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8492   effect(KILL cr);
8493 
8494   format %{ "salq    $dst, $shift" %}
8495   opcode(0xD3, 0x4); /* D3 /4 */
8496   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8497   ins_pipe(ialu_mem_reg);
8498 %}
8499 
8500 // Arithmetic shift right by one
8501 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8502 %{
8503   match(Set dst (RShiftL dst shift));
8504   effect(KILL cr);
8505 
8506   format %{ "sarq    $dst, $shift" %}
8507   opcode(0xD1, 0x7); /* D1 /7 */
8508   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8509   ins_pipe(ialu_reg);
8510 %}
8511 
8512 // Arithmetic shift right by one
8513 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8514 %{
8515   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8516   effect(KILL cr);
8517 
8518   format %{ "sarq    $dst, $shift" %}
8519   opcode(0xD1, 0x7); /* D1 /7 */
8520   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8521   ins_pipe(ialu_mem_imm);
8522 %}
8523 
8524 // Arithmetic Shift Right by 8-bit immediate
8525 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8526 %{
8527   match(Set dst (RShiftL dst shift));
8528   effect(KILL cr);
8529 
8530   format %{ "sarq    $dst, $shift" %}
8531   opcode(0xC1, 0x7); /* C1 /7 ib */
8532   ins_encode(reg_opc_imm_wide(dst, shift));
8533   ins_pipe(ialu_mem_imm);
8534 %}
8535 
8536 // Arithmetic Shift Right by 8-bit immediate
8537 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8538 %{
8539   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8540   effect(KILL cr);
8541 
8542   format %{ "sarq    $dst, $shift" %}
8543   opcode(0xC1, 0x7); /* C1 /7 ib */
8544   ins_encode(REX_mem_wide(dst), OpcP,
8545              RM_opc_mem(secondary, dst), Con8or32(shift));
8546   ins_pipe(ialu_mem_imm);
8547 %}
8548 
8549 // Arithmetic Shift Right by variable
8550 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8551 %{
8552   match(Set dst (RShiftL dst shift));
8553   effect(KILL cr);
8554 
8555   format %{ "sarq    $dst, $shift" %}
8556   opcode(0xD3, 0x7); /* D3 /7 */
8557   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8558   ins_pipe(ialu_reg_reg);
8559 %}
8560 
8561 // Arithmetic Shift Right by variable
8562 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8563 %{
8564   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8565   effect(KILL cr);
8566 
8567   format %{ "sarq    $dst, $shift" %}
8568   opcode(0xD3, 0x7); /* D3 /7 */
8569   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8570   ins_pipe(ialu_mem_reg);
8571 %}
8572 
8573 // Logical shift right by one
8574 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8575 %{
8576   match(Set dst (URShiftL dst shift));
8577   effect(KILL cr);
8578 
8579   format %{ "shrq    $dst, $shift" %}
8580   opcode(0xD1, 0x5); /* D1 /5 */
8581   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8582   ins_pipe(ialu_reg);
8583 %}
8584 
8585 // Logical shift right by one
8586 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8587 %{
8588   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8589   effect(KILL cr);
8590 
8591   format %{ "shrq    $dst, $shift" %}
8592   opcode(0xD1, 0x5); /* D1 /5 */
8593   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8594   ins_pipe(ialu_mem_imm);
8595 %}
8596 
8597 // Logical Shift Right by 8-bit immediate
8598 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8599 %{
8600   match(Set dst (URShiftL dst shift));
8601   effect(KILL cr);
8602 
8603   format %{ "shrq    $dst, $shift" %}
8604   opcode(0xC1, 0x5); /* C1 /5 ib */
8605   ins_encode(reg_opc_imm_wide(dst, shift));
8606   ins_pipe(ialu_reg);
8607 %}
8608 
8609 
8610 // Logical Shift Right by 8-bit immediate
8611 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8612 %{
8613   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8614   effect(KILL cr);
8615 
8616   format %{ "shrq    $dst, $shift" %}
8617   opcode(0xC1, 0x5); /* C1 /5 ib */
8618   ins_encode(REX_mem_wide(dst), OpcP,
8619              RM_opc_mem(secondary, dst), Con8or32(shift));
8620   ins_pipe(ialu_mem_imm);
8621 %}
8622 
8623 // Logical Shift Right by variable
8624 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8625 %{
8626   match(Set dst (URShiftL dst shift));
8627   effect(KILL cr);
8628 
8629   format %{ "shrq    $dst, $shift" %}
8630   opcode(0xD3, 0x5); /* D3 /5 */
8631   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8632   ins_pipe(ialu_reg_reg);
8633 %}
8634 
8635 // Logical Shift Right by variable
8636 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8637 %{
8638   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8639   effect(KILL cr);
8640 
8641   format %{ "shrq    $dst, $shift" %}
8642   opcode(0xD3, 0x5); /* D3 /5 */
8643   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8644   ins_pipe(ialu_mem_reg);
8645 %}
8646 
8647 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8648 // This idiom is used by the compiler for the i2b bytecode.
8649 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8650 %{
8651   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8652 
8653   format %{ "movsbl  $dst, $src\t# i2b" %}
8654   opcode(0x0F, 0xBE);
8655   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8656   ins_pipe(ialu_reg_reg);
8657 %}
8658 
8659 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8660 // This idiom is used by the compiler the i2s bytecode.
8661 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8662 %{
8663   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8664 
8665   format %{ "movswl  $dst, $src\t# i2s" %}
8666   opcode(0x0F, 0xBF);
8667   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8668   ins_pipe(ialu_reg_reg);
8669 %}
8670 
8671 // ROL/ROR instructions
8672 
8673 // ROL expand
8674 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8675   effect(KILL cr, USE_DEF dst);
8676 
8677   format %{ "roll    $dst" %}
8678   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8679   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8680   ins_pipe(ialu_reg);
8681 %}
8682 
8683 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8684   effect(USE_DEF dst, USE shift, KILL cr);
8685 
8686   format %{ "roll    $dst, $shift" %}
8687   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8688   ins_encode( reg_opc_imm(dst, shift) );
8689   ins_pipe(ialu_reg);
8690 %}
8691 
8692 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8693 %{
8694   effect(USE_DEF dst, USE shift, KILL cr);
8695 
8696   format %{ "roll    $dst, $shift" %}
8697   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8698   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8699   ins_pipe(ialu_reg_reg);
8700 %}
8701 // end of ROL expand
8702 
8703 // Rotate Left by one
8704 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8705 %{
8706   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8707 
8708   expand %{
8709     rolI_rReg_imm1(dst, cr);
8710   %}
8711 %}
8712 
8713 // Rotate Left by 8-bit immediate
8714 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8715 %{
8716   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8717   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8718 
8719   expand %{
8720     rolI_rReg_imm8(dst, lshift, cr);
8721   %}
8722 %}
8723 
8724 // Rotate Left by variable
8725 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8726 %{
8727   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8728 
8729   expand %{
8730     rolI_rReg_CL(dst, shift, cr);
8731   %}
8732 %}
8733 
8734 // Rotate Left by variable
8735 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8736 %{
8737   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8738 
8739   expand %{
8740     rolI_rReg_CL(dst, shift, cr);
8741   %}
8742 %}
8743 
8744 // ROR expand
8745 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8746 %{
8747   effect(USE_DEF dst, KILL cr);
8748 
8749   format %{ "rorl    $dst" %}
8750   opcode(0xD1, 0x1); /* D1 /1 */
8751   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8752   ins_pipe(ialu_reg);
8753 %}
8754 
8755 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8756 %{
8757   effect(USE_DEF dst, USE shift, KILL cr);
8758 
8759   format %{ "rorl    $dst, $shift" %}
8760   opcode(0xC1, 0x1); /* C1 /1 ib */
8761   ins_encode(reg_opc_imm(dst, shift));
8762   ins_pipe(ialu_reg);
8763 %}
8764 
8765 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8766 %{
8767   effect(USE_DEF dst, USE shift, KILL cr);
8768 
8769   format %{ "rorl    $dst, $shift" %}
8770   opcode(0xD3, 0x1); /* D3 /1 */
8771   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8772   ins_pipe(ialu_reg_reg);
8773 %}
8774 // end of ROR expand
8775 
8776 // Rotate Right by one
8777 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8778 %{
8779   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8780 
8781   expand %{
8782     rorI_rReg_imm1(dst, cr);
8783   %}
8784 %}
8785 
8786 // Rotate Right by 8-bit immediate
8787 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8788 %{
8789   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8790   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8791 
8792   expand %{
8793     rorI_rReg_imm8(dst, rshift, cr);
8794   %}
8795 %}
8796 
8797 // Rotate Right by variable
8798 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8799 %{
8800   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8801 
8802   expand %{
8803     rorI_rReg_CL(dst, shift, cr);
8804   %}
8805 %}
8806 
8807 // Rotate Right by variable
8808 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8809 %{
8810   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8811 
8812   expand %{
8813     rorI_rReg_CL(dst, shift, cr);
8814   %}
8815 %}
8816 
8817 // for long rotate
8818 // ROL expand
8819 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8820   effect(USE_DEF dst, KILL cr);
8821 
8822   format %{ "rolq    $dst" %}
8823   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8824   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8825   ins_pipe(ialu_reg);
8826 %}
8827 
8828 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8829   effect(USE_DEF dst, USE shift, KILL cr);
8830 
8831   format %{ "rolq    $dst, $shift" %}
8832   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8833   ins_encode( reg_opc_imm_wide(dst, shift) );
8834   ins_pipe(ialu_reg);
8835 %}
8836 
8837 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8838 %{
8839   effect(USE_DEF dst, USE shift, KILL cr);
8840 
8841   format %{ "rolq    $dst, $shift" %}
8842   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8843   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8844   ins_pipe(ialu_reg_reg);
8845 %}
8846 // end of ROL expand
8847 
8848 // Rotate Left by one
8849 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8850 %{
8851   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8852 
8853   expand %{
8854     rolL_rReg_imm1(dst, cr);
8855   %}
8856 %}
8857 
8858 // Rotate Left by 8-bit immediate
8859 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8860 %{
8861   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8862   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8863 
8864   expand %{
8865     rolL_rReg_imm8(dst, lshift, cr);
8866   %}
8867 %}
8868 
8869 // Rotate Left by variable
8870 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8871 %{
8872   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8873 
8874   expand %{
8875     rolL_rReg_CL(dst, shift, cr);
8876   %}
8877 %}
8878 
8879 // Rotate Left by variable
8880 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8881 %{
8882   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8883 
8884   expand %{
8885     rolL_rReg_CL(dst, shift, cr);
8886   %}
8887 %}
8888 
8889 // ROR expand
8890 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8891 %{
8892   effect(USE_DEF dst, KILL cr);
8893 
8894   format %{ "rorq    $dst" %}
8895   opcode(0xD1, 0x1); /* D1 /1 */
8896   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8897   ins_pipe(ialu_reg);
8898 %}
8899 
8900 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8901 %{
8902   effect(USE_DEF dst, USE shift, KILL cr);
8903 
8904   format %{ "rorq    $dst, $shift" %}
8905   opcode(0xC1, 0x1); /* C1 /1 ib */
8906   ins_encode(reg_opc_imm_wide(dst, shift));
8907   ins_pipe(ialu_reg);
8908 %}
8909 
8910 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8911 %{
8912   effect(USE_DEF dst, USE shift, KILL cr);
8913 
8914   format %{ "rorq    $dst, $shift" %}
8915   opcode(0xD3, 0x1); /* D3 /1 */
8916   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8917   ins_pipe(ialu_reg_reg);
8918 %}
8919 // end of ROR expand
8920 
8921 // Rotate Right by one
8922 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8923 %{
8924   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8925 
8926   expand %{
8927     rorL_rReg_imm1(dst, cr);
8928   %}
8929 %}
8930 
8931 // Rotate Right by 8-bit immediate
8932 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8933 %{
8934   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8935   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8936 
8937   expand %{
8938     rorL_rReg_imm8(dst, rshift, cr);
8939   %}
8940 %}
8941 
8942 // Rotate Right by variable
8943 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8944 %{
8945   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8946 
8947   expand %{
8948     rorL_rReg_CL(dst, shift, cr);
8949   %}
8950 %}
8951 
8952 // Rotate Right by variable
8953 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8954 %{
8955   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8956 
8957   expand %{
8958     rorL_rReg_CL(dst, shift, cr);
8959   %}
8960 %}
8961 
8962 // Logical Instructions
8963 
8964 // Integer Logical Instructions
8965 
8966 // And Instructions
8967 // And Register with Register
8968 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8969 %{
8970   match(Set dst (AndI dst src));
8971   effect(KILL cr);
8972 
8973   format %{ "andl    $dst, $src\t# int" %}
8974   opcode(0x23);
8975   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8976   ins_pipe(ialu_reg_reg);
8977 %}
8978 
8979 // And Register with Immediate 255
8980 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8981 %{
8982   match(Set dst (AndI dst src));
8983 
8984   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8985   opcode(0x0F, 0xB6);
8986   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8987   ins_pipe(ialu_reg);
8988 %}
8989 
8990 // And Register with Immediate 255 and promote to long
8991 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8992 %{
8993   match(Set dst (ConvI2L (AndI src mask)));
8994 
8995   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8996   opcode(0x0F, 0xB6);
8997   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8998   ins_pipe(ialu_reg);
8999 %}
9000 
9001 // And Register with Immediate 65535
9002 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9003 %{
9004   match(Set dst (AndI dst src));
9005 
9006   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9007   opcode(0x0F, 0xB7);
9008   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9009   ins_pipe(ialu_reg);
9010 %}
9011 
9012 // And Register with Immediate 65535 and promote to long
9013 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9014 %{
9015   match(Set dst (ConvI2L (AndI src mask)));
9016 
9017   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9018   opcode(0x0F, 0xB7);
9019   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9020   ins_pipe(ialu_reg);
9021 %}
9022 
9023 // And Register with Immediate
9024 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9025 %{
9026   match(Set dst (AndI dst src));
9027   effect(KILL cr);
9028 
9029   format %{ "andl    $dst, $src\t# int" %}
9030   opcode(0x81, 0x04); /* Opcode 81 /4 */
9031   ins_encode(OpcSErm(dst, src), Con8or32(src));
9032   ins_pipe(ialu_reg);
9033 %}
9034 
9035 // And Register with Memory
9036 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9037 %{
9038   match(Set dst (AndI dst (LoadI src)));
9039   effect(KILL cr);
9040 
9041   ins_cost(125);
9042   format %{ "andl    $dst, $src\t# int" %}
9043   opcode(0x23);
9044   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9045   ins_pipe(ialu_reg_mem);
9046 %}
9047 
9048 // And Memory with Register
9049 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9050 %{
9051   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9052   effect(KILL cr);
9053 
9054   ins_cost(150);
9055   format %{ "andl    $dst, $src\t# int" %}
9056   opcode(0x21); /* Opcode 21 /r */
9057   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9058   ins_pipe(ialu_mem_reg);
9059 %}
9060 
9061 // And Memory with Immediate
9062 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9063 %{
9064   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9065   effect(KILL cr);
9066 
9067   ins_cost(125);
9068   format %{ "andl    $dst, $src\t# int" %}
9069   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9070   ins_encode(REX_mem(dst), OpcSE(src),
9071              RM_opc_mem(secondary, dst), Con8or32(src));
9072   ins_pipe(ialu_mem_imm);
9073 %}
9074 
9075 // BMI1 instructions
9076 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9077   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9078   predicate(UseBMI1Instructions);
9079   effect(KILL cr);
9080 
9081   ins_cost(125);
9082   format %{ "andnl  $dst, $src1, $src2" %}
9083 
9084   ins_encode %{
9085     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9086   %}
9087   ins_pipe(ialu_reg_mem);
9088 %}
9089 
9090 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9091   match(Set dst (AndI (XorI src1 minus_1) src2));
9092   predicate(UseBMI1Instructions);
9093   effect(KILL cr);
9094 
9095   format %{ "andnl  $dst, $src1, $src2" %}
9096 
9097   ins_encode %{
9098     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9099   %}
9100   ins_pipe(ialu_reg);
9101 %}
9102 
9103 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9104   match(Set dst (AndI (SubI imm_zero src) src));
9105   predicate(UseBMI1Instructions);
9106   effect(KILL cr);
9107 
9108   format %{ "blsil  $dst, $src" %}
9109 
9110   ins_encode %{
9111     __ blsil($dst$$Register, $src$$Register);
9112   %}
9113   ins_pipe(ialu_reg);
9114 %}
9115 
9116 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9117   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9118   predicate(UseBMI1Instructions);
9119   effect(KILL cr);
9120 
9121   ins_cost(125);
9122   format %{ "blsil  $dst, $src" %}
9123 
9124   ins_encode %{
9125     __ blsil($dst$$Register, $src$$Address);
9126   %}
9127   ins_pipe(ialu_reg_mem);
9128 %}
9129 
9130 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9131 %{
9132   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9133   predicate(UseBMI1Instructions);
9134   effect(KILL cr);
9135 
9136   ins_cost(125);
9137   format %{ "blsmskl $dst, $src" %}
9138 
9139   ins_encode %{
9140     __ blsmskl($dst$$Register, $src$$Address);
9141   %}
9142   ins_pipe(ialu_reg_mem);
9143 %}
9144 
9145 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9146 %{
9147   match(Set dst (XorI (AddI src minus_1) src));
9148   predicate(UseBMI1Instructions);
9149   effect(KILL cr);
9150 
9151   format %{ "blsmskl $dst, $src" %}
9152 
9153   ins_encode %{
9154     __ blsmskl($dst$$Register, $src$$Register);
9155   %}
9156 
9157   ins_pipe(ialu_reg);
9158 %}
9159 
9160 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9161 %{
9162   match(Set dst (AndI (AddI src minus_1) src) );
9163   predicate(UseBMI1Instructions);
9164   effect(KILL cr);
9165 
9166   format %{ "blsrl  $dst, $src" %}
9167 
9168   ins_encode %{
9169     __ blsrl($dst$$Register, $src$$Register);
9170   %}
9171 
9172   ins_pipe(ialu_reg_mem);
9173 %}
9174 
9175 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9176 %{
9177   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9178   predicate(UseBMI1Instructions);
9179   effect(KILL cr);
9180 
9181   ins_cost(125);
9182   format %{ "blsrl  $dst, $src" %}
9183 
9184   ins_encode %{
9185     __ blsrl($dst$$Register, $src$$Address);
9186   %}
9187 
9188   ins_pipe(ialu_reg);
9189 %}
9190 
9191 // Or Instructions
9192 // Or Register with Register
9193 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9194 %{
9195   match(Set dst (OrI dst src));
9196   effect(KILL cr);
9197 
9198   format %{ "orl     $dst, $src\t# int" %}
9199   opcode(0x0B);
9200   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9201   ins_pipe(ialu_reg_reg);
9202 %}
9203 
9204 // Or Register with Immediate
9205 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9206 %{
9207   match(Set dst (OrI dst src));
9208   effect(KILL cr);
9209 
9210   format %{ "orl     $dst, $src\t# int" %}
9211   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9212   ins_encode(OpcSErm(dst, src), Con8or32(src));
9213   ins_pipe(ialu_reg);
9214 %}
9215 
9216 // Or Register with Memory
9217 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9218 %{
9219   match(Set dst (OrI dst (LoadI src)));
9220   effect(KILL cr);
9221 
9222   ins_cost(125);
9223   format %{ "orl     $dst, $src\t# int" %}
9224   opcode(0x0B);
9225   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9226   ins_pipe(ialu_reg_mem);
9227 %}
9228 
9229 // Or Memory with Register
9230 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9231 %{
9232   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9233   effect(KILL cr);
9234 
9235   ins_cost(150);
9236   format %{ "orl     $dst, $src\t# int" %}
9237   opcode(0x09); /* Opcode 09 /r */
9238   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9239   ins_pipe(ialu_mem_reg);
9240 %}
9241 
9242 // Or Memory with Immediate
9243 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9244 %{
9245   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9246   effect(KILL cr);
9247 
9248   ins_cost(125);
9249   format %{ "orl     $dst, $src\t# int" %}
9250   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9251   ins_encode(REX_mem(dst), OpcSE(src),
9252              RM_opc_mem(secondary, dst), Con8or32(src));
9253   ins_pipe(ialu_mem_imm);
9254 %}
9255 
9256 // Xor Instructions
9257 // Xor Register with Register
9258 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9259 %{
9260   match(Set dst (XorI dst src));
9261   effect(KILL cr);
9262 
9263   format %{ "xorl    $dst, $src\t# int" %}
9264   opcode(0x33);
9265   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9266   ins_pipe(ialu_reg_reg);
9267 %}
9268 
9269 // Xor Register with Immediate -1
9270 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9271   match(Set dst (XorI dst imm));
9272 
9273   format %{ "not    $dst" %}
9274   ins_encode %{
9275      __ notl($dst$$Register);
9276   %}
9277   ins_pipe(ialu_reg);
9278 %}
9279 
9280 // Xor Register with Immediate
9281 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9282 %{
9283   match(Set dst (XorI dst src));
9284   effect(KILL cr);
9285 
9286   format %{ "xorl    $dst, $src\t# int" %}
9287   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9288   ins_encode(OpcSErm(dst, src), Con8or32(src));
9289   ins_pipe(ialu_reg);
9290 %}
9291 
9292 // Xor Register with Memory
9293 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9294 %{
9295   match(Set dst (XorI dst (LoadI src)));
9296   effect(KILL cr);
9297 
9298   ins_cost(125);
9299   format %{ "xorl    $dst, $src\t# int" %}
9300   opcode(0x33);
9301   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9302   ins_pipe(ialu_reg_mem);
9303 %}
9304 
9305 // Xor Memory with Register
9306 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9307 %{
9308   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9309   effect(KILL cr);
9310 
9311   ins_cost(150);
9312   format %{ "xorl    $dst, $src\t# int" %}
9313   opcode(0x31); /* Opcode 31 /r */
9314   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9315   ins_pipe(ialu_mem_reg);
9316 %}
9317 
9318 // Xor Memory with Immediate
9319 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9320 %{
9321   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9322   effect(KILL cr);
9323 
9324   ins_cost(125);
9325   format %{ "xorl    $dst, $src\t# int" %}
9326   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9327   ins_encode(REX_mem(dst), OpcSE(src),
9328              RM_opc_mem(secondary, dst), Con8or32(src));
9329   ins_pipe(ialu_mem_imm);
9330 %}
9331 
9332 
9333 // Long Logical Instructions
9334 
9335 // And Instructions
9336 // And Register with Register
9337 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9338 %{
9339   match(Set dst (AndL dst src));
9340   effect(KILL cr);
9341 
9342   format %{ "andq    $dst, $src\t# long" %}
9343   opcode(0x23);
9344   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9345   ins_pipe(ialu_reg_reg);
9346 %}
9347 
9348 // And Register with Immediate 255
9349 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9350 %{
9351   match(Set dst (AndL dst src));
9352 
9353   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9354   opcode(0x0F, 0xB6);
9355   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9356   ins_pipe(ialu_reg);
9357 %}
9358 
9359 // And Register with Immediate 65535
9360 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9361 %{
9362   match(Set dst (AndL dst src));
9363 
9364   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9365   opcode(0x0F, 0xB7);
9366   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9367   ins_pipe(ialu_reg);
9368 %}
9369 
9370 // And Register with Immediate
9371 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9372 %{
9373   match(Set dst (AndL dst src));
9374   effect(KILL cr);
9375 
9376   format %{ "andq    $dst, $src\t# long" %}
9377   opcode(0x81, 0x04); /* Opcode 81 /4 */
9378   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9379   ins_pipe(ialu_reg);
9380 %}
9381 
9382 // And Register with Memory
9383 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9384 %{
9385   match(Set dst (AndL dst (LoadL src)));
9386   effect(KILL cr);
9387 
9388   ins_cost(125);
9389   format %{ "andq    $dst, $src\t# long" %}
9390   opcode(0x23);
9391   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9392   ins_pipe(ialu_reg_mem);
9393 %}
9394 
9395 // And Memory with Register
9396 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9397 %{
9398   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9399   effect(KILL cr);
9400 
9401   ins_cost(150);
9402   format %{ "andq    $dst, $src\t# long" %}
9403   opcode(0x21); /* Opcode 21 /r */
9404   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9405   ins_pipe(ialu_mem_reg);
9406 %}
9407 
9408 // And Memory with Immediate
9409 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9410 %{
9411   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9412   effect(KILL cr);
9413 
9414   ins_cost(125);
9415   format %{ "andq    $dst, $src\t# long" %}
9416   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9417   ins_encode(REX_mem_wide(dst), OpcSE(src),
9418              RM_opc_mem(secondary, dst), Con8or32(src));
9419   ins_pipe(ialu_mem_imm);
9420 %}
9421 
9422 // BMI1 instructions
9423 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9424   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9425   predicate(UseBMI1Instructions);
9426   effect(KILL cr);
9427 
9428   ins_cost(125);
9429   format %{ "andnq  $dst, $src1, $src2" %}
9430 
9431   ins_encode %{
9432     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9433   %}
9434   ins_pipe(ialu_reg_mem);
9435 %}
9436 
9437 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9438   match(Set dst (AndL (XorL src1 minus_1) src2));
9439   predicate(UseBMI1Instructions);
9440   effect(KILL cr);
9441 
9442   format %{ "andnq  $dst, $src1, $src2" %}
9443 
9444   ins_encode %{
9445   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9446   %}
9447   ins_pipe(ialu_reg_mem);
9448 %}
9449 
9450 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9451   match(Set dst (AndL (SubL imm_zero src) src));
9452   predicate(UseBMI1Instructions);
9453   effect(KILL cr);
9454 
9455   format %{ "blsiq  $dst, $src" %}
9456 
9457   ins_encode %{
9458     __ blsiq($dst$$Register, $src$$Register);
9459   %}
9460   ins_pipe(ialu_reg);
9461 %}
9462 
9463 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9464   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9465   predicate(UseBMI1Instructions);
9466   effect(KILL cr);
9467 
9468   ins_cost(125);
9469   format %{ "blsiq  $dst, $src" %}
9470 
9471   ins_encode %{
9472     __ blsiq($dst$$Register, $src$$Address);
9473   %}
9474   ins_pipe(ialu_reg_mem);
9475 %}
9476 
9477 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9478 %{
9479   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9480   predicate(UseBMI1Instructions);
9481   effect(KILL cr);
9482 
9483   ins_cost(125);
9484   format %{ "blsmskq $dst, $src" %}
9485 
9486   ins_encode %{
9487     __ blsmskq($dst$$Register, $src$$Address);
9488   %}
9489   ins_pipe(ialu_reg_mem);
9490 %}
9491 
9492 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9493 %{
9494   match(Set dst (XorL (AddL src minus_1) src));
9495   predicate(UseBMI1Instructions);
9496   effect(KILL cr);
9497 
9498   format %{ "blsmskq $dst, $src" %}
9499 
9500   ins_encode %{
9501     __ blsmskq($dst$$Register, $src$$Register);
9502   %}
9503 
9504   ins_pipe(ialu_reg);
9505 %}
9506 
9507 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9508 %{
9509   match(Set dst (AndL (AddL src minus_1) src) );
9510   predicate(UseBMI1Instructions);
9511   effect(KILL cr);
9512 
9513   format %{ "blsrq  $dst, $src" %}
9514 
9515   ins_encode %{
9516     __ blsrq($dst$$Register, $src$$Register);
9517   %}
9518 
9519   ins_pipe(ialu_reg);
9520 %}
9521 
9522 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9523 %{
9524   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9525   predicate(UseBMI1Instructions);
9526   effect(KILL cr);
9527 
9528   ins_cost(125);
9529   format %{ "blsrq  $dst, $src" %}
9530 
9531   ins_encode %{
9532     __ blsrq($dst$$Register, $src$$Address);
9533   %}
9534 
9535   ins_pipe(ialu_reg);
9536 %}
9537 
9538 // Or Instructions
9539 // Or Register with Register
9540 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9541 %{
9542   match(Set dst (OrL dst src));
9543   effect(KILL cr);
9544 
9545   format %{ "orq     $dst, $src\t# long" %}
9546   opcode(0x0B);
9547   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9548   ins_pipe(ialu_reg_reg);
9549 %}
9550 
9551 // Use any_RegP to match R15 (TLS register) without spilling.
9552 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9553   match(Set dst (OrL dst (CastP2X src)));
9554   effect(KILL cr);
9555 
9556   format %{ "orq     $dst, $src\t# long" %}
9557   opcode(0x0B);
9558   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9559   ins_pipe(ialu_reg_reg);
9560 %}
9561 
9562 
9563 // Or Register with Immediate
9564 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9565 %{
9566   match(Set dst (OrL dst src));
9567   effect(KILL cr);
9568 
9569   format %{ "orq     $dst, $src\t# long" %}
9570   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9571   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9572   ins_pipe(ialu_reg);
9573 %}
9574 
9575 // Or Register with Memory
9576 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9577 %{
9578   match(Set dst (OrL dst (LoadL src)));
9579   effect(KILL cr);
9580 
9581   ins_cost(125);
9582   format %{ "orq     $dst, $src\t# long" %}
9583   opcode(0x0B);
9584   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9585   ins_pipe(ialu_reg_mem);
9586 %}
9587 
9588 // Or Memory with Register
9589 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9590 %{
9591   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9592   effect(KILL cr);
9593 
9594   ins_cost(150);
9595   format %{ "orq     $dst, $src\t# long" %}
9596   opcode(0x09); /* Opcode 09 /r */
9597   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9598   ins_pipe(ialu_mem_reg);
9599 %}
9600 
9601 // Or Memory with Immediate
9602 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9603 %{
9604   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9605   effect(KILL cr);
9606 
9607   ins_cost(125);
9608   format %{ "orq     $dst, $src\t# long" %}
9609   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9610   ins_encode(REX_mem_wide(dst), OpcSE(src),
9611              RM_opc_mem(secondary, dst), Con8or32(src));
9612   ins_pipe(ialu_mem_imm);
9613 %}
9614 
9615 // Xor Instructions
9616 // Xor Register with Register
9617 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9618 %{
9619   match(Set dst (XorL dst src));
9620   effect(KILL cr);
9621 
9622   format %{ "xorq    $dst, $src\t# long" %}
9623   opcode(0x33);
9624   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9625   ins_pipe(ialu_reg_reg);
9626 %}
9627 
9628 // Xor Register with Immediate -1
9629 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9630   match(Set dst (XorL dst imm));
9631 
9632   format %{ "notq   $dst" %}
9633   ins_encode %{
9634      __ notq($dst$$Register);
9635   %}
9636   ins_pipe(ialu_reg);
9637 %}
9638 
9639 // Xor Register with Immediate
9640 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9641 %{
9642   match(Set dst (XorL dst src));
9643   effect(KILL cr);
9644 
9645   format %{ "xorq    $dst, $src\t# long" %}
9646   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9647   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9648   ins_pipe(ialu_reg);
9649 %}
9650 
9651 // Xor Register with Memory
9652 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9653 %{
9654   match(Set dst (XorL dst (LoadL src)));
9655   effect(KILL cr);
9656 
9657   ins_cost(125);
9658   format %{ "xorq    $dst, $src\t# long" %}
9659   opcode(0x33);
9660   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9661   ins_pipe(ialu_reg_mem);
9662 %}
9663 
9664 // Xor Memory with Register
9665 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9666 %{
9667   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9668   effect(KILL cr);
9669 
9670   ins_cost(150);
9671   format %{ "xorq    $dst, $src\t# long" %}
9672   opcode(0x31); /* Opcode 31 /r */
9673   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9674   ins_pipe(ialu_mem_reg);
9675 %}
9676 
9677 // Xor Memory with Immediate
9678 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9679 %{
9680   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9681   effect(KILL cr);
9682 
9683   ins_cost(125);
9684   format %{ "xorq    $dst, $src\t# long" %}
9685   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9686   ins_encode(REX_mem_wide(dst), OpcSE(src),
9687              RM_opc_mem(secondary, dst), Con8or32(src));
9688   ins_pipe(ialu_mem_imm);
9689 %}
9690 
9691 // Convert Int to Boolean
9692 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9693 %{
9694   match(Set dst (Conv2B src));
9695   effect(KILL cr);
9696 
9697   format %{ "testl   $src, $src\t# ci2b\n\t"
9698             "setnz   $dst\n\t"
9699             "movzbl  $dst, $dst" %}
9700   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9701              setNZ_reg(dst),
9702              REX_reg_breg(dst, dst), // movzbl
9703              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9704   ins_pipe(pipe_slow); // XXX
9705 %}
9706 
9707 // Convert Pointer to Boolean
9708 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9709 %{
9710   match(Set dst (Conv2B src));
9711   effect(KILL cr);
9712 
9713   format %{ "testq   $src, $src\t# cp2b\n\t"
9714             "setnz   $dst\n\t"
9715             "movzbl  $dst, $dst" %}
9716   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9717              setNZ_reg(dst),
9718              REX_reg_breg(dst, dst), // movzbl
9719              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9720   ins_pipe(pipe_slow); // XXX
9721 %}
9722 
9723 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9724 %{
9725   match(Set dst (CmpLTMask p q));
9726   effect(KILL cr);
9727 
9728   ins_cost(400);
9729   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9730             "setlt   $dst\n\t"
9731             "movzbl  $dst, $dst\n\t"
9732             "negl    $dst" %}
9733   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9734              setLT_reg(dst),
9735              REX_reg_breg(dst, dst), // movzbl
9736              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9737              neg_reg(dst));
9738   ins_pipe(pipe_slow);
9739 %}
9740 
9741 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9742 %{
9743   match(Set dst (CmpLTMask dst zero));
9744   effect(KILL cr);
9745 
9746   ins_cost(100);
9747   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9748   ins_encode %{
9749   __ sarl($dst$$Register, 31);
9750   %}
9751   ins_pipe(ialu_reg);
9752 %}
9753 
9754 /* Better to save a register than avoid a branch */
9755 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9756 %{
9757   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9758   effect(KILL cr);
9759   ins_cost(300);
9760   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9761             "jge    done\n\t"
9762             "addl   $p,$y\n"
9763             "done:  " %}
9764   ins_encode %{
9765     Register Rp = $p$$Register;
9766     Register Rq = $q$$Register;
9767     Register Ry = $y$$Register;
9768     Label done;
9769     __ subl(Rp, Rq);
9770     __ jccb(Assembler::greaterEqual, done);
9771     __ addl(Rp, Ry);
9772     __ bind(done);
9773   %}
9774   ins_pipe(pipe_cmplt);
9775 %}
9776 
9777 /* Better to save a register than avoid a branch */
9778 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9779 %{
9780   match(Set y (AndI (CmpLTMask p q) y));
9781   effect(KILL cr);
9782 
9783   ins_cost(300);
9784 
9785   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9786             "jlt      done\n\t"
9787             "xorl     $y, $y\n"
9788             "done:  " %}
9789   ins_encode %{
9790     Register Rp = $p$$Register;
9791     Register Rq = $q$$Register;
9792     Register Ry = $y$$Register;
9793     Label done;
9794     __ cmpl(Rp, Rq);
9795     __ jccb(Assembler::less, done);
9796     __ xorl(Ry, Ry);
9797     __ bind(done);
9798   %}
9799   ins_pipe(pipe_cmplt);
9800 %}
9801 
9802 
9803 //---------- FP Instructions------------------------------------------------
9804 
9805 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9806 %{
9807   match(Set cr (CmpF src1 src2));
9808 
9809   ins_cost(145);
9810   format %{ "ucomiss $src1, $src2\n\t"
9811             "jnp,s   exit\n\t"
9812             "pushfq\t# saw NaN, set CF\n\t"
9813             "andq    [rsp], #0xffffff2b\n\t"
9814             "popfq\n"
9815     "exit:" %}
9816   ins_encode %{
9817     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9818     emit_cmpfp_fixup(_masm);
9819   %}
9820   ins_pipe(pipe_slow);
9821 %}
9822 
9823 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9824   match(Set cr (CmpF src1 src2));
9825 
9826   ins_cost(100);
9827   format %{ "ucomiss $src1, $src2" %}
9828   ins_encode %{
9829     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9830   %}
9831   ins_pipe(pipe_slow);
9832 %}
9833 
9834 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9835 %{
9836   match(Set cr (CmpF src1 (LoadF src2)));
9837 
9838   ins_cost(145);
9839   format %{ "ucomiss $src1, $src2\n\t"
9840             "jnp,s   exit\n\t"
9841             "pushfq\t# saw NaN, set CF\n\t"
9842             "andq    [rsp], #0xffffff2b\n\t"
9843             "popfq\n"
9844     "exit:" %}
9845   ins_encode %{
9846     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9847     emit_cmpfp_fixup(_masm);
9848   %}
9849   ins_pipe(pipe_slow);
9850 %}
9851 
9852 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9853   match(Set cr (CmpF src1 (LoadF src2)));
9854 
9855   ins_cost(100);
9856   format %{ "ucomiss $src1, $src2" %}
9857   ins_encode %{
9858     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9859   %}
9860   ins_pipe(pipe_slow);
9861 %}
9862 
9863 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9864   match(Set cr (CmpF src con));
9865 
9866   ins_cost(145);
9867   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9868             "jnp,s   exit\n\t"
9869             "pushfq\t# saw NaN, set CF\n\t"
9870             "andq    [rsp], #0xffffff2b\n\t"
9871             "popfq\n"
9872     "exit:" %}
9873   ins_encode %{
9874     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9875     emit_cmpfp_fixup(_masm);
9876   %}
9877   ins_pipe(pipe_slow);
9878 %}
9879 
9880 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9881   match(Set cr (CmpF src con));
9882   ins_cost(100);
9883   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9884   ins_encode %{
9885     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9886   %}
9887   ins_pipe(pipe_slow);
9888 %}
9889 
9890 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9891 %{
9892   match(Set cr (CmpD src1 src2));
9893 
9894   ins_cost(145);
9895   format %{ "ucomisd $src1, $src2\n\t"
9896             "jnp,s   exit\n\t"
9897             "pushfq\t# saw NaN, set CF\n\t"
9898             "andq    [rsp], #0xffffff2b\n\t"
9899             "popfq\n"
9900     "exit:" %}
9901   ins_encode %{
9902     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9903     emit_cmpfp_fixup(_masm);
9904   %}
9905   ins_pipe(pipe_slow);
9906 %}
9907 
9908 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9909   match(Set cr (CmpD src1 src2));
9910 
9911   ins_cost(100);
9912   format %{ "ucomisd $src1, $src2 test" %}
9913   ins_encode %{
9914     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9915   %}
9916   ins_pipe(pipe_slow);
9917 %}
9918 
9919 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9920 %{
9921   match(Set cr (CmpD src1 (LoadD src2)));
9922 
9923   ins_cost(145);
9924   format %{ "ucomisd $src1, $src2\n\t"
9925             "jnp,s   exit\n\t"
9926             "pushfq\t# saw NaN, set CF\n\t"
9927             "andq    [rsp], #0xffffff2b\n\t"
9928             "popfq\n"
9929     "exit:" %}
9930   ins_encode %{
9931     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9932     emit_cmpfp_fixup(_masm);
9933   %}
9934   ins_pipe(pipe_slow);
9935 %}
9936 
9937 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9938   match(Set cr (CmpD src1 (LoadD src2)));
9939 
9940   ins_cost(100);
9941   format %{ "ucomisd $src1, $src2" %}
9942   ins_encode %{
9943     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9944   %}
9945   ins_pipe(pipe_slow);
9946 %}
9947 
9948 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9949   match(Set cr (CmpD src con));
9950 
9951   ins_cost(145);
9952   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9953             "jnp,s   exit\n\t"
9954             "pushfq\t# saw NaN, set CF\n\t"
9955             "andq    [rsp], #0xffffff2b\n\t"
9956             "popfq\n"
9957     "exit:" %}
9958   ins_encode %{
9959     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9960     emit_cmpfp_fixup(_masm);
9961   %}
9962   ins_pipe(pipe_slow);
9963 %}
9964 
9965 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9966   match(Set cr (CmpD src con));
9967   ins_cost(100);
9968   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9969   ins_encode %{
9970     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9971   %}
9972   ins_pipe(pipe_slow);
9973 %}
9974 
9975 // Compare into -1,0,1
9976 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9977 %{
9978   match(Set dst (CmpF3 src1 src2));
9979   effect(KILL cr);
9980 
9981   ins_cost(275);
9982   format %{ "ucomiss $src1, $src2\n\t"
9983             "movl    $dst, #-1\n\t"
9984             "jp,s    done\n\t"
9985             "jb,s    done\n\t"
9986             "setne   $dst\n\t"
9987             "movzbl  $dst, $dst\n"
9988     "done:" %}
9989   ins_encode %{
9990     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9991     emit_cmpfp3(_masm, $dst$$Register);
9992   %}
9993   ins_pipe(pipe_slow);
9994 %}
9995 
9996 // Compare into -1,0,1
9997 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9998 %{
9999   match(Set dst (CmpF3 src1 (LoadF src2)));
10000   effect(KILL cr);
10001 
10002   ins_cost(275);
10003   format %{ "ucomiss $src1, $src2\n\t"
10004             "movl    $dst, #-1\n\t"
10005             "jp,s    done\n\t"
10006             "jb,s    done\n\t"
10007             "setne   $dst\n\t"
10008             "movzbl  $dst, $dst\n"
10009     "done:" %}
10010   ins_encode %{
10011     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10012     emit_cmpfp3(_masm, $dst$$Register);
10013   %}
10014   ins_pipe(pipe_slow);
10015 %}
10016 
10017 // Compare into -1,0,1
10018 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10019   match(Set dst (CmpF3 src con));
10020   effect(KILL cr);
10021 
10022   ins_cost(275);
10023   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10024             "movl    $dst, #-1\n\t"
10025             "jp,s    done\n\t"
10026             "jb,s    done\n\t"
10027             "setne   $dst\n\t"
10028             "movzbl  $dst, $dst\n"
10029     "done:" %}
10030   ins_encode %{
10031     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10032     emit_cmpfp3(_masm, $dst$$Register);
10033   %}
10034   ins_pipe(pipe_slow);
10035 %}
10036 
10037 // Compare into -1,0,1
10038 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10039 %{
10040   match(Set dst (CmpD3 src1 src2));
10041   effect(KILL cr);
10042 
10043   ins_cost(275);
10044   format %{ "ucomisd $src1, $src2\n\t"
10045             "movl    $dst, #-1\n\t"
10046             "jp,s    done\n\t"
10047             "jb,s    done\n\t"
10048             "setne   $dst\n\t"
10049             "movzbl  $dst, $dst\n"
10050     "done:" %}
10051   ins_encode %{
10052     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10053     emit_cmpfp3(_masm, $dst$$Register);
10054   %}
10055   ins_pipe(pipe_slow);
10056 %}
10057 
10058 // Compare into -1,0,1
10059 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10060 %{
10061   match(Set dst (CmpD3 src1 (LoadD src2)));
10062   effect(KILL cr);
10063 
10064   ins_cost(275);
10065   format %{ "ucomisd $src1, $src2\n\t"
10066             "movl    $dst, #-1\n\t"
10067             "jp,s    done\n\t"
10068             "jb,s    done\n\t"
10069             "setne   $dst\n\t"
10070             "movzbl  $dst, $dst\n"
10071     "done:" %}
10072   ins_encode %{
10073     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10074     emit_cmpfp3(_masm, $dst$$Register);
10075   %}
10076   ins_pipe(pipe_slow);
10077 %}
10078 
10079 // Compare into -1,0,1
10080 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10081   match(Set dst (CmpD3 src con));
10082   effect(KILL cr);
10083 
10084   ins_cost(275);
10085   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10086             "movl    $dst, #-1\n\t"
10087             "jp,s    done\n\t"
10088             "jb,s    done\n\t"
10089             "setne   $dst\n\t"
10090             "movzbl  $dst, $dst\n"
10091     "done:" %}
10092   ins_encode %{
10093     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10094     emit_cmpfp3(_masm, $dst$$Register);
10095   %}
10096   ins_pipe(pipe_slow);
10097 %}
10098 
10099 //----------Arithmetic Conversion Instructions---------------------------------
10100 
10101 instruct roundFloat_nop(regF dst)
10102 %{
10103   match(Set dst (RoundFloat dst));
10104 
10105   ins_cost(0);
10106   ins_encode();
10107   ins_pipe(empty);
10108 %}
10109 
10110 instruct roundDouble_nop(regD dst)
10111 %{
10112   match(Set dst (RoundDouble dst));
10113 
10114   ins_cost(0);
10115   ins_encode();
10116   ins_pipe(empty);
10117 %}
10118 
10119 instruct convF2D_reg_reg(regD dst, regF src)
10120 %{
10121   match(Set dst (ConvF2D src));
10122 
10123   format %{ "cvtss2sd $dst, $src" %}
10124   ins_encode %{
10125     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10126   %}
10127   ins_pipe(pipe_slow); // XXX
10128 %}
10129 
10130 instruct convF2D_reg_mem(regD dst, memory src)
10131 %{
10132   match(Set dst (ConvF2D (LoadF src)));
10133 
10134   format %{ "cvtss2sd $dst, $src" %}
10135   ins_encode %{
10136     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10137   %}
10138   ins_pipe(pipe_slow); // XXX
10139 %}
10140 
10141 instruct convD2F_reg_reg(regF dst, regD src)
10142 %{
10143   match(Set dst (ConvD2F src));
10144 
10145   format %{ "cvtsd2ss $dst, $src" %}
10146   ins_encode %{
10147     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10148   %}
10149   ins_pipe(pipe_slow); // XXX
10150 %}
10151 
10152 instruct convD2F_reg_mem(regF dst, memory src)
10153 %{
10154   match(Set dst (ConvD2F (LoadD src)));
10155 
10156   format %{ "cvtsd2ss $dst, $src" %}
10157   ins_encode %{
10158     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10159   %}
10160   ins_pipe(pipe_slow); // XXX
10161 %}
10162 
10163 // XXX do mem variants
10164 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10165 %{
10166   match(Set dst (ConvF2I src));
10167   effect(KILL cr);
10168 
10169   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10170             "cmpl    $dst, #0x80000000\n\t"
10171             "jne,s   done\n\t"
10172             "subq    rsp, #8\n\t"
10173             "movss   [rsp], $src\n\t"
10174             "call    f2i_fixup\n\t"
10175             "popq    $dst\n"
10176     "done:   "%}
10177   ins_encode %{
10178     Label done;
10179     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10180     __ cmpl($dst$$Register, 0x80000000);
10181     __ jccb(Assembler::notEqual, done);
10182     __ subptr(rsp, 8);
10183     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10184     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10185     __ pop($dst$$Register);
10186     __ bind(done);
10187   %}
10188   ins_pipe(pipe_slow);
10189 %}
10190 
10191 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10192 %{
10193   match(Set dst (ConvF2L src));
10194   effect(KILL cr);
10195 
10196   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10197             "cmpq    $dst, [0x8000000000000000]\n\t"
10198             "jne,s   done\n\t"
10199             "subq    rsp, #8\n\t"
10200             "movss   [rsp], $src\n\t"
10201             "call    f2l_fixup\n\t"
10202             "popq    $dst\n"
10203     "done:   "%}
10204   ins_encode %{
10205     Label done;
10206     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10207     __ cmp64($dst$$Register,
10208              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10209     __ jccb(Assembler::notEqual, done);
10210     __ subptr(rsp, 8);
10211     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10212     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10213     __ pop($dst$$Register);
10214     __ bind(done);
10215   %}
10216   ins_pipe(pipe_slow);
10217 %}
10218 
10219 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10220 %{
10221   match(Set dst (ConvD2I src));
10222   effect(KILL cr);
10223 
10224   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10225             "cmpl    $dst, #0x80000000\n\t"
10226             "jne,s   done\n\t"
10227             "subq    rsp, #8\n\t"
10228             "movsd   [rsp], $src\n\t"
10229             "call    d2i_fixup\n\t"
10230             "popq    $dst\n"
10231     "done:   "%}
10232   ins_encode %{
10233     Label done;
10234     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10235     __ cmpl($dst$$Register, 0x80000000);
10236     __ jccb(Assembler::notEqual, done);
10237     __ subptr(rsp, 8);
10238     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10239     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10240     __ pop($dst$$Register);
10241     __ bind(done);
10242   %}
10243   ins_pipe(pipe_slow);
10244 %}
10245 
10246 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10247 %{
10248   match(Set dst (ConvD2L src));
10249   effect(KILL cr);
10250 
10251   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10252             "cmpq    $dst, [0x8000000000000000]\n\t"
10253             "jne,s   done\n\t"
10254             "subq    rsp, #8\n\t"
10255             "movsd   [rsp], $src\n\t"
10256             "call    d2l_fixup\n\t"
10257             "popq    $dst\n"
10258     "done:   "%}
10259   ins_encode %{
10260     Label done;
10261     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10262     __ cmp64($dst$$Register,
10263              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10264     __ jccb(Assembler::notEqual, done);
10265     __ subptr(rsp, 8);
10266     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10267     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10268     __ pop($dst$$Register);
10269     __ bind(done);
10270   %}
10271   ins_pipe(pipe_slow);
10272 %}
10273 
10274 instruct convI2F_reg_reg(regF dst, rRegI src)
10275 %{
10276   predicate(!UseXmmI2F);
10277   match(Set dst (ConvI2F src));
10278 
10279   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10280   ins_encode %{
10281     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10282   %}
10283   ins_pipe(pipe_slow); // XXX
10284 %}
10285 
10286 instruct convI2F_reg_mem(regF dst, memory src)
10287 %{
10288   match(Set dst (ConvI2F (LoadI src)));
10289 
10290   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10291   ins_encode %{
10292     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10293   %}
10294   ins_pipe(pipe_slow); // XXX
10295 %}
10296 
10297 instruct convI2D_reg_reg(regD dst, rRegI src)
10298 %{
10299   predicate(!UseXmmI2D);
10300   match(Set dst (ConvI2D src));
10301 
10302   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10303   ins_encode %{
10304     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10305   %}
10306   ins_pipe(pipe_slow); // XXX
10307 %}
10308 
10309 instruct convI2D_reg_mem(regD dst, memory src)
10310 %{
10311   match(Set dst (ConvI2D (LoadI src)));
10312 
10313   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10314   ins_encode %{
10315     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10316   %}
10317   ins_pipe(pipe_slow); // XXX
10318 %}
10319 
10320 instruct convXI2F_reg(regF dst, rRegI src)
10321 %{
10322   predicate(UseXmmI2F);
10323   match(Set dst (ConvI2F src));
10324 
10325   format %{ "movdl $dst, $src\n\t"
10326             "cvtdq2psl $dst, $dst\t# i2f" %}
10327   ins_encode %{
10328     __ movdl($dst$$XMMRegister, $src$$Register);
10329     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10330   %}
10331   ins_pipe(pipe_slow); // XXX
10332 %}
10333 
10334 instruct convXI2D_reg(regD dst, rRegI src)
10335 %{
10336   predicate(UseXmmI2D);
10337   match(Set dst (ConvI2D src));
10338 
10339   format %{ "movdl $dst, $src\n\t"
10340             "cvtdq2pdl $dst, $dst\t# i2d" %}
10341   ins_encode %{
10342     __ movdl($dst$$XMMRegister, $src$$Register);
10343     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10344   %}
10345   ins_pipe(pipe_slow); // XXX
10346 %}
10347 
10348 instruct convL2F_reg_reg(regF dst, rRegL src)
10349 %{
10350   match(Set dst (ConvL2F src));
10351 
10352   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10353   ins_encode %{
10354     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10355   %}
10356   ins_pipe(pipe_slow); // XXX
10357 %}
10358 
10359 instruct convL2F_reg_mem(regF dst, memory src)
10360 %{
10361   match(Set dst (ConvL2F (LoadL src)));
10362 
10363   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10364   ins_encode %{
10365     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10366   %}
10367   ins_pipe(pipe_slow); // XXX
10368 %}
10369 
10370 instruct convL2D_reg_reg(regD dst, rRegL src)
10371 %{
10372   match(Set dst (ConvL2D src));
10373 
10374   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10375   ins_encode %{
10376     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10377   %}
10378   ins_pipe(pipe_slow); // XXX
10379 %}
10380 
10381 instruct convL2D_reg_mem(regD dst, memory src)
10382 %{
10383   match(Set dst (ConvL2D (LoadL src)));
10384 
10385   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10386   ins_encode %{
10387     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10388   %}
10389   ins_pipe(pipe_slow); // XXX
10390 %}
10391 
10392 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10393 %{
10394   match(Set dst (ConvI2L src));
10395 
10396   ins_cost(125);
10397   format %{ "movslq  $dst, $src\t# i2l" %}
10398   ins_encode %{
10399     __ movslq($dst$$Register, $src$$Register);
10400   %}
10401   ins_pipe(ialu_reg_reg);
10402 %}
10403 
10404 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10405 // %{
10406 //   match(Set dst (ConvI2L src));
10407 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10408 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10409 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10410 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10411 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10412 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10413 
10414 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10415 //   ins_encode(enc_copy(dst, src));
10416 // //   opcode(0x63); // needs REX.W
10417 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10418 //   ins_pipe(ialu_reg_reg);
10419 // %}
10420 
10421 // Zero-extend convert int to long
10422 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10423 %{
10424   match(Set dst (AndL (ConvI2L src) mask));
10425 
10426   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10427   ins_encode %{
10428     if ($dst$$reg != $src$$reg) {
10429       __ movl($dst$$Register, $src$$Register);
10430     }
10431   %}
10432   ins_pipe(ialu_reg_reg);
10433 %}
10434 
10435 // Zero-extend convert int to long
10436 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10437 %{
10438   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10439 
10440   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10441   ins_encode %{
10442     __ movl($dst$$Register, $src$$Address);
10443   %}
10444   ins_pipe(ialu_reg_mem);
10445 %}
10446 
10447 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10448 %{
10449   match(Set dst (AndL src mask));
10450 
10451   format %{ "movl    $dst, $src\t# zero-extend long" %}
10452   ins_encode %{
10453     __ movl($dst$$Register, $src$$Register);
10454   %}
10455   ins_pipe(ialu_reg_reg);
10456 %}
10457 
10458 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10459 %{
10460   match(Set dst (ConvL2I src));
10461 
10462   format %{ "movl    $dst, $src\t# l2i" %}
10463   ins_encode %{
10464     __ movl($dst$$Register, $src$$Register);
10465   %}
10466   ins_pipe(ialu_reg_reg);
10467 %}
10468 
10469 
10470 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10471   match(Set dst (MoveF2I src));
10472   effect(DEF dst, USE src);
10473 
10474   ins_cost(125);
10475   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10476   ins_encode %{
10477     __ movl($dst$$Register, Address(rsp, $src$$disp));
10478   %}
10479   ins_pipe(ialu_reg_mem);
10480 %}
10481 
10482 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10483   match(Set dst (MoveI2F src));
10484   effect(DEF dst, USE src);
10485 
10486   ins_cost(125);
10487   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10488   ins_encode %{
10489     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10490   %}
10491   ins_pipe(pipe_slow);
10492 %}
10493 
10494 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10495   match(Set dst (MoveD2L src));
10496   effect(DEF dst, USE src);
10497 
10498   ins_cost(125);
10499   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10500   ins_encode %{
10501     __ movq($dst$$Register, Address(rsp, $src$$disp));
10502   %}
10503   ins_pipe(ialu_reg_mem);
10504 %}
10505 
10506 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10507   predicate(!UseXmmLoadAndClearUpper);
10508   match(Set dst (MoveL2D src));
10509   effect(DEF dst, USE src);
10510 
10511   ins_cost(125);
10512   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10513   ins_encode %{
10514     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10515   %}
10516   ins_pipe(pipe_slow);
10517 %}
10518 
10519 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10520   predicate(UseXmmLoadAndClearUpper);
10521   match(Set dst (MoveL2D src));
10522   effect(DEF dst, USE src);
10523 
10524   ins_cost(125);
10525   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10526   ins_encode %{
10527     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10528   %}
10529   ins_pipe(pipe_slow);
10530 %}
10531 
10532 
10533 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10534   match(Set dst (MoveF2I src));
10535   effect(DEF dst, USE src);
10536 
10537   ins_cost(95); // XXX
10538   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10539   ins_encode %{
10540     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10541   %}
10542   ins_pipe(pipe_slow);
10543 %}
10544 
10545 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10546   match(Set dst (MoveI2F src));
10547   effect(DEF dst, USE src);
10548 
10549   ins_cost(100);
10550   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10551   ins_encode %{
10552     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10553   %}
10554   ins_pipe( ialu_mem_reg );
10555 %}
10556 
10557 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10558   match(Set dst (MoveD2L src));
10559   effect(DEF dst, USE src);
10560 
10561   ins_cost(95); // XXX
10562   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10563   ins_encode %{
10564     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10565   %}
10566   ins_pipe(pipe_slow);
10567 %}
10568 
10569 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10570   match(Set dst (MoveL2D src));
10571   effect(DEF dst, USE src);
10572 
10573   ins_cost(100);
10574   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10575   ins_encode %{
10576     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10577   %}
10578   ins_pipe(ialu_mem_reg);
10579 %}
10580 
10581 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10582   match(Set dst (MoveF2I src));
10583   effect(DEF dst, USE src);
10584   ins_cost(85);
10585   format %{ "movd    $dst,$src\t# MoveF2I" %}
10586   ins_encode %{
10587     __ movdl($dst$$Register, $src$$XMMRegister);
10588   %}
10589   ins_pipe( pipe_slow );
10590 %}
10591 
10592 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10593   match(Set dst (MoveD2L src));
10594   effect(DEF dst, USE src);
10595   ins_cost(85);
10596   format %{ "movd    $dst,$src\t# MoveD2L" %}
10597   ins_encode %{
10598     __ movdq($dst$$Register, $src$$XMMRegister);
10599   %}
10600   ins_pipe( pipe_slow );
10601 %}
10602 
10603 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10604   match(Set dst (MoveI2F src));
10605   effect(DEF dst, USE src);
10606   ins_cost(100);
10607   format %{ "movd    $dst,$src\t# MoveI2F" %}
10608   ins_encode %{
10609     __ movdl($dst$$XMMRegister, $src$$Register);
10610   %}
10611   ins_pipe( pipe_slow );
10612 %}
10613 
10614 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10615   match(Set dst (MoveL2D src));
10616   effect(DEF dst, USE src);
10617   ins_cost(100);
10618   format %{ "movd    $dst,$src\t# MoveL2D" %}
10619   ins_encode %{
10620      __ movdq($dst$$XMMRegister, $src$$Register);
10621   %}
10622   ins_pipe( pipe_slow );
10623 %}
10624 
10625 
10626 // =======================================================================
10627 // fast clearing of an array
10628 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10629                   rFlagsReg cr)
10630 %{
10631   predicate(!((ClearArrayNode*)n)->is_large());
10632   match(Set dummy (ClearArray cnt base));
10633   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10634 
10635   format %{ $$template
10636     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10637     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10638     $$emit$$"jg      LARGE\n\t"
10639     $$emit$$"dec     rcx\n\t"
10640     $$emit$$"js      DONE\t# Zero length\n\t"
10641     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10642     $$emit$$"dec     rcx\n\t"
10643     $$emit$$"jge     LOOP\n\t"
10644     $$emit$$"jmp     DONE\n\t"
10645     $$emit$$"# LARGE:\n\t"
10646     if (UseFastStosb) {
10647        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10648        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10649     } else {
10650        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10651     }
10652     $$emit$$"# DONE"
10653   %}
10654   ins_encode %{
10655     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, false);
10656   %}
10657   ins_pipe(pipe_slow);
10658 %}
10659 
10660 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10661                   rFlagsReg cr)
10662 %{
10663   predicate(((ClearArrayNode*)n)->is_large());
10664   match(Set dummy (ClearArray cnt base));
10665   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10666 
10667   format %{ $$template
10668     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10669     if (UseFastStosb) {
10670        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10671        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10672     } else {
10673        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10674     }
10675   %}
10676   ins_encode %{
10677     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, true);
10678   %}
10679   ins_pipe(pipe_slow);
10680 %}
10681 
10682 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10683                          rax_RegI result, regD tmp1, rFlagsReg cr)
10684 %{
10685   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10686   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10687   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10688 
10689   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10690   ins_encode %{
10691     __ string_compare($str1$$Register, $str2$$Register,
10692                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10693                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10694   %}
10695   ins_pipe( pipe_slow );
10696 %}
10697 
10698 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10699                          rax_RegI result, regD tmp1, rFlagsReg cr)
10700 %{
10701   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10702   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10703   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10704 
10705   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10706   ins_encode %{
10707     __ string_compare($str1$$Register, $str2$$Register,
10708                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10709                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10710   %}
10711   ins_pipe( pipe_slow );
10712 %}
10713 
10714 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10715                           rax_RegI result, regD tmp1, rFlagsReg cr)
10716 %{
10717   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10718   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10719   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10720 
10721   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10722   ins_encode %{
10723     __ string_compare($str1$$Register, $str2$$Register,
10724                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10725                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10726   %}
10727   ins_pipe( pipe_slow );
10728 %}
10729 
10730 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10731                           rax_RegI result, regD tmp1, rFlagsReg cr)
10732 %{
10733   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10734   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10735   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10736 
10737   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10738   ins_encode %{
10739     __ string_compare($str2$$Register, $str1$$Register,
10740                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10741                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10742   %}
10743   ins_pipe( pipe_slow );
10744 %}
10745 
10746 // fast search of substring with known size.
10747 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10748                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10749 %{
10750   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10751   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10752   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10753 
10754   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10755   ins_encode %{
10756     int icnt2 = (int)$int_cnt2$$constant;
10757     if (icnt2 >= 16) {
10758       // IndexOf for constant substrings with size >= 16 elements
10759       // which don't need to be loaded through stack.
10760       __ string_indexofC8($str1$$Register, $str2$$Register,
10761                           $cnt1$$Register, $cnt2$$Register,
10762                           icnt2, $result$$Register,
10763                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10764     } else {
10765       // Small strings are loaded through stack if they cross page boundary.
10766       __ string_indexof($str1$$Register, $str2$$Register,
10767                         $cnt1$$Register, $cnt2$$Register,
10768                         icnt2, $result$$Register,
10769                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10770     }
10771   %}
10772   ins_pipe( pipe_slow );
10773 %}
10774 
10775 // fast search of substring with known size.
10776 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10777                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10778 %{
10779   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10780   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10781   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10782 
10783   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10784   ins_encode %{
10785     int icnt2 = (int)$int_cnt2$$constant;
10786     if (icnt2 >= 8) {
10787       // IndexOf for constant substrings with size >= 8 elements
10788       // which don't need to be loaded through stack.
10789       __ string_indexofC8($str1$$Register, $str2$$Register,
10790                           $cnt1$$Register, $cnt2$$Register,
10791                           icnt2, $result$$Register,
10792                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10793     } else {
10794       // Small strings are loaded through stack if they cross page boundary.
10795       __ string_indexof($str1$$Register, $str2$$Register,
10796                         $cnt1$$Register, $cnt2$$Register,
10797                         icnt2, $result$$Register,
10798                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10799     }
10800   %}
10801   ins_pipe( pipe_slow );
10802 %}
10803 
10804 // fast search of substring with known size.
10805 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10806                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10807 %{
10808   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10809   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10810   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10811 
10812   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10813   ins_encode %{
10814     int icnt2 = (int)$int_cnt2$$constant;
10815     if (icnt2 >= 8) {
10816       // IndexOf for constant substrings with size >= 8 elements
10817       // which don't need to be loaded through stack.
10818       __ string_indexofC8($str1$$Register, $str2$$Register,
10819                           $cnt1$$Register, $cnt2$$Register,
10820                           icnt2, $result$$Register,
10821                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10822     } else {
10823       // Small strings are loaded through stack if they cross page boundary.
10824       __ string_indexof($str1$$Register, $str2$$Register,
10825                         $cnt1$$Register, $cnt2$$Register,
10826                         icnt2, $result$$Register,
10827                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10828     }
10829   %}
10830   ins_pipe( pipe_slow );
10831 %}
10832 
10833 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10834                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10835 %{
10836   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10837   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10838   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10839 
10840   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10841   ins_encode %{
10842     __ string_indexof($str1$$Register, $str2$$Register,
10843                       $cnt1$$Register, $cnt2$$Register,
10844                       (-1), $result$$Register,
10845                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10846   %}
10847   ins_pipe( pipe_slow );
10848 %}
10849 
10850 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10851                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10852 %{
10853   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10854   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10855   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10856 
10857   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10858   ins_encode %{
10859     __ string_indexof($str1$$Register, $str2$$Register,
10860                       $cnt1$$Register, $cnt2$$Register,
10861                       (-1), $result$$Register,
10862                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10863   %}
10864   ins_pipe( pipe_slow );
10865 %}
10866 
10867 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10868                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10869 %{
10870   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10871   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10872   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10873 
10874   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10875   ins_encode %{
10876     __ string_indexof($str1$$Register, $str2$$Register,
10877                       $cnt1$$Register, $cnt2$$Register,
10878                       (-1), $result$$Register,
10879                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10880   %}
10881   ins_pipe( pipe_slow );
10882 %}
10883 
10884 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10885                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10886 %{
10887   predicate(UseSSE42Intrinsics);
10888   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10889   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10890   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10891   ins_encode %{
10892     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10893                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10894   %}
10895   ins_pipe( pipe_slow );
10896 %}
10897 
10898 // fast string equals
10899 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10900                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10901 %{
10902   match(Set result (StrEquals (Binary str1 str2) cnt));
10903   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10904 
10905   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10906   ins_encode %{
10907     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10908                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10909                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10910   %}
10911   ins_pipe( pipe_slow );
10912 %}
10913 
10914 // fast array equals
10915 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10916                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10917 %{
10918   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10919   match(Set result (AryEq ary1 ary2));
10920   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10921 
10922   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10923   ins_encode %{
10924     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10925                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10926                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10927   %}
10928   ins_pipe( pipe_slow );
10929 %}
10930 
10931 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10932                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10933 %{
10934   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10935   match(Set result (AryEq ary1 ary2));
10936   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10937 
10938   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10939   ins_encode %{
10940     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10941                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10942                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10943   %}
10944   ins_pipe( pipe_slow );
10945 %}
10946 
10947 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10948                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10949 %{
10950   match(Set result (HasNegatives ary1 len));
10951   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10952 
10953   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10954   ins_encode %{
10955     __ has_negatives($ary1$$Register, $len$$Register,
10956                      $result$$Register, $tmp3$$Register,
10957                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10958   %}
10959   ins_pipe( pipe_slow );
10960 %}
10961 
10962 // fast char[] to byte[] compression
10963 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10964                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10965   match(Set result (StrCompressedCopy src (Binary dst len)));
10966   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10967 
10968   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10969   ins_encode %{
10970     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10971                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10972                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10973   %}
10974   ins_pipe( pipe_slow );
10975 %}
10976 
10977 // fast byte[] to char[] inflation
10978 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10979                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10980   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10981   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10982 
10983   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10984   ins_encode %{
10985     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10986                           $tmp1$$XMMRegister, $tmp2$$Register);
10987   %}
10988   ins_pipe( pipe_slow );
10989 %}
10990 
10991 // encode char[] to byte[] in ISO_8859_1
10992 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10993                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10994                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10995   match(Set result (EncodeISOArray src (Binary dst len)));
10996   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10997 
10998   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10999   ins_encode %{
11000     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11001                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11002                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11003   %}
11004   ins_pipe( pipe_slow );
11005 %}
11006 
11007 //----------Overflow Math Instructions-----------------------------------------
11008 
11009 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11010 %{
11011   match(Set cr (OverflowAddI op1 op2));
11012   effect(DEF cr, USE_KILL op1, USE op2);
11013 
11014   format %{ "addl    $op1, $op2\t# overflow check int" %}
11015 
11016   ins_encode %{
11017     __ addl($op1$$Register, $op2$$Register);
11018   %}
11019   ins_pipe(ialu_reg_reg);
11020 %}
11021 
11022 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11023 %{
11024   match(Set cr (OverflowAddI op1 op2));
11025   effect(DEF cr, USE_KILL op1, USE op2);
11026 
11027   format %{ "addl    $op1, $op2\t# overflow check int" %}
11028 
11029   ins_encode %{
11030     __ addl($op1$$Register, $op2$$constant);
11031   %}
11032   ins_pipe(ialu_reg_reg);
11033 %}
11034 
11035 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11036 %{
11037   match(Set cr (OverflowAddL op1 op2));
11038   effect(DEF cr, USE_KILL op1, USE op2);
11039 
11040   format %{ "addq    $op1, $op2\t# overflow check long" %}
11041   ins_encode %{
11042     __ addq($op1$$Register, $op2$$Register);
11043   %}
11044   ins_pipe(ialu_reg_reg);
11045 %}
11046 
11047 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11048 %{
11049   match(Set cr (OverflowAddL op1 op2));
11050   effect(DEF cr, USE_KILL op1, USE op2);
11051 
11052   format %{ "addq    $op1, $op2\t# overflow check long" %}
11053   ins_encode %{
11054     __ addq($op1$$Register, $op2$$constant);
11055   %}
11056   ins_pipe(ialu_reg_reg);
11057 %}
11058 
11059 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11060 %{
11061   match(Set cr (OverflowSubI op1 op2));
11062 
11063   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11064   ins_encode %{
11065     __ cmpl($op1$$Register, $op2$$Register);
11066   %}
11067   ins_pipe(ialu_reg_reg);
11068 %}
11069 
11070 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11071 %{
11072   match(Set cr (OverflowSubI op1 op2));
11073 
11074   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11075   ins_encode %{
11076     __ cmpl($op1$$Register, $op2$$constant);
11077   %}
11078   ins_pipe(ialu_reg_reg);
11079 %}
11080 
11081 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11082 %{
11083   match(Set cr (OverflowSubL op1 op2));
11084 
11085   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11086   ins_encode %{
11087     __ cmpq($op1$$Register, $op2$$Register);
11088   %}
11089   ins_pipe(ialu_reg_reg);
11090 %}
11091 
11092 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11093 %{
11094   match(Set cr (OverflowSubL op1 op2));
11095 
11096   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11097   ins_encode %{
11098     __ cmpq($op1$$Register, $op2$$constant);
11099   %}
11100   ins_pipe(ialu_reg_reg);
11101 %}
11102 
11103 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11104 %{
11105   match(Set cr (OverflowSubI zero op2));
11106   effect(DEF cr, USE_KILL op2);
11107 
11108   format %{ "negl    $op2\t# overflow check int" %}
11109   ins_encode %{
11110     __ negl($op2$$Register);
11111   %}
11112   ins_pipe(ialu_reg_reg);
11113 %}
11114 
11115 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11116 %{
11117   match(Set cr (OverflowSubL zero op2));
11118   effect(DEF cr, USE_KILL op2);
11119 
11120   format %{ "negq    $op2\t# overflow check long" %}
11121   ins_encode %{
11122     __ negq($op2$$Register);
11123   %}
11124   ins_pipe(ialu_reg_reg);
11125 %}
11126 
11127 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11128 %{
11129   match(Set cr (OverflowMulI op1 op2));
11130   effect(DEF cr, USE_KILL op1, USE op2);
11131 
11132   format %{ "imull    $op1, $op2\t# overflow check int" %}
11133   ins_encode %{
11134     __ imull($op1$$Register, $op2$$Register);
11135   %}
11136   ins_pipe(ialu_reg_reg_alu0);
11137 %}
11138 
11139 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11140 %{
11141   match(Set cr (OverflowMulI op1 op2));
11142   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11143 
11144   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11145   ins_encode %{
11146     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11147   %}
11148   ins_pipe(ialu_reg_reg_alu0);
11149 %}
11150 
11151 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11152 %{
11153   match(Set cr (OverflowMulL op1 op2));
11154   effect(DEF cr, USE_KILL op1, USE op2);
11155 
11156   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11157   ins_encode %{
11158     __ imulq($op1$$Register, $op2$$Register);
11159   %}
11160   ins_pipe(ialu_reg_reg_alu0);
11161 %}
11162 
11163 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11164 %{
11165   match(Set cr (OverflowMulL op1 op2));
11166   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11167 
11168   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11169   ins_encode %{
11170     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11171   %}
11172   ins_pipe(ialu_reg_reg_alu0);
11173 %}
11174 
11175 
11176 //----------Control Flow Instructions------------------------------------------
11177 // Signed compare Instructions
11178 
11179 // XXX more variants!!
11180 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11181 %{
11182   match(Set cr (CmpI op1 op2));
11183   effect(DEF cr, USE op1, USE op2);
11184 
11185   format %{ "cmpl    $op1, $op2" %}
11186   opcode(0x3B);  /* Opcode 3B /r */
11187   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11188   ins_pipe(ialu_cr_reg_reg);
11189 %}
11190 
11191 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11192 %{
11193   match(Set cr (CmpI op1 op2));
11194 
11195   format %{ "cmpl    $op1, $op2" %}
11196   opcode(0x81, 0x07); /* Opcode 81 /7 */
11197   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11198   ins_pipe(ialu_cr_reg_imm);
11199 %}
11200 
11201 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11202 %{
11203   match(Set cr (CmpI op1 (LoadI op2)));
11204 
11205   ins_cost(500); // XXX
11206   format %{ "cmpl    $op1, $op2" %}
11207   opcode(0x3B); /* Opcode 3B /r */
11208   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11209   ins_pipe(ialu_cr_reg_mem);
11210 %}
11211 
11212 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11213 %{
11214   match(Set cr (CmpI src zero));
11215 
11216   format %{ "testl   $src, $src" %}
11217   opcode(0x85);
11218   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11219   ins_pipe(ialu_cr_reg_imm);
11220 %}
11221 
11222 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11223 %{
11224   match(Set cr (CmpI (AndI src con) zero));
11225 
11226   format %{ "testl   $src, $con" %}
11227   opcode(0xF7, 0x00);
11228   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11229   ins_pipe(ialu_cr_reg_imm);
11230 %}
11231 
11232 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11233 %{
11234   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11235 
11236   format %{ "testl   $src, $mem" %}
11237   opcode(0x85);
11238   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11239   ins_pipe(ialu_cr_reg_mem);
11240 %}
11241 
11242 // Unsigned compare Instructions; really, same as signed except they
11243 // produce an rFlagsRegU instead of rFlagsReg.
11244 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11245 %{
11246   match(Set cr (CmpU op1 op2));
11247 
11248   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11249   opcode(0x3B); /* Opcode 3B /r */
11250   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11251   ins_pipe(ialu_cr_reg_reg);
11252 %}
11253 
11254 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11255 %{
11256   match(Set cr (CmpU op1 op2));
11257 
11258   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11259   opcode(0x81,0x07); /* Opcode 81 /7 */
11260   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11261   ins_pipe(ialu_cr_reg_imm);
11262 %}
11263 
11264 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11265 %{
11266   match(Set cr (CmpU op1 (LoadI op2)));
11267 
11268   ins_cost(500); // XXX
11269   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11270   opcode(0x3B); /* Opcode 3B /r */
11271   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11272   ins_pipe(ialu_cr_reg_mem);
11273 %}
11274 
11275 // // // Cisc-spilled version of cmpU_rReg
11276 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11277 // //%{
11278 // //  match(Set cr (CmpU (LoadI op1) op2));
11279 // //
11280 // //  format %{ "CMPu   $op1,$op2" %}
11281 // //  ins_cost(500);
11282 // //  opcode(0x39);  /* Opcode 39 /r */
11283 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11284 // //%}
11285 
11286 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11287 %{
11288   match(Set cr (CmpU src zero));
11289 
11290   format %{ "testl  $src, $src\t# unsigned" %}
11291   opcode(0x85);
11292   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11293   ins_pipe(ialu_cr_reg_imm);
11294 %}
11295 
11296 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11297 %{
11298   match(Set cr (CmpP op1 op2));
11299 
11300   format %{ "cmpq    $op1, $op2\t# ptr" %}
11301   opcode(0x3B); /* Opcode 3B /r */
11302   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11303   ins_pipe(ialu_cr_reg_reg);
11304 %}
11305 
11306 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11307 %{
11308   match(Set cr (CmpP op1 (LoadP op2)));
11309 
11310   ins_cost(500); // XXX
11311   format %{ "cmpq    $op1, $op2\t# ptr" %}
11312   opcode(0x3B); /* Opcode 3B /r */
11313   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11314   ins_pipe(ialu_cr_reg_mem);
11315 %}
11316 
11317 // // // Cisc-spilled version of cmpP_rReg
11318 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11319 // //%{
11320 // //  match(Set cr (CmpP (LoadP op1) op2));
11321 // //
11322 // //  format %{ "CMPu   $op1,$op2" %}
11323 // //  ins_cost(500);
11324 // //  opcode(0x39);  /* Opcode 39 /r */
11325 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11326 // //%}
11327 
11328 // XXX this is generalized by compP_rReg_mem???
11329 // Compare raw pointer (used in out-of-heap check).
11330 // Only works because non-oop pointers must be raw pointers
11331 // and raw pointers have no anti-dependencies.
11332 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11333 %{
11334   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11335   match(Set cr (CmpP op1 (LoadP op2)));
11336 
11337   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11338   opcode(0x3B); /* Opcode 3B /r */
11339   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11340   ins_pipe(ialu_cr_reg_mem);
11341 %}
11342 
11343 // This will generate a signed flags result. This should be OK since
11344 // any compare to a zero should be eq/neq.
11345 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11346 %{
11347   match(Set cr (CmpP src zero));
11348 
11349   format %{ "testq   $src, $src\t# ptr" %}
11350   opcode(0x85);
11351   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11352   ins_pipe(ialu_cr_reg_imm);
11353 %}
11354 
11355 // This will generate a signed flags result. This should be OK since
11356 // any compare to a zero should be eq/neq.
11357 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11358 %{
11359   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11360   match(Set cr (CmpP (LoadP op) zero));
11361 
11362   ins_cost(500); // XXX
11363   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11364   opcode(0xF7); /* Opcode F7 /0 */
11365   ins_encode(REX_mem_wide(op),
11366              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11367   ins_pipe(ialu_cr_reg_imm);
11368 %}
11369 
11370 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11371 %{
11372   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11373   match(Set cr (CmpP (LoadP mem) zero));
11374 
11375   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11376   ins_encode %{
11377     __ cmpq(r12, $mem$$Address);
11378   %}
11379   ins_pipe(ialu_cr_reg_mem);
11380 %}
11381 
11382 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11383 %{
11384   match(Set cr (CmpN op1 op2));
11385 
11386   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11387   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11388   ins_pipe(ialu_cr_reg_reg);
11389 %}
11390 
11391 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11392 %{
11393   match(Set cr (CmpN src (LoadN mem)));
11394 
11395   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11396   ins_encode %{
11397     __ cmpl($src$$Register, $mem$$Address);
11398   %}
11399   ins_pipe(ialu_cr_reg_mem);
11400 %}
11401 
11402 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11403   match(Set cr (CmpN op1 op2));
11404 
11405   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11406   ins_encode %{
11407     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11408   %}
11409   ins_pipe(ialu_cr_reg_imm);
11410 %}
11411 
11412 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11413 %{
11414   match(Set cr (CmpN src (LoadN mem)));
11415 
11416   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11417   ins_encode %{
11418     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11419   %}
11420   ins_pipe(ialu_cr_reg_mem);
11421 %}
11422 
11423 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11424   match(Set cr (CmpN op1 op2));
11425 
11426   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11427   ins_encode %{
11428     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11429   %}
11430   ins_pipe(ialu_cr_reg_imm);
11431 %}
11432 
11433 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11434 %{
11435   match(Set cr (CmpN src (LoadNKlass mem)));
11436 
11437   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11438   ins_encode %{
11439     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11440   %}
11441   ins_pipe(ialu_cr_reg_mem);
11442 %}
11443 
11444 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11445   match(Set cr (CmpN src zero));
11446 
11447   format %{ "testl   $src, $src\t# compressed ptr" %}
11448   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11449   ins_pipe(ialu_cr_reg_imm);
11450 %}
11451 
11452 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11453 %{
11454   predicate(Universe::narrow_oop_base() != NULL);
11455   match(Set cr (CmpN (LoadN mem) zero));
11456 
11457   ins_cost(500); // XXX
11458   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11459   ins_encode %{
11460     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11461   %}
11462   ins_pipe(ialu_cr_reg_mem);
11463 %}
11464 
11465 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11466 %{
11467   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11468   match(Set cr (CmpN (LoadN mem) zero));
11469 
11470   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11471   ins_encode %{
11472     __ cmpl(r12, $mem$$Address);
11473   %}
11474   ins_pipe(ialu_cr_reg_mem);
11475 %}
11476 
11477 // Yanked all unsigned pointer compare operations.
11478 // Pointer compares are done with CmpP which is already unsigned.
11479 
11480 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11481 %{
11482   match(Set cr (CmpL op1 op2));
11483 
11484   format %{ "cmpq    $op1, $op2" %}
11485   opcode(0x3B);  /* Opcode 3B /r */
11486   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11487   ins_pipe(ialu_cr_reg_reg);
11488 %}
11489 
11490 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11491 %{
11492   match(Set cr (CmpL op1 op2));
11493 
11494   format %{ "cmpq    $op1, $op2" %}
11495   opcode(0x81, 0x07); /* Opcode 81 /7 */
11496   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11497   ins_pipe(ialu_cr_reg_imm);
11498 %}
11499 
11500 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11501 %{
11502   match(Set cr (CmpL op1 (LoadL op2)));
11503 
11504   format %{ "cmpq    $op1, $op2" %}
11505   opcode(0x3B); /* Opcode 3B /r */
11506   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11507   ins_pipe(ialu_cr_reg_mem);
11508 %}
11509 
11510 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11511 %{
11512   match(Set cr (CmpL src zero));
11513 
11514   format %{ "testq   $src, $src" %}
11515   opcode(0x85);
11516   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11517   ins_pipe(ialu_cr_reg_imm);
11518 %}
11519 
11520 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11521 %{
11522   match(Set cr (CmpL (AndL src con) zero));
11523 
11524   format %{ "testq   $src, $con\t# long" %}
11525   opcode(0xF7, 0x00);
11526   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11527   ins_pipe(ialu_cr_reg_imm);
11528 %}
11529 
11530 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11531 %{
11532   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11533 
11534   format %{ "testq   $src, $mem" %}
11535   opcode(0x85);
11536   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11537   ins_pipe(ialu_cr_reg_mem);
11538 %}
11539 
11540 // Manifest a CmpL result in an integer register.  Very painful.
11541 // This is the test to avoid.
11542 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11543 %{
11544   match(Set dst (CmpL3 src1 src2));
11545   effect(KILL flags);
11546 
11547   ins_cost(275); // XXX
11548   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11549             "movl    $dst, -1\n\t"
11550             "jl,s    done\n\t"
11551             "setne   $dst\n\t"
11552             "movzbl  $dst, $dst\n\t"
11553     "done:" %}
11554   ins_encode(cmpl3_flag(src1, src2, dst));
11555   ins_pipe(pipe_slow);
11556 %}
11557 
11558 // Unsigned long compare Instructions; really, same as signed long except they
11559 // produce an rFlagsRegU instead of rFlagsReg.
11560 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11561 %{
11562   match(Set cr (CmpUL op1 op2));
11563 
11564   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11565   opcode(0x3B);  /* Opcode 3B /r */
11566   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11567   ins_pipe(ialu_cr_reg_reg);
11568 %}
11569 
11570 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11571 %{
11572   match(Set cr (CmpUL op1 op2));
11573 
11574   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11575   opcode(0x81, 0x07); /* Opcode 81 /7 */
11576   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11577   ins_pipe(ialu_cr_reg_imm);
11578 %}
11579 
11580 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11581 %{
11582   match(Set cr (CmpUL op1 (LoadL op2)));
11583 
11584   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11585   opcode(0x3B); /* Opcode 3B /r */
11586   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11587   ins_pipe(ialu_cr_reg_mem);
11588 %}
11589 
11590 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11591 %{
11592   match(Set cr (CmpUL src zero));
11593 
11594   format %{ "testq   $src, $src\t# unsigned" %}
11595   opcode(0x85);
11596   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11597   ins_pipe(ialu_cr_reg_imm);
11598 %}
11599 
11600 //----------Max and Min--------------------------------------------------------
11601 // Min Instructions
11602 
11603 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11604 %{
11605   effect(USE_DEF dst, USE src, USE cr);
11606 
11607   format %{ "cmovlgt $dst, $src\t# min" %}
11608   opcode(0x0F, 0x4F);
11609   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11610   ins_pipe(pipe_cmov_reg);
11611 %}
11612 
11613 
11614 instruct minI_rReg(rRegI dst, rRegI src)
11615 %{
11616   match(Set dst (MinI dst src));
11617 
11618   ins_cost(200);
11619   expand %{
11620     rFlagsReg cr;
11621     compI_rReg(cr, dst, src);
11622     cmovI_reg_g(dst, src, cr);
11623   %}
11624 %}
11625 
11626 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11627 %{
11628   effect(USE_DEF dst, USE src, USE cr);
11629 
11630   format %{ "cmovllt $dst, $src\t# max" %}
11631   opcode(0x0F, 0x4C);
11632   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11633   ins_pipe(pipe_cmov_reg);
11634 %}
11635 
11636 
11637 instruct maxI_rReg(rRegI dst, rRegI src)
11638 %{
11639   match(Set dst (MaxI dst src));
11640 
11641   ins_cost(200);
11642   expand %{
11643     rFlagsReg cr;
11644     compI_rReg(cr, dst, src);
11645     cmovI_reg_l(dst, src, cr);
11646   %}
11647 %}
11648 
11649 // ============================================================================
11650 // Branch Instructions
11651 
11652 // Jump Direct - Label defines a relative address from JMP+1
11653 instruct jmpDir(label labl)
11654 %{
11655   match(Goto);
11656   effect(USE labl);
11657 
11658   ins_cost(300);
11659   format %{ "jmp     $labl" %}
11660   size(5);
11661   ins_encode %{
11662     Label* L = $labl$$label;
11663     __ jmp(*L, false); // Always long jump
11664   %}
11665   ins_pipe(pipe_jmp);
11666 %}
11667 
11668 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11669 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11670 %{
11671   match(If cop cr);
11672   effect(USE labl);
11673 
11674   ins_cost(300);
11675   format %{ "j$cop     $labl" %}
11676   size(6);
11677   ins_encode %{
11678     Label* L = $labl$$label;
11679     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11680   %}
11681   ins_pipe(pipe_jcc);
11682 %}
11683 
11684 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11685 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11686 %{
11687   predicate(!n->has_vector_mask_set());
11688   match(CountedLoopEnd cop cr);
11689   effect(USE labl);
11690 
11691   ins_cost(300);
11692   format %{ "j$cop     $labl\t# loop end" %}
11693   size(6);
11694   ins_encode %{
11695     Label* L = $labl$$label;
11696     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11697   %}
11698   ins_pipe(pipe_jcc);
11699 %}
11700 
11701 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11702 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11703   predicate(!n->has_vector_mask_set());
11704   match(CountedLoopEnd cop cmp);
11705   effect(USE labl);
11706 
11707   ins_cost(300);
11708   format %{ "j$cop,u   $labl\t# loop end" %}
11709   size(6);
11710   ins_encode %{
11711     Label* L = $labl$$label;
11712     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11713   %}
11714   ins_pipe(pipe_jcc);
11715 %}
11716 
11717 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11718   predicate(!n->has_vector_mask_set());
11719   match(CountedLoopEnd cop cmp);
11720   effect(USE labl);
11721 
11722   ins_cost(200);
11723   format %{ "j$cop,u   $labl\t# loop end" %}
11724   size(6);
11725   ins_encode %{
11726     Label* L = $labl$$label;
11727     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11728   %}
11729   ins_pipe(pipe_jcc);
11730 %}
11731 
11732 // mask version
11733 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11734 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
11735 %{
11736   predicate(n->has_vector_mask_set());
11737   match(CountedLoopEnd cop cr);
11738   effect(USE labl);
11739 
11740   ins_cost(400);
11741   format %{ "j$cop     $labl\t# loop end\n\t"
11742             "restorevectmask \t# vector mask restore for loops" %}
11743   size(10);
11744   ins_encode %{
11745     Label* L = $labl$$label;
11746     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11747     __ restorevectmask();
11748   %}
11749   ins_pipe(pipe_jcc);
11750 %}
11751 
11752 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11753 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11754   predicate(n->has_vector_mask_set());
11755   match(CountedLoopEnd cop cmp);
11756   effect(USE labl);
11757 
11758   ins_cost(400);
11759   format %{ "j$cop,u   $labl\t# loop end\n\t"
11760             "restorevectmask \t# vector mask restore for loops" %}
11761   size(10);
11762   ins_encode %{
11763     Label* L = $labl$$label;
11764     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11765     __ restorevectmask();
11766   %}
11767   ins_pipe(pipe_jcc);
11768 %}
11769 
11770 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11771   predicate(n->has_vector_mask_set());
11772   match(CountedLoopEnd cop cmp);
11773   effect(USE labl);
11774 
11775   ins_cost(300);
11776   format %{ "j$cop,u   $labl\t# loop end\n\t"
11777             "restorevectmask \t# vector mask restore for loops" %}
11778   size(10);
11779   ins_encode %{
11780     Label* L = $labl$$label;
11781     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11782     __ restorevectmask();
11783   %}
11784   ins_pipe(pipe_jcc);
11785 %}
11786 
11787 // Jump Direct Conditional - using unsigned comparison
11788 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11789   match(If cop cmp);
11790   effect(USE labl);
11791 
11792   ins_cost(300);
11793   format %{ "j$cop,u  $labl" %}
11794   size(6);
11795   ins_encode %{
11796     Label* L = $labl$$label;
11797     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11798   %}
11799   ins_pipe(pipe_jcc);
11800 %}
11801 
11802 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11803   match(If cop cmp);
11804   effect(USE labl);
11805 
11806   ins_cost(200);
11807   format %{ "j$cop,u  $labl" %}
11808   size(6);
11809   ins_encode %{
11810     Label* L = $labl$$label;
11811     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11812   %}
11813   ins_pipe(pipe_jcc);
11814 %}
11815 
11816 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11817   match(If cop cmp);
11818   effect(USE labl);
11819 
11820   ins_cost(200);
11821   format %{ $$template
11822     if ($cop$$cmpcode == Assembler::notEqual) {
11823       $$emit$$"jp,u   $labl\n\t"
11824       $$emit$$"j$cop,u   $labl"
11825     } else {
11826       $$emit$$"jp,u   done\n\t"
11827       $$emit$$"j$cop,u   $labl\n\t"
11828       $$emit$$"done:"
11829     }
11830   %}
11831   ins_encode %{
11832     Label* l = $labl$$label;
11833     if ($cop$$cmpcode == Assembler::notEqual) {
11834       __ jcc(Assembler::parity, *l, false);
11835       __ jcc(Assembler::notEqual, *l, false);
11836     } else if ($cop$$cmpcode == Assembler::equal) {
11837       Label done;
11838       __ jccb(Assembler::parity, done);
11839       __ jcc(Assembler::equal, *l, false);
11840       __ bind(done);
11841     } else {
11842        ShouldNotReachHere();
11843     }
11844   %}
11845   ins_pipe(pipe_jcc);
11846 %}
11847 
11848 // ============================================================================
11849 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11850 // superklass array for an instance of the superklass.  Set a hidden
11851 // internal cache on a hit (cache is checked with exposed code in
11852 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11853 // encoding ALSO sets flags.
11854 
11855 instruct partialSubtypeCheck(rdi_RegP result,
11856                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11857                              rFlagsReg cr)
11858 %{
11859   match(Set result (PartialSubtypeCheck sub super));
11860   effect(KILL rcx, KILL cr);
11861 
11862   ins_cost(1100);  // slightly larger than the next version
11863   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11864             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11865             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11866             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11867             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11868             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11869             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11870     "miss:\t" %}
11871 
11872   opcode(0x1); // Force a XOR of RDI
11873   ins_encode(enc_PartialSubtypeCheck());
11874   ins_pipe(pipe_slow);
11875 %}
11876 
11877 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11878                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11879                                      immP0 zero,
11880                                      rdi_RegP result)
11881 %{
11882   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11883   effect(KILL rcx, KILL result);
11884 
11885   ins_cost(1000);
11886   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11887             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11888             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11889             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11890             "jne,s   miss\t\t# Missed: flags nz\n\t"
11891             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11892     "miss:\t" %}
11893 
11894   opcode(0x0); // No need to XOR RDI
11895   ins_encode(enc_PartialSubtypeCheck());
11896   ins_pipe(pipe_slow);
11897 %}
11898 
11899 // ============================================================================
11900 // Branch Instructions -- short offset versions
11901 //
11902 // These instructions are used to replace jumps of a long offset (the default
11903 // match) with jumps of a shorter offset.  These instructions are all tagged
11904 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11905 // match rules in general matching.  Instead, the ADLC generates a conversion
11906 // method in the MachNode which can be used to do in-place replacement of the
11907 // long variant with the shorter variant.  The compiler will determine if a
11908 // branch can be taken by the is_short_branch_offset() predicate in the machine
11909 // specific code section of the file.
11910 
11911 // Jump Direct - Label defines a relative address from JMP+1
11912 instruct jmpDir_short(label labl) %{
11913   match(Goto);
11914   effect(USE labl);
11915 
11916   ins_cost(300);
11917   format %{ "jmp,s   $labl" %}
11918   size(2);
11919   ins_encode %{
11920     Label* L = $labl$$label;
11921     __ jmpb(*L);
11922   %}
11923   ins_pipe(pipe_jmp);
11924   ins_short_branch(1);
11925 %}
11926 
11927 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11928 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11929   match(If cop cr);
11930   effect(USE labl);
11931 
11932   ins_cost(300);
11933   format %{ "j$cop,s   $labl" %}
11934   size(2);
11935   ins_encode %{
11936     Label* L = $labl$$label;
11937     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11938   %}
11939   ins_pipe(pipe_jcc);
11940   ins_short_branch(1);
11941 %}
11942 
11943 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11944 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11945   match(CountedLoopEnd cop cr);
11946   effect(USE labl);
11947 
11948   ins_cost(300);
11949   format %{ "j$cop,s   $labl\t# loop end" %}
11950   size(2);
11951   ins_encode %{
11952     Label* L = $labl$$label;
11953     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11954   %}
11955   ins_pipe(pipe_jcc);
11956   ins_short_branch(1);
11957 %}
11958 
11959 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11960 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11961   match(CountedLoopEnd cop cmp);
11962   effect(USE labl);
11963 
11964   ins_cost(300);
11965   format %{ "j$cop,us  $labl\t# loop end" %}
11966   size(2);
11967   ins_encode %{
11968     Label* L = $labl$$label;
11969     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11970   %}
11971   ins_pipe(pipe_jcc);
11972   ins_short_branch(1);
11973 %}
11974 
11975 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11976   match(CountedLoopEnd cop cmp);
11977   effect(USE labl);
11978 
11979   ins_cost(300);
11980   format %{ "j$cop,us  $labl\t# loop end" %}
11981   size(2);
11982   ins_encode %{
11983     Label* L = $labl$$label;
11984     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11985   %}
11986   ins_pipe(pipe_jcc);
11987   ins_short_branch(1);
11988 %}
11989 
11990 // Jump Direct Conditional - using unsigned comparison
11991 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11992   match(If cop cmp);
11993   effect(USE labl);
11994 
11995   ins_cost(300);
11996   format %{ "j$cop,us  $labl" %}
11997   size(2);
11998   ins_encode %{
11999     Label* L = $labl$$label;
12000     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12001   %}
12002   ins_pipe(pipe_jcc);
12003   ins_short_branch(1);
12004 %}
12005 
12006 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12007   match(If cop cmp);
12008   effect(USE labl);
12009 
12010   ins_cost(300);
12011   format %{ "j$cop,us  $labl" %}
12012   size(2);
12013   ins_encode %{
12014     Label* L = $labl$$label;
12015     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12016   %}
12017   ins_pipe(pipe_jcc);
12018   ins_short_branch(1);
12019 %}
12020 
12021 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12022   match(If cop cmp);
12023   effect(USE labl);
12024 
12025   ins_cost(300);
12026   format %{ $$template
12027     if ($cop$$cmpcode == Assembler::notEqual) {
12028       $$emit$$"jp,u,s   $labl\n\t"
12029       $$emit$$"j$cop,u,s   $labl"
12030     } else {
12031       $$emit$$"jp,u,s   done\n\t"
12032       $$emit$$"j$cop,u,s  $labl\n\t"
12033       $$emit$$"done:"
12034     }
12035   %}
12036   size(4);
12037   ins_encode %{
12038     Label* l = $labl$$label;
12039     if ($cop$$cmpcode == Assembler::notEqual) {
12040       __ jccb(Assembler::parity, *l);
12041       __ jccb(Assembler::notEqual, *l);
12042     } else if ($cop$$cmpcode == Assembler::equal) {
12043       Label done;
12044       __ jccb(Assembler::parity, done);
12045       __ jccb(Assembler::equal, *l);
12046       __ bind(done);
12047     } else {
12048        ShouldNotReachHere();
12049     }
12050   %}
12051   ins_pipe(pipe_jcc);
12052   ins_short_branch(1);
12053 %}
12054 
12055 // ============================================================================
12056 // inlined locking and unlocking
12057 
12058 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12059   predicate(Compile::current()->use_rtm());
12060   match(Set cr (FastLock object box));
12061   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12062   ins_cost(300);
12063   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12064   ins_encode %{
12065     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12066                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12067                  _counters, _rtm_counters, _stack_rtm_counters,
12068                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12069                  true, ra_->C->profile_rtm());
12070   %}
12071   ins_pipe(pipe_slow);
12072 %}
12073 
12074 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12075   predicate(!Compile::current()->use_rtm());
12076   match(Set cr (FastLock object box));
12077   effect(TEMP tmp, TEMP scr, USE_KILL box);
12078   ins_cost(300);
12079   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12080   ins_encode %{
12081     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12082                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12083   %}
12084   ins_pipe(pipe_slow);
12085 %}
12086 
12087 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12088   match(Set cr (FastUnlock object box));
12089   effect(TEMP tmp, USE_KILL box);
12090   ins_cost(300);
12091   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12092   ins_encode %{
12093     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12094   %}
12095   ins_pipe(pipe_slow);
12096 %}
12097 
12098 
12099 // ============================================================================
12100 // Safepoint Instructions
12101 instruct safePoint_poll(rFlagsReg cr)
12102 %{
12103   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12104   match(SafePoint);
12105   effect(KILL cr);
12106 
12107   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12108             "# Safepoint: poll for GC" %}
12109   ins_cost(125);
12110   ins_encode %{
12111     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12112     __ testl(rax, addr);
12113   %}
12114   ins_pipe(ialu_reg_mem);
12115 %}
12116 
12117 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12118 %{
12119   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12120   match(SafePoint poll);
12121   effect(KILL cr, USE poll);
12122 
12123   format %{ "testl  rax, [$poll]\t"
12124             "# Safepoint: poll for GC" %}
12125   ins_cost(125);
12126   ins_encode %{
12127     __ relocate(relocInfo::poll_type);
12128     __ testl(rax, Address($poll$$Register, 0));
12129   %}
12130   ins_pipe(ialu_reg_mem);
12131 %}
12132 
12133 instruct safePoint_poll_tls(rFlagsReg cr, rex_RegP poll)
12134 %{
12135   predicate(SafepointMechanism::uses_thread_local_poll());
12136   match(SafePoint poll);
12137   effect(KILL cr, USE poll);
12138 
12139   format %{ "testl  rax, [$poll]\t"
12140             "# Safepoint: poll for GC" %}
12141   ins_cost(125);
12142   size(3); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12143   ins_encode %{
12144     __ relocate(relocInfo::poll_type);
12145     address pre_pc = __ pc();
12146     __ testl(rax, Address($poll$$Register, 0));
12147     address post_pc = __ pc();
12148     guarantee(pre_pc[0] == 0x41 && pre_pc[1] == 0x85, "must emit #rex test-ax [reg]");
12149   %}
12150   ins_pipe(ialu_reg_mem);
12151 %}
12152 
12153 // ============================================================================
12154 // Procedure Call/Return Instructions
12155 // Call Java Static Instruction
12156 // Note: If this code changes, the corresponding ret_addr_offset() and
12157 //       compute_padding() functions will have to be adjusted.
12158 instruct CallStaticJavaDirect(method meth) %{
12159   match(CallStaticJava);
12160   effect(USE meth);
12161 
12162   ins_cost(300);
12163   format %{ "call,static " %}
12164   opcode(0xE8); /* E8 cd */
12165   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12166   ins_pipe(pipe_slow);
12167   ins_alignment(4);
12168 %}
12169 
12170 // Call Java Dynamic Instruction
12171 // Note: If this code changes, the corresponding ret_addr_offset() and
12172 //       compute_padding() functions will have to be adjusted.
12173 instruct CallDynamicJavaDirect(method meth)
12174 %{
12175   match(CallDynamicJava);
12176   effect(USE meth);
12177 
12178   ins_cost(300);
12179   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12180             "call,dynamic " %}
12181   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12182   ins_pipe(pipe_slow);
12183   ins_alignment(4);
12184 %}
12185 
12186 // Call Runtime Instruction
12187 instruct CallRuntimeDirect(method meth)
12188 %{
12189   match(CallRuntime);
12190   effect(USE meth);
12191 
12192   ins_cost(300);
12193   format %{ "call,runtime " %}
12194   ins_encode(clear_avx, Java_To_Runtime(meth));
12195   ins_pipe(pipe_slow);
12196 %}
12197 
12198 // Call runtime without safepoint
12199 instruct CallLeafDirect(method meth)
12200 %{
12201   match(CallLeaf);
12202   effect(USE meth);
12203 
12204   ins_cost(300);
12205   format %{ "call_leaf,runtime " %}
12206   ins_encode(clear_avx, Java_To_Runtime(meth));
12207   ins_pipe(pipe_slow);
12208 %}
12209 
12210 // Call runtime without safepoint
12211 instruct CallLeafNoFPDirect(method meth)
12212 %{
12213   match(CallLeafNoFP);
12214   effect(USE meth);
12215 
12216   ins_cost(300);
12217   format %{ "call_leaf_nofp,runtime " %}
12218   ins_encode(clear_avx, Java_To_Runtime(meth));
12219   ins_pipe(pipe_slow);
12220 %}
12221 
12222 // Return Instruction
12223 // Remove the return address & jump to it.
12224 // Notice: We always emit a nop after a ret to make sure there is room
12225 // for safepoint patching
12226 instruct Ret()
12227 %{
12228   match(Return);
12229 
12230   format %{ "ret" %}
12231   opcode(0xC3);
12232   ins_encode(OpcP);
12233   ins_pipe(pipe_jmp);
12234 %}
12235 
12236 // Tail Call; Jump from runtime stub to Java code.
12237 // Also known as an 'interprocedural jump'.
12238 // Target of jump will eventually return to caller.
12239 // TailJump below removes the return address.
12240 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12241 %{
12242   match(TailCall jump_target method_oop);
12243 
12244   ins_cost(300);
12245   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12246   opcode(0xFF, 0x4); /* Opcode FF /4 */
12247   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12248   ins_pipe(pipe_jmp);
12249 %}
12250 
12251 // Tail Jump; remove the return address; jump to target.
12252 // TailCall above leaves the return address around.
12253 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12254 %{
12255   match(TailJump jump_target ex_oop);
12256 
12257   ins_cost(300);
12258   format %{ "popq    rdx\t# pop return address\n\t"
12259             "jmp     $jump_target" %}
12260   opcode(0xFF, 0x4); /* Opcode FF /4 */
12261   ins_encode(Opcode(0x5a), // popq rdx
12262              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12263   ins_pipe(pipe_jmp);
12264 %}
12265 
12266 // Create exception oop: created by stack-crawling runtime code.
12267 // Created exception is now available to this handler, and is setup
12268 // just prior to jumping to this handler.  No code emitted.
12269 instruct CreateException(rax_RegP ex_oop)
12270 %{
12271   match(Set ex_oop (CreateEx));
12272 
12273   size(0);
12274   // use the following format syntax
12275   format %{ "# exception oop is in rax; no code emitted" %}
12276   ins_encode();
12277   ins_pipe(empty);
12278 %}
12279 
12280 // Rethrow exception:
12281 // The exception oop will come in the first argument position.
12282 // Then JUMP (not call) to the rethrow stub code.
12283 instruct RethrowException()
12284 %{
12285   match(Rethrow);
12286 
12287   // use the following format syntax
12288   format %{ "jmp     rethrow_stub" %}
12289   ins_encode(enc_rethrow);
12290   ins_pipe(pipe_jmp);
12291 %}
12292 
12293 
12294 // ============================================================================
12295 // This name is KNOWN by the ADLC and cannot be changed.
12296 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12297 // for this guy.
12298 instruct tlsLoadP(r15_RegP dst) %{
12299   match(Set dst (ThreadLocal));
12300   effect(DEF dst);
12301 
12302   size(0);
12303   format %{ "# TLS is in R15" %}
12304   ins_encode( /*empty encoding*/ );
12305   ins_pipe(ialu_reg_reg);
12306 %}
12307 
12308 
12309 //----------PEEPHOLE RULES-----------------------------------------------------
12310 // These must follow all instruction definitions as they use the names
12311 // defined in the instructions definitions.
12312 //
12313 // peepmatch ( root_instr_name [preceding_instruction]* );
12314 //
12315 // peepconstraint %{
12316 // (instruction_number.operand_name relational_op instruction_number.operand_name
12317 //  [, ...] );
12318 // // instruction numbers are zero-based using left to right order in peepmatch
12319 //
12320 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12321 // // provide an instruction_number.operand_name for each operand that appears
12322 // // in the replacement instruction's match rule
12323 //
12324 // ---------VM FLAGS---------------------------------------------------------
12325 //
12326 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12327 //
12328 // Each peephole rule is given an identifying number starting with zero and
12329 // increasing by one in the order seen by the parser.  An individual peephole
12330 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12331 // on the command-line.
12332 //
12333 // ---------CURRENT LIMITATIONS----------------------------------------------
12334 //
12335 // Only match adjacent instructions in same basic block
12336 // Only equality constraints
12337 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12338 // Only one replacement instruction
12339 //
12340 // ---------EXAMPLE----------------------------------------------------------
12341 //
12342 // // pertinent parts of existing instructions in architecture description
12343 // instruct movI(rRegI dst, rRegI src)
12344 // %{
12345 //   match(Set dst (CopyI src));
12346 // %}
12347 //
12348 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12349 // %{
12350 //   match(Set dst (AddI dst src));
12351 //   effect(KILL cr);
12352 // %}
12353 //
12354 // // Change (inc mov) to lea
12355 // peephole %{
12356 //   // increment preceeded by register-register move
12357 //   peepmatch ( incI_rReg movI );
12358 //   // require that the destination register of the increment
12359 //   // match the destination register of the move
12360 //   peepconstraint ( 0.dst == 1.dst );
12361 //   // construct a replacement instruction that sets
12362 //   // the destination to ( move's source register + one )
12363 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12364 // %}
12365 //
12366 
12367 // Implementation no longer uses movX instructions since
12368 // machine-independent system no longer uses CopyX nodes.
12369 //
12370 // peephole
12371 // %{
12372 //   peepmatch (incI_rReg movI);
12373 //   peepconstraint (0.dst == 1.dst);
12374 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12375 // %}
12376 
12377 // peephole
12378 // %{
12379 //   peepmatch (decI_rReg movI);
12380 //   peepconstraint (0.dst == 1.dst);
12381 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12382 // %}
12383 
12384 // peephole
12385 // %{
12386 //   peepmatch (addI_rReg_imm movI);
12387 //   peepconstraint (0.dst == 1.dst);
12388 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12389 // %}
12390 
12391 // peephole
12392 // %{
12393 //   peepmatch (incL_rReg movL);
12394 //   peepconstraint (0.dst == 1.dst);
12395 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12396 // %}
12397 
12398 // peephole
12399 // %{
12400 //   peepmatch (decL_rReg movL);
12401 //   peepconstraint (0.dst == 1.dst);
12402 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12403 // %}
12404 
12405 // peephole
12406 // %{
12407 //   peepmatch (addL_rReg_imm movL);
12408 //   peepconstraint (0.dst == 1.dst);
12409 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12410 // %}
12411 
12412 // peephole
12413 // %{
12414 //   peepmatch (addP_rReg_imm movP);
12415 //   peepconstraint (0.dst == 1.dst);
12416 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12417 // %}
12418 
12419 // // Change load of spilled value to only a spill
12420 // instruct storeI(memory mem, rRegI src)
12421 // %{
12422 //   match(Set mem (StoreI mem src));
12423 // %}
12424 //
12425 // instruct loadI(rRegI dst, memory mem)
12426 // %{
12427 //   match(Set dst (LoadI mem));
12428 // %}
12429 //
12430 
12431 peephole
12432 %{
12433   peepmatch (loadI storeI);
12434   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12435   peepreplace (storeI(1.mem 1.mem 1.src));
12436 %}
12437 
12438 peephole
12439 %{
12440   peepmatch (loadL storeL);
12441   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12442   peepreplace (storeL(1.mem 1.mem 1.src));
12443 %}
12444 
12445 //----------SMARTSPILL RULES---------------------------------------------------
12446 // These must follow all instruction definitions as they use the names
12447 // defined in the instructions definitions.