1 //
   2 // Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // The registers which can be used for
 321 // a thread local safepoint poll
 322 // * R12 is reserved for heap base
 323 // * R13 cannot be encoded for addressing without an offset byte
 324 // * R15 is reserved for the JavaThread
 325 reg_class ptr_rex_reg(R8,  R8_H,
 326                       R9,  R9_H,
 327                       R10, R10_H,
 328                       R11, R11_H,
 329                       R14, R14_H);
 330 
 331 
 332 // Class for all long registers (excluding RSP)
 333 reg_class long_reg_with_rbp(RAX, RAX_H,
 334                             RDX, RDX_H,
 335                             RBP, RBP_H,
 336                             RDI, RDI_H,
 337                             RSI, RSI_H,
 338                             RCX, RCX_H,
 339                             RBX, RBX_H,
 340                             R8,  R8_H,
 341                             R9,  R9_H,
 342                             R10, R10_H,
 343                             R11, R11_H,
 344                             R13, R13_H,
 345                             R14, R14_H);
 346 
 347 // Class for all long registers (excluding RSP and RBP)
 348 reg_class long_reg_no_rbp(RAX, RAX_H,
 349                           RDX, RDX_H,
 350                           RDI, RDI_H,
 351                           RSI, RSI_H,
 352                           RCX, RCX_H,
 353                           RBX, RBX_H,
 354                           R8,  R8_H,
 355                           R9,  R9_H,
 356                           R10, R10_H,
 357                           R11, R11_H,
 358                           R13, R13_H,
 359                           R14, R14_H);
 360 
 361 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 362 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 363 
 364 // Class for all long registers (excluding RAX, RDX and RSP)
 365 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 366                                        RDI, RDI_H,
 367                                        RSI, RSI_H,
 368                                        RCX, RCX_H,
 369                                        RBX, RBX_H,
 370                                        R8,  R8_H,
 371                                        R9,  R9_H,
 372                                        R10, R10_H,
 373                                        R11, R11_H,
 374                                        R13, R13_H,
 375                                        R14, R14_H);
 376 
 377 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 378 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 379                                      RSI, RSI_H,
 380                                      RCX, RCX_H,
 381                                      RBX, RBX_H,
 382                                      R8,  R8_H,
 383                                      R9,  R9_H,
 384                                      R10, R10_H,
 385                                      R11, R11_H,
 386                                      R13, R13_H,
 387                                      R14, R14_H);
 388 
 389 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 390 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 391 
 392 // Class for all long registers (excluding RCX and RSP)
 393 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 394                                    RDI, RDI_H,
 395                                    RSI, RSI_H,
 396                                    RAX, RAX_H,
 397                                    RDX, RDX_H,
 398                                    RBX, RBX_H,
 399                                    R8,  R8_H,
 400                                    R9,  R9_H,
 401                                    R10, R10_H,
 402                                    R11, R11_H,
 403                                    R13, R13_H,
 404                                    R14, R14_H);
 405 
 406 // Class for all long registers (excluding RCX, RSP, and RBP)
 407 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 408                                  RSI, RSI_H,
 409                                  RAX, RAX_H,
 410                                  RDX, RDX_H,
 411                                  RBX, RBX_H,
 412                                  R8,  R8_H,
 413                                  R9,  R9_H,
 414                                  R10, R10_H,
 415                                  R11, R11_H,
 416                                  R13, R13_H,
 417                                  R14, R14_H);
 418 
 419 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 420 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 421 
 422 // Singleton class for RAX long register
 423 reg_class long_rax_reg(RAX, RAX_H);
 424 
 425 // Singleton class for RCX long register
 426 reg_class long_rcx_reg(RCX, RCX_H);
 427 
 428 // Singleton class for RDX long register
 429 reg_class long_rdx_reg(RDX, RDX_H);
 430 
 431 // Class for all int registers (excluding RSP)
 432 reg_class int_reg_with_rbp(RAX,
 433                            RDX,
 434                            RBP,
 435                            RDI,
 436                            RSI,
 437                            RCX,
 438                            RBX,
 439                            R8,
 440                            R9,
 441                            R10,
 442                            R11,
 443                            R13,
 444                            R14);
 445 
 446 // Class for all int registers (excluding RSP and RBP)
 447 reg_class int_reg_no_rbp(RAX,
 448                          RDX,
 449                          RDI,
 450                          RSI,
 451                          RCX,
 452                          RBX,
 453                          R8,
 454                          R9,
 455                          R10,
 456                          R11,
 457                          R13,
 458                          R14);
 459 
 460 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 461 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 462 
 463 // Class for all int registers (excluding RCX and RSP)
 464 reg_class int_no_rcx_reg_with_rbp(RAX,
 465                                   RDX,
 466                                   RBP,
 467                                   RDI,
 468                                   RSI,
 469                                   RBX,
 470                                   R8,
 471                                   R9,
 472                                   R10,
 473                                   R11,
 474                                   R13,
 475                                   R14);
 476 
 477 // Class for all int registers (excluding RCX, RSP, and RBP)
 478 reg_class int_no_rcx_reg_no_rbp(RAX,
 479                                 RDX,
 480                                 RDI,
 481                                 RSI,
 482                                 RBX,
 483                                 R8,
 484                                 R9,
 485                                 R10,
 486                                 R11,
 487                                 R13,
 488                                 R14);
 489 
 490 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 491 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 492 
 493 // Class for all int registers (excluding RAX, RDX, and RSP)
 494 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 495                                       RDI,
 496                                       RSI,
 497                                       RCX,
 498                                       RBX,
 499                                       R8,
 500                                       R9,
 501                                       R10,
 502                                       R11,
 503                                       R13,
 504                                       R14);
 505 
 506 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 507 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 508                                     RSI,
 509                                     RCX,
 510                                     RBX,
 511                                     R8,
 512                                     R9,
 513                                     R10,
 514                                     R11,
 515                                     R13,
 516                                     R14);
 517 
 518 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 519 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 520 
 521 // Singleton class for RAX int register
 522 reg_class int_rax_reg(RAX);
 523 
 524 // Singleton class for RBX int register
 525 reg_class int_rbx_reg(RBX);
 526 
 527 // Singleton class for RCX int register
 528 reg_class int_rcx_reg(RCX);
 529 
 530 // Singleton class for RCX int register
 531 reg_class int_rdx_reg(RDX);
 532 
 533 // Singleton class for RCX int register
 534 reg_class int_rdi_reg(RDI);
 535 
 536 // Singleton class for instruction pointer
 537 // reg_class ip_reg(RIP);
 538 
 539 %}
 540 
 541 source_hpp %{
 542 #if INCLUDE_ZGC
 543 #include "gc/z/zBarrierSetAssembler.hpp"
 544 #endif
 545 %}
 546 
 547 //----------SOURCE BLOCK-------------------------------------------------------
 548 // This is a block of C++ code which provides values, functions, and
 549 // definitions necessary in the rest of the architecture description
 550 source %{
 551 #define   RELOC_IMM64    Assembler::imm_operand
 552 #define   RELOC_DISP32   Assembler::disp32_operand
 553 
 554 #define __ _masm.
 555 
 556 static bool generate_vzeroupper(Compile* C) {
 557   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 558 }
 559 
 560 static int clear_avx_size() {
 561   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 562 }
 563 
 564 // !!!!! Special hack to get all types of calls to specify the byte offset
 565 //       from the start of the call to the point where the return address
 566 //       will point.
 567 int MachCallStaticJavaNode::ret_addr_offset()
 568 {
 569   int offset = 5; // 5 bytes from start of call to where return address points
 570   offset += clear_avx_size();
 571   return offset;
 572 }
 573 
 574 int MachCallDynamicJavaNode::ret_addr_offset()
 575 {
 576   int offset = 15; // 15 bytes from start of call to where return address points
 577   offset += clear_avx_size();
 578   return offset;
 579 }
 580 
 581 int MachCallRuntimeNode::ret_addr_offset() {
 582   int offset = 13; // movq r10,#addr; callq (r10)
 583   offset += clear_avx_size();
 584   return offset;
 585 }
 586 
 587 // Indicate if the safepoint node needs the polling page as an input,
 588 // it does if the polling page is more than disp32 away.
 589 bool SafePointNode::needs_polling_address_input()
 590 {
 591   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 592 }
 593 
 594 //
 595 // Compute padding required for nodes which need alignment
 596 //
 597 
 598 // The address of the call instruction needs to be 4-byte aligned to
 599 // ensure that it does not span a cache line so that it can be patched.
 600 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 601 {
 602   current_offset += clear_avx_size(); // skip vzeroupper
 603   current_offset += 1; // skip call opcode byte
 604   return align_up(current_offset, alignment_required()) - current_offset;
 605 }
 606 
 607 // The address of the call instruction needs to be 4-byte aligned to
 608 // ensure that it does not span a cache line so that it can be patched.
 609 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 610 {
 611   current_offset += clear_avx_size(); // skip vzeroupper
 612   current_offset += 11; // skip movq instruction + call opcode byte
 613   return align_up(current_offset, alignment_required()) - current_offset;
 614 }
 615 
 616 // EMIT_RM()
 617 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 618   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 619   cbuf.insts()->emit_int8(c);
 620 }
 621 
 622 // EMIT_CC()
 623 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 624   unsigned char c = (unsigned char) (f1 | f2);
 625   cbuf.insts()->emit_int8(c);
 626 }
 627 
 628 // EMIT_OPCODE()
 629 void emit_opcode(CodeBuffer &cbuf, int code) {
 630   cbuf.insts()->emit_int8((unsigned char) code);
 631 }
 632 
 633 // EMIT_OPCODE() w/ relocation information
 634 void emit_opcode(CodeBuffer &cbuf,
 635                  int code, relocInfo::relocType reloc, int offset, int format)
 636 {
 637   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 638   emit_opcode(cbuf, code);
 639 }
 640 
 641 // EMIT_D8()
 642 void emit_d8(CodeBuffer &cbuf, int d8) {
 643   cbuf.insts()->emit_int8((unsigned char) d8);
 644 }
 645 
 646 // EMIT_D16()
 647 void emit_d16(CodeBuffer &cbuf, int d16) {
 648   cbuf.insts()->emit_int16(d16);
 649 }
 650 
 651 // EMIT_D32()
 652 void emit_d32(CodeBuffer &cbuf, int d32) {
 653   cbuf.insts()->emit_int32(d32);
 654 }
 655 
 656 // EMIT_D64()
 657 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 658   cbuf.insts()->emit_int64(d64);
 659 }
 660 
 661 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 662 void emit_d32_reloc(CodeBuffer& cbuf,
 663                     int d32,
 664                     relocInfo::relocType reloc,
 665                     int format)
 666 {
 667   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 668   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 669   cbuf.insts()->emit_int32(d32);
 670 }
 671 
 672 // emit 32 bit value and construct relocation entry from RelocationHolder
 673 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 674 #ifdef ASSERT
 675   if (rspec.reloc()->type() == relocInfo::oop_type &&
 676       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 677     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 678     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 679   }
 680 #endif
 681   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 682   cbuf.insts()->emit_int32(d32);
 683 }
 684 
 685 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 686   address next_ip = cbuf.insts_end() + 4;
 687   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 688                  external_word_Relocation::spec(addr),
 689                  RELOC_DISP32);
 690 }
 691 
 692 
 693 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 694 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 695   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 696   cbuf.insts()->emit_int64(d64);
 697 }
 698 
 699 // emit 64 bit value and construct relocation entry from RelocationHolder
 700 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 701 #ifdef ASSERT
 702   if (rspec.reloc()->type() == relocInfo::oop_type &&
 703       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 704     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 705     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 706            "cannot embed scavengable oops in code");
 707   }
 708 #endif
 709   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 710   cbuf.insts()->emit_int64(d64);
 711 }
 712 
 713 // Access stack slot for load or store
 714 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 715 {
 716   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 717   if (-0x80 <= disp && disp < 0x80) {
 718     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 719     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 720     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 721   } else {
 722     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 723     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 724     emit_d32(cbuf, disp);     // Displacement // R/M byte
 725   }
 726 }
 727 
 728    // rRegI ereg, memory mem) %{    // emit_reg_mem
 729 void encode_RegMem(CodeBuffer &cbuf,
 730                    int reg,
 731                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 732 {
 733   assert(disp_reloc == relocInfo::none, "cannot have disp");
 734   int regenc = reg & 7;
 735   int baseenc = base & 7;
 736   int indexenc = index & 7;
 737 
 738   // There is no index & no scale, use form without SIB byte
 739   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 740     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 741     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 742       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 743     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 744       // If 8-bit displacement, mode 0x1
 745       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 746       emit_d8(cbuf, disp);
 747     } else {
 748       // If 32-bit displacement
 749       if (base == -1) { // Special flag for absolute address
 750         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 751         if (disp_reloc != relocInfo::none) {
 752           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 753         } else {
 754           emit_d32(cbuf, disp);
 755         }
 756       } else {
 757         // Normal base + offset
 758         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 759         if (disp_reloc != relocInfo::none) {
 760           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 761         } else {
 762           emit_d32(cbuf, disp);
 763         }
 764       }
 765     }
 766   } else {
 767     // Else, encode with the SIB byte
 768     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 769     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 770       // If no displacement
 771       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 772       emit_rm(cbuf, scale, indexenc, baseenc);
 773     } else {
 774       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 775         // If 8-bit displacement, mode 0x1
 776         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 777         emit_rm(cbuf, scale, indexenc, baseenc);
 778         emit_d8(cbuf, disp);
 779       } else {
 780         // If 32-bit displacement
 781         if (base == 0x04 ) {
 782           emit_rm(cbuf, 0x2, regenc, 0x4);
 783           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 784         } else {
 785           emit_rm(cbuf, 0x2, regenc, 0x4);
 786           emit_rm(cbuf, scale, indexenc, baseenc); // *
 787         }
 788         if (disp_reloc != relocInfo::none) {
 789           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 790         } else {
 791           emit_d32(cbuf, disp);
 792         }
 793       }
 794     }
 795   }
 796 }
 797 
 798 // This could be in MacroAssembler but it's fairly C2 specific
 799 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 800   Label exit;
 801   __ jccb(Assembler::noParity, exit);
 802   __ pushf();
 803   //
 804   // comiss/ucomiss instructions set ZF,PF,CF flags and
 805   // zero OF,AF,SF for NaN values.
 806   // Fixup flags by zeroing ZF,PF so that compare of NaN
 807   // values returns 'less than' result (CF is set).
 808   // Leave the rest of flags unchanged.
 809   //
 810   //    7 6 5 4 3 2 1 0
 811   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 812   //    0 0 1 0 1 0 1 1   (0x2B)
 813   //
 814   __ andq(Address(rsp, 0), 0xffffff2b);
 815   __ popf();
 816   __ bind(exit);
 817 }
 818 
 819 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 820   Label done;
 821   __ movl(dst, -1);
 822   __ jcc(Assembler::parity, done);
 823   __ jcc(Assembler::below, done);
 824   __ setb(Assembler::notEqual, dst);
 825   __ movzbl(dst, dst);
 826   __ bind(done);
 827 }
 828 
 829 
 830 //=============================================================================
 831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 832 
 833 int Compile::ConstantTable::calculate_table_base_offset() const {
 834   return 0;  // absolute addressing, no offset
 835 }
 836 
 837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 839   ShouldNotReachHere();
 840 }
 841 
 842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 843   // Empty encoding
 844 }
 845 
 846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 847   return 0;
 848 }
 849 
 850 #ifndef PRODUCT
 851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 852   st->print("# MachConstantBaseNode (empty encoding)");
 853 }
 854 #endif
 855 
 856 
 857 //=============================================================================
 858 #ifndef PRODUCT
 859 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 860   Compile* C = ra_->C;
 861 
 862   int framesize = C->frame_size_in_bytes();
 863   int bangsize = C->bang_size_in_bytes();
 864   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 865   // Remove wordSize for return addr which is already pushed.
 866   framesize -= wordSize;
 867 
 868   if (C->need_stack_bang(bangsize)) {
 869     framesize -= wordSize;
 870     st->print("# stack bang (%d bytes)", bangsize);
 871     st->print("\n\t");
 872     st->print("pushq   rbp\t# Save rbp");
 873     if (PreserveFramePointer) {
 874         st->print("\n\t");
 875         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 876     }
 877     if (framesize) {
 878       st->print("\n\t");
 879       st->print("subq    rsp, #%d\t# Create frame",framesize);
 880     }
 881   } else {
 882     st->print("subq    rsp, #%d\t# Create frame",framesize);
 883     st->print("\n\t");
 884     framesize -= wordSize;
 885     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 886     if (PreserveFramePointer) {
 887       st->print("\n\t");
 888       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 889       if (framesize > 0) {
 890         st->print("\n\t");
 891         st->print("addq    rbp, #%d", framesize);
 892       }
 893     }
 894   }
 895 
 896   if (VerifyStackAtCalls) {
 897     st->print("\n\t");
 898     framesize -= wordSize;
 899     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 900 #ifdef ASSERT
 901     st->print("\n\t");
 902     st->print("# stack alignment check");
 903 #endif
 904   }
 905   st->cr();
 906 }
 907 #endif
 908 
 909 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 910   Compile* C = ra_->C;
 911   MacroAssembler _masm(&cbuf);
 912 
 913   int framesize = C->frame_size_in_bytes();
 914   int bangsize = C->bang_size_in_bytes();
 915 
 916   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 917 
 918   C->set_frame_complete(cbuf.insts_size());
 919 
 920   if (C->has_mach_constant_base_node()) {
 921     // NOTE: We set the table base offset here because users might be
 922     // emitted before MachConstantBaseNode.
 923     Compile::ConstantTable& constant_table = C->constant_table();
 924     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 925   }
 926 }
 927 
 928 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 929 {
 930   return MachNode::size(ra_); // too many variables; just compute it
 931                               // the hard way
 932 }
 933 
 934 int MachPrologNode::reloc() const
 935 {
 936   return 0; // a large enough number
 937 }
 938 
 939 //=============================================================================
 940 #ifndef PRODUCT
 941 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 942 {
 943   Compile* C = ra_->C;
 944   if (generate_vzeroupper(C)) {
 945     st->print("vzeroupper");
 946     st->cr(); st->print("\t");
 947   }
 948 
 949   int framesize = C->frame_size_in_bytes();
 950   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 951   // Remove word for return adr already pushed
 952   // and RBP
 953   framesize -= 2*wordSize;
 954 
 955   if (framesize) {
 956     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 957     st->print("\t");
 958   }
 959 
 960   st->print_cr("popq   rbp");
 961   if (do_polling() && C->is_method_compilation()) {
 962     st->print("\t");
 963     if (SafepointMechanism::uses_thread_local_poll()) {
 964       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 965                    "testl  rax, [rscratch1]\t"
 966                    "# Safepoint: poll for GC");
 967     } else if (Assembler::is_polling_page_far()) {
 968       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 969                    "testl  rax, [rscratch1]\t"
 970                    "# Safepoint: poll for GC");
 971     } else {
 972       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 973                    "# Safepoint: poll for GC");
 974     }
 975   }
 976 }
 977 #endif
 978 
 979 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 980 {
 981   Compile* C = ra_->C;
 982   MacroAssembler _masm(&cbuf);
 983 
 984   if (generate_vzeroupper(C)) {
 985     // Clear upper bits of YMM registers when current compiled code uses
 986     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 987     __ vzeroupper();
 988   }
 989 
 990   int framesize = C->frame_size_in_bytes();
 991   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 992   // Remove word for return adr already pushed
 993   // and RBP
 994   framesize -= 2*wordSize;
 995 
 996   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 997 
 998   if (framesize) {
 999     emit_opcode(cbuf, Assembler::REX_W);
1000     if (framesize < 0x80) {
1001       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
1002       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1003       emit_d8(cbuf, framesize);
1004     } else {
1005       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1006       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1007       emit_d32(cbuf, framesize);
1008     }
1009   }
1010 
1011   // popq rbp
1012   emit_opcode(cbuf, 0x58 | RBP_enc);
1013 
1014   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1015     __ reserved_stack_check();
1016   }
1017 
1018   if (do_polling() && C->is_method_compilation()) {
1019     MacroAssembler _masm(&cbuf);
1020     if (SafepointMechanism::uses_thread_local_poll()) {
1021       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1022       __ relocate(relocInfo::poll_return_type);
1023       __ testl(rax, Address(rscratch1, 0));
1024     } else {
1025       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1026       if (Assembler::is_polling_page_far()) {
1027         __ lea(rscratch1, polling_page);
1028         __ relocate(relocInfo::poll_return_type);
1029         __ testl(rax, Address(rscratch1, 0));
1030       } else {
1031         __ testl(rax, polling_page);
1032       }
1033     }
1034   }
1035 }
1036 
1037 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1038 {
1039   return MachNode::size(ra_); // too many variables; just compute it
1040                               // the hard way
1041 }
1042 
1043 int MachEpilogNode::reloc() const
1044 {
1045   return 2; // a large enough number
1046 }
1047 
1048 const Pipeline* MachEpilogNode::pipeline() const
1049 {
1050   return MachNode::pipeline_class();
1051 }
1052 
1053 int MachEpilogNode::safepoint_offset() const
1054 {
1055   return 0;
1056 }
1057 
1058 //=============================================================================
1059 
1060 enum RC {
1061   rc_bad,
1062   rc_int,
1063   rc_float,
1064   rc_stack
1065 };
1066 
1067 static enum RC rc_class(OptoReg::Name reg)
1068 {
1069   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1070 
1071   if (OptoReg::is_stack(reg)) return rc_stack;
1072 
1073   VMReg r = OptoReg::as_VMReg(reg);
1074 
1075   if (r->is_Register()) return rc_int;
1076 
1077   assert(r->is_XMMRegister(), "must be");
1078   return rc_float;
1079 }
1080 
1081 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1082 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1083                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1084 
1085 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1086                             int stack_offset, int reg, uint ireg, outputStream* st);
1087 
1088 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1089                                       int dst_offset, uint ireg, outputStream* st) {
1090   if (cbuf) {
1091     MacroAssembler _masm(cbuf);
1092     switch (ireg) {
1093     case Op_VecS:
1094       __ movq(Address(rsp, -8), rax);
1095       __ movl(rax, Address(rsp, src_offset));
1096       __ movl(Address(rsp, dst_offset), rax);
1097       __ movq(rax, Address(rsp, -8));
1098       break;
1099     case Op_VecD:
1100       __ pushq(Address(rsp, src_offset));
1101       __ popq (Address(rsp, dst_offset));
1102       break;
1103     case Op_VecX:
1104       __ pushq(Address(rsp, src_offset));
1105       __ popq (Address(rsp, dst_offset));
1106       __ pushq(Address(rsp, src_offset+8));
1107       __ popq (Address(rsp, dst_offset+8));
1108       break;
1109     case Op_VecY:
1110       __ vmovdqu(Address(rsp, -32), xmm0);
1111       __ vmovdqu(xmm0, Address(rsp, src_offset));
1112       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1113       __ vmovdqu(xmm0, Address(rsp, -32));
1114       break;
1115     case Op_VecZ:
1116       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1117       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1118       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1119       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1120       break;
1121     default:
1122       ShouldNotReachHere();
1123     }
1124 #ifndef PRODUCT
1125   } else {
1126     switch (ireg) {
1127     case Op_VecS:
1128       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1129                 "movl    rax, [rsp + #%d]\n\t"
1130                 "movl    [rsp + #%d], rax\n\t"
1131                 "movq    rax, [rsp - #8]",
1132                 src_offset, dst_offset);
1133       break;
1134     case Op_VecD:
1135       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1136                 "popq    [rsp + #%d]",
1137                 src_offset, dst_offset);
1138       break;
1139      case Op_VecX:
1140       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1141                 "popq    [rsp + #%d]\n\t"
1142                 "pushq   [rsp + #%d]\n\t"
1143                 "popq    [rsp + #%d]",
1144                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1145       break;
1146     case Op_VecY:
1147       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1148                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1149                 "vmovdqu [rsp + #%d], xmm0\n\t"
1150                 "vmovdqu xmm0, [rsp - #32]",
1151                 src_offset, dst_offset);
1152       break;
1153     case Op_VecZ:
1154       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1155                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1156                 "vmovdqu [rsp + #%d], xmm0\n\t"
1157                 "vmovdqu xmm0, [rsp - #64]",
1158                 src_offset, dst_offset);
1159       break;
1160     default:
1161       ShouldNotReachHere();
1162     }
1163 #endif
1164   }
1165 }
1166 
1167 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1168                                        PhaseRegAlloc* ra_,
1169                                        bool do_size,
1170                                        outputStream* st) const {
1171   assert(cbuf != NULL || st  != NULL, "sanity");
1172   // Get registers to move
1173   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1174   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1175   OptoReg::Name dst_second = ra_->get_reg_second(this);
1176   OptoReg::Name dst_first = ra_->get_reg_first(this);
1177 
1178   enum RC src_second_rc = rc_class(src_second);
1179   enum RC src_first_rc = rc_class(src_first);
1180   enum RC dst_second_rc = rc_class(dst_second);
1181   enum RC dst_first_rc = rc_class(dst_first);
1182 
1183   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1184          "must move at least 1 register" );
1185 
1186   if (src_first == dst_first && src_second == dst_second) {
1187     // Self copy, no move
1188     return 0;
1189   }
1190   if (bottom_type()->isa_vect() != NULL) {
1191     uint ireg = ideal_reg();
1192     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1193     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1194     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1195       // mem -> mem
1196       int src_offset = ra_->reg2offset(src_first);
1197       int dst_offset = ra_->reg2offset(dst_first);
1198       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1199     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1200       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1201     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1202       int stack_offset = ra_->reg2offset(dst_first);
1203       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1204     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1205       int stack_offset = ra_->reg2offset(src_first);
1206       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1207     } else {
1208       ShouldNotReachHere();
1209     }
1210     return 0;
1211   }
1212   if (src_first_rc == rc_stack) {
1213     // mem ->
1214     if (dst_first_rc == rc_stack) {
1215       // mem -> mem
1216       assert(src_second != dst_first, "overlap");
1217       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1218           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1219         // 64-bit
1220         int src_offset = ra_->reg2offset(src_first);
1221         int dst_offset = ra_->reg2offset(dst_first);
1222         if (cbuf) {
1223           MacroAssembler _masm(cbuf);
1224           __ pushq(Address(rsp, src_offset));
1225           __ popq (Address(rsp, dst_offset));
1226 #ifndef PRODUCT
1227         } else {
1228           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1229                     "popq    [rsp + #%d]",
1230                      src_offset, dst_offset);
1231 #endif
1232         }
1233       } else {
1234         // 32-bit
1235         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1236         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1237         // No pushl/popl, so:
1238         int src_offset = ra_->reg2offset(src_first);
1239         int dst_offset = ra_->reg2offset(dst_first);
1240         if (cbuf) {
1241           MacroAssembler _masm(cbuf);
1242           __ movq(Address(rsp, -8), rax);
1243           __ movl(rax, Address(rsp, src_offset));
1244           __ movl(Address(rsp, dst_offset), rax);
1245           __ movq(rax, Address(rsp, -8));
1246 #ifndef PRODUCT
1247         } else {
1248           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1249                     "movl    rax, [rsp + #%d]\n\t"
1250                     "movl    [rsp + #%d], rax\n\t"
1251                     "movq    rax, [rsp - #8]",
1252                      src_offset, dst_offset);
1253 #endif
1254         }
1255       }
1256       return 0;
1257     } else if (dst_first_rc == rc_int) {
1258       // mem -> gpr
1259       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1260           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1261         // 64-bit
1262         int offset = ra_->reg2offset(src_first);
1263         if (cbuf) {
1264           MacroAssembler _masm(cbuf);
1265           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("movq    %s, [rsp + #%d]\t# spill",
1269                      Matcher::regName[dst_first],
1270                      offset);
1271 #endif
1272         }
1273       } else {
1274         // 32-bit
1275         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1276         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1277         int offset = ra_->reg2offset(src_first);
1278         if (cbuf) {
1279           MacroAssembler _masm(cbuf);
1280           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1281 #ifndef PRODUCT
1282         } else {
1283           st->print("movl    %s, [rsp + #%d]\t# spill",
1284                      Matcher::regName[dst_first],
1285                      offset);
1286 #endif
1287         }
1288       }
1289       return 0;
1290     } else if (dst_first_rc == rc_float) {
1291       // mem-> xmm
1292       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1293           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1294         // 64-bit
1295         int offset = ra_->reg2offset(src_first);
1296         if (cbuf) {
1297           MacroAssembler _masm(cbuf);
1298           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1299 #ifndef PRODUCT
1300         } else {
1301           st->print("%s  %s, [rsp + #%d]\t# spill",
1302                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1303                      Matcher::regName[dst_first],
1304                      offset);
1305 #endif
1306         }
1307       } else {
1308         // 32-bit
1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1311         int offset = ra_->reg2offset(src_first);
1312         if (cbuf) {
1313           MacroAssembler _masm(cbuf);
1314           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1315 #ifndef PRODUCT
1316         } else {
1317           st->print("movss   %s, [rsp + #%d]\t# spill",
1318                      Matcher::regName[dst_first],
1319                      offset);
1320 #endif
1321         }
1322       }
1323       return 0;
1324     }
1325   } else if (src_first_rc == rc_int) {
1326     // gpr ->
1327     if (dst_first_rc == rc_stack) {
1328       // gpr -> mem
1329       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1330           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1331         // 64-bit
1332         int offset = ra_->reg2offset(dst_first);
1333         if (cbuf) {
1334           MacroAssembler _masm(cbuf);
1335           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1336 #ifndef PRODUCT
1337         } else {
1338           st->print("movq    [rsp + #%d], %s\t# spill",
1339                      offset,
1340                      Matcher::regName[src_first]);
1341 #endif
1342         }
1343       } else {
1344         // 32-bit
1345         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1346         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1347         int offset = ra_->reg2offset(dst_first);
1348         if (cbuf) {
1349           MacroAssembler _masm(cbuf);
1350           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1351 #ifndef PRODUCT
1352         } else {
1353           st->print("movl    [rsp + #%d], %s\t# spill",
1354                      offset,
1355                      Matcher::regName[src_first]);
1356 #endif
1357         }
1358       }
1359       return 0;
1360     } else if (dst_first_rc == rc_int) {
1361       // gpr -> gpr
1362       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1363           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1364         // 64-bit
1365         if (cbuf) {
1366           MacroAssembler _masm(cbuf);
1367           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1368                   as_Register(Matcher::_regEncode[src_first]));
1369 #ifndef PRODUCT
1370         } else {
1371           st->print("movq    %s, %s\t# spill",
1372                      Matcher::regName[dst_first],
1373                      Matcher::regName[src_first]);
1374 #endif
1375         }
1376         return 0;
1377       } else {
1378         // 32-bit
1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1381         if (cbuf) {
1382           MacroAssembler _masm(cbuf);
1383           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1384                   as_Register(Matcher::_regEncode[src_first]));
1385 #ifndef PRODUCT
1386         } else {
1387           st->print("movl    %s, %s\t# spill",
1388                      Matcher::regName[dst_first],
1389                      Matcher::regName[src_first]);
1390 #endif
1391         }
1392         return 0;
1393       }
1394     } else if (dst_first_rc == rc_float) {
1395       // gpr -> xmm
1396       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1397           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1398         // 64-bit
1399         if (cbuf) {
1400           MacroAssembler _masm(cbuf);
1401           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1402 #ifndef PRODUCT
1403         } else {
1404           st->print("movdq   %s, %s\t# spill",
1405                      Matcher::regName[dst_first],
1406                      Matcher::regName[src_first]);
1407 #endif
1408         }
1409       } else {
1410         // 32-bit
1411         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1412         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1413         if (cbuf) {
1414           MacroAssembler _masm(cbuf);
1415           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1416 #ifndef PRODUCT
1417         } else {
1418           st->print("movdl   %s, %s\t# spill",
1419                      Matcher::regName[dst_first],
1420                      Matcher::regName[src_first]);
1421 #endif
1422         }
1423       }
1424       return 0;
1425     }
1426   } else if (src_first_rc == rc_float) {
1427     // xmm ->
1428     if (dst_first_rc == rc_stack) {
1429       // xmm -> mem
1430       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1431           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1432         // 64-bit
1433         int offset = ra_->reg2offset(dst_first);
1434         if (cbuf) {
1435           MacroAssembler _masm(cbuf);
1436           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1437 #ifndef PRODUCT
1438         } else {
1439           st->print("movsd   [rsp + #%d], %s\t# spill",
1440                      offset,
1441                      Matcher::regName[src_first]);
1442 #endif
1443         }
1444       } else {
1445         // 32-bit
1446         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1447         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1448         int offset = ra_->reg2offset(dst_first);
1449         if (cbuf) {
1450           MacroAssembler _masm(cbuf);
1451           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1452 #ifndef PRODUCT
1453         } else {
1454           st->print("movss   [rsp + #%d], %s\t# spill",
1455                      offset,
1456                      Matcher::regName[src_first]);
1457 #endif
1458         }
1459       }
1460       return 0;
1461     } else if (dst_first_rc == rc_int) {
1462       // xmm -> gpr
1463       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1464           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1465         // 64-bit
1466         if (cbuf) {
1467           MacroAssembler _masm(cbuf);
1468           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1469 #ifndef PRODUCT
1470         } else {
1471           st->print("movdq   %s, %s\t# spill",
1472                      Matcher::regName[dst_first],
1473                      Matcher::regName[src_first]);
1474 #endif
1475         }
1476       } else {
1477         // 32-bit
1478         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1479         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1480         if (cbuf) {
1481           MacroAssembler _masm(cbuf);
1482           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1483 #ifndef PRODUCT
1484         } else {
1485           st->print("movdl   %s, %s\t# spill",
1486                      Matcher::regName[dst_first],
1487                      Matcher::regName[src_first]);
1488 #endif
1489         }
1490       }
1491       return 0;
1492     } else if (dst_first_rc == rc_float) {
1493       // xmm -> xmm
1494       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1495           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1496         // 64-bit
1497         if (cbuf) {
1498           MacroAssembler _masm(cbuf);
1499           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1500 #ifndef PRODUCT
1501         } else {
1502           st->print("%s  %s, %s\t# spill",
1503                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1504                      Matcher::regName[dst_first],
1505                      Matcher::regName[src_first]);
1506 #endif
1507         }
1508       } else {
1509         // 32-bit
1510         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1511         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1512         if (cbuf) {
1513           MacroAssembler _masm(cbuf);
1514           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1515 #ifndef PRODUCT
1516         } else {
1517           st->print("%s  %s, %s\t# spill",
1518                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1519                      Matcher::regName[dst_first],
1520                      Matcher::regName[src_first]);
1521 #endif
1522         }
1523       }
1524       return 0;
1525     }
1526   }
1527 
1528   assert(0," foo ");
1529   Unimplemented();
1530   return 0;
1531 }
1532 
1533 #ifndef PRODUCT
1534 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1535   implementation(NULL, ra_, false, st);
1536 }
1537 #endif
1538 
1539 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1540   implementation(&cbuf, ra_, false, NULL);
1541 }
1542 
1543 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1544   return MachNode::size(ra_);
1545 }
1546 
1547 //=============================================================================
1548 #ifndef PRODUCT
1549 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1550 {
1551   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1552   int reg = ra_->get_reg_first(this);
1553   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1554             Matcher::regName[reg], offset);
1555 }
1556 #endif
1557 
1558 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1559 {
1560   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1561   int reg = ra_->get_encode(this);
1562   if (offset >= 0x80) {
1563     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1564     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1565     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1566     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1567     emit_d32(cbuf, offset);
1568   } else {
1569     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1570     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1571     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1572     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1573     emit_d8(cbuf, offset);
1574   }
1575 }
1576 
1577 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1578 {
1579   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1580   return (offset < 0x80) ? 5 : 8; // REX
1581 }
1582 
1583 //=============================================================================
1584 #ifndef PRODUCT
1585 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1586 {
1587   if (UseCompressedClassPointers) {
1588     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1589     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1590     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1591   } else {
1592     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1593                  "# Inline cache check");
1594   }
1595   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1596   st->print_cr("\tnop\t# nops to align entry point");
1597 }
1598 #endif
1599 
1600 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1601 {
1602   MacroAssembler masm(&cbuf);
1603   uint insts_size = cbuf.insts_size();
1604   if (UseCompressedClassPointers) {
1605     masm.load_klass(rscratch1, j_rarg0);
1606     masm.cmpptr(rax, rscratch1);
1607   } else {
1608     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1609   }
1610 
1611   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1612 
1613   /* WARNING these NOPs are critical so that verified entry point is properly
1614      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1615   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1616   if (OptoBreakpoint) {
1617     // Leave space for int3
1618     nops_cnt -= 1;
1619   }
1620   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1621   if (nops_cnt > 0)
1622     masm.nop(nops_cnt);
1623 }
1624 
1625 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1626 {
1627   return MachNode::size(ra_); // too many variables; just compute it
1628                               // the hard way
1629 }
1630 
1631 
1632 //=============================================================================
1633 
1634 int Matcher::regnum_to_fpu_offset(int regnum)
1635 {
1636   return regnum - 32; // The FP registers are in the second chunk
1637 }
1638 
1639 // This is UltraSparc specific, true just means we have fast l2f conversion
1640 const bool Matcher::convL2FSupported(void) {
1641   return true;
1642 }
1643 
1644 // Is this branch offset short enough that a short branch can be used?
1645 //
1646 // NOTE: If the platform does not provide any short branch variants, then
1647 //       this method should return false for offset 0.
1648 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1649   // The passed offset is relative to address of the branch.
1650   // On 86 a branch displacement is calculated relative to address
1651   // of a next instruction.
1652   offset -= br_size;
1653 
1654   // the short version of jmpConUCF2 contains multiple branches,
1655   // making the reach slightly less
1656   if (rule == jmpConUCF2_rule)
1657     return (-126 <= offset && offset <= 125);
1658   return (-128 <= offset && offset <= 127);
1659 }
1660 
1661 const bool Matcher::isSimpleConstant64(jlong value) {
1662   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1663   //return value == (int) value;  // Cf. storeImmL and immL32.
1664 
1665   // Probably always true, even if a temp register is required.
1666   return true;
1667 }
1668 
1669 // The ecx parameter to rep stosq for the ClearArray node is in words.
1670 const bool Matcher::init_array_count_is_in_bytes = false;
1671 
1672 // No additional cost for CMOVL.
1673 const int Matcher::long_cmove_cost() { return 0; }
1674 
1675 // No CMOVF/CMOVD with SSE2
1676 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1677 
1678 // Does the CPU require late expand (see block.cpp for description of late expand)?
1679 const bool Matcher::require_postalloc_expand = false;
1680 
1681 // Do we need to mask the count passed to shift instructions or does
1682 // the cpu only look at the lower 5/6 bits anyway?
1683 const bool Matcher::need_masked_shift_count = false;
1684 
1685 bool Matcher::narrow_oop_use_complex_address() {
1686   assert(UseCompressedOops, "only for compressed oops code");
1687   return (LogMinObjAlignmentInBytes <= 3);
1688 }
1689 
1690 bool Matcher::narrow_klass_use_complex_address() {
1691   assert(UseCompressedClassPointers, "only for compressed klass code");
1692   return (LogKlassAlignmentInBytes <= 3);
1693 }
1694 
1695 bool Matcher::const_oop_prefer_decode() {
1696   // Prefer ConN+DecodeN over ConP.
1697   return true;
1698 }
1699 
1700 bool Matcher::const_klass_prefer_decode() {
1701   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1702   //       or condisider the following:
1703   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1704   //return Universe::narrow_klass_base() == NULL;
1705   return true;
1706 }
1707 
1708 // Is it better to copy float constants, or load them directly from
1709 // memory?  Intel can load a float constant from a direct address,
1710 // requiring no extra registers.  Most RISCs will have to materialize
1711 // an address into a register first, so they would do better to copy
1712 // the constant from stack.
1713 const bool Matcher::rematerialize_float_constants = true; // XXX
1714 
1715 // If CPU can load and store mis-aligned doubles directly then no
1716 // fixup is needed.  Else we split the double into 2 integer pieces
1717 // and move it piece-by-piece.  Only happens when passing doubles into
1718 // C code as the Java calling convention forces doubles to be aligned.
1719 const bool Matcher::misaligned_doubles_ok = true;
1720 
1721 // No-op on amd64
1722 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1723 
1724 // Advertise here if the CPU requires explicit rounding operations to
1725 // implement the UseStrictFP mode.
1726 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1727 
1728 // Are floats conerted to double when stored to stack during deoptimization?
1729 // On x64 it is stored without convertion so we can use normal access.
1730 bool Matcher::float_in_double() { return false; }
1731 
1732 // Do ints take an entire long register or just half?
1733 const bool Matcher::int_in_long = true;
1734 
1735 // Return whether or not this register is ever used as an argument.
1736 // This function is used on startup to build the trampoline stubs in
1737 // generateOptoStub.  Registers not mentioned will be killed by the VM
1738 // call in the trampoline, and arguments in those registers not be
1739 // available to the callee.
1740 bool Matcher::can_be_java_arg(int reg)
1741 {
1742   return
1743     reg ==  RDI_num || reg == RDI_H_num ||
1744     reg ==  RSI_num || reg == RSI_H_num ||
1745     reg ==  RDX_num || reg == RDX_H_num ||
1746     reg ==  RCX_num || reg == RCX_H_num ||
1747     reg ==   R8_num || reg ==  R8_H_num ||
1748     reg ==   R9_num || reg ==  R9_H_num ||
1749     reg ==  R12_num || reg == R12_H_num ||
1750     reg == XMM0_num || reg == XMM0b_num ||
1751     reg == XMM1_num || reg == XMM1b_num ||
1752     reg == XMM2_num || reg == XMM2b_num ||
1753     reg == XMM3_num || reg == XMM3b_num ||
1754     reg == XMM4_num || reg == XMM4b_num ||
1755     reg == XMM5_num || reg == XMM5b_num ||
1756     reg == XMM6_num || reg == XMM6b_num ||
1757     reg == XMM7_num || reg == XMM7b_num;
1758 }
1759 
1760 bool Matcher::is_spillable_arg(int reg)
1761 {
1762   return can_be_java_arg(reg);
1763 }
1764 
1765 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1766   // In 64 bit mode a code which use multiply when
1767   // devisor is constant is faster than hardware
1768   // DIV instruction (it uses MulHiL).
1769   return false;
1770 }
1771 
1772 // Register for DIVI projection of divmodI
1773 RegMask Matcher::divI_proj_mask() {
1774   return INT_RAX_REG_mask();
1775 }
1776 
1777 // Register for MODI projection of divmodI
1778 RegMask Matcher::modI_proj_mask() {
1779   return INT_RDX_REG_mask();
1780 }
1781 
1782 // Register for DIVL projection of divmodL
1783 RegMask Matcher::divL_proj_mask() {
1784   return LONG_RAX_REG_mask();
1785 }
1786 
1787 // Register for MODL projection of divmodL
1788 RegMask Matcher::modL_proj_mask() {
1789   return LONG_RDX_REG_mask();
1790 }
1791 
1792 // Register for saving SP into on method handle invokes. Not used on x86_64.
1793 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1794     return NO_REG_mask();
1795 }
1796 
1797 %}
1798 
1799 //----------ENCODING BLOCK-----------------------------------------------------
1800 // This block specifies the encoding classes used by the compiler to
1801 // output byte streams.  Encoding classes are parameterized macros
1802 // used by Machine Instruction Nodes in order to generate the bit
1803 // encoding of the instruction.  Operands specify their base encoding
1804 // interface with the interface keyword.  There are currently
1805 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1806 // COND_INTER.  REG_INTER causes an operand to generate a function
1807 // which returns its register number when queried.  CONST_INTER causes
1808 // an operand to generate a function which returns the value of the
1809 // constant when queried.  MEMORY_INTER causes an operand to generate
1810 // four functions which return the Base Register, the Index Register,
1811 // the Scale Value, and the Offset Value of the operand when queried.
1812 // COND_INTER causes an operand to generate six functions which return
1813 // the encoding code (ie - encoding bits for the instruction)
1814 // associated with each basic boolean condition for a conditional
1815 // instruction.
1816 //
1817 // Instructions specify two basic values for encoding.  Again, a
1818 // function is available to check if the constant displacement is an
1819 // oop. They use the ins_encode keyword to specify their encoding
1820 // classes (which must be a sequence of enc_class names, and their
1821 // parameters, specified in the encoding block), and they use the
1822 // opcode keyword to specify, in order, their primary, secondary, and
1823 // tertiary opcode.  Only the opcode sections which a particular
1824 // instruction needs for encoding need to be specified.
1825 encode %{
1826   // Build emit functions for each basic byte or larger field in the
1827   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1828   // from C++ code in the enc_class source block.  Emit functions will
1829   // live in the main source block for now.  In future, we can
1830   // generalize this by adding a syntax that specifies the sizes of
1831   // fields in an order, so that the adlc can build the emit functions
1832   // automagically
1833 
1834   // Emit primary opcode
1835   enc_class OpcP
1836   %{
1837     emit_opcode(cbuf, $primary);
1838   %}
1839 
1840   // Emit secondary opcode
1841   enc_class OpcS
1842   %{
1843     emit_opcode(cbuf, $secondary);
1844   %}
1845 
1846   // Emit tertiary opcode
1847   enc_class OpcT
1848   %{
1849     emit_opcode(cbuf, $tertiary);
1850   %}
1851 
1852   // Emit opcode directly
1853   enc_class Opcode(immI d8)
1854   %{
1855     emit_opcode(cbuf, $d8$$constant);
1856   %}
1857 
1858   // Emit size prefix
1859   enc_class SizePrefix
1860   %{
1861     emit_opcode(cbuf, 0x66);
1862   %}
1863 
1864   enc_class reg(rRegI reg)
1865   %{
1866     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1867   %}
1868 
1869   enc_class reg_reg(rRegI dst, rRegI src)
1870   %{
1871     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1872   %}
1873 
1874   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1875   %{
1876     emit_opcode(cbuf, $opcode$$constant);
1877     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1878   %}
1879 
1880   enc_class cdql_enc(no_rax_rdx_RegI div)
1881   %{
1882     // Full implementation of Java idiv and irem; checks for
1883     // special case as described in JVM spec., p.243 & p.271.
1884     //
1885     //         normal case                           special case
1886     //
1887     // input : rax: dividend                         min_int
1888     //         reg: divisor                          -1
1889     //
1890     // output: rax: quotient  (= rax idiv reg)       min_int
1891     //         rdx: remainder (= rax irem reg)       0
1892     //
1893     //  Code sequnce:
1894     //
1895     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1896     //    5:   75 07/08                jne    e <normal>
1897     //    7:   33 d2                   xor    %edx,%edx
1898     //  [div >= 8 -> offset + 1]
1899     //  [REX_B]
1900     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1901     //    c:   74 03/04                je     11 <done>
1902     // 000000000000000e <normal>:
1903     //    e:   99                      cltd
1904     //  [div >= 8 -> offset + 1]
1905     //  [REX_B]
1906     //    f:   f7 f9                   idiv   $div
1907     // 0000000000000011 <done>:
1908 
1909     // cmp    $0x80000000,%eax
1910     emit_opcode(cbuf, 0x3d);
1911     emit_d8(cbuf, 0x00);
1912     emit_d8(cbuf, 0x00);
1913     emit_d8(cbuf, 0x00);
1914     emit_d8(cbuf, 0x80);
1915 
1916     // jne    e <normal>
1917     emit_opcode(cbuf, 0x75);
1918     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1919 
1920     // xor    %edx,%edx
1921     emit_opcode(cbuf, 0x33);
1922     emit_d8(cbuf, 0xD2);
1923 
1924     // cmp    $0xffffffffffffffff,%ecx
1925     if ($div$$reg >= 8) {
1926       emit_opcode(cbuf, Assembler::REX_B);
1927     }
1928     emit_opcode(cbuf, 0x83);
1929     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1930     emit_d8(cbuf, 0xFF);
1931 
1932     // je     11 <done>
1933     emit_opcode(cbuf, 0x74);
1934     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1935 
1936     // <normal>
1937     // cltd
1938     emit_opcode(cbuf, 0x99);
1939 
1940     // idivl (note: must be emitted by the user of this rule)
1941     // <done>
1942   %}
1943 
1944   enc_class cdqq_enc(no_rax_rdx_RegL div)
1945   %{
1946     // Full implementation of Java ldiv and lrem; checks for
1947     // special case as described in JVM spec., p.243 & p.271.
1948     //
1949     //         normal case                           special case
1950     //
1951     // input : rax: dividend                         min_long
1952     //         reg: divisor                          -1
1953     //
1954     // output: rax: quotient  (= rax idiv reg)       min_long
1955     //         rdx: remainder (= rax irem reg)       0
1956     //
1957     //  Code sequnce:
1958     //
1959     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1960     //    7:   00 00 80
1961     //    a:   48 39 d0                cmp    %rdx,%rax
1962     //    d:   75 08                   jne    17 <normal>
1963     //    f:   33 d2                   xor    %edx,%edx
1964     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1965     //   15:   74 05                   je     1c <done>
1966     // 0000000000000017 <normal>:
1967     //   17:   48 99                   cqto
1968     //   19:   48 f7 f9                idiv   $div
1969     // 000000000000001c <done>:
1970 
1971     // mov    $0x8000000000000000,%rdx
1972     emit_opcode(cbuf, Assembler::REX_W);
1973     emit_opcode(cbuf, 0xBA);
1974     emit_d8(cbuf, 0x00);
1975     emit_d8(cbuf, 0x00);
1976     emit_d8(cbuf, 0x00);
1977     emit_d8(cbuf, 0x00);
1978     emit_d8(cbuf, 0x00);
1979     emit_d8(cbuf, 0x00);
1980     emit_d8(cbuf, 0x00);
1981     emit_d8(cbuf, 0x80);
1982 
1983     // cmp    %rdx,%rax
1984     emit_opcode(cbuf, Assembler::REX_W);
1985     emit_opcode(cbuf, 0x39);
1986     emit_d8(cbuf, 0xD0);
1987 
1988     // jne    17 <normal>
1989     emit_opcode(cbuf, 0x75);
1990     emit_d8(cbuf, 0x08);
1991 
1992     // xor    %edx,%edx
1993     emit_opcode(cbuf, 0x33);
1994     emit_d8(cbuf, 0xD2);
1995 
1996     // cmp    $0xffffffffffffffff,$div
1997     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1998     emit_opcode(cbuf, 0x83);
1999     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2000     emit_d8(cbuf, 0xFF);
2001 
2002     // je     1e <done>
2003     emit_opcode(cbuf, 0x74);
2004     emit_d8(cbuf, 0x05);
2005 
2006     // <normal>
2007     // cqto
2008     emit_opcode(cbuf, Assembler::REX_W);
2009     emit_opcode(cbuf, 0x99);
2010 
2011     // idivq (note: must be emitted by the user of this rule)
2012     // <done>
2013   %}
2014 
2015   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2016   enc_class OpcSE(immI imm)
2017   %{
2018     // Emit primary opcode and set sign-extend bit
2019     // Check for 8-bit immediate, and set sign extend bit in opcode
2020     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2021       emit_opcode(cbuf, $primary | 0x02);
2022     } else {
2023       // 32-bit immediate
2024       emit_opcode(cbuf, $primary);
2025     }
2026   %}
2027 
2028   enc_class OpcSErm(rRegI dst, immI imm)
2029   %{
2030     // OpcSEr/m
2031     int dstenc = $dst$$reg;
2032     if (dstenc >= 8) {
2033       emit_opcode(cbuf, Assembler::REX_B);
2034       dstenc -= 8;
2035     }
2036     // Emit primary opcode and set sign-extend bit
2037     // Check for 8-bit immediate, and set sign extend bit in opcode
2038     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2039       emit_opcode(cbuf, $primary | 0x02);
2040     } else {
2041       // 32-bit immediate
2042       emit_opcode(cbuf, $primary);
2043     }
2044     // Emit r/m byte with secondary opcode, after primary opcode.
2045     emit_rm(cbuf, 0x3, $secondary, dstenc);
2046   %}
2047 
2048   enc_class OpcSErm_wide(rRegL dst, immI imm)
2049   %{
2050     // OpcSEr/m
2051     int dstenc = $dst$$reg;
2052     if (dstenc < 8) {
2053       emit_opcode(cbuf, Assembler::REX_W);
2054     } else {
2055       emit_opcode(cbuf, Assembler::REX_WB);
2056       dstenc -= 8;
2057     }
2058     // Emit primary opcode and set sign-extend bit
2059     // Check for 8-bit immediate, and set sign extend bit in opcode
2060     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2061       emit_opcode(cbuf, $primary | 0x02);
2062     } else {
2063       // 32-bit immediate
2064       emit_opcode(cbuf, $primary);
2065     }
2066     // Emit r/m byte with secondary opcode, after primary opcode.
2067     emit_rm(cbuf, 0x3, $secondary, dstenc);
2068   %}
2069 
2070   enc_class Con8or32(immI imm)
2071   %{
2072     // Check for 8-bit immediate, and set sign extend bit in opcode
2073     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2074       $$$emit8$imm$$constant;
2075     } else {
2076       // 32-bit immediate
2077       $$$emit32$imm$$constant;
2078     }
2079   %}
2080 
2081   enc_class opc2_reg(rRegI dst)
2082   %{
2083     // BSWAP
2084     emit_cc(cbuf, $secondary, $dst$$reg);
2085   %}
2086 
2087   enc_class opc3_reg(rRegI dst)
2088   %{
2089     // BSWAP
2090     emit_cc(cbuf, $tertiary, $dst$$reg);
2091   %}
2092 
2093   enc_class reg_opc(rRegI div)
2094   %{
2095     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2096     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2097   %}
2098 
2099   enc_class enc_cmov(cmpOp cop)
2100   %{
2101     // CMOV
2102     $$$emit8$primary;
2103     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2104   %}
2105 
2106   enc_class enc_PartialSubtypeCheck()
2107   %{
2108     Register Rrdi = as_Register(RDI_enc); // result register
2109     Register Rrax = as_Register(RAX_enc); // super class
2110     Register Rrcx = as_Register(RCX_enc); // killed
2111     Register Rrsi = as_Register(RSI_enc); // sub class
2112     Label miss;
2113     const bool set_cond_codes = true;
2114 
2115     MacroAssembler _masm(&cbuf);
2116     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2117                                      NULL, &miss,
2118                                      /*set_cond_codes:*/ true);
2119     if ($primary) {
2120       __ xorptr(Rrdi, Rrdi);
2121     }
2122     __ bind(miss);
2123   %}
2124 
2125   enc_class clear_avx %{
2126     debug_only(int off0 = cbuf.insts_size());
2127     if (generate_vzeroupper(Compile::current())) {
2128       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2129       // Clear upper bits of YMM registers when current compiled code uses
2130       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2131       MacroAssembler _masm(&cbuf);
2132       __ vzeroupper();
2133     }
2134     debug_only(int off1 = cbuf.insts_size());
2135     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2136   %}
2137 
2138   enc_class Java_To_Runtime(method meth) %{
2139     // No relocation needed
2140     MacroAssembler _masm(&cbuf);
2141     __ mov64(r10, (int64_t) $meth$$method);
2142     __ call(r10);
2143   %}
2144 
2145   enc_class Java_To_Interpreter(method meth)
2146   %{
2147     // CALL Java_To_Interpreter
2148     // This is the instruction starting address for relocation info.
2149     cbuf.set_insts_mark();
2150     $$$emit8$primary;
2151     // CALL directly to the runtime
2152     emit_d32_reloc(cbuf,
2153                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2154                    runtime_call_Relocation::spec(),
2155                    RELOC_DISP32);
2156   %}
2157 
2158   enc_class Java_Static_Call(method meth)
2159   %{
2160     // JAVA STATIC CALL
2161     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2162     // determine who we intended to call.
2163     cbuf.set_insts_mark();
2164     $$$emit8$primary;
2165 
2166     if (!_method) {
2167       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2168                      runtime_call_Relocation::spec(),
2169                      RELOC_DISP32);
2170     } else {
2171       int method_index = resolved_method_index(cbuf);
2172       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2173                                                   : static_call_Relocation::spec(method_index);
2174       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2175                      rspec, RELOC_DISP32);
2176       // Emit stubs for static call.
2177       address mark = cbuf.insts_mark();
2178       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2179       if (stub == NULL) {
2180         ciEnv::current()->record_failure("CodeCache is full");
2181         return;
2182       }
2183 #if INCLUDE_AOT
2184       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2185 #endif
2186     }
2187   %}
2188 
2189   enc_class Java_Dynamic_Call(method meth) %{
2190     MacroAssembler _masm(&cbuf);
2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2192   %}
2193 
2194   enc_class Java_Compiled_Call(method meth)
2195   %{
2196     // JAVA COMPILED CALL
2197     int disp = in_bytes(Method:: from_compiled_offset());
2198 
2199     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2200     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2201 
2202     // callq *disp(%rax)
2203     cbuf.set_insts_mark();
2204     $$$emit8$primary;
2205     if (disp < 0x80) {
2206       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2207       emit_d8(cbuf, disp); // Displacement
2208     } else {
2209       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2210       emit_d32(cbuf, disp); // Displacement
2211     }
2212   %}
2213 
2214   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2215   %{
2216     // SAL, SAR, SHR
2217     int dstenc = $dst$$reg;
2218     if (dstenc >= 8) {
2219       emit_opcode(cbuf, Assembler::REX_B);
2220       dstenc -= 8;
2221     }
2222     $$$emit8$primary;
2223     emit_rm(cbuf, 0x3, $secondary, dstenc);
2224     $$$emit8$shift$$constant;
2225   %}
2226 
2227   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2228   %{
2229     // SAL, SAR, SHR
2230     int dstenc = $dst$$reg;
2231     if (dstenc < 8) {
2232       emit_opcode(cbuf, Assembler::REX_W);
2233     } else {
2234       emit_opcode(cbuf, Assembler::REX_WB);
2235       dstenc -= 8;
2236     }
2237     $$$emit8$primary;
2238     emit_rm(cbuf, 0x3, $secondary, dstenc);
2239     $$$emit8$shift$$constant;
2240   %}
2241 
2242   enc_class load_immI(rRegI dst, immI src)
2243   %{
2244     int dstenc = $dst$$reg;
2245     if (dstenc >= 8) {
2246       emit_opcode(cbuf, Assembler::REX_B);
2247       dstenc -= 8;
2248     }
2249     emit_opcode(cbuf, 0xB8 | dstenc);
2250     $$$emit32$src$$constant;
2251   %}
2252 
2253   enc_class load_immL(rRegL dst, immL src)
2254   %{
2255     int dstenc = $dst$$reg;
2256     if (dstenc < 8) {
2257       emit_opcode(cbuf, Assembler::REX_W);
2258     } else {
2259       emit_opcode(cbuf, Assembler::REX_WB);
2260       dstenc -= 8;
2261     }
2262     emit_opcode(cbuf, 0xB8 | dstenc);
2263     emit_d64(cbuf, $src$$constant);
2264   %}
2265 
2266   enc_class load_immUL32(rRegL dst, immUL32 src)
2267   %{
2268     // same as load_immI, but this time we care about zeroes in the high word
2269     int dstenc = $dst$$reg;
2270     if (dstenc >= 8) {
2271       emit_opcode(cbuf, Assembler::REX_B);
2272       dstenc -= 8;
2273     }
2274     emit_opcode(cbuf, 0xB8 | dstenc);
2275     $$$emit32$src$$constant;
2276   %}
2277 
2278   enc_class load_immL32(rRegL dst, immL32 src)
2279   %{
2280     int dstenc = $dst$$reg;
2281     if (dstenc < 8) {
2282       emit_opcode(cbuf, Assembler::REX_W);
2283     } else {
2284       emit_opcode(cbuf, Assembler::REX_WB);
2285       dstenc -= 8;
2286     }
2287     emit_opcode(cbuf, 0xC7);
2288     emit_rm(cbuf, 0x03, 0x00, dstenc);
2289     $$$emit32$src$$constant;
2290   %}
2291 
2292   enc_class load_immP31(rRegP dst, immP32 src)
2293   %{
2294     // same as load_immI, but this time we care about zeroes in the high word
2295     int dstenc = $dst$$reg;
2296     if (dstenc >= 8) {
2297       emit_opcode(cbuf, Assembler::REX_B);
2298       dstenc -= 8;
2299     }
2300     emit_opcode(cbuf, 0xB8 | dstenc);
2301     $$$emit32$src$$constant;
2302   %}
2303 
2304   enc_class load_immP(rRegP dst, immP src)
2305   %{
2306     int dstenc = $dst$$reg;
2307     if (dstenc < 8) {
2308       emit_opcode(cbuf, Assembler::REX_W);
2309     } else {
2310       emit_opcode(cbuf, Assembler::REX_WB);
2311       dstenc -= 8;
2312     }
2313     emit_opcode(cbuf, 0xB8 | dstenc);
2314     // This next line should be generated from ADLC
2315     if ($src->constant_reloc() != relocInfo::none) {
2316       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2317     } else {
2318       emit_d64(cbuf, $src$$constant);
2319     }
2320   %}
2321 
2322   enc_class Con32(immI src)
2323   %{
2324     // Output immediate
2325     $$$emit32$src$$constant;
2326   %}
2327 
2328   enc_class Con32F_as_bits(immF src)
2329   %{
2330     // Output Float immediate bits
2331     jfloat jf = $src$$constant;
2332     jint jf_as_bits = jint_cast(jf);
2333     emit_d32(cbuf, jf_as_bits);
2334   %}
2335 
2336   enc_class Con16(immI src)
2337   %{
2338     // Output immediate
2339     $$$emit16$src$$constant;
2340   %}
2341 
2342   // How is this different from Con32??? XXX
2343   enc_class Con_d32(immI src)
2344   %{
2345     emit_d32(cbuf,$src$$constant);
2346   %}
2347 
2348   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2349     // Output immediate memory reference
2350     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2351     emit_d32(cbuf, 0x00);
2352   %}
2353 
2354   enc_class lock_prefix()
2355   %{
2356     if (os::is_MP()) {
2357       emit_opcode(cbuf, 0xF0); // lock
2358     }
2359   %}
2360 
2361   enc_class REX_mem(memory mem)
2362   %{
2363     if ($mem$$base >= 8) {
2364       if ($mem$$index < 8) {
2365         emit_opcode(cbuf, Assembler::REX_B);
2366       } else {
2367         emit_opcode(cbuf, Assembler::REX_XB);
2368       }
2369     } else {
2370       if ($mem$$index >= 8) {
2371         emit_opcode(cbuf, Assembler::REX_X);
2372       }
2373     }
2374   %}
2375 
2376   enc_class REX_mem_wide(memory mem)
2377   %{
2378     if ($mem$$base >= 8) {
2379       if ($mem$$index < 8) {
2380         emit_opcode(cbuf, Assembler::REX_WB);
2381       } else {
2382         emit_opcode(cbuf, Assembler::REX_WXB);
2383       }
2384     } else {
2385       if ($mem$$index < 8) {
2386         emit_opcode(cbuf, Assembler::REX_W);
2387       } else {
2388         emit_opcode(cbuf, Assembler::REX_WX);
2389       }
2390     }
2391   %}
2392 
2393   // for byte regs
2394   enc_class REX_breg(rRegI reg)
2395   %{
2396     if ($reg$$reg >= 4) {
2397       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2398     }
2399   %}
2400 
2401   // for byte regs
2402   enc_class REX_reg_breg(rRegI dst, rRegI src)
2403   %{
2404     if ($dst$$reg < 8) {
2405       if ($src$$reg >= 4) {
2406         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2407       }
2408     } else {
2409       if ($src$$reg < 8) {
2410         emit_opcode(cbuf, Assembler::REX_R);
2411       } else {
2412         emit_opcode(cbuf, Assembler::REX_RB);
2413       }
2414     }
2415   %}
2416 
2417   // for byte regs
2418   enc_class REX_breg_mem(rRegI reg, memory mem)
2419   %{
2420     if ($reg$$reg < 8) {
2421       if ($mem$$base < 8) {
2422         if ($mem$$index >= 8) {
2423           emit_opcode(cbuf, Assembler::REX_X);
2424         } else if ($reg$$reg >= 4) {
2425           emit_opcode(cbuf, Assembler::REX);
2426         }
2427       } else {
2428         if ($mem$$index < 8) {
2429           emit_opcode(cbuf, Assembler::REX_B);
2430         } else {
2431           emit_opcode(cbuf, Assembler::REX_XB);
2432         }
2433       }
2434     } else {
2435       if ($mem$$base < 8) {
2436         if ($mem$$index < 8) {
2437           emit_opcode(cbuf, Assembler::REX_R);
2438         } else {
2439           emit_opcode(cbuf, Assembler::REX_RX);
2440         }
2441       } else {
2442         if ($mem$$index < 8) {
2443           emit_opcode(cbuf, Assembler::REX_RB);
2444         } else {
2445           emit_opcode(cbuf, Assembler::REX_RXB);
2446         }
2447       }
2448     }
2449   %}
2450 
2451   enc_class REX_reg(rRegI reg)
2452   %{
2453     if ($reg$$reg >= 8) {
2454       emit_opcode(cbuf, Assembler::REX_B);
2455     }
2456   %}
2457 
2458   enc_class REX_reg_wide(rRegI reg)
2459   %{
2460     if ($reg$$reg < 8) {
2461       emit_opcode(cbuf, Assembler::REX_W);
2462     } else {
2463       emit_opcode(cbuf, Assembler::REX_WB);
2464     }
2465   %}
2466 
2467   enc_class REX_reg_reg(rRegI dst, rRegI src)
2468   %{
2469     if ($dst$$reg < 8) {
2470       if ($src$$reg >= 8) {
2471         emit_opcode(cbuf, Assembler::REX_B);
2472       }
2473     } else {
2474       if ($src$$reg < 8) {
2475         emit_opcode(cbuf, Assembler::REX_R);
2476       } else {
2477         emit_opcode(cbuf, Assembler::REX_RB);
2478       }
2479     }
2480   %}
2481 
2482   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2483   %{
2484     if ($dst$$reg < 8) {
2485       if ($src$$reg < 8) {
2486         emit_opcode(cbuf, Assembler::REX_W);
2487       } else {
2488         emit_opcode(cbuf, Assembler::REX_WB);
2489       }
2490     } else {
2491       if ($src$$reg < 8) {
2492         emit_opcode(cbuf, Assembler::REX_WR);
2493       } else {
2494         emit_opcode(cbuf, Assembler::REX_WRB);
2495       }
2496     }
2497   %}
2498 
2499   enc_class REX_reg_mem(rRegI reg, memory mem)
2500   %{
2501     if ($reg$$reg < 8) {
2502       if ($mem$$base < 8) {
2503         if ($mem$$index >= 8) {
2504           emit_opcode(cbuf, Assembler::REX_X);
2505         }
2506       } else {
2507         if ($mem$$index < 8) {
2508           emit_opcode(cbuf, Assembler::REX_B);
2509         } else {
2510           emit_opcode(cbuf, Assembler::REX_XB);
2511         }
2512       }
2513     } else {
2514       if ($mem$$base < 8) {
2515         if ($mem$$index < 8) {
2516           emit_opcode(cbuf, Assembler::REX_R);
2517         } else {
2518           emit_opcode(cbuf, Assembler::REX_RX);
2519         }
2520       } else {
2521         if ($mem$$index < 8) {
2522           emit_opcode(cbuf, Assembler::REX_RB);
2523         } else {
2524           emit_opcode(cbuf, Assembler::REX_RXB);
2525         }
2526       }
2527     }
2528   %}
2529 
2530   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2531   %{
2532     if ($reg$$reg < 8) {
2533       if ($mem$$base < 8) {
2534         if ($mem$$index < 8) {
2535           emit_opcode(cbuf, Assembler::REX_W);
2536         } else {
2537           emit_opcode(cbuf, Assembler::REX_WX);
2538         }
2539       } else {
2540         if ($mem$$index < 8) {
2541           emit_opcode(cbuf, Assembler::REX_WB);
2542         } else {
2543           emit_opcode(cbuf, Assembler::REX_WXB);
2544         }
2545       }
2546     } else {
2547       if ($mem$$base < 8) {
2548         if ($mem$$index < 8) {
2549           emit_opcode(cbuf, Assembler::REX_WR);
2550         } else {
2551           emit_opcode(cbuf, Assembler::REX_WRX);
2552         }
2553       } else {
2554         if ($mem$$index < 8) {
2555           emit_opcode(cbuf, Assembler::REX_WRB);
2556         } else {
2557           emit_opcode(cbuf, Assembler::REX_WRXB);
2558         }
2559       }
2560     }
2561   %}
2562 
2563   enc_class reg_mem(rRegI ereg, memory mem)
2564   %{
2565     // High registers handle in encode_RegMem
2566     int reg = $ereg$$reg;
2567     int base = $mem$$base;
2568     int index = $mem$$index;
2569     int scale = $mem$$scale;
2570     int disp = $mem$$disp;
2571     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2572 
2573     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2574   %}
2575 
2576   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2577   %{
2578     int rm_byte_opcode = $rm_opcode$$constant;
2579 
2580     // High registers handle in encode_RegMem
2581     int base = $mem$$base;
2582     int index = $mem$$index;
2583     int scale = $mem$$scale;
2584     int displace = $mem$$disp;
2585 
2586     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2587                                             // working with static
2588                                             // globals
2589     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2590                   disp_reloc);
2591   %}
2592 
2593   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2594   %{
2595     int reg_encoding = $dst$$reg;
2596     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2597     int index        = 0x04;            // 0x04 indicates no index
2598     int scale        = 0x00;            // 0x00 indicates no scale
2599     int displace     = $src1$$constant; // 0x00 indicates no displacement
2600     relocInfo::relocType disp_reloc = relocInfo::none;
2601     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2602                   disp_reloc);
2603   %}
2604 
2605   enc_class neg_reg(rRegI dst)
2606   %{
2607     int dstenc = $dst$$reg;
2608     if (dstenc >= 8) {
2609       emit_opcode(cbuf, Assembler::REX_B);
2610       dstenc -= 8;
2611     }
2612     // NEG $dst
2613     emit_opcode(cbuf, 0xF7);
2614     emit_rm(cbuf, 0x3, 0x03, dstenc);
2615   %}
2616 
2617   enc_class neg_reg_wide(rRegI dst)
2618   %{
2619     int dstenc = $dst$$reg;
2620     if (dstenc < 8) {
2621       emit_opcode(cbuf, Assembler::REX_W);
2622     } else {
2623       emit_opcode(cbuf, Assembler::REX_WB);
2624       dstenc -= 8;
2625     }
2626     // NEG $dst
2627     emit_opcode(cbuf, 0xF7);
2628     emit_rm(cbuf, 0x3, 0x03, dstenc);
2629   %}
2630 
2631   enc_class setLT_reg(rRegI dst)
2632   %{
2633     int dstenc = $dst$$reg;
2634     if (dstenc >= 8) {
2635       emit_opcode(cbuf, Assembler::REX_B);
2636       dstenc -= 8;
2637     } else if (dstenc >= 4) {
2638       emit_opcode(cbuf, Assembler::REX);
2639     }
2640     // SETLT $dst
2641     emit_opcode(cbuf, 0x0F);
2642     emit_opcode(cbuf, 0x9C);
2643     emit_rm(cbuf, 0x3, 0x0, dstenc);
2644   %}
2645 
2646   enc_class setNZ_reg(rRegI dst)
2647   %{
2648     int dstenc = $dst$$reg;
2649     if (dstenc >= 8) {
2650       emit_opcode(cbuf, Assembler::REX_B);
2651       dstenc -= 8;
2652     } else if (dstenc >= 4) {
2653       emit_opcode(cbuf, Assembler::REX);
2654     }
2655     // SETNZ $dst
2656     emit_opcode(cbuf, 0x0F);
2657     emit_opcode(cbuf, 0x95);
2658     emit_rm(cbuf, 0x3, 0x0, dstenc);
2659   %}
2660 
2661 
2662   // Compare the lonogs and set -1, 0, or 1 into dst
2663   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2664   %{
2665     int src1enc = $src1$$reg;
2666     int src2enc = $src2$$reg;
2667     int dstenc = $dst$$reg;
2668 
2669     // cmpq $src1, $src2
2670     if (src1enc < 8) {
2671       if (src2enc < 8) {
2672         emit_opcode(cbuf, Assembler::REX_W);
2673       } else {
2674         emit_opcode(cbuf, Assembler::REX_WB);
2675       }
2676     } else {
2677       if (src2enc < 8) {
2678         emit_opcode(cbuf, Assembler::REX_WR);
2679       } else {
2680         emit_opcode(cbuf, Assembler::REX_WRB);
2681       }
2682     }
2683     emit_opcode(cbuf, 0x3B);
2684     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2685 
2686     // movl $dst, -1
2687     if (dstenc >= 8) {
2688       emit_opcode(cbuf, Assembler::REX_B);
2689     }
2690     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2691     emit_d32(cbuf, -1);
2692 
2693     // jl,s done
2694     emit_opcode(cbuf, 0x7C);
2695     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2696 
2697     // setne $dst
2698     if (dstenc >= 4) {
2699       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2700     }
2701     emit_opcode(cbuf, 0x0F);
2702     emit_opcode(cbuf, 0x95);
2703     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2704 
2705     // movzbl $dst, $dst
2706     if (dstenc >= 4) {
2707       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2708     }
2709     emit_opcode(cbuf, 0x0F);
2710     emit_opcode(cbuf, 0xB6);
2711     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2712   %}
2713 
2714   enc_class Push_ResultXD(regD dst) %{
2715     MacroAssembler _masm(&cbuf);
2716     __ fstp_d(Address(rsp, 0));
2717     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2718     __ addptr(rsp, 8);
2719   %}
2720 
2721   enc_class Push_SrcXD(regD src) %{
2722     MacroAssembler _masm(&cbuf);
2723     __ subptr(rsp, 8);
2724     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2725     __ fld_d(Address(rsp, 0));
2726   %}
2727 
2728 
2729   enc_class enc_rethrow()
2730   %{
2731     cbuf.set_insts_mark();
2732     emit_opcode(cbuf, 0xE9); // jmp entry
2733     emit_d32_reloc(cbuf,
2734                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2735                    runtime_call_Relocation::spec(),
2736                    RELOC_DISP32);
2737   %}
2738 
2739 %}
2740 
2741 
2742 
2743 //----------FRAME--------------------------------------------------------------
2744 // Definition of frame structure and management information.
2745 //
2746 //  S T A C K   L A Y O U T    Allocators stack-slot number
2747 //                             |   (to get allocators register number
2748 //  G  Owned by    |        |  v    add OptoReg::stack0())
2749 //  r   CALLER     |        |
2750 //  o     |        +--------+      pad to even-align allocators stack-slot
2751 //  w     V        |  pad0  |        numbers; owned by CALLER
2752 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2753 //  h     ^        |   in   |  5
2754 //        |        |  args  |  4   Holes in incoming args owned by SELF
2755 //  |     |        |        |  3
2756 //  |     |        +--------+
2757 //  V     |        | old out|      Empty on Intel, window on Sparc
2758 //        |    old |preserve|      Must be even aligned.
2759 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2760 //        |        |   in   |  3   area for Intel ret address
2761 //     Owned by    |preserve|      Empty on Sparc.
2762 //       SELF      +--------+
2763 //        |        |  pad2  |  2   pad to align old SP
2764 //        |        +--------+  1
2765 //        |        | locks  |  0
2766 //        |        +--------+----> OptoReg::stack0(), even aligned
2767 //        |        |  pad1  | 11   pad to align new SP
2768 //        |        +--------+
2769 //        |        |        | 10
2770 //        |        | spills |  9   spills
2771 //        V        |        |  8   (pad0 slot for callee)
2772 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2773 //        ^        |  out   |  7
2774 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2775 //     Owned by    +--------+
2776 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2777 //        |    new |preserve|      Must be even-aligned.
2778 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2779 //        |        |        |
2780 //
2781 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2782 //         known from SELF's arguments and the Java calling convention.
2783 //         Region 6-7 is determined per call site.
2784 // Note 2: If the calling convention leaves holes in the incoming argument
2785 //         area, those holes are owned by SELF.  Holes in the outgoing area
2786 //         are owned by the CALLEE.  Holes should not be nessecary in the
2787 //         incoming area, as the Java calling convention is completely under
2788 //         the control of the AD file.  Doubles can be sorted and packed to
2789 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2790 //         varargs C calling conventions.
2791 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2792 //         even aligned with pad0 as needed.
2793 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2794 //         region 6-11 is even aligned; it may be padded out more so that
2795 //         the region from SP to FP meets the minimum stack alignment.
2796 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2797 //         alignment.  Region 11, pad1, may be dynamically extended so that
2798 //         SP meets the minimum alignment.
2799 
2800 frame
2801 %{
2802   // What direction does stack grow in (assumed to be same for C & Java)
2803   stack_direction(TOWARDS_LOW);
2804 
2805   // These three registers define part of the calling convention
2806   // between compiled code and the interpreter.
2807   inline_cache_reg(RAX);                // Inline Cache Register
2808   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2809                                         // calling interpreter
2810 
2811   // Optional: name the operand used by cisc-spilling to access
2812   // [stack_pointer + offset]
2813   cisc_spilling_operand_name(indOffset32);
2814 
2815   // Number of stack slots consumed by locking an object
2816   sync_stack_slots(2);
2817 
2818   // Compiled code's Frame Pointer
2819   frame_pointer(RSP);
2820 
2821   // Interpreter stores its frame pointer in a register which is
2822   // stored to the stack by I2CAdaptors.
2823   // I2CAdaptors convert from interpreted java to compiled java.
2824   interpreter_frame_pointer(RBP);
2825 
2826   // Stack alignment requirement
2827   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2828 
2829   // Number of stack slots between incoming argument block and the start of
2830   // a new frame.  The PROLOG must add this many slots to the stack.  The
2831   // EPILOG must remove this many slots.  amd64 needs two slots for
2832   // return address.
2833   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2834 
2835   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2836   // for calls to C.  Supports the var-args backing area for register parms.
2837   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2838 
2839   // The after-PROLOG location of the return address.  Location of
2840   // return address specifies a type (REG or STACK) and a number
2841   // representing the register number (i.e. - use a register name) or
2842   // stack slot.
2843   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2844   // Otherwise, it is above the locks and verification slot and alignment word
2845   return_addr(STACK - 2 +
2846               align_up((Compile::current()->in_preserve_stack_slots() +
2847                         Compile::current()->fixed_slots()),
2848                        stack_alignment_in_slots()));
2849 
2850   // Body of function which returns an integer array locating
2851   // arguments either in registers or in stack slots.  Passed an array
2852   // of ideal registers called "sig" and a "length" count.  Stack-slot
2853   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2854   // arguments for a CALLEE.  Incoming stack arguments are
2855   // automatically biased by the preserve_stack_slots field above.
2856 
2857   calling_convention
2858   %{
2859     // No difference between ingoing/outgoing just pass false
2860     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2861   %}
2862 
2863   c_calling_convention
2864   %{
2865     // This is obviously always outgoing
2866     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2867   %}
2868 
2869   // Location of compiled Java return values.  Same as C for now.
2870   return_value
2871   %{
2872     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2873            "only return normal values");
2874 
2875     static const int lo[Op_RegL + 1] = {
2876       0,
2877       0,
2878       RAX_num,  // Op_RegN
2879       RAX_num,  // Op_RegI
2880       RAX_num,  // Op_RegP
2881       XMM0_num, // Op_RegF
2882       XMM0_num, // Op_RegD
2883       RAX_num   // Op_RegL
2884     };
2885     static const int hi[Op_RegL + 1] = {
2886       0,
2887       0,
2888       OptoReg::Bad, // Op_RegN
2889       OptoReg::Bad, // Op_RegI
2890       RAX_H_num,    // Op_RegP
2891       OptoReg::Bad, // Op_RegF
2892       XMM0b_num,    // Op_RegD
2893       RAX_H_num     // Op_RegL
2894     };
2895     // Excluded flags and vector registers.
2896     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2897     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2898   %}
2899 %}
2900 
2901 //----------ATTRIBUTES---------------------------------------------------------
2902 //----------Operand Attributes-------------------------------------------------
2903 op_attrib op_cost(0);        // Required cost attribute
2904 
2905 //----------Instruction Attributes---------------------------------------------
2906 ins_attrib ins_cost(100);       // Required cost attribute
2907 ins_attrib ins_size(8);         // Required size attribute (in bits)
2908 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2909                                 // a non-matching short branch variant
2910                                 // of some long branch?
2911 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2912                                 // be a power of 2) specifies the
2913                                 // alignment that some part of the
2914                                 // instruction (not necessarily the
2915                                 // start) requires.  If > 1, a
2916                                 // compute_padding() function must be
2917                                 // provided for the instruction
2918 
2919 //----------OPERANDS-----------------------------------------------------------
2920 // Operand definitions must precede instruction definitions for correct parsing
2921 // in the ADLC because operands constitute user defined types which are used in
2922 // instruction definitions.
2923 
2924 //----------Simple Operands----------------------------------------------------
2925 // Immediate Operands
2926 // Integer Immediate
2927 operand immI()
2928 %{
2929   match(ConI);
2930 
2931   op_cost(10);
2932   format %{ %}
2933   interface(CONST_INTER);
2934 %}
2935 
2936 // Constant for test vs zero
2937 operand immI0()
2938 %{
2939   predicate(n->get_int() == 0);
2940   match(ConI);
2941 
2942   op_cost(0);
2943   format %{ %}
2944   interface(CONST_INTER);
2945 %}
2946 
2947 // Constant for increment
2948 operand immI1()
2949 %{
2950   predicate(n->get_int() == 1);
2951   match(ConI);
2952 
2953   op_cost(0);
2954   format %{ %}
2955   interface(CONST_INTER);
2956 %}
2957 
2958 // Constant for decrement
2959 operand immI_M1()
2960 %{
2961   predicate(n->get_int() == -1);
2962   match(ConI);
2963 
2964   op_cost(0);
2965   format %{ %}
2966   interface(CONST_INTER);
2967 %}
2968 
2969 // Valid scale values for addressing modes
2970 operand immI2()
2971 %{
2972   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2973   match(ConI);
2974 
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 operand immI8()
2980 %{
2981   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2982   match(ConI);
2983 
2984   op_cost(5);
2985   format %{ %}
2986   interface(CONST_INTER);
2987 %}
2988 
2989 operand immU8()
2990 %{
2991   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2992   match(ConI);
2993 
2994   op_cost(5);
2995   format %{ %}
2996   interface(CONST_INTER);
2997 %}
2998 
2999 operand immI16()
3000 %{
3001   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3002   match(ConI);
3003 
3004   op_cost(10);
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 // Int Immediate non-negative
3010 operand immU31()
3011 %{
3012   predicate(n->get_int() >= 0);
3013   match(ConI);
3014 
3015   op_cost(0);
3016   format %{ %}
3017   interface(CONST_INTER);
3018 %}
3019 
3020 // Constant for long shifts
3021 operand immI_32()
3022 %{
3023   predicate( n->get_int() == 32 );
3024   match(ConI);
3025 
3026   op_cost(0);
3027   format %{ %}
3028   interface(CONST_INTER);
3029 %}
3030 
3031 // Constant for long shifts
3032 operand immI_64()
3033 %{
3034   predicate( n->get_int() == 64 );
3035   match(ConI);
3036 
3037   op_cost(0);
3038   format %{ %}
3039   interface(CONST_INTER);
3040 %}
3041 
3042 // Pointer Immediate
3043 operand immP()
3044 %{
3045   match(ConP);
3046 
3047   op_cost(10);
3048   format %{ %}
3049   interface(CONST_INTER);
3050 %}
3051 
3052 // NULL Pointer Immediate
3053 operand immP0()
3054 %{
3055   predicate(n->get_ptr() == 0);
3056   match(ConP);
3057 
3058   op_cost(5);
3059   format %{ %}
3060   interface(CONST_INTER);
3061 %}
3062 
3063 // Pointer Immediate
3064 operand immN() %{
3065   match(ConN);
3066 
3067   op_cost(10);
3068   format %{ %}
3069   interface(CONST_INTER);
3070 %}
3071 
3072 operand immNKlass() %{
3073   match(ConNKlass);
3074 
3075   op_cost(10);
3076   format %{ %}
3077   interface(CONST_INTER);
3078 %}
3079 
3080 // NULL Pointer Immediate
3081 operand immN0() %{
3082   predicate(n->get_narrowcon() == 0);
3083   match(ConN);
3084 
3085   op_cost(5);
3086   format %{ %}
3087   interface(CONST_INTER);
3088 %}
3089 
3090 operand immP31()
3091 %{
3092   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3093             && (n->get_ptr() >> 31) == 0);
3094   match(ConP);
3095 
3096   op_cost(5);
3097   format %{ %}
3098   interface(CONST_INTER);
3099 %}
3100 
3101 
3102 // Long Immediate
3103 operand immL()
3104 %{
3105   match(ConL);
3106 
3107   op_cost(20);
3108   format %{ %}
3109   interface(CONST_INTER);
3110 %}
3111 
3112 // Long Immediate 8-bit
3113 operand immL8()
3114 %{
3115   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3116   match(ConL);
3117 
3118   op_cost(5);
3119   format %{ %}
3120   interface(CONST_INTER);
3121 %}
3122 
3123 // Long Immediate 32-bit unsigned
3124 operand immUL32()
3125 %{
3126   predicate(n->get_long() == (unsigned int) (n->get_long()));
3127   match(ConL);
3128 
3129   op_cost(10);
3130   format %{ %}
3131   interface(CONST_INTER);
3132 %}
3133 
3134 // Long Immediate 32-bit signed
3135 operand immL32()
3136 %{
3137   predicate(n->get_long() == (int) (n->get_long()));
3138   match(ConL);
3139 
3140   op_cost(15);
3141   format %{ %}
3142   interface(CONST_INTER);
3143 %}
3144 
3145 // Long Immediate zero
3146 operand immL0()
3147 %{
3148   predicate(n->get_long() == 0L);
3149   match(ConL);
3150 
3151   op_cost(10);
3152   format %{ %}
3153   interface(CONST_INTER);
3154 %}
3155 
3156 // Constant for increment
3157 operand immL1()
3158 %{
3159   predicate(n->get_long() == 1);
3160   match(ConL);
3161 
3162   format %{ %}
3163   interface(CONST_INTER);
3164 %}
3165 
3166 // Constant for decrement
3167 operand immL_M1()
3168 %{
3169   predicate(n->get_long() == -1);
3170   match(ConL);
3171 
3172   format %{ %}
3173   interface(CONST_INTER);
3174 %}
3175 
3176 // Long Immediate: the value 10
3177 operand immL10()
3178 %{
3179   predicate(n->get_long() == 10);
3180   match(ConL);
3181 
3182   format %{ %}
3183   interface(CONST_INTER);
3184 %}
3185 
3186 // Long immediate from 0 to 127.
3187 // Used for a shorter form of long mul by 10.
3188 operand immL_127()
3189 %{
3190   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3191   match(ConL);
3192 
3193   op_cost(10);
3194   format %{ %}
3195   interface(CONST_INTER);
3196 %}
3197 
3198 // Long Immediate: low 32-bit mask
3199 operand immL_32bits()
3200 %{
3201   predicate(n->get_long() == 0xFFFFFFFFL);
3202   match(ConL);
3203   op_cost(20);
3204 
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 // Float Immediate zero
3210 operand immF0()
3211 %{
3212   predicate(jint_cast(n->getf()) == 0);
3213   match(ConF);
3214 
3215   op_cost(5);
3216   format %{ %}
3217   interface(CONST_INTER);
3218 %}
3219 
3220 // Float Immediate
3221 operand immF()
3222 %{
3223   match(ConF);
3224 
3225   op_cost(15);
3226   format %{ %}
3227   interface(CONST_INTER);
3228 %}
3229 
3230 // Double Immediate zero
3231 operand immD0()
3232 %{
3233   predicate(jlong_cast(n->getd()) == 0);
3234   match(ConD);
3235 
3236   op_cost(5);
3237   format %{ %}
3238   interface(CONST_INTER);
3239 %}
3240 
3241 // Double Immediate
3242 operand immD()
3243 %{
3244   match(ConD);
3245 
3246   op_cost(15);
3247   format %{ %}
3248   interface(CONST_INTER);
3249 %}
3250 
3251 // Immediates for special shifts (sign extend)
3252 
3253 // Constants for increment
3254 operand immI_16()
3255 %{
3256   predicate(n->get_int() == 16);
3257   match(ConI);
3258 
3259   format %{ %}
3260   interface(CONST_INTER);
3261 %}
3262 
3263 operand immI_24()
3264 %{
3265   predicate(n->get_int() == 24);
3266   match(ConI);
3267 
3268   format %{ %}
3269   interface(CONST_INTER);
3270 %}
3271 
3272 // Constant for byte-wide masking
3273 operand immI_255()
3274 %{
3275   predicate(n->get_int() == 255);
3276   match(ConI);
3277 
3278   format %{ %}
3279   interface(CONST_INTER);
3280 %}
3281 
3282 // Constant for short-wide masking
3283 operand immI_65535()
3284 %{
3285   predicate(n->get_int() == 65535);
3286   match(ConI);
3287 
3288   format %{ %}
3289   interface(CONST_INTER);
3290 %}
3291 
3292 // Constant for byte-wide masking
3293 operand immL_255()
3294 %{
3295   predicate(n->get_long() == 255);
3296   match(ConL);
3297 
3298   format %{ %}
3299   interface(CONST_INTER);
3300 %}
3301 
3302 // Constant for short-wide masking
3303 operand immL_65535()
3304 %{
3305   predicate(n->get_long() == 65535);
3306   match(ConL);
3307 
3308   format %{ %}
3309   interface(CONST_INTER);
3310 %}
3311 
3312 // Register Operands
3313 // Integer Register
3314 operand rRegI()
3315 %{
3316   constraint(ALLOC_IN_RC(int_reg));
3317   match(RegI);
3318 
3319   match(rax_RegI);
3320   match(rbx_RegI);
3321   match(rcx_RegI);
3322   match(rdx_RegI);
3323   match(rdi_RegI);
3324 
3325   format %{ %}
3326   interface(REG_INTER);
3327 %}
3328 
3329 // Special Registers
3330 operand rax_RegI()
3331 %{
3332   constraint(ALLOC_IN_RC(int_rax_reg));
3333   match(RegI);
3334   match(rRegI);
3335 
3336   format %{ "RAX" %}
3337   interface(REG_INTER);
3338 %}
3339 
3340 // Special Registers
3341 operand rbx_RegI()
3342 %{
3343   constraint(ALLOC_IN_RC(int_rbx_reg));
3344   match(RegI);
3345   match(rRegI);
3346 
3347   format %{ "RBX" %}
3348   interface(REG_INTER);
3349 %}
3350 
3351 operand rcx_RegI()
3352 %{
3353   constraint(ALLOC_IN_RC(int_rcx_reg));
3354   match(RegI);
3355   match(rRegI);
3356 
3357   format %{ "RCX" %}
3358   interface(REG_INTER);
3359 %}
3360 
3361 operand rdx_RegI()
3362 %{
3363   constraint(ALLOC_IN_RC(int_rdx_reg));
3364   match(RegI);
3365   match(rRegI);
3366 
3367   format %{ "RDX" %}
3368   interface(REG_INTER);
3369 %}
3370 
3371 operand rdi_RegI()
3372 %{
3373   constraint(ALLOC_IN_RC(int_rdi_reg));
3374   match(RegI);
3375   match(rRegI);
3376 
3377   format %{ "RDI" %}
3378   interface(REG_INTER);
3379 %}
3380 
3381 operand no_rcx_RegI()
3382 %{
3383   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3384   match(RegI);
3385   match(rax_RegI);
3386   match(rbx_RegI);
3387   match(rdx_RegI);
3388   match(rdi_RegI);
3389 
3390   format %{ %}
3391   interface(REG_INTER);
3392 %}
3393 
3394 operand no_rax_rdx_RegI()
3395 %{
3396   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3397   match(RegI);
3398   match(rbx_RegI);
3399   match(rcx_RegI);
3400   match(rdi_RegI);
3401 
3402   format %{ %}
3403   interface(REG_INTER);
3404 %}
3405 
3406 // Pointer Register
3407 operand any_RegP()
3408 %{
3409   constraint(ALLOC_IN_RC(any_reg));
3410   match(RegP);
3411   match(rax_RegP);
3412   match(rbx_RegP);
3413   match(rdi_RegP);
3414   match(rsi_RegP);
3415   match(rbp_RegP);
3416   match(r15_RegP);
3417   match(rRegP);
3418 
3419   format %{ %}
3420   interface(REG_INTER);
3421 %}
3422 
3423 operand rRegP()
3424 %{
3425   constraint(ALLOC_IN_RC(ptr_reg));
3426   match(RegP);
3427   match(rax_RegP);
3428   match(rbx_RegP);
3429   match(rdi_RegP);
3430   match(rsi_RegP);
3431   match(rbp_RegP);  // See Q&A below about
3432   match(r15_RegP);  // r15_RegP and rbp_RegP.
3433 
3434   format %{ %}
3435   interface(REG_INTER);
3436 %}
3437 
3438 operand rRegN() %{
3439   constraint(ALLOC_IN_RC(int_reg));
3440   match(RegN);
3441 
3442   format %{ %}
3443   interface(REG_INTER);
3444 %}
3445 
3446 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3447 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3448 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3449 // The output of an instruction is controlled by the allocator, which respects
3450 // register class masks, not match rules.  Unless an instruction mentions
3451 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3452 // by the allocator as an input.
3453 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3454 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3455 // result, RBP is not included in the output of the instruction either.
3456 
3457 operand no_rax_RegP()
3458 %{
3459   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3460   match(RegP);
3461   match(rbx_RegP);
3462   match(rsi_RegP);
3463   match(rdi_RegP);
3464 
3465   format %{ %}
3466   interface(REG_INTER);
3467 %}
3468 
3469 // This operand is not allowed to use RBP even if
3470 // RBP is not used to hold the frame pointer.
3471 operand no_rbp_RegP()
3472 %{
3473   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3474   match(RegP);
3475   match(rbx_RegP);
3476   match(rsi_RegP);
3477   match(rdi_RegP);
3478 
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 operand no_rax_rbx_RegP()
3484 %{
3485   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3486   match(RegP);
3487   match(rsi_RegP);
3488   match(rdi_RegP);
3489 
3490   format %{ %}
3491   interface(REG_INTER);
3492 %}
3493 
3494 // Special Registers
3495 // Return a pointer value
3496 operand rax_RegP()
3497 %{
3498   constraint(ALLOC_IN_RC(ptr_rax_reg));
3499   match(RegP);
3500   match(rRegP);
3501 
3502   format %{ %}
3503   interface(REG_INTER);
3504 %}
3505 
3506 // Special Registers
3507 // Return a compressed pointer value
3508 operand rax_RegN()
3509 %{
3510   constraint(ALLOC_IN_RC(int_rax_reg));
3511   match(RegN);
3512   match(rRegN);
3513 
3514   format %{ %}
3515   interface(REG_INTER);
3516 %}
3517 
3518 // Used in AtomicAdd
3519 operand rbx_RegP()
3520 %{
3521   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3522   match(RegP);
3523   match(rRegP);
3524 
3525   format %{ %}
3526   interface(REG_INTER);
3527 %}
3528 
3529 operand rsi_RegP()
3530 %{
3531   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3532   match(RegP);
3533   match(rRegP);
3534 
3535   format %{ %}
3536   interface(REG_INTER);
3537 %}
3538 
3539 // Used in rep stosq
3540 operand rdi_RegP()
3541 %{
3542   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3543   match(RegP);
3544   match(rRegP);
3545 
3546   format %{ %}
3547   interface(REG_INTER);
3548 %}
3549 
3550 operand r15_RegP()
3551 %{
3552   constraint(ALLOC_IN_RC(ptr_r15_reg));
3553   match(RegP);
3554   match(rRegP);
3555 
3556   format %{ %}
3557   interface(REG_INTER);
3558 %}
3559 
3560 operand rex_RegP()
3561 %{
3562   constraint(ALLOC_IN_RC(ptr_rex_reg));
3563   match(RegP);
3564   match(rRegP);
3565 
3566   format %{ %}
3567   interface(REG_INTER);
3568 %}
3569 
3570 operand rRegL()
3571 %{
3572   constraint(ALLOC_IN_RC(long_reg));
3573   match(RegL);
3574   match(rax_RegL);
3575   match(rdx_RegL);
3576 
3577   format %{ %}
3578   interface(REG_INTER);
3579 %}
3580 
3581 // Special Registers
3582 operand no_rax_rdx_RegL()
3583 %{
3584   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3585   match(RegL);
3586   match(rRegL);
3587 
3588   format %{ %}
3589   interface(REG_INTER);
3590 %}
3591 
3592 operand no_rax_RegL()
3593 %{
3594   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3595   match(RegL);
3596   match(rRegL);
3597   match(rdx_RegL);
3598 
3599   format %{ %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 operand no_rcx_RegL()
3604 %{
3605   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3606   match(RegL);
3607   match(rRegL);
3608 
3609   format %{ %}
3610   interface(REG_INTER);
3611 %}
3612 
3613 operand rax_RegL()
3614 %{
3615   constraint(ALLOC_IN_RC(long_rax_reg));
3616   match(RegL);
3617   match(rRegL);
3618 
3619   format %{ "RAX" %}
3620   interface(REG_INTER);
3621 %}
3622 
3623 operand rcx_RegL()
3624 %{
3625   constraint(ALLOC_IN_RC(long_rcx_reg));
3626   match(RegL);
3627   match(rRegL);
3628 
3629   format %{ %}
3630   interface(REG_INTER);
3631 %}
3632 
3633 operand rdx_RegL()
3634 %{
3635   constraint(ALLOC_IN_RC(long_rdx_reg));
3636   match(RegL);
3637   match(rRegL);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 // Flags register, used as output of compare instructions
3644 operand rFlagsReg()
3645 %{
3646   constraint(ALLOC_IN_RC(int_flags));
3647   match(RegFlags);
3648 
3649   format %{ "RFLAGS" %}
3650   interface(REG_INTER);
3651 %}
3652 
3653 // Flags register, used as output of FLOATING POINT compare instructions
3654 operand rFlagsRegU()
3655 %{
3656   constraint(ALLOC_IN_RC(int_flags));
3657   match(RegFlags);
3658 
3659   format %{ "RFLAGS_U" %}
3660   interface(REG_INTER);
3661 %}
3662 
3663 operand rFlagsRegUCF() %{
3664   constraint(ALLOC_IN_RC(int_flags));
3665   match(RegFlags);
3666   predicate(false);
3667 
3668   format %{ "RFLAGS_U_CF" %}
3669   interface(REG_INTER);
3670 %}
3671 
3672 // Float register operands
3673 operand regF() %{
3674    constraint(ALLOC_IN_RC(float_reg));
3675    match(RegF);
3676 
3677    format %{ %}
3678    interface(REG_INTER);
3679 %}
3680 
3681 // Double register operands
3682 operand regD() %{
3683    constraint(ALLOC_IN_RC(double_reg));
3684    match(RegD);
3685 
3686    format %{ %}
3687    interface(REG_INTER);
3688 %}
3689 
3690 // Vectors
3691 operand vecS() %{
3692   constraint(ALLOC_IN_RC(vectors_reg));
3693   match(VecS);
3694 
3695   format %{ %}
3696   interface(REG_INTER);
3697 %}
3698 
3699 operand vecD() %{
3700   constraint(ALLOC_IN_RC(vectord_reg));
3701   match(VecD);
3702 
3703   format %{ %}
3704   interface(REG_INTER);
3705 %}
3706 
3707 operand vecX() %{
3708   constraint(ALLOC_IN_RC(vectorx_reg));
3709   match(VecX);
3710 
3711   format %{ %}
3712   interface(REG_INTER);
3713 %}
3714 
3715 operand vecY() %{
3716   constraint(ALLOC_IN_RC(vectory_reg));
3717   match(VecY);
3718 
3719   format %{ %}
3720   interface(REG_INTER);
3721 %}
3722 
3723 //----------Memory Operands----------------------------------------------------
3724 // Direct Memory Operand
3725 // operand direct(immP addr)
3726 // %{
3727 //   match(addr);
3728 
3729 //   format %{ "[$addr]" %}
3730 //   interface(MEMORY_INTER) %{
3731 //     base(0xFFFFFFFF);
3732 //     index(0x4);
3733 //     scale(0x0);
3734 //     disp($addr);
3735 //   %}
3736 // %}
3737 
3738 // Indirect Memory Operand
3739 operand indirect(any_RegP reg)
3740 %{
3741   constraint(ALLOC_IN_RC(ptr_reg));
3742   match(reg);
3743 
3744   format %{ "[$reg]" %}
3745   interface(MEMORY_INTER) %{
3746     base($reg);
3747     index(0x4);
3748     scale(0x0);
3749     disp(0x0);
3750   %}
3751 %}
3752 
3753 // Indirect Memory Plus Short Offset Operand
3754 operand indOffset8(any_RegP reg, immL8 off)
3755 %{
3756   constraint(ALLOC_IN_RC(ptr_reg));
3757   match(AddP reg off);
3758 
3759   format %{ "[$reg + $off (8-bit)]" %}
3760   interface(MEMORY_INTER) %{
3761     base($reg);
3762     index(0x4);
3763     scale(0x0);
3764     disp($off);
3765   %}
3766 %}
3767 
3768 // Indirect Memory Plus Long Offset Operand
3769 operand indOffset32(any_RegP reg, immL32 off)
3770 %{
3771   constraint(ALLOC_IN_RC(ptr_reg));
3772   match(AddP reg off);
3773 
3774   format %{ "[$reg + $off (32-bit)]" %}
3775   interface(MEMORY_INTER) %{
3776     base($reg);
3777     index(0x4);
3778     scale(0x0);
3779     disp($off);
3780   %}
3781 %}
3782 
3783 // Indirect Memory Plus Index Register Plus Offset Operand
3784 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3785 %{
3786   constraint(ALLOC_IN_RC(ptr_reg));
3787   match(AddP (AddP reg lreg) off);
3788 
3789   op_cost(10);
3790   format %{"[$reg + $off + $lreg]" %}
3791   interface(MEMORY_INTER) %{
3792     base($reg);
3793     index($lreg);
3794     scale(0x0);
3795     disp($off);
3796   %}
3797 %}
3798 
3799 // Indirect Memory Plus Index Register Plus Offset Operand
3800 operand indIndex(any_RegP reg, rRegL lreg)
3801 %{
3802   constraint(ALLOC_IN_RC(ptr_reg));
3803   match(AddP reg lreg);
3804 
3805   op_cost(10);
3806   format %{"[$reg + $lreg]" %}
3807   interface(MEMORY_INTER) %{
3808     base($reg);
3809     index($lreg);
3810     scale(0x0);
3811     disp(0x0);
3812   %}
3813 %}
3814 
3815 // Indirect Memory Times Scale Plus Index Register
3816 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3817 %{
3818   constraint(ALLOC_IN_RC(ptr_reg));
3819   match(AddP reg (LShiftL lreg scale));
3820 
3821   op_cost(10);
3822   format %{"[$reg + $lreg << $scale]" %}
3823   interface(MEMORY_INTER) %{
3824     base($reg);
3825     index($lreg);
3826     scale($scale);
3827     disp(0x0);
3828   %}
3829 %}
3830 
3831 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3832 %{
3833   constraint(ALLOC_IN_RC(ptr_reg));
3834   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3835   match(AddP reg (LShiftL (ConvI2L idx) scale));
3836 
3837   op_cost(10);
3838   format %{"[$reg + pos $idx << $scale]" %}
3839   interface(MEMORY_INTER) %{
3840     base($reg);
3841     index($idx);
3842     scale($scale);
3843     disp(0x0);
3844   %}
3845 %}
3846 
3847 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3848 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3849 %{
3850   constraint(ALLOC_IN_RC(ptr_reg));
3851   match(AddP (AddP reg (LShiftL lreg scale)) off);
3852 
3853   op_cost(10);
3854   format %{"[$reg + $off + $lreg << $scale]" %}
3855   interface(MEMORY_INTER) %{
3856     base($reg);
3857     index($lreg);
3858     scale($scale);
3859     disp($off);
3860   %}
3861 %}
3862 
3863 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3864 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3865 %{
3866   constraint(ALLOC_IN_RC(ptr_reg));
3867   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3868   match(AddP (AddP reg (ConvI2L idx)) off);
3869 
3870   op_cost(10);
3871   format %{"[$reg + $off + $idx]" %}
3872   interface(MEMORY_INTER) %{
3873     base($reg);
3874     index($idx);
3875     scale(0x0);
3876     disp($off);
3877   %}
3878 %}
3879 
3880 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3881 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3882 %{
3883   constraint(ALLOC_IN_RC(ptr_reg));
3884   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3885   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3886 
3887   op_cost(10);
3888   format %{"[$reg + $off + $idx << $scale]" %}
3889   interface(MEMORY_INTER) %{
3890     base($reg);
3891     index($idx);
3892     scale($scale);
3893     disp($off);
3894   %}
3895 %}
3896 
3897 // Indirect Narrow Oop Plus Offset Operand
3898 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3899 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3900 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3901   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3902   constraint(ALLOC_IN_RC(ptr_reg));
3903   match(AddP (DecodeN reg) off);
3904 
3905   op_cost(10);
3906   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3907   interface(MEMORY_INTER) %{
3908     base(0xc); // R12
3909     index($reg);
3910     scale(0x3);
3911     disp($off);
3912   %}
3913 %}
3914 
3915 // Indirect Memory Operand
3916 operand indirectNarrow(rRegN reg)
3917 %{
3918   predicate(Universe::narrow_oop_shift() == 0);
3919   constraint(ALLOC_IN_RC(ptr_reg));
3920   match(DecodeN reg);
3921 
3922   format %{ "[$reg]" %}
3923   interface(MEMORY_INTER) %{
3924     base($reg);
3925     index(0x4);
3926     scale(0x0);
3927     disp(0x0);
3928   %}
3929 %}
3930 
3931 // Indirect Memory Plus Short Offset Operand
3932 operand indOffset8Narrow(rRegN reg, immL8 off)
3933 %{
3934   predicate(Universe::narrow_oop_shift() == 0);
3935   constraint(ALLOC_IN_RC(ptr_reg));
3936   match(AddP (DecodeN reg) off);
3937 
3938   format %{ "[$reg + $off (8-bit)]" %}
3939   interface(MEMORY_INTER) %{
3940     base($reg);
3941     index(0x4);
3942     scale(0x0);
3943     disp($off);
3944   %}
3945 %}
3946 
3947 // Indirect Memory Plus Long Offset Operand
3948 operand indOffset32Narrow(rRegN reg, immL32 off)
3949 %{
3950   predicate(Universe::narrow_oop_shift() == 0);
3951   constraint(ALLOC_IN_RC(ptr_reg));
3952   match(AddP (DecodeN reg) off);
3953 
3954   format %{ "[$reg + $off (32-bit)]" %}
3955   interface(MEMORY_INTER) %{
3956     base($reg);
3957     index(0x4);
3958     scale(0x0);
3959     disp($off);
3960   %}
3961 %}
3962 
3963 // Indirect Memory Plus Index Register Plus Offset Operand
3964 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3965 %{
3966   predicate(Universe::narrow_oop_shift() == 0);
3967   constraint(ALLOC_IN_RC(ptr_reg));
3968   match(AddP (AddP (DecodeN reg) lreg) off);
3969 
3970   op_cost(10);
3971   format %{"[$reg + $off + $lreg]" %}
3972   interface(MEMORY_INTER) %{
3973     base($reg);
3974     index($lreg);
3975     scale(0x0);
3976     disp($off);
3977   %}
3978 %}
3979 
3980 // Indirect Memory Plus Index Register Plus Offset Operand
3981 operand indIndexNarrow(rRegN reg, rRegL lreg)
3982 %{
3983   predicate(Universe::narrow_oop_shift() == 0);
3984   constraint(ALLOC_IN_RC(ptr_reg));
3985   match(AddP (DecodeN reg) lreg);
3986 
3987   op_cost(10);
3988   format %{"[$reg + $lreg]" %}
3989   interface(MEMORY_INTER) %{
3990     base($reg);
3991     index($lreg);
3992     scale(0x0);
3993     disp(0x0);
3994   %}
3995 %}
3996 
3997 // Indirect Memory Times Scale Plus Index Register
3998 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3999 %{
4000   predicate(Universe::narrow_oop_shift() == 0);
4001   constraint(ALLOC_IN_RC(ptr_reg));
4002   match(AddP (DecodeN reg) (LShiftL lreg scale));
4003 
4004   op_cost(10);
4005   format %{"[$reg + $lreg << $scale]" %}
4006   interface(MEMORY_INTER) %{
4007     base($reg);
4008     index($lreg);
4009     scale($scale);
4010     disp(0x0);
4011   %}
4012 %}
4013 
4014 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4015 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4016 %{
4017   predicate(Universe::narrow_oop_shift() == 0);
4018   constraint(ALLOC_IN_RC(ptr_reg));
4019   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4020 
4021   op_cost(10);
4022   format %{"[$reg + $off + $lreg << $scale]" %}
4023   interface(MEMORY_INTER) %{
4024     base($reg);
4025     index($lreg);
4026     scale($scale);
4027     disp($off);
4028   %}
4029 %}
4030 
4031 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4032 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4033 %{
4034   constraint(ALLOC_IN_RC(ptr_reg));
4035   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4036   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4037 
4038   op_cost(10);
4039   format %{"[$reg + $off + $idx]" %}
4040   interface(MEMORY_INTER) %{
4041     base($reg);
4042     index($idx);
4043     scale(0x0);
4044     disp($off);
4045   %}
4046 %}
4047 
4048 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4049 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4050 %{
4051   constraint(ALLOC_IN_RC(ptr_reg));
4052   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4053   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4054 
4055   op_cost(10);
4056   format %{"[$reg + $off + $idx << $scale]" %}
4057   interface(MEMORY_INTER) %{
4058     base($reg);
4059     index($idx);
4060     scale($scale);
4061     disp($off);
4062   %}
4063 %}
4064 
4065 //----------Special Memory Operands--------------------------------------------
4066 // Stack Slot Operand - This operand is used for loading and storing temporary
4067 //                      values on the stack where a match requires a value to
4068 //                      flow through memory.
4069 operand stackSlotP(sRegP reg)
4070 %{
4071   constraint(ALLOC_IN_RC(stack_slots));
4072   // No match rule because this operand is only generated in matching
4073 
4074   format %{ "[$reg]" %}
4075   interface(MEMORY_INTER) %{
4076     base(0x4);   // RSP
4077     index(0x4);  // No Index
4078     scale(0x0);  // No Scale
4079     disp($reg);  // Stack Offset
4080   %}
4081 %}
4082 
4083 operand stackSlotI(sRegI reg)
4084 %{
4085   constraint(ALLOC_IN_RC(stack_slots));
4086   // No match rule because this operand is only generated in matching
4087 
4088   format %{ "[$reg]" %}
4089   interface(MEMORY_INTER) %{
4090     base(0x4);   // RSP
4091     index(0x4);  // No Index
4092     scale(0x0);  // No Scale
4093     disp($reg);  // Stack Offset
4094   %}
4095 %}
4096 
4097 operand stackSlotF(sRegF reg)
4098 %{
4099   constraint(ALLOC_IN_RC(stack_slots));
4100   // No match rule because this operand is only generated in matching
4101 
4102   format %{ "[$reg]" %}
4103   interface(MEMORY_INTER) %{
4104     base(0x4);   // RSP
4105     index(0x4);  // No Index
4106     scale(0x0);  // No Scale
4107     disp($reg);  // Stack Offset
4108   %}
4109 %}
4110 
4111 operand stackSlotD(sRegD reg)
4112 %{
4113   constraint(ALLOC_IN_RC(stack_slots));
4114   // No match rule because this operand is only generated in matching
4115 
4116   format %{ "[$reg]" %}
4117   interface(MEMORY_INTER) %{
4118     base(0x4);   // RSP
4119     index(0x4);  // No Index
4120     scale(0x0);  // No Scale
4121     disp($reg);  // Stack Offset
4122   %}
4123 %}
4124 operand stackSlotL(sRegL reg)
4125 %{
4126   constraint(ALLOC_IN_RC(stack_slots));
4127   // No match rule because this operand is only generated in matching
4128 
4129   format %{ "[$reg]" %}
4130   interface(MEMORY_INTER) %{
4131     base(0x4);   // RSP
4132     index(0x4);  // No Index
4133     scale(0x0);  // No Scale
4134     disp($reg);  // Stack Offset
4135   %}
4136 %}
4137 
4138 //----------Conditional Branch Operands----------------------------------------
4139 // Comparison Op  - This is the operation of the comparison, and is limited to
4140 //                  the following set of codes:
4141 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4142 //
4143 // Other attributes of the comparison, such as unsignedness, are specified
4144 // by the comparison instruction that sets a condition code flags register.
4145 // That result is represented by a flags operand whose subtype is appropriate
4146 // to the unsignedness (etc.) of the comparison.
4147 //
4148 // Later, the instruction which matches both the Comparison Op (a Bool) and
4149 // the flags (produced by the Cmp) specifies the coding of the comparison op
4150 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4151 
4152 // Comparision Code
4153 operand cmpOp()
4154 %{
4155   match(Bool);
4156 
4157   format %{ "" %}
4158   interface(COND_INTER) %{
4159     equal(0x4, "e");
4160     not_equal(0x5, "ne");
4161     less(0xC, "l");
4162     greater_equal(0xD, "ge");
4163     less_equal(0xE, "le");
4164     greater(0xF, "g");
4165     overflow(0x0, "o");
4166     no_overflow(0x1, "no");
4167   %}
4168 %}
4169 
4170 // Comparison Code, unsigned compare.  Used by FP also, with
4171 // C2 (unordered) turned into GT or LT already.  The other bits
4172 // C0 and C3 are turned into Carry & Zero flags.
4173 operand cmpOpU()
4174 %{
4175   match(Bool);
4176 
4177   format %{ "" %}
4178   interface(COND_INTER) %{
4179     equal(0x4, "e");
4180     not_equal(0x5, "ne");
4181     less(0x2, "b");
4182     greater_equal(0x3, "nb");
4183     less_equal(0x6, "be");
4184     greater(0x7, "nbe");
4185     overflow(0x0, "o");
4186     no_overflow(0x1, "no");
4187   %}
4188 %}
4189 
4190 
4191 // Floating comparisons that don't require any fixup for the unordered case
4192 operand cmpOpUCF() %{
4193   match(Bool);
4194   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4195             n->as_Bool()->_test._test == BoolTest::ge ||
4196             n->as_Bool()->_test._test == BoolTest::le ||
4197             n->as_Bool()->_test._test == BoolTest::gt);
4198   format %{ "" %}
4199   interface(COND_INTER) %{
4200     equal(0x4, "e");
4201     not_equal(0x5, "ne");
4202     less(0x2, "b");
4203     greater_equal(0x3, "nb");
4204     less_equal(0x6, "be");
4205     greater(0x7, "nbe");
4206     overflow(0x0, "o");
4207     no_overflow(0x1, "no");
4208   %}
4209 %}
4210 
4211 
4212 // Floating comparisons that can be fixed up with extra conditional jumps
4213 operand cmpOpUCF2() %{
4214   match(Bool);
4215   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4216             n->as_Bool()->_test._test == BoolTest::eq);
4217   format %{ "" %}
4218   interface(COND_INTER) %{
4219     equal(0x4, "e");
4220     not_equal(0x5, "ne");
4221     less(0x2, "b");
4222     greater_equal(0x3, "nb");
4223     less_equal(0x6, "be");
4224     greater(0x7, "nbe");
4225     overflow(0x0, "o");
4226     no_overflow(0x1, "no");
4227   %}
4228 %}
4229 
4230 // Operands for bound floating pointer register arguments
4231 operand rxmm0() %{
4232   constraint(ALLOC_IN_RC(xmm0_reg));  match(VecX);
4233   predicate((UseSSE > 0) && (UseAVX<= 2));  format%{%}  interface(REG_INTER);
4234 %}
4235 operand rxmm1() %{
4236   constraint(ALLOC_IN_RC(xmm1_reg));  match(VecX);
4237   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4238 %}
4239 operand rxmm2() %{
4240   constraint(ALLOC_IN_RC(xmm2_reg));  match(VecX);
4241   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4242 %}
4243 operand rxmm3() %{
4244   constraint(ALLOC_IN_RC(xmm3_reg));  match(VecX);
4245   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4246 %}
4247 operand rxmm4() %{
4248   constraint(ALLOC_IN_RC(xmm4_reg));  match(VecX);
4249   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4250 %}
4251 operand rxmm5() %{
4252   constraint(ALLOC_IN_RC(xmm5_reg));  match(VecX);
4253   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4254 %}
4255 operand rxmm6() %{
4256   constraint(ALLOC_IN_RC(xmm6_reg));  match(VecX);
4257   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4258 %}
4259 operand rxmm7() %{
4260   constraint(ALLOC_IN_RC(xmm7_reg));  match(VecX);
4261   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4262 %}
4263 operand rxmm8() %{
4264   constraint(ALLOC_IN_RC(xmm8_reg));  match(VecX);
4265   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4266 %}
4267 operand rxmm9() %{
4268   constraint(ALLOC_IN_RC(xmm9_reg));  match(VecX);
4269   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4270 %}
4271 operand rxmm10() %{
4272   constraint(ALLOC_IN_RC(xmm10_reg));  match(VecX);
4273   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4274 %}
4275 operand rxmm11() %{
4276   constraint(ALLOC_IN_RC(xmm11_reg));  match(VecX);
4277   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4278 %}
4279 operand rxmm12() %{
4280   constraint(ALLOC_IN_RC(xmm12_reg));  match(VecX);
4281   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4282 %}
4283 operand rxmm13() %{
4284   constraint(ALLOC_IN_RC(xmm13_reg));  match(VecX);
4285   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4286 %}
4287 operand rxmm14() %{
4288   constraint(ALLOC_IN_RC(xmm14_reg));  match(VecX);
4289   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4290 %}
4291 operand rxmm15() %{
4292   constraint(ALLOC_IN_RC(xmm15_reg));  match(VecX);
4293   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4294 %}
4295 operand rxmm16() %{
4296   constraint(ALLOC_IN_RC(xmm16_reg));  match(VecX);
4297   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4298 %}
4299 operand rxmm17() %{
4300   constraint(ALLOC_IN_RC(xmm17_reg));  match(VecX);
4301   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4302 %}
4303 operand rxmm18() %{
4304   constraint(ALLOC_IN_RC(xmm18_reg));  match(VecX);
4305   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4306 %}
4307 operand rxmm19() %{
4308   constraint(ALLOC_IN_RC(xmm19_reg));  match(VecX);
4309   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4310 %}
4311 operand rxmm20() %{
4312   constraint(ALLOC_IN_RC(xmm20_reg));  match(VecX);
4313   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4314 %}
4315 operand rxmm21() %{
4316   constraint(ALLOC_IN_RC(xmm21_reg));  match(VecX);
4317   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4318 %}
4319 operand rxmm22() %{
4320   constraint(ALLOC_IN_RC(xmm22_reg));  match(VecX);
4321   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4322 %}
4323 operand rxmm23() %{
4324   constraint(ALLOC_IN_RC(xmm23_reg));  match(VecX);
4325   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4326 %}
4327 operand rxmm24() %{
4328   constraint(ALLOC_IN_RC(xmm24_reg));  match(VecX);
4329   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4330 %}
4331 operand rxmm25() %{
4332   constraint(ALLOC_IN_RC(xmm25_reg));  match(VecX);
4333   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4334 %}
4335 operand rxmm26() %{
4336   constraint(ALLOC_IN_RC(xmm26_reg));  match(VecX);
4337   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4338 %}
4339 operand rxmm27() %{
4340   constraint(ALLOC_IN_RC(xmm27_reg));  match(VecX);
4341   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4342 %}
4343 operand rxmm28() %{
4344   constraint(ALLOC_IN_RC(xmm28_reg));  match(VecX);
4345   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4346 %}
4347 operand rxmm29() %{
4348   constraint(ALLOC_IN_RC(xmm29_reg));  match(VecX);
4349   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4350 %}
4351 operand rxmm30() %{
4352   constraint(ALLOC_IN_RC(xmm30_reg));  match(VecX);
4353   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4354 %}
4355 operand rxmm31() %{
4356   constraint(ALLOC_IN_RC(xmm31_reg));  match(VecX);
4357   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4358 %}
4359 
4360 //----------OPERAND CLASSES----------------------------------------------------
4361 // Operand Classes are groups of operands that are used as to simplify
4362 // instruction definitions by not requiring the AD writer to specify separate
4363 // instructions for every form of operand when the instruction accepts
4364 // multiple operand types with the same basic encoding and format.  The classic
4365 // case of this is memory operands.
4366 
4367 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4368                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4369                indCompressedOopOffset,
4370                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4371                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4372                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4373 
4374 //----------PIPELINE-----------------------------------------------------------
4375 // Rules which define the behavior of the target architectures pipeline.
4376 pipeline %{
4377 
4378 //----------ATTRIBUTES---------------------------------------------------------
4379 attributes %{
4380   variable_size_instructions;        // Fixed size instructions
4381   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4382   instruction_unit_size = 1;         // An instruction is 1 bytes long
4383   instruction_fetch_unit_size = 16;  // The processor fetches one line
4384   instruction_fetch_units = 1;       // of 16 bytes
4385 
4386   // List of nop instructions
4387   nops( MachNop );
4388 %}
4389 
4390 //----------RESOURCES----------------------------------------------------------
4391 // Resources are the functional units available to the machine
4392 
4393 // Generic P2/P3 pipeline
4394 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4395 // 3 instructions decoded per cycle.
4396 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4397 // 3 ALU op, only ALU0 handles mul instructions.
4398 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4399            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4400            BR, FPU,
4401            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4402 
4403 //----------PIPELINE DESCRIPTION-----------------------------------------------
4404 // Pipeline Description specifies the stages in the machine's pipeline
4405 
4406 // Generic P2/P3 pipeline
4407 pipe_desc(S0, S1, S2, S3, S4, S5);
4408 
4409 //----------PIPELINE CLASSES---------------------------------------------------
4410 // Pipeline Classes describe the stages in which input and output are
4411 // referenced by the hardware pipeline.
4412 
4413 // Naming convention: ialu or fpu
4414 // Then: _reg
4415 // Then: _reg if there is a 2nd register
4416 // Then: _long if it's a pair of instructions implementing a long
4417 // Then: _fat if it requires the big decoder
4418 //   Or: _mem if it requires the big decoder and a memory unit.
4419 
4420 // Integer ALU reg operation
4421 pipe_class ialu_reg(rRegI dst)
4422 %{
4423     single_instruction;
4424     dst    : S4(write);
4425     dst    : S3(read);
4426     DECODE : S0;        // any decoder
4427     ALU    : S3;        // any alu
4428 %}
4429 
4430 // Long ALU reg operation
4431 pipe_class ialu_reg_long(rRegL dst)
4432 %{
4433     instruction_count(2);
4434     dst    : S4(write);
4435     dst    : S3(read);
4436     DECODE : S0(2);     // any 2 decoders
4437     ALU    : S3(2);     // both alus
4438 %}
4439 
4440 // Integer ALU reg operation using big decoder
4441 pipe_class ialu_reg_fat(rRegI dst)
4442 %{
4443     single_instruction;
4444     dst    : S4(write);
4445     dst    : S3(read);
4446     D0     : S0;        // big decoder only
4447     ALU    : S3;        // any alu
4448 %}
4449 
4450 // Long ALU reg operation using big decoder
4451 pipe_class ialu_reg_long_fat(rRegL dst)
4452 %{
4453     instruction_count(2);
4454     dst    : S4(write);
4455     dst    : S3(read);
4456     D0     : S0(2);     // big decoder only; twice
4457     ALU    : S3(2);     // any 2 alus
4458 %}
4459 
4460 // Integer ALU reg-reg operation
4461 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4462 %{
4463     single_instruction;
4464     dst    : S4(write);
4465     src    : S3(read);
4466     DECODE : S0;        // any decoder
4467     ALU    : S3;        // any alu
4468 %}
4469 
4470 // Long ALU reg-reg operation
4471 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4472 %{
4473     instruction_count(2);
4474     dst    : S4(write);
4475     src    : S3(read);
4476     DECODE : S0(2);     // any 2 decoders
4477     ALU    : S3(2);     // both alus
4478 %}
4479 
4480 // Integer ALU reg-reg operation
4481 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4482 %{
4483     single_instruction;
4484     dst    : S4(write);
4485     src    : S3(read);
4486     D0     : S0;        // big decoder only
4487     ALU    : S3;        // any alu
4488 %}
4489 
4490 // Long ALU reg-reg operation
4491 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4492 %{
4493     instruction_count(2);
4494     dst    : S4(write);
4495     src    : S3(read);
4496     D0     : S0(2);     // big decoder only; twice
4497     ALU    : S3(2);     // both alus
4498 %}
4499 
4500 // Integer ALU reg-mem operation
4501 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4502 %{
4503     single_instruction;
4504     dst    : S5(write);
4505     mem    : S3(read);
4506     D0     : S0;        // big decoder only
4507     ALU    : S4;        // any alu
4508     MEM    : S3;        // any mem
4509 %}
4510 
4511 // Integer mem operation (prefetch)
4512 pipe_class ialu_mem(memory mem)
4513 %{
4514     single_instruction;
4515     mem    : S3(read);
4516     D0     : S0;        // big decoder only
4517     MEM    : S3;        // any mem
4518 %}
4519 
4520 // Integer Store to Memory
4521 pipe_class ialu_mem_reg(memory mem, rRegI src)
4522 %{
4523     single_instruction;
4524     mem    : S3(read);
4525     src    : S5(read);
4526     D0     : S0;        // big decoder only
4527     ALU    : S4;        // any alu
4528     MEM    : S3;
4529 %}
4530 
4531 // // Long Store to Memory
4532 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4533 // %{
4534 //     instruction_count(2);
4535 //     mem    : S3(read);
4536 //     src    : S5(read);
4537 //     D0     : S0(2);          // big decoder only; twice
4538 //     ALU    : S4(2);     // any 2 alus
4539 //     MEM    : S3(2);  // Both mems
4540 // %}
4541 
4542 // Integer Store to Memory
4543 pipe_class ialu_mem_imm(memory mem)
4544 %{
4545     single_instruction;
4546     mem    : S3(read);
4547     D0     : S0;        // big decoder only
4548     ALU    : S4;        // any alu
4549     MEM    : S3;
4550 %}
4551 
4552 // Integer ALU0 reg-reg operation
4553 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4554 %{
4555     single_instruction;
4556     dst    : S4(write);
4557     src    : S3(read);
4558     D0     : S0;        // Big decoder only
4559     ALU0   : S3;        // only alu0
4560 %}
4561 
4562 // Integer ALU0 reg-mem operation
4563 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4564 %{
4565     single_instruction;
4566     dst    : S5(write);
4567     mem    : S3(read);
4568     D0     : S0;        // big decoder only
4569     ALU0   : S4;        // ALU0 only
4570     MEM    : S3;        // any mem
4571 %}
4572 
4573 // Integer ALU reg-reg operation
4574 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4575 %{
4576     single_instruction;
4577     cr     : S4(write);
4578     src1   : S3(read);
4579     src2   : S3(read);
4580     DECODE : S0;        // any decoder
4581     ALU    : S3;        // any alu
4582 %}
4583 
4584 // Integer ALU reg-imm operation
4585 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4586 %{
4587     single_instruction;
4588     cr     : S4(write);
4589     src1   : S3(read);
4590     DECODE : S0;        // any decoder
4591     ALU    : S3;        // any alu
4592 %}
4593 
4594 // Integer ALU reg-mem operation
4595 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4596 %{
4597     single_instruction;
4598     cr     : S4(write);
4599     src1   : S3(read);
4600     src2   : S3(read);
4601     D0     : S0;        // big decoder only
4602     ALU    : S4;        // any alu
4603     MEM    : S3;
4604 %}
4605 
4606 // Conditional move reg-reg
4607 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4608 %{
4609     instruction_count(4);
4610     y      : S4(read);
4611     q      : S3(read);
4612     p      : S3(read);
4613     DECODE : S0(4);     // any decoder
4614 %}
4615 
4616 // Conditional move reg-reg
4617 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4618 %{
4619     single_instruction;
4620     dst    : S4(write);
4621     src    : S3(read);
4622     cr     : S3(read);
4623     DECODE : S0;        // any decoder
4624 %}
4625 
4626 // Conditional move reg-mem
4627 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4628 %{
4629     single_instruction;
4630     dst    : S4(write);
4631     src    : S3(read);
4632     cr     : S3(read);
4633     DECODE : S0;        // any decoder
4634     MEM    : S3;
4635 %}
4636 
4637 // Conditional move reg-reg long
4638 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4639 %{
4640     single_instruction;
4641     dst    : S4(write);
4642     src    : S3(read);
4643     cr     : S3(read);
4644     DECODE : S0(2);     // any 2 decoders
4645 %}
4646 
4647 // XXX
4648 // // Conditional move double reg-reg
4649 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4650 // %{
4651 //     single_instruction;
4652 //     dst    : S4(write);
4653 //     src    : S3(read);
4654 //     cr     : S3(read);
4655 //     DECODE : S0;     // any decoder
4656 // %}
4657 
4658 // Float reg-reg operation
4659 pipe_class fpu_reg(regD dst)
4660 %{
4661     instruction_count(2);
4662     dst    : S3(read);
4663     DECODE : S0(2);     // any 2 decoders
4664     FPU    : S3;
4665 %}
4666 
4667 // Float reg-reg operation
4668 pipe_class fpu_reg_reg(regD dst, regD src)
4669 %{
4670     instruction_count(2);
4671     dst    : S4(write);
4672     src    : S3(read);
4673     DECODE : S0(2);     // any 2 decoders
4674     FPU    : S3;
4675 %}
4676 
4677 // Float reg-reg operation
4678 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4679 %{
4680     instruction_count(3);
4681     dst    : S4(write);
4682     src1   : S3(read);
4683     src2   : S3(read);
4684     DECODE : S0(3);     // any 3 decoders
4685     FPU    : S3(2);
4686 %}
4687 
4688 // Float reg-reg operation
4689 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4690 %{
4691     instruction_count(4);
4692     dst    : S4(write);
4693     src1   : S3(read);
4694     src2   : S3(read);
4695     src3   : S3(read);
4696     DECODE : S0(4);     // any 3 decoders
4697     FPU    : S3(2);
4698 %}
4699 
4700 // Float reg-reg operation
4701 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4702 %{
4703     instruction_count(4);
4704     dst    : S4(write);
4705     src1   : S3(read);
4706     src2   : S3(read);
4707     src3   : S3(read);
4708     DECODE : S1(3);     // any 3 decoders
4709     D0     : S0;        // Big decoder only
4710     FPU    : S3(2);
4711     MEM    : S3;
4712 %}
4713 
4714 // Float reg-mem operation
4715 pipe_class fpu_reg_mem(regD dst, memory mem)
4716 %{
4717     instruction_count(2);
4718     dst    : S5(write);
4719     mem    : S3(read);
4720     D0     : S0;        // big decoder only
4721     DECODE : S1;        // any decoder for FPU POP
4722     FPU    : S4;
4723     MEM    : S3;        // any mem
4724 %}
4725 
4726 // Float reg-mem operation
4727 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4728 %{
4729     instruction_count(3);
4730     dst    : S5(write);
4731     src1   : S3(read);
4732     mem    : S3(read);
4733     D0     : S0;        // big decoder only
4734     DECODE : S1(2);     // any decoder for FPU POP
4735     FPU    : S4;
4736     MEM    : S3;        // any mem
4737 %}
4738 
4739 // Float mem-reg operation
4740 pipe_class fpu_mem_reg(memory mem, regD src)
4741 %{
4742     instruction_count(2);
4743     src    : S5(read);
4744     mem    : S3(read);
4745     DECODE : S0;        // any decoder for FPU PUSH
4746     D0     : S1;        // big decoder only
4747     FPU    : S4;
4748     MEM    : S3;        // any mem
4749 %}
4750 
4751 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4752 %{
4753     instruction_count(3);
4754     src1   : S3(read);
4755     src2   : S3(read);
4756     mem    : S3(read);
4757     DECODE : S0(2);     // any decoder for FPU PUSH
4758     D0     : S1;        // big decoder only
4759     FPU    : S4;
4760     MEM    : S3;        // any mem
4761 %}
4762 
4763 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4764 %{
4765     instruction_count(3);
4766     src1   : S3(read);
4767     src2   : S3(read);
4768     mem    : S4(read);
4769     DECODE : S0;        // any decoder for FPU PUSH
4770     D0     : S0(2);     // big decoder only
4771     FPU    : S4;
4772     MEM    : S3(2);     // any mem
4773 %}
4774 
4775 pipe_class fpu_mem_mem(memory dst, memory src1)
4776 %{
4777     instruction_count(2);
4778     src1   : S3(read);
4779     dst    : S4(read);
4780     D0     : S0(2);     // big decoder only
4781     MEM    : S3(2);     // any mem
4782 %}
4783 
4784 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4785 %{
4786     instruction_count(3);
4787     src1   : S3(read);
4788     src2   : S3(read);
4789     dst    : S4(read);
4790     D0     : S0(3);     // big decoder only
4791     FPU    : S4;
4792     MEM    : S3(3);     // any mem
4793 %}
4794 
4795 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4796 %{
4797     instruction_count(3);
4798     src1   : S4(read);
4799     mem    : S4(read);
4800     DECODE : S0;        // any decoder for FPU PUSH
4801     D0     : S0(2);     // big decoder only
4802     FPU    : S4;
4803     MEM    : S3(2);     // any mem
4804 %}
4805 
4806 // Float load constant
4807 pipe_class fpu_reg_con(regD dst)
4808 %{
4809     instruction_count(2);
4810     dst    : S5(write);
4811     D0     : S0;        // big decoder only for the load
4812     DECODE : S1;        // any decoder for FPU POP
4813     FPU    : S4;
4814     MEM    : S3;        // any mem
4815 %}
4816 
4817 // Float load constant
4818 pipe_class fpu_reg_reg_con(regD dst, regD src)
4819 %{
4820     instruction_count(3);
4821     dst    : S5(write);
4822     src    : S3(read);
4823     D0     : S0;        // big decoder only for the load
4824     DECODE : S1(2);     // any decoder for FPU POP
4825     FPU    : S4;
4826     MEM    : S3;        // any mem
4827 %}
4828 
4829 // UnConditional branch
4830 pipe_class pipe_jmp(label labl)
4831 %{
4832     single_instruction;
4833     BR   : S3;
4834 %}
4835 
4836 // Conditional branch
4837 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4838 %{
4839     single_instruction;
4840     cr    : S1(read);
4841     BR    : S3;
4842 %}
4843 
4844 // Allocation idiom
4845 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4846 %{
4847     instruction_count(1); force_serialization;
4848     fixed_latency(6);
4849     heap_ptr : S3(read);
4850     DECODE   : S0(3);
4851     D0       : S2;
4852     MEM      : S3;
4853     ALU      : S3(2);
4854     dst      : S5(write);
4855     BR       : S5;
4856 %}
4857 
4858 // Generic big/slow expanded idiom
4859 pipe_class pipe_slow()
4860 %{
4861     instruction_count(10); multiple_bundles; force_serialization;
4862     fixed_latency(100);
4863     D0  : S0(2);
4864     MEM : S3(2);
4865 %}
4866 
4867 // The real do-nothing guy
4868 pipe_class empty()
4869 %{
4870     instruction_count(0);
4871 %}
4872 
4873 // Define the class for the Nop node
4874 define
4875 %{
4876    MachNop = empty;
4877 %}
4878 
4879 %}
4880 
4881 //----------INSTRUCTIONS-------------------------------------------------------
4882 //
4883 // match      -- States which machine-independent subtree may be replaced
4884 //               by this instruction.
4885 // ins_cost   -- The estimated cost of this instruction is used by instruction
4886 //               selection to identify a minimum cost tree of machine
4887 //               instructions that matches a tree of machine-independent
4888 //               instructions.
4889 // format     -- A string providing the disassembly for this instruction.
4890 //               The value of an instruction's operand may be inserted
4891 //               by referring to it with a '$' prefix.
4892 // opcode     -- Three instruction opcodes may be provided.  These are referred
4893 //               to within an encode class as $primary, $secondary, and $tertiary
4894 //               rrspectively.  The primary opcode is commonly used to
4895 //               indicate the type of machine instruction, while secondary
4896 //               and tertiary are often used for prefix options or addressing
4897 //               modes.
4898 // ins_encode -- A list of encode classes with parameters. The encode class
4899 //               name must have been defined in an 'enc_class' specification
4900 //               in the encode section of the architecture description.
4901 
4902 
4903 //----------Load/Store/Move Instructions---------------------------------------
4904 //----------Load Instructions--------------------------------------------------
4905 
4906 // Load Byte (8 bit signed)
4907 instruct loadB(rRegI dst, memory mem)
4908 %{
4909   match(Set dst (LoadB mem));
4910 
4911   ins_cost(125);
4912   format %{ "movsbl  $dst, $mem\t# byte" %}
4913 
4914   ins_encode %{
4915     __ movsbl($dst$$Register, $mem$$Address);
4916   %}
4917 
4918   ins_pipe(ialu_reg_mem);
4919 %}
4920 
4921 // Load Byte (8 bit signed) into Long Register
4922 instruct loadB2L(rRegL dst, memory mem)
4923 %{
4924   match(Set dst (ConvI2L (LoadB mem)));
4925 
4926   ins_cost(125);
4927   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4928 
4929   ins_encode %{
4930     __ movsbq($dst$$Register, $mem$$Address);
4931   %}
4932 
4933   ins_pipe(ialu_reg_mem);
4934 %}
4935 
4936 // Load Unsigned Byte (8 bit UNsigned)
4937 instruct loadUB(rRegI dst, memory mem)
4938 %{
4939   match(Set dst (LoadUB mem));
4940 
4941   ins_cost(125);
4942   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4943 
4944   ins_encode %{
4945     __ movzbl($dst$$Register, $mem$$Address);
4946   %}
4947 
4948   ins_pipe(ialu_reg_mem);
4949 %}
4950 
4951 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4952 instruct loadUB2L(rRegL dst, memory mem)
4953 %{
4954   match(Set dst (ConvI2L (LoadUB mem)));
4955 
4956   ins_cost(125);
4957   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4958 
4959   ins_encode %{
4960     __ movzbq($dst$$Register, $mem$$Address);
4961   %}
4962 
4963   ins_pipe(ialu_reg_mem);
4964 %}
4965 
4966 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4967 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4968   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4969   effect(KILL cr);
4970 
4971   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4972             "andl    $dst, right_n_bits($mask, 8)" %}
4973   ins_encode %{
4974     Register Rdst = $dst$$Register;
4975     __ movzbq(Rdst, $mem$$Address);
4976     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4977   %}
4978   ins_pipe(ialu_reg_mem);
4979 %}
4980 
4981 // Load Short (16 bit signed)
4982 instruct loadS(rRegI dst, memory mem)
4983 %{
4984   match(Set dst (LoadS mem));
4985 
4986   ins_cost(125);
4987   format %{ "movswl $dst, $mem\t# short" %}
4988 
4989   ins_encode %{
4990     __ movswl($dst$$Register, $mem$$Address);
4991   %}
4992 
4993   ins_pipe(ialu_reg_mem);
4994 %}
4995 
4996 // Load Short (16 bit signed) to Byte (8 bit signed)
4997 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4998   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4999 
5000   ins_cost(125);
5001   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5002   ins_encode %{
5003     __ movsbl($dst$$Register, $mem$$Address);
5004   %}
5005   ins_pipe(ialu_reg_mem);
5006 %}
5007 
5008 // Load Short (16 bit signed) into Long Register
5009 instruct loadS2L(rRegL dst, memory mem)
5010 %{
5011   match(Set dst (ConvI2L (LoadS mem)));
5012 
5013   ins_cost(125);
5014   format %{ "movswq $dst, $mem\t# short -> long" %}
5015 
5016   ins_encode %{
5017     __ movswq($dst$$Register, $mem$$Address);
5018   %}
5019 
5020   ins_pipe(ialu_reg_mem);
5021 %}
5022 
5023 // Load Unsigned Short/Char (16 bit UNsigned)
5024 instruct loadUS(rRegI dst, memory mem)
5025 %{
5026   match(Set dst (LoadUS mem));
5027 
5028   ins_cost(125);
5029   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5030 
5031   ins_encode %{
5032     __ movzwl($dst$$Register, $mem$$Address);
5033   %}
5034 
5035   ins_pipe(ialu_reg_mem);
5036 %}
5037 
5038 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5039 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5040   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5041 
5042   ins_cost(125);
5043   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5044   ins_encode %{
5045     __ movsbl($dst$$Register, $mem$$Address);
5046   %}
5047   ins_pipe(ialu_reg_mem);
5048 %}
5049 
5050 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5051 instruct loadUS2L(rRegL dst, memory mem)
5052 %{
5053   match(Set dst (ConvI2L (LoadUS mem)));
5054 
5055   ins_cost(125);
5056   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5057 
5058   ins_encode %{
5059     __ movzwq($dst$$Register, $mem$$Address);
5060   %}
5061 
5062   ins_pipe(ialu_reg_mem);
5063 %}
5064 
5065 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5066 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5067   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5068 
5069   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5070   ins_encode %{
5071     __ movzbq($dst$$Register, $mem$$Address);
5072   %}
5073   ins_pipe(ialu_reg_mem);
5074 %}
5075 
5076 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5077 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5078   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5079   effect(KILL cr);
5080 
5081   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5082             "andl    $dst, right_n_bits($mask, 16)" %}
5083   ins_encode %{
5084     Register Rdst = $dst$$Register;
5085     __ movzwq(Rdst, $mem$$Address);
5086     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5087   %}
5088   ins_pipe(ialu_reg_mem);
5089 %}
5090 
5091 // Load Integer
5092 instruct loadI(rRegI dst, memory mem)
5093 %{
5094   match(Set dst (LoadI mem));
5095 
5096   ins_cost(125);
5097   format %{ "movl    $dst, $mem\t# int" %}
5098 
5099   ins_encode %{
5100     __ movl($dst$$Register, $mem$$Address);
5101   %}
5102 
5103   ins_pipe(ialu_reg_mem);
5104 %}
5105 
5106 // Load Integer (32 bit signed) to Byte (8 bit signed)
5107 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5108   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5109 
5110   ins_cost(125);
5111   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5112   ins_encode %{
5113     __ movsbl($dst$$Register, $mem$$Address);
5114   %}
5115   ins_pipe(ialu_reg_mem);
5116 %}
5117 
5118 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5119 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5120   match(Set dst (AndI (LoadI mem) mask));
5121 
5122   ins_cost(125);
5123   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5124   ins_encode %{
5125     __ movzbl($dst$$Register, $mem$$Address);
5126   %}
5127   ins_pipe(ialu_reg_mem);
5128 %}
5129 
5130 // Load Integer (32 bit signed) to Short (16 bit signed)
5131 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5132   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5133 
5134   ins_cost(125);
5135   format %{ "movswl  $dst, $mem\t# int -> short" %}
5136   ins_encode %{
5137     __ movswl($dst$$Register, $mem$$Address);
5138   %}
5139   ins_pipe(ialu_reg_mem);
5140 %}
5141 
5142 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5143 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5144   match(Set dst (AndI (LoadI mem) mask));
5145 
5146   ins_cost(125);
5147   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5148   ins_encode %{
5149     __ movzwl($dst$$Register, $mem$$Address);
5150   %}
5151   ins_pipe(ialu_reg_mem);
5152 %}
5153 
5154 // Load Integer into Long Register
5155 instruct loadI2L(rRegL dst, memory mem)
5156 %{
5157   match(Set dst (ConvI2L (LoadI mem)));
5158 
5159   ins_cost(125);
5160   format %{ "movslq  $dst, $mem\t# int -> long" %}
5161 
5162   ins_encode %{
5163     __ movslq($dst$$Register, $mem$$Address);
5164   %}
5165 
5166   ins_pipe(ialu_reg_mem);
5167 %}
5168 
5169 // Load Integer with mask 0xFF into Long Register
5170 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5171   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5172 
5173   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5174   ins_encode %{
5175     __ movzbq($dst$$Register, $mem$$Address);
5176   %}
5177   ins_pipe(ialu_reg_mem);
5178 %}
5179 
5180 // Load Integer with mask 0xFFFF into Long Register
5181 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5182   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5183 
5184   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5185   ins_encode %{
5186     __ movzwq($dst$$Register, $mem$$Address);
5187   %}
5188   ins_pipe(ialu_reg_mem);
5189 %}
5190 
5191 // Load Integer with a 31-bit mask into Long Register
5192 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5193   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5194   effect(KILL cr);
5195 
5196   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5197             "andl    $dst, $mask" %}
5198   ins_encode %{
5199     Register Rdst = $dst$$Register;
5200     __ movl(Rdst, $mem$$Address);
5201     __ andl(Rdst, $mask$$constant);
5202   %}
5203   ins_pipe(ialu_reg_mem);
5204 %}
5205 
5206 // Load Unsigned Integer into Long Register
5207 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5208 %{
5209   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5210 
5211   ins_cost(125);
5212   format %{ "movl    $dst, $mem\t# uint -> long" %}
5213 
5214   ins_encode %{
5215     __ movl($dst$$Register, $mem$$Address);
5216   %}
5217 
5218   ins_pipe(ialu_reg_mem);
5219 %}
5220 
5221 // Load Long
5222 instruct loadL(rRegL dst, memory mem)
5223 %{
5224   match(Set dst (LoadL mem));
5225 
5226   ins_cost(125);
5227   format %{ "movq    $dst, $mem\t# long" %}
5228 
5229   ins_encode %{
5230     __ movq($dst$$Register, $mem$$Address);
5231   %}
5232 
5233   ins_pipe(ialu_reg_mem); // XXX
5234 %}
5235 
5236 // Load Range
5237 instruct loadRange(rRegI dst, memory mem)
5238 %{
5239   match(Set dst (LoadRange mem));
5240 
5241   ins_cost(125); // XXX
5242   format %{ "movl    $dst, $mem\t# range" %}
5243   opcode(0x8B);
5244   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5245   ins_pipe(ialu_reg_mem);
5246 %}
5247 
5248 // Load Pointer
5249 instruct loadP(rRegP dst, memory mem)
5250 %{
5251   match(Set dst (LoadP mem));
5252 
5253   ins_cost(125); // XXX
5254   format %{ "movq    $dst, $mem\t# ptr" %}
5255   opcode(0x8B);
5256   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5257   ins_pipe(ialu_reg_mem); // XXX
5258 %}
5259 
5260 // Load Compressed Pointer
5261 instruct loadN(rRegN dst, memory mem)
5262 %{
5263    match(Set dst (LoadN mem));
5264 
5265    ins_cost(125); // XXX
5266    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5267    ins_encode %{
5268      __ movl($dst$$Register, $mem$$Address);
5269    %}
5270    ins_pipe(ialu_reg_mem); // XXX
5271 %}
5272 
5273 
5274 // Load Klass Pointer
5275 instruct loadKlass(rRegP dst, memory mem)
5276 %{
5277   match(Set dst (LoadKlass mem));
5278 
5279   ins_cost(125); // XXX
5280   format %{ "movq    $dst, $mem\t# class" %}
5281   opcode(0x8B);
5282   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5283   ins_pipe(ialu_reg_mem); // XXX
5284 %}
5285 
5286 // Load narrow Klass Pointer
5287 instruct loadNKlass(rRegN dst, memory mem)
5288 %{
5289   match(Set dst (LoadNKlass mem));
5290 
5291   ins_cost(125); // XXX
5292   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5293   ins_encode %{
5294     __ movl($dst$$Register, $mem$$Address);
5295   %}
5296   ins_pipe(ialu_reg_mem); // XXX
5297 %}
5298 
5299 // Load Float
5300 instruct loadF(regF dst, memory mem)
5301 %{
5302   match(Set dst (LoadF mem));
5303 
5304   ins_cost(145); // XXX
5305   format %{ "movss   $dst, $mem\t# float" %}
5306   ins_encode %{
5307     __ movflt($dst$$XMMRegister, $mem$$Address);
5308   %}
5309   ins_pipe(pipe_slow); // XXX
5310 %}
5311 
5312 // Load Double
5313 instruct loadD_partial(regD dst, memory mem)
5314 %{
5315   predicate(!UseXmmLoadAndClearUpper);
5316   match(Set dst (LoadD mem));
5317 
5318   ins_cost(145); // XXX
5319   format %{ "movlpd  $dst, $mem\t# double" %}
5320   ins_encode %{
5321     __ movdbl($dst$$XMMRegister, $mem$$Address);
5322   %}
5323   ins_pipe(pipe_slow); // XXX
5324 %}
5325 
5326 instruct loadD(regD dst, memory mem)
5327 %{
5328   predicate(UseXmmLoadAndClearUpper);
5329   match(Set dst (LoadD mem));
5330 
5331   ins_cost(145); // XXX
5332   format %{ "movsd   $dst, $mem\t# double" %}
5333   ins_encode %{
5334     __ movdbl($dst$$XMMRegister, $mem$$Address);
5335   %}
5336   ins_pipe(pipe_slow); // XXX
5337 %}
5338 
5339 // Load Effective Address
5340 instruct leaP8(rRegP dst, indOffset8 mem)
5341 %{
5342   match(Set dst mem);
5343 
5344   ins_cost(110); // XXX
5345   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5346   opcode(0x8D);
5347   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5348   ins_pipe(ialu_reg_reg_fat);
5349 %}
5350 
5351 instruct leaP32(rRegP dst, indOffset32 mem)
5352 %{
5353   match(Set dst mem);
5354 
5355   ins_cost(110);
5356   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5357   opcode(0x8D);
5358   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5359   ins_pipe(ialu_reg_reg_fat);
5360 %}
5361 
5362 // instruct leaPIdx(rRegP dst, indIndex mem)
5363 // %{
5364 //   match(Set dst mem);
5365 
5366 //   ins_cost(110);
5367 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5368 //   opcode(0x8D);
5369 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5370 //   ins_pipe(ialu_reg_reg_fat);
5371 // %}
5372 
5373 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5374 %{
5375   match(Set dst mem);
5376 
5377   ins_cost(110);
5378   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5379   opcode(0x8D);
5380   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5381   ins_pipe(ialu_reg_reg_fat);
5382 %}
5383 
5384 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5385 %{
5386   match(Set dst mem);
5387 
5388   ins_cost(110);
5389   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5390   opcode(0x8D);
5391   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5392   ins_pipe(ialu_reg_reg_fat);
5393 %}
5394 
5395 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5396 %{
5397   match(Set dst mem);
5398 
5399   ins_cost(110);
5400   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5401   opcode(0x8D);
5402   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5403   ins_pipe(ialu_reg_reg_fat);
5404 %}
5405 
5406 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5407 %{
5408   match(Set dst mem);
5409 
5410   ins_cost(110);
5411   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5412   opcode(0x8D);
5413   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5414   ins_pipe(ialu_reg_reg_fat);
5415 %}
5416 
5417 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5418 %{
5419   match(Set dst mem);
5420 
5421   ins_cost(110);
5422   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5423   opcode(0x8D);
5424   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5425   ins_pipe(ialu_reg_reg_fat);
5426 %}
5427 
5428 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5429 %{
5430   match(Set dst mem);
5431 
5432   ins_cost(110);
5433   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5434   opcode(0x8D);
5435   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5436   ins_pipe(ialu_reg_reg_fat);
5437 %}
5438 
5439 // Load Effective Address which uses Narrow (32-bits) oop
5440 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5441 %{
5442   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5443   match(Set dst mem);
5444 
5445   ins_cost(110);
5446   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5447   opcode(0x8D);
5448   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5449   ins_pipe(ialu_reg_reg_fat);
5450 %}
5451 
5452 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5453 %{
5454   predicate(Universe::narrow_oop_shift() == 0);
5455   match(Set dst mem);
5456 
5457   ins_cost(110); // XXX
5458   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5459   opcode(0x8D);
5460   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5461   ins_pipe(ialu_reg_reg_fat);
5462 %}
5463 
5464 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5465 %{
5466   predicate(Universe::narrow_oop_shift() == 0);
5467   match(Set dst mem);
5468 
5469   ins_cost(110);
5470   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5471   opcode(0x8D);
5472   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5473   ins_pipe(ialu_reg_reg_fat);
5474 %}
5475 
5476 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5477 %{
5478   predicate(Universe::narrow_oop_shift() == 0);
5479   match(Set dst mem);
5480 
5481   ins_cost(110);
5482   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5483   opcode(0x8D);
5484   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5485   ins_pipe(ialu_reg_reg_fat);
5486 %}
5487 
5488 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5489 %{
5490   predicate(Universe::narrow_oop_shift() == 0);
5491   match(Set dst mem);
5492 
5493   ins_cost(110);
5494   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5495   opcode(0x8D);
5496   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5497   ins_pipe(ialu_reg_reg_fat);
5498 %}
5499 
5500 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5501 %{
5502   predicate(Universe::narrow_oop_shift() == 0);
5503   match(Set dst mem);
5504 
5505   ins_cost(110);
5506   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5507   opcode(0x8D);
5508   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5509   ins_pipe(ialu_reg_reg_fat);
5510 %}
5511 
5512 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5513 %{
5514   predicate(Universe::narrow_oop_shift() == 0);
5515   match(Set dst mem);
5516 
5517   ins_cost(110);
5518   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5519   opcode(0x8D);
5520   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5521   ins_pipe(ialu_reg_reg_fat);
5522 %}
5523 
5524 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5525 %{
5526   predicate(Universe::narrow_oop_shift() == 0);
5527   match(Set dst mem);
5528 
5529   ins_cost(110);
5530   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5531   opcode(0x8D);
5532   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5533   ins_pipe(ialu_reg_reg_fat);
5534 %}
5535 
5536 instruct loadConI(rRegI dst, immI src)
5537 %{
5538   match(Set dst src);
5539 
5540   format %{ "movl    $dst, $src\t# int" %}
5541   ins_encode(load_immI(dst, src));
5542   ins_pipe(ialu_reg_fat); // XXX
5543 %}
5544 
5545 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5546 %{
5547   match(Set dst src);
5548   effect(KILL cr);
5549 
5550   ins_cost(50);
5551   format %{ "xorl    $dst, $dst\t# int" %}
5552   opcode(0x33); /* + rd */
5553   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5554   ins_pipe(ialu_reg);
5555 %}
5556 
5557 instruct loadConL(rRegL dst, immL src)
5558 %{
5559   match(Set dst src);
5560 
5561   ins_cost(150);
5562   format %{ "movq    $dst, $src\t# long" %}
5563   ins_encode(load_immL(dst, src));
5564   ins_pipe(ialu_reg);
5565 %}
5566 
5567 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5568 %{
5569   match(Set dst src);
5570   effect(KILL cr);
5571 
5572   ins_cost(50);
5573   format %{ "xorl    $dst, $dst\t# long" %}
5574   opcode(0x33); /* + rd */
5575   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5576   ins_pipe(ialu_reg); // XXX
5577 %}
5578 
5579 instruct loadConUL32(rRegL dst, immUL32 src)
5580 %{
5581   match(Set dst src);
5582 
5583   ins_cost(60);
5584   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5585   ins_encode(load_immUL32(dst, src));
5586   ins_pipe(ialu_reg);
5587 %}
5588 
5589 instruct loadConL32(rRegL dst, immL32 src)
5590 %{
5591   match(Set dst src);
5592 
5593   ins_cost(70);
5594   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5595   ins_encode(load_immL32(dst, src));
5596   ins_pipe(ialu_reg);
5597 %}
5598 
5599 instruct loadConP(rRegP dst, immP con) %{
5600   match(Set dst con);
5601 
5602   format %{ "movq    $dst, $con\t# ptr" %}
5603   ins_encode(load_immP(dst, con));
5604   ins_pipe(ialu_reg_fat); // XXX
5605 %}
5606 
5607 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5608 %{
5609   match(Set dst src);
5610   effect(KILL cr);
5611 
5612   ins_cost(50);
5613   format %{ "xorl    $dst, $dst\t# ptr" %}
5614   opcode(0x33); /* + rd */
5615   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5616   ins_pipe(ialu_reg);
5617 %}
5618 
5619 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5620 %{
5621   match(Set dst src);
5622   effect(KILL cr);
5623 
5624   ins_cost(60);
5625   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5626   ins_encode(load_immP31(dst, src));
5627   ins_pipe(ialu_reg);
5628 %}
5629 
5630 instruct loadConF(regF dst, immF con) %{
5631   match(Set dst con);
5632   ins_cost(125);
5633   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5634   ins_encode %{
5635     __ movflt($dst$$XMMRegister, $constantaddress($con));
5636   %}
5637   ins_pipe(pipe_slow);
5638 %}
5639 
5640 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5641   match(Set dst src);
5642   effect(KILL cr);
5643   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5644   ins_encode %{
5645     __ xorq($dst$$Register, $dst$$Register);
5646   %}
5647   ins_pipe(ialu_reg);
5648 %}
5649 
5650 instruct loadConN(rRegN dst, immN src) %{
5651   match(Set dst src);
5652 
5653   ins_cost(125);
5654   format %{ "movl    $dst, $src\t# compressed ptr" %}
5655   ins_encode %{
5656     address con = (address)$src$$constant;
5657     if (con == NULL) {
5658       ShouldNotReachHere();
5659     } else {
5660       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5661     }
5662   %}
5663   ins_pipe(ialu_reg_fat); // XXX
5664 %}
5665 
5666 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5667   match(Set dst src);
5668 
5669   ins_cost(125);
5670   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5671   ins_encode %{
5672     address con = (address)$src$$constant;
5673     if (con == NULL) {
5674       ShouldNotReachHere();
5675     } else {
5676       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5677     }
5678   %}
5679   ins_pipe(ialu_reg_fat); // XXX
5680 %}
5681 
5682 instruct loadConF0(regF dst, immF0 src)
5683 %{
5684   match(Set dst src);
5685   ins_cost(100);
5686 
5687   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5688   ins_encode %{
5689     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5690   %}
5691   ins_pipe(pipe_slow);
5692 %}
5693 
5694 // Use the same format since predicate() can not be used here.
5695 instruct loadConD(regD dst, immD con) %{
5696   match(Set dst con);
5697   ins_cost(125);
5698   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5699   ins_encode %{
5700     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5701   %}
5702   ins_pipe(pipe_slow);
5703 %}
5704 
5705 instruct loadConD0(regD dst, immD0 src)
5706 %{
5707   match(Set dst src);
5708   ins_cost(100);
5709 
5710   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5711   ins_encode %{
5712     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5713   %}
5714   ins_pipe(pipe_slow);
5715 %}
5716 
5717 instruct loadSSI(rRegI dst, stackSlotI src)
5718 %{
5719   match(Set dst src);
5720 
5721   ins_cost(125);
5722   format %{ "movl    $dst, $src\t# int stk" %}
5723   opcode(0x8B);
5724   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5725   ins_pipe(ialu_reg_mem);
5726 %}
5727 
5728 instruct loadSSL(rRegL dst, stackSlotL src)
5729 %{
5730   match(Set dst src);
5731 
5732   ins_cost(125);
5733   format %{ "movq    $dst, $src\t# long stk" %}
5734   opcode(0x8B);
5735   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5736   ins_pipe(ialu_reg_mem);
5737 %}
5738 
5739 instruct loadSSP(rRegP dst, stackSlotP src)
5740 %{
5741   match(Set dst src);
5742 
5743   ins_cost(125);
5744   format %{ "movq    $dst, $src\t# ptr stk" %}
5745   opcode(0x8B);
5746   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5747   ins_pipe(ialu_reg_mem);
5748 %}
5749 
5750 instruct loadSSF(regF dst, stackSlotF src)
5751 %{
5752   match(Set dst src);
5753 
5754   ins_cost(125);
5755   format %{ "movss   $dst, $src\t# float stk" %}
5756   ins_encode %{
5757     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5758   %}
5759   ins_pipe(pipe_slow); // XXX
5760 %}
5761 
5762 // Use the same format since predicate() can not be used here.
5763 instruct loadSSD(regD dst, stackSlotD src)
5764 %{
5765   match(Set dst src);
5766 
5767   ins_cost(125);
5768   format %{ "movsd   $dst, $src\t# double stk" %}
5769   ins_encode  %{
5770     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5771   %}
5772   ins_pipe(pipe_slow); // XXX
5773 %}
5774 
5775 // Prefetch instructions for allocation.
5776 // Must be safe to execute with invalid address (cannot fault).
5777 
5778 instruct prefetchAlloc( memory mem ) %{
5779   predicate(AllocatePrefetchInstr==3);
5780   match(PrefetchAllocation mem);
5781   ins_cost(125);
5782 
5783   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5784   ins_encode %{
5785     __ prefetchw($mem$$Address);
5786   %}
5787   ins_pipe(ialu_mem);
5788 %}
5789 
5790 instruct prefetchAllocNTA( memory mem ) %{
5791   predicate(AllocatePrefetchInstr==0);
5792   match(PrefetchAllocation mem);
5793   ins_cost(125);
5794 
5795   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5796   ins_encode %{
5797     __ prefetchnta($mem$$Address);
5798   %}
5799   ins_pipe(ialu_mem);
5800 %}
5801 
5802 instruct prefetchAllocT0( memory mem ) %{
5803   predicate(AllocatePrefetchInstr==1);
5804   match(PrefetchAllocation mem);
5805   ins_cost(125);
5806 
5807   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5808   ins_encode %{
5809     __ prefetcht0($mem$$Address);
5810   %}
5811   ins_pipe(ialu_mem);
5812 %}
5813 
5814 instruct prefetchAllocT2( memory mem ) %{
5815   predicate(AllocatePrefetchInstr==2);
5816   match(PrefetchAllocation mem);
5817   ins_cost(125);
5818 
5819   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5820   ins_encode %{
5821     __ prefetcht2($mem$$Address);
5822   %}
5823   ins_pipe(ialu_mem);
5824 %}
5825 
5826 //----------Store Instructions-------------------------------------------------
5827 
5828 // Store Byte
5829 instruct storeB(memory mem, rRegI src)
5830 %{
5831   match(Set mem (StoreB mem src));
5832 
5833   ins_cost(125); // XXX
5834   format %{ "movb    $mem, $src\t# byte" %}
5835   opcode(0x88);
5836   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5837   ins_pipe(ialu_mem_reg);
5838 %}
5839 
5840 // Store Char/Short
5841 instruct storeC(memory mem, rRegI src)
5842 %{
5843   match(Set mem (StoreC mem src));
5844 
5845   ins_cost(125); // XXX
5846   format %{ "movw    $mem, $src\t# char/short" %}
5847   opcode(0x89);
5848   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5849   ins_pipe(ialu_mem_reg);
5850 %}
5851 
5852 // Store Integer
5853 instruct storeI(memory mem, rRegI src)
5854 %{
5855   match(Set mem (StoreI mem src));
5856 
5857   ins_cost(125); // XXX
5858   format %{ "movl    $mem, $src\t# int" %}
5859   opcode(0x89);
5860   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5861   ins_pipe(ialu_mem_reg);
5862 %}
5863 
5864 // Store Long
5865 instruct storeL(memory mem, rRegL src)
5866 %{
5867   match(Set mem (StoreL mem src));
5868 
5869   ins_cost(125); // XXX
5870   format %{ "movq    $mem, $src\t# long" %}
5871   opcode(0x89);
5872   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5873   ins_pipe(ialu_mem_reg); // XXX
5874 %}
5875 
5876 // Store Pointer
5877 instruct storeP(memory mem, any_RegP src)
5878 %{
5879   match(Set mem (StoreP mem src));
5880 
5881   ins_cost(125); // XXX
5882   format %{ "movq    $mem, $src\t# ptr" %}
5883   opcode(0x89);
5884   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5885   ins_pipe(ialu_mem_reg);
5886 %}
5887 
5888 instruct storeImmP0(memory mem, immP0 zero)
5889 %{
5890   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5891   match(Set mem (StoreP mem zero));
5892 
5893   ins_cost(125); // XXX
5894   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5895   ins_encode %{
5896     __ movq($mem$$Address, r12);
5897   %}
5898   ins_pipe(ialu_mem_reg);
5899 %}
5900 
5901 // Store NULL Pointer, mark word, or other simple pointer constant.
5902 instruct storeImmP(memory mem, immP31 src)
5903 %{
5904   match(Set mem (StoreP mem src));
5905 
5906   ins_cost(150); // XXX
5907   format %{ "movq    $mem, $src\t# ptr" %}
5908   opcode(0xC7); /* C7 /0 */
5909   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5910   ins_pipe(ialu_mem_imm);
5911 %}
5912 
5913 // Store Compressed Pointer
5914 instruct storeN(memory mem, rRegN src)
5915 %{
5916   match(Set mem (StoreN mem src));
5917 
5918   ins_cost(125); // XXX
5919   format %{ "movl    $mem, $src\t# compressed ptr" %}
5920   ins_encode %{
5921     __ movl($mem$$Address, $src$$Register);
5922   %}
5923   ins_pipe(ialu_mem_reg);
5924 %}
5925 
5926 instruct storeNKlass(memory mem, rRegN src)
5927 %{
5928   match(Set mem (StoreNKlass mem src));
5929 
5930   ins_cost(125); // XXX
5931   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5932   ins_encode %{
5933     __ movl($mem$$Address, $src$$Register);
5934   %}
5935   ins_pipe(ialu_mem_reg);
5936 %}
5937 
5938 instruct storeImmN0(memory mem, immN0 zero)
5939 %{
5940   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5941   match(Set mem (StoreN mem zero));
5942 
5943   ins_cost(125); // XXX
5944   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5945   ins_encode %{
5946     __ movl($mem$$Address, r12);
5947   %}
5948   ins_pipe(ialu_mem_reg);
5949 %}
5950 
5951 instruct storeImmN(memory mem, immN src)
5952 %{
5953   match(Set mem (StoreN mem src));
5954 
5955   ins_cost(150); // XXX
5956   format %{ "movl    $mem, $src\t# compressed ptr" %}
5957   ins_encode %{
5958     address con = (address)$src$$constant;
5959     if (con == NULL) {
5960       __ movl($mem$$Address, (int32_t)0);
5961     } else {
5962       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5963     }
5964   %}
5965   ins_pipe(ialu_mem_imm);
5966 %}
5967 
5968 instruct storeImmNKlass(memory mem, immNKlass src)
5969 %{
5970   match(Set mem (StoreNKlass mem src));
5971 
5972   ins_cost(150); // XXX
5973   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5974   ins_encode %{
5975     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5976   %}
5977   ins_pipe(ialu_mem_imm);
5978 %}
5979 
5980 // Store Integer Immediate
5981 instruct storeImmI0(memory mem, immI0 zero)
5982 %{
5983   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5984   match(Set mem (StoreI mem zero));
5985 
5986   ins_cost(125); // XXX
5987   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5988   ins_encode %{
5989     __ movl($mem$$Address, r12);
5990   %}
5991   ins_pipe(ialu_mem_reg);
5992 %}
5993 
5994 instruct storeImmI(memory mem, immI src)
5995 %{
5996   match(Set mem (StoreI mem src));
5997 
5998   ins_cost(150);
5999   format %{ "movl    $mem, $src\t# int" %}
6000   opcode(0xC7); /* C7 /0 */
6001   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6002   ins_pipe(ialu_mem_imm);
6003 %}
6004 
6005 // Store Long Immediate
6006 instruct storeImmL0(memory mem, immL0 zero)
6007 %{
6008   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6009   match(Set mem (StoreL mem zero));
6010 
6011   ins_cost(125); // XXX
6012   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6013   ins_encode %{
6014     __ movq($mem$$Address, r12);
6015   %}
6016   ins_pipe(ialu_mem_reg);
6017 %}
6018 
6019 instruct storeImmL(memory mem, immL32 src)
6020 %{
6021   match(Set mem (StoreL mem src));
6022 
6023   ins_cost(150);
6024   format %{ "movq    $mem, $src\t# long" %}
6025   opcode(0xC7); /* C7 /0 */
6026   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6027   ins_pipe(ialu_mem_imm);
6028 %}
6029 
6030 // Store Short/Char Immediate
6031 instruct storeImmC0(memory mem, immI0 zero)
6032 %{
6033   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6034   match(Set mem (StoreC mem zero));
6035 
6036   ins_cost(125); // XXX
6037   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6038   ins_encode %{
6039     __ movw($mem$$Address, r12);
6040   %}
6041   ins_pipe(ialu_mem_reg);
6042 %}
6043 
6044 instruct storeImmI16(memory mem, immI16 src)
6045 %{
6046   predicate(UseStoreImmI16);
6047   match(Set mem (StoreC mem src));
6048 
6049   ins_cost(150);
6050   format %{ "movw    $mem, $src\t# short/char" %}
6051   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6052   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6053   ins_pipe(ialu_mem_imm);
6054 %}
6055 
6056 // Store Byte Immediate
6057 instruct storeImmB0(memory mem, immI0 zero)
6058 %{
6059   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6060   match(Set mem (StoreB mem zero));
6061 
6062   ins_cost(125); // XXX
6063   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6064   ins_encode %{
6065     __ movb($mem$$Address, r12);
6066   %}
6067   ins_pipe(ialu_mem_reg);
6068 %}
6069 
6070 instruct storeImmB(memory mem, immI8 src)
6071 %{
6072   match(Set mem (StoreB mem src));
6073 
6074   ins_cost(150); // XXX
6075   format %{ "movb    $mem, $src\t# byte" %}
6076   opcode(0xC6); /* C6 /0 */
6077   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6078   ins_pipe(ialu_mem_imm);
6079 %}
6080 
6081 // Store CMS card-mark Immediate
6082 instruct storeImmCM0_reg(memory mem, immI0 zero)
6083 %{
6084   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6085   match(Set mem (StoreCM mem zero));
6086 
6087   ins_cost(125); // XXX
6088   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6089   ins_encode %{
6090     __ movb($mem$$Address, r12);
6091   %}
6092   ins_pipe(ialu_mem_reg);
6093 %}
6094 
6095 instruct storeImmCM0(memory mem, immI0 src)
6096 %{
6097   match(Set mem (StoreCM mem src));
6098 
6099   ins_cost(150); // XXX
6100   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6101   opcode(0xC6); /* C6 /0 */
6102   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6103   ins_pipe(ialu_mem_imm);
6104 %}
6105 
6106 // Store Float
6107 instruct storeF(memory mem, regF src)
6108 %{
6109   match(Set mem (StoreF mem src));
6110 
6111   ins_cost(95); // XXX
6112   format %{ "movss   $mem, $src\t# float" %}
6113   ins_encode %{
6114     __ movflt($mem$$Address, $src$$XMMRegister);
6115   %}
6116   ins_pipe(pipe_slow); // XXX
6117 %}
6118 
6119 // Store immediate Float value (it is faster than store from XMM register)
6120 instruct storeF0(memory mem, immF0 zero)
6121 %{
6122   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6123   match(Set mem (StoreF mem zero));
6124 
6125   ins_cost(25); // XXX
6126   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6127   ins_encode %{
6128     __ movl($mem$$Address, r12);
6129   %}
6130   ins_pipe(ialu_mem_reg);
6131 %}
6132 
6133 instruct storeF_imm(memory mem, immF src)
6134 %{
6135   match(Set mem (StoreF mem src));
6136 
6137   ins_cost(50);
6138   format %{ "movl    $mem, $src\t# float" %}
6139   opcode(0xC7); /* C7 /0 */
6140   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6141   ins_pipe(ialu_mem_imm);
6142 %}
6143 
6144 // Store Double
6145 instruct storeD(memory mem, regD src)
6146 %{
6147   match(Set mem (StoreD mem src));
6148 
6149   ins_cost(95); // XXX
6150   format %{ "movsd   $mem, $src\t# double" %}
6151   ins_encode %{
6152     __ movdbl($mem$$Address, $src$$XMMRegister);
6153   %}
6154   ins_pipe(pipe_slow); // XXX
6155 %}
6156 
6157 // Store immediate double 0.0 (it is faster than store from XMM register)
6158 instruct storeD0_imm(memory mem, immD0 src)
6159 %{
6160   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6161   match(Set mem (StoreD mem src));
6162 
6163   ins_cost(50);
6164   format %{ "movq    $mem, $src\t# double 0." %}
6165   opcode(0xC7); /* C7 /0 */
6166   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6167   ins_pipe(ialu_mem_imm);
6168 %}
6169 
6170 instruct storeD0(memory mem, immD0 zero)
6171 %{
6172   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6173   match(Set mem (StoreD mem zero));
6174 
6175   ins_cost(25); // XXX
6176   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6177   ins_encode %{
6178     __ movq($mem$$Address, r12);
6179   %}
6180   ins_pipe(ialu_mem_reg);
6181 %}
6182 
6183 instruct storeSSI(stackSlotI dst, rRegI src)
6184 %{
6185   match(Set dst src);
6186 
6187   ins_cost(100);
6188   format %{ "movl    $dst, $src\t# int stk" %}
6189   opcode(0x89);
6190   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6191   ins_pipe( ialu_mem_reg );
6192 %}
6193 
6194 instruct storeSSL(stackSlotL dst, rRegL src)
6195 %{
6196   match(Set dst src);
6197 
6198   ins_cost(100);
6199   format %{ "movq    $dst, $src\t# long stk" %}
6200   opcode(0x89);
6201   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6202   ins_pipe(ialu_mem_reg);
6203 %}
6204 
6205 instruct storeSSP(stackSlotP dst, rRegP src)
6206 %{
6207   match(Set dst src);
6208 
6209   ins_cost(100);
6210   format %{ "movq    $dst, $src\t# ptr stk" %}
6211   opcode(0x89);
6212   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6213   ins_pipe(ialu_mem_reg);
6214 %}
6215 
6216 instruct storeSSF(stackSlotF dst, regF src)
6217 %{
6218   match(Set dst src);
6219 
6220   ins_cost(95); // XXX
6221   format %{ "movss   $dst, $src\t# float stk" %}
6222   ins_encode %{
6223     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6224   %}
6225   ins_pipe(pipe_slow); // XXX
6226 %}
6227 
6228 instruct storeSSD(stackSlotD dst, regD src)
6229 %{
6230   match(Set dst src);
6231 
6232   ins_cost(95); // XXX
6233   format %{ "movsd   $dst, $src\t# double stk" %}
6234   ins_encode %{
6235     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6236   %}
6237   ins_pipe(pipe_slow); // XXX
6238 %}
6239 
6240 instruct cacheWB(indirect addr)
6241 %{
6242   match(CacheWB addr);
6243 
6244   ins_cost(100);
6245   format %{"cache wb $addr" %}
6246   ins_encode %{
6247     assert($addr->index_position() < 0, "should be");
6248     assert($addr$$disp == 0, "should be");
6249     __ cache_wb(Address($addr$$base$$Register, 0));
6250   %}
6251   ins_pipe(pipe_slow); // XXX
6252 %}
6253 
6254 instruct cacheWBPreSync()
6255 %{
6256   match(CacheWBPreSync);
6257 
6258   ins_cost(100);
6259   format %{"cache wb presync" %}
6260   ins_encode %{
6261     __ cache_wbsync(true);
6262   %}
6263   ins_pipe(pipe_slow); // XXX
6264 %}
6265 
6266 instruct cacheWBPostSync()
6267 %{
6268   match(CacheWBPostSync);
6269 
6270   ins_cost(100);
6271   format %{"cache wb postsync" %}
6272   ins_encode %{
6273     __ cache_wbsync(false);
6274   %}
6275   ins_pipe(pipe_slow); // XXX
6276 %}
6277 
6278 //----------BSWAP Instructions-------------------------------------------------
6279 instruct bytes_reverse_int(rRegI dst) %{
6280   match(Set dst (ReverseBytesI dst));
6281 
6282   format %{ "bswapl  $dst" %}
6283   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6284   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6285   ins_pipe( ialu_reg );
6286 %}
6287 
6288 instruct bytes_reverse_long(rRegL dst) %{
6289   match(Set dst (ReverseBytesL dst));
6290 
6291   format %{ "bswapq  $dst" %}
6292   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6293   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6294   ins_pipe( ialu_reg);
6295 %}
6296 
6297 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6298   match(Set dst (ReverseBytesUS dst));
6299   effect(KILL cr);
6300 
6301   format %{ "bswapl  $dst\n\t"
6302             "shrl    $dst,16\n\t" %}
6303   ins_encode %{
6304     __ bswapl($dst$$Register);
6305     __ shrl($dst$$Register, 16);
6306   %}
6307   ins_pipe( ialu_reg );
6308 %}
6309 
6310 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6311   match(Set dst (ReverseBytesS dst));
6312   effect(KILL cr);
6313 
6314   format %{ "bswapl  $dst\n\t"
6315             "sar     $dst,16\n\t" %}
6316   ins_encode %{
6317     __ bswapl($dst$$Register);
6318     __ sarl($dst$$Register, 16);
6319   %}
6320   ins_pipe( ialu_reg );
6321 %}
6322 
6323 //---------- Zeros Count Instructions ------------------------------------------
6324 
6325 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6326   predicate(UseCountLeadingZerosInstruction);
6327   match(Set dst (CountLeadingZerosI src));
6328   effect(KILL cr);
6329 
6330   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6331   ins_encode %{
6332     __ lzcntl($dst$$Register, $src$$Register);
6333   %}
6334   ins_pipe(ialu_reg);
6335 %}
6336 
6337 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6338   predicate(!UseCountLeadingZerosInstruction);
6339   match(Set dst (CountLeadingZerosI src));
6340   effect(KILL cr);
6341 
6342   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6343             "jnz     skip\n\t"
6344             "movl    $dst, -1\n"
6345       "skip:\n\t"
6346             "negl    $dst\n\t"
6347             "addl    $dst, 31" %}
6348   ins_encode %{
6349     Register Rdst = $dst$$Register;
6350     Register Rsrc = $src$$Register;
6351     Label skip;
6352     __ bsrl(Rdst, Rsrc);
6353     __ jccb(Assembler::notZero, skip);
6354     __ movl(Rdst, -1);
6355     __ bind(skip);
6356     __ negl(Rdst);
6357     __ addl(Rdst, BitsPerInt - 1);
6358   %}
6359   ins_pipe(ialu_reg);
6360 %}
6361 
6362 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6363   predicate(UseCountLeadingZerosInstruction);
6364   match(Set dst (CountLeadingZerosL src));
6365   effect(KILL cr);
6366 
6367   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6368   ins_encode %{
6369     __ lzcntq($dst$$Register, $src$$Register);
6370   %}
6371   ins_pipe(ialu_reg);
6372 %}
6373 
6374 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6375   predicate(!UseCountLeadingZerosInstruction);
6376   match(Set dst (CountLeadingZerosL src));
6377   effect(KILL cr);
6378 
6379   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6380             "jnz     skip\n\t"
6381             "movl    $dst, -1\n"
6382       "skip:\n\t"
6383             "negl    $dst\n\t"
6384             "addl    $dst, 63" %}
6385   ins_encode %{
6386     Register Rdst = $dst$$Register;
6387     Register Rsrc = $src$$Register;
6388     Label skip;
6389     __ bsrq(Rdst, Rsrc);
6390     __ jccb(Assembler::notZero, skip);
6391     __ movl(Rdst, -1);
6392     __ bind(skip);
6393     __ negl(Rdst);
6394     __ addl(Rdst, BitsPerLong - 1);
6395   %}
6396   ins_pipe(ialu_reg);
6397 %}
6398 
6399 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6400   predicate(UseCountTrailingZerosInstruction);
6401   match(Set dst (CountTrailingZerosI src));
6402   effect(KILL cr);
6403 
6404   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6405   ins_encode %{
6406     __ tzcntl($dst$$Register, $src$$Register);
6407   %}
6408   ins_pipe(ialu_reg);
6409 %}
6410 
6411 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6412   predicate(!UseCountTrailingZerosInstruction);
6413   match(Set dst (CountTrailingZerosI src));
6414   effect(KILL cr);
6415 
6416   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6417             "jnz     done\n\t"
6418             "movl    $dst, 32\n"
6419       "done:" %}
6420   ins_encode %{
6421     Register Rdst = $dst$$Register;
6422     Label done;
6423     __ bsfl(Rdst, $src$$Register);
6424     __ jccb(Assembler::notZero, done);
6425     __ movl(Rdst, BitsPerInt);
6426     __ bind(done);
6427   %}
6428   ins_pipe(ialu_reg);
6429 %}
6430 
6431 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6432   predicate(UseCountTrailingZerosInstruction);
6433   match(Set dst (CountTrailingZerosL src));
6434   effect(KILL cr);
6435 
6436   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6437   ins_encode %{
6438     __ tzcntq($dst$$Register, $src$$Register);
6439   %}
6440   ins_pipe(ialu_reg);
6441 %}
6442 
6443 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6444   predicate(!UseCountTrailingZerosInstruction);
6445   match(Set dst (CountTrailingZerosL src));
6446   effect(KILL cr);
6447 
6448   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6449             "jnz     done\n\t"
6450             "movl    $dst, 64\n"
6451       "done:" %}
6452   ins_encode %{
6453     Register Rdst = $dst$$Register;
6454     Label done;
6455     __ bsfq(Rdst, $src$$Register);
6456     __ jccb(Assembler::notZero, done);
6457     __ movl(Rdst, BitsPerLong);
6458     __ bind(done);
6459   %}
6460   ins_pipe(ialu_reg);
6461 %}
6462 
6463 
6464 //---------- Population Count Instructions -------------------------------------
6465 
6466 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6467   predicate(UsePopCountInstruction);
6468   match(Set dst (PopCountI src));
6469   effect(KILL cr);
6470 
6471   format %{ "popcnt  $dst, $src" %}
6472   ins_encode %{
6473     __ popcntl($dst$$Register, $src$$Register);
6474   %}
6475   ins_pipe(ialu_reg);
6476 %}
6477 
6478 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6479   predicate(UsePopCountInstruction);
6480   match(Set dst (PopCountI (LoadI mem)));
6481   effect(KILL cr);
6482 
6483   format %{ "popcnt  $dst, $mem" %}
6484   ins_encode %{
6485     __ popcntl($dst$$Register, $mem$$Address);
6486   %}
6487   ins_pipe(ialu_reg);
6488 %}
6489 
6490 // Note: Long.bitCount(long) returns an int.
6491 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6492   predicate(UsePopCountInstruction);
6493   match(Set dst (PopCountL src));
6494   effect(KILL cr);
6495 
6496   format %{ "popcnt  $dst, $src" %}
6497   ins_encode %{
6498     __ popcntq($dst$$Register, $src$$Register);
6499   %}
6500   ins_pipe(ialu_reg);
6501 %}
6502 
6503 // Note: Long.bitCount(long) returns an int.
6504 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6505   predicate(UsePopCountInstruction);
6506   match(Set dst (PopCountL (LoadL mem)));
6507   effect(KILL cr);
6508 
6509   format %{ "popcnt  $dst, $mem" %}
6510   ins_encode %{
6511     __ popcntq($dst$$Register, $mem$$Address);
6512   %}
6513   ins_pipe(ialu_reg);
6514 %}
6515 
6516 
6517 //----------MemBar Instructions-----------------------------------------------
6518 // Memory barrier flavors
6519 
6520 instruct membar_acquire()
6521 %{
6522   match(MemBarAcquire);
6523   match(LoadFence);
6524   ins_cost(0);
6525 
6526   size(0);
6527   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6528   ins_encode();
6529   ins_pipe(empty);
6530 %}
6531 
6532 instruct membar_acquire_lock()
6533 %{
6534   match(MemBarAcquireLock);
6535   ins_cost(0);
6536 
6537   size(0);
6538   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6539   ins_encode();
6540   ins_pipe(empty);
6541 %}
6542 
6543 instruct membar_release()
6544 %{
6545   match(MemBarRelease);
6546   match(StoreFence);
6547   ins_cost(0);
6548 
6549   size(0);
6550   format %{ "MEMBAR-release ! (empty encoding)" %}
6551   ins_encode();
6552   ins_pipe(empty);
6553 %}
6554 
6555 instruct membar_release_lock()
6556 %{
6557   match(MemBarReleaseLock);
6558   ins_cost(0);
6559 
6560   size(0);
6561   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6562   ins_encode();
6563   ins_pipe(empty);
6564 %}
6565 
6566 instruct membar_volatile(rFlagsReg cr) %{
6567   match(MemBarVolatile);
6568   effect(KILL cr);
6569   ins_cost(400);
6570 
6571   format %{
6572     $$template
6573     if (os::is_MP()) {
6574       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6575     } else {
6576       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6577     }
6578   %}
6579   ins_encode %{
6580     __ membar(Assembler::StoreLoad);
6581   %}
6582   ins_pipe(pipe_slow);
6583 %}
6584 
6585 instruct unnecessary_membar_volatile()
6586 %{
6587   match(MemBarVolatile);
6588   predicate(Matcher::post_store_load_barrier(n));
6589   ins_cost(0);
6590 
6591   size(0);
6592   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6593   ins_encode();
6594   ins_pipe(empty);
6595 %}
6596 
6597 instruct membar_storestore() %{
6598   match(MemBarStoreStore);
6599   ins_cost(0);
6600 
6601   size(0);
6602   format %{ "MEMBAR-storestore (empty encoding)" %}
6603   ins_encode( );
6604   ins_pipe(empty);
6605 %}
6606 
6607 //----------Move Instructions--------------------------------------------------
6608 
6609 instruct castX2P(rRegP dst, rRegL src)
6610 %{
6611   match(Set dst (CastX2P src));
6612 
6613   format %{ "movq    $dst, $src\t# long->ptr" %}
6614   ins_encode %{
6615     if ($dst$$reg != $src$$reg) {
6616       __ movptr($dst$$Register, $src$$Register);
6617     }
6618   %}
6619   ins_pipe(ialu_reg_reg); // XXX
6620 %}
6621 
6622 instruct castP2X(rRegL dst, rRegP src)
6623 %{
6624   match(Set dst (CastP2X src));
6625 
6626   format %{ "movq    $dst, $src\t# ptr -> long" %}
6627   ins_encode %{
6628     if ($dst$$reg != $src$$reg) {
6629       __ movptr($dst$$Register, $src$$Register);
6630     }
6631   %}
6632   ins_pipe(ialu_reg_reg); // XXX
6633 %}
6634 
6635 // Convert oop into int for vectors alignment masking
6636 instruct convP2I(rRegI dst, rRegP src)
6637 %{
6638   match(Set dst (ConvL2I (CastP2X src)));
6639 
6640   format %{ "movl    $dst, $src\t# ptr -> int" %}
6641   ins_encode %{
6642     __ movl($dst$$Register, $src$$Register);
6643   %}
6644   ins_pipe(ialu_reg_reg); // XXX
6645 %}
6646 
6647 // Convert compressed oop into int for vectors alignment masking
6648 // in case of 32bit oops (heap < 4Gb).
6649 instruct convN2I(rRegI dst, rRegN src)
6650 %{
6651   predicate(Universe::narrow_oop_shift() == 0);
6652   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6653 
6654   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6655   ins_encode %{
6656     __ movl($dst$$Register, $src$$Register);
6657   %}
6658   ins_pipe(ialu_reg_reg); // XXX
6659 %}
6660 
6661 // Convert oop pointer into compressed form
6662 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6663   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6664   match(Set dst (EncodeP src));
6665   effect(KILL cr);
6666   format %{ "encode_heap_oop $dst,$src" %}
6667   ins_encode %{
6668     Register s = $src$$Register;
6669     Register d = $dst$$Register;
6670     if (s != d) {
6671       __ movq(d, s);
6672     }
6673     __ encode_heap_oop(d);
6674   %}
6675   ins_pipe(ialu_reg_long);
6676 %}
6677 
6678 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6679   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6680   match(Set dst (EncodeP src));
6681   effect(KILL cr);
6682   format %{ "encode_heap_oop_not_null $dst,$src" %}
6683   ins_encode %{
6684     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6685   %}
6686   ins_pipe(ialu_reg_long);
6687 %}
6688 
6689 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6690   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6691             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6692   match(Set dst (DecodeN src));
6693   effect(KILL cr);
6694   format %{ "decode_heap_oop $dst,$src" %}
6695   ins_encode %{
6696     Register s = $src$$Register;
6697     Register d = $dst$$Register;
6698     if (s != d) {
6699       __ movq(d, s);
6700     }
6701     __ decode_heap_oop(d);
6702   %}
6703   ins_pipe(ialu_reg_long);
6704 %}
6705 
6706 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6707   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6708             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6709   match(Set dst (DecodeN src));
6710   effect(KILL cr);
6711   format %{ "decode_heap_oop_not_null $dst,$src" %}
6712   ins_encode %{
6713     Register s = $src$$Register;
6714     Register d = $dst$$Register;
6715     if (s != d) {
6716       __ decode_heap_oop_not_null(d, s);
6717     } else {
6718       __ decode_heap_oop_not_null(d);
6719     }
6720   %}
6721   ins_pipe(ialu_reg_long);
6722 %}
6723 
6724 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6725   match(Set dst (EncodePKlass src));
6726   effect(KILL cr);
6727   format %{ "encode_klass_not_null $dst,$src" %}
6728   ins_encode %{
6729     __ encode_klass_not_null($dst$$Register, $src$$Register);
6730   %}
6731   ins_pipe(ialu_reg_long);
6732 %}
6733 
6734 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6735   match(Set dst (DecodeNKlass src));
6736   effect(KILL cr);
6737   format %{ "decode_klass_not_null $dst,$src" %}
6738   ins_encode %{
6739     Register s = $src$$Register;
6740     Register d = $dst$$Register;
6741     if (s != d) {
6742       __ decode_klass_not_null(d, s);
6743     } else {
6744       __ decode_klass_not_null(d);
6745     }
6746   %}
6747   ins_pipe(ialu_reg_long);
6748 %}
6749 
6750 
6751 //----------Conditional Move---------------------------------------------------
6752 // Jump
6753 // dummy instruction for generating temp registers
6754 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6755   match(Jump (LShiftL switch_val shift));
6756   ins_cost(350);
6757   predicate(false);
6758   effect(TEMP dest);
6759 
6760   format %{ "leaq    $dest, [$constantaddress]\n\t"
6761             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6762   ins_encode %{
6763     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6764     // to do that and the compiler is using that register as one it can allocate.
6765     // So we build it all by hand.
6766     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6767     // ArrayAddress dispatch(table, index);
6768     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6769     __ lea($dest$$Register, $constantaddress);
6770     __ jmp(dispatch);
6771   %}
6772   ins_pipe(pipe_jmp);
6773 %}
6774 
6775 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6776   match(Jump (AddL (LShiftL switch_val shift) offset));
6777   ins_cost(350);
6778   effect(TEMP dest);
6779 
6780   format %{ "leaq    $dest, [$constantaddress]\n\t"
6781             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6782   ins_encode %{
6783     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6784     // to do that and the compiler is using that register as one it can allocate.
6785     // So we build it all by hand.
6786     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6787     // ArrayAddress dispatch(table, index);
6788     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6789     __ lea($dest$$Register, $constantaddress);
6790     __ jmp(dispatch);
6791   %}
6792   ins_pipe(pipe_jmp);
6793 %}
6794 
6795 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6796   match(Jump switch_val);
6797   ins_cost(350);
6798   effect(TEMP dest);
6799 
6800   format %{ "leaq    $dest, [$constantaddress]\n\t"
6801             "jmp     [$dest + $switch_val]\n\t" %}
6802   ins_encode %{
6803     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6804     // to do that and the compiler is using that register as one it can allocate.
6805     // So we build it all by hand.
6806     // Address index(noreg, switch_reg, Address::times_1);
6807     // ArrayAddress dispatch(table, index);
6808     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6809     __ lea($dest$$Register, $constantaddress);
6810     __ jmp(dispatch);
6811   %}
6812   ins_pipe(pipe_jmp);
6813 %}
6814 
6815 // Conditional move
6816 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6817 %{
6818   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6819 
6820   ins_cost(200); // XXX
6821   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6822   opcode(0x0F, 0x40);
6823   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6824   ins_pipe(pipe_cmov_reg);
6825 %}
6826 
6827 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6828   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6829 
6830   ins_cost(200); // XXX
6831   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6832   opcode(0x0F, 0x40);
6833   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6834   ins_pipe(pipe_cmov_reg);
6835 %}
6836 
6837 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6838   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6839   ins_cost(200);
6840   expand %{
6841     cmovI_regU(cop, cr, dst, src);
6842   %}
6843 %}
6844 
6845 // Conditional move
6846 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6847   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6848 
6849   ins_cost(250); // XXX
6850   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6851   opcode(0x0F, 0x40);
6852   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6853   ins_pipe(pipe_cmov_mem);
6854 %}
6855 
6856 // Conditional move
6857 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6858 %{
6859   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6860 
6861   ins_cost(250); // XXX
6862   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6863   opcode(0x0F, 0x40);
6864   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6865   ins_pipe(pipe_cmov_mem);
6866 %}
6867 
6868 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6869   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6870   ins_cost(250);
6871   expand %{
6872     cmovI_memU(cop, cr, dst, src);
6873   %}
6874 %}
6875 
6876 // Conditional move
6877 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6878 %{
6879   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6880 
6881   ins_cost(200); // XXX
6882   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6883   opcode(0x0F, 0x40);
6884   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6885   ins_pipe(pipe_cmov_reg);
6886 %}
6887 
6888 // Conditional move
6889 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6890 %{
6891   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6892 
6893   ins_cost(200); // XXX
6894   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6895   opcode(0x0F, 0x40);
6896   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6897   ins_pipe(pipe_cmov_reg);
6898 %}
6899 
6900 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6901   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6902   ins_cost(200);
6903   expand %{
6904     cmovN_regU(cop, cr, dst, src);
6905   %}
6906 %}
6907 
6908 // Conditional move
6909 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6910 %{
6911   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6912 
6913   ins_cost(200); // XXX
6914   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6915   opcode(0x0F, 0x40);
6916   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6917   ins_pipe(pipe_cmov_reg);  // XXX
6918 %}
6919 
6920 // Conditional move
6921 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6922 %{
6923   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6924 
6925   ins_cost(200); // XXX
6926   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6927   opcode(0x0F, 0x40);
6928   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6929   ins_pipe(pipe_cmov_reg); // XXX
6930 %}
6931 
6932 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6933   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6934   ins_cost(200);
6935   expand %{
6936     cmovP_regU(cop, cr, dst, src);
6937   %}
6938 %}
6939 
6940 // DISABLED: Requires the ADLC to emit a bottom_type call that
6941 // correctly meets the two pointer arguments; one is an incoming
6942 // register but the other is a memory operand.  ALSO appears to
6943 // be buggy with implicit null checks.
6944 //
6945 //// Conditional move
6946 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6947 //%{
6948 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6949 //  ins_cost(250);
6950 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6951 //  opcode(0x0F,0x40);
6952 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6953 //  ins_pipe( pipe_cmov_mem );
6954 //%}
6955 //
6956 //// Conditional move
6957 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6958 //%{
6959 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6960 //  ins_cost(250);
6961 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6962 //  opcode(0x0F,0x40);
6963 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6964 //  ins_pipe( pipe_cmov_mem );
6965 //%}
6966 
6967 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6968 %{
6969   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6970 
6971   ins_cost(200); // XXX
6972   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6973   opcode(0x0F, 0x40);
6974   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6975   ins_pipe(pipe_cmov_reg);  // XXX
6976 %}
6977 
6978 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6979 %{
6980   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6981 
6982   ins_cost(200); // XXX
6983   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6984   opcode(0x0F, 0x40);
6985   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6986   ins_pipe(pipe_cmov_mem);  // XXX
6987 %}
6988 
6989 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6990 %{
6991   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6992 
6993   ins_cost(200); // XXX
6994   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6995   opcode(0x0F, 0x40);
6996   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6997   ins_pipe(pipe_cmov_reg); // XXX
6998 %}
6999 
7000 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7001   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7002   ins_cost(200);
7003   expand %{
7004     cmovL_regU(cop, cr, dst, src);
7005   %}
7006 %}
7007 
7008 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7009 %{
7010   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7011 
7012   ins_cost(200); // XXX
7013   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7014   opcode(0x0F, 0x40);
7015   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7016   ins_pipe(pipe_cmov_mem); // XXX
7017 %}
7018 
7019 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7020   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7021   ins_cost(200);
7022   expand %{
7023     cmovL_memU(cop, cr, dst, src);
7024   %}
7025 %}
7026 
7027 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7028 %{
7029   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7030 
7031   ins_cost(200); // XXX
7032   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7033             "movss     $dst, $src\n"
7034     "skip:" %}
7035   ins_encode %{
7036     Label Lskip;
7037     // Invert sense of branch from sense of CMOV
7038     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7039     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7040     __ bind(Lskip);
7041   %}
7042   ins_pipe(pipe_slow);
7043 %}
7044 
7045 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7046 // %{
7047 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7048 
7049 //   ins_cost(200); // XXX
7050 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7051 //             "movss     $dst, $src\n"
7052 //     "skip:" %}
7053 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7054 //   ins_pipe(pipe_slow);
7055 // %}
7056 
7057 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7058 %{
7059   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7060 
7061   ins_cost(200); // XXX
7062   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7063             "movss     $dst, $src\n"
7064     "skip:" %}
7065   ins_encode %{
7066     Label Lskip;
7067     // Invert sense of branch from sense of CMOV
7068     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7069     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7070     __ bind(Lskip);
7071   %}
7072   ins_pipe(pipe_slow);
7073 %}
7074 
7075 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7076   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7077   ins_cost(200);
7078   expand %{
7079     cmovF_regU(cop, cr, dst, src);
7080   %}
7081 %}
7082 
7083 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7084 %{
7085   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7086 
7087   ins_cost(200); // XXX
7088   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7089             "movsd     $dst, $src\n"
7090     "skip:" %}
7091   ins_encode %{
7092     Label Lskip;
7093     // Invert sense of branch from sense of CMOV
7094     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7095     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7096     __ bind(Lskip);
7097   %}
7098   ins_pipe(pipe_slow);
7099 %}
7100 
7101 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7102 %{
7103   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7104 
7105   ins_cost(200); // XXX
7106   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7107             "movsd     $dst, $src\n"
7108     "skip:" %}
7109   ins_encode %{
7110     Label Lskip;
7111     // Invert sense of branch from sense of CMOV
7112     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7113     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7114     __ bind(Lskip);
7115   %}
7116   ins_pipe(pipe_slow);
7117 %}
7118 
7119 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7120   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7121   ins_cost(200);
7122   expand %{
7123     cmovD_regU(cop, cr, dst, src);
7124   %}
7125 %}
7126 
7127 //----------Arithmetic Instructions--------------------------------------------
7128 //----------Addition Instructions----------------------------------------------
7129 
7130 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7131 %{
7132   match(Set dst (AddI dst src));
7133   effect(KILL cr);
7134 
7135   format %{ "addl    $dst, $src\t# int" %}
7136   opcode(0x03);
7137   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7138   ins_pipe(ialu_reg_reg);
7139 %}
7140 
7141 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7142 %{
7143   match(Set dst (AddI dst src));
7144   effect(KILL cr);
7145 
7146   format %{ "addl    $dst, $src\t# int" %}
7147   opcode(0x81, 0x00); /* /0 id */
7148   ins_encode(OpcSErm(dst, src), Con8or32(src));
7149   ins_pipe( ialu_reg );
7150 %}
7151 
7152 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7153 %{
7154   match(Set dst (AddI dst (LoadI src)));
7155   effect(KILL cr);
7156 
7157   ins_cost(125); // XXX
7158   format %{ "addl    $dst, $src\t# int" %}
7159   opcode(0x03);
7160   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7161   ins_pipe(ialu_reg_mem);
7162 %}
7163 
7164 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7165 %{
7166   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7167   effect(KILL cr);
7168 
7169   ins_cost(150); // XXX
7170   format %{ "addl    $dst, $src\t# int" %}
7171   opcode(0x01); /* Opcode 01 /r */
7172   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7173   ins_pipe(ialu_mem_reg);
7174 %}
7175 
7176 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7177 %{
7178   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7179   effect(KILL cr);
7180 
7181   ins_cost(125); // XXX
7182   format %{ "addl    $dst, $src\t# int" %}
7183   opcode(0x81); /* Opcode 81 /0 id */
7184   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7185   ins_pipe(ialu_mem_imm);
7186 %}
7187 
7188 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7189 %{
7190   predicate(UseIncDec);
7191   match(Set dst (AddI dst src));
7192   effect(KILL cr);
7193 
7194   format %{ "incl    $dst\t# int" %}
7195   opcode(0xFF, 0x00); // FF /0
7196   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7197   ins_pipe(ialu_reg);
7198 %}
7199 
7200 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7201 %{
7202   predicate(UseIncDec);
7203   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7204   effect(KILL cr);
7205 
7206   ins_cost(125); // XXX
7207   format %{ "incl    $dst\t# int" %}
7208   opcode(0xFF); /* Opcode FF /0 */
7209   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7210   ins_pipe(ialu_mem_imm);
7211 %}
7212 
7213 // XXX why does that use AddI
7214 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7215 %{
7216   predicate(UseIncDec);
7217   match(Set dst (AddI dst src));
7218   effect(KILL cr);
7219 
7220   format %{ "decl    $dst\t# int" %}
7221   opcode(0xFF, 0x01); // FF /1
7222   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7223   ins_pipe(ialu_reg);
7224 %}
7225 
7226 // XXX why does that use AddI
7227 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7228 %{
7229   predicate(UseIncDec);
7230   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7231   effect(KILL cr);
7232 
7233   ins_cost(125); // XXX
7234   format %{ "decl    $dst\t# int" %}
7235   opcode(0xFF); /* Opcode FF /1 */
7236   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7237   ins_pipe(ialu_mem_imm);
7238 %}
7239 
7240 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7241 %{
7242   match(Set dst (AddI src0 src1));
7243 
7244   ins_cost(110);
7245   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7246   opcode(0x8D); /* 0x8D /r */
7247   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7248   ins_pipe(ialu_reg_reg);
7249 %}
7250 
7251 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7252 %{
7253   match(Set dst (AddL dst src));
7254   effect(KILL cr);
7255 
7256   format %{ "addq    $dst, $src\t# long" %}
7257   opcode(0x03);
7258   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7259   ins_pipe(ialu_reg_reg);
7260 %}
7261 
7262 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7263 %{
7264   match(Set dst (AddL dst src));
7265   effect(KILL cr);
7266 
7267   format %{ "addq    $dst, $src\t# long" %}
7268   opcode(0x81, 0x00); /* /0 id */
7269   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7270   ins_pipe( ialu_reg );
7271 %}
7272 
7273 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7274 %{
7275   match(Set dst (AddL dst (LoadL src)));
7276   effect(KILL cr);
7277 
7278   ins_cost(125); // XXX
7279   format %{ "addq    $dst, $src\t# long" %}
7280   opcode(0x03);
7281   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7282   ins_pipe(ialu_reg_mem);
7283 %}
7284 
7285 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7286 %{
7287   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7288   effect(KILL cr);
7289 
7290   ins_cost(150); // XXX
7291   format %{ "addq    $dst, $src\t# long" %}
7292   opcode(0x01); /* Opcode 01 /r */
7293   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7294   ins_pipe(ialu_mem_reg);
7295 %}
7296 
7297 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7298 %{
7299   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7300   effect(KILL cr);
7301 
7302   ins_cost(125); // XXX
7303   format %{ "addq    $dst, $src\t# long" %}
7304   opcode(0x81); /* Opcode 81 /0 id */
7305   ins_encode(REX_mem_wide(dst),
7306              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7307   ins_pipe(ialu_mem_imm);
7308 %}
7309 
7310 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7311 %{
7312   predicate(UseIncDec);
7313   match(Set dst (AddL dst src));
7314   effect(KILL cr);
7315 
7316   format %{ "incq    $dst\t# long" %}
7317   opcode(0xFF, 0x00); // FF /0
7318   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7319   ins_pipe(ialu_reg);
7320 %}
7321 
7322 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7323 %{
7324   predicate(UseIncDec);
7325   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7326   effect(KILL cr);
7327 
7328   ins_cost(125); // XXX
7329   format %{ "incq    $dst\t# long" %}
7330   opcode(0xFF); /* Opcode FF /0 */
7331   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7332   ins_pipe(ialu_mem_imm);
7333 %}
7334 
7335 // XXX why does that use AddL
7336 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7337 %{
7338   predicate(UseIncDec);
7339   match(Set dst (AddL dst src));
7340   effect(KILL cr);
7341 
7342   format %{ "decq    $dst\t# long" %}
7343   opcode(0xFF, 0x01); // FF /1
7344   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7345   ins_pipe(ialu_reg);
7346 %}
7347 
7348 // XXX why does that use AddL
7349 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7350 %{
7351   predicate(UseIncDec);
7352   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7353   effect(KILL cr);
7354 
7355   ins_cost(125); // XXX
7356   format %{ "decq    $dst\t# long" %}
7357   opcode(0xFF); /* Opcode FF /1 */
7358   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7359   ins_pipe(ialu_mem_imm);
7360 %}
7361 
7362 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7363 %{
7364   match(Set dst (AddL src0 src1));
7365 
7366   ins_cost(110);
7367   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7368   opcode(0x8D); /* 0x8D /r */
7369   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7370   ins_pipe(ialu_reg_reg);
7371 %}
7372 
7373 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7374 %{
7375   match(Set dst (AddP dst src));
7376   effect(KILL cr);
7377 
7378   format %{ "addq    $dst, $src\t# ptr" %}
7379   opcode(0x03);
7380   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7381   ins_pipe(ialu_reg_reg);
7382 %}
7383 
7384 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7385 %{
7386   match(Set dst (AddP dst src));
7387   effect(KILL cr);
7388 
7389   format %{ "addq    $dst, $src\t# ptr" %}
7390   opcode(0x81, 0x00); /* /0 id */
7391   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7392   ins_pipe( ialu_reg );
7393 %}
7394 
7395 // XXX addP mem ops ????
7396 
7397 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7398 %{
7399   match(Set dst (AddP src0 src1));
7400 
7401   ins_cost(110);
7402   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7403   opcode(0x8D); /* 0x8D /r */
7404   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7405   ins_pipe(ialu_reg_reg);
7406 %}
7407 
7408 instruct checkCastPP(rRegP dst)
7409 %{
7410   match(Set dst (CheckCastPP dst));
7411 
7412   size(0);
7413   format %{ "# checkcastPP of $dst" %}
7414   ins_encode(/* empty encoding */);
7415   ins_pipe(empty);
7416 %}
7417 
7418 instruct castPP(rRegP dst)
7419 %{
7420   match(Set dst (CastPP dst));
7421 
7422   size(0);
7423   format %{ "# castPP of $dst" %}
7424   ins_encode(/* empty encoding */);
7425   ins_pipe(empty);
7426 %}
7427 
7428 instruct castII(rRegI dst)
7429 %{
7430   match(Set dst (CastII dst));
7431 
7432   size(0);
7433   format %{ "# castII of $dst" %}
7434   ins_encode(/* empty encoding */);
7435   ins_cost(0);
7436   ins_pipe(empty);
7437 %}
7438 
7439 // LoadP-locked same as a regular LoadP when used with compare-swap
7440 instruct loadPLocked(rRegP dst, memory mem)
7441 %{
7442   match(Set dst (LoadPLocked mem));
7443 
7444   ins_cost(125); // XXX
7445   format %{ "movq    $dst, $mem\t# ptr locked" %}
7446   opcode(0x8B);
7447   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7448   ins_pipe(ialu_reg_mem); // XXX
7449 %}
7450 
7451 // Conditional-store of the updated heap-top.
7452 // Used during allocation of the shared heap.
7453 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7454 
7455 instruct storePConditional(memory heap_top_ptr,
7456                            rax_RegP oldval, rRegP newval,
7457                            rFlagsReg cr)
7458 %{
7459   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7460 
7461   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7462             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7463   opcode(0x0F, 0xB1);
7464   ins_encode(lock_prefix,
7465              REX_reg_mem_wide(newval, heap_top_ptr),
7466              OpcP, OpcS,
7467              reg_mem(newval, heap_top_ptr));
7468   ins_pipe(pipe_cmpxchg);
7469 %}
7470 
7471 // Conditional-store of an int value.
7472 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7473 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7474 %{
7475   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7476   effect(KILL oldval);
7477 
7478   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7479   opcode(0x0F, 0xB1);
7480   ins_encode(lock_prefix,
7481              REX_reg_mem(newval, mem),
7482              OpcP, OpcS,
7483              reg_mem(newval, mem));
7484   ins_pipe(pipe_cmpxchg);
7485 %}
7486 
7487 // Conditional-store of a long value.
7488 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7489 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7490 %{
7491   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7492   effect(KILL oldval);
7493 
7494   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7495   opcode(0x0F, 0xB1);
7496   ins_encode(lock_prefix,
7497              REX_reg_mem_wide(newval, mem),
7498              OpcP, OpcS,
7499              reg_mem(newval, mem));
7500   ins_pipe(pipe_cmpxchg);
7501 %}
7502 
7503 
7504 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7505 instruct compareAndSwapP(rRegI res,
7506                          memory mem_ptr,
7507                          rax_RegP oldval, rRegP newval,
7508                          rFlagsReg cr)
7509 %{
7510   predicate(VM_Version::supports_cx8());
7511   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7512   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7513   effect(KILL cr, KILL oldval);
7514 
7515   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7516             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7517             "sete    $res\n\t"
7518             "movzbl  $res, $res" %}
7519   opcode(0x0F, 0xB1);
7520   ins_encode(lock_prefix,
7521              REX_reg_mem_wide(newval, mem_ptr),
7522              OpcP, OpcS,
7523              reg_mem(newval, mem_ptr),
7524              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7525              REX_reg_breg(res, res), // movzbl
7526              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7527   ins_pipe( pipe_cmpxchg );
7528 %}
7529 
7530 instruct compareAndSwapL(rRegI res,
7531                          memory mem_ptr,
7532                          rax_RegL oldval, rRegL newval,
7533                          rFlagsReg cr)
7534 %{
7535   predicate(VM_Version::supports_cx8());
7536   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7537   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7538   effect(KILL cr, KILL oldval);
7539 
7540   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7541             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7542             "sete    $res\n\t"
7543             "movzbl  $res, $res" %}
7544   opcode(0x0F, 0xB1);
7545   ins_encode(lock_prefix,
7546              REX_reg_mem_wide(newval, mem_ptr),
7547              OpcP, OpcS,
7548              reg_mem(newval, mem_ptr),
7549              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7550              REX_reg_breg(res, res), // movzbl
7551              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7552   ins_pipe( pipe_cmpxchg );
7553 %}
7554 
7555 instruct compareAndSwapI(rRegI res,
7556                          memory mem_ptr,
7557                          rax_RegI oldval, rRegI newval,
7558                          rFlagsReg cr)
7559 %{
7560   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7561   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7562   effect(KILL cr, KILL oldval);
7563 
7564   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7565             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7566             "sete    $res\n\t"
7567             "movzbl  $res, $res" %}
7568   opcode(0x0F, 0xB1);
7569   ins_encode(lock_prefix,
7570              REX_reg_mem(newval, mem_ptr),
7571              OpcP, OpcS,
7572              reg_mem(newval, mem_ptr),
7573              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7574              REX_reg_breg(res, res), // movzbl
7575              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7576   ins_pipe( pipe_cmpxchg );
7577 %}
7578 
7579 instruct compareAndSwapB(rRegI res,
7580                          memory mem_ptr,
7581                          rax_RegI oldval, rRegI newval,
7582                          rFlagsReg cr)
7583 %{
7584   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7585   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7586   effect(KILL cr, KILL oldval);
7587 
7588   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7589             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7590             "sete    $res\n\t"
7591             "movzbl  $res, $res" %}
7592   opcode(0x0F, 0xB0);
7593   ins_encode(lock_prefix,
7594              REX_breg_mem(newval, mem_ptr),
7595              OpcP, OpcS,
7596              reg_mem(newval, mem_ptr),
7597              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7598              REX_reg_breg(res, res), // movzbl
7599              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7600   ins_pipe( pipe_cmpxchg );
7601 %}
7602 
7603 instruct compareAndSwapS(rRegI res,
7604                          memory mem_ptr,
7605                          rax_RegI oldval, rRegI newval,
7606                          rFlagsReg cr)
7607 %{
7608   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7609   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7610   effect(KILL cr, KILL oldval);
7611 
7612   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7613             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7614             "sete    $res\n\t"
7615             "movzbl  $res, $res" %}
7616   opcode(0x0F, 0xB1);
7617   ins_encode(lock_prefix,
7618              SizePrefix,
7619              REX_reg_mem(newval, mem_ptr),
7620              OpcP, OpcS,
7621              reg_mem(newval, mem_ptr),
7622              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7623              REX_reg_breg(res, res), // movzbl
7624              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7625   ins_pipe( pipe_cmpxchg );
7626 %}
7627 
7628 instruct compareAndSwapN(rRegI res,
7629                           memory mem_ptr,
7630                           rax_RegN oldval, rRegN newval,
7631                           rFlagsReg cr) %{
7632   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7633   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7634   effect(KILL cr, KILL oldval);
7635 
7636   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7637             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7638             "sete    $res\n\t"
7639             "movzbl  $res, $res" %}
7640   opcode(0x0F, 0xB1);
7641   ins_encode(lock_prefix,
7642              REX_reg_mem(newval, mem_ptr),
7643              OpcP, OpcS,
7644              reg_mem(newval, mem_ptr),
7645              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7646              REX_reg_breg(res, res), // movzbl
7647              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7648   ins_pipe( pipe_cmpxchg );
7649 %}
7650 
7651 instruct compareAndExchangeB(
7652                          memory mem_ptr,
7653                          rax_RegI oldval, rRegI newval,
7654                          rFlagsReg cr)
7655 %{
7656   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7657   effect(KILL cr);
7658 
7659   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7660             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7661   opcode(0x0F, 0xB0);
7662   ins_encode(lock_prefix,
7663              REX_breg_mem(newval, mem_ptr),
7664              OpcP, OpcS,
7665              reg_mem(newval, mem_ptr) // lock cmpxchg
7666              );
7667   ins_pipe( pipe_cmpxchg );
7668 %}
7669 
7670 instruct compareAndExchangeS(
7671                          memory mem_ptr,
7672                          rax_RegI oldval, rRegI newval,
7673                          rFlagsReg cr)
7674 %{
7675   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7676   effect(KILL cr);
7677 
7678   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7679             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7680   opcode(0x0F, 0xB1);
7681   ins_encode(lock_prefix,
7682              SizePrefix,
7683              REX_reg_mem(newval, mem_ptr),
7684              OpcP, OpcS,
7685              reg_mem(newval, mem_ptr) // lock cmpxchg
7686              );
7687   ins_pipe( pipe_cmpxchg );
7688 %}
7689 
7690 instruct compareAndExchangeI(
7691                          memory mem_ptr,
7692                          rax_RegI oldval, rRegI newval,
7693                          rFlagsReg cr)
7694 %{
7695   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7696   effect(KILL cr);
7697 
7698   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7699             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7700   opcode(0x0F, 0xB1);
7701   ins_encode(lock_prefix,
7702              REX_reg_mem(newval, mem_ptr),
7703              OpcP, OpcS,
7704              reg_mem(newval, mem_ptr) // lock cmpxchg
7705              );
7706   ins_pipe( pipe_cmpxchg );
7707 %}
7708 
7709 instruct compareAndExchangeL(
7710                          memory mem_ptr,
7711                          rax_RegL oldval, rRegL newval,
7712                          rFlagsReg cr)
7713 %{
7714   predicate(VM_Version::supports_cx8());
7715   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7716   effect(KILL cr);
7717 
7718   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7719             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7720   opcode(0x0F, 0xB1);
7721   ins_encode(lock_prefix,
7722              REX_reg_mem_wide(newval, mem_ptr),
7723              OpcP, OpcS,
7724              reg_mem(newval, mem_ptr)  // lock cmpxchg
7725             );
7726   ins_pipe( pipe_cmpxchg );
7727 %}
7728 
7729 instruct compareAndExchangeN(
7730                           memory mem_ptr,
7731                           rax_RegN oldval, rRegN newval,
7732                           rFlagsReg cr) %{
7733   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7734   effect(KILL cr);
7735 
7736   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7737             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7738   opcode(0x0F, 0xB1);
7739   ins_encode(lock_prefix,
7740              REX_reg_mem(newval, mem_ptr),
7741              OpcP, OpcS,
7742              reg_mem(newval, mem_ptr)  // lock cmpxchg
7743           );
7744   ins_pipe( pipe_cmpxchg );
7745 %}
7746 
7747 instruct compareAndExchangeP(
7748                          memory mem_ptr,
7749                          rax_RegP oldval, rRegP newval,
7750                          rFlagsReg cr)
7751 %{
7752   predicate(VM_Version::supports_cx8());
7753   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7754   effect(KILL cr);
7755 
7756   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7757             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7758   opcode(0x0F, 0xB1);
7759   ins_encode(lock_prefix,
7760              REX_reg_mem_wide(newval, mem_ptr),
7761              OpcP, OpcS,
7762              reg_mem(newval, mem_ptr)  // lock cmpxchg
7763           );
7764   ins_pipe( pipe_cmpxchg );
7765 %}
7766 
7767 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7768   predicate(n->as_LoadStore()->result_not_used());
7769   match(Set dummy (GetAndAddB mem add));
7770   effect(KILL cr);
7771   format %{ "ADDB  [$mem],$add" %}
7772   ins_encode %{
7773     if (os::is_MP()) { __ lock(); }
7774     __ addb($mem$$Address, $add$$constant);
7775   %}
7776   ins_pipe( pipe_cmpxchg );
7777 %}
7778 
7779 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7780   match(Set newval (GetAndAddB mem newval));
7781   effect(KILL cr);
7782   format %{ "XADDB  [$mem],$newval" %}
7783   ins_encode %{
7784     if (os::is_MP()) { __ lock(); }
7785     __ xaddb($mem$$Address, $newval$$Register);
7786   %}
7787   ins_pipe( pipe_cmpxchg );
7788 %}
7789 
7790 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7791   predicate(n->as_LoadStore()->result_not_used());
7792   match(Set dummy (GetAndAddS mem add));
7793   effect(KILL cr);
7794   format %{ "ADDW  [$mem],$add" %}
7795   ins_encode %{
7796     if (os::is_MP()) { __ lock(); }
7797     __ addw($mem$$Address, $add$$constant);
7798   %}
7799   ins_pipe( pipe_cmpxchg );
7800 %}
7801 
7802 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7803   match(Set newval (GetAndAddS mem newval));
7804   effect(KILL cr);
7805   format %{ "XADDW  [$mem],$newval" %}
7806   ins_encode %{
7807     if (os::is_MP()) { __ lock(); }
7808     __ xaddw($mem$$Address, $newval$$Register);
7809   %}
7810   ins_pipe( pipe_cmpxchg );
7811 %}
7812 
7813 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7814   predicate(n->as_LoadStore()->result_not_used());
7815   match(Set dummy (GetAndAddI mem add));
7816   effect(KILL cr);
7817   format %{ "ADDL  [$mem],$add" %}
7818   ins_encode %{
7819     if (os::is_MP()) { __ lock(); }
7820     __ addl($mem$$Address, $add$$constant);
7821   %}
7822   ins_pipe( pipe_cmpxchg );
7823 %}
7824 
7825 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7826   match(Set newval (GetAndAddI mem newval));
7827   effect(KILL cr);
7828   format %{ "XADDL  [$mem],$newval" %}
7829   ins_encode %{
7830     if (os::is_MP()) { __ lock(); }
7831     __ xaddl($mem$$Address, $newval$$Register);
7832   %}
7833   ins_pipe( pipe_cmpxchg );
7834 %}
7835 
7836 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7837   predicate(n->as_LoadStore()->result_not_used());
7838   match(Set dummy (GetAndAddL mem add));
7839   effect(KILL cr);
7840   format %{ "ADDQ  [$mem],$add" %}
7841   ins_encode %{
7842     if (os::is_MP()) { __ lock(); }
7843     __ addq($mem$$Address, $add$$constant);
7844   %}
7845   ins_pipe( pipe_cmpxchg );
7846 %}
7847 
7848 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7849   match(Set newval (GetAndAddL mem newval));
7850   effect(KILL cr);
7851   format %{ "XADDQ  [$mem],$newval" %}
7852   ins_encode %{
7853     if (os::is_MP()) { __ lock(); }
7854     __ xaddq($mem$$Address, $newval$$Register);
7855   %}
7856   ins_pipe( pipe_cmpxchg );
7857 %}
7858 
7859 instruct xchgB( memory mem, rRegI newval) %{
7860   match(Set newval (GetAndSetB mem newval));
7861   format %{ "XCHGB  $newval,[$mem]" %}
7862   ins_encode %{
7863     __ xchgb($newval$$Register, $mem$$Address);
7864   %}
7865   ins_pipe( pipe_cmpxchg );
7866 %}
7867 
7868 instruct xchgS( memory mem, rRegI newval) %{
7869   match(Set newval (GetAndSetS mem newval));
7870   format %{ "XCHGW  $newval,[$mem]" %}
7871   ins_encode %{
7872     __ xchgw($newval$$Register, $mem$$Address);
7873   %}
7874   ins_pipe( pipe_cmpxchg );
7875 %}
7876 
7877 instruct xchgI( memory mem, rRegI newval) %{
7878   match(Set newval (GetAndSetI mem newval));
7879   format %{ "XCHGL  $newval,[$mem]" %}
7880   ins_encode %{
7881     __ xchgl($newval$$Register, $mem$$Address);
7882   %}
7883   ins_pipe( pipe_cmpxchg );
7884 %}
7885 
7886 instruct xchgL( memory mem, rRegL newval) %{
7887   match(Set newval (GetAndSetL mem newval));
7888   format %{ "XCHGL  $newval,[$mem]" %}
7889   ins_encode %{
7890     __ xchgq($newval$$Register, $mem$$Address);
7891   %}
7892   ins_pipe( pipe_cmpxchg );
7893 %}
7894 
7895 instruct xchgP( memory mem, rRegP newval) %{
7896   match(Set newval (GetAndSetP mem newval));
7897   format %{ "XCHGQ  $newval,[$mem]" %}
7898   ins_encode %{
7899     __ xchgq($newval$$Register, $mem$$Address);
7900   %}
7901   ins_pipe( pipe_cmpxchg );
7902 %}
7903 
7904 instruct xchgN( memory mem, rRegN newval) %{
7905   match(Set newval (GetAndSetN mem newval));
7906   format %{ "XCHGL  $newval,$mem]" %}
7907   ins_encode %{
7908     __ xchgl($newval$$Register, $mem$$Address);
7909   %}
7910   ins_pipe( pipe_cmpxchg );
7911 %}
7912 
7913 //----------Subtraction Instructions-------------------------------------------
7914 
7915 // Integer Subtraction Instructions
7916 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7917 %{
7918   match(Set dst (SubI dst src));
7919   effect(KILL cr);
7920 
7921   format %{ "subl    $dst, $src\t# int" %}
7922   opcode(0x2B);
7923   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7924   ins_pipe(ialu_reg_reg);
7925 %}
7926 
7927 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7928 %{
7929   match(Set dst (SubI dst src));
7930   effect(KILL cr);
7931 
7932   format %{ "subl    $dst, $src\t# int" %}
7933   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7934   ins_encode(OpcSErm(dst, src), Con8or32(src));
7935   ins_pipe(ialu_reg);
7936 %}
7937 
7938 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7939 %{
7940   match(Set dst (SubI dst (LoadI src)));
7941   effect(KILL cr);
7942 
7943   ins_cost(125);
7944   format %{ "subl    $dst, $src\t# int" %}
7945   opcode(0x2B);
7946   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7947   ins_pipe(ialu_reg_mem);
7948 %}
7949 
7950 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7951 %{
7952   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7953   effect(KILL cr);
7954 
7955   ins_cost(150);
7956   format %{ "subl    $dst, $src\t# int" %}
7957   opcode(0x29); /* Opcode 29 /r */
7958   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7959   ins_pipe(ialu_mem_reg);
7960 %}
7961 
7962 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7963 %{
7964   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7965   effect(KILL cr);
7966 
7967   ins_cost(125); // XXX
7968   format %{ "subl    $dst, $src\t# int" %}
7969   opcode(0x81); /* Opcode 81 /5 id */
7970   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7971   ins_pipe(ialu_mem_imm);
7972 %}
7973 
7974 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7975 %{
7976   match(Set dst (SubL dst src));
7977   effect(KILL cr);
7978 
7979   format %{ "subq    $dst, $src\t# long" %}
7980   opcode(0x2B);
7981   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7982   ins_pipe(ialu_reg_reg);
7983 %}
7984 
7985 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7986 %{
7987   match(Set dst (SubL dst src));
7988   effect(KILL cr);
7989 
7990   format %{ "subq    $dst, $src\t# long" %}
7991   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7992   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7993   ins_pipe(ialu_reg);
7994 %}
7995 
7996 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7997 %{
7998   match(Set dst (SubL dst (LoadL src)));
7999   effect(KILL cr);
8000 
8001   ins_cost(125);
8002   format %{ "subq    $dst, $src\t# long" %}
8003   opcode(0x2B);
8004   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8005   ins_pipe(ialu_reg_mem);
8006 %}
8007 
8008 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8009 %{
8010   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8011   effect(KILL cr);
8012 
8013   ins_cost(150);
8014   format %{ "subq    $dst, $src\t# long" %}
8015   opcode(0x29); /* Opcode 29 /r */
8016   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8017   ins_pipe(ialu_mem_reg);
8018 %}
8019 
8020 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8021 %{
8022   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8023   effect(KILL cr);
8024 
8025   ins_cost(125); // XXX
8026   format %{ "subq    $dst, $src\t# long" %}
8027   opcode(0x81); /* Opcode 81 /5 id */
8028   ins_encode(REX_mem_wide(dst),
8029              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8030   ins_pipe(ialu_mem_imm);
8031 %}
8032 
8033 // Subtract from a pointer
8034 // XXX hmpf???
8035 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8036 %{
8037   match(Set dst (AddP dst (SubI zero src)));
8038   effect(KILL cr);
8039 
8040   format %{ "subq    $dst, $src\t# ptr - int" %}
8041   opcode(0x2B);
8042   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8043   ins_pipe(ialu_reg_reg);
8044 %}
8045 
8046 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8047 %{
8048   match(Set dst (SubI zero dst));
8049   effect(KILL cr);
8050 
8051   format %{ "negl    $dst\t# int" %}
8052   opcode(0xF7, 0x03);  // Opcode F7 /3
8053   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8054   ins_pipe(ialu_reg);
8055 %}
8056 
8057 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8058 %{
8059   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8060   effect(KILL cr);
8061 
8062   format %{ "negl    $dst\t# int" %}
8063   opcode(0xF7, 0x03);  // Opcode F7 /3
8064   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8065   ins_pipe(ialu_reg);
8066 %}
8067 
8068 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8069 %{
8070   match(Set dst (SubL zero dst));
8071   effect(KILL cr);
8072 
8073   format %{ "negq    $dst\t# long" %}
8074   opcode(0xF7, 0x03);  // Opcode F7 /3
8075   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8076   ins_pipe(ialu_reg);
8077 %}
8078 
8079 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8080 %{
8081   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8082   effect(KILL cr);
8083 
8084   format %{ "negq    $dst\t# long" %}
8085   opcode(0xF7, 0x03);  // Opcode F7 /3
8086   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8087   ins_pipe(ialu_reg);
8088 %}
8089 
8090 //----------Multiplication/Division Instructions-------------------------------
8091 // Integer Multiplication Instructions
8092 // Multiply Register
8093 
8094 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8095 %{
8096   match(Set dst (MulI dst src));
8097   effect(KILL cr);
8098 
8099   ins_cost(300);
8100   format %{ "imull   $dst, $src\t# int" %}
8101   opcode(0x0F, 0xAF);
8102   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8103   ins_pipe(ialu_reg_reg_alu0);
8104 %}
8105 
8106 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8107 %{
8108   match(Set dst (MulI src imm));
8109   effect(KILL cr);
8110 
8111   ins_cost(300);
8112   format %{ "imull   $dst, $src, $imm\t# int" %}
8113   opcode(0x69); /* 69 /r id */
8114   ins_encode(REX_reg_reg(dst, src),
8115              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8116   ins_pipe(ialu_reg_reg_alu0);
8117 %}
8118 
8119 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8120 %{
8121   match(Set dst (MulI dst (LoadI src)));
8122   effect(KILL cr);
8123 
8124   ins_cost(350);
8125   format %{ "imull   $dst, $src\t# int" %}
8126   opcode(0x0F, 0xAF);
8127   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8128   ins_pipe(ialu_reg_mem_alu0);
8129 %}
8130 
8131 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8132 %{
8133   match(Set dst (MulI (LoadI src) imm));
8134   effect(KILL cr);
8135 
8136   ins_cost(300);
8137   format %{ "imull   $dst, $src, $imm\t# int" %}
8138   opcode(0x69); /* 69 /r id */
8139   ins_encode(REX_reg_mem(dst, src),
8140              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8141   ins_pipe(ialu_reg_mem_alu0);
8142 %}
8143 
8144 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8145 %{
8146   match(Set dst (MulL dst src));
8147   effect(KILL cr);
8148 
8149   ins_cost(300);
8150   format %{ "imulq   $dst, $src\t# long" %}
8151   opcode(0x0F, 0xAF);
8152   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8153   ins_pipe(ialu_reg_reg_alu0);
8154 %}
8155 
8156 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8157 %{
8158   match(Set dst (MulL src imm));
8159   effect(KILL cr);
8160 
8161   ins_cost(300);
8162   format %{ "imulq   $dst, $src, $imm\t# long" %}
8163   opcode(0x69); /* 69 /r id */
8164   ins_encode(REX_reg_reg_wide(dst, src),
8165              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8166   ins_pipe(ialu_reg_reg_alu0);
8167 %}
8168 
8169 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8170 %{
8171   match(Set dst (MulL dst (LoadL src)));
8172   effect(KILL cr);
8173 
8174   ins_cost(350);
8175   format %{ "imulq   $dst, $src\t# long" %}
8176   opcode(0x0F, 0xAF);
8177   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8178   ins_pipe(ialu_reg_mem_alu0);
8179 %}
8180 
8181 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8182 %{
8183   match(Set dst (MulL (LoadL src) imm));
8184   effect(KILL cr);
8185 
8186   ins_cost(300);
8187   format %{ "imulq   $dst, $src, $imm\t# long" %}
8188   opcode(0x69); /* 69 /r id */
8189   ins_encode(REX_reg_mem_wide(dst, src),
8190              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8191   ins_pipe(ialu_reg_mem_alu0);
8192 %}
8193 
8194 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8195 %{
8196   match(Set dst (MulHiL src rax));
8197   effect(USE_KILL rax, KILL cr);
8198 
8199   ins_cost(300);
8200   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8201   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8202   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8203   ins_pipe(ialu_reg_reg_alu0);
8204 %}
8205 
8206 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8207                    rFlagsReg cr)
8208 %{
8209   match(Set rax (DivI rax div));
8210   effect(KILL rdx, KILL cr);
8211 
8212   ins_cost(30*100+10*100); // XXX
8213   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8214             "jne,s   normal\n\t"
8215             "xorl    rdx, rdx\n\t"
8216             "cmpl    $div, -1\n\t"
8217             "je,s    done\n"
8218     "normal: cdql\n\t"
8219             "idivl   $div\n"
8220     "done:"        %}
8221   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8222   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8223   ins_pipe(ialu_reg_reg_alu0);
8224 %}
8225 
8226 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8227                    rFlagsReg cr)
8228 %{
8229   match(Set rax (DivL rax div));
8230   effect(KILL rdx, KILL cr);
8231 
8232   ins_cost(30*100+10*100); // XXX
8233   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8234             "cmpq    rax, rdx\n\t"
8235             "jne,s   normal\n\t"
8236             "xorl    rdx, rdx\n\t"
8237             "cmpq    $div, -1\n\t"
8238             "je,s    done\n"
8239     "normal: cdqq\n\t"
8240             "idivq   $div\n"
8241     "done:"        %}
8242   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8243   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8244   ins_pipe(ialu_reg_reg_alu0);
8245 %}
8246 
8247 // Integer DIVMOD with Register, both quotient and mod results
8248 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8249                              rFlagsReg cr)
8250 %{
8251   match(DivModI rax div);
8252   effect(KILL cr);
8253 
8254   ins_cost(30*100+10*100); // XXX
8255   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8256             "jne,s   normal\n\t"
8257             "xorl    rdx, rdx\n\t"
8258             "cmpl    $div, -1\n\t"
8259             "je,s    done\n"
8260     "normal: cdql\n\t"
8261             "idivl   $div\n"
8262     "done:"        %}
8263   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8264   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8265   ins_pipe(pipe_slow);
8266 %}
8267 
8268 // Long DIVMOD with Register, both quotient and mod results
8269 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8270                              rFlagsReg cr)
8271 %{
8272   match(DivModL rax div);
8273   effect(KILL cr);
8274 
8275   ins_cost(30*100+10*100); // XXX
8276   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8277             "cmpq    rax, rdx\n\t"
8278             "jne,s   normal\n\t"
8279             "xorl    rdx, rdx\n\t"
8280             "cmpq    $div, -1\n\t"
8281             "je,s    done\n"
8282     "normal: cdqq\n\t"
8283             "idivq   $div\n"
8284     "done:"        %}
8285   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8286   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8287   ins_pipe(pipe_slow);
8288 %}
8289 
8290 //----------- DivL-By-Constant-Expansions--------------------------------------
8291 // DivI cases are handled by the compiler
8292 
8293 // Magic constant, reciprocal of 10
8294 instruct loadConL_0x6666666666666667(rRegL dst)
8295 %{
8296   effect(DEF dst);
8297 
8298   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8299   ins_encode(load_immL(dst, 0x6666666666666667));
8300   ins_pipe(ialu_reg);
8301 %}
8302 
8303 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8304 %{
8305   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8306 
8307   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8308   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8309   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8310   ins_pipe(ialu_reg_reg_alu0);
8311 %}
8312 
8313 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8314 %{
8315   effect(USE_DEF dst, KILL cr);
8316 
8317   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8318   opcode(0xC1, 0x7); /* C1 /7 ib */
8319   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8320   ins_pipe(ialu_reg);
8321 %}
8322 
8323 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8324 %{
8325   effect(USE_DEF dst, KILL cr);
8326 
8327   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8328   opcode(0xC1, 0x7); /* C1 /7 ib */
8329   ins_encode(reg_opc_imm_wide(dst, 0x2));
8330   ins_pipe(ialu_reg);
8331 %}
8332 
8333 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8334 %{
8335   match(Set dst (DivL src div));
8336 
8337   ins_cost((5+8)*100);
8338   expand %{
8339     rax_RegL rax;                     // Killed temp
8340     rFlagsReg cr;                     // Killed
8341     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8342     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8343     sarL_rReg_63(src, cr);            // sarq  src, 63
8344     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8345     subL_rReg(dst, src, cr);          // subl  rdx, src
8346   %}
8347 %}
8348 
8349 //-----------------------------------------------------------------------------
8350 
8351 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8352                    rFlagsReg cr)
8353 %{
8354   match(Set rdx (ModI rax div));
8355   effect(KILL rax, KILL cr);
8356 
8357   ins_cost(300); // XXX
8358   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8359             "jne,s   normal\n\t"
8360             "xorl    rdx, rdx\n\t"
8361             "cmpl    $div, -1\n\t"
8362             "je,s    done\n"
8363     "normal: cdql\n\t"
8364             "idivl   $div\n"
8365     "done:"        %}
8366   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8367   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8368   ins_pipe(ialu_reg_reg_alu0);
8369 %}
8370 
8371 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8372                    rFlagsReg cr)
8373 %{
8374   match(Set rdx (ModL rax div));
8375   effect(KILL rax, KILL cr);
8376 
8377   ins_cost(300); // XXX
8378   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8379             "cmpq    rax, rdx\n\t"
8380             "jne,s   normal\n\t"
8381             "xorl    rdx, rdx\n\t"
8382             "cmpq    $div, -1\n\t"
8383             "je,s    done\n"
8384     "normal: cdqq\n\t"
8385             "idivq   $div\n"
8386     "done:"        %}
8387   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8388   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8389   ins_pipe(ialu_reg_reg_alu0);
8390 %}
8391 
8392 // Integer Shift Instructions
8393 // Shift Left by one
8394 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8395 %{
8396   match(Set dst (LShiftI dst shift));
8397   effect(KILL cr);
8398 
8399   format %{ "sall    $dst, $shift" %}
8400   opcode(0xD1, 0x4); /* D1 /4 */
8401   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8402   ins_pipe(ialu_reg);
8403 %}
8404 
8405 // Shift Left by one
8406 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8407 %{
8408   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8409   effect(KILL cr);
8410 
8411   format %{ "sall    $dst, $shift\t" %}
8412   opcode(0xD1, 0x4); /* D1 /4 */
8413   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8414   ins_pipe(ialu_mem_imm);
8415 %}
8416 
8417 // Shift Left by 8-bit immediate
8418 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8419 %{
8420   match(Set dst (LShiftI dst shift));
8421   effect(KILL cr);
8422 
8423   format %{ "sall    $dst, $shift" %}
8424   opcode(0xC1, 0x4); /* C1 /4 ib */
8425   ins_encode(reg_opc_imm(dst, shift));
8426   ins_pipe(ialu_reg);
8427 %}
8428 
8429 // Shift Left by 8-bit immediate
8430 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8431 %{
8432   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8433   effect(KILL cr);
8434 
8435   format %{ "sall    $dst, $shift" %}
8436   opcode(0xC1, 0x4); /* C1 /4 ib */
8437   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8438   ins_pipe(ialu_mem_imm);
8439 %}
8440 
8441 // Shift Left by variable
8442 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8443 %{
8444   match(Set dst (LShiftI dst shift));
8445   effect(KILL cr);
8446 
8447   format %{ "sall    $dst, $shift" %}
8448   opcode(0xD3, 0x4); /* D3 /4 */
8449   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8450   ins_pipe(ialu_reg_reg);
8451 %}
8452 
8453 // Shift Left by variable
8454 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8455 %{
8456   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8457   effect(KILL cr);
8458 
8459   format %{ "sall    $dst, $shift" %}
8460   opcode(0xD3, 0x4); /* D3 /4 */
8461   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8462   ins_pipe(ialu_mem_reg);
8463 %}
8464 
8465 // Arithmetic shift right by one
8466 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8467 %{
8468   match(Set dst (RShiftI dst shift));
8469   effect(KILL cr);
8470 
8471   format %{ "sarl    $dst, $shift" %}
8472   opcode(0xD1, 0x7); /* D1 /7 */
8473   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8474   ins_pipe(ialu_reg);
8475 %}
8476 
8477 // Arithmetic shift right by one
8478 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8479 %{
8480   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8481   effect(KILL cr);
8482 
8483   format %{ "sarl    $dst, $shift" %}
8484   opcode(0xD1, 0x7); /* D1 /7 */
8485   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8486   ins_pipe(ialu_mem_imm);
8487 %}
8488 
8489 // Arithmetic Shift Right by 8-bit immediate
8490 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8491 %{
8492   match(Set dst (RShiftI dst shift));
8493   effect(KILL cr);
8494 
8495   format %{ "sarl    $dst, $shift" %}
8496   opcode(0xC1, 0x7); /* C1 /7 ib */
8497   ins_encode(reg_opc_imm(dst, shift));
8498   ins_pipe(ialu_mem_imm);
8499 %}
8500 
8501 // Arithmetic Shift Right by 8-bit immediate
8502 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8503 %{
8504   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8505   effect(KILL cr);
8506 
8507   format %{ "sarl    $dst, $shift" %}
8508   opcode(0xC1, 0x7); /* C1 /7 ib */
8509   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8510   ins_pipe(ialu_mem_imm);
8511 %}
8512 
8513 // Arithmetic Shift Right by variable
8514 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8515 %{
8516   match(Set dst (RShiftI dst shift));
8517   effect(KILL cr);
8518 
8519   format %{ "sarl    $dst, $shift" %}
8520   opcode(0xD3, 0x7); /* D3 /7 */
8521   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8522   ins_pipe(ialu_reg_reg);
8523 %}
8524 
8525 // Arithmetic Shift Right by variable
8526 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8527 %{
8528   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8529   effect(KILL cr);
8530 
8531   format %{ "sarl    $dst, $shift" %}
8532   opcode(0xD3, 0x7); /* D3 /7 */
8533   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8534   ins_pipe(ialu_mem_reg);
8535 %}
8536 
8537 // Logical shift right by one
8538 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8539 %{
8540   match(Set dst (URShiftI dst shift));
8541   effect(KILL cr);
8542 
8543   format %{ "shrl    $dst, $shift" %}
8544   opcode(0xD1, 0x5); /* D1 /5 */
8545   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8546   ins_pipe(ialu_reg);
8547 %}
8548 
8549 // Logical shift right by one
8550 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8551 %{
8552   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8553   effect(KILL cr);
8554 
8555   format %{ "shrl    $dst, $shift" %}
8556   opcode(0xD1, 0x5); /* D1 /5 */
8557   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8558   ins_pipe(ialu_mem_imm);
8559 %}
8560 
8561 // Logical Shift Right by 8-bit immediate
8562 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8563 %{
8564   match(Set dst (URShiftI dst shift));
8565   effect(KILL cr);
8566 
8567   format %{ "shrl    $dst, $shift" %}
8568   opcode(0xC1, 0x5); /* C1 /5 ib */
8569   ins_encode(reg_opc_imm(dst, shift));
8570   ins_pipe(ialu_reg);
8571 %}
8572 
8573 // Logical Shift Right by 8-bit immediate
8574 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8575 %{
8576   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8577   effect(KILL cr);
8578 
8579   format %{ "shrl    $dst, $shift" %}
8580   opcode(0xC1, 0x5); /* C1 /5 ib */
8581   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8582   ins_pipe(ialu_mem_imm);
8583 %}
8584 
8585 // Logical Shift Right by variable
8586 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8587 %{
8588   match(Set dst (URShiftI dst shift));
8589   effect(KILL cr);
8590 
8591   format %{ "shrl    $dst, $shift" %}
8592   opcode(0xD3, 0x5); /* D3 /5 */
8593   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8594   ins_pipe(ialu_reg_reg);
8595 %}
8596 
8597 // Logical Shift Right by variable
8598 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8599 %{
8600   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8601   effect(KILL cr);
8602 
8603   format %{ "shrl    $dst, $shift" %}
8604   opcode(0xD3, 0x5); /* D3 /5 */
8605   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8606   ins_pipe(ialu_mem_reg);
8607 %}
8608 
8609 // Long Shift Instructions
8610 // Shift Left by one
8611 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8612 %{
8613   match(Set dst (LShiftL dst shift));
8614   effect(KILL cr);
8615 
8616   format %{ "salq    $dst, $shift" %}
8617   opcode(0xD1, 0x4); /* D1 /4 */
8618   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8619   ins_pipe(ialu_reg);
8620 %}
8621 
8622 // Shift Left by one
8623 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8624 %{
8625   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8626   effect(KILL cr);
8627 
8628   format %{ "salq    $dst, $shift" %}
8629   opcode(0xD1, 0x4); /* D1 /4 */
8630   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8631   ins_pipe(ialu_mem_imm);
8632 %}
8633 
8634 // Shift Left by 8-bit immediate
8635 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8636 %{
8637   match(Set dst (LShiftL dst shift));
8638   effect(KILL cr);
8639 
8640   format %{ "salq    $dst, $shift" %}
8641   opcode(0xC1, 0x4); /* C1 /4 ib */
8642   ins_encode(reg_opc_imm_wide(dst, shift));
8643   ins_pipe(ialu_reg);
8644 %}
8645 
8646 // Shift Left by 8-bit immediate
8647 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8648 %{
8649   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8650   effect(KILL cr);
8651 
8652   format %{ "salq    $dst, $shift" %}
8653   opcode(0xC1, 0x4); /* C1 /4 ib */
8654   ins_encode(REX_mem_wide(dst), OpcP,
8655              RM_opc_mem(secondary, dst), Con8or32(shift));
8656   ins_pipe(ialu_mem_imm);
8657 %}
8658 
8659 // Shift Left by variable
8660 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8661 %{
8662   match(Set dst (LShiftL dst shift));
8663   effect(KILL cr);
8664 
8665   format %{ "salq    $dst, $shift" %}
8666   opcode(0xD3, 0x4); /* D3 /4 */
8667   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8668   ins_pipe(ialu_reg_reg);
8669 %}
8670 
8671 // Shift Left by variable
8672 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8673 %{
8674   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8675   effect(KILL cr);
8676 
8677   format %{ "salq    $dst, $shift" %}
8678   opcode(0xD3, 0x4); /* D3 /4 */
8679   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8680   ins_pipe(ialu_mem_reg);
8681 %}
8682 
8683 // Arithmetic shift right by one
8684 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8685 %{
8686   match(Set dst (RShiftL dst shift));
8687   effect(KILL cr);
8688 
8689   format %{ "sarq    $dst, $shift" %}
8690   opcode(0xD1, 0x7); /* D1 /7 */
8691   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8692   ins_pipe(ialu_reg);
8693 %}
8694 
8695 // Arithmetic shift right by one
8696 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8697 %{
8698   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8699   effect(KILL cr);
8700 
8701   format %{ "sarq    $dst, $shift" %}
8702   opcode(0xD1, 0x7); /* D1 /7 */
8703   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8704   ins_pipe(ialu_mem_imm);
8705 %}
8706 
8707 // Arithmetic Shift Right by 8-bit immediate
8708 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8709 %{
8710   match(Set dst (RShiftL dst shift));
8711   effect(KILL cr);
8712 
8713   format %{ "sarq    $dst, $shift" %}
8714   opcode(0xC1, 0x7); /* C1 /7 ib */
8715   ins_encode(reg_opc_imm_wide(dst, shift));
8716   ins_pipe(ialu_mem_imm);
8717 %}
8718 
8719 // Arithmetic Shift Right by 8-bit immediate
8720 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8721 %{
8722   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8723   effect(KILL cr);
8724 
8725   format %{ "sarq    $dst, $shift" %}
8726   opcode(0xC1, 0x7); /* C1 /7 ib */
8727   ins_encode(REX_mem_wide(dst), OpcP,
8728              RM_opc_mem(secondary, dst), Con8or32(shift));
8729   ins_pipe(ialu_mem_imm);
8730 %}
8731 
8732 // Arithmetic Shift Right by variable
8733 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8734 %{
8735   match(Set dst (RShiftL dst shift));
8736   effect(KILL cr);
8737 
8738   format %{ "sarq    $dst, $shift" %}
8739   opcode(0xD3, 0x7); /* D3 /7 */
8740   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8741   ins_pipe(ialu_reg_reg);
8742 %}
8743 
8744 // Arithmetic Shift Right by variable
8745 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8746 %{
8747   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8748   effect(KILL cr);
8749 
8750   format %{ "sarq    $dst, $shift" %}
8751   opcode(0xD3, 0x7); /* D3 /7 */
8752   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8753   ins_pipe(ialu_mem_reg);
8754 %}
8755 
8756 // Logical shift right by one
8757 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8758 %{
8759   match(Set dst (URShiftL dst shift));
8760   effect(KILL cr);
8761 
8762   format %{ "shrq    $dst, $shift" %}
8763   opcode(0xD1, 0x5); /* D1 /5 */
8764   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8765   ins_pipe(ialu_reg);
8766 %}
8767 
8768 // Logical shift right by one
8769 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8770 %{
8771   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8772   effect(KILL cr);
8773 
8774   format %{ "shrq    $dst, $shift" %}
8775   opcode(0xD1, 0x5); /* D1 /5 */
8776   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8777   ins_pipe(ialu_mem_imm);
8778 %}
8779 
8780 // Logical Shift Right by 8-bit immediate
8781 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8782 %{
8783   match(Set dst (URShiftL dst shift));
8784   effect(KILL cr);
8785 
8786   format %{ "shrq    $dst, $shift" %}
8787   opcode(0xC1, 0x5); /* C1 /5 ib */
8788   ins_encode(reg_opc_imm_wide(dst, shift));
8789   ins_pipe(ialu_reg);
8790 %}
8791 
8792 
8793 // Logical Shift Right by 8-bit immediate
8794 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8795 %{
8796   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8797   effect(KILL cr);
8798 
8799   format %{ "shrq    $dst, $shift" %}
8800   opcode(0xC1, 0x5); /* C1 /5 ib */
8801   ins_encode(REX_mem_wide(dst), OpcP,
8802              RM_opc_mem(secondary, dst), Con8or32(shift));
8803   ins_pipe(ialu_mem_imm);
8804 %}
8805 
8806 // Logical Shift Right by variable
8807 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8808 %{
8809   match(Set dst (URShiftL dst shift));
8810   effect(KILL cr);
8811 
8812   format %{ "shrq    $dst, $shift" %}
8813   opcode(0xD3, 0x5); /* D3 /5 */
8814   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8815   ins_pipe(ialu_reg_reg);
8816 %}
8817 
8818 // Logical Shift Right by variable
8819 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8820 %{
8821   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8822   effect(KILL cr);
8823 
8824   format %{ "shrq    $dst, $shift" %}
8825   opcode(0xD3, 0x5); /* D3 /5 */
8826   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8827   ins_pipe(ialu_mem_reg);
8828 %}
8829 
8830 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8831 // This idiom is used by the compiler for the i2b bytecode.
8832 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8833 %{
8834   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8835 
8836   format %{ "movsbl  $dst, $src\t# i2b" %}
8837   opcode(0x0F, 0xBE);
8838   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8839   ins_pipe(ialu_reg_reg);
8840 %}
8841 
8842 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8843 // This idiom is used by the compiler the i2s bytecode.
8844 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8845 %{
8846   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8847 
8848   format %{ "movswl  $dst, $src\t# i2s" %}
8849   opcode(0x0F, 0xBF);
8850   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8851   ins_pipe(ialu_reg_reg);
8852 %}
8853 
8854 // ROL/ROR instructions
8855 
8856 // ROL expand
8857 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8858   effect(KILL cr, USE_DEF dst);
8859 
8860   format %{ "roll    $dst" %}
8861   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8862   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8863   ins_pipe(ialu_reg);
8864 %}
8865 
8866 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8867   effect(USE_DEF dst, USE shift, KILL cr);
8868 
8869   format %{ "roll    $dst, $shift" %}
8870   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8871   ins_encode( reg_opc_imm(dst, shift) );
8872   ins_pipe(ialu_reg);
8873 %}
8874 
8875 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8876 %{
8877   effect(USE_DEF dst, USE shift, KILL cr);
8878 
8879   format %{ "roll    $dst, $shift" %}
8880   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8881   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8882   ins_pipe(ialu_reg_reg);
8883 %}
8884 // end of ROL expand
8885 
8886 // Rotate Left by one
8887 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8888 %{
8889   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8890 
8891   expand %{
8892     rolI_rReg_imm1(dst, cr);
8893   %}
8894 %}
8895 
8896 // Rotate Left by 8-bit immediate
8897 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8898 %{
8899   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8900   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8901 
8902   expand %{
8903     rolI_rReg_imm8(dst, lshift, cr);
8904   %}
8905 %}
8906 
8907 // Rotate Left by variable
8908 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8909 %{
8910   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8911 
8912   expand %{
8913     rolI_rReg_CL(dst, shift, cr);
8914   %}
8915 %}
8916 
8917 // Rotate Left by variable
8918 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8919 %{
8920   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8921 
8922   expand %{
8923     rolI_rReg_CL(dst, shift, cr);
8924   %}
8925 %}
8926 
8927 // ROR expand
8928 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8929 %{
8930   effect(USE_DEF dst, KILL cr);
8931 
8932   format %{ "rorl    $dst" %}
8933   opcode(0xD1, 0x1); /* D1 /1 */
8934   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8935   ins_pipe(ialu_reg);
8936 %}
8937 
8938 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8939 %{
8940   effect(USE_DEF dst, USE shift, KILL cr);
8941 
8942   format %{ "rorl    $dst, $shift" %}
8943   opcode(0xC1, 0x1); /* C1 /1 ib */
8944   ins_encode(reg_opc_imm(dst, shift));
8945   ins_pipe(ialu_reg);
8946 %}
8947 
8948 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8949 %{
8950   effect(USE_DEF dst, USE shift, KILL cr);
8951 
8952   format %{ "rorl    $dst, $shift" %}
8953   opcode(0xD3, 0x1); /* D3 /1 */
8954   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8955   ins_pipe(ialu_reg_reg);
8956 %}
8957 // end of ROR expand
8958 
8959 // Rotate Right by one
8960 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8961 %{
8962   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8963 
8964   expand %{
8965     rorI_rReg_imm1(dst, cr);
8966   %}
8967 %}
8968 
8969 // Rotate Right by 8-bit immediate
8970 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8971 %{
8972   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8973   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8974 
8975   expand %{
8976     rorI_rReg_imm8(dst, rshift, cr);
8977   %}
8978 %}
8979 
8980 // Rotate Right by variable
8981 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8982 %{
8983   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8984 
8985   expand %{
8986     rorI_rReg_CL(dst, shift, cr);
8987   %}
8988 %}
8989 
8990 // Rotate Right by variable
8991 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8992 %{
8993   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8994 
8995   expand %{
8996     rorI_rReg_CL(dst, shift, cr);
8997   %}
8998 %}
8999 
9000 // for long rotate
9001 // ROL expand
9002 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9003   effect(USE_DEF dst, KILL cr);
9004 
9005   format %{ "rolq    $dst" %}
9006   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9007   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9008   ins_pipe(ialu_reg);
9009 %}
9010 
9011 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9012   effect(USE_DEF dst, USE shift, KILL cr);
9013 
9014   format %{ "rolq    $dst, $shift" %}
9015   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9016   ins_encode( reg_opc_imm_wide(dst, shift) );
9017   ins_pipe(ialu_reg);
9018 %}
9019 
9020 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9021 %{
9022   effect(USE_DEF dst, USE shift, KILL cr);
9023 
9024   format %{ "rolq    $dst, $shift" %}
9025   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9026   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9027   ins_pipe(ialu_reg_reg);
9028 %}
9029 // end of ROL expand
9030 
9031 // Rotate Left by one
9032 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9033 %{
9034   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9035 
9036   expand %{
9037     rolL_rReg_imm1(dst, cr);
9038   %}
9039 %}
9040 
9041 // Rotate Left by 8-bit immediate
9042 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9043 %{
9044   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9045   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9046 
9047   expand %{
9048     rolL_rReg_imm8(dst, lshift, cr);
9049   %}
9050 %}
9051 
9052 // Rotate Left by variable
9053 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9054 %{
9055   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9056 
9057   expand %{
9058     rolL_rReg_CL(dst, shift, cr);
9059   %}
9060 %}
9061 
9062 // Rotate Left by variable
9063 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9064 %{
9065   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9066 
9067   expand %{
9068     rolL_rReg_CL(dst, shift, cr);
9069   %}
9070 %}
9071 
9072 // ROR expand
9073 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9074 %{
9075   effect(USE_DEF dst, KILL cr);
9076 
9077   format %{ "rorq    $dst" %}
9078   opcode(0xD1, 0x1); /* D1 /1 */
9079   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9080   ins_pipe(ialu_reg);
9081 %}
9082 
9083 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9084 %{
9085   effect(USE_DEF dst, USE shift, KILL cr);
9086 
9087   format %{ "rorq    $dst, $shift" %}
9088   opcode(0xC1, 0x1); /* C1 /1 ib */
9089   ins_encode(reg_opc_imm_wide(dst, shift));
9090   ins_pipe(ialu_reg);
9091 %}
9092 
9093 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9094 %{
9095   effect(USE_DEF dst, USE shift, KILL cr);
9096 
9097   format %{ "rorq    $dst, $shift" %}
9098   opcode(0xD3, 0x1); /* D3 /1 */
9099   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9100   ins_pipe(ialu_reg_reg);
9101 %}
9102 // end of ROR expand
9103 
9104 // Rotate Right by one
9105 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9106 %{
9107   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9108 
9109   expand %{
9110     rorL_rReg_imm1(dst, cr);
9111   %}
9112 %}
9113 
9114 // Rotate Right by 8-bit immediate
9115 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9116 %{
9117   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9118   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9119 
9120   expand %{
9121     rorL_rReg_imm8(dst, rshift, cr);
9122   %}
9123 %}
9124 
9125 // Rotate Right by variable
9126 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9127 %{
9128   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9129 
9130   expand %{
9131     rorL_rReg_CL(dst, shift, cr);
9132   %}
9133 %}
9134 
9135 // Rotate Right by variable
9136 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9137 %{
9138   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9139 
9140   expand %{
9141     rorL_rReg_CL(dst, shift, cr);
9142   %}
9143 %}
9144 
9145 // Logical Instructions
9146 
9147 // Integer Logical Instructions
9148 
9149 // And Instructions
9150 // And Register with Register
9151 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9152 %{
9153   match(Set dst (AndI dst src));
9154   effect(KILL cr);
9155 
9156   format %{ "andl    $dst, $src\t# int" %}
9157   opcode(0x23);
9158   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9159   ins_pipe(ialu_reg_reg);
9160 %}
9161 
9162 // And Register with Immediate 255
9163 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9164 %{
9165   match(Set dst (AndI dst src));
9166 
9167   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9168   opcode(0x0F, 0xB6);
9169   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9170   ins_pipe(ialu_reg);
9171 %}
9172 
9173 // And Register with Immediate 255 and promote to long
9174 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9175 %{
9176   match(Set dst (ConvI2L (AndI src mask)));
9177 
9178   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9179   opcode(0x0F, 0xB6);
9180   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9181   ins_pipe(ialu_reg);
9182 %}
9183 
9184 // And Register with Immediate 65535
9185 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9186 %{
9187   match(Set dst (AndI dst src));
9188 
9189   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9190   opcode(0x0F, 0xB7);
9191   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9192   ins_pipe(ialu_reg);
9193 %}
9194 
9195 // And Register with Immediate 65535 and promote to long
9196 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9197 %{
9198   match(Set dst (ConvI2L (AndI src mask)));
9199 
9200   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9201   opcode(0x0F, 0xB7);
9202   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9203   ins_pipe(ialu_reg);
9204 %}
9205 
9206 // And Register with Immediate
9207 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9208 %{
9209   match(Set dst (AndI dst src));
9210   effect(KILL cr);
9211 
9212   format %{ "andl    $dst, $src\t# int" %}
9213   opcode(0x81, 0x04); /* Opcode 81 /4 */
9214   ins_encode(OpcSErm(dst, src), Con8or32(src));
9215   ins_pipe(ialu_reg);
9216 %}
9217 
9218 // And Register with Memory
9219 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9220 %{
9221   match(Set dst (AndI dst (LoadI src)));
9222   effect(KILL cr);
9223 
9224   ins_cost(125);
9225   format %{ "andl    $dst, $src\t# int" %}
9226   opcode(0x23);
9227   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9228   ins_pipe(ialu_reg_mem);
9229 %}
9230 
9231 // And Memory with Register
9232 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9233 %{
9234   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9235   effect(KILL cr);
9236 
9237   ins_cost(150);
9238   format %{ "andl    $dst, $src\t# int" %}
9239   opcode(0x21); /* Opcode 21 /r */
9240   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9241   ins_pipe(ialu_mem_reg);
9242 %}
9243 
9244 // And Memory with Immediate
9245 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9246 %{
9247   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9248   effect(KILL cr);
9249 
9250   ins_cost(125);
9251   format %{ "andl    $dst, $src\t# int" %}
9252   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9253   ins_encode(REX_mem(dst), OpcSE(src),
9254              RM_opc_mem(secondary, dst), Con8or32(src));
9255   ins_pipe(ialu_mem_imm);
9256 %}
9257 
9258 // BMI1 instructions
9259 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9260   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9261   predicate(UseBMI1Instructions);
9262   effect(KILL cr);
9263 
9264   ins_cost(125);
9265   format %{ "andnl  $dst, $src1, $src2" %}
9266 
9267   ins_encode %{
9268     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9269   %}
9270   ins_pipe(ialu_reg_mem);
9271 %}
9272 
9273 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9274   match(Set dst (AndI (XorI src1 minus_1) src2));
9275   predicate(UseBMI1Instructions);
9276   effect(KILL cr);
9277 
9278   format %{ "andnl  $dst, $src1, $src2" %}
9279 
9280   ins_encode %{
9281     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9282   %}
9283   ins_pipe(ialu_reg);
9284 %}
9285 
9286 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9287   match(Set dst (AndI (SubI imm_zero src) src));
9288   predicate(UseBMI1Instructions);
9289   effect(KILL cr);
9290 
9291   format %{ "blsil  $dst, $src" %}
9292 
9293   ins_encode %{
9294     __ blsil($dst$$Register, $src$$Register);
9295   %}
9296   ins_pipe(ialu_reg);
9297 %}
9298 
9299 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9300   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9301   predicate(UseBMI1Instructions);
9302   effect(KILL cr);
9303 
9304   ins_cost(125);
9305   format %{ "blsil  $dst, $src" %}
9306 
9307   ins_encode %{
9308     __ blsil($dst$$Register, $src$$Address);
9309   %}
9310   ins_pipe(ialu_reg_mem);
9311 %}
9312 
9313 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9314 %{
9315   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9316   predicate(UseBMI1Instructions);
9317   effect(KILL cr);
9318 
9319   ins_cost(125);
9320   format %{ "blsmskl $dst, $src" %}
9321 
9322   ins_encode %{
9323     __ blsmskl($dst$$Register, $src$$Address);
9324   %}
9325   ins_pipe(ialu_reg_mem);
9326 %}
9327 
9328 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9329 %{
9330   match(Set dst (XorI (AddI src minus_1) src));
9331   predicate(UseBMI1Instructions);
9332   effect(KILL cr);
9333 
9334   format %{ "blsmskl $dst, $src" %}
9335 
9336   ins_encode %{
9337     __ blsmskl($dst$$Register, $src$$Register);
9338   %}
9339 
9340   ins_pipe(ialu_reg);
9341 %}
9342 
9343 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9344 %{
9345   match(Set dst (AndI (AddI src minus_1) src) );
9346   predicate(UseBMI1Instructions);
9347   effect(KILL cr);
9348 
9349   format %{ "blsrl  $dst, $src" %}
9350 
9351   ins_encode %{
9352     __ blsrl($dst$$Register, $src$$Register);
9353   %}
9354 
9355   ins_pipe(ialu_reg_mem);
9356 %}
9357 
9358 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9359 %{
9360   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9361   predicate(UseBMI1Instructions);
9362   effect(KILL cr);
9363 
9364   ins_cost(125);
9365   format %{ "blsrl  $dst, $src" %}
9366 
9367   ins_encode %{
9368     __ blsrl($dst$$Register, $src$$Address);
9369   %}
9370 
9371   ins_pipe(ialu_reg);
9372 %}
9373 
9374 // Or Instructions
9375 // Or Register with Register
9376 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9377 %{
9378   match(Set dst (OrI dst src));
9379   effect(KILL cr);
9380 
9381   format %{ "orl     $dst, $src\t# int" %}
9382   opcode(0x0B);
9383   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9384   ins_pipe(ialu_reg_reg);
9385 %}
9386 
9387 // Or Register with Immediate
9388 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9389 %{
9390   match(Set dst (OrI dst src));
9391   effect(KILL cr);
9392 
9393   format %{ "orl     $dst, $src\t# int" %}
9394   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9395   ins_encode(OpcSErm(dst, src), Con8or32(src));
9396   ins_pipe(ialu_reg);
9397 %}
9398 
9399 // Or Register with Memory
9400 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9401 %{
9402   match(Set dst (OrI dst (LoadI src)));
9403   effect(KILL cr);
9404 
9405   ins_cost(125);
9406   format %{ "orl     $dst, $src\t# int" %}
9407   opcode(0x0B);
9408   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9409   ins_pipe(ialu_reg_mem);
9410 %}
9411 
9412 // Or Memory with Register
9413 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9414 %{
9415   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9416   effect(KILL cr);
9417 
9418   ins_cost(150);
9419   format %{ "orl     $dst, $src\t# int" %}
9420   opcode(0x09); /* Opcode 09 /r */
9421   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9422   ins_pipe(ialu_mem_reg);
9423 %}
9424 
9425 // Or Memory with Immediate
9426 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9427 %{
9428   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9429   effect(KILL cr);
9430 
9431   ins_cost(125);
9432   format %{ "orl     $dst, $src\t# int" %}
9433   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9434   ins_encode(REX_mem(dst), OpcSE(src),
9435              RM_opc_mem(secondary, dst), Con8or32(src));
9436   ins_pipe(ialu_mem_imm);
9437 %}
9438 
9439 // Xor Instructions
9440 // Xor Register with Register
9441 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9442 %{
9443   match(Set dst (XorI dst src));
9444   effect(KILL cr);
9445 
9446   format %{ "xorl    $dst, $src\t# int" %}
9447   opcode(0x33);
9448   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9449   ins_pipe(ialu_reg_reg);
9450 %}
9451 
9452 // Xor Register with Immediate -1
9453 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9454   match(Set dst (XorI dst imm));
9455 
9456   format %{ "not    $dst" %}
9457   ins_encode %{
9458      __ notl($dst$$Register);
9459   %}
9460   ins_pipe(ialu_reg);
9461 %}
9462 
9463 // Xor Register with Immediate
9464 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9465 %{
9466   match(Set dst (XorI dst src));
9467   effect(KILL cr);
9468 
9469   format %{ "xorl    $dst, $src\t# int" %}
9470   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9471   ins_encode(OpcSErm(dst, src), Con8or32(src));
9472   ins_pipe(ialu_reg);
9473 %}
9474 
9475 // Xor Register with Memory
9476 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9477 %{
9478   match(Set dst (XorI dst (LoadI src)));
9479   effect(KILL cr);
9480 
9481   ins_cost(125);
9482   format %{ "xorl    $dst, $src\t# int" %}
9483   opcode(0x33);
9484   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9485   ins_pipe(ialu_reg_mem);
9486 %}
9487 
9488 // Xor Memory with Register
9489 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9490 %{
9491   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9492   effect(KILL cr);
9493 
9494   ins_cost(150);
9495   format %{ "xorl    $dst, $src\t# int" %}
9496   opcode(0x31); /* Opcode 31 /r */
9497   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9498   ins_pipe(ialu_mem_reg);
9499 %}
9500 
9501 // Xor Memory with Immediate
9502 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9503 %{
9504   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9505   effect(KILL cr);
9506 
9507   ins_cost(125);
9508   format %{ "xorl    $dst, $src\t# int" %}
9509   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9510   ins_encode(REX_mem(dst), OpcSE(src),
9511              RM_opc_mem(secondary, dst), Con8or32(src));
9512   ins_pipe(ialu_mem_imm);
9513 %}
9514 
9515 
9516 // Long Logical Instructions
9517 
9518 // And Instructions
9519 // And Register with Register
9520 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9521 %{
9522   match(Set dst (AndL dst src));
9523   effect(KILL cr);
9524 
9525   format %{ "andq    $dst, $src\t# long" %}
9526   opcode(0x23);
9527   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9528   ins_pipe(ialu_reg_reg);
9529 %}
9530 
9531 // And Register with Immediate 255
9532 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9533 %{
9534   match(Set dst (AndL dst src));
9535 
9536   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9537   opcode(0x0F, 0xB6);
9538   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9539   ins_pipe(ialu_reg);
9540 %}
9541 
9542 // And Register with Immediate 65535
9543 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9544 %{
9545   match(Set dst (AndL dst src));
9546 
9547   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9548   opcode(0x0F, 0xB7);
9549   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9550   ins_pipe(ialu_reg);
9551 %}
9552 
9553 // And Register with Immediate
9554 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9555 %{
9556   match(Set dst (AndL dst src));
9557   effect(KILL cr);
9558 
9559   format %{ "andq    $dst, $src\t# long" %}
9560   opcode(0x81, 0x04); /* Opcode 81 /4 */
9561   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9562   ins_pipe(ialu_reg);
9563 %}
9564 
9565 // And Register with Memory
9566 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9567 %{
9568   match(Set dst (AndL dst (LoadL src)));
9569   effect(KILL cr);
9570 
9571   ins_cost(125);
9572   format %{ "andq    $dst, $src\t# long" %}
9573   opcode(0x23);
9574   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9575   ins_pipe(ialu_reg_mem);
9576 %}
9577 
9578 // And Memory with Register
9579 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9580 %{
9581   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9582   effect(KILL cr);
9583 
9584   ins_cost(150);
9585   format %{ "andq    $dst, $src\t# long" %}
9586   opcode(0x21); /* Opcode 21 /r */
9587   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9588   ins_pipe(ialu_mem_reg);
9589 %}
9590 
9591 // And Memory with Immediate
9592 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9593 %{
9594   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9595   effect(KILL cr);
9596 
9597   ins_cost(125);
9598   format %{ "andq    $dst, $src\t# long" %}
9599   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9600   ins_encode(REX_mem_wide(dst), OpcSE(src),
9601              RM_opc_mem(secondary, dst), Con8or32(src));
9602   ins_pipe(ialu_mem_imm);
9603 %}
9604 
9605 // BMI1 instructions
9606 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9607   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9608   predicate(UseBMI1Instructions);
9609   effect(KILL cr);
9610 
9611   ins_cost(125);
9612   format %{ "andnq  $dst, $src1, $src2" %}
9613 
9614   ins_encode %{
9615     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9616   %}
9617   ins_pipe(ialu_reg_mem);
9618 %}
9619 
9620 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9621   match(Set dst (AndL (XorL src1 minus_1) src2));
9622   predicate(UseBMI1Instructions);
9623   effect(KILL cr);
9624 
9625   format %{ "andnq  $dst, $src1, $src2" %}
9626 
9627   ins_encode %{
9628   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9629   %}
9630   ins_pipe(ialu_reg_mem);
9631 %}
9632 
9633 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9634   match(Set dst (AndL (SubL imm_zero src) src));
9635   predicate(UseBMI1Instructions);
9636   effect(KILL cr);
9637 
9638   format %{ "blsiq  $dst, $src" %}
9639 
9640   ins_encode %{
9641     __ blsiq($dst$$Register, $src$$Register);
9642   %}
9643   ins_pipe(ialu_reg);
9644 %}
9645 
9646 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9647   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9648   predicate(UseBMI1Instructions);
9649   effect(KILL cr);
9650 
9651   ins_cost(125);
9652   format %{ "blsiq  $dst, $src" %}
9653 
9654   ins_encode %{
9655     __ blsiq($dst$$Register, $src$$Address);
9656   %}
9657   ins_pipe(ialu_reg_mem);
9658 %}
9659 
9660 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9661 %{
9662   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9663   predicate(UseBMI1Instructions);
9664   effect(KILL cr);
9665 
9666   ins_cost(125);
9667   format %{ "blsmskq $dst, $src" %}
9668 
9669   ins_encode %{
9670     __ blsmskq($dst$$Register, $src$$Address);
9671   %}
9672   ins_pipe(ialu_reg_mem);
9673 %}
9674 
9675 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9676 %{
9677   match(Set dst (XorL (AddL src minus_1) src));
9678   predicate(UseBMI1Instructions);
9679   effect(KILL cr);
9680 
9681   format %{ "blsmskq $dst, $src" %}
9682 
9683   ins_encode %{
9684     __ blsmskq($dst$$Register, $src$$Register);
9685   %}
9686 
9687   ins_pipe(ialu_reg);
9688 %}
9689 
9690 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9691 %{
9692   match(Set dst (AndL (AddL src minus_1) src) );
9693   predicate(UseBMI1Instructions);
9694   effect(KILL cr);
9695 
9696   format %{ "blsrq  $dst, $src" %}
9697 
9698   ins_encode %{
9699     __ blsrq($dst$$Register, $src$$Register);
9700   %}
9701 
9702   ins_pipe(ialu_reg);
9703 %}
9704 
9705 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9706 %{
9707   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9708   predicate(UseBMI1Instructions);
9709   effect(KILL cr);
9710 
9711   ins_cost(125);
9712   format %{ "blsrq  $dst, $src" %}
9713 
9714   ins_encode %{
9715     __ blsrq($dst$$Register, $src$$Address);
9716   %}
9717 
9718   ins_pipe(ialu_reg);
9719 %}
9720 
9721 // Or Instructions
9722 // Or Register with Register
9723 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9724 %{
9725   match(Set dst (OrL dst src));
9726   effect(KILL cr);
9727 
9728   format %{ "orq     $dst, $src\t# long" %}
9729   opcode(0x0B);
9730   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9731   ins_pipe(ialu_reg_reg);
9732 %}
9733 
9734 // Use any_RegP to match R15 (TLS register) without spilling.
9735 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9736   match(Set dst (OrL dst (CastP2X src)));
9737   effect(KILL cr);
9738 
9739   format %{ "orq     $dst, $src\t# long" %}
9740   opcode(0x0B);
9741   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9742   ins_pipe(ialu_reg_reg);
9743 %}
9744 
9745 
9746 // Or Register with Immediate
9747 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9748 %{
9749   match(Set dst (OrL dst src));
9750   effect(KILL cr);
9751 
9752   format %{ "orq     $dst, $src\t# long" %}
9753   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9754   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9755   ins_pipe(ialu_reg);
9756 %}
9757 
9758 // Or Register with Memory
9759 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9760 %{
9761   match(Set dst (OrL dst (LoadL src)));
9762   effect(KILL cr);
9763 
9764   ins_cost(125);
9765   format %{ "orq     $dst, $src\t# long" %}
9766   opcode(0x0B);
9767   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9768   ins_pipe(ialu_reg_mem);
9769 %}
9770 
9771 // Or Memory with Register
9772 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9773 %{
9774   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9775   effect(KILL cr);
9776 
9777   ins_cost(150);
9778   format %{ "orq     $dst, $src\t# long" %}
9779   opcode(0x09); /* Opcode 09 /r */
9780   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9781   ins_pipe(ialu_mem_reg);
9782 %}
9783 
9784 // Or Memory with Immediate
9785 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9786 %{
9787   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9788   effect(KILL cr);
9789 
9790   ins_cost(125);
9791   format %{ "orq     $dst, $src\t# long" %}
9792   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9793   ins_encode(REX_mem_wide(dst), OpcSE(src),
9794              RM_opc_mem(secondary, dst), Con8or32(src));
9795   ins_pipe(ialu_mem_imm);
9796 %}
9797 
9798 // Xor Instructions
9799 // Xor Register with Register
9800 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9801 %{
9802   match(Set dst (XorL dst src));
9803   effect(KILL cr);
9804 
9805   format %{ "xorq    $dst, $src\t# long" %}
9806   opcode(0x33);
9807   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9808   ins_pipe(ialu_reg_reg);
9809 %}
9810 
9811 // Xor Register with Immediate -1
9812 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9813   match(Set dst (XorL dst imm));
9814 
9815   format %{ "notq   $dst" %}
9816   ins_encode %{
9817      __ notq($dst$$Register);
9818   %}
9819   ins_pipe(ialu_reg);
9820 %}
9821 
9822 // Xor Register with Immediate
9823 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9824 %{
9825   match(Set dst (XorL dst src));
9826   effect(KILL cr);
9827 
9828   format %{ "xorq    $dst, $src\t# long" %}
9829   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9830   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9831   ins_pipe(ialu_reg);
9832 %}
9833 
9834 // Xor Register with Memory
9835 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9836 %{
9837   match(Set dst (XorL dst (LoadL src)));
9838   effect(KILL cr);
9839 
9840   ins_cost(125);
9841   format %{ "xorq    $dst, $src\t# long" %}
9842   opcode(0x33);
9843   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9844   ins_pipe(ialu_reg_mem);
9845 %}
9846 
9847 // Xor Memory with Register
9848 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9849 %{
9850   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9851   effect(KILL cr);
9852 
9853   ins_cost(150);
9854   format %{ "xorq    $dst, $src\t# long" %}
9855   opcode(0x31); /* Opcode 31 /r */
9856   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9857   ins_pipe(ialu_mem_reg);
9858 %}
9859 
9860 // Xor Memory with Immediate
9861 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9862 %{
9863   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9864   effect(KILL cr);
9865 
9866   ins_cost(125);
9867   format %{ "xorq    $dst, $src\t# long" %}
9868   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9869   ins_encode(REX_mem_wide(dst), OpcSE(src),
9870              RM_opc_mem(secondary, dst), Con8or32(src));
9871   ins_pipe(ialu_mem_imm);
9872 %}
9873 
9874 // Convert Int to Boolean
9875 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9876 %{
9877   match(Set dst (Conv2B src));
9878   effect(KILL cr);
9879 
9880   format %{ "testl   $src, $src\t# ci2b\n\t"
9881             "setnz   $dst\n\t"
9882             "movzbl  $dst, $dst" %}
9883   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9884              setNZ_reg(dst),
9885              REX_reg_breg(dst, dst), // movzbl
9886              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9887   ins_pipe(pipe_slow); // XXX
9888 %}
9889 
9890 // Convert Pointer to Boolean
9891 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9892 %{
9893   match(Set dst (Conv2B src));
9894   effect(KILL cr);
9895 
9896   format %{ "testq   $src, $src\t# cp2b\n\t"
9897             "setnz   $dst\n\t"
9898             "movzbl  $dst, $dst" %}
9899   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9900              setNZ_reg(dst),
9901              REX_reg_breg(dst, dst), // movzbl
9902              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9903   ins_pipe(pipe_slow); // XXX
9904 %}
9905 
9906 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9907 %{
9908   match(Set dst (CmpLTMask p q));
9909   effect(KILL cr);
9910 
9911   ins_cost(400);
9912   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9913             "setlt   $dst\n\t"
9914             "movzbl  $dst, $dst\n\t"
9915             "negl    $dst" %}
9916   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9917              setLT_reg(dst),
9918              REX_reg_breg(dst, dst), // movzbl
9919              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9920              neg_reg(dst));
9921   ins_pipe(pipe_slow);
9922 %}
9923 
9924 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9925 %{
9926   match(Set dst (CmpLTMask dst zero));
9927   effect(KILL cr);
9928 
9929   ins_cost(100);
9930   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9931   ins_encode %{
9932   __ sarl($dst$$Register, 31);
9933   %}
9934   ins_pipe(ialu_reg);
9935 %}
9936 
9937 /* Better to save a register than avoid a branch */
9938 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9939 %{
9940   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9941   effect(KILL cr);
9942   ins_cost(300);
9943   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9944             "jge    done\n\t"
9945             "addl   $p,$y\n"
9946             "done:  " %}
9947   ins_encode %{
9948     Register Rp = $p$$Register;
9949     Register Rq = $q$$Register;
9950     Register Ry = $y$$Register;
9951     Label done;
9952     __ subl(Rp, Rq);
9953     __ jccb(Assembler::greaterEqual, done);
9954     __ addl(Rp, Ry);
9955     __ bind(done);
9956   %}
9957   ins_pipe(pipe_cmplt);
9958 %}
9959 
9960 /* Better to save a register than avoid a branch */
9961 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9962 %{
9963   match(Set y (AndI (CmpLTMask p q) y));
9964   effect(KILL cr);
9965 
9966   ins_cost(300);
9967 
9968   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9969             "jlt      done\n\t"
9970             "xorl     $y, $y\n"
9971             "done:  " %}
9972   ins_encode %{
9973     Register Rp = $p$$Register;
9974     Register Rq = $q$$Register;
9975     Register Ry = $y$$Register;
9976     Label done;
9977     __ cmpl(Rp, Rq);
9978     __ jccb(Assembler::less, done);
9979     __ xorl(Ry, Ry);
9980     __ bind(done);
9981   %}
9982   ins_pipe(pipe_cmplt);
9983 %}
9984 
9985 
9986 //---------- FP Instructions------------------------------------------------
9987 
9988 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9989 %{
9990   match(Set cr (CmpF src1 src2));
9991 
9992   ins_cost(145);
9993   format %{ "ucomiss $src1, $src2\n\t"
9994             "jnp,s   exit\n\t"
9995             "pushfq\t# saw NaN, set CF\n\t"
9996             "andq    [rsp], #0xffffff2b\n\t"
9997             "popfq\n"
9998     "exit:" %}
9999   ins_encode %{
10000     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10001     emit_cmpfp_fixup(_masm);
10002   %}
10003   ins_pipe(pipe_slow);
10004 %}
10005 
10006 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10007   match(Set cr (CmpF src1 src2));
10008 
10009   ins_cost(100);
10010   format %{ "ucomiss $src1, $src2" %}
10011   ins_encode %{
10012     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10013   %}
10014   ins_pipe(pipe_slow);
10015 %}
10016 
10017 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10018 %{
10019   match(Set cr (CmpF src1 (LoadF src2)));
10020 
10021   ins_cost(145);
10022   format %{ "ucomiss $src1, $src2\n\t"
10023             "jnp,s   exit\n\t"
10024             "pushfq\t# saw NaN, set CF\n\t"
10025             "andq    [rsp], #0xffffff2b\n\t"
10026             "popfq\n"
10027     "exit:" %}
10028   ins_encode %{
10029     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10030     emit_cmpfp_fixup(_masm);
10031   %}
10032   ins_pipe(pipe_slow);
10033 %}
10034 
10035 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10036   match(Set cr (CmpF src1 (LoadF src2)));
10037 
10038   ins_cost(100);
10039   format %{ "ucomiss $src1, $src2" %}
10040   ins_encode %{
10041     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10042   %}
10043   ins_pipe(pipe_slow);
10044 %}
10045 
10046 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10047   match(Set cr (CmpF src con));
10048 
10049   ins_cost(145);
10050   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10051             "jnp,s   exit\n\t"
10052             "pushfq\t# saw NaN, set CF\n\t"
10053             "andq    [rsp], #0xffffff2b\n\t"
10054             "popfq\n"
10055     "exit:" %}
10056   ins_encode %{
10057     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10058     emit_cmpfp_fixup(_masm);
10059   %}
10060   ins_pipe(pipe_slow);
10061 %}
10062 
10063 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10064   match(Set cr (CmpF src con));
10065   ins_cost(100);
10066   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10067   ins_encode %{
10068     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10069   %}
10070   ins_pipe(pipe_slow);
10071 %}
10072 
10073 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10074 %{
10075   match(Set cr (CmpD src1 src2));
10076 
10077   ins_cost(145);
10078   format %{ "ucomisd $src1, $src2\n\t"
10079             "jnp,s   exit\n\t"
10080             "pushfq\t# saw NaN, set CF\n\t"
10081             "andq    [rsp], #0xffffff2b\n\t"
10082             "popfq\n"
10083     "exit:" %}
10084   ins_encode %{
10085     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10086     emit_cmpfp_fixup(_masm);
10087   %}
10088   ins_pipe(pipe_slow);
10089 %}
10090 
10091 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10092   match(Set cr (CmpD src1 src2));
10093 
10094   ins_cost(100);
10095   format %{ "ucomisd $src1, $src2 test" %}
10096   ins_encode %{
10097     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10098   %}
10099   ins_pipe(pipe_slow);
10100 %}
10101 
10102 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10103 %{
10104   match(Set cr (CmpD src1 (LoadD src2)));
10105 
10106   ins_cost(145);
10107   format %{ "ucomisd $src1, $src2\n\t"
10108             "jnp,s   exit\n\t"
10109             "pushfq\t# saw NaN, set CF\n\t"
10110             "andq    [rsp], #0xffffff2b\n\t"
10111             "popfq\n"
10112     "exit:" %}
10113   ins_encode %{
10114     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10115     emit_cmpfp_fixup(_masm);
10116   %}
10117   ins_pipe(pipe_slow);
10118 %}
10119 
10120 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10121   match(Set cr (CmpD src1 (LoadD src2)));
10122 
10123   ins_cost(100);
10124   format %{ "ucomisd $src1, $src2" %}
10125   ins_encode %{
10126     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10127   %}
10128   ins_pipe(pipe_slow);
10129 %}
10130 
10131 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10132   match(Set cr (CmpD src con));
10133 
10134   ins_cost(145);
10135   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10136             "jnp,s   exit\n\t"
10137             "pushfq\t# saw NaN, set CF\n\t"
10138             "andq    [rsp], #0xffffff2b\n\t"
10139             "popfq\n"
10140     "exit:" %}
10141   ins_encode %{
10142     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10143     emit_cmpfp_fixup(_masm);
10144   %}
10145   ins_pipe(pipe_slow);
10146 %}
10147 
10148 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10149   match(Set cr (CmpD src con));
10150   ins_cost(100);
10151   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10152   ins_encode %{
10153     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10154   %}
10155   ins_pipe(pipe_slow);
10156 %}
10157 
10158 // Compare into -1,0,1
10159 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10160 %{
10161   match(Set dst (CmpF3 src1 src2));
10162   effect(KILL cr);
10163 
10164   ins_cost(275);
10165   format %{ "ucomiss $src1, $src2\n\t"
10166             "movl    $dst, #-1\n\t"
10167             "jp,s    done\n\t"
10168             "jb,s    done\n\t"
10169             "setne   $dst\n\t"
10170             "movzbl  $dst, $dst\n"
10171     "done:" %}
10172   ins_encode %{
10173     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10174     emit_cmpfp3(_masm, $dst$$Register);
10175   %}
10176   ins_pipe(pipe_slow);
10177 %}
10178 
10179 // Compare into -1,0,1
10180 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10181 %{
10182   match(Set dst (CmpF3 src1 (LoadF src2)));
10183   effect(KILL cr);
10184 
10185   ins_cost(275);
10186   format %{ "ucomiss $src1, $src2\n\t"
10187             "movl    $dst, #-1\n\t"
10188             "jp,s    done\n\t"
10189             "jb,s    done\n\t"
10190             "setne   $dst\n\t"
10191             "movzbl  $dst, $dst\n"
10192     "done:" %}
10193   ins_encode %{
10194     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10195     emit_cmpfp3(_masm, $dst$$Register);
10196   %}
10197   ins_pipe(pipe_slow);
10198 %}
10199 
10200 // Compare into -1,0,1
10201 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10202   match(Set dst (CmpF3 src con));
10203   effect(KILL cr);
10204 
10205   ins_cost(275);
10206   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10207             "movl    $dst, #-1\n\t"
10208             "jp,s    done\n\t"
10209             "jb,s    done\n\t"
10210             "setne   $dst\n\t"
10211             "movzbl  $dst, $dst\n"
10212     "done:" %}
10213   ins_encode %{
10214     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10215     emit_cmpfp3(_masm, $dst$$Register);
10216   %}
10217   ins_pipe(pipe_slow);
10218 %}
10219 
10220 // Compare into -1,0,1
10221 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10222 %{
10223   match(Set dst (CmpD3 src1 src2));
10224   effect(KILL cr);
10225 
10226   ins_cost(275);
10227   format %{ "ucomisd $src1, $src2\n\t"
10228             "movl    $dst, #-1\n\t"
10229             "jp,s    done\n\t"
10230             "jb,s    done\n\t"
10231             "setne   $dst\n\t"
10232             "movzbl  $dst, $dst\n"
10233     "done:" %}
10234   ins_encode %{
10235     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10236     emit_cmpfp3(_masm, $dst$$Register);
10237   %}
10238   ins_pipe(pipe_slow);
10239 %}
10240 
10241 // Compare into -1,0,1
10242 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10243 %{
10244   match(Set dst (CmpD3 src1 (LoadD src2)));
10245   effect(KILL cr);
10246 
10247   ins_cost(275);
10248   format %{ "ucomisd $src1, $src2\n\t"
10249             "movl    $dst, #-1\n\t"
10250             "jp,s    done\n\t"
10251             "jb,s    done\n\t"
10252             "setne   $dst\n\t"
10253             "movzbl  $dst, $dst\n"
10254     "done:" %}
10255   ins_encode %{
10256     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10257     emit_cmpfp3(_masm, $dst$$Register);
10258   %}
10259   ins_pipe(pipe_slow);
10260 %}
10261 
10262 // Compare into -1,0,1
10263 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10264   match(Set dst (CmpD3 src con));
10265   effect(KILL cr);
10266 
10267   ins_cost(275);
10268   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10269             "movl    $dst, #-1\n\t"
10270             "jp,s    done\n\t"
10271             "jb,s    done\n\t"
10272             "setne   $dst\n\t"
10273             "movzbl  $dst, $dst\n"
10274     "done:" %}
10275   ins_encode %{
10276     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10277     emit_cmpfp3(_masm, $dst$$Register);
10278   %}
10279   ins_pipe(pipe_slow);
10280 %}
10281 
10282 //----------Arithmetic Conversion Instructions---------------------------------
10283 
10284 instruct roundFloat_nop(regF dst)
10285 %{
10286   match(Set dst (RoundFloat dst));
10287 
10288   ins_cost(0);
10289   ins_encode();
10290   ins_pipe(empty);
10291 %}
10292 
10293 instruct roundDouble_nop(regD dst)
10294 %{
10295   match(Set dst (RoundDouble dst));
10296 
10297   ins_cost(0);
10298   ins_encode();
10299   ins_pipe(empty);
10300 %}
10301 
10302 instruct convF2D_reg_reg(regD dst, regF src)
10303 %{
10304   match(Set dst (ConvF2D src));
10305 
10306   format %{ "cvtss2sd $dst, $src" %}
10307   ins_encode %{
10308     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10309   %}
10310   ins_pipe(pipe_slow); // XXX
10311 %}
10312 
10313 instruct convF2D_reg_mem(regD dst, memory src)
10314 %{
10315   match(Set dst (ConvF2D (LoadF src)));
10316 
10317   format %{ "cvtss2sd $dst, $src" %}
10318   ins_encode %{
10319     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10320   %}
10321   ins_pipe(pipe_slow); // XXX
10322 %}
10323 
10324 instruct convD2F_reg_reg(regF dst, regD src)
10325 %{
10326   match(Set dst (ConvD2F src));
10327 
10328   format %{ "cvtsd2ss $dst, $src" %}
10329   ins_encode %{
10330     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10331   %}
10332   ins_pipe(pipe_slow); // XXX
10333 %}
10334 
10335 instruct convD2F_reg_mem(regF dst, memory src)
10336 %{
10337   match(Set dst (ConvD2F (LoadD src)));
10338 
10339   format %{ "cvtsd2ss $dst, $src" %}
10340   ins_encode %{
10341     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10342   %}
10343   ins_pipe(pipe_slow); // XXX
10344 %}
10345 
10346 // XXX do mem variants
10347 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10348 %{
10349   match(Set dst (ConvF2I src));
10350   effect(KILL cr);
10351 
10352   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10353             "cmpl    $dst, #0x80000000\n\t"
10354             "jne,s   done\n\t"
10355             "subq    rsp, #8\n\t"
10356             "movss   [rsp], $src\n\t"
10357             "call    f2i_fixup\n\t"
10358             "popq    $dst\n"
10359     "done:   "%}
10360   ins_encode %{
10361     Label done;
10362     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10363     __ cmpl($dst$$Register, 0x80000000);
10364     __ jccb(Assembler::notEqual, done);
10365     __ subptr(rsp, 8);
10366     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10367     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10368     __ pop($dst$$Register);
10369     __ bind(done);
10370   %}
10371   ins_pipe(pipe_slow);
10372 %}
10373 
10374 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10375 %{
10376   match(Set dst (ConvF2L src));
10377   effect(KILL cr);
10378 
10379   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10380             "cmpq    $dst, [0x8000000000000000]\n\t"
10381             "jne,s   done\n\t"
10382             "subq    rsp, #8\n\t"
10383             "movss   [rsp], $src\n\t"
10384             "call    f2l_fixup\n\t"
10385             "popq    $dst\n"
10386     "done:   "%}
10387   ins_encode %{
10388     Label done;
10389     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10390     __ cmp64($dst$$Register,
10391              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10392     __ jccb(Assembler::notEqual, done);
10393     __ subptr(rsp, 8);
10394     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10395     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10396     __ pop($dst$$Register);
10397     __ bind(done);
10398   %}
10399   ins_pipe(pipe_slow);
10400 %}
10401 
10402 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10403 %{
10404   match(Set dst (ConvD2I src));
10405   effect(KILL cr);
10406 
10407   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10408             "cmpl    $dst, #0x80000000\n\t"
10409             "jne,s   done\n\t"
10410             "subq    rsp, #8\n\t"
10411             "movsd   [rsp], $src\n\t"
10412             "call    d2i_fixup\n\t"
10413             "popq    $dst\n"
10414     "done:   "%}
10415   ins_encode %{
10416     Label done;
10417     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10418     __ cmpl($dst$$Register, 0x80000000);
10419     __ jccb(Assembler::notEqual, done);
10420     __ subptr(rsp, 8);
10421     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10422     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10423     __ pop($dst$$Register);
10424     __ bind(done);
10425   %}
10426   ins_pipe(pipe_slow);
10427 %}
10428 
10429 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10430 %{
10431   match(Set dst (ConvD2L src));
10432   effect(KILL cr);
10433 
10434   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10435             "cmpq    $dst, [0x8000000000000000]\n\t"
10436             "jne,s   done\n\t"
10437             "subq    rsp, #8\n\t"
10438             "movsd   [rsp], $src\n\t"
10439             "call    d2l_fixup\n\t"
10440             "popq    $dst\n"
10441     "done:   "%}
10442   ins_encode %{
10443     Label done;
10444     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10445     __ cmp64($dst$$Register,
10446              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10447     __ jccb(Assembler::notEqual, done);
10448     __ subptr(rsp, 8);
10449     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10450     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10451     __ pop($dst$$Register);
10452     __ bind(done);
10453   %}
10454   ins_pipe(pipe_slow);
10455 %}
10456 
10457 instruct convI2F_reg_reg(regF dst, rRegI src)
10458 %{
10459   predicate(!UseXmmI2F);
10460   match(Set dst (ConvI2F src));
10461 
10462   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10463   ins_encode %{
10464     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10465   %}
10466   ins_pipe(pipe_slow); // XXX
10467 %}
10468 
10469 instruct convI2F_reg_mem(regF dst, memory src)
10470 %{
10471   match(Set dst (ConvI2F (LoadI src)));
10472 
10473   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10474   ins_encode %{
10475     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10476   %}
10477   ins_pipe(pipe_slow); // XXX
10478 %}
10479 
10480 instruct convI2D_reg_reg(regD dst, rRegI src)
10481 %{
10482   predicate(!UseXmmI2D);
10483   match(Set dst (ConvI2D src));
10484 
10485   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10486   ins_encode %{
10487     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10488   %}
10489   ins_pipe(pipe_slow); // XXX
10490 %}
10491 
10492 instruct convI2D_reg_mem(regD dst, memory src)
10493 %{
10494   match(Set dst (ConvI2D (LoadI src)));
10495 
10496   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10497   ins_encode %{
10498     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10499   %}
10500   ins_pipe(pipe_slow); // XXX
10501 %}
10502 
10503 instruct convXI2F_reg(regF dst, rRegI src)
10504 %{
10505   predicate(UseXmmI2F);
10506   match(Set dst (ConvI2F src));
10507 
10508   format %{ "movdl $dst, $src\n\t"
10509             "cvtdq2psl $dst, $dst\t# i2f" %}
10510   ins_encode %{
10511     __ movdl($dst$$XMMRegister, $src$$Register);
10512     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10513   %}
10514   ins_pipe(pipe_slow); // XXX
10515 %}
10516 
10517 instruct convXI2D_reg(regD dst, rRegI src)
10518 %{
10519   predicate(UseXmmI2D);
10520   match(Set dst (ConvI2D src));
10521 
10522   format %{ "movdl $dst, $src\n\t"
10523             "cvtdq2pdl $dst, $dst\t# i2d" %}
10524   ins_encode %{
10525     __ movdl($dst$$XMMRegister, $src$$Register);
10526     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10527   %}
10528   ins_pipe(pipe_slow); // XXX
10529 %}
10530 
10531 instruct convL2F_reg_reg(regF dst, rRegL src)
10532 %{
10533   match(Set dst (ConvL2F src));
10534 
10535   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10536   ins_encode %{
10537     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10538   %}
10539   ins_pipe(pipe_slow); // XXX
10540 %}
10541 
10542 instruct convL2F_reg_mem(regF dst, memory src)
10543 %{
10544   match(Set dst (ConvL2F (LoadL src)));
10545 
10546   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10547   ins_encode %{
10548     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10549   %}
10550   ins_pipe(pipe_slow); // XXX
10551 %}
10552 
10553 instruct convL2D_reg_reg(regD dst, rRegL src)
10554 %{
10555   match(Set dst (ConvL2D src));
10556 
10557   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10558   ins_encode %{
10559     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10560   %}
10561   ins_pipe(pipe_slow); // XXX
10562 %}
10563 
10564 instruct convL2D_reg_mem(regD dst, memory src)
10565 %{
10566   match(Set dst (ConvL2D (LoadL src)));
10567 
10568   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10569   ins_encode %{
10570     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10571   %}
10572   ins_pipe(pipe_slow); // XXX
10573 %}
10574 
10575 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10576 %{
10577   match(Set dst (ConvI2L src));
10578 
10579   ins_cost(125);
10580   format %{ "movslq  $dst, $src\t# i2l" %}
10581   ins_encode %{
10582     __ movslq($dst$$Register, $src$$Register);
10583   %}
10584   ins_pipe(ialu_reg_reg);
10585 %}
10586 
10587 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10588 // %{
10589 //   match(Set dst (ConvI2L src));
10590 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10591 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10592 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10593 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10594 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10595 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10596 
10597 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10598 //   ins_encode(enc_copy(dst, src));
10599 // //   opcode(0x63); // needs REX.W
10600 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10601 //   ins_pipe(ialu_reg_reg);
10602 // %}
10603 
10604 // Zero-extend convert int to long
10605 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10606 %{
10607   match(Set dst (AndL (ConvI2L src) mask));
10608 
10609   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10610   ins_encode %{
10611     if ($dst$$reg != $src$$reg) {
10612       __ movl($dst$$Register, $src$$Register);
10613     }
10614   %}
10615   ins_pipe(ialu_reg_reg);
10616 %}
10617 
10618 // Zero-extend convert int to long
10619 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10620 %{
10621   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10622 
10623   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10624   ins_encode %{
10625     __ movl($dst$$Register, $src$$Address);
10626   %}
10627   ins_pipe(ialu_reg_mem);
10628 %}
10629 
10630 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10631 %{
10632   match(Set dst (AndL src mask));
10633 
10634   format %{ "movl    $dst, $src\t# zero-extend long" %}
10635   ins_encode %{
10636     __ movl($dst$$Register, $src$$Register);
10637   %}
10638   ins_pipe(ialu_reg_reg);
10639 %}
10640 
10641 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10642 %{
10643   match(Set dst (ConvL2I src));
10644 
10645   format %{ "movl    $dst, $src\t# l2i" %}
10646   ins_encode %{
10647     __ movl($dst$$Register, $src$$Register);
10648   %}
10649   ins_pipe(ialu_reg_reg);
10650 %}
10651 
10652 
10653 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10654   match(Set dst (MoveF2I src));
10655   effect(DEF dst, USE src);
10656 
10657   ins_cost(125);
10658   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10659   ins_encode %{
10660     __ movl($dst$$Register, Address(rsp, $src$$disp));
10661   %}
10662   ins_pipe(ialu_reg_mem);
10663 %}
10664 
10665 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10666   match(Set dst (MoveI2F src));
10667   effect(DEF dst, USE src);
10668 
10669   ins_cost(125);
10670   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10671   ins_encode %{
10672     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10673   %}
10674   ins_pipe(pipe_slow);
10675 %}
10676 
10677 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10678   match(Set dst (MoveD2L src));
10679   effect(DEF dst, USE src);
10680 
10681   ins_cost(125);
10682   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10683   ins_encode %{
10684     __ movq($dst$$Register, Address(rsp, $src$$disp));
10685   %}
10686   ins_pipe(ialu_reg_mem);
10687 %}
10688 
10689 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10690   predicate(!UseXmmLoadAndClearUpper);
10691   match(Set dst (MoveL2D src));
10692   effect(DEF dst, USE src);
10693 
10694   ins_cost(125);
10695   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10696   ins_encode %{
10697     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10698   %}
10699   ins_pipe(pipe_slow);
10700 %}
10701 
10702 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10703   predicate(UseXmmLoadAndClearUpper);
10704   match(Set dst (MoveL2D src));
10705   effect(DEF dst, USE src);
10706 
10707   ins_cost(125);
10708   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10709   ins_encode %{
10710     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10711   %}
10712   ins_pipe(pipe_slow);
10713 %}
10714 
10715 
10716 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10717   match(Set dst (MoveF2I src));
10718   effect(DEF dst, USE src);
10719 
10720   ins_cost(95); // XXX
10721   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10722   ins_encode %{
10723     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10724   %}
10725   ins_pipe(pipe_slow);
10726 %}
10727 
10728 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10729   match(Set dst (MoveI2F src));
10730   effect(DEF dst, USE src);
10731 
10732   ins_cost(100);
10733   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10734   ins_encode %{
10735     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10736   %}
10737   ins_pipe( ialu_mem_reg );
10738 %}
10739 
10740 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10741   match(Set dst (MoveD2L src));
10742   effect(DEF dst, USE src);
10743 
10744   ins_cost(95); // XXX
10745   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10746   ins_encode %{
10747     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10748   %}
10749   ins_pipe(pipe_slow);
10750 %}
10751 
10752 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10753   match(Set dst (MoveL2D src));
10754   effect(DEF dst, USE src);
10755 
10756   ins_cost(100);
10757   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10758   ins_encode %{
10759     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10760   %}
10761   ins_pipe(ialu_mem_reg);
10762 %}
10763 
10764 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10765   match(Set dst (MoveF2I src));
10766   effect(DEF dst, USE src);
10767   ins_cost(85);
10768   format %{ "movd    $dst,$src\t# MoveF2I" %}
10769   ins_encode %{
10770     __ movdl($dst$$Register, $src$$XMMRegister);
10771   %}
10772   ins_pipe( pipe_slow );
10773 %}
10774 
10775 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10776   match(Set dst (MoveD2L src));
10777   effect(DEF dst, USE src);
10778   ins_cost(85);
10779   format %{ "movd    $dst,$src\t# MoveD2L" %}
10780   ins_encode %{
10781     __ movdq($dst$$Register, $src$$XMMRegister);
10782   %}
10783   ins_pipe( pipe_slow );
10784 %}
10785 
10786 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10787   match(Set dst (MoveI2F src));
10788   effect(DEF dst, USE src);
10789   ins_cost(100);
10790   format %{ "movd    $dst,$src\t# MoveI2F" %}
10791   ins_encode %{
10792     __ movdl($dst$$XMMRegister, $src$$Register);
10793   %}
10794   ins_pipe( pipe_slow );
10795 %}
10796 
10797 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10798   match(Set dst (MoveL2D src));
10799   effect(DEF dst, USE src);
10800   ins_cost(100);
10801   format %{ "movd    $dst,$src\t# MoveL2D" %}
10802   ins_encode %{
10803      __ movdq($dst$$XMMRegister, $src$$Register);
10804   %}
10805   ins_pipe( pipe_slow );
10806 %}
10807 
10808 
10809 // =======================================================================
10810 // fast clearing of an array
10811 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10812                   Universe dummy, rFlagsReg cr)
10813 %{
10814   predicate(!((ClearArrayNode*)n)->is_large());
10815   match(Set dummy (ClearArray cnt base));
10816   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10817 
10818   format %{ $$template
10819     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10820     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10821     $$emit$$"jg      LARGE\n\t"
10822     $$emit$$"dec     rcx\n\t"
10823     $$emit$$"js      DONE\t# Zero length\n\t"
10824     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10825     $$emit$$"dec     rcx\n\t"
10826     $$emit$$"jge     LOOP\n\t"
10827     $$emit$$"jmp     DONE\n\t"
10828     $$emit$$"# LARGE:\n\t"
10829     if (UseFastStosb) {
10830        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10831        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10832     } else if (UseXMMForObjInit) {
10833        $$emit$$"mov     rdi,rax\n\t"
10834        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10835        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10836        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10837        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10838        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10839        $$emit$$"add     0x40,rax\n\t"
10840        $$emit$$"# L_zero_64_bytes:\n\t"
10841        $$emit$$"sub     0x8,rcx\n\t"
10842        $$emit$$"jge     L_loop\n\t"
10843        $$emit$$"add     0x4,rcx\n\t"
10844        $$emit$$"jl      L_tail\n\t"
10845        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10846        $$emit$$"add     0x20,rax\n\t"
10847        $$emit$$"sub     0x4,rcx\n\t"
10848        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10849        $$emit$$"add     0x4,rcx\n\t"
10850        $$emit$$"jle     L_end\n\t"
10851        $$emit$$"dec     rcx\n\t"
10852        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10853        $$emit$$"vmovq   xmm0,(rax)\n\t"
10854        $$emit$$"add     0x8,rax\n\t"
10855        $$emit$$"dec     rcx\n\t"
10856        $$emit$$"jge     L_sloop\n\t"
10857        $$emit$$"# L_end:\n\t"
10858     } else {
10859        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10860     }
10861     $$emit$$"# DONE"
10862   %}
10863   ins_encode %{
10864     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10865                  $tmp$$XMMRegister, false);
10866   %}
10867   ins_pipe(pipe_slow);
10868 %}
10869 
10870 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero, 
10871                         Universe dummy, rFlagsReg cr)
10872 %{
10873   predicate(((ClearArrayNode*)n)->is_large());
10874   match(Set dummy (ClearArray cnt base));
10875   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10876 
10877   format %{ $$template
10878     if (UseFastStosb) {
10879        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10880        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10881        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10882     } else if (UseXMMForObjInit) {
10883        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10884        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10885        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10886        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10887        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10888        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10889        $$emit$$"add     0x40,rax\n\t"
10890        $$emit$$"# L_zero_64_bytes:\n\t"
10891        $$emit$$"sub     0x8,rcx\n\t"
10892        $$emit$$"jge     L_loop\n\t"
10893        $$emit$$"add     0x4,rcx\n\t"
10894        $$emit$$"jl      L_tail\n\t"
10895        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10896        $$emit$$"add     0x20,rax\n\t"
10897        $$emit$$"sub     0x4,rcx\n\t"
10898        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10899        $$emit$$"add     0x4,rcx\n\t"
10900        $$emit$$"jle     L_end\n\t"
10901        $$emit$$"dec     rcx\n\t"
10902        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10903        $$emit$$"vmovq   xmm0,(rax)\n\t"
10904        $$emit$$"add     0x8,rax\n\t"
10905        $$emit$$"dec     rcx\n\t"
10906        $$emit$$"jge     L_sloop\n\t"
10907        $$emit$$"# L_end:\n\t"
10908     } else {
10909        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10910        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10911     }
10912   %}
10913   ins_encode %{
10914     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, 
10915                  $tmp$$XMMRegister, true);
10916   %}
10917   ins_pipe(pipe_slow);
10918 %}
10919 
10920 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10921                          rax_RegI result, regD tmp1, rFlagsReg cr)
10922 %{
10923   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10924   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10925   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10926 
10927   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10928   ins_encode %{
10929     __ string_compare($str1$$Register, $str2$$Register,
10930                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10931                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10932   %}
10933   ins_pipe( pipe_slow );
10934 %}
10935 
10936 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10937                          rax_RegI result, regD tmp1, rFlagsReg cr)
10938 %{
10939   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10940   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10941   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10942 
10943   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10944   ins_encode %{
10945     __ string_compare($str1$$Register, $str2$$Register,
10946                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10947                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10948   %}
10949   ins_pipe( pipe_slow );
10950 %}
10951 
10952 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10953                           rax_RegI result, regD tmp1, rFlagsReg cr)
10954 %{
10955   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10956   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10957   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10958 
10959   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10960   ins_encode %{
10961     __ string_compare($str1$$Register, $str2$$Register,
10962                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10963                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10964   %}
10965   ins_pipe( pipe_slow );
10966 %}
10967 
10968 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10969                           rax_RegI result, regD tmp1, rFlagsReg cr)
10970 %{
10971   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10972   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10973   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10974 
10975   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10976   ins_encode %{
10977     __ string_compare($str2$$Register, $str1$$Register,
10978                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10979                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10980   %}
10981   ins_pipe( pipe_slow );
10982 %}
10983 
10984 // fast search of substring with known size.
10985 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10986                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10987 %{
10988   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10989   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10990   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10991 
10992   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10993   ins_encode %{
10994     int icnt2 = (int)$int_cnt2$$constant;
10995     if (icnt2 >= 16) {
10996       // IndexOf for constant substrings with size >= 16 elements
10997       // which don't need to be loaded through stack.
10998       __ string_indexofC8($str1$$Register, $str2$$Register,
10999                           $cnt1$$Register, $cnt2$$Register,
11000                           icnt2, $result$$Register,
11001                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11002     } else {
11003       // Small strings are loaded through stack if they cross page boundary.
11004       __ string_indexof($str1$$Register, $str2$$Register,
11005                         $cnt1$$Register, $cnt2$$Register,
11006                         icnt2, $result$$Register,
11007                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11008     }
11009   %}
11010   ins_pipe( pipe_slow );
11011 %}
11012 
11013 // fast search of substring with known size.
11014 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11015                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11016 %{
11017   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11018   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11019   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11020 
11021   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11022   ins_encode %{
11023     int icnt2 = (int)$int_cnt2$$constant;
11024     if (icnt2 >= 8) {
11025       // IndexOf for constant substrings with size >= 8 elements
11026       // which don't need to be loaded through stack.
11027       __ string_indexofC8($str1$$Register, $str2$$Register,
11028                           $cnt1$$Register, $cnt2$$Register,
11029                           icnt2, $result$$Register,
11030                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11031     } else {
11032       // Small strings are loaded through stack if they cross page boundary.
11033       __ string_indexof($str1$$Register, $str2$$Register,
11034                         $cnt1$$Register, $cnt2$$Register,
11035                         icnt2, $result$$Register,
11036                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11037     }
11038   %}
11039   ins_pipe( pipe_slow );
11040 %}
11041 
11042 // fast search of substring with known size.
11043 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11044                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11045 %{
11046   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11047   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11048   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11049 
11050   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11051   ins_encode %{
11052     int icnt2 = (int)$int_cnt2$$constant;
11053     if (icnt2 >= 8) {
11054       // IndexOf for constant substrings with size >= 8 elements
11055       // which don't need to be loaded through stack.
11056       __ string_indexofC8($str1$$Register, $str2$$Register,
11057                           $cnt1$$Register, $cnt2$$Register,
11058                           icnt2, $result$$Register,
11059                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11060     } else {
11061       // Small strings are loaded through stack if they cross page boundary.
11062       __ string_indexof($str1$$Register, $str2$$Register,
11063                         $cnt1$$Register, $cnt2$$Register,
11064                         icnt2, $result$$Register,
11065                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11066     }
11067   %}
11068   ins_pipe( pipe_slow );
11069 %}
11070 
11071 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11072                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11073 %{
11074   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11075   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11076   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11077 
11078   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11079   ins_encode %{
11080     __ string_indexof($str1$$Register, $str2$$Register,
11081                       $cnt1$$Register, $cnt2$$Register,
11082                       (-1), $result$$Register,
11083                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11084   %}
11085   ins_pipe( pipe_slow );
11086 %}
11087 
11088 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11089                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11090 %{
11091   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11092   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11093   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11094 
11095   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11096   ins_encode %{
11097     __ string_indexof($str1$$Register, $str2$$Register,
11098                       $cnt1$$Register, $cnt2$$Register,
11099                       (-1), $result$$Register,
11100                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11101   %}
11102   ins_pipe( pipe_slow );
11103 %}
11104 
11105 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11106                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11107 %{
11108   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11109   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11110   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11111 
11112   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11113   ins_encode %{
11114     __ string_indexof($str1$$Register, $str2$$Register,
11115                       $cnt1$$Register, $cnt2$$Register,
11116                       (-1), $result$$Register,
11117                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11118   %}
11119   ins_pipe( pipe_slow );
11120 %}
11121 
11122 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11123                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
11124 %{
11125   predicate(UseSSE42Intrinsics);
11126   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11127   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11128   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11129   ins_encode %{
11130     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11131                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11132   %}
11133   ins_pipe( pipe_slow );
11134 %}
11135 
11136 // fast string equals
11137 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11138                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11139 %{
11140   match(Set result (StrEquals (Binary str1 str2) cnt));
11141   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11142 
11143   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11144   ins_encode %{
11145     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11146                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11147                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11148   %}
11149   ins_pipe( pipe_slow );
11150 %}
11151 
11152 // fast array equals
11153 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11154                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11155 %{
11156   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11157   match(Set result (AryEq ary1 ary2));
11158   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11159 
11160   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11161   ins_encode %{
11162     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11163                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11164                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11165   %}
11166   ins_pipe( pipe_slow );
11167 %}
11168 
11169 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11170                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11171 %{
11172   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11173   match(Set result (AryEq ary1 ary2));
11174   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11175 
11176   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11177   ins_encode %{
11178     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11179                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11180                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11181   %}
11182   ins_pipe( pipe_slow );
11183 %}
11184 
11185 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11186                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11187 %{
11188   match(Set result (HasNegatives ary1 len));
11189   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11190 
11191   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11192   ins_encode %{
11193     __ has_negatives($ary1$$Register, $len$$Register,
11194                      $result$$Register, $tmp3$$Register,
11195                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11196   %}
11197   ins_pipe( pipe_slow );
11198 %}
11199 
11200 // fast char[] to byte[] compression
11201 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11202                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11203   match(Set result (StrCompressedCopy src (Binary dst len)));
11204   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11205 
11206   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11207   ins_encode %{
11208     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11209                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11210                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11211   %}
11212   ins_pipe( pipe_slow );
11213 %}
11214 
11215 // fast byte[] to char[] inflation
11216 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11217                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11218   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11219   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11220 
11221   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11222   ins_encode %{
11223     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11224                           $tmp1$$XMMRegister, $tmp2$$Register);
11225   %}
11226   ins_pipe( pipe_slow );
11227 %}
11228 
11229 // encode char[] to byte[] in ISO_8859_1
11230 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11231                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11232                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11233   match(Set result (EncodeISOArray src (Binary dst len)));
11234   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11235 
11236   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11237   ins_encode %{
11238     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11239                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11240                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11241   %}
11242   ins_pipe( pipe_slow );
11243 %}
11244 
11245 //----------Overflow Math Instructions-----------------------------------------
11246 
11247 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11248 %{
11249   match(Set cr (OverflowAddI op1 op2));
11250   effect(DEF cr, USE_KILL op1, USE op2);
11251 
11252   format %{ "addl    $op1, $op2\t# overflow check int" %}
11253 
11254   ins_encode %{
11255     __ addl($op1$$Register, $op2$$Register);
11256   %}
11257   ins_pipe(ialu_reg_reg);
11258 %}
11259 
11260 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11261 %{
11262   match(Set cr (OverflowAddI op1 op2));
11263   effect(DEF cr, USE_KILL op1, USE op2);
11264 
11265   format %{ "addl    $op1, $op2\t# overflow check int" %}
11266 
11267   ins_encode %{
11268     __ addl($op1$$Register, $op2$$constant);
11269   %}
11270   ins_pipe(ialu_reg_reg);
11271 %}
11272 
11273 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11274 %{
11275   match(Set cr (OverflowAddL op1 op2));
11276   effect(DEF cr, USE_KILL op1, USE op2);
11277 
11278   format %{ "addq    $op1, $op2\t# overflow check long" %}
11279   ins_encode %{
11280     __ addq($op1$$Register, $op2$$Register);
11281   %}
11282   ins_pipe(ialu_reg_reg);
11283 %}
11284 
11285 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11286 %{
11287   match(Set cr (OverflowAddL op1 op2));
11288   effect(DEF cr, USE_KILL op1, USE op2);
11289 
11290   format %{ "addq    $op1, $op2\t# overflow check long" %}
11291   ins_encode %{
11292     __ addq($op1$$Register, $op2$$constant);
11293   %}
11294   ins_pipe(ialu_reg_reg);
11295 %}
11296 
11297 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11298 %{
11299   match(Set cr (OverflowSubI op1 op2));
11300 
11301   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11302   ins_encode %{
11303     __ cmpl($op1$$Register, $op2$$Register);
11304   %}
11305   ins_pipe(ialu_reg_reg);
11306 %}
11307 
11308 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11309 %{
11310   match(Set cr (OverflowSubI op1 op2));
11311 
11312   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11313   ins_encode %{
11314     __ cmpl($op1$$Register, $op2$$constant);
11315   %}
11316   ins_pipe(ialu_reg_reg);
11317 %}
11318 
11319 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11320 %{
11321   match(Set cr (OverflowSubL op1 op2));
11322 
11323   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11324   ins_encode %{
11325     __ cmpq($op1$$Register, $op2$$Register);
11326   %}
11327   ins_pipe(ialu_reg_reg);
11328 %}
11329 
11330 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11331 %{
11332   match(Set cr (OverflowSubL op1 op2));
11333 
11334   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11335   ins_encode %{
11336     __ cmpq($op1$$Register, $op2$$constant);
11337   %}
11338   ins_pipe(ialu_reg_reg);
11339 %}
11340 
11341 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11342 %{
11343   match(Set cr (OverflowSubI zero op2));
11344   effect(DEF cr, USE_KILL op2);
11345 
11346   format %{ "negl    $op2\t# overflow check int" %}
11347   ins_encode %{
11348     __ negl($op2$$Register);
11349   %}
11350   ins_pipe(ialu_reg_reg);
11351 %}
11352 
11353 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11354 %{
11355   match(Set cr (OverflowSubL zero op2));
11356   effect(DEF cr, USE_KILL op2);
11357 
11358   format %{ "negq    $op2\t# overflow check long" %}
11359   ins_encode %{
11360     __ negq($op2$$Register);
11361   %}
11362   ins_pipe(ialu_reg_reg);
11363 %}
11364 
11365 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11366 %{
11367   match(Set cr (OverflowMulI op1 op2));
11368   effect(DEF cr, USE_KILL op1, USE op2);
11369 
11370   format %{ "imull    $op1, $op2\t# overflow check int" %}
11371   ins_encode %{
11372     __ imull($op1$$Register, $op2$$Register);
11373   %}
11374   ins_pipe(ialu_reg_reg_alu0);
11375 %}
11376 
11377 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11378 %{
11379   match(Set cr (OverflowMulI op1 op2));
11380   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11381 
11382   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11383   ins_encode %{
11384     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11385   %}
11386   ins_pipe(ialu_reg_reg_alu0);
11387 %}
11388 
11389 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11390 %{
11391   match(Set cr (OverflowMulL op1 op2));
11392   effect(DEF cr, USE_KILL op1, USE op2);
11393 
11394   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11395   ins_encode %{
11396     __ imulq($op1$$Register, $op2$$Register);
11397   %}
11398   ins_pipe(ialu_reg_reg_alu0);
11399 %}
11400 
11401 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11402 %{
11403   match(Set cr (OverflowMulL op1 op2));
11404   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11405 
11406   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11407   ins_encode %{
11408     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11409   %}
11410   ins_pipe(ialu_reg_reg_alu0);
11411 %}
11412 
11413 
11414 //----------Control Flow Instructions------------------------------------------
11415 // Signed compare Instructions
11416 
11417 // XXX more variants!!
11418 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11419 %{
11420   match(Set cr (CmpI op1 op2));
11421   effect(DEF cr, USE op1, USE op2);
11422 
11423   format %{ "cmpl    $op1, $op2" %}
11424   opcode(0x3B);  /* Opcode 3B /r */
11425   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11426   ins_pipe(ialu_cr_reg_reg);
11427 %}
11428 
11429 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11430 %{
11431   match(Set cr (CmpI op1 op2));
11432 
11433   format %{ "cmpl    $op1, $op2" %}
11434   opcode(0x81, 0x07); /* Opcode 81 /7 */
11435   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11436   ins_pipe(ialu_cr_reg_imm);
11437 %}
11438 
11439 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11440 %{
11441   match(Set cr (CmpI op1 (LoadI op2)));
11442 
11443   ins_cost(500); // XXX
11444   format %{ "cmpl    $op1, $op2" %}
11445   opcode(0x3B); /* Opcode 3B /r */
11446   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11447   ins_pipe(ialu_cr_reg_mem);
11448 %}
11449 
11450 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11451 %{
11452   match(Set cr (CmpI src zero));
11453 
11454   format %{ "testl   $src, $src" %}
11455   opcode(0x85);
11456   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11457   ins_pipe(ialu_cr_reg_imm);
11458 %}
11459 
11460 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11461 %{
11462   match(Set cr (CmpI (AndI src con) zero));
11463 
11464   format %{ "testl   $src, $con" %}
11465   opcode(0xF7, 0x00);
11466   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11467   ins_pipe(ialu_cr_reg_imm);
11468 %}
11469 
11470 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11471 %{
11472   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11473 
11474   format %{ "testl   $src, $mem" %}
11475   opcode(0x85);
11476   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11477   ins_pipe(ialu_cr_reg_mem);
11478 %}
11479 
11480 // Unsigned compare Instructions; really, same as signed except they
11481 // produce an rFlagsRegU instead of rFlagsReg.
11482 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11483 %{
11484   match(Set cr (CmpU op1 op2));
11485 
11486   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11487   opcode(0x3B); /* Opcode 3B /r */
11488   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11489   ins_pipe(ialu_cr_reg_reg);
11490 %}
11491 
11492 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11493 %{
11494   match(Set cr (CmpU op1 op2));
11495 
11496   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11497   opcode(0x81,0x07); /* Opcode 81 /7 */
11498   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11499   ins_pipe(ialu_cr_reg_imm);
11500 %}
11501 
11502 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11503 %{
11504   match(Set cr (CmpU op1 (LoadI op2)));
11505 
11506   ins_cost(500); // XXX
11507   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11508   opcode(0x3B); /* Opcode 3B /r */
11509   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11510   ins_pipe(ialu_cr_reg_mem);
11511 %}
11512 
11513 // // // Cisc-spilled version of cmpU_rReg
11514 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11515 // //%{
11516 // //  match(Set cr (CmpU (LoadI op1) op2));
11517 // //
11518 // //  format %{ "CMPu   $op1,$op2" %}
11519 // //  ins_cost(500);
11520 // //  opcode(0x39);  /* Opcode 39 /r */
11521 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11522 // //%}
11523 
11524 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11525 %{
11526   match(Set cr (CmpU src zero));
11527 
11528   format %{ "testl  $src, $src\t# unsigned" %}
11529   opcode(0x85);
11530   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11531   ins_pipe(ialu_cr_reg_imm);
11532 %}
11533 
11534 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11535 %{
11536   match(Set cr (CmpP op1 op2));
11537 
11538   format %{ "cmpq    $op1, $op2\t# ptr" %}
11539   opcode(0x3B); /* Opcode 3B /r */
11540   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11541   ins_pipe(ialu_cr_reg_reg);
11542 %}
11543 
11544 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11545 %{
11546   match(Set cr (CmpP op1 (LoadP op2)));
11547 
11548   ins_cost(500); // XXX
11549   format %{ "cmpq    $op1, $op2\t# ptr" %}
11550   opcode(0x3B); /* Opcode 3B /r */
11551   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11552   ins_pipe(ialu_cr_reg_mem);
11553 %}
11554 
11555 // // // Cisc-spilled version of cmpP_rReg
11556 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11557 // //%{
11558 // //  match(Set cr (CmpP (LoadP op1) op2));
11559 // //
11560 // //  format %{ "CMPu   $op1,$op2" %}
11561 // //  ins_cost(500);
11562 // //  opcode(0x39);  /* Opcode 39 /r */
11563 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11564 // //%}
11565 
11566 // XXX this is generalized by compP_rReg_mem???
11567 // Compare raw pointer (used in out-of-heap check).
11568 // Only works because non-oop pointers must be raw pointers
11569 // and raw pointers have no anti-dependencies.
11570 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11571 %{
11572   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11573   match(Set cr (CmpP op1 (LoadP op2)));
11574 
11575   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11576   opcode(0x3B); /* Opcode 3B /r */
11577   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11578   ins_pipe(ialu_cr_reg_mem);
11579 %}
11580 
11581 // This will generate a signed flags result. This should be OK since
11582 // any compare to a zero should be eq/neq.
11583 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11584 %{
11585   match(Set cr (CmpP src zero));
11586 
11587   format %{ "testq   $src, $src\t# ptr" %}
11588   opcode(0x85);
11589   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11590   ins_pipe(ialu_cr_reg_imm);
11591 %}
11592 
11593 // This will generate a signed flags result. This should be OK since
11594 // any compare to a zero should be eq/neq.
11595 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11596 %{
11597   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11598   match(Set cr (CmpP (LoadP op) zero));
11599 
11600   ins_cost(500); // XXX
11601   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11602   opcode(0xF7); /* Opcode F7 /0 */
11603   ins_encode(REX_mem_wide(op),
11604              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11605   ins_pipe(ialu_cr_reg_imm);
11606 %}
11607 
11608 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11609 %{
11610   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11611   match(Set cr (CmpP (LoadP mem) zero));
11612 
11613   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11614   ins_encode %{
11615     __ cmpq(r12, $mem$$Address);
11616   %}
11617   ins_pipe(ialu_cr_reg_mem);
11618 %}
11619 
11620 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11621 %{
11622   match(Set cr (CmpN op1 op2));
11623 
11624   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11625   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11626   ins_pipe(ialu_cr_reg_reg);
11627 %}
11628 
11629 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11630 %{
11631   match(Set cr (CmpN src (LoadN mem)));
11632 
11633   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11634   ins_encode %{
11635     __ cmpl($src$$Register, $mem$$Address);
11636   %}
11637   ins_pipe(ialu_cr_reg_mem);
11638 %}
11639 
11640 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11641   match(Set cr (CmpN op1 op2));
11642 
11643   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11644   ins_encode %{
11645     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11646   %}
11647   ins_pipe(ialu_cr_reg_imm);
11648 %}
11649 
11650 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11651 %{
11652   match(Set cr (CmpN src (LoadN mem)));
11653 
11654   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11655   ins_encode %{
11656     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11657   %}
11658   ins_pipe(ialu_cr_reg_mem);
11659 %}
11660 
11661 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11662   match(Set cr (CmpN op1 op2));
11663 
11664   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11665   ins_encode %{
11666     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11667   %}
11668   ins_pipe(ialu_cr_reg_imm);
11669 %}
11670 
11671 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11672 %{
11673   match(Set cr (CmpN src (LoadNKlass mem)));
11674 
11675   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11676   ins_encode %{
11677     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11678   %}
11679   ins_pipe(ialu_cr_reg_mem);
11680 %}
11681 
11682 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11683   match(Set cr (CmpN src zero));
11684 
11685   format %{ "testl   $src, $src\t# compressed ptr" %}
11686   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11687   ins_pipe(ialu_cr_reg_imm);
11688 %}
11689 
11690 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11691 %{
11692   predicate(Universe::narrow_oop_base() != NULL);
11693   match(Set cr (CmpN (LoadN mem) zero));
11694 
11695   ins_cost(500); // XXX
11696   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11697   ins_encode %{
11698     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11699   %}
11700   ins_pipe(ialu_cr_reg_mem);
11701 %}
11702 
11703 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11704 %{
11705   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11706   match(Set cr (CmpN (LoadN mem) zero));
11707 
11708   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11709   ins_encode %{
11710     __ cmpl(r12, $mem$$Address);
11711   %}
11712   ins_pipe(ialu_cr_reg_mem);
11713 %}
11714 
11715 // Yanked all unsigned pointer compare operations.
11716 // Pointer compares are done with CmpP which is already unsigned.
11717 
11718 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11719 %{
11720   match(Set cr (CmpL op1 op2));
11721 
11722   format %{ "cmpq    $op1, $op2" %}
11723   opcode(0x3B);  /* Opcode 3B /r */
11724   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11725   ins_pipe(ialu_cr_reg_reg);
11726 %}
11727 
11728 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11729 %{
11730   match(Set cr (CmpL op1 op2));
11731 
11732   format %{ "cmpq    $op1, $op2" %}
11733   opcode(0x81, 0x07); /* Opcode 81 /7 */
11734   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11735   ins_pipe(ialu_cr_reg_imm);
11736 %}
11737 
11738 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11739 %{
11740   match(Set cr (CmpL op1 (LoadL op2)));
11741 
11742   format %{ "cmpq    $op1, $op2" %}
11743   opcode(0x3B); /* Opcode 3B /r */
11744   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11745   ins_pipe(ialu_cr_reg_mem);
11746 %}
11747 
11748 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11749 %{
11750   match(Set cr (CmpL src zero));
11751 
11752   format %{ "testq   $src, $src" %}
11753   opcode(0x85);
11754   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11755   ins_pipe(ialu_cr_reg_imm);
11756 %}
11757 
11758 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11759 %{
11760   match(Set cr (CmpL (AndL src con) zero));
11761 
11762   format %{ "testq   $src, $con\t# long" %}
11763   opcode(0xF7, 0x00);
11764   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11765   ins_pipe(ialu_cr_reg_imm);
11766 %}
11767 
11768 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11769 %{
11770   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11771 
11772   format %{ "testq   $src, $mem" %}
11773   opcode(0x85);
11774   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11775   ins_pipe(ialu_cr_reg_mem);
11776 %}
11777 
11778 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11779 %{
11780   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11781 
11782   format %{ "testq   $src, $mem" %}
11783   opcode(0x85);
11784   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11785   ins_pipe(ialu_cr_reg_mem);
11786 %}
11787 
11788 // Manifest a CmpL result in an integer register.  Very painful.
11789 // This is the test to avoid.
11790 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11791 %{
11792   match(Set dst (CmpL3 src1 src2));
11793   effect(KILL flags);
11794 
11795   ins_cost(275); // XXX
11796   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11797             "movl    $dst, -1\n\t"
11798             "jl,s    done\n\t"
11799             "setne   $dst\n\t"
11800             "movzbl  $dst, $dst\n\t"
11801     "done:" %}
11802   ins_encode(cmpl3_flag(src1, src2, dst));
11803   ins_pipe(pipe_slow);
11804 %}
11805 
11806 // Unsigned long compare Instructions; really, same as signed long except they
11807 // produce an rFlagsRegU instead of rFlagsReg.
11808 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11809 %{
11810   match(Set cr (CmpUL op1 op2));
11811 
11812   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11813   opcode(0x3B);  /* Opcode 3B /r */
11814   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11815   ins_pipe(ialu_cr_reg_reg);
11816 %}
11817 
11818 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11819 %{
11820   match(Set cr (CmpUL op1 op2));
11821 
11822   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11823   opcode(0x81, 0x07); /* Opcode 81 /7 */
11824   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11825   ins_pipe(ialu_cr_reg_imm);
11826 %}
11827 
11828 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11829 %{
11830   match(Set cr (CmpUL op1 (LoadL op2)));
11831 
11832   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11833   opcode(0x3B); /* Opcode 3B /r */
11834   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11835   ins_pipe(ialu_cr_reg_mem);
11836 %}
11837 
11838 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11839 %{
11840   match(Set cr (CmpUL src zero));
11841 
11842   format %{ "testq   $src, $src\t# unsigned" %}
11843   opcode(0x85);
11844   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11845   ins_pipe(ialu_cr_reg_imm);
11846 %}
11847 
11848 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11849 %{
11850   match(Set cr (CmpI (LoadB mem) imm));
11851 
11852   ins_cost(125);
11853   format %{ "cmpb    $mem, $imm" %}
11854   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11855   ins_pipe(ialu_cr_reg_mem);
11856 %}
11857 
11858 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
11859 %{
11860   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11861 
11862   ins_cost(125);
11863   format %{ "testb   $mem, $imm" %}
11864   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11865   ins_pipe(ialu_cr_reg_mem);
11866 %}
11867 
11868 //----------Max and Min--------------------------------------------------------
11869 // Min Instructions
11870 
11871 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11872 %{
11873   effect(USE_DEF dst, USE src, USE cr);
11874 
11875   format %{ "cmovlgt $dst, $src\t# min" %}
11876   opcode(0x0F, 0x4F);
11877   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11878   ins_pipe(pipe_cmov_reg);
11879 %}
11880 
11881 
11882 instruct minI_rReg(rRegI dst, rRegI src)
11883 %{
11884   match(Set dst (MinI dst src));
11885 
11886   ins_cost(200);
11887   expand %{
11888     rFlagsReg cr;
11889     compI_rReg(cr, dst, src);
11890     cmovI_reg_g(dst, src, cr);
11891   %}
11892 %}
11893 
11894 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11895 %{
11896   effect(USE_DEF dst, USE src, USE cr);
11897 
11898   format %{ "cmovllt $dst, $src\t# max" %}
11899   opcode(0x0F, 0x4C);
11900   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11901   ins_pipe(pipe_cmov_reg);
11902 %}
11903 
11904 
11905 instruct maxI_rReg(rRegI dst, rRegI src)
11906 %{
11907   match(Set dst (MaxI dst src));
11908 
11909   ins_cost(200);
11910   expand %{
11911     rFlagsReg cr;
11912     compI_rReg(cr, dst, src);
11913     cmovI_reg_l(dst, src, cr);
11914   %}
11915 %}
11916 
11917 // ============================================================================
11918 // Branch Instructions
11919 
11920 // Jump Direct - Label defines a relative address from JMP+1
11921 instruct jmpDir(label labl)
11922 %{
11923   match(Goto);
11924   effect(USE labl);
11925 
11926   ins_cost(300);
11927   format %{ "jmp     $labl" %}
11928   size(5);
11929   ins_encode %{
11930     Label* L = $labl$$label;
11931     __ jmp(*L, false); // Always long jump
11932   %}
11933   ins_pipe(pipe_jmp);
11934 %}
11935 
11936 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11937 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11938 %{
11939   match(If cop cr);
11940   effect(USE labl);
11941 
11942   ins_cost(300);
11943   format %{ "j$cop     $labl" %}
11944   size(6);
11945   ins_encode %{
11946     Label* L = $labl$$label;
11947     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11948   %}
11949   ins_pipe(pipe_jcc);
11950 %}
11951 
11952 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11953 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11954 %{
11955   predicate(!n->has_vector_mask_set());
11956   match(CountedLoopEnd cop cr);
11957   effect(USE labl);
11958 
11959   ins_cost(300);
11960   format %{ "j$cop     $labl\t# loop end" %}
11961   size(6);
11962   ins_encode %{
11963     Label* L = $labl$$label;
11964     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11965   %}
11966   ins_pipe(pipe_jcc);
11967 %}
11968 
11969 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11970 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11971   predicate(!n->has_vector_mask_set());
11972   match(CountedLoopEnd cop cmp);
11973   effect(USE labl);
11974 
11975   ins_cost(300);
11976   format %{ "j$cop,u   $labl\t# loop end" %}
11977   size(6);
11978   ins_encode %{
11979     Label* L = $labl$$label;
11980     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11981   %}
11982   ins_pipe(pipe_jcc);
11983 %}
11984 
11985 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11986   predicate(!n->has_vector_mask_set());
11987   match(CountedLoopEnd cop cmp);
11988   effect(USE labl);
11989 
11990   ins_cost(200);
11991   format %{ "j$cop,u   $labl\t# loop end" %}
11992   size(6);
11993   ins_encode %{
11994     Label* L = $labl$$label;
11995     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11996   %}
11997   ins_pipe(pipe_jcc);
11998 %}
11999 
12000 // mask version
12001 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12002 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12003 %{
12004   predicate(n->has_vector_mask_set());
12005   match(CountedLoopEnd cop cr);
12006   effect(USE labl);
12007 
12008   ins_cost(400);
12009   format %{ "j$cop     $labl\t# loop end\n\t"
12010             "restorevectmask \t# vector mask restore for loops" %}
12011   size(10);
12012   ins_encode %{
12013     Label* L = $labl$$label;
12014     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12015     __ restorevectmask();
12016   %}
12017   ins_pipe(pipe_jcc);
12018 %}
12019 
12020 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12021 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12022   predicate(n->has_vector_mask_set());
12023   match(CountedLoopEnd cop cmp);
12024   effect(USE labl);
12025 
12026   ins_cost(400);
12027   format %{ "j$cop,u   $labl\t# loop end\n\t"
12028             "restorevectmask \t# vector mask restore for loops" %}
12029   size(10);
12030   ins_encode %{
12031     Label* L = $labl$$label;
12032     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12033     __ restorevectmask();
12034   %}
12035   ins_pipe(pipe_jcc);
12036 %}
12037 
12038 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12039   predicate(n->has_vector_mask_set());
12040   match(CountedLoopEnd cop cmp);
12041   effect(USE labl);
12042 
12043   ins_cost(300);
12044   format %{ "j$cop,u   $labl\t# loop end\n\t"
12045             "restorevectmask \t# vector mask restore for loops" %}
12046   size(10);
12047   ins_encode %{
12048     Label* L = $labl$$label;
12049     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12050     __ restorevectmask();
12051   %}
12052   ins_pipe(pipe_jcc);
12053 %}
12054 
12055 // Jump Direct Conditional - using unsigned comparison
12056 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12057   match(If cop cmp);
12058   effect(USE labl);
12059 
12060   ins_cost(300);
12061   format %{ "j$cop,u  $labl" %}
12062   size(6);
12063   ins_encode %{
12064     Label* L = $labl$$label;
12065     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12066   %}
12067   ins_pipe(pipe_jcc);
12068 %}
12069 
12070 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12071   match(If cop cmp);
12072   effect(USE labl);
12073 
12074   ins_cost(200);
12075   format %{ "j$cop,u  $labl" %}
12076   size(6);
12077   ins_encode %{
12078     Label* L = $labl$$label;
12079     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12080   %}
12081   ins_pipe(pipe_jcc);
12082 %}
12083 
12084 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12085   match(If cop cmp);
12086   effect(USE labl);
12087 
12088   ins_cost(200);
12089   format %{ $$template
12090     if ($cop$$cmpcode == Assembler::notEqual) {
12091       $$emit$$"jp,u   $labl\n\t"
12092       $$emit$$"j$cop,u   $labl"
12093     } else {
12094       $$emit$$"jp,u   done\n\t"
12095       $$emit$$"j$cop,u   $labl\n\t"
12096       $$emit$$"done:"
12097     }
12098   %}
12099   ins_encode %{
12100     Label* l = $labl$$label;
12101     if ($cop$$cmpcode == Assembler::notEqual) {
12102       __ jcc(Assembler::parity, *l, false);
12103       __ jcc(Assembler::notEqual, *l, false);
12104     } else if ($cop$$cmpcode == Assembler::equal) {
12105       Label done;
12106       __ jccb(Assembler::parity, done);
12107       __ jcc(Assembler::equal, *l, false);
12108       __ bind(done);
12109     } else {
12110        ShouldNotReachHere();
12111     }
12112   %}
12113   ins_pipe(pipe_jcc);
12114 %}
12115 
12116 // ============================================================================
12117 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12118 // superklass array for an instance of the superklass.  Set a hidden
12119 // internal cache on a hit (cache is checked with exposed code in
12120 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12121 // encoding ALSO sets flags.
12122 
12123 instruct partialSubtypeCheck(rdi_RegP result,
12124                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12125                              rFlagsReg cr)
12126 %{
12127   match(Set result (PartialSubtypeCheck sub super));
12128   effect(KILL rcx, KILL cr);
12129 
12130   ins_cost(1100);  // slightly larger than the next version
12131   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12132             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12133             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12134             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12135             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12136             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12137             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12138     "miss:\t" %}
12139 
12140   opcode(0x1); // Force a XOR of RDI
12141   ins_encode(enc_PartialSubtypeCheck());
12142   ins_pipe(pipe_slow);
12143 %}
12144 
12145 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12146                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12147                                      immP0 zero,
12148                                      rdi_RegP result)
12149 %{
12150   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12151   effect(KILL rcx, KILL result);
12152 
12153   ins_cost(1000);
12154   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12155             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12156             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12157             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12158             "jne,s   miss\t\t# Missed: flags nz\n\t"
12159             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12160     "miss:\t" %}
12161 
12162   opcode(0x0); // No need to XOR RDI
12163   ins_encode(enc_PartialSubtypeCheck());
12164   ins_pipe(pipe_slow);
12165 %}
12166 
12167 // ============================================================================
12168 // Branch Instructions -- short offset versions
12169 //
12170 // These instructions are used to replace jumps of a long offset (the default
12171 // match) with jumps of a shorter offset.  These instructions are all tagged
12172 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12173 // match rules in general matching.  Instead, the ADLC generates a conversion
12174 // method in the MachNode which can be used to do in-place replacement of the
12175 // long variant with the shorter variant.  The compiler will determine if a
12176 // branch can be taken by the is_short_branch_offset() predicate in the machine
12177 // specific code section of the file.
12178 
12179 // Jump Direct - Label defines a relative address from JMP+1
12180 instruct jmpDir_short(label labl) %{
12181   match(Goto);
12182   effect(USE labl);
12183 
12184   ins_cost(300);
12185   format %{ "jmp,s   $labl" %}
12186   size(2);
12187   ins_encode %{
12188     Label* L = $labl$$label;
12189     __ jmpb(*L);
12190   %}
12191   ins_pipe(pipe_jmp);
12192   ins_short_branch(1);
12193 %}
12194 
12195 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12196 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12197   match(If cop cr);
12198   effect(USE labl);
12199 
12200   ins_cost(300);
12201   format %{ "j$cop,s   $labl" %}
12202   size(2);
12203   ins_encode %{
12204     Label* L = $labl$$label;
12205     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12206   %}
12207   ins_pipe(pipe_jcc);
12208   ins_short_branch(1);
12209 %}
12210 
12211 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12212 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12213   match(CountedLoopEnd cop cr);
12214   effect(USE labl);
12215 
12216   ins_cost(300);
12217   format %{ "j$cop,s   $labl\t# loop end" %}
12218   size(2);
12219   ins_encode %{
12220     Label* L = $labl$$label;
12221     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12222   %}
12223   ins_pipe(pipe_jcc);
12224   ins_short_branch(1);
12225 %}
12226 
12227 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12228 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12229   match(CountedLoopEnd cop cmp);
12230   effect(USE labl);
12231 
12232   ins_cost(300);
12233   format %{ "j$cop,us  $labl\t# loop end" %}
12234   size(2);
12235   ins_encode %{
12236     Label* L = $labl$$label;
12237     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12238   %}
12239   ins_pipe(pipe_jcc);
12240   ins_short_branch(1);
12241 %}
12242 
12243 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12244   match(CountedLoopEnd cop cmp);
12245   effect(USE labl);
12246 
12247   ins_cost(300);
12248   format %{ "j$cop,us  $labl\t# loop end" %}
12249   size(2);
12250   ins_encode %{
12251     Label* L = $labl$$label;
12252     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12253   %}
12254   ins_pipe(pipe_jcc);
12255   ins_short_branch(1);
12256 %}
12257 
12258 // Jump Direct Conditional - using unsigned comparison
12259 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12260   match(If cop cmp);
12261   effect(USE labl);
12262 
12263   ins_cost(300);
12264   format %{ "j$cop,us  $labl" %}
12265   size(2);
12266   ins_encode %{
12267     Label* L = $labl$$label;
12268     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12269   %}
12270   ins_pipe(pipe_jcc);
12271   ins_short_branch(1);
12272 %}
12273 
12274 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12275   match(If cop cmp);
12276   effect(USE labl);
12277 
12278   ins_cost(300);
12279   format %{ "j$cop,us  $labl" %}
12280   size(2);
12281   ins_encode %{
12282     Label* L = $labl$$label;
12283     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12284   %}
12285   ins_pipe(pipe_jcc);
12286   ins_short_branch(1);
12287 %}
12288 
12289 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12290   match(If cop cmp);
12291   effect(USE labl);
12292 
12293   ins_cost(300);
12294   format %{ $$template
12295     if ($cop$$cmpcode == Assembler::notEqual) {
12296       $$emit$$"jp,u,s   $labl\n\t"
12297       $$emit$$"j$cop,u,s   $labl"
12298     } else {
12299       $$emit$$"jp,u,s   done\n\t"
12300       $$emit$$"j$cop,u,s  $labl\n\t"
12301       $$emit$$"done:"
12302     }
12303   %}
12304   size(4);
12305   ins_encode %{
12306     Label* l = $labl$$label;
12307     if ($cop$$cmpcode == Assembler::notEqual) {
12308       __ jccb(Assembler::parity, *l);
12309       __ jccb(Assembler::notEqual, *l);
12310     } else if ($cop$$cmpcode == Assembler::equal) {
12311       Label done;
12312       __ jccb(Assembler::parity, done);
12313       __ jccb(Assembler::equal, *l);
12314       __ bind(done);
12315     } else {
12316        ShouldNotReachHere();
12317     }
12318   %}
12319   ins_pipe(pipe_jcc);
12320   ins_short_branch(1);
12321 %}
12322 
12323 // ============================================================================
12324 // inlined locking and unlocking
12325 
12326 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12327   predicate(Compile::current()->use_rtm());
12328   match(Set cr (FastLock object box));
12329   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12330   ins_cost(300);
12331   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12332   ins_encode %{
12333     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12334                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12335                  _counters, _rtm_counters, _stack_rtm_counters,
12336                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12337                  true, ra_->C->profile_rtm());
12338   %}
12339   ins_pipe(pipe_slow);
12340 %}
12341 
12342 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12343   predicate(!Compile::current()->use_rtm());
12344   match(Set cr (FastLock object box));
12345   effect(TEMP tmp, TEMP scr, USE_KILL box);
12346   ins_cost(300);
12347   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12348   ins_encode %{
12349     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12350                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12351   %}
12352   ins_pipe(pipe_slow);
12353 %}
12354 
12355 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12356   match(Set cr (FastUnlock object box));
12357   effect(TEMP tmp, USE_KILL box);
12358   ins_cost(300);
12359   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12360   ins_encode %{
12361     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12362   %}
12363   ins_pipe(pipe_slow);
12364 %}
12365 
12366 
12367 // ============================================================================
12368 // Safepoint Instructions
12369 instruct safePoint_poll(rFlagsReg cr)
12370 %{
12371   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12372   match(SafePoint);
12373   effect(KILL cr);
12374 
12375   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12376             "# Safepoint: poll for GC" %}
12377   ins_cost(125);
12378   ins_encode %{
12379     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12380     __ testl(rax, addr);
12381   %}
12382   ins_pipe(ialu_reg_mem);
12383 %}
12384 
12385 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12386 %{
12387   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12388   match(SafePoint poll);
12389   effect(KILL cr, USE poll);
12390 
12391   format %{ "testl  rax, [$poll]\t"
12392             "# Safepoint: poll for GC" %}
12393   ins_cost(125);
12394   ins_encode %{
12395     __ relocate(relocInfo::poll_type);
12396     __ testl(rax, Address($poll$$Register, 0));
12397   %}
12398   ins_pipe(ialu_reg_mem);
12399 %}
12400 
12401 instruct safePoint_poll_tls(rFlagsReg cr, rex_RegP poll)
12402 %{
12403   predicate(SafepointMechanism::uses_thread_local_poll());
12404   match(SafePoint poll);
12405   effect(KILL cr, USE poll);
12406 
12407   format %{ "testl  rax, [$poll]\t"
12408             "# Safepoint: poll for GC" %}
12409   ins_cost(125);
12410   size(3); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12411   ins_encode %{
12412     __ relocate(relocInfo::poll_type);
12413     address pre_pc = __ pc();
12414     __ testl(rax, Address($poll$$Register, 0));
12415     address post_pc = __ pc();
12416     guarantee(pre_pc[0] == 0x41 && pre_pc[1] == 0x85, "must emit #rex test-ax [reg]");
12417   %}
12418   ins_pipe(ialu_reg_mem);
12419 %}
12420 
12421 // ============================================================================
12422 // Procedure Call/Return Instructions
12423 // Call Java Static Instruction
12424 // Note: If this code changes, the corresponding ret_addr_offset() and
12425 //       compute_padding() functions will have to be adjusted.
12426 instruct CallStaticJavaDirect(method meth) %{
12427   match(CallStaticJava);
12428   effect(USE meth);
12429 
12430   ins_cost(300);
12431   format %{ "call,static " %}
12432   opcode(0xE8); /* E8 cd */
12433   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12434   ins_pipe(pipe_slow);
12435   ins_alignment(4);
12436 %}
12437 
12438 // Call Java Dynamic Instruction
12439 // Note: If this code changes, the corresponding ret_addr_offset() and
12440 //       compute_padding() functions will have to be adjusted.
12441 instruct CallDynamicJavaDirect(method meth)
12442 %{
12443   match(CallDynamicJava);
12444   effect(USE meth);
12445 
12446   ins_cost(300);
12447   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12448             "call,dynamic " %}
12449   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12450   ins_pipe(pipe_slow);
12451   ins_alignment(4);
12452 %}
12453 
12454 // Call Runtime Instruction
12455 instruct CallRuntimeDirect(method meth)
12456 %{
12457   match(CallRuntime);
12458   effect(USE meth);
12459 
12460   ins_cost(300);
12461   format %{ "call,runtime " %}
12462   ins_encode(clear_avx, Java_To_Runtime(meth));
12463   ins_pipe(pipe_slow);
12464 %}
12465 
12466 // Call runtime without safepoint
12467 instruct CallLeafDirect(method meth)
12468 %{
12469   match(CallLeaf);
12470   effect(USE meth);
12471 
12472   ins_cost(300);
12473   format %{ "call_leaf,runtime " %}
12474   ins_encode(clear_avx, Java_To_Runtime(meth));
12475   ins_pipe(pipe_slow);
12476 %}
12477 
12478 // Call runtime without safepoint
12479 instruct CallLeafNoFPDirect(method meth)
12480 %{
12481   match(CallLeafNoFP);
12482   effect(USE meth);
12483 
12484   ins_cost(300);
12485   format %{ "call_leaf_nofp,runtime " %}
12486   ins_encode(clear_avx, Java_To_Runtime(meth));
12487   ins_pipe(pipe_slow);
12488 %}
12489 
12490 // Return Instruction
12491 // Remove the return address & jump to it.
12492 // Notice: We always emit a nop after a ret to make sure there is room
12493 // for safepoint patching
12494 instruct Ret()
12495 %{
12496   match(Return);
12497 
12498   format %{ "ret" %}
12499   opcode(0xC3);
12500   ins_encode(OpcP);
12501   ins_pipe(pipe_jmp);
12502 %}
12503 
12504 // Tail Call; Jump from runtime stub to Java code.
12505 // Also known as an 'interprocedural jump'.
12506 // Target of jump will eventually return to caller.
12507 // TailJump below removes the return address.
12508 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12509 %{
12510   match(TailCall jump_target method_oop);
12511 
12512   ins_cost(300);
12513   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12514   opcode(0xFF, 0x4); /* Opcode FF /4 */
12515   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12516   ins_pipe(pipe_jmp);
12517 %}
12518 
12519 // Tail Jump; remove the return address; jump to target.
12520 // TailCall above leaves the return address around.
12521 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12522 %{
12523   match(TailJump jump_target ex_oop);
12524 
12525   ins_cost(300);
12526   format %{ "popq    rdx\t# pop return address\n\t"
12527             "jmp     $jump_target" %}
12528   opcode(0xFF, 0x4); /* Opcode FF /4 */
12529   ins_encode(Opcode(0x5a), // popq rdx
12530              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12531   ins_pipe(pipe_jmp);
12532 %}
12533 
12534 // Create exception oop: created by stack-crawling runtime code.
12535 // Created exception is now available to this handler, and is setup
12536 // just prior to jumping to this handler.  No code emitted.
12537 instruct CreateException(rax_RegP ex_oop)
12538 %{
12539   match(Set ex_oop (CreateEx));
12540 
12541   size(0);
12542   // use the following format syntax
12543   format %{ "# exception oop is in rax; no code emitted" %}
12544   ins_encode();
12545   ins_pipe(empty);
12546 %}
12547 
12548 // Rethrow exception:
12549 // The exception oop will come in the first argument position.
12550 // Then JUMP (not call) to the rethrow stub code.
12551 instruct RethrowException()
12552 %{
12553   match(Rethrow);
12554 
12555   // use the following format syntax
12556   format %{ "jmp     rethrow_stub" %}
12557   ins_encode(enc_rethrow);
12558   ins_pipe(pipe_jmp);
12559 %}
12560 
12561 //
12562 // Execute ZGC load barrier (strong) slow path
12563 //
12564 
12565 // When running without XMM regs
12566 instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12567 
12568   match(Set dst (LoadBarrierSlowReg mem));
12569   predicate(MaxVectorSize < 16);
12570 
12571   effect(DEF dst, KILL cr);
12572 
12573   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12574   ins_encode %{
12575 #if INCLUDE_ZGC
12576     Register d = $dst$$Register;
12577     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12578 
12579     assert(d != r12, "Can't be R12!");
12580     assert(d != r15, "Can't be R15!");
12581     assert(d != rsp, "Can't be RSP!");
12582 
12583     __ lea(d, $mem$$Address);
12584     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12585 #else
12586     ShouldNotReachHere();
12587 #endif
12588   %}
12589   ins_pipe(pipe_slow);
12590 %}
12591 
12592 // For XMM and YMM enabled processors
12593 instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12594                                      rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12595                                      rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12596                                      rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12597                                      rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12598 
12599   match(Set dst (LoadBarrierSlowReg mem));
12600   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12601 
12602   effect(DEF dst, KILL cr,
12603          KILL x0, KILL x1, KILL x2, KILL x3,
12604          KILL x4, KILL x5, KILL x6, KILL x7,
12605          KILL x8, KILL x9, KILL x10, KILL x11,
12606          KILL x12, KILL x13, KILL x14, KILL x15);
12607 
12608   format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
12609   ins_encode %{
12610 #if INCLUDE_ZGC
12611     Register d = $dst$$Register;
12612     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12613 
12614     assert(d != r12, "Can't be R12!");
12615     assert(d != r15, "Can't be R15!");
12616     assert(d != rsp, "Can't be RSP!");
12617 
12618     __ lea(d, $mem$$Address);
12619     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12620 #else
12621     ShouldNotReachHere();
12622 #endif
12623   %}
12624   ins_pipe(pipe_slow);
12625 %}
12626 
12627 // For ZMM enabled processors
12628 instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12629                                rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12630                                rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12631                                rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12632                                rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12633                                rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12634                                rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12635                                rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12636                                rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12637 
12638   match(Set dst (LoadBarrierSlowReg mem));
12639   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12640 
12641   effect(DEF dst, KILL cr,
12642          KILL x0, KILL x1, KILL x2, KILL x3,
12643          KILL x4, KILL x5, KILL x6, KILL x7,
12644          KILL x8, KILL x9, KILL x10, KILL x11,
12645          KILL x12, KILL x13, KILL x14, KILL x15,
12646          KILL x16, KILL x17, KILL x18, KILL x19,
12647          KILL x20, KILL x21, KILL x22, KILL x23,
12648          KILL x24, KILL x25, KILL x26, KILL x27,
12649          KILL x28, KILL x29, KILL x30, KILL x31);
12650 
12651   format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
12652   ins_encode %{
12653 #if INCLUDE_ZGC
12654     Register d = $dst$$Register;
12655     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12656 
12657     assert(d != r12, "Can't be R12!");
12658     assert(d != r15, "Can't be R15!");
12659     assert(d != rsp, "Can't be RSP!");
12660 
12661     __ lea(d, $mem$$Address);
12662     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12663 #else
12664     ShouldNotReachHere();
12665 #endif
12666   %}
12667   ins_pipe(pipe_slow);
12668 %}
12669 
12670 //
12671 // Execute ZGC load barrier (weak) slow path
12672 //
12673 
12674 // When running without XMM regs
12675 instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12676 
12677   match(Set dst (LoadBarrierSlowReg mem));
12678   predicate(MaxVectorSize < 16);
12679 
12680   effect(DEF dst, KILL cr);
12681 
12682   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12683   ins_encode %{
12684 #if INCLUDE_ZGC
12685     Register d = $dst$$Register;
12686     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12687 
12688     assert(d != r12, "Can't be R12!");
12689     assert(d != r15, "Can't be R15!");
12690     assert(d != rsp, "Can't be RSP!");
12691 
12692     __ lea(d, $mem$$Address);
12693     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12694 #else
12695     ShouldNotReachHere();
12696 #endif
12697   %}
12698   ins_pipe(pipe_slow);
12699 %}
12700 
12701 // For XMM and YMM enabled processors
12702 instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12703                                          rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12704                                          rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12705                                          rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12706                                          rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12707 
12708   match(Set dst (LoadBarrierWeakSlowReg mem));
12709   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12710 
12711   effect(DEF dst, KILL cr,
12712          KILL x0, KILL x1, KILL x2, KILL x3,
12713          KILL x4, KILL x5, KILL x6, KILL x7,
12714          KILL x8, KILL x9, KILL x10, KILL x11,
12715          KILL x12, KILL x13, KILL x14, KILL x15);
12716 
12717   format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
12718   ins_encode %{
12719 #if INCLUDE_ZGC
12720     Register d = $dst$$Register;
12721     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12722 
12723     assert(d != r12, "Can't be R12!");
12724     assert(d != r15, "Can't be R15!");
12725     assert(d != rsp, "Can't be RSP!");
12726 
12727     __ lea(d,$mem$$Address);
12728     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12729 #else
12730     ShouldNotReachHere();
12731 #endif
12732   %}
12733   ins_pipe(pipe_slow);
12734 %}
12735 
12736 // For ZMM enabled processors
12737 instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12738                                    rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12739                                    rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12740                                    rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12741                                    rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12742                                    rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12743                                    rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12744                                    rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12745                                    rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12746 
12747   match(Set dst (LoadBarrierWeakSlowReg mem));
12748   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12749 
12750   effect(DEF dst, KILL cr,
12751          KILL x0, KILL x1, KILL x2, KILL x3,
12752          KILL x4, KILL x5, KILL x6, KILL x7,
12753          KILL x8, KILL x9, KILL x10, KILL x11,
12754          KILL x12, KILL x13, KILL x14, KILL x15,
12755          KILL x16, KILL x17, KILL x18, KILL x19,
12756          KILL x20, KILL x21, KILL x22, KILL x23,
12757          KILL x24, KILL x25, KILL x26, KILL x27,
12758          KILL x28, KILL x29, KILL x30, KILL x31);
12759 
12760   format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
12761   ins_encode %{
12762 #if INCLUDE_ZGC
12763     Register d = $dst$$Register;
12764     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12765 
12766     assert(d != r12, "Can't be R12!");
12767     assert(d != r15, "Can't be R15!");
12768     assert(d != rsp, "Can't be RSP!");
12769 
12770     __ lea(d,$mem$$Address);
12771     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12772 #else
12773     ShouldNotReachHere();
12774 #endif
12775   %}
12776   ins_pipe(pipe_slow);
12777 %}
12778 
12779 // ============================================================================
12780 // This name is KNOWN by the ADLC and cannot be changed.
12781 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12782 // for this guy.
12783 instruct tlsLoadP(r15_RegP dst) %{
12784   match(Set dst (ThreadLocal));
12785   effect(DEF dst);
12786 
12787   size(0);
12788   format %{ "# TLS is in R15" %}
12789   ins_encode( /*empty encoding*/ );
12790   ins_pipe(ialu_reg_reg);
12791 %}
12792 
12793 
12794 //----------PEEPHOLE RULES-----------------------------------------------------
12795 // These must follow all instruction definitions as they use the names
12796 // defined in the instructions definitions.
12797 //
12798 // peepmatch ( root_instr_name [preceding_instruction]* );
12799 //
12800 // peepconstraint %{
12801 // (instruction_number.operand_name relational_op instruction_number.operand_name
12802 //  [, ...] );
12803 // // instruction numbers are zero-based using left to right order in peepmatch
12804 //
12805 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12806 // // provide an instruction_number.operand_name for each operand that appears
12807 // // in the replacement instruction's match rule
12808 //
12809 // ---------VM FLAGS---------------------------------------------------------
12810 //
12811 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12812 //
12813 // Each peephole rule is given an identifying number starting with zero and
12814 // increasing by one in the order seen by the parser.  An individual peephole
12815 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12816 // on the command-line.
12817 //
12818 // ---------CURRENT LIMITATIONS----------------------------------------------
12819 //
12820 // Only match adjacent instructions in same basic block
12821 // Only equality constraints
12822 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12823 // Only one replacement instruction
12824 //
12825 // ---------EXAMPLE----------------------------------------------------------
12826 //
12827 // // pertinent parts of existing instructions in architecture description
12828 // instruct movI(rRegI dst, rRegI src)
12829 // %{
12830 //   match(Set dst (CopyI src));
12831 // %}
12832 //
12833 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12834 // %{
12835 //   match(Set dst (AddI dst src));
12836 //   effect(KILL cr);
12837 // %}
12838 //
12839 // // Change (inc mov) to lea
12840 // peephole %{
12841 //   // increment preceeded by register-register move
12842 //   peepmatch ( incI_rReg movI );
12843 //   // require that the destination register of the increment
12844 //   // match the destination register of the move
12845 //   peepconstraint ( 0.dst == 1.dst );
12846 //   // construct a replacement instruction that sets
12847 //   // the destination to ( move's source register + one )
12848 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12849 // %}
12850 //
12851 
12852 // Implementation no longer uses movX instructions since
12853 // machine-independent system no longer uses CopyX nodes.
12854 //
12855 // peephole
12856 // %{
12857 //   peepmatch (incI_rReg movI);
12858 //   peepconstraint (0.dst == 1.dst);
12859 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12860 // %}
12861 
12862 // peephole
12863 // %{
12864 //   peepmatch (decI_rReg movI);
12865 //   peepconstraint (0.dst == 1.dst);
12866 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12867 // %}
12868 
12869 // peephole
12870 // %{
12871 //   peepmatch (addI_rReg_imm movI);
12872 //   peepconstraint (0.dst == 1.dst);
12873 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12874 // %}
12875 
12876 // peephole
12877 // %{
12878 //   peepmatch (incL_rReg movL);
12879 //   peepconstraint (0.dst == 1.dst);
12880 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12881 // %}
12882 
12883 // peephole
12884 // %{
12885 //   peepmatch (decL_rReg movL);
12886 //   peepconstraint (0.dst == 1.dst);
12887 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12888 // %}
12889 
12890 // peephole
12891 // %{
12892 //   peepmatch (addL_rReg_imm movL);
12893 //   peepconstraint (0.dst == 1.dst);
12894 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12895 // %}
12896 
12897 // peephole
12898 // %{
12899 //   peepmatch (addP_rReg_imm movP);
12900 //   peepconstraint (0.dst == 1.dst);
12901 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12902 // %}
12903 
12904 // // Change load of spilled value to only a spill
12905 // instruct storeI(memory mem, rRegI src)
12906 // %{
12907 //   match(Set mem (StoreI mem src));
12908 // %}
12909 //
12910 // instruct loadI(rRegI dst, memory mem)
12911 // %{
12912 //   match(Set dst (LoadI mem));
12913 // %}
12914 //
12915 
12916 peephole
12917 %{
12918   peepmatch (loadI storeI);
12919   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12920   peepreplace (storeI(1.mem 1.mem 1.src));
12921 %}
12922 
12923 peephole
12924 %{
12925   peepmatch (loadL storeL);
12926   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12927   peepreplace (storeL(1.mem 1.mem 1.src));
12928 %}
12929 
12930 //----------SMARTSPILL RULES---------------------------------------------------
12931 // These must follow all instruction definitions as they use the names
12932 // defined in the instructions definitions.
--- EOF ---