1 //
   2 // Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // The registers which can be used for
 321 // a thread local safepoint poll
 322 // * R12 is reserved for heap base
 323 // * R13 cannot be encoded for addressing without an offset byte
 324 // * R15 is reserved for the JavaThread
 325 reg_class ptr_rex_reg(R8,  R8_H,
 326                       R9,  R9_H,
 327                       R10, R10_H,
 328                       R11, R11_H,
 329                       R14, R14_H);
 330 
 331 
 332 // Class for all long registers (excluding RSP)
 333 reg_class long_reg_with_rbp(RAX, RAX_H,
 334                             RDX, RDX_H,
 335                             RBP, RBP_H,
 336                             RDI, RDI_H,
 337                             RSI, RSI_H,
 338                             RCX, RCX_H,
 339                             RBX, RBX_H,
 340                             R8,  R8_H,
 341                             R9,  R9_H,
 342                             R10, R10_H,
 343                             R11, R11_H,
 344                             R13, R13_H,
 345                             R14, R14_H);
 346 
 347 // Class for all long registers (excluding RSP and RBP)
 348 reg_class long_reg_no_rbp(RAX, RAX_H,
 349                           RDX, RDX_H,
 350                           RDI, RDI_H,
 351                           RSI, RSI_H,
 352                           RCX, RCX_H,
 353                           RBX, RBX_H,
 354                           R8,  R8_H,
 355                           R9,  R9_H,
 356                           R10, R10_H,
 357                           R11, R11_H,
 358                           R13, R13_H,
 359                           R14, R14_H);
 360 
 361 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 362 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 363 
 364 // Class for all long registers (excluding RAX, RDX and RSP)
 365 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 366                                        RDI, RDI_H,
 367                                        RSI, RSI_H,
 368                                        RCX, RCX_H,
 369                                        RBX, RBX_H,
 370                                        R8,  R8_H,
 371                                        R9,  R9_H,
 372                                        R10, R10_H,
 373                                        R11, R11_H,
 374                                        R13, R13_H,
 375                                        R14, R14_H);
 376 
 377 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 378 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 379                                      RSI, RSI_H,
 380                                      RCX, RCX_H,
 381                                      RBX, RBX_H,
 382                                      R8,  R8_H,
 383                                      R9,  R9_H,
 384                                      R10, R10_H,
 385                                      R11, R11_H,
 386                                      R13, R13_H,
 387                                      R14, R14_H);
 388 
 389 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 390 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 391 
 392 // Class for all long registers (excluding RCX and RSP)
 393 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 394                                    RDI, RDI_H,
 395                                    RSI, RSI_H,
 396                                    RAX, RAX_H,
 397                                    RDX, RDX_H,
 398                                    RBX, RBX_H,
 399                                    R8,  R8_H,
 400                                    R9,  R9_H,
 401                                    R10, R10_H,
 402                                    R11, R11_H,
 403                                    R13, R13_H,
 404                                    R14, R14_H);
 405 
 406 // Class for all long registers (excluding RCX, RSP, and RBP)
 407 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 408                                  RSI, RSI_H,
 409                                  RAX, RAX_H,
 410                                  RDX, RDX_H,
 411                                  RBX, RBX_H,
 412                                  R8,  R8_H,
 413                                  R9,  R9_H,
 414                                  R10, R10_H,
 415                                  R11, R11_H,
 416                                  R13, R13_H,
 417                                  R14, R14_H);
 418 
 419 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 420 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 421 
 422 // Singleton class for RAX long register
 423 reg_class long_rax_reg(RAX, RAX_H);
 424 
 425 // Singleton class for RCX long register
 426 reg_class long_rcx_reg(RCX, RCX_H);
 427 
 428 // Singleton class for RDX long register
 429 reg_class long_rdx_reg(RDX, RDX_H);
 430 
 431 // Class for all int registers (excluding RSP)
 432 reg_class int_reg_with_rbp(RAX,
 433                            RDX,
 434                            RBP,
 435                            RDI,
 436                            RSI,
 437                            RCX,
 438                            RBX,
 439                            R8,
 440                            R9,
 441                            R10,
 442                            R11,
 443                            R13,
 444                            R14);
 445 
 446 // Class for all int registers (excluding RSP and RBP)
 447 reg_class int_reg_no_rbp(RAX,
 448                          RDX,
 449                          RDI,
 450                          RSI,
 451                          RCX,
 452                          RBX,
 453                          R8,
 454                          R9,
 455                          R10,
 456                          R11,
 457                          R13,
 458                          R14);
 459 
 460 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 461 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 462 
 463 // Class for all int registers (excluding RCX and RSP)
 464 reg_class int_no_rcx_reg_with_rbp(RAX,
 465                                   RDX,
 466                                   RBP,
 467                                   RDI,
 468                                   RSI,
 469                                   RBX,
 470                                   R8,
 471                                   R9,
 472                                   R10,
 473                                   R11,
 474                                   R13,
 475                                   R14);
 476 
 477 // Class for all int registers (excluding RCX, RSP, and RBP)
 478 reg_class int_no_rcx_reg_no_rbp(RAX,
 479                                 RDX,
 480                                 RDI,
 481                                 RSI,
 482                                 RBX,
 483                                 R8,
 484                                 R9,
 485                                 R10,
 486                                 R11,
 487                                 R13,
 488                                 R14);
 489 
 490 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 491 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 492 
 493 // Class for all int registers (excluding RAX, RDX, and RSP)
 494 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 495                                       RDI,
 496                                       RSI,
 497                                       RCX,
 498                                       RBX,
 499                                       R8,
 500                                       R9,
 501                                       R10,
 502                                       R11,
 503                                       R13,
 504                                       R14);
 505 
 506 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 507 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 508                                     RSI,
 509                                     RCX,
 510                                     RBX,
 511                                     R8,
 512                                     R9,
 513                                     R10,
 514                                     R11,
 515                                     R13,
 516                                     R14);
 517 
 518 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 519 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 520 
 521 // Singleton class for RAX int register
 522 reg_class int_rax_reg(RAX);
 523 
 524 // Singleton class for RBX int register
 525 reg_class int_rbx_reg(RBX);
 526 
 527 // Singleton class for RCX int register
 528 reg_class int_rcx_reg(RCX);
 529 
 530 // Singleton class for RCX int register
 531 reg_class int_rdx_reg(RDX);
 532 
 533 // Singleton class for RCX int register
 534 reg_class int_rdi_reg(RDI);
 535 
 536 // Singleton class for instruction pointer
 537 // reg_class ip_reg(RIP);
 538 
 539 %}
 540 
 541 source_hpp %{
 542 #if INCLUDE_ZGC
 543 #include "gc/z/zBarrierSetAssembler.hpp"
 544 #endif
 545 %}
 546 
 547 //----------SOURCE BLOCK-------------------------------------------------------
 548 // This is a block of C++ code which provides values, functions, and
 549 // definitions necessary in the rest of the architecture description
 550 source %{
 551 #define   RELOC_IMM64    Assembler::imm_operand
 552 #define   RELOC_DISP32   Assembler::disp32_operand
 553 
 554 #define __ _masm.
 555 
 556 static bool generate_vzeroupper(Compile* C) {
 557   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 558 }
 559 
 560 static int clear_avx_size() {
 561   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 562 }
 563 
 564 // !!!!! Special hack to get all types of calls to specify the byte offset
 565 //       from the start of the call to the point where the return address
 566 //       will point.
 567 int MachCallStaticJavaNode::ret_addr_offset()
 568 {
 569   int offset = 5; // 5 bytes from start of call to where return address points
 570   offset += clear_avx_size();
 571   return offset;
 572 }
 573 
 574 int MachCallDynamicJavaNode::ret_addr_offset()
 575 {
 576   int offset = 15; // 15 bytes from start of call to where return address points
 577   offset += clear_avx_size();
 578   return offset;
 579 }
 580 
 581 int MachCallRuntimeNode::ret_addr_offset() {
 582   int offset = 13; // movq r10,#addr; callq (r10)
 583   offset += clear_avx_size();
 584   return offset;
 585 }
 586 
 587 // Indicate if the safepoint node needs the polling page as an input,
 588 // it does if the polling page is more than disp32 away.
 589 bool SafePointNode::needs_polling_address_input()
 590 {
 591   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 592 }
 593 
 594 //
 595 // Compute padding required for nodes which need alignment
 596 //
 597 
 598 // The address of the call instruction needs to be 4-byte aligned to
 599 // ensure that it does not span a cache line so that it can be patched.
 600 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 601 {
 602   current_offset += clear_avx_size(); // skip vzeroupper
 603   current_offset += 1; // skip call opcode byte
 604   return align_up(current_offset, alignment_required()) - current_offset;
 605 }
 606 
 607 // The address of the call instruction needs to be 4-byte aligned to
 608 // ensure that it does not span a cache line so that it can be patched.
 609 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 610 {
 611   current_offset += clear_avx_size(); // skip vzeroupper
 612   current_offset += 11; // skip movq instruction + call opcode byte
 613   return align_up(current_offset, alignment_required()) - current_offset;
 614 }
 615 
 616 // EMIT_RM()
 617 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 618   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 619   cbuf.insts()->emit_int8(c);
 620 }
 621 
 622 // EMIT_CC()
 623 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 624   unsigned char c = (unsigned char) (f1 | f2);
 625   cbuf.insts()->emit_int8(c);
 626 }
 627 
 628 // EMIT_OPCODE()
 629 void emit_opcode(CodeBuffer &cbuf, int code) {
 630   cbuf.insts()->emit_int8((unsigned char) code);
 631 }
 632 
 633 // EMIT_OPCODE() w/ relocation information
 634 void emit_opcode(CodeBuffer &cbuf,
 635                  int code, relocInfo::relocType reloc, int offset, int format)
 636 {
 637   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 638   emit_opcode(cbuf, code);
 639 }
 640 
 641 // EMIT_D8()
 642 void emit_d8(CodeBuffer &cbuf, int d8) {
 643   cbuf.insts()->emit_int8((unsigned char) d8);
 644 }
 645 
 646 // EMIT_D16()
 647 void emit_d16(CodeBuffer &cbuf, int d16) {
 648   cbuf.insts()->emit_int16(d16);
 649 }
 650 
 651 // EMIT_D32()
 652 void emit_d32(CodeBuffer &cbuf, int d32) {
 653   cbuf.insts()->emit_int32(d32);
 654 }
 655 
 656 // EMIT_D64()
 657 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 658   cbuf.insts()->emit_int64(d64);
 659 }
 660 
 661 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 662 void emit_d32_reloc(CodeBuffer& cbuf,
 663                     int d32,
 664                     relocInfo::relocType reloc,
 665                     int format)
 666 {
 667   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 668   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 669   cbuf.insts()->emit_int32(d32);
 670 }
 671 
 672 // emit 32 bit value and construct relocation entry from RelocationHolder
 673 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 674 #ifdef ASSERT
 675   if (rspec.reloc()->type() == relocInfo::oop_type &&
 676       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 677     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 678     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 679   }
 680 #endif
 681   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 682   cbuf.insts()->emit_int32(d32);
 683 }
 684 
 685 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 686   address next_ip = cbuf.insts_end() + 4;
 687   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 688                  external_word_Relocation::spec(addr),
 689                  RELOC_DISP32);
 690 }
 691 
 692 
 693 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 694 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 695   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 696   cbuf.insts()->emit_int64(d64);
 697 }
 698 
 699 // emit 64 bit value and construct relocation entry from RelocationHolder
 700 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 701 #ifdef ASSERT
 702   if (rspec.reloc()->type() == relocInfo::oop_type &&
 703       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 704     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 705     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 706            "cannot embed scavengable oops in code");
 707   }
 708 #endif
 709   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 710   cbuf.insts()->emit_int64(d64);
 711 }
 712 
 713 // Access stack slot for load or store
 714 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 715 {
 716   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 717   if (-0x80 <= disp && disp < 0x80) {
 718     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 719     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 720     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 721   } else {
 722     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 723     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 724     emit_d32(cbuf, disp);     // Displacement // R/M byte
 725   }
 726 }
 727 
 728    // rRegI ereg, memory mem) %{    // emit_reg_mem
 729 void encode_RegMem(CodeBuffer &cbuf,
 730                    int reg,
 731                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 732 {
 733   assert(disp_reloc == relocInfo::none, "cannot have disp");
 734   int regenc = reg & 7;
 735   int baseenc = base & 7;
 736   int indexenc = index & 7;
 737 
 738   // There is no index & no scale, use form without SIB byte
 739   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 740     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 741     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 742       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 743     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 744       // If 8-bit displacement, mode 0x1
 745       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 746       emit_d8(cbuf, disp);
 747     } else {
 748       // If 32-bit displacement
 749       if (base == -1) { // Special flag for absolute address
 750         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 751         if (disp_reloc != relocInfo::none) {
 752           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 753         } else {
 754           emit_d32(cbuf, disp);
 755         }
 756       } else {
 757         // Normal base + offset
 758         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 759         if (disp_reloc != relocInfo::none) {
 760           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 761         } else {
 762           emit_d32(cbuf, disp);
 763         }
 764       }
 765     }
 766   } else {
 767     // Else, encode with the SIB byte
 768     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 769     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 770       // If no displacement
 771       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 772       emit_rm(cbuf, scale, indexenc, baseenc);
 773     } else {
 774       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 775         // If 8-bit displacement, mode 0x1
 776         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 777         emit_rm(cbuf, scale, indexenc, baseenc);
 778         emit_d8(cbuf, disp);
 779       } else {
 780         // If 32-bit displacement
 781         if (base == 0x04 ) {
 782           emit_rm(cbuf, 0x2, regenc, 0x4);
 783           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 784         } else {
 785           emit_rm(cbuf, 0x2, regenc, 0x4);
 786           emit_rm(cbuf, scale, indexenc, baseenc); // *
 787         }
 788         if (disp_reloc != relocInfo::none) {
 789           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 790         } else {
 791           emit_d32(cbuf, disp);
 792         }
 793       }
 794     }
 795   }
 796 }
 797 
 798 // This could be in MacroAssembler but it's fairly C2 specific
 799 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 800   Label exit;
 801   __ jccb(Assembler::noParity, exit);
 802   __ pushf();
 803   //
 804   // comiss/ucomiss instructions set ZF,PF,CF flags and
 805   // zero OF,AF,SF for NaN values.
 806   // Fixup flags by zeroing ZF,PF so that compare of NaN
 807   // values returns 'less than' result (CF is set).
 808   // Leave the rest of flags unchanged.
 809   //
 810   //    7 6 5 4 3 2 1 0
 811   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 812   //    0 0 1 0 1 0 1 1   (0x2B)
 813   //
 814   __ andq(Address(rsp, 0), 0xffffff2b);
 815   __ popf();
 816   __ bind(exit);
 817 }
 818 
 819 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 820   Label done;
 821   __ movl(dst, -1);
 822   __ jcc(Assembler::parity, done);
 823   __ jcc(Assembler::below, done);
 824   __ setb(Assembler::notEqual, dst);
 825   __ movzbl(dst, dst);
 826   __ bind(done);
 827 }
 828 
 829 
 830 //=============================================================================
 831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 832 
 833 int Compile::ConstantTable::calculate_table_base_offset() const {
 834   return 0;  // absolute addressing, no offset
 835 }
 836 
 837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 839   ShouldNotReachHere();
 840 }
 841 
 842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 843   // Empty encoding
 844 }
 845 
 846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 847   return 0;
 848 }
 849 
 850 #ifndef PRODUCT
 851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 852   st->print("# MachConstantBaseNode (empty encoding)");
 853 }
 854 #endif
 855 
 856 
 857 //=============================================================================
 858 #ifndef PRODUCT
 859 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 860   Compile* C = ra_->C;
 861 
 862   int framesize = C->frame_size_in_bytes();
 863   int bangsize = C->bang_size_in_bytes();
 864   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 865   // Remove wordSize for return addr which is already pushed.
 866   framesize -= wordSize;
 867 
 868   if (C->need_stack_bang(bangsize)) {
 869     framesize -= wordSize;
 870     st->print("# stack bang (%d bytes)", bangsize);
 871     st->print("\n\t");
 872     st->print("pushq   rbp\t# Save rbp");
 873     if (PreserveFramePointer) {
 874         st->print("\n\t");
 875         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 876     }
 877     if (framesize) {
 878       st->print("\n\t");
 879       st->print("subq    rsp, #%d\t# Create frame",framesize);
 880     }
 881   } else {
 882     st->print("subq    rsp, #%d\t# Create frame",framesize);
 883     st->print("\n\t");
 884     framesize -= wordSize;
 885     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 886     if (PreserveFramePointer) {
 887       st->print("\n\t");
 888       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 889       if (framesize > 0) {
 890         st->print("\n\t");
 891         st->print("addq    rbp, #%d", framesize);
 892       }
 893     }
 894   }
 895 
 896   if (VerifyStackAtCalls) {
 897     st->print("\n\t");
 898     framesize -= wordSize;
 899     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 900 #ifdef ASSERT
 901     st->print("\n\t");
 902     st->print("# stack alignment check");
 903 #endif
 904   }
 905   st->cr();
 906 }
 907 #endif
 908 
 909 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 910   Compile* C = ra_->C;
 911   MacroAssembler _masm(&cbuf);
 912 
 913   int framesize = C->frame_size_in_bytes();
 914   int bangsize = C->bang_size_in_bytes();
 915 
 916   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 917 
 918   C->set_frame_complete(cbuf.insts_size());
 919 
 920   if (C->has_mach_constant_base_node()) {
 921     // NOTE: We set the table base offset here because users might be
 922     // emitted before MachConstantBaseNode.
 923     Compile::ConstantTable& constant_table = C->constant_table();
 924     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 925   }
 926 }
 927 
 928 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 929 {
 930   return MachNode::size(ra_); // too many variables; just compute it
 931                               // the hard way
 932 }
 933 
 934 int MachPrologNode::reloc() const
 935 {
 936   return 0; // a large enough number
 937 }
 938 
 939 //=============================================================================
 940 #ifndef PRODUCT
 941 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 942 {
 943   Compile* C = ra_->C;
 944   if (generate_vzeroupper(C)) {
 945     st->print("vzeroupper");
 946     st->cr(); st->print("\t");
 947   }
 948 
 949   int framesize = C->frame_size_in_bytes();
 950   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 951   // Remove word for return adr already pushed
 952   // and RBP
 953   framesize -= 2*wordSize;
 954 
 955   if (framesize) {
 956     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 957     st->print("\t");
 958   }
 959 
 960   st->print_cr("popq   rbp");
 961   if (do_polling() && C->is_method_compilation()) {
 962     st->print("\t");
 963     if (SafepointMechanism::uses_thread_local_poll()) {
 964       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 965                    "testl  rax, [rscratch1]\t"
 966                    "# Safepoint: poll for GC");
 967     } else if (Assembler::is_polling_page_far()) {
 968       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 969                    "testl  rax, [rscratch1]\t"
 970                    "# Safepoint: poll for GC");
 971     } else {
 972       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 973                    "# Safepoint: poll for GC");
 974     }
 975   }
 976 }
 977 #endif
 978 
 979 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 980 {
 981   Compile* C = ra_->C;
 982   MacroAssembler _masm(&cbuf);
 983 
 984   if (generate_vzeroupper(C)) {
 985     // Clear upper bits of YMM registers when current compiled code uses
 986     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 987     __ vzeroupper();
 988   }
 989 
 990   int framesize = C->frame_size_in_bytes();
 991   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 992   // Remove word for return adr already pushed
 993   // and RBP
 994   framesize -= 2*wordSize;
 995 
 996   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 997 
 998   if (framesize) {
 999     emit_opcode(cbuf, Assembler::REX_W);
1000     if (framesize < 0x80) {
1001       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
1002       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1003       emit_d8(cbuf, framesize);
1004     } else {
1005       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1006       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1007       emit_d32(cbuf, framesize);
1008     }
1009   }
1010 
1011   // popq rbp
1012   emit_opcode(cbuf, 0x58 | RBP_enc);
1013 
1014   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1015     __ reserved_stack_check();
1016   }
1017 
1018   if (do_polling() && C->is_method_compilation()) {
1019     MacroAssembler _masm(&cbuf);
1020     if (SafepointMechanism::uses_thread_local_poll()) {
1021       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1022       __ relocate(relocInfo::poll_return_type);
1023       __ testl(rax, Address(rscratch1, 0));
1024     } else {
1025       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1026       if (Assembler::is_polling_page_far()) {
1027         __ lea(rscratch1, polling_page);
1028         __ relocate(relocInfo::poll_return_type);
1029         __ testl(rax, Address(rscratch1, 0));
1030       } else {
1031         __ testl(rax, polling_page);
1032       }
1033     }
1034   }
1035 }
1036 
1037 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1038 {
1039   return MachNode::size(ra_); // too many variables; just compute it
1040                               // the hard way
1041 }
1042 
1043 int MachEpilogNode::reloc() const
1044 {
1045   return 2; // a large enough number
1046 }
1047 
1048 const Pipeline* MachEpilogNode::pipeline() const
1049 {
1050   return MachNode::pipeline_class();
1051 }
1052 
1053 int MachEpilogNode::safepoint_offset() const
1054 {
1055   return 0;
1056 }
1057 
1058 //=============================================================================
1059 
1060 enum RC {
1061   rc_bad,
1062   rc_int,
1063   rc_float,
1064   rc_stack
1065 };
1066 
1067 static enum RC rc_class(OptoReg::Name reg)
1068 {
1069   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1070 
1071   if (OptoReg::is_stack(reg)) return rc_stack;
1072 
1073   VMReg r = OptoReg::as_VMReg(reg);
1074 
1075   if (r->is_Register()) return rc_int;
1076 
1077   assert(r->is_XMMRegister(), "must be");
1078   return rc_float;
1079 }
1080 
1081 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1082 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1083                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1084 
1085 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1086                             int stack_offset, int reg, uint ireg, outputStream* st);
1087 
1088 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1089                                       int dst_offset, uint ireg, outputStream* st) {
1090   if (cbuf) {
1091     MacroAssembler _masm(cbuf);
1092     switch (ireg) {
1093     case Op_VecS:
1094       __ movq(Address(rsp, -8), rax);
1095       __ movl(rax, Address(rsp, src_offset));
1096       __ movl(Address(rsp, dst_offset), rax);
1097       __ movq(rax, Address(rsp, -8));
1098       break;
1099     case Op_VecD:
1100       __ pushq(Address(rsp, src_offset));
1101       __ popq (Address(rsp, dst_offset));
1102       break;
1103     case Op_VecX:
1104       __ pushq(Address(rsp, src_offset));
1105       __ popq (Address(rsp, dst_offset));
1106       __ pushq(Address(rsp, src_offset+8));
1107       __ popq (Address(rsp, dst_offset+8));
1108       break;
1109     case Op_VecY:
1110       __ vmovdqu(Address(rsp, -32), xmm0);
1111       __ vmovdqu(xmm0, Address(rsp, src_offset));
1112       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1113       __ vmovdqu(xmm0, Address(rsp, -32));
1114       break;
1115     case Op_VecZ:
1116       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1117       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1118       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1119       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1120       break;
1121     default:
1122       ShouldNotReachHere();
1123     }
1124 #ifndef PRODUCT
1125   } else {
1126     switch (ireg) {
1127     case Op_VecS:
1128       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1129                 "movl    rax, [rsp + #%d]\n\t"
1130                 "movl    [rsp + #%d], rax\n\t"
1131                 "movq    rax, [rsp - #8]",
1132                 src_offset, dst_offset);
1133       break;
1134     case Op_VecD:
1135       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1136                 "popq    [rsp + #%d]",
1137                 src_offset, dst_offset);
1138       break;
1139      case Op_VecX:
1140       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1141                 "popq    [rsp + #%d]\n\t"
1142                 "pushq   [rsp + #%d]\n\t"
1143                 "popq    [rsp + #%d]",
1144                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1145       break;
1146     case Op_VecY:
1147       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1148                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1149                 "vmovdqu [rsp + #%d], xmm0\n\t"
1150                 "vmovdqu xmm0, [rsp - #32]",
1151                 src_offset, dst_offset);
1152       break;
1153     case Op_VecZ:
1154       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1155                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1156                 "vmovdqu [rsp + #%d], xmm0\n\t"
1157                 "vmovdqu xmm0, [rsp - #64]",
1158                 src_offset, dst_offset);
1159       break;
1160     default:
1161       ShouldNotReachHere();
1162     }
1163 #endif
1164   }
1165 }
1166 
1167 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1168                                        PhaseRegAlloc* ra_,
1169                                        bool do_size,
1170                                        outputStream* st) const {
1171   assert(cbuf != NULL || st  != NULL, "sanity");
1172   // Get registers to move
1173   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1174   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1175   OptoReg::Name dst_second = ra_->get_reg_second(this);
1176   OptoReg::Name dst_first = ra_->get_reg_first(this);
1177 
1178   enum RC src_second_rc = rc_class(src_second);
1179   enum RC src_first_rc = rc_class(src_first);
1180   enum RC dst_second_rc = rc_class(dst_second);
1181   enum RC dst_first_rc = rc_class(dst_first);
1182 
1183   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1184          "must move at least 1 register" );
1185 
1186   if (src_first == dst_first && src_second == dst_second) {
1187     // Self copy, no move
1188     return 0;
1189   }
1190   if (bottom_type()->isa_vect() != NULL) {
1191     uint ireg = ideal_reg();
1192     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1193     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1194     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1195       // mem -> mem
1196       int src_offset = ra_->reg2offset(src_first);
1197       int dst_offset = ra_->reg2offset(dst_first);
1198       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1199     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1200       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1201     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1202       int stack_offset = ra_->reg2offset(dst_first);
1203       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1204     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1205       int stack_offset = ra_->reg2offset(src_first);
1206       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1207     } else {
1208       ShouldNotReachHere();
1209     }
1210     return 0;
1211   }
1212   if (src_first_rc == rc_stack) {
1213     // mem ->
1214     if (dst_first_rc == rc_stack) {
1215       // mem -> mem
1216       assert(src_second != dst_first, "overlap");
1217       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1218           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1219         // 64-bit
1220         int src_offset = ra_->reg2offset(src_first);
1221         int dst_offset = ra_->reg2offset(dst_first);
1222         if (cbuf) {
1223           MacroAssembler _masm(cbuf);
1224           __ pushq(Address(rsp, src_offset));
1225           __ popq (Address(rsp, dst_offset));
1226 #ifndef PRODUCT
1227         } else {
1228           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1229                     "popq    [rsp + #%d]",
1230                      src_offset, dst_offset);
1231 #endif
1232         }
1233       } else {
1234         // 32-bit
1235         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1236         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1237         // No pushl/popl, so:
1238         int src_offset = ra_->reg2offset(src_first);
1239         int dst_offset = ra_->reg2offset(dst_first);
1240         if (cbuf) {
1241           MacroAssembler _masm(cbuf);
1242           __ movq(Address(rsp, -8), rax);
1243           __ movl(rax, Address(rsp, src_offset));
1244           __ movl(Address(rsp, dst_offset), rax);
1245           __ movq(rax, Address(rsp, -8));
1246 #ifndef PRODUCT
1247         } else {
1248           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1249                     "movl    rax, [rsp + #%d]\n\t"
1250                     "movl    [rsp + #%d], rax\n\t"
1251                     "movq    rax, [rsp - #8]",
1252                      src_offset, dst_offset);
1253 #endif
1254         }
1255       }
1256       return 0;
1257     } else if (dst_first_rc == rc_int) {
1258       // mem -> gpr
1259       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1260           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1261         // 64-bit
1262         int offset = ra_->reg2offset(src_first);
1263         if (cbuf) {
1264           MacroAssembler _masm(cbuf);
1265           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("movq    %s, [rsp + #%d]\t# spill",
1269                      Matcher::regName[dst_first],
1270                      offset);
1271 #endif
1272         }
1273       } else {
1274         // 32-bit
1275         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1276         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1277         int offset = ra_->reg2offset(src_first);
1278         if (cbuf) {
1279           MacroAssembler _masm(cbuf);
1280           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1281 #ifndef PRODUCT
1282         } else {
1283           st->print("movl    %s, [rsp + #%d]\t# spill",
1284                      Matcher::regName[dst_first],
1285                      offset);
1286 #endif
1287         }
1288       }
1289       return 0;
1290     } else if (dst_first_rc == rc_float) {
1291       // mem-> xmm
1292       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1293           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1294         // 64-bit
1295         int offset = ra_->reg2offset(src_first);
1296         if (cbuf) {
1297           MacroAssembler _masm(cbuf);
1298           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1299 #ifndef PRODUCT
1300         } else {
1301           st->print("%s  %s, [rsp + #%d]\t# spill",
1302                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1303                      Matcher::regName[dst_first],
1304                      offset);
1305 #endif
1306         }
1307       } else {
1308         // 32-bit
1309         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1310         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1311         int offset = ra_->reg2offset(src_first);
1312         if (cbuf) {
1313           MacroAssembler _masm(cbuf);
1314           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1315 #ifndef PRODUCT
1316         } else {
1317           st->print("movss   %s, [rsp + #%d]\t# spill",
1318                      Matcher::regName[dst_first],
1319                      offset);
1320 #endif
1321         }
1322       }
1323       return 0;
1324     }
1325   } else if (src_first_rc == rc_int) {
1326     // gpr ->
1327     if (dst_first_rc == rc_stack) {
1328       // gpr -> mem
1329       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1330           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1331         // 64-bit
1332         int offset = ra_->reg2offset(dst_first);
1333         if (cbuf) {
1334           MacroAssembler _masm(cbuf);
1335           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1336 #ifndef PRODUCT
1337         } else {
1338           st->print("movq    [rsp + #%d], %s\t# spill",
1339                      offset,
1340                      Matcher::regName[src_first]);
1341 #endif
1342         }
1343       } else {
1344         // 32-bit
1345         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1346         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1347         int offset = ra_->reg2offset(dst_first);
1348         if (cbuf) {
1349           MacroAssembler _masm(cbuf);
1350           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1351 #ifndef PRODUCT
1352         } else {
1353           st->print("movl    [rsp + #%d], %s\t# spill",
1354                      offset,
1355                      Matcher::regName[src_first]);
1356 #endif
1357         }
1358       }
1359       return 0;
1360     } else if (dst_first_rc == rc_int) {
1361       // gpr -> gpr
1362       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1363           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1364         // 64-bit
1365         if (cbuf) {
1366           MacroAssembler _masm(cbuf);
1367           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1368                   as_Register(Matcher::_regEncode[src_first]));
1369 #ifndef PRODUCT
1370         } else {
1371           st->print("movq    %s, %s\t# spill",
1372                      Matcher::regName[dst_first],
1373                      Matcher::regName[src_first]);
1374 #endif
1375         }
1376         return 0;
1377       } else {
1378         // 32-bit
1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1381         if (cbuf) {
1382           MacroAssembler _masm(cbuf);
1383           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1384                   as_Register(Matcher::_regEncode[src_first]));
1385 #ifndef PRODUCT
1386         } else {
1387           st->print("movl    %s, %s\t# spill",
1388                      Matcher::regName[dst_first],
1389                      Matcher::regName[src_first]);
1390 #endif
1391         }
1392         return 0;
1393       }
1394     } else if (dst_first_rc == rc_float) {
1395       // gpr -> xmm
1396       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1397           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1398         // 64-bit
1399         if (cbuf) {
1400           MacroAssembler _masm(cbuf);
1401           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1402 #ifndef PRODUCT
1403         } else {
1404           st->print("movdq   %s, %s\t# spill",
1405                      Matcher::regName[dst_first],
1406                      Matcher::regName[src_first]);
1407 #endif
1408         }
1409       } else {
1410         // 32-bit
1411         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1412         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1413         if (cbuf) {
1414           MacroAssembler _masm(cbuf);
1415           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1416 #ifndef PRODUCT
1417         } else {
1418           st->print("movdl   %s, %s\t# spill",
1419                      Matcher::regName[dst_first],
1420                      Matcher::regName[src_first]);
1421 #endif
1422         }
1423       }
1424       return 0;
1425     }
1426   } else if (src_first_rc == rc_float) {
1427     // xmm ->
1428     if (dst_first_rc == rc_stack) {
1429       // xmm -> mem
1430       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1431           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1432         // 64-bit
1433         int offset = ra_->reg2offset(dst_first);
1434         if (cbuf) {
1435           MacroAssembler _masm(cbuf);
1436           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1437 #ifndef PRODUCT
1438         } else {
1439           st->print("movsd   [rsp + #%d], %s\t# spill",
1440                      offset,
1441                      Matcher::regName[src_first]);
1442 #endif
1443         }
1444       } else {
1445         // 32-bit
1446         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1447         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1448         int offset = ra_->reg2offset(dst_first);
1449         if (cbuf) {
1450           MacroAssembler _masm(cbuf);
1451           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1452 #ifndef PRODUCT
1453         } else {
1454           st->print("movss   [rsp + #%d], %s\t# spill",
1455                      offset,
1456                      Matcher::regName[src_first]);
1457 #endif
1458         }
1459       }
1460       return 0;
1461     } else if (dst_first_rc == rc_int) {
1462       // xmm -> gpr
1463       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1464           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1465         // 64-bit
1466         if (cbuf) {
1467           MacroAssembler _masm(cbuf);
1468           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1469 #ifndef PRODUCT
1470         } else {
1471           st->print("movdq   %s, %s\t# spill",
1472                      Matcher::regName[dst_first],
1473                      Matcher::regName[src_first]);
1474 #endif
1475         }
1476       } else {
1477         // 32-bit
1478         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1479         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1480         if (cbuf) {
1481           MacroAssembler _masm(cbuf);
1482           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1483 #ifndef PRODUCT
1484         } else {
1485           st->print("movdl   %s, %s\t# spill",
1486                      Matcher::regName[dst_first],
1487                      Matcher::regName[src_first]);
1488 #endif
1489         }
1490       }
1491       return 0;
1492     } else if (dst_first_rc == rc_float) {
1493       // xmm -> xmm
1494       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1495           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1496         // 64-bit
1497         if (cbuf) {
1498           MacroAssembler _masm(cbuf);
1499           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1500 #ifndef PRODUCT
1501         } else {
1502           st->print("%s  %s, %s\t# spill",
1503                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1504                      Matcher::regName[dst_first],
1505                      Matcher::regName[src_first]);
1506 #endif
1507         }
1508       } else {
1509         // 32-bit
1510         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1511         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1512         if (cbuf) {
1513           MacroAssembler _masm(cbuf);
1514           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1515 #ifndef PRODUCT
1516         } else {
1517           st->print("%s  %s, %s\t# spill",
1518                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1519                      Matcher::regName[dst_first],
1520                      Matcher::regName[src_first]);
1521 #endif
1522         }
1523       }
1524       return 0;
1525     }
1526   }
1527 
1528   assert(0," foo ");
1529   Unimplemented();
1530   return 0;
1531 }
1532 
1533 #ifndef PRODUCT
1534 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1535   implementation(NULL, ra_, false, st);
1536 }
1537 #endif
1538 
1539 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1540   implementation(&cbuf, ra_, false, NULL);
1541 }
1542 
1543 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1544   return MachNode::size(ra_);
1545 }
1546 
1547 //=============================================================================
1548 #ifndef PRODUCT
1549 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1550 {
1551   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1552   int reg = ra_->get_reg_first(this);
1553   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1554             Matcher::regName[reg], offset);
1555 }
1556 #endif
1557 
1558 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1559 {
1560   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1561   int reg = ra_->get_encode(this);
1562   if (offset >= 0x80) {
1563     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1564     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1565     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1566     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1567     emit_d32(cbuf, offset);
1568   } else {
1569     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1570     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1571     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1572     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1573     emit_d8(cbuf, offset);
1574   }
1575 }
1576 
1577 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1578 {
1579   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1580   return (offset < 0x80) ? 5 : 8; // REX
1581 }
1582 
1583 //=============================================================================
1584 #ifndef PRODUCT
1585 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1586 {
1587   if (UseCompressedClassPointers) {
1588     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1589     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1590     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1591   } else {
1592     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1593                  "# Inline cache check");
1594   }
1595   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1596   st->print_cr("\tnop\t# nops to align entry point");
1597 }
1598 #endif
1599 
1600 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1601 {
1602   MacroAssembler masm(&cbuf);
1603   uint insts_size = cbuf.insts_size();
1604   if (UseCompressedClassPointers) {
1605     masm.load_klass(rscratch1, j_rarg0);
1606     masm.cmpptr(rax, rscratch1);
1607   } else {
1608     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1609   }
1610 
1611   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1612 
1613   /* WARNING these NOPs are critical so that verified entry point is properly
1614      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1615   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1616   if (OptoBreakpoint) {
1617     // Leave space for int3
1618     nops_cnt -= 1;
1619   }
1620   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1621   if (nops_cnt > 0)
1622     masm.nop(nops_cnt);
1623 }
1624 
1625 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1626 {
1627   return MachNode::size(ra_); // too many variables; just compute it
1628                               // the hard way
1629 }
1630 
1631 
1632 //=============================================================================
1633 
1634 int Matcher::regnum_to_fpu_offset(int regnum)
1635 {
1636   return regnum - 32; // The FP registers are in the second chunk
1637 }
1638 
1639 // This is UltraSparc specific, true just means we have fast l2f conversion
1640 const bool Matcher::convL2FSupported(void) {
1641   return true;
1642 }
1643 
1644 // Is this branch offset short enough that a short branch can be used?
1645 //
1646 // NOTE: If the platform does not provide any short branch variants, then
1647 //       this method should return false for offset 0.
1648 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1649   // The passed offset is relative to address of the branch.
1650   // On 86 a branch displacement is calculated relative to address
1651   // of a next instruction.
1652   offset -= br_size;
1653 
1654   // the short version of jmpConUCF2 contains multiple branches,
1655   // making the reach slightly less
1656   if (rule == jmpConUCF2_rule)
1657     return (-126 <= offset && offset <= 125);
1658   return (-128 <= offset && offset <= 127);
1659 }
1660 
1661 const bool Matcher::isSimpleConstant64(jlong value) {
1662   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1663   //return value == (int) value;  // Cf. storeImmL and immL32.
1664 
1665   // Probably always true, even if a temp register is required.
1666   return true;
1667 }
1668 
1669 // The ecx parameter to rep stosq for the ClearArray node is in words.
1670 const bool Matcher::init_array_count_is_in_bytes = false;
1671 
1672 // No additional cost for CMOVL.
1673 const int Matcher::long_cmove_cost() { return 0; }
1674 
1675 // No CMOVF/CMOVD with SSE2
1676 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1677 
1678 // Does the CPU require late expand (see block.cpp for description of late expand)?
1679 const bool Matcher::require_postalloc_expand = false;
1680 
1681 // Do we need to mask the count passed to shift instructions or does
1682 // the cpu only look at the lower 5/6 bits anyway?
1683 const bool Matcher::need_masked_shift_count = false;
1684 
1685 bool Matcher::narrow_oop_use_complex_address() {
1686   assert(UseCompressedOops, "only for compressed oops code");
1687   return (LogMinObjAlignmentInBytes <= 3);
1688 }
1689 
1690 bool Matcher::narrow_klass_use_complex_address() {
1691   assert(UseCompressedClassPointers, "only for compressed klass code");
1692   return (LogKlassAlignmentInBytes <= 3);
1693 }
1694 
1695 bool Matcher::const_oop_prefer_decode() {
1696   // Prefer ConN+DecodeN over ConP.
1697   return true;
1698 }
1699 
1700 bool Matcher::const_klass_prefer_decode() {
1701   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1702   //       or condisider the following:
1703   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1704   //return Universe::narrow_klass_base() == NULL;
1705   return true;
1706 }
1707 
1708 // Is it better to copy float constants, or load them directly from
1709 // memory?  Intel can load a float constant from a direct address,
1710 // requiring no extra registers.  Most RISCs will have to materialize
1711 // an address into a register first, so they would do better to copy
1712 // the constant from stack.
1713 const bool Matcher::rematerialize_float_constants = true; // XXX
1714 
1715 // If CPU can load and store mis-aligned doubles directly then no
1716 // fixup is needed.  Else we split the double into 2 integer pieces
1717 // and move it piece-by-piece.  Only happens when passing doubles into
1718 // C code as the Java calling convention forces doubles to be aligned.
1719 const bool Matcher::misaligned_doubles_ok = true;
1720 
1721 // No-op on amd64
1722 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1723 
1724 // Advertise here if the CPU requires explicit rounding operations to
1725 // implement the UseStrictFP mode.
1726 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1727 
1728 // Are floats conerted to double when stored to stack during deoptimization?
1729 // On x64 it is stored without convertion so we can use normal access.
1730 bool Matcher::float_in_double() { return false; }
1731 
1732 // Do ints take an entire long register or just half?
1733 const bool Matcher::int_in_long = true;
1734 
1735 // Return whether or not this register is ever used as an argument.
1736 // This function is used on startup to build the trampoline stubs in
1737 // generateOptoStub.  Registers not mentioned will be killed by the VM
1738 // call in the trampoline, and arguments in those registers not be
1739 // available to the callee.
1740 bool Matcher::can_be_java_arg(int reg)
1741 {
1742   return
1743     reg ==  RDI_num || reg == RDI_H_num ||
1744     reg ==  RSI_num || reg == RSI_H_num ||
1745     reg ==  RDX_num || reg == RDX_H_num ||
1746     reg ==  RCX_num || reg == RCX_H_num ||
1747     reg ==   R8_num || reg ==  R8_H_num ||
1748     reg ==   R9_num || reg ==  R9_H_num ||
1749     reg ==  R12_num || reg == R12_H_num ||
1750     reg == XMM0_num || reg == XMM0b_num ||
1751     reg == XMM1_num || reg == XMM1b_num ||
1752     reg == XMM2_num || reg == XMM2b_num ||
1753     reg == XMM3_num || reg == XMM3b_num ||
1754     reg == XMM4_num || reg == XMM4b_num ||
1755     reg == XMM5_num || reg == XMM5b_num ||
1756     reg == XMM6_num || reg == XMM6b_num ||
1757     reg == XMM7_num || reg == XMM7b_num;
1758 }
1759 
1760 bool Matcher::is_spillable_arg(int reg)
1761 {
1762   return can_be_java_arg(reg);
1763 }
1764 
1765 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1766   // In 64 bit mode a code which use multiply when
1767   // devisor is constant is faster than hardware
1768   // DIV instruction (it uses MulHiL).
1769   return false;
1770 }
1771 
1772 // Register for DIVI projection of divmodI
1773 RegMask Matcher::divI_proj_mask() {
1774   return INT_RAX_REG_mask();
1775 }
1776 
1777 // Register for MODI projection of divmodI
1778 RegMask Matcher::modI_proj_mask() {
1779   return INT_RDX_REG_mask();
1780 }
1781 
1782 // Register for DIVL projection of divmodL
1783 RegMask Matcher::divL_proj_mask() {
1784   return LONG_RAX_REG_mask();
1785 }
1786 
1787 // Register for MODL projection of divmodL
1788 RegMask Matcher::modL_proj_mask() {
1789   return LONG_RDX_REG_mask();
1790 }
1791 
1792 // Register for saving SP into on method handle invokes. Not used on x86_64.
1793 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1794     return NO_REG_mask();
1795 }
1796 
1797 %}
1798 
1799 //----------ENCODING BLOCK-----------------------------------------------------
1800 // This block specifies the encoding classes used by the compiler to
1801 // output byte streams.  Encoding classes are parameterized macros
1802 // used by Machine Instruction Nodes in order to generate the bit
1803 // encoding of the instruction.  Operands specify their base encoding
1804 // interface with the interface keyword.  There are currently
1805 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1806 // COND_INTER.  REG_INTER causes an operand to generate a function
1807 // which returns its register number when queried.  CONST_INTER causes
1808 // an operand to generate a function which returns the value of the
1809 // constant when queried.  MEMORY_INTER causes an operand to generate
1810 // four functions which return the Base Register, the Index Register,
1811 // the Scale Value, and the Offset Value of the operand when queried.
1812 // COND_INTER causes an operand to generate six functions which return
1813 // the encoding code (ie - encoding bits for the instruction)
1814 // associated with each basic boolean condition for a conditional
1815 // instruction.
1816 //
1817 // Instructions specify two basic values for encoding.  Again, a
1818 // function is available to check if the constant displacement is an
1819 // oop. They use the ins_encode keyword to specify their encoding
1820 // classes (which must be a sequence of enc_class names, and their
1821 // parameters, specified in the encoding block), and they use the
1822 // opcode keyword to specify, in order, their primary, secondary, and
1823 // tertiary opcode.  Only the opcode sections which a particular
1824 // instruction needs for encoding need to be specified.
1825 encode %{
1826   // Build emit functions for each basic byte or larger field in the
1827   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1828   // from C++ code in the enc_class source block.  Emit functions will
1829   // live in the main source block for now.  In future, we can
1830   // generalize this by adding a syntax that specifies the sizes of
1831   // fields in an order, so that the adlc can build the emit functions
1832   // automagically
1833 
1834   // Emit primary opcode
1835   enc_class OpcP
1836   %{
1837     emit_opcode(cbuf, $primary);
1838   %}
1839 
1840   // Emit secondary opcode
1841   enc_class OpcS
1842   %{
1843     emit_opcode(cbuf, $secondary);
1844   %}
1845 
1846   // Emit tertiary opcode
1847   enc_class OpcT
1848   %{
1849     emit_opcode(cbuf, $tertiary);
1850   %}
1851 
1852   // Emit opcode directly
1853   enc_class Opcode(immI d8)
1854   %{
1855     emit_opcode(cbuf, $d8$$constant);
1856   %}
1857 
1858   // Emit size prefix
1859   enc_class SizePrefix
1860   %{
1861     emit_opcode(cbuf, 0x66);
1862   %}
1863 
1864   enc_class reg(rRegI reg)
1865   %{
1866     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1867   %}
1868 
1869   enc_class reg_reg(rRegI dst, rRegI src)
1870   %{
1871     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1872   %}
1873 
1874   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1875   %{
1876     emit_opcode(cbuf, $opcode$$constant);
1877     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1878   %}
1879 
1880   enc_class cdql_enc(no_rax_rdx_RegI div)
1881   %{
1882     // Full implementation of Java idiv and irem; checks for
1883     // special case as described in JVM spec., p.243 & p.271.
1884     //
1885     //         normal case                           special case
1886     //
1887     // input : rax: dividend                         min_int
1888     //         reg: divisor                          -1
1889     //
1890     // output: rax: quotient  (= rax idiv reg)       min_int
1891     //         rdx: remainder (= rax irem reg)       0
1892     //
1893     //  Code sequnce:
1894     //
1895     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1896     //    5:   75 07/08                jne    e <normal>
1897     //    7:   33 d2                   xor    %edx,%edx
1898     //  [div >= 8 -> offset + 1]
1899     //  [REX_B]
1900     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1901     //    c:   74 03/04                je     11 <done>
1902     // 000000000000000e <normal>:
1903     //    e:   99                      cltd
1904     //  [div >= 8 -> offset + 1]
1905     //  [REX_B]
1906     //    f:   f7 f9                   idiv   $div
1907     // 0000000000000011 <done>:
1908 
1909     // cmp    $0x80000000,%eax
1910     emit_opcode(cbuf, 0x3d);
1911     emit_d8(cbuf, 0x00);
1912     emit_d8(cbuf, 0x00);
1913     emit_d8(cbuf, 0x00);
1914     emit_d8(cbuf, 0x80);
1915 
1916     // jne    e <normal>
1917     emit_opcode(cbuf, 0x75);
1918     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1919 
1920     // xor    %edx,%edx
1921     emit_opcode(cbuf, 0x33);
1922     emit_d8(cbuf, 0xD2);
1923 
1924     // cmp    $0xffffffffffffffff,%ecx
1925     if ($div$$reg >= 8) {
1926       emit_opcode(cbuf, Assembler::REX_B);
1927     }
1928     emit_opcode(cbuf, 0x83);
1929     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1930     emit_d8(cbuf, 0xFF);
1931 
1932     // je     11 <done>
1933     emit_opcode(cbuf, 0x74);
1934     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1935 
1936     // <normal>
1937     // cltd
1938     emit_opcode(cbuf, 0x99);
1939 
1940     // idivl (note: must be emitted by the user of this rule)
1941     // <done>
1942   %}
1943 
1944   enc_class cdqq_enc(no_rax_rdx_RegL div)
1945   %{
1946     // Full implementation of Java ldiv and lrem; checks for
1947     // special case as described in JVM spec., p.243 & p.271.
1948     //
1949     //         normal case                           special case
1950     //
1951     // input : rax: dividend                         min_long
1952     //         reg: divisor                          -1
1953     //
1954     // output: rax: quotient  (= rax idiv reg)       min_long
1955     //         rdx: remainder (= rax irem reg)       0
1956     //
1957     //  Code sequnce:
1958     //
1959     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1960     //    7:   00 00 80
1961     //    a:   48 39 d0                cmp    %rdx,%rax
1962     //    d:   75 08                   jne    17 <normal>
1963     //    f:   33 d2                   xor    %edx,%edx
1964     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1965     //   15:   74 05                   je     1c <done>
1966     // 0000000000000017 <normal>:
1967     //   17:   48 99                   cqto
1968     //   19:   48 f7 f9                idiv   $div
1969     // 000000000000001c <done>:
1970 
1971     // mov    $0x8000000000000000,%rdx
1972     emit_opcode(cbuf, Assembler::REX_W);
1973     emit_opcode(cbuf, 0xBA);
1974     emit_d8(cbuf, 0x00);
1975     emit_d8(cbuf, 0x00);
1976     emit_d8(cbuf, 0x00);
1977     emit_d8(cbuf, 0x00);
1978     emit_d8(cbuf, 0x00);
1979     emit_d8(cbuf, 0x00);
1980     emit_d8(cbuf, 0x00);
1981     emit_d8(cbuf, 0x80);
1982 
1983     // cmp    %rdx,%rax
1984     emit_opcode(cbuf, Assembler::REX_W);
1985     emit_opcode(cbuf, 0x39);
1986     emit_d8(cbuf, 0xD0);
1987 
1988     // jne    17 <normal>
1989     emit_opcode(cbuf, 0x75);
1990     emit_d8(cbuf, 0x08);
1991 
1992     // xor    %edx,%edx
1993     emit_opcode(cbuf, 0x33);
1994     emit_d8(cbuf, 0xD2);
1995 
1996     // cmp    $0xffffffffffffffff,$div
1997     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1998     emit_opcode(cbuf, 0x83);
1999     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
2000     emit_d8(cbuf, 0xFF);
2001 
2002     // je     1e <done>
2003     emit_opcode(cbuf, 0x74);
2004     emit_d8(cbuf, 0x05);
2005 
2006     // <normal>
2007     // cqto
2008     emit_opcode(cbuf, Assembler::REX_W);
2009     emit_opcode(cbuf, 0x99);
2010 
2011     // idivq (note: must be emitted by the user of this rule)
2012     // <done>
2013   %}
2014 
2015   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2016   enc_class OpcSE(immI imm)
2017   %{
2018     // Emit primary opcode and set sign-extend bit
2019     // Check for 8-bit immediate, and set sign extend bit in opcode
2020     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2021       emit_opcode(cbuf, $primary | 0x02);
2022     } else {
2023       // 32-bit immediate
2024       emit_opcode(cbuf, $primary);
2025     }
2026   %}
2027 
2028   enc_class OpcSErm(rRegI dst, immI imm)
2029   %{
2030     // OpcSEr/m
2031     int dstenc = $dst$$reg;
2032     if (dstenc >= 8) {
2033       emit_opcode(cbuf, Assembler::REX_B);
2034       dstenc -= 8;
2035     }
2036     // Emit primary opcode and set sign-extend bit
2037     // Check for 8-bit immediate, and set sign extend bit in opcode
2038     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2039       emit_opcode(cbuf, $primary | 0x02);
2040     } else {
2041       // 32-bit immediate
2042       emit_opcode(cbuf, $primary);
2043     }
2044     // Emit r/m byte with secondary opcode, after primary opcode.
2045     emit_rm(cbuf, 0x3, $secondary, dstenc);
2046   %}
2047 
2048   enc_class OpcSErm_wide(rRegL dst, immI imm)
2049   %{
2050     // OpcSEr/m
2051     int dstenc = $dst$$reg;
2052     if (dstenc < 8) {
2053       emit_opcode(cbuf, Assembler::REX_W);
2054     } else {
2055       emit_opcode(cbuf, Assembler::REX_WB);
2056       dstenc -= 8;
2057     }
2058     // Emit primary opcode and set sign-extend bit
2059     // Check for 8-bit immediate, and set sign extend bit in opcode
2060     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2061       emit_opcode(cbuf, $primary | 0x02);
2062     } else {
2063       // 32-bit immediate
2064       emit_opcode(cbuf, $primary);
2065     }
2066     // Emit r/m byte with secondary opcode, after primary opcode.
2067     emit_rm(cbuf, 0x3, $secondary, dstenc);
2068   %}
2069 
2070   enc_class Con8or32(immI imm)
2071   %{
2072     // Check for 8-bit immediate, and set sign extend bit in opcode
2073     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2074       $$$emit8$imm$$constant;
2075     } else {
2076       // 32-bit immediate
2077       $$$emit32$imm$$constant;
2078     }
2079   %}
2080 
2081   enc_class opc2_reg(rRegI dst)
2082   %{
2083     // BSWAP
2084     emit_cc(cbuf, $secondary, $dst$$reg);
2085   %}
2086 
2087   enc_class opc3_reg(rRegI dst)
2088   %{
2089     // BSWAP
2090     emit_cc(cbuf, $tertiary, $dst$$reg);
2091   %}
2092 
2093   enc_class reg_opc(rRegI div)
2094   %{
2095     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2096     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2097   %}
2098 
2099   enc_class enc_cmov(cmpOp cop)
2100   %{
2101     // CMOV
2102     $$$emit8$primary;
2103     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2104   %}
2105 
2106   enc_class enc_PartialSubtypeCheck()
2107   %{
2108     Register Rrdi = as_Register(RDI_enc); // result register
2109     Register Rrax = as_Register(RAX_enc); // super class
2110     Register Rrcx = as_Register(RCX_enc); // killed
2111     Register Rrsi = as_Register(RSI_enc); // sub class
2112     Label miss;
2113     const bool set_cond_codes = true;
2114 
2115     MacroAssembler _masm(&cbuf);
2116     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2117                                      NULL, &miss,
2118                                      /*set_cond_codes:*/ true);
2119     if ($primary) {
2120       __ xorptr(Rrdi, Rrdi);
2121     }
2122     __ bind(miss);
2123   %}
2124 
2125   enc_class clear_avx %{
2126     debug_only(int off0 = cbuf.insts_size());
2127     if (generate_vzeroupper(Compile::current())) {
2128       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2129       // Clear upper bits of YMM registers when current compiled code uses
2130       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2131       MacroAssembler _masm(&cbuf);
2132       __ vzeroupper();
2133     }
2134     debug_only(int off1 = cbuf.insts_size());
2135     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2136   %}
2137 
2138   enc_class Java_To_Runtime(method meth) %{
2139     // No relocation needed
2140     MacroAssembler _masm(&cbuf);
2141     __ mov64(r10, (int64_t) $meth$$method);
2142     __ call(r10);
2143   %}
2144 
2145   enc_class Java_To_Interpreter(method meth)
2146   %{
2147     // CALL Java_To_Interpreter
2148     // This is the instruction starting address for relocation info.
2149     cbuf.set_insts_mark();
2150     $$$emit8$primary;
2151     // CALL directly to the runtime
2152     emit_d32_reloc(cbuf,
2153                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2154                    runtime_call_Relocation::spec(),
2155                    RELOC_DISP32);
2156   %}
2157 
2158   enc_class Java_Static_Call(method meth)
2159   %{
2160     // JAVA STATIC CALL
2161     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2162     // determine who we intended to call.
2163     cbuf.set_insts_mark();
2164     $$$emit8$primary;
2165 
2166     if (!_method) {
2167       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2168                      runtime_call_Relocation::spec(),
2169                      RELOC_DISP32);
2170     } else {
2171       int method_index = resolved_method_index(cbuf);
2172       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2173                                                   : static_call_Relocation::spec(method_index);
2174       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2175                      rspec, RELOC_DISP32);
2176       // Emit stubs for static call.
2177       address mark = cbuf.insts_mark();
2178       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2179       if (stub == NULL) {
2180         ciEnv::current()->record_failure("CodeCache is full");
2181         return;
2182       }
2183 #if INCLUDE_AOT
2184       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2185 #endif
2186     }
2187   %}
2188 
2189   enc_class Java_Dynamic_Call(method meth) %{
2190     MacroAssembler _masm(&cbuf);
2191     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2192   %}
2193 
2194   enc_class Java_Compiled_Call(method meth)
2195   %{
2196     // JAVA COMPILED CALL
2197     int disp = in_bytes(Method:: from_compiled_offset());
2198 
2199     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2200     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2201 
2202     // callq *disp(%rax)
2203     cbuf.set_insts_mark();
2204     $$$emit8$primary;
2205     if (disp < 0x80) {
2206       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2207       emit_d8(cbuf, disp); // Displacement
2208     } else {
2209       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2210       emit_d32(cbuf, disp); // Displacement
2211     }
2212   %}
2213 
2214   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2215   %{
2216     // SAL, SAR, SHR
2217     int dstenc = $dst$$reg;
2218     if (dstenc >= 8) {
2219       emit_opcode(cbuf, Assembler::REX_B);
2220       dstenc -= 8;
2221     }
2222     $$$emit8$primary;
2223     emit_rm(cbuf, 0x3, $secondary, dstenc);
2224     $$$emit8$shift$$constant;
2225   %}
2226 
2227   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2228   %{
2229     // SAL, SAR, SHR
2230     int dstenc = $dst$$reg;
2231     if (dstenc < 8) {
2232       emit_opcode(cbuf, Assembler::REX_W);
2233     } else {
2234       emit_opcode(cbuf, Assembler::REX_WB);
2235       dstenc -= 8;
2236     }
2237     $$$emit8$primary;
2238     emit_rm(cbuf, 0x3, $secondary, dstenc);
2239     $$$emit8$shift$$constant;
2240   %}
2241 
2242   enc_class load_immI(rRegI dst, immI src)
2243   %{
2244     int dstenc = $dst$$reg;
2245     if (dstenc >= 8) {
2246       emit_opcode(cbuf, Assembler::REX_B);
2247       dstenc -= 8;
2248     }
2249     emit_opcode(cbuf, 0xB8 | dstenc);
2250     $$$emit32$src$$constant;
2251   %}
2252 
2253   enc_class load_immL(rRegL dst, immL src)
2254   %{
2255     int dstenc = $dst$$reg;
2256     if (dstenc < 8) {
2257       emit_opcode(cbuf, Assembler::REX_W);
2258     } else {
2259       emit_opcode(cbuf, Assembler::REX_WB);
2260       dstenc -= 8;
2261     }
2262     emit_opcode(cbuf, 0xB8 | dstenc);
2263     emit_d64(cbuf, $src$$constant);
2264   %}
2265 
2266   enc_class load_immUL32(rRegL dst, immUL32 src)
2267   %{
2268     // same as load_immI, but this time we care about zeroes in the high word
2269     int dstenc = $dst$$reg;
2270     if (dstenc >= 8) {
2271       emit_opcode(cbuf, Assembler::REX_B);
2272       dstenc -= 8;
2273     }
2274     emit_opcode(cbuf, 0xB8 | dstenc);
2275     $$$emit32$src$$constant;
2276   %}
2277 
2278   enc_class load_immL32(rRegL dst, immL32 src)
2279   %{
2280     int dstenc = $dst$$reg;
2281     if (dstenc < 8) {
2282       emit_opcode(cbuf, Assembler::REX_W);
2283     } else {
2284       emit_opcode(cbuf, Assembler::REX_WB);
2285       dstenc -= 8;
2286     }
2287     emit_opcode(cbuf, 0xC7);
2288     emit_rm(cbuf, 0x03, 0x00, dstenc);
2289     $$$emit32$src$$constant;
2290   %}
2291 
2292   enc_class load_immP31(rRegP dst, immP32 src)
2293   %{
2294     // same as load_immI, but this time we care about zeroes in the high word
2295     int dstenc = $dst$$reg;
2296     if (dstenc >= 8) {
2297       emit_opcode(cbuf, Assembler::REX_B);
2298       dstenc -= 8;
2299     }
2300     emit_opcode(cbuf, 0xB8 | dstenc);
2301     $$$emit32$src$$constant;
2302   %}
2303 
2304   enc_class load_immP(rRegP dst, immP src)
2305   %{
2306     int dstenc = $dst$$reg;
2307     if (dstenc < 8) {
2308       emit_opcode(cbuf, Assembler::REX_W);
2309     } else {
2310       emit_opcode(cbuf, Assembler::REX_WB);
2311       dstenc -= 8;
2312     }
2313     emit_opcode(cbuf, 0xB8 | dstenc);
2314     // This next line should be generated from ADLC
2315     if ($src->constant_reloc() != relocInfo::none) {
2316       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2317     } else {
2318       emit_d64(cbuf, $src$$constant);
2319     }
2320   %}
2321 
2322   enc_class Con32(immI src)
2323   %{
2324     // Output immediate
2325     $$$emit32$src$$constant;
2326   %}
2327 
2328   enc_class Con32F_as_bits(immF src)
2329   %{
2330     // Output Float immediate bits
2331     jfloat jf = $src$$constant;
2332     jint jf_as_bits = jint_cast(jf);
2333     emit_d32(cbuf, jf_as_bits);
2334   %}
2335 
2336   enc_class Con16(immI src)
2337   %{
2338     // Output immediate
2339     $$$emit16$src$$constant;
2340   %}
2341 
2342   // How is this different from Con32??? XXX
2343   enc_class Con_d32(immI src)
2344   %{
2345     emit_d32(cbuf,$src$$constant);
2346   %}
2347 
2348   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2349     // Output immediate memory reference
2350     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2351     emit_d32(cbuf, 0x00);
2352   %}
2353 
2354   enc_class lock_prefix()
2355   %{
2356     if (os::is_MP()) {
2357       emit_opcode(cbuf, 0xF0); // lock
2358     }
2359   %}
2360 
2361   enc_class REX_mem(memory mem)
2362   %{
2363     if ($mem$$base >= 8) {
2364       if ($mem$$index < 8) {
2365         emit_opcode(cbuf, Assembler::REX_B);
2366       } else {
2367         emit_opcode(cbuf, Assembler::REX_XB);
2368       }
2369     } else {
2370       if ($mem$$index >= 8) {
2371         emit_opcode(cbuf, Assembler::REX_X);
2372       }
2373     }
2374   %}
2375 
2376   enc_class REX_mem_wide(memory mem)
2377   %{
2378     if ($mem$$base >= 8) {
2379       if ($mem$$index < 8) {
2380         emit_opcode(cbuf, Assembler::REX_WB);
2381       } else {
2382         emit_opcode(cbuf, Assembler::REX_WXB);
2383       }
2384     } else {
2385       if ($mem$$index < 8) {
2386         emit_opcode(cbuf, Assembler::REX_W);
2387       } else {
2388         emit_opcode(cbuf, Assembler::REX_WX);
2389       }
2390     }
2391   %}
2392 
2393   // for byte regs
2394   enc_class REX_breg(rRegI reg)
2395   %{
2396     if ($reg$$reg >= 4) {
2397       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2398     }
2399   %}
2400 
2401   // for byte regs
2402   enc_class REX_reg_breg(rRegI dst, rRegI src)
2403   %{
2404     if ($dst$$reg < 8) {
2405       if ($src$$reg >= 4) {
2406         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2407       }
2408     } else {
2409       if ($src$$reg < 8) {
2410         emit_opcode(cbuf, Assembler::REX_R);
2411       } else {
2412         emit_opcode(cbuf, Assembler::REX_RB);
2413       }
2414     }
2415   %}
2416 
2417   // for byte regs
2418   enc_class REX_breg_mem(rRegI reg, memory mem)
2419   %{
2420     if ($reg$$reg < 8) {
2421       if ($mem$$base < 8) {
2422         if ($mem$$index >= 8) {
2423           emit_opcode(cbuf, Assembler::REX_X);
2424         } else if ($reg$$reg >= 4) {
2425           emit_opcode(cbuf, Assembler::REX);
2426         }
2427       } else {
2428         if ($mem$$index < 8) {
2429           emit_opcode(cbuf, Assembler::REX_B);
2430         } else {
2431           emit_opcode(cbuf, Assembler::REX_XB);
2432         }
2433       }
2434     } else {
2435       if ($mem$$base < 8) {
2436         if ($mem$$index < 8) {
2437           emit_opcode(cbuf, Assembler::REX_R);
2438         } else {
2439           emit_opcode(cbuf, Assembler::REX_RX);
2440         }
2441       } else {
2442         if ($mem$$index < 8) {
2443           emit_opcode(cbuf, Assembler::REX_RB);
2444         } else {
2445           emit_opcode(cbuf, Assembler::REX_RXB);
2446         }
2447       }
2448     }
2449   %}
2450 
2451   enc_class REX_reg(rRegI reg)
2452   %{
2453     if ($reg$$reg >= 8) {
2454       emit_opcode(cbuf, Assembler::REX_B);
2455     }
2456   %}
2457 
2458   enc_class REX_reg_wide(rRegI reg)
2459   %{
2460     if ($reg$$reg < 8) {
2461       emit_opcode(cbuf, Assembler::REX_W);
2462     } else {
2463       emit_opcode(cbuf, Assembler::REX_WB);
2464     }
2465   %}
2466 
2467   enc_class REX_reg_reg(rRegI dst, rRegI src)
2468   %{
2469     if ($dst$$reg < 8) {
2470       if ($src$$reg >= 8) {
2471         emit_opcode(cbuf, Assembler::REX_B);
2472       }
2473     } else {
2474       if ($src$$reg < 8) {
2475         emit_opcode(cbuf, Assembler::REX_R);
2476       } else {
2477         emit_opcode(cbuf, Assembler::REX_RB);
2478       }
2479     }
2480   %}
2481 
2482   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2483   %{
2484     if ($dst$$reg < 8) {
2485       if ($src$$reg < 8) {
2486         emit_opcode(cbuf, Assembler::REX_W);
2487       } else {
2488         emit_opcode(cbuf, Assembler::REX_WB);
2489       }
2490     } else {
2491       if ($src$$reg < 8) {
2492         emit_opcode(cbuf, Assembler::REX_WR);
2493       } else {
2494         emit_opcode(cbuf, Assembler::REX_WRB);
2495       }
2496     }
2497   %}
2498 
2499   enc_class REX_reg_mem(rRegI reg, memory mem)
2500   %{
2501     if ($reg$$reg < 8) {
2502       if ($mem$$base < 8) {
2503         if ($mem$$index >= 8) {
2504           emit_opcode(cbuf, Assembler::REX_X);
2505         }
2506       } else {
2507         if ($mem$$index < 8) {
2508           emit_opcode(cbuf, Assembler::REX_B);
2509         } else {
2510           emit_opcode(cbuf, Assembler::REX_XB);
2511         }
2512       }
2513     } else {
2514       if ($mem$$base < 8) {
2515         if ($mem$$index < 8) {
2516           emit_opcode(cbuf, Assembler::REX_R);
2517         } else {
2518           emit_opcode(cbuf, Assembler::REX_RX);
2519         }
2520       } else {
2521         if ($mem$$index < 8) {
2522           emit_opcode(cbuf, Assembler::REX_RB);
2523         } else {
2524           emit_opcode(cbuf, Assembler::REX_RXB);
2525         }
2526       }
2527     }
2528   %}
2529 
2530   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2531   %{
2532     if ($reg$$reg < 8) {
2533       if ($mem$$base < 8) {
2534         if ($mem$$index < 8) {
2535           emit_opcode(cbuf, Assembler::REX_W);
2536         } else {
2537           emit_opcode(cbuf, Assembler::REX_WX);
2538         }
2539       } else {
2540         if ($mem$$index < 8) {
2541           emit_opcode(cbuf, Assembler::REX_WB);
2542         } else {
2543           emit_opcode(cbuf, Assembler::REX_WXB);
2544         }
2545       }
2546     } else {
2547       if ($mem$$base < 8) {
2548         if ($mem$$index < 8) {
2549           emit_opcode(cbuf, Assembler::REX_WR);
2550         } else {
2551           emit_opcode(cbuf, Assembler::REX_WRX);
2552         }
2553       } else {
2554         if ($mem$$index < 8) {
2555           emit_opcode(cbuf, Assembler::REX_WRB);
2556         } else {
2557           emit_opcode(cbuf, Assembler::REX_WRXB);
2558         }
2559       }
2560     }
2561   %}
2562 
2563   enc_class reg_mem(rRegI ereg, memory mem)
2564   %{
2565     // High registers handle in encode_RegMem
2566     int reg = $ereg$$reg;
2567     int base = $mem$$base;
2568     int index = $mem$$index;
2569     int scale = $mem$$scale;
2570     int disp = $mem$$disp;
2571     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2572 
2573     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2574   %}
2575 
2576   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2577   %{
2578     int rm_byte_opcode = $rm_opcode$$constant;
2579 
2580     // High registers handle in encode_RegMem
2581     int base = $mem$$base;
2582     int index = $mem$$index;
2583     int scale = $mem$$scale;
2584     int displace = $mem$$disp;
2585 
2586     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2587                                             // working with static
2588                                             // globals
2589     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2590                   disp_reloc);
2591   %}
2592 
2593   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2594   %{
2595     int reg_encoding = $dst$$reg;
2596     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2597     int index        = 0x04;            // 0x04 indicates no index
2598     int scale        = 0x00;            // 0x00 indicates no scale
2599     int displace     = $src1$$constant; // 0x00 indicates no displacement
2600     relocInfo::relocType disp_reloc = relocInfo::none;
2601     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2602                   disp_reloc);
2603   %}
2604 
2605   enc_class neg_reg(rRegI dst)
2606   %{
2607     int dstenc = $dst$$reg;
2608     if (dstenc >= 8) {
2609       emit_opcode(cbuf, Assembler::REX_B);
2610       dstenc -= 8;
2611     }
2612     // NEG $dst
2613     emit_opcode(cbuf, 0xF7);
2614     emit_rm(cbuf, 0x3, 0x03, dstenc);
2615   %}
2616 
2617   enc_class neg_reg_wide(rRegI dst)
2618   %{
2619     int dstenc = $dst$$reg;
2620     if (dstenc < 8) {
2621       emit_opcode(cbuf, Assembler::REX_W);
2622     } else {
2623       emit_opcode(cbuf, Assembler::REX_WB);
2624       dstenc -= 8;
2625     }
2626     // NEG $dst
2627     emit_opcode(cbuf, 0xF7);
2628     emit_rm(cbuf, 0x3, 0x03, dstenc);
2629   %}
2630 
2631   enc_class setLT_reg(rRegI dst)
2632   %{
2633     int dstenc = $dst$$reg;
2634     if (dstenc >= 8) {
2635       emit_opcode(cbuf, Assembler::REX_B);
2636       dstenc -= 8;
2637     } else if (dstenc >= 4) {
2638       emit_opcode(cbuf, Assembler::REX);
2639     }
2640     // SETLT $dst
2641     emit_opcode(cbuf, 0x0F);
2642     emit_opcode(cbuf, 0x9C);
2643     emit_rm(cbuf, 0x3, 0x0, dstenc);
2644   %}
2645 
2646   enc_class setNZ_reg(rRegI dst)
2647   %{
2648     int dstenc = $dst$$reg;
2649     if (dstenc >= 8) {
2650       emit_opcode(cbuf, Assembler::REX_B);
2651       dstenc -= 8;
2652     } else if (dstenc >= 4) {
2653       emit_opcode(cbuf, Assembler::REX);
2654     }
2655     // SETNZ $dst
2656     emit_opcode(cbuf, 0x0F);
2657     emit_opcode(cbuf, 0x95);
2658     emit_rm(cbuf, 0x3, 0x0, dstenc);
2659   %}
2660 
2661 
2662   // Compare the lonogs and set -1, 0, or 1 into dst
2663   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2664   %{
2665     int src1enc = $src1$$reg;
2666     int src2enc = $src2$$reg;
2667     int dstenc = $dst$$reg;
2668 
2669     // cmpq $src1, $src2
2670     if (src1enc < 8) {
2671       if (src2enc < 8) {
2672         emit_opcode(cbuf, Assembler::REX_W);
2673       } else {
2674         emit_opcode(cbuf, Assembler::REX_WB);
2675       }
2676     } else {
2677       if (src2enc < 8) {
2678         emit_opcode(cbuf, Assembler::REX_WR);
2679       } else {
2680         emit_opcode(cbuf, Assembler::REX_WRB);
2681       }
2682     }
2683     emit_opcode(cbuf, 0x3B);
2684     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2685 
2686     // movl $dst, -1
2687     if (dstenc >= 8) {
2688       emit_opcode(cbuf, Assembler::REX_B);
2689     }
2690     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2691     emit_d32(cbuf, -1);
2692 
2693     // jl,s done
2694     emit_opcode(cbuf, 0x7C);
2695     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2696 
2697     // setne $dst
2698     if (dstenc >= 4) {
2699       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2700     }
2701     emit_opcode(cbuf, 0x0F);
2702     emit_opcode(cbuf, 0x95);
2703     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2704 
2705     // movzbl $dst, $dst
2706     if (dstenc >= 4) {
2707       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2708     }
2709     emit_opcode(cbuf, 0x0F);
2710     emit_opcode(cbuf, 0xB6);
2711     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2712   %}
2713 
2714   enc_class Push_ResultXD(regD dst) %{
2715     MacroAssembler _masm(&cbuf);
2716     __ fstp_d(Address(rsp, 0));
2717     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2718     __ addptr(rsp, 8);
2719   %}
2720 
2721   enc_class Push_SrcXD(regD src) %{
2722     MacroAssembler _masm(&cbuf);
2723     __ subptr(rsp, 8);
2724     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2725     __ fld_d(Address(rsp, 0));
2726   %}
2727 
2728 
2729   enc_class enc_rethrow()
2730   %{
2731     cbuf.set_insts_mark();
2732     emit_opcode(cbuf, 0xE9); // jmp entry
2733     emit_d32_reloc(cbuf,
2734                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2735                    runtime_call_Relocation::spec(),
2736                    RELOC_DISP32);
2737   %}
2738 
2739 %}
2740 
2741 
2742 
2743 //----------FRAME--------------------------------------------------------------
2744 // Definition of frame structure and management information.
2745 //
2746 //  S T A C K   L A Y O U T    Allocators stack-slot number
2747 //                             |   (to get allocators register number
2748 //  G  Owned by    |        |  v    add OptoReg::stack0())
2749 //  r   CALLER     |        |
2750 //  o     |        +--------+      pad to even-align allocators stack-slot
2751 //  w     V        |  pad0  |        numbers; owned by CALLER
2752 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2753 //  h     ^        |   in   |  5
2754 //        |        |  args  |  4   Holes in incoming args owned by SELF
2755 //  |     |        |        |  3
2756 //  |     |        +--------+
2757 //  V     |        | old out|      Empty on Intel, window on Sparc
2758 //        |    old |preserve|      Must be even aligned.
2759 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2760 //        |        |   in   |  3   area for Intel ret address
2761 //     Owned by    |preserve|      Empty on Sparc.
2762 //       SELF      +--------+
2763 //        |        |  pad2  |  2   pad to align old SP
2764 //        |        +--------+  1
2765 //        |        | locks  |  0
2766 //        |        +--------+----> OptoReg::stack0(), even aligned
2767 //        |        |  pad1  | 11   pad to align new SP
2768 //        |        +--------+
2769 //        |        |        | 10
2770 //        |        | spills |  9   spills
2771 //        V        |        |  8   (pad0 slot for callee)
2772 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2773 //        ^        |  out   |  7
2774 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2775 //     Owned by    +--------+
2776 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2777 //        |    new |preserve|      Must be even-aligned.
2778 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2779 //        |        |        |
2780 //
2781 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2782 //         known from SELF's arguments and the Java calling convention.
2783 //         Region 6-7 is determined per call site.
2784 // Note 2: If the calling convention leaves holes in the incoming argument
2785 //         area, those holes are owned by SELF.  Holes in the outgoing area
2786 //         are owned by the CALLEE.  Holes should not be nessecary in the
2787 //         incoming area, as the Java calling convention is completely under
2788 //         the control of the AD file.  Doubles can be sorted and packed to
2789 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2790 //         varargs C calling conventions.
2791 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2792 //         even aligned with pad0 as needed.
2793 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2794 //         region 6-11 is even aligned; it may be padded out more so that
2795 //         the region from SP to FP meets the minimum stack alignment.
2796 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2797 //         alignment.  Region 11, pad1, may be dynamically extended so that
2798 //         SP meets the minimum alignment.
2799 
2800 frame
2801 %{
2802   // What direction does stack grow in (assumed to be same for C & Java)
2803   stack_direction(TOWARDS_LOW);
2804 
2805   // These three registers define part of the calling convention
2806   // between compiled code and the interpreter.
2807   inline_cache_reg(RAX);                // Inline Cache Register
2808   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2809                                         // calling interpreter
2810 
2811   // Optional: name the operand used by cisc-spilling to access
2812   // [stack_pointer + offset]
2813   cisc_spilling_operand_name(indOffset32);
2814 
2815   // Number of stack slots consumed by locking an object
2816   sync_stack_slots(2);
2817 
2818   // Compiled code's Frame Pointer
2819   frame_pointer(RSP);
2820 
2821   // Interpreter stores its frame pointer in a register which is
2822   // stored to the stack by I2CAdaptors.
2823   // I2CAdaptors convert from interpreted java to compiled java.
2824   interpreter_frame_pointer(RBP);
2825 
2826   // Stack alignment requirement
2827   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2828 
2829   // Number of stack slots between incoming argument block and the start of
2830   // a new frame.  The PROLOG must add this many slots to the stack.  The
2831   // EPILOG must remove this many slots.  amd64 needs two slots for
2832   // return address.
2833   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2834 
2835   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2836   // for calls to C.  Supports the var-args backing area for register parms.
2837   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2838 
2839   // The after-PROLOG location of the return address.  Location of
2840   // return address specifies a type (REG or STACK) and a number
2841   // representing the register number (i.e. - use a register name) or
2842   // stack slot.
2843   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2844   // Otherwise, it is above the locks and verification slot and alignment word
2845   return_addr(STACK - 2 +
2846               align_up((Compile::current()->in_preserve_stack_slots() +
2847                         Compile::current()->fixed_slots()),
2848                        stack_alignment_in_slots()));
2849 
2850   // Body of function which returns an integer array locating
2851   // arguments either in registers or in stack slots.  Passed an array
2852   // of ideal registers called "sig" and a "length" count.  Stack-slot
2853   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2854   // arguments for a CALLEE.  Incoming stack arguments are
2855   // automatically biased by the preserve_stack_slots field above.
2856 
2857   calling_convention
2858   %{
2859     // No difference between ingoing/outgoing just pass false
2860     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2861   %}
2862 
2863   c_calling_convention
2864   %{
2865     // This is obviously always outgoing
2866     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2867   %}
2868 
2869   // Location of compiled Java return values.  Same as C for now.
2870   return_value
2871   %{
2872     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2873            "only return normal values");
2874 
2875     static const int lo[Op_RegL + 1] = {
2876       0,
2877       0,
2878       RAX_num,  // Op_RegN
2879       RAX_num,  // Op_RegI
2880       RAX_num,  // Op_RegP
2881       XMM0_num, // Op_RegF
2882       XMM0_num, // Op_RegD
2883       RAX_num   // Op_RegL
2884     };
2885     static const int hi[Op_RegL + 1] = {
2886       0,
2887       0,
2888       OptoReg::Bad, // Op_RegN
2889       OptoReg::Bad, // Op_RegI
2890       RAX_H_num,    // Op_RegP
2891       OptoReg::Bad, // Op_RegF
2892       XMM0b_num,    // Op_RegD
2893       RAX_H_num     // Op_RegL
2894     };
2895     // Excluded flags and vector registers.
2896     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2897     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2898   %}
2899 %}
2900 
2901 //----------ATTRIBUTES---------------------------------------------------------
2902 //----------Operand Attributes-------------------------------------------------
2903 op_attrib op_cost(0);        // Required cost attribute
2904 
2905 //----------Instruction Attributes---------------------------------------------
2906 ins_attrib ins_cost(100);       // Required cost attribute
2907 ins_attrib ins_size(8);         // Required size attribute (in bits)
2908 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2909                                 // a non-matching short branch variant
2910                                 // of some long branch?
2911 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2912                                 // be a power of 2) specifies the
2913                                 // alignment that some part of the
2914                                 // instruction (not necessarily the
2915                                 // start) requires.  If > 1, a
2916                                 // compute_padding() function must be
2917                                 // provided for the instruction
2918 
2919 //----------OPERANDS-----------------------------------------------------------
2920 // Operand definitions must precede instruction definitions for correct parsing
2921 // in the ADLC because operands constitute user defined types which are used in
2922 // instruction definitions.
2923 
2924 //----------Simple Operands----------------------------------------------------
2925 // Immediate Operands
2926 // Integer Immediate
2927 operand immI()
2928 %{
2929   match(ConI);
2930 
2931   op_cost(10);
2932   format %{ %}
2933   interface(CONST_INTER);
2934 %}
2935 
2936 // Constant for test vs zero
2937 operand immI0()
2938 %{
2939   predicate(n->get_int() == 0);
2940   match(ConI);
2941 
2942   op_cost(0);
2943   format %{ %}
2944   interface(CONST_INTER);
2945 %}
2946 
2947 // Constant for increment
2948 operand immI1()
2949 %{
2950   predicate(n->get_int() == 1);
2951   match(ConI);
2952 
2953   op_cost(0);
2954   format %{ %}
2955   interface(CONST_INTER);
2956 %}
2957 
2958 // Constant for decrement
2959 operand immI_M1()
2960 %{
2961   predicate(n->get_int() == -1);
2962   match(ConI);
2963 
2964   op_cost(0);
2965   format %{ %}
2966   interface(CONST_INTER);
2967 %}
2968 
2969 // Valid scale values for addressing modes
2970 operand immI2()
2971 %{
2972   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2973   match(ConI);
2974 
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 operand immI8()
2980 %{
2981   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2982   match(ConI);
2983 
2984   op_cost(5);
2985   format %{ %}
2986   interface(CONST_INTER);
2987 %}
2988 
2989 operand immU8()
2990 %{
2991   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2992   match(ConI);
2993 
2994   op_cost(5);
2995   format %{ %}
2996   interface(CONST_INTER);
2997 %}
2998 
2999 operand immI16()
3000 %{
3001   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
3002   match(ConI);
3003 
3004   op_cost(10);
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 // Int Immediate non-negative
3010 operand immU31()
3011 %{
3012   predicate(n->get_int() >= 0);
3013   match(ConI);
3014 
3015   op_cost(0);
3016   format %{ %}
3017   interface(CONST_INTER);
3018 %}
3019 
3020 // Constant for long shifts
3021 operand immI_32()
3022 %{
3023   predicate( n->get_int() == 32 );
3024   match(ConI);
3025 
3026   op_cost(0);
3027   format %{ %}
3028   interface(CONST_INTER);
3029 %}
3030 
3031 // Constant for long shifts
3032 operand immI_64()
3033 %{
3034   predicate( n->get_int() == 64 );
3035   match(ConI);
3036 
3037   op_cost(0);
3038   format %{ %}
3039   interface(CONST_INTER);
3040 %}
3041 
3042 // Pointer Immediate
3043 operand immP()
3044 %{
3045   match(ConP);
3046 
3047   op_cost(10);
3048   format %{ %}
3049   interface(CONST_INTER);
3050 %}
3051 
3052 // NULL Pointer Immediate
3053 operand immP0()
3054 %{
3055   predicate(n->get_ptr() == 0);
3056   match(ConP);
3057 
3058   op_cost(5);
3059   format %{ %}
3060   interface(CONST_INTER);
3061 %}
3062 
3063 // Pointer Immediate
3064 operand immN() %{
3065   match(ConN);
3066 
3067   op_cost(10);
3068   format %{ %}
3069   interface(CONST_INTER);
3070 %}
3071 
3072 operand immNKlass() %{
3073   match(ConNKlass);
3074 
3075   op_cost(10);
3076   format %{ %}
3077   interface(CONST_INTER);
3078 %}
3079 
3080 // NULL Pointer Immediate
3081 operand immN0() %{
3082   predicate(n->get_narrowcon() == 0);
3083   match(ConN);
3084 
3085   op_cost(5);
3086   format %{ %}
3087   interface(CONST_INTER);
3088 %}
3089 
3090 operand immP31()
3091 %{
3092   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3093             && (n->get_ptr() >> 31) == 0);
3094   match(ConP);
3095 
3096   op_cost(5);
3097   format %{ %}
3098   interface(CONST_INTER);
3099 %}
3100 
3101 
3102 // Long Immediate
3103 operand immL()
3104 %{
3105   match(ConL);
3106 
3107   op_cost(20);
3108   format %{ %}
3109   interface(CONST_INTER);
3110 %}
3111 
3112 // Long Immediate 8-bit
3113 operand immL8()
3114 %{
3115   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3116   match(ConL);
3117 
3118   op_cost(5);
3119   format %{ %}
3120   interface(CONST_INTER);
3121 %}
3122 
3123 // Long Immediate 32-bit unsigned
3124 operand immUL32()
3125 %{
3126   predicate(n->get_long() == (unsigned int) (n->get_long()));
3127   match(ConL);
3128 
3129   op_cost(10);
3130   format %{ %}
3131   interface(CONST_INTER);
3132 %}
3133 
3134 // Long Immediate 32-bit signed
3135 operand immL32()
3136 %{
3137   predicate(n->get_long() == (int) (n->get_long()));
3138   match(ConL);
3139 
3140   op_cost(15);
3141   format %{ %}
3142   interface(CONST_INTER);
3143 %}
3144 
3145 // Long Immediate zero
3146 operand immL0()
3147 %{
3148   predicate(n->get_long() == 0L);
3149   match(ConL);
3150 
3151   op_cost(10);
3152   format %{ %}
3153   interface(CONST_INTER);
3154 %}
3155 
3156 // Constant for increment
3157 operand immL1()
3158 %{
3159   predicate(n->get_long() == 1);
3160   match(ConL);
3161 
3162   format %{ %}
3163   interface(CONST_INTER);
3164 %}
3165 
3166 // Constant for decrement
3167 operand immL_M1()
3168 %{
3169   predicate(n->get_long() == -1);
3170   match(ConL);
3171 
3172   format %{ %}
3173   interface(CONST_INTER);
3174 %}
3175 
3176 // Long Immediate: the value 10
3177 operand immL10()
3178 %{
3179   predicate(n->get_long() == 10);
3180   match(ConL);
3181 
3182   format %{ %}
3183   interface(CONST_INTER);
3184 %}
3185 
3186 // Long immediate from 0 to 127.
3187 // Used for a shorter form of long mul by 10.
3188 operand immL_127()
3189 %{
3190   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3191   match(ConL);
3192 
3193   op_cost(10);
3194   format %{ %}
3195   interface(CONST_INTER);
3196 %}
3197 
3198 // Long Immediate: low 32-bit mask
3199 operand immL_32bits()
3200 %{
3201   predicate(n->get_long() == 0xFFFFFFFFL);
3202   match(ConL);
3203   op_cost(20);
3204 
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 // Float Immediate zero
3210 operand immF0()
3211 %{
3212   predicate(jint_cast(n->getf()) == 0);
3213   match(ConF);
3214 
3215   op_cost(5);
3216   format %{ %}
3217   interface(CONST_INTER);
3218 %}
3219 
3220 // Float Immediate
3221 operand immF()
3222 %{
3223   match(ConF);
3224 
3225   op_cost(15);
3226   format %{ %}
3227   interface(CONST_INTER);
3228 %}
3229 
3230 // Double Immediate zero
3231 operand immD0()
3232 %{
3233   predicate(jlong_cast(n->getd()) == 0);
3234   match(ConD);
3235 
3236   op_cost(5);
3237   format %{ %}
3238   interface(CONST_INTER);
3239 %}
3240 
3241 // Double Immediate
3242 operand immD()
3243 %{
3244   match(ConD);
3245 
3246   op_cost(15);
3247   format %{ %}
3248   interface(CONST_INTER);
3249 %}
3250 
3251 // Immediates for special shifts (sign extend)
3252 
3253 // Constants for increment
3254 operand immI_16()
3255 %{
3256   predicate(n->get_int() == 16);
3257   match(ConI);
3258 
3259   format %{ %}
3260   interface(CONST_INTER);
3261 %}
3262 
3263 operand immI_24()
3264 %{
3265   predicate(n->get_int() == 24);
3266   match(ConI);
3267 
3268   format %{ %}
3269   interface(CONST_INTER);
3270 %}
3271 
3272 // Constant for byte-wide masking
3273 operand immI_255()
3274 %{
3275   predicate(n->get_int() == 255);
3276   match(ConI);
3277 
3278   format %{ %}
3279   interface(CONST_INTER);
3280 %}
3281 
3282 // Constant for short-wide masking
3283 operand immI_65535()
3284 %{
3285   predicate(n->get_int() == 65535);
3286   match(ConI);
3287 
3288   format %{ %}
3289   interface(CONST_INTER);
3290 %}
3291 
3292 // Constant for byte-wide masking
3293 operand immL_255()
3294 %{
3295   predicate(n->get_long() == 255);
3296   match(ConL);
3297 
3298   format %{ %}
3299   interface(CONST_INTER);
3300 %}
3301 
3302 // Constant for short-wide masking
3303 operand immL_65535()
3304 %{
3305   predicate(n->get_long() == 65535);
3306   match(ConL);
3307 
3308   format %{ %}
3309   interface(CONST_INTER);
3310 %}
3311 
3312 // Register Operands
3313 // Integer Register
3314 operand rRegI()
3315 %{
3316   constraint(ALLOC_IN_RC(int_reg));
3317   match(RegI);
3318 
3319   match(rax_RegI);
3320   match(rbx_RegI);
3321   match(rcx_RegI);
3322   match(rdx_RegI);
3323   match(rdi_RegI);
3324 
3325   format %{ %}
3326   interface(REG_INTER);
3327 %}
3328 
3329 // Special Registers
3330 operand rax_RegI()
3331 %{
3332   constraint(ALLOC_IN_RC(int_rax_reg));
3333   match(RegI);
3334   match(rRegI);
3335 
3336   format %{ "RAX" %}
3337   interface(REG_INTER);
3338 %}
3339 
3340 // Special Registers
3341 operand rbx_RegI()
3342 %{
3343   constraint(ALLOC_IN_RC(int_rbx_reg));
3344   match(RegI);
3345   match(rRegI);
3346 
3347   format %{ "RBX" %}
3348   interface(REG_INTER);
3349 %}
3350 
3351 operand rcx_RegI()
3352 %{
3353   constraint(ALLOC_IN_RC(int_rcx_reg));
3354   match(RegI);
3355   match(rRegI);
3356 
3357   format %{ "RCX" %}
3358   interface(REG_INTER);
3359 %}
3360 
3361 operand rdx_RegI()
3362 %{
3363   constraint(ALLOC_IN_RC(int_rdx_reg));
3364   match(RegI);
3365   match(rRegI);
3366 
3367   format %{ "RDX" %}
3368   interface(REG_INTER);
3369 %}
3370 
3371 operand rdi_RegI()
3372 %{
3373   constraint(ALLOC_IN_RC(int_rdi_reg));
3374   match(RegI);
3375   match(rRegI);
3376 
3377   format %{ "RDI" %}
3378   interface(REG_INTER);
3379 %}
3380 
3381 operand no_rcx_RegI()
3382 %{
3383   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3384   match(RegI);
3385   match(rax_RegI);
3386   match(rbx_RegI);
3387   match(rdx_RegI);
3388   match(rdi_RegI);
3389 
3390   format %{ %}
3391   interface(REG_INTER);
3392 %}
3393 
3394 operand no_rax_rdx_RegI()
3395 %{
3396   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3397   match(RegI);
3398   match(rbx_RegI);
3399   match(rcx_RegI);
3400   match(rdi_RegI);
3401 
3402   format %{ %}
3403   interface(REG_INTER);
3404 %}
3405 
3406 // Pointer Register
3407 operand any_RegP()
3408 %{
3409   constraint(ALLOC_IN_RC(any_reg));
3410   match(RegP);
3411   match(rax_RegP);
3412   match(rbx_RegP);
3413   match(rdi_RegP);
3414   match(rsi_RegP);
3415   match(rbp_RegP);
3416   match(r15_RegP);
3417   match(rRegP);
3418 
3419   format %{ %}
3420   interface(REG_INTER);
3421 %}
3422 
3423 operand rRegP()
3424 %{
3425   constraint(ALLOC_IN_RC(ptr_reg));
3426   match(RegP);
3427   match(rax_RegP);
3428   match(rbx_RegP);
3429   match(rdi_RegP);
3430   match(rsi_RegP);
3431   match(rbp_RegP);  // See Q&A below about
3432   match(r15_RegP);  // r15_RegP and rbp_RegP.
3433 
3434   format %{ %}
3435   interface(REG_INTER);
3436 %}
3437 
3438 operand rRegN() %{
3439   constraint(ALLOC_IN_RC(int_reg));
3440   match(RegN);
3441 
3442   format %{ %}
3443   interface(REG_INTER);
3444 %}
3445 
3446 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3447 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3448 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3449 // The output of an instruction is controlled by the allocator, which respects
3450 // register class masks, not match rules.  Unless an instruction mentions
3451 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3452 // by the allocator as an input.
3453 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3454 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3455 // result, RBP is not included in the output of the instruction either.
3456 
3457 operand no_rax_RegP()
3458 %{
3459   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3460   match(RegP);
3461   match(rbx_RegP);
3462   match(rsi_RegP);
3463   match(rdi_RegP);
3464 
3465   format %{ %}
3466   interface(REG_INTER);
3467 %}
3468 
3469 // This operand is not allowed to use RBP even if
3470 // RBP is not used to hold the frame pointer.
3471 operand no_rbp_RegP()
3472 %{
3473   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3474   match(RegP);
3475   match(rbx_RegP);
3476   match(rsi_RegP);
3477   match(rdi_RegP);
3478 
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 operand no_rax_rbx_RegP()
3484 %{
3485   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3486   match(RegP);
3487   match(rsi_RegP);
3488   match(rdi_RegP);
3489 
3490   format %{ %}
3491   interface(REG_INTER);
3492 %}
3493 
3494 // Special Registers
3495 // Return a pointer value
3496 operand rax_RegP()
3497 %{
3498   constraint(ALLOC_IN_RC(ptr_rax_reg));
3499   match(RegP);
3500   match(rRegP);
3501 
3502   format %{ %}
3503   interface(REG_INTER);
3504 %}
3505 
3506 // Special Registers
3507 // Return a compressed pointer value
3508 operand rax_RegN()
3509 %{
3510   constraint(ALLOC_IN_RC(int_rax_reg));
3511   match(RegN);
3512   match(rRegN);
3513 
3514   format %{ %}
3515   interface(REG_INTER);
3516 %}
3517 
3518 // Used in AtomicAdd
3519 operand rbx_RegP()
3520 %{
3521   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3522   match(RegP);
3523   match(rRegP);
3524 
3525   format %{ %}
3526   interface(REG_INTER);
3527 %}
3528 
3529 operand rsi_RegP()
3530 %{
3531   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3532   match(RegP);
3533   match(rRegP);
3534 
3535   format %{ %}
3536   interface(REG_INTER);
3537 %}
3538 
3539 // Used in rep stosq
3540 operand rdi_RegP()
3541 %{
3542   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3543   match(RegP);
3544   match(rRegP);
3545 
3546   format %{ %}
3547   interface(REG_INTER);
3548 %}
3549 
3550 operand r15_RegP()
3551 %{
3552   constraint(ALLOC_IN_RC(ptr_r15_reg));
3553   match(RegP);
3554   match(rRegP);
3555 
3556   format %{ %}
3557   interface(REG_INTER);
3558 %}
3559 
3560 operand rex_RegP()
3561 %{
3562   constraint(ALLOC_IN_RC(ptr_rex_reg));
3563   match(RegP);
3564   match(rRegP);
3565 
3566   format %{ %}
3567   interface(REG_INTER);
3568 %}
3569 
3570 operand rRegL()
3571 %{
3572   constraint(ALLOC_IN_RC(long_reg));
3573   match(RegL);
3574   match(rax_RegL);
3575   match(rdx_RegL);
3576 
3577   format %{ %}
3578   interface(REG_INTER);
3579 %}
3580 
3581 // Special Registers
3582 operand no_rax_rdx_RegL()
3583 %{
3584   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3585   match(RegL);
3586   match(rRegL);
3587 
3588   format %{ %}
3589   interface(REG_INTER);
3590 %}
3591 
3592 operand no_rax_RegL()
3593 %{
3594   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3595   match(RegL);
3596   match(rRegL);
3597   match(rdx_RegL);
3598 
3599   format %{ %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 operand no_rcx_RegL()
3604 %{
3605   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3606   match(RegL);
3607   match(rRegL);
3608 
3609   format %{ %}
3610   interface(REG_INTER);
3611 %}
3612 
3613 operand rax_RegL()
3614 %{
3615   constraint(ALLOC_IN_RC(long_rax_reg));
3616   match(RegL);
3617   match(rRegL);
3618 
3619   format %{ "RAX" %}
3620   interface(REG_INTER);
3621 %}
3622 
3623 operand rcx_RegL()
3624 %{
3625   constraint(ALLOC_IN_RC(long_rcx_reg));
3626   match(RegL);
3627   match(rRegL);
3628 
3629   format %{ %}
3630   interface(REG_INTER);
3631 %}
3632 
3633 operand rdx_RegL()
3634 %{
3635   constraint(ALLOC_IN_RC(long_rdx_reg));
3636   match(RegL);
3637   match(rRegL);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 // Flags register, used as output of compare instructions
3644 operand rFlagsReg()
3645 %{
3646   constraint(ALLOC_IN_RC(int_flags));
3647   match(RegFlags);
3648 
3649   format %{ "RFLAGS" %}
3650   interface(REG_INTER);
3651 %}
3652 
3653 // Flags register, used as output of FLOATING POINT compare instructions
3654 operand rFlagsRegU()
3655 %{
3656   constraint(ALLOC_IN_RC(int_flags));
3657   match(RegFlags);
3658 
3659   format %{ "RFLAGS_U" %}
3660   interface(REG_INTER);
3661 %}
3662 
3663 operand rFlagsRegUCF() %{
3664   constraint(ALLOC_IN_RC(int_flags));
3665   match(RegFlags);
3666   predicate(false);
3667 
3668   format %{ "RFLAGS_U_CF" %}
3669   interface(REG_INTER);
3670 %}
3671 
3672 // Float register operands
3673 operand regF() %{
3674    constraint(ALLOC_IN_RC(float_reg));
3675    match(RegF);
3676 
3677    format %{ %}
3678    interface(REG_INTER);
3679 %}
3680 
3681 // Double register operands
3682 operand regD() %{
3683    constraint(ALLOC_IN_RC(double_reg));
3684    match(RegD);
3685 
3686    format %{ %}
3687    interface(REG_INTER);
3688 %}
3689 
3690 // Vectors
3691 operand vecS() %{
3692   constraint(ALLOC_IN_RC(vectors_reg));
3693   match(VecS);
3694 
3695   format %{ %}
3696   interface(REG_INTER);
3697 %}
3698 
3699 operand vecD() %{
3700   constraint(ALLOC_IN_RC(vectord_reg));
3701   match(VecD);
3702 
3703   format %{ %}
3704   interface(REG_INTER);
3705 %}
3706 
3707 operand vecX() %{
3708   constraint(ALLOC_IN_RC(vectorx_reg));
3709   match(VecX);
3710 
3711   format %{ %}
3712   interface(REG_INTER);
3713 %}
3714 
3715 operand vecY() %{
3716   constraint(ALLOC_IN_RC(vectory_reg));
3717   match(VecY);
3718 
3719   format %{ %}
3720   interface(REG_INTER);
3721 %}
3722 
3723 //----------Memory Operands----------------------------------------------------
3724 // Direct Memory Operand
3725 // operand direct(immP addr)
3726 // %{
3727 //   match(addr);
3728 
3729 //   format %{ "[$addr]" %}
3730 //   interface(MEMORY_INTER) %{
3731 //     base(0xFFFFFFFF);
3732 //     index(0x4);
3733 //     scale(0x0);
3734 //     disp($addr);
3735 //   %}
3736 // %}
3737 
3738 // Indirect Memory Operand
3739 operand indirect(any_RegP reg)
3740 %{
3741   constraint(ALLOC_IN_RC(ptr_reg));
3742   match(reg);
3743 
3744   format %{ "[$reg]" %}
3745   interface(MEMORY_INTER) %{
3746     base($reg);
3747     index(0x4);
3748     scale(0x0);
3749     disp(0x0);
3750   %}
3751 %}
3752 
3753 // Indirect Memory Plus Short Offset Operand
3754 operand indOffset8(any_RegP reg, immL8 off)
3755 %{
3756   constraint(ALLOC_IN_RC(ptr_reg));
3757   match(AddP reg off);
3758 
3759   format %{ "[$reg + $off (8-bit)]" %}
3760   interface(MEMORY_INTER) %{
3761     base($reg);
3762     index(0x4);
3763     scale(0x0);
3764     disp($off);
3765   %}
3766 %}
3767 
3768 // Indirect Memory Plus Long Offset Operand
3769 operand indOffset32(any_RegP reg, immL32 off)
3770 %{
3771   constraint(ALLOC_IN_RC(ptr_reg));
3772   match(AddP reg off);
3773 
3774   format %{ "[$reg + $off (32-bit)]" %}
3775   interface(MEMORY_INTER) %{
3776     base($reg);
3777     index(0x4);
3778     scale(0x0);
3779     disp($off);
3780   %}
3781 %}
3782 
3783 // Indirect Memory Plus Index Register Plus Offset Operand
3784 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3785 %{
3786   constraint(ALLOC_IN_RC(ptr_reg));
3787   match(AddP (AddP reg lreg) off);
3788 
3789   op_cost(10);
3790   format %{"[$reg + $off + $lreg]" %}
3791   interface(MEMORY_INTER) %{
3792     base($reg);
3793     index($lreg);
3794     scale(0x0);
3795     disp($off);
3796   %}
3797 %}
3798 
3799 // Indirect Memory Plus Index Register Plus Offset Operand
3800 operand indIndex(any_RegP reg, rRegL lreg)
3801 %{
3802   constraint(ALLOC_IN_RC(ptr_reg));
3803   match(AddP reg lreg);
3804 
3805   op_cost(10);
3806   format %{"[$reg + $lreg]" %}
3807   interface(MEMORY_INTER) %{
3808     base($reg);
3809     index($lreg);
3810     scale(0x0);
3811     disp(0x0);
3812   %}
3813 %}
3814 
3815 // Indirect Memory Times Scale Plus Index Register
3816 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3817 %{
3818   constraint(ALLOC_IN_RC(ptr_reg));
3819   match(AddP reg (LShiftL lreg scale));
3820 
3821   op_cost(10);
3822   format %{"[$reg + $lreg << $scale]" %}
3823   interface(MEMORY_INTER) %{
3824     base($reg);
3825     index($lreg);
3826     scale($scale);
3827     disp(0x0);
3828   %}
3829 %}
3830 
3831 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3832 %{
3833   constraint(ALLOC_IN_RC(ptr_reg));
3834   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3835   match(AddP reg (LShiftL (ConvI2L idx) scale));
3836 
3837   op_cost(10);
3838   format %{"[$reg + pos $idx << $scale]" %}
3839   interface(MEMORY_INTER) %{
3840     base($reg);
3841     index($idx);
3842     scale($scale);
3843     disp(0x0);
3844   %}
3845 %}
3846 
3847 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3848 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3849 %{
3850   constraint(ALLOC_IN_RC(ptr_reg));
3851   match(AddP (AddP reg (LShiftL lreg scale)) off);
3852 
3853   op_cost(10);
3854   format %{"[$reg + $off + $lreg << $scale]" %}
3855   interface(MEMORY_INTER) %{
3856     base($reg);
3857     index($lreg);
3858     scale($scale);
3859     disp($off);
3860   %}
3861 %}
3862 
3863 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3864 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3865 %{
3866   constraint(ALLOC_IN_RC(ptr_reg));
3867   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3868   match(AddP (AddP reg (ConvI2L idx)) off);
3869 
3870   op_cost(10);
3871   format %{"[$reg + $off + $idx]" %}
3872   interface(MEMORY_INTER) %{
3873     base($reg);
3874     index($idx);
3875     scale(0x0);
3876     disp($off);
3877   %}
3878 %}
3879 
3880 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3881 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3882 %{
3883   constraint(ALLOC_IN_RC(ptr_reg));
3884   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3885   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3886 
3887   op_cost(10);
3888   format %{"[$reg + $off + $idx << $scale]" %}
3889   interface(MEMORY_INTER) %{
3890     base($reg);
3891     index($idx);
3892     scale($scale);
3893     disp($off);
3894   %}
3895 %}
3896 
3897 // Indirect Narrow Oop Plus Offset Operand
3898 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3899 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3900 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3901   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3902   constraint(ALLOC_IN_RC(ptr_reg));
3903   match(AddP (DecodeN reg) off);
3904 
3905   op_cost(10);
3906   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3907   interface(MEMORY_INTER) %{
3908     base(0xc); // R12
3909     index($reg);
3910     scale(0x3);
3911     disp($off);
3912   %}
3913 %}
3914 
3915 // Indirect Memory Operand
3916 operand indirectNarrow(rRegN reg)
3917 %{
3918   predicate(Universe::narrow_oop_shift() == 0);
3919   constraint(ALLOC_IN_RC(ptr_reg));
3920   match(DecodeN reg);
3921 
3922   format %{ "[$reg]" %}
3923   interface(MEMORY_INTER) %{
3924     base($reg);
3925     index(0x4);
3926     scale(0x0);
3927     disp(0x0);
3928   %}
3929 %}
3930 
3931 // Indirect Memory Plus Short Offset Operand
3932 operand indOffset8Narrow(rRegN reg, immL8 off)
3933 %{
3934   predicate(Universe::narrow_oop_shift() == 0);
3935   constraint(ALLOC_IN_RC(ptr_reg));
3936   match(AddP (DecodeN reg) off);
3937 
3938   format %{ "[$reg + $off (8-bit)]" %}
3939   interface(MEMORY_INTER) %{
3940     base($reg);
3941     index(0x4);
3942     scale(0x0);
3943     disp($off);
3944   %}
3945 %}
3946 
3947 // Indirect Memory Plus Long Offset Operand
3948 operand indOffset32Narrow(rRegN reg, immL32 off)
3949 %{
3950   predicate(Universe::narrow_oop_shift() == 0);
3951   constraint(ALLOC_IN_RC(ptr_reg));
3952   match(AddP (DecodeN reg) off);
3953 
3954   format %{ "[$reg + $off (32-bit)]" %}
3955   interface(MEMORY_INTER) %{
3956     base($reg);
3957     index(0x4);
3958     scale(0x0);
3959     disp($off);
3960   %}
3961 %}
3962 
3963 // Indirect Memory Plus Index Register Plus Offset Operand
3964 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3965 %{
3966   predicate(Universe::narrow_oop_shift() == 0);
3967   constraint(ALLOC_IN_RC(ptr_reg));
3968   match(AddP (AddP (DecodeN reg) lreg) off);
3969 
3970   op_cost(10);
3971   format %{"[$reg + $off + $lreg]" %}
3972   interface(MEMORY_INTER) %{
3973     base($reg);
3974     index($lreg);
3975     scale(0x0);
3976     disp($off);
3977   %}
3978 %}
3979 
3980 // Indirect Memory Plus Index Register Plus Offset Operand
3981 operand indIndexNarrow(rRegN reg, rRegL lreg)
3982 %{
3983   predicate(Universe::narrow_oop_shift() == 0);
3984   constraint(ALLOC_IN_RC(ptr_reg));
3985   match(AddP (DecodeN reg) lreg);
3986 
3987   op_cost(10);
3988   format %{"[$reg + $lreg]" %}
3989   interface(MEMORY_INTER) %{
3990     base($reg);
3991     index($lreg);
3992     scale(0x0);
3993     disp(0x0);
3994   %}
3995 %}
3996 
3997 // Indirect Memory Times Scale Plus Index Register
3998 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3999 %{
4000   predicate(Universe::narrow_oop_shift() == 0);
4001   constraint(ALLOC_IN_RC(ptr_reg));
4002   match(AddP (DecodeN reg) (LShiftL lreg scale));
4003 
4004   op_cost(10);
4005   format %{"[$reg + $lreg << $scale]" %}
4006   interface(MEMORY_INTER) %{
4007     base($reg);
4008     index($lreg);
4009     scale($scale);
4010     disp(0x0);
4011   %}
4012 %}
4013 
4014 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4015 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4016 %{
4017   predicate(Universe::narrow_oop_shift() == 0);
4018   constraint(ALLOC_IN_RC(ptr_reg));
4019   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4020 
4021   op_cost(10);
4022   format %{"[$reg + $off + $lreg << $scale]" %}
4023   interface(MEMORY_INTER) %{
4024     base($reg);
4025     index($lreg);
4026     scale($scale);
4027     disp($off);
4028   %}
4029 %}
4030 
4031 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4032 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4033 %{
4034   constraint(ALLOC_IN_RC(ptr_reg));
4035   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4036   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4037 
4038   op_cost(10);
4039   format %{"[$reg + $off + $idx]" %}
4040   interface(MEMORY_INTER) %{
4041     base($reg);
4042     index($idx);
4043     scale(0x0);
4044     disp($off);
4045   %}
4046 %}
4047 
4048 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4049 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4050 %{
4051   constraint(ALLOC_IN_RC(ptr_reg));
4052   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4053   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4054 
4055   op_cost(10);
4056   format %{"[$reg + $off + $idx << $scale]" %}
4057   interface(MEMORY_INTER) %{
4058     base($reg);
4059     index($idx);
4060     scale($scale);
4061     disp($off);
4062   %}
4063 %}
4064 
4065 //----------Special Memory Operands--------------------------------------------
4066 // Stack Slot Operand - This operand is used for loading and storing temporary
4067 //                      values on the stack where a match requires a value to
4068 //                      flow through memory.
4069 operand stackSlotP(sRegP reg)
4070 %{
4071   constraint(ALLOC_IN_RC(stack_slots));
4072   // No match rule because this operand is only generated in matching
4073 
4074   format %{ "[$reg]" %}
4075   interface(MEMORY_INTER) %{
4076     base(0x4);   // RSP
4077     index(0x4);  // No Index
4078     scale(0x0);  // No Scale
4079     disp($reg);  // Stack Offset
4080   %}
4081 %}
4082 
4083 operand stackSlotI(sRegI reg)
4084 %{
4085   constraint(ALLOC_IN_RC(stack_slots));
4086   // No match rule because this operand is only generated in matching
4087 
4088   format %{ "[$reg]" %}
4089   interface(MEMORY_INTER) %{
4090     base(0x4);   // RSP
4091     index(0x4);  // No Index
4092     scale(0x0);  // No Scale
4093     disp($reg);  // Stack Offset
4094   %}
4095 %}
4096 
4097 operand stackSlotF(sRegF reg)
4098 %{
4099   constraint(ALLOC_IN_RC(stack_slots));
4100   // No match rule because this operand is only generated in matching
4101 
4102   format %{ "[$reg]" %}
4103   interface(MEMORY_INTER) %{
4104     base(0x4);   // RSP
4105     index(0x4);  // No Index
4106     scale(0x0);  // No Scale
4107     disp($reg);  // Stack Offset
4108   %}
4109 %}
4110 
4111 operand stackSlotD(sRegD reg)
4112 %{
4113   constraint(ALLOC_IN_RC(stack_slots));
4114   // No match rule because this operand is only generated in matching
4115 
4116   format %{ "[$reg]" %}
4117   interface(MEMORY_INTER) %{
4118     base(0x4);   // RSP
4119     index(0x4);  // No Index
4120     scale(0x0);  // No Scale
4121     disp($reg);  // Stack Offset
4122   %}
4123 %}
4124 operand stackSlotL(sRegL reg)
4125 %{
4126   constraint(ALLOC_IN_RC(stack_slots));
4127   // No match rule because this operand is only generated in matching
4128 
4129   format %{ "[$reg]" %}
4130   interface(MEMORY_INTER) %{
4131     base(0x4);   // RSP
4132     index(0x4);  // No Index
4133     scale(0x0);  // No Scale
4134     disp($reg);  // Stack Offset
4135   %}
4136 %}
4137 
4138 //----------Conditional Branch Operands----------------------------------------
4139 // Comparison Op  - This is the operation of the comparison, and is limited to
4140 //                  the following set of codes:
4141 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4142 //
4143 // Other attributes of the comparison, such as unsignedness, are specified
4144 // by the comparison instruction that sets a condition code flags register.
4145 // That result is represented by a flags operand whose subtype is appropriate
4146 // to the unsignedness (etc.) of the comparison.
4147 //
4148 // Later, the instruction which matches both the Comparison Op (a Bool) and
4149 // the flags (produced by the Cmp) specifies the coding of the comparison op
4150 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4151 
4152 // Comparision Code
4153 operand cmpOp()
4154 %{
4155   match(Bool);
4156 
4157   format %{ "" %}
4158   interface(COND_INTER) %{
4159     equal(0x4, "e");
4160     not_equal(0x5, "ne");
4161     less(0xC, "l");
4162     greater_equal(0xD, "ge");
4163     less_equal(0xE, "le");
4164     greater(0xF, "g");
4165     overflow(0x0, "o");
4166     no_overflow(0x1, "no");
4167   %}
4168 %}
4169 
4170 // Comparison Code, unsigned compare.  Used by FP also, with
4171 // C2 (unordered) turned into GT or LT already.  The other bits
4172 // C0 and C3 are turned into Carry & Zero flags.
4173 operand cmpOpU()
4174 %{
4175   match(Bool);
4176 
4177   format %{ "" %}
4178   interface(COND_INTER) %{
4179     equal(0x4, "e");
4180     not_equal(0x5, "ne");
4181     less(0x2, "b");
4182     greater_equal(0x3, "nb");
4183     less_equal(0x6, "be");
4184     greater(0x7, "nbe");
4185     overflow(0x0, "o");
4186     no_overflow(0x1, "no");
4187   %}
4188 %}
4189 
4190 
4191 // Floating comparisons that don't require any fixup for the unordered case
4192 operand cmpOpUCF() %{
4193   match(Bool);
4194   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4195             n->as_Bool()->_test._test == BoolTest::ge ||
4196             n->as_Bool()->_test._test == BoolTest::le ||
4197             n->as_Bool()->_test._test == BoolTest::gt);
4198   format %{ "" %}
4199   interface(COND_INTER) %{
4200     equal(0x4, "e");
4201     not_equal(0x5, "ne");
4202     less(0x2, "b");
4203     greater_equal(0x3, "nb");
4204     less_equal(0x6, "be");
4205     greater(0x7, "nbe");
4206     overflow(0x0, "o");
4207     no_overflow(0x1, "no");
4208   %}
4209 %}
4210 
4211 
4212 // Floating comparisons that can be fixed up with extra conditional jumps
4213 operand cmpOpUCF2() %{
4214   match(Bool);
4215   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4216             n->as_Bool()->_test._test == BoolTest::eq);
4217   format %{ "" %}
4218   interface(COND_INTER) %{
4219     equal(0x4, "e");
4220     not_equal(0x5, "ne");
4221     less(0x2, "b");
4222     greater_equal(0x3, "nb");
4223     less_equal(0x6, "be");
4224     greater(0x7, "nbe");
4225     overflow(0x0, "o");
4226     no_overflow(0x1, "no");
4227   %}
4228 %}
4229 
4230 // Operands for bound floating pointer register arguments
4231 operand rxmm0() %{
4232   constraint(ALLOC_IN_RC(xmm0_reg));  match(VecX);
4233   predicate((UseSSE > 0) && (UseAVX<= 2));  format%{%}  interface(REG_INTER);
4234 %}
4235 operand rxmm1() %{
4236   constraint(ALLOC_IN_RC(xmm1_reg));  match(VecX);
4237   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4238 %}
4239 operand rxmm2() %{
4240   constraint(ALLOC_IN_RC(xmm2_reg));  match(VecX);
4241   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4242 %}
4243 operand rxmm3() %{
4244   constraint(ALLOC_IN_RC(xmm3_reg));  match(VecX);
4245   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4246 %}
4247 operand rxmm4() %{
4248   constraint(ALLOC_IN_RC(xmm4_reg));  match(VecX);
4249   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4250 %}
4251 operand rxmm5() %{
4252   constraint(ALLOC_IN_RC(xmm5_reg));  match(VecX);
4253   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4254 %}
4255 operand rxmm6() %{
4256   constraint(ALLOC_IN_RC(xmm6_reg));  match(VecX);
4257   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4258 %}
4259 operand rxmm7() %{
4260   constraint(ALLOC_IN_RC(xmm7_reg));  match(VecX);
4261   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4262 %}
4263 operand rxmm8() %{
4264   constraint(ALLOC_IN_RC(xmm8_reg));  match(VecX);
4265   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4266 %}
4267 operand rxmm9() %{
4268   constraint(ALLOC_IN_RC(xmm9_reg));  match(VecX);
4269   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4270 %}
4271 operand rxmm10() %{
4272   constraint(ALLOC_IN_RC(xmm10_reg));  match(VecX);
4273   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4274 %}
4275 operand rxmm11() %{
4276   constraint(ALLOC_IN_RC(xmm11_reg));  match(VecX);
4277   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4278 %}
4279 operand rxmm12() %{
4280   constraint(ALLOC_IN_RC(xmm12_reg));  match(VecX);
4281   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4282 %}
4283 operand rxmm13() %{
4284   constraint(ALLOC_IN_RC(xmm13_reg));  match(VecX);
4285   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4286 %}
4287 operand rxmm14() %{
4288   constraint(ALLOC_IN_RC(xmm14_reg));  match(VecX);
4289   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4290 %}
4291 operand rxmm15() %{
4292   constraint(ALLOC_IN_RC(xmm15_reg));  match(VecX);
4293   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4294 %}
4295 operand rxmm16() %{
4296   constraint(ALLOC_IN_RC(xmm16_reg));  match(VecX);
4297   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4298 %}
4299 operand rxmm17() %{
4300   constraint(ALLOC_IN_RC(xmm17_reg));  match(VecX);
4301   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4302 %}
4303 operand rxmm18() %{
4304   constraint(ALLOC_IN_RC(xmm18_reg));  match(VecX);
4305   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4306 %}
4307 operand rxmm19() %{
4308   constraint(ALLOC_IN_RC(xmm19_reg));  match(VecX);
4309   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4310 %}
4311 operand rxmm20() %{
4312   constraint(ALLOC_IN_RC(xmm20_reg));  match(VecX);
4313   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4314 %}
4315 operand rxmm21() %{
4316   constraint(ALLOC_IN_RC(xmm21_reg));  match(VecX);
4317   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4318 %}
4319 operand rxmm22() %{
4320   constraint(ALLOC_IN_RC(xmm22_reg));  match(VecX);
4321   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4322 %}
4323 operand rxmm23() %{
4324   constraint(ALLOC_IN_RC(xmm23_reg));  match(VecX);
4325   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4326 %}
4327 operand rxmm24() %{
4328   constraint(ALLOC_IN_RC(xmm24_reg));  match(VecX);
4329   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4330 %}
4331 operand rxmm25() %{
4332   constraint(ALLOC_IN_RC(xmm25_reg));  match(VecX);
4333   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4334 %}
4335 operand rxmm26() %{
4336   constraint(ALLOC_IN_RC(xmm26_reg));  match(VecX);
4337   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4338 %}
4339 operand rxmm27() %{
4340   constraint(ALLOC_IN_RC(xmm27_reg));  match(VecX);
4341   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4342 %}
4343 operand rxmm28() %{
4344   constraint(ALLOC_IN_RC(xmm28_reg));  match(VecX);
4345   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4346 %}
4347 operand rxmm29() %{
4348   constraint(ALLOC_IN_RC(xmm29_reg));  match(VecX);
4349   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4350 %}
4351 operand rxmm30() %{
4352   constraint(ALLOC_IN_RC(xmm30_reg));  match(VecX);
4353   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4354 %}
4355 operand rxmm31() %{
4356   constraint(ALLOC_IN_RC(xmm31_reg));  match(VecX);
4357   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4358 %}
4359 
4360 //----------OPERAND CLASSES----------------------------------------------------
4361 // Operand Classes are groups of operands that are used as to simplify
4362 // instruction definitions by not requiring the AD writer to specify separate
4363 // instructions for every form of operand when the instruction accepts
4364 // multiple operand types with the same basic encoding and format.  The classic
4365 // case of this is memory operands.
4366 
4367 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4368                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4369                indCompressedOopOffset,
4370                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4371                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4372                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4373 
4374 //----------PIPELINE-----------------------------------------------------------
4375 // Rules which define the behavior of the target architectures pipeline.
4376 pipeline %{
4377 
4378 //----------ATTRIBUTES---------------------------------------------------------
4379 attributes %{
4380   variable_size_instructions;        // Fixed size instructions
4381   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4382   instruction_unit_size = 1;         // An instruction is 1 bytes long
4383   instruction_fetch_unit_size = 16;  // The processor fetches one line
4384   instruction_fetch_units = 1;       // of 16 bytes
4385 
4386   // List of nop instructions
4387   nops( MachNop );
4388 %}
4389 
4390 //----------RESOURCES----------------------------------------------------------
4391 // Resources are the functional units available to the machine
4392 
4393 // Generic P2/P3 pipeline
4394 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4395 // 3 instructions decoded per cycle.
4396 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4397 // 3 ALU op, only ALU0 handles mul instructions.
4398 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4399            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4400            BR, FPU,
4401            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4402 
4403 //----------PIPELINE DESCRIPTION-----------------------------------------------
4404 // Pipeline Description specifies the stages in the machine's pipeline
4405 
4406 // Generic P2/P3 pipeline
4407 pipe_desc(S0, S1, S2, S3, S4, S5);
4408 
4409 //----------PIPELINE CLASSES---------------------------------------------------
4410 // Pipeline Classes describe the stages in which input and output are
4411 // referenced by the hardware pipeline.
4412 
4413 // Naming convention: ialu or fpu
4414 // Then: _reg
4415 // Then: _reg if there is a 2nd register
4416 // Then: _long if it's a pair of instructions implementing a long
4417 // Then: _fat if it requires the big decoder
4418 //   Or: _mem if it requires the big decoder and a memory unit.
4419 
4420 // Integer ALU reg operation
4421 pipe_class ialu_reg(rRegI dst)
4422 %{
4423     single_instruction;
4424     dst    : S4(write);
4425     dst    : S3(read);
4426     DECODE : S0;        // any decoder
4427     ALU    : S3;        // any alu
4428 %}
4429 
4430 // Long ALU reg operation
4431 pipe_class ialu_reg_long(rRegL dst)
4432 %{
4433     instruction_count(2);
4434     dst    : S4(write);
4435     dst    : S3(read);
4436     DECODE : S0(2);     // any 2 decoders
4437     ALU    : S3(2);     // both alus
4438 %}
4439 
4440 // Integer ALU reg operation using big decoder
4441 pipe_class ialu_reg_fat(rRegI dst)
4442 %{
4443     single_instruction;
4444     dst    : S4(write);
4445     dst    : S3(read);
4446     D0     : S0;        // big decoder only
4447     ALU    : S3;        // any alu
4448 %}
4449 
4450 // Long ALU reg operation using big decoder
4451 pipe_class ialu_reg_long_fat(rRegL dst)
4452 %{
4453     instruction_count(2);
4454     dst    : S4(write);
4455     dst    : S3(read);
4456     D0     : S0(2);     // big decoder only; twice
4457     ALU    : S3(2);     // any 2 alus
4458 %}
4459 
4460 // Integer ALU reg-reg operation
4461 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4462 %{
4463     single_instruction;
4464     dst    : S4(write);
4465     src    : S3(read);
4466     DECODE : S0;        // any decoder
4467     ALU    : S3;        // any alu
4468 %}
4469 
4470 // Long ALU reg-reg operation
4471 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4472 %{
4473     instruction_count(2);
4474     dst    : S4(write);
4475     src    : S3(read);
4476     DECODE : S0(2);     // any 2 decoders
4477     ALU    : S3(2);     // both alus
4478 %}
4479 
4480 // Integer ALU reg-reg operation
4481 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4482 %{
4483     single_instruction;
4484     dst    : S4(write);
4485     src    : S3(read);
4486     D0     : S0;        // big decoder only
4487     ALU    : S3;        // any alu
4488 %}
4489 
4490 // Long ALU reg-reg operation
4491 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4492 %{
4493     instruction_count(2);
4494     dst    : S4(write);
4495     src    : S3(read);
4496     D0     : S0(2);     // big decoder only; twice
4497     ALU    : S3(2);     // both alus
4498 %}
4499 
4500 // Integer ALU reg-mem operation
4501 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4502 %{
4503     single_instruction;
4504     dst    : S5(write);
4505     mem    : S3(read);
4506     D0     : S0;        // big decoder only
4507     ALU    : S4;        // any alu
4508     MEM    : S3;        // any mem
4509 %}
4510 
4511 // Integer mem operation (prefetch)
4512 pipe_class ialu_mem(memory mem)
4513 %{
4514     single_instruction;
4515     mem    : S3(read);
4516     D0     : S0;        // big decoder only
4517     MEM    : S3;        // any mem
4518 %}
4519 
4520 // Integer Store to Memory
4521 pipe_class ialu_mem_reg(memory mem, rRegI src)
4522 %{
4523     single_instruction;
4524     mem    : S3(read);
4525     src    : S5(read);
4526     D0     : S0;        // big decoder only
4527     ALU    : S4;        // any alu
4528     MEM    : S3;
4529 %}
4530 
4531 // // Long Store to Memory
4532 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4533 // %{
4534 //     instruction_count(2);
4535 //     mem    : S3(read);
4536 //     src    : S5(read);
4537 //     D0     : S0(2);          // big decoder only; twice
4538 //     ALU    : S4(2);     // any 2 alus
4539 //     MEM    : S3(2);  // Both mems
4540 // %}
4541 
4542 // Integer Store to Memory
4543 pipe_class ialu_mem_imm(memory mem)
4544 %{
4545     single_instruction;
4546     mem    : S3(read);
4547     D0     : S0;        // big decoder only
4548     ALU    : S4;        // any alu
4549     MEM    : S3;
4550 %}
4551 
4552 // Integer ALU0 reg-reg operation
4553 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4554 %{
4555     single_instruction;
4556     dst    : S4(write);
4557     src    : S3(read);
4558     D0     : S0;        // Big decoder only
4559     ALU0   : S3;        // only alu0
4560 %}
4561 
4562 // Integer ALU0 reg-mem operation
4563 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4564 %{
4565     single_instruction;
4566     dst    : S5(write);
4567     mem    : S3(read);
4568     D0     : S0;        // big decoder only
4569     ALU0   : S4;        // ALU0 only
4570     MEM    : S3;        // any mem
4571 %}
4572 
4573 // Integer ALU reg-reg operation
4574 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4575 %{
4576     single_instruction;
4577     cr     : S4(write);
4578     src1   : S3(read);
4579     src2   : S3(read);
4580     DECODE : S0;        // any decoder
4581     ALU    : S3;        // any alu
4582 %}
4583 
4584 // Integer ALU reg-imm operation
4585 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4586 %{
4587     single_instruction;
4588     cr     : S4(write);
4589     src1   : S3(read);
4590     DECODE : S0;        // any decoder
4591     ALU    : S3;        // any alu
4592 %}
4593 
4594 // Integer ALU reg-mem operation
4595 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4596 %{
4597     single_instruction;
4598     cr     : S4(write);
4599     src1   : S3(read);
4600     src2   : S3(read);
4601     D0     : S0;        // big decoder only
4602     ALU    : S4;        // any alu
4603     MEM    : S3;
4604 %}
4605 
4606 // Conditional move reg-reg
4607 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4608 %{
4609     instruction_count(4);
4610     y      : S4(read);
4611     q      : S3(read);
4612     p      : S3(read);
4613     DECODE : S0(4);     // any decoder
4614 %}
4615 
4616 // Conditional move reg-reg
4617 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4618 %{
4619     single_instruction;
4620     dst    : S4(write);
4621     src    : S3(read);
4622     cr     : S3(read);
4623     DECODE : S0;        // any decoder
4624 %}
4625 
4626 // Conditional move reg-mem
4627 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4628 %{
4629     single_instruction;
4630     dst    : S4(write);
4631     src    : S3(read);
4632     cr     : S3(read);
4633     DECODE : S0;        // any decoder
4634     MEM    : S3;
4635 %}
4636 
4637 // Conditional move reg-reg long
4638 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4639 %{
4640     single_instruction;
4641     dst    : S4(write);
4642     src    : S3(read);
4643     cr     : S3(read);
4644     DECODE : S0(2);     // any 2 decoders
4645 %}
4646 
4647 // XXX
4648 // // Conditional move double reg-reg
4649 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4650 // %{
4651 //     single_instruction;
4652 //     dst    : S4(write);
4653 //     src    : S3(read);
4654 //     cr     : S3(read);
4655 //     DECODE : S0;     // any decoder
4656 // %}
4657 
4658 // Float reg-reg operation
4659 pipe_class fpu_reg(regD dst)
4660 %{
4661     instruction_count(2);
4662     dst    : S3(read);
4663     DECODE : S0(2);     // any 2 decoders
4664     FPU    : S3;
4665 %}
4666 
4667 // Float reg-reg operation
4668 pipe_class fpu_reg_reg(regD dst, regD src)
4669 %{
4670     instruction_count(2);
4671     dst    : S4(write);
4672     src    : S3(read);
4673     DECODE : S0(2);     // any 2 decoders
4674     FPU    : S3;
4675 %}
4676 
4677 // Float reg-reg operation
4678 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4679 %{
4680     instruction_count(3);
4681     dst    : S4(write);
4682     src1   : S3(read);
4683     src2   : S3(read);
4684     DECODE : S0(3);     // any 3 decoders
4685     FPU    : S3(2);
4686 %}
4687 
4688 // Float reg-reg operation
4689 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4690 %{
4691     instruction_count(4);
4692     dst    : S4(write);
4693     src1   : S3(read);
4694     src2   : S3(read);
4695     src3   : S3(read);
4696     DECODE : S0(4);     // any 3 decoders
4697     FPU    : S3(2);
4698 %}
4699 
4700 // Float reg-reg operation
4701 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4702 %{
4703     instruction_count(4);
4704     dst    : S4(write);
4705     src1   : S3(read);
4706     src2   : S3(read);
4707     src3   : S3(read);
4708     DECODE : S1(3);     // any 3 decoders
4709     D0     : S0;        // Big decoder only
4710     FPU    : S3(2);
4711     MEM    : S3;
4712 %}
4713 
4714 // Float reg-mem operation
4715 pipe_class fpu_reg_mem(regD dst, memory mem)
4716 %{
4717     instruction_count(2);
4718     dst    : S5(write);
4719     mem    : S3(read);
4720     D0     : S0;        // big decoder only
4721     DECODE : S1;        // any decoder for FPU POP
4722     FPU    : S4;
4723     MEM    : S3;        // any mem
4724 %}
4725 
4726 // Float reg-mem operation
4727 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4728 %{
4729     instruction_count(3);
4730     dst    : S5(write);
4731     src1   : S3(read);
4732     mem    : S3(read);
4733     D0     : S0;        // big decoder only
4734     DECODE : S1(2);     // any decoder for FPU POP
4735     FPU    : S4;
4736     MEM    : S3;        // any mem
4737 %}
4738 
4739 // Float mem-reg operation
4740 pipe_class fpu_mem_reg(memory mem, regD src)
4741 %{
4742     instruction_count(2);
4743     src    : S5(read);
4744     mem    : S3(read);
4745     DECODE : S0;        // any decoder for FPU PUSH
4746     D0     : S1;        // big decoder only
4747     FPU    : S4;
4748     MEM    : S3;        // any mem
4749 %}
4750 
4751 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4752 %{
4753     instruction_count(3);
4754     src1   : S3(read);
4755     src2   : S3(read);
4756     mem    : S3(read);
4757     DECODE : S0(2);     // any decoder for FPU PUSH
4758     D0     : S1;        // big decoder only
4759     FPU    : S4;
4760     MEM    : S3;        // any mem
4761 %}
4762 
4763 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4764 %{
4765     instruction_count(3);
4766     src1   : S3(read);
4767     src2   : S3(read);
4768     mem    : S4(read);
4769     DECODE : S0;        // any decoder for FPU PUSH
4770     D0     : S0(2);     // big decoder only
4771     FPU    : S4;
4772     MEM    : S3(2);     // any mem
4773 %}
4774 
4775 pipe_class fpu_mem_mem(memory dst, memory src1)
4776 %{
4777     instruction_count(2);
4778     src1   : S3(read);
4779     dst    : S4(read);
4780     D0     : S0(2);     // big decoder only
4781     MEM    : S3(2);     // any mem
4782 %}
4783 
4784 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4785 %{
4786     instruction_count(3);
4787     src1   : S3(read);
4788     src2   : S3(read);
4789     dst    : S4(read);
4790     D0     : S0(3);     // big decoder only
4791     FPU    : S4;
4792     MEM    : S3(3);     // any mem
4793 %}
4794 
4795 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4796 %{
4797     instruction_count(3);
4798     src1   : S4(read);
4799     mem    : S4(read);
4800     DECODE : S0;        // any decoder for FPU PUSH
4801     D0     : S0(2);     // big decoder only
4802     FPU    : S4;
4803     MEM    : S3(2);     // any mem
4804 %}
4805 
4806 // Float load constant
4807 pipe_class fpu_reg_con(regD dst)
4808 %{
4809     instruction_count(2);
4810     dst    : S5(write);
4811     D0     : S0;        // big decoder only for the load
4812     DECODE : S1;        // any decoder for FPU POP
4813     FPU    : S4;
4814     MEM    : S3;        // any mem
4815 %}
4816 
4817 // Float load constant
4818 pipe_class fpu_reg_reg_con(regD dst, regD src)
4819 %{
4820     instruction_count(3);
4821     dst    : S5(write);
4822     src    : S3(read);
4823     D0     : S0;        // big decoder only for the load
4824     DECODE : S1(2);     // any decoder for FPU POP
4825     FPU    : S4;
4826     MEM    : S3;        // any mem
4827 %}
4828 
4829 // UnConditional branch
4830 pipe_class pipe_jmp(label labl)
4831 %{
4832     single_instruction;
4833     BR   : S3;
4834 %}
4835 
4836 // Conditional branch
4837 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4838 %{
4839     single_instruction;
4840     cr    : S1(read);
4841     BR    : S3;
4842 %}
4843 
4844 // Allocation idiom
4845 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4846 %{
4847     instruction_count(1); force_serialization;
4848     fixed_latency(6);
4849     heap_ptr : S3(read);
4850     DECODE   : S0(3);
4851     D0       : S2;
4852     MEM      : S3;
4853     ALU      : S3(2);
4854     dst      : S5(write);
4855     BR       : S5;
4856 %}
4857 
4858 // Generic big/slow expanded idiom
4859 pipe_class pipe_slow()
4860 %{
4861     instruction_count(10); multiple_bundles; force_serialization;
4862     fixed_latency(100);
4863     D0  : S0(2);
4864     MEM : S3(2);
4865 %}
4866 
4867 // The real do-nothing guy
4868 pipe_class empty()
4869 %{
4870     instruction_count(0);
4871 %}
4872 
4873 // Define the class for the Nop node
4874 define
4875 %{
4876    MachNop = empty;
4877 %}
4878 
4879 %}
4880 
4881 //----------INSTRUCTIONS-------------------------------------------------------
4882 //
4883 // match      -- States which machine-independent subtree may be replaced
4884 //               by this instruction.
4885 // ins_cost   -- The estimated cost of this instruction is used by instruction
4886 //               selection to identify a minimum cost tree of machine
4887 //               instructions that matches a tree of machine-independent
4888 //               instructions.
4889 // format     -- A string providing the disassembly for this instruction.
4890 //               The value of an instruction's operand may be inserted
4891 //               by referring to it with a '$' prefix.
4892 // opcode     -- Three instruction opcodes may be provided.  These are referred
4893 //               to within an encode class as $primary, $secondary, and $tertiary
4894 //               rrspectively.  The primary opcode is commonly used to
4895 //               indicate the type of machine instruction, while secondary
4896 //               and tertiary are often used for prefix options or addressing
4897 //               modes.
4898 // ins_encode -- A list of encode classes with parameters. The encode class
4899 //               name must have been defined in an 'enc_class' specification
4900 //               in the encode section of the architecture description.
4901 
4902 
4903 //----------Load/Store/Move Instructions---------------------------------------
4904 //----------Load Instructions--------------------------------------------------
4905 
4906 // Load Byte (8 bit signed)
4907 instruct loadB(rRegI dst, memory mem)
4908 %{
4909   match(Set dst (LoadB mem));
4910 
4911   ins_cost(125);
4912   format %{ "movsbl  $dst, $mem\t# byte" %}
4913 
4914   ins_encode %{
4915     __ movsbl($dst$$Register, $mem$$Address);
4916   %}
4917 
4918   ins_pipe(ialu_reg_mem);
4919 %}
4920 
4921 // Load Byte (8 bit signed) into Long Register
4922 instruct loadB2L(rRegL dst, memory mem)
4923 %{
4924   match(Set dst (ConvI2L (LoadB mem)));
4925 
4926   ins_cost(125);
4927   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4928 
4929   ins_encode %{
4930     __ movsbq($dst$$Register, $mem$$Address);
4931   %}
4932 
4933   ins_pipe(ialu_reg_mem);
4934 %}
4935 
4936 // Load Unsigned Byte (8 bit UNsigned)
4937 instruct loadUB(rRegI dst, memory mem)
4938 %{
4939   match(Set dst (LoadUB mem));
4940 
4941   ins_cost(125);
4942   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4943 
4944   ins_encode %{
4945     __ movzbl($dst$$Register, $mem$$Address);
4946   %}
4947 
4948   ins_pipe(ialu_reg_mem);
4949 %}
4950 
4951 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4952 instruct loadUB2L(rRegL dst, memory mem)
4953 %{
4954   match(Set dst (ConvI2L (LoadUB mem)));
4955 
4956   ins_cost(125);
4957   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4958 
4959   ins_encode %{
4960     __ movzbq($dst$$Register, $mem$$Address);
4961   %}
4962 
4963   ins_pipe(ialu_reg_mem);
4964 %}
4965 
4966 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4967 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4968   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4969   effect(KILL cr);
4970 
4971   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4972             "andl    $dst, right_n_bits($mask, 8)" %}
4973   ins_encode %{
4974     Register Rdst = $dst$$Register;
4975     __ movzbq(Rdst, $mem$$Address);
4976     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4977   %}
4978   ins_pipe(ialu_reg_mem);
4979 %}
4980 
4981 // Load Short (16 bit signed)
4982 instruct loadS(rRegI dst, memory mem)
4983 %{
4984   match(Set dst (LoadS mem));
4985 
4986   ins_cost(125);
4987   format %{ "movswl $dst, $mem\t# short" %}
4988 
4989   ins_encode %{
4990     __ movswl($dst$$Register, $mem$$Address);
4991   %}
4992 
4993   ins_pipe(ialu_reg_mem);
4994 %}
4995 
4996 // Load Short (16 bit signed) to Byte (8 bit signed)
4997 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4998   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4999 
5000   ins_cost(125);
5001   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5002   ins_encode %{
5003     __ movsbl($dst$$Register, $mem$$Address);
5004   %}
5005   ins_pipe(ialu_reg_mem);
5006 %}
5007 
5008 // Load Short (16 bit signed) into Long Register
5009 instruct loadS2L(rRegL dst, memory mem)
5010 %{
5011   match(Set dst (ConvI2L (LoadS mem)));
5012 
5013   ins_cost(125);
5014   format %{ "movswq $dst, $mem\t# short -> long" %}
5015 
5016   ins_encode %{
5017     __ movswq($dst$$Register, $mem$$Address);
5018   %}
5019 
5020   ins_pipe(ialu_reg_mem);
5021 %}
5022 
5023 // Load Unsigned Short/Char (16 bit UNsigned)
5024 instruct loadUS(rRegI dst, memory mem)
5025 %{
5026   match(Set dst (LoadUS mem));
5027 
5028   ins_cost(125);
5029   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5030 
5031   ins_encode %{
5032     __ movzwl($dst$$Register, $mem$$Address);
5033   %}
5034 
5035   ins_pipe(ialu_reg_mem);
5036 %}
5037 
5038 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5039 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5040   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5041 
5042   ins_cost(125);
5043   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5044   ins_encode %{
5045     __ movsbl($dst$$Register, $mem$$Address);
5046   %}
5047   ins_pipe(ialu_reg_mem);
5048 %}
5049 
5050 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5051 instruct loadUS2L(rRegL dst, memory mem)
5052 %{
5053   match(Set dst (ConvI2L (LoadUS mem)));
5054 
5055   ins_cost(125);
5056   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5057 
5058   ins_encode %{
5059     __ movzwq($dst$$Register, $mem$$Address);
5060   %}
5061 
5062   ins_pipe(ialu_reg_mem);
5063 %}
5064 
5065 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5066 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5067   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5068 
5069   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5070   ins_encode %{
5071     __ movzbq($dst$$Register, $mem$$Address);
5072   %}
5073   ins_pipe(ialu_reg_mem);
5074 %}
5075 
5076 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5077 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5078   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5079   effect(KILL cr);
5080 
5081   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5082             "andl    $dst, right_n_bits($mask, 16)" %}
5083   ins_encode %{
5084     Register Rdst = $dst$$Register;
5085     __ movzwq(Rdst, $mem$$Address);
5086     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5087   %}
5088   ins_pipe(ialu_reg_mem);
5089 %}
5090 
5091 // Load Integer
5092 instruct loadI(rRegI dst, memory mem)
5093 %{
5094   match(Set dst (LoadI mem));
5095 
5096   ins_cost(125);
5097   format %{ "movl    $dst, $mem\t# int" %}
5098 
5099   ins_encode %{
5100     __ movl($dst$$Register, $mem$$Address);
5101   %}
5102 
5103   ins_pipe(ialu_reg_mem);
5104 %}
5105 
5106 // Load Integer (32 bit signed) to Byte (8 bit signed)
5107 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5108   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5109 
5110   ins_cost(125);
5111   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5112   ins_encode %{
5113     __ movsbl($dst$$Register, $mem$$Address);
5114   %}
5115   ins_pipe(ialu_reg_mem);
5116 %}
5117 
5118 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5119 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5120   match(Set dst (AndI (LoadI mem) mask));
5121 
5122   ins_cost(125);
5123   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5124   ins_encode %{
5125     __ movzbl($dst$$Register, $mem$$Address);
5126   %}
5127   ins_pipe(ialu_reg_mem);
5128 %}
5129 
5130 // Load Integer (32 bit signed) to Short (16 bit signed)
5131 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5132   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5133 
5134   ins_cost(125);
5135   format %{ "movswl  $dst, $mem\t# int -> short" %}
5136   ins_encode %{
5137     __ movswl($dst$$Register, $mem$$Address);
5138   %}
5139   ins_pipe(ialu_reg_mem);
5140 %}
5141 
5142 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5143 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5144   match(Set dst (AndI (LoadI mem) mask));
5145 
5146   ins_cost(125);
5147   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5148   ins_encode %{
5149     __ movzwl($dst$$Register, $mem$$Address);
5150   %}
5151   ins_pipe(ialu_reg_mem);
5152 %}
5153 
5154 // Load Integer into Long Register
5155 instruct loadI2L(rRegL dst, memory mem)
5156 %{
5157   match(Set dst (ConvI2L (LoadI mem)));
5158 
5159   ins_cost(125);
5160   format %{ "movslq  $dst, $mem\t# int -> long" %}
5161 
5162   ins_encode %{
5163     __ movslq($dst$$Register, $mem$$Address);
5164   %}
5165 
5166   ins_pipe(ialu_reg_mem);
5167 %}
5168 
5169 // Load Integer with mask 0xFF into Long Register
5170 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5171   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5172 
5173   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5174   ins_encode %{
5175     __ movzbq($dst$$Register, $mem$$Address);
5176   %}
5177   ins_pipe(ialu_reg_mem);
5178 %}
5179 
5180 // Load Integer with mask 0xFFFF into Long Register
5181 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5182   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5183 
5184   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5185   ins_encode %{
5186     __ movzwq($dst$$Register, $mem$$Address);
5187   %}
5188   ins_pipe(ialu_reg_mem);
5189 %}
5190 
5191 // Load Integer with a 31-bit mask into Long Register
5192 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5193   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5194   effect(KILL cr);
5195 
5196   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5197             "andl    $dst, $mask" %}
5198   ins_encode %{
5199     Register Rdst = $dst$$Register;
5200     __ movl(Rdst, $mem$$Address);
5201     __ andl(Rdst, $mask$$constant);
5202   %}
5203   ins_pipe(ialu_reg_mem);
5204 %}
5205 
5206 // Load Unsigned Integer into Long Register
5207 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5208 %{
5209   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5210 
5211   ins_cost(125);
5212   format %{ "movl    $dst, $mem\t# uint -> long" %}
5213 
5214   ins_encode %{
5215     __ movl($dst$$Register, $mem$$Address);
5216   %}
5217 
5218   ins_pipe(ialu_reg_mem);
5219 %}
5220 
5221 // Load Long
5222 instruct loadL(rRegL dst, memory mem)
5223 %{
5224   match(Set dst (LoadL mem));
5225 
5226   ins_cost(125);
5227   format %{ "movq    $dst, $mem\t# long" %}
5228 
5229   ins_encode %{
5230     __ movq($dst$$Register, $mem$$Address);
5231   %}
5232 
5233   ins_pipe(ialu_reg_mem); // XXX
5234 %}
5235 
5236 // Load Range
5237 instruct loadRange(rRegI dst, memory mem)
5238 %{
5239   match(Set dst (LoadRange mem));
5240 
5241   ins_cost(125); // XXX
5242   format %{ "movl    $dst, $mem\t# range" %}
5243   opcode(0x8B);
5244   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5245   ins_pipe(ialu_reg_mem);
5246 %}
5247 
5248 // Load Pointer
5249 instruct loadP(rRegP dst, memory mem)
5250 %{
5251   match(Set dst (LoadP mem));
5252 
5253   ins_cost(125); // XXX
5254   format %{ "movq    $dst, $mem\t# ptr" %}
5255   opcode(0x8B);
5256   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5257   ins_pipe(ialu_reg_mem); // XXX
5258 %}
5259 
5260 // Load Compressed Pointer
5261 instruct loadN(rRegN dst, memory mem)
5262 %{
5263    match(Set dst (LoadN mem));
5264 
5265    ins_cost(125); // XXX
5266    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5267    ins_encode %{
5268      __ movl($dst$$Register, $mem$$Address);
5269    %}
5270    ins_pipe(ialu_reg_mem); // XXX
5271 %}
5272 
5273 
5274 // Load Klass Pointer
5275 instruct loadKlass(rRegP dst, memory mem)
5276 %{
5277   match(Set dst (LoadKlass mem));
5278 
5279   ins_cost(125); // XXX
5280   format %{ "movq    $dst, $mem\t# class" %}
5281   opcode(0x8B);
5282   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5283   ins_pipe(ialu_reg_mem); // XXX
5284 %}
5285 
5286 // Load narrow Klass Pointer
5287 instruct loadNKlass(rRegN dst, memory mem)
5288 %{
5289   match(Set dst (LoadNKlass mem));
5290 
5291   ins_cost(125); // XXX
5292   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5293   ins_encode %{
5294     __ movl($dst$$Register, $mem$$Address);
5295   %}
5296   ins_pipe(ialu_reg_mem); // XXX
5297 %}
5298 
5299 // Load Float
5300 instruct loadF(regF dst, memory mem)
5301 %{
5302   match(Set dst (LoadF mem));
5303 
5304   ins_cost(145); // XXX
5305   format %{ "movss   $dst, $mem\t# float" %}
5306   ins_encode %{
5307     __ movflt($dst$$XMMRegister, $mem$$Address);
5308   %}
5309   ins_pipe(pipe_slow); // XXX
5310 %}
5311 
5312 // Load Double
5313 instruct loadD_partial(regD dst, memory mem)
5314 %{
5315   predicate(!UseXmmLoadAndClearUpper);
5316   match(Set dst (LoadD mem));
5317 
5318   ins_cost(145); // XXX
5319   format %{ "movlpd  $dst, $mem\t# double" %}
5320   ins_encode %{
5321     __ movdbl($dst$$XMMRegister, $mem$$Address);
5322   %}
5323   ins_pipe(pipe_slow); // XXX
5324 %}
5325 
5326 instruct loadD(regD dst, memory mem)
5327 %{
5328   predicate(UseXmmLoadAndClearUpper);
5329   match(Set dst (LoadD mem));
5330 
5331   ins_cost(145); // XXX
5332   format %{ "movsd   $dst, $mem\t# double" %}
5333   ins_encode %{
5334     __ movdbl($dst$$XMMRegister, $mem$$Address);
5335   %}
5336   ins_pipe(pipe_slow); // XXX
5337 %}
5338 
5339 // Load Effective Address
5340 instruct leaP8(rRegP dst, indOffset8 mem)
5341 %{
5342   match(Set dst mem);
5343 
5344   ins_cost(110); // XXX
5345   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5346   opcode(0x8D);
5347   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5348   ins_pipe(ialu_reg_reg_fat);
5349 %}
5350 
5351 instruct leaP32(rRegP dst, indOffset32 mem)
5352 %{
5353   match(Set dst mem);
5354 
5355   ins_cost(110);
5356   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5357   opcode(0x8D);
5358   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5359   ins_pipe(ialu_reg_reg_fat);
5360 %}
5361 
5362 // instruct leaPIdx(rRegP dst, indIndex mem)
5363 // %{
5364 //   match(Set dst mem);
5365 
5366 //   ins_cost(110);
5367 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5368 //   opcode(0x8D);
5369 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5370 //   ins_pipe(ialu_reg_reg_fat);
5371 // %}
5372 
5373 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5374 %{
5375   match(Set dst mem);
5376 
5377   ins_cost(110);
5378   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5379   opcode(0x8D);
5380   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5381   ins_pipe(ialu_reg_reg_fat);
5382 %}
5383 
5384 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5385 %{
5386   match(Set dst mem);
5387 
5388   ins_cost(110);
5389   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5390   opcode(0x8D);
5391   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5392   ins_pipe(ialu_reg_reg_fat);
5393 %}
5394 
5395 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5396 %{
5397   match(Set dst mem);
5398 
5399   ins_cost(110);
5400   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5401   opcode(0x8D);
5402   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5403   ins_pipe(ialu_reg_reg_fat);
5404 %}
5405 
5406 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5407 %{
5408   match(Set dst mem);
5409 
5410   ins_cost(110);
5411   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5412   opcode(0x8D);
5413   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5414   ins_pipe(ialu_reg_reg_fat);
5415 %}
5416 
5417 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5418 %{
5419   match(Set dst mem);
5420 
5421   ins_cost(110);
5422   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5423   opcode(0x8D);
5424   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5425   ins_pipe(ialu_reg_reg_fat);
5426 %}
5427 
5428 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5429 %{
5430   match(Set dst mem);
5431 
5432   ins_cost(110);
5433   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5434   opcode(0x8D);
5435   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5436   ins_pipe(ialu_reg_reg_fat);
5437 %}
5438 
5439 // Load Effective Address which uses Narrow (32-bits) oop
5440 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5441 %{
5442   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5443   match(Set dst mem);
5444 
5445   ins_cost(110);
5446   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5447   opcode(0x8D);
5448   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5449   ins_pipe(ialu_reg_reg_fat);
5450 %}
5451 
5452 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5453 %{
5454   predicate(Universe::narrow_oop_shift() == 0);
5455   match(Set dst mem);
5456 
5457   ins_cost(110); // XXX
5458   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5459   opcode(0x8D);
5460   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5461   ins_pipe(ialu_reg_reg_fat);
5462 %}
5463 
5464 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5465 %{
5466   predicate(Universe::narrow_oop_shift() == 0);
5467   match(Set dst mem);
5468 
5469   ins_cost(110);
5470   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5471   opcode(0x8D);
5472   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5473   ins_pipe(ialu_reg_reg_fat);
5474 %}
5475 
5476 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5477 %{
5478   predicate(Universe::narrow_oop_shift() == 0);
5479   match(Set dst mem);
5480 
5481   ins_cost(110);
5482   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5483   opcode(0x8D);
5484   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5485   ins_pipe(ialu_reg_reg_fat);
5486 %}
5487 
5488 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5489 %{
5490   predicate(Universe::narrow_oop_shift() == 0);
5491   match(Set dst mem);
5492 
5493   ins_cost(110);
5494   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5495   opcode(0x8D);
5496   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5497   ins_pipe(ialu_reg_reg_fat);
5498 %}
5499 
5500 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5501 %{
5502   predicate(Universe::narrow_oop_shift() == 0);
5503   match(Set dst mem);
5504 
5505   ins_cost(110);
5506   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5507   opcode(0x8D);
5508   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5509   ins_pipe(ialu_reg_reg_fat);
5510 %}
5511 
5512 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5513 %{
5514   predicate(Universe::narrow_oop_shift() == 0);
5515   match(Set dst mem);
5516 
5517   ins_cost(110);
5518   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5519   opcode(0x8D);
5520   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5521   ins_pipe(ialu_reg_reg_fat);
5522 %}
5523 
5524 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5525 %{
5526   predicate(Universe::narrow_oop_shift() == 0);
5527   match(Set dst mem);
5528 
5529   ins_cost(110);
5530   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5531   opcode(0x8D);
5532   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5533   ins_pipe(ialu_reg_reg_fat);
5534 %}
5535 
5536 instruct loadConI(rRegI dst, immI src)
5537 %{
5538   match(Set dst src);
5539 
5540   format %{ "movl    $dst, $src\t# int" %}
5541   ins_encode(load_immI(dst, src));
5542   ins_pipe(ialu_reg_fat); // XXX
5543 %}
5544 
5545 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5546 %{
5547   match(Set dst src);
5548   effect(KILL cr);
5549 
5550   ins_cost(50);
5551   format %{ "xorl    $dst, $dst\t# int" %}
5552   opcode(0x33); /* + rd */
5553   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5554   ins_pipe(ialu_reg);
5555 %}
5556 
5557 instruct loadConL(rRegL dst, immL src)
5558 %{
5559   match(Set dst src);
5560 
5561   ins_cost(150);
5562   format %{ "movq    $dst, $src\t# long" %}
5563   ins_encode(load_immL(dst, src));
5564   ins_pipe(ialu_reg);
5565 %}
5566 
5567 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5568 %{
5569   match(Set dst src);
5570   effect(KILL cr);
5571 
5572   ins_cost(50);
5573   format %{ "xorl    $dst, $dst\t# long" %}
5574   opcode(0x33); /* + rd */
5575   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5576   ins_pipe(ialu_reg); // XXX
5577 %}
5578 
5579 instruct loadConUL32(rRegL dst, immUL32 src)
5580 %{
5581   match(Set dst src);
5582 
5583   ins_cost(60);
5584   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5585   ins_encode(load_immUL32(dst, src));
5586   ins_pipe(ialu_reg);
5587 %}
5588 
5589 instruct loadConL32(rRegL dst, immL32 src)
5590 %{
5591   match(Set dst src);
5592 
5593   ins_cost(70);
5594   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5595   ins_encode(load_immL32(dst, src));
5596   ins_pipe(ialu_reg);
5597 %}
5598 
5599 instruct loadConP(rRegP dst, immP con) %{
5600   match(Set dst con);
5601 
5602   format %{ "movq    $dst, $con\t# ptr" %}
5603   ins_encode(load_immP(dst, con));
5604   ins_pipe(ialu_reg_fat); // XXX
5605 %}
5606 
5607 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5608 %{
5609   match(Set dst src);
5610   effect(KILL cr);
5611 
5612   ins_cost(50);
5613   format %{ "xorl    $dst, $dst\t# ptr" %}
5614   opcode(0x33); /* + rd */
5615   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5616   ins_pipe(ialu_reg);
5617 %}
5618 
5619 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5620 %{
5621   match(Set dst src);
5622   effect(KILL cr);
5623 
5624   ins_cost(60);
5625   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5626   ins_encode(load_immP31(dst, src));
5627   ins_pipe(ialu_reg);
5628 %}
5629 
5630 instruct loadConF(regF dst, immF con) %{
5631   match(Set dst con);
5632   ins_cost(125);
5633   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5634   ins_encode %{
5635     __ movflt($dst$$XMMRegister, $constantaddress($con));
5636   %}
5637   ins_pipe(pipe_slow);
5638 %}
5639 
5640 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5641   match(Set dst src);
5642   effect(KILL cr);
5643   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5644   ins_encode %{
5645     __ xorq($dst$$Register, $dst$$Register);
5646   %}
5647   ins_pipe(ialu_reg);
5648 %}
5649 
5650 instruct loadConN(rRegN dst, immN src) %{
5651   match(Set dst src);
5652 
5653   ins_cost(125);
5654   format %{ "movl    $dst, $src\t# compressed ptr" %}
5655   ins_encode %{
5656     address con = (address)$src$$constant;
5657     if (con == NULL) {
5658       ShouldNotReachHere();
5659     } else {
5660       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5661     }
5662   %}
5663   ins_pipe(ialu_reg_fat); // XXX
5664 %}
5665 
5666 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5667   match(Set dst src);
5668 
5669   ins_cost(125);
5670   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5671   ins_encode %{
5672     address con = (address)$src$$constant;
5673     if (con == NULL) {
5674       ShouldNotReachHere();
5675     } else {
5676       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5677     }
5678   %}
5679   ins_pipe(ialu_reg_fat); // XXX
5680 %}
5681 
5682 instruct loadConF0(regF dst, immF0 src)
5683 %{
5684   match(Set dst src);
5685   ins_cost(100);
5686 
5687   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5688   ins_encode %{
5689     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5690   %}
5691   ins_pipe(pipe_slow);
5692 %}
5693 
5694 // Use the same format since predicate() can not be used here.
5695 instruct loadConD(regD dst, immD con) %{
5696   match(Set dst con);
5697   ins_cost(125);
5698   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5699   ins_encode %{
5700     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5701   %}
5702   ins_pipe(pipe_slow);
5703 %}
5704 
5705 instruct loadConD0(regD dst, immD0 src)
5706 %{
5707   match(Set dst src);
5708   ins_cost(100);
5709 
5710   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5711   ins_encode %{
5712     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5713   %}
5714   ins_pipe(pipe_slow);
5715 %}
5716 
5717 instruct loadSSI(rRegI dst, stackSlotI src)
5718 %{
5719   match(Set dst src);
5720 
5721   ins_cost(125);
5722   format %{ "movl    $dst, $src\t# int stk" %}
5723   opcode(0x8B);
5724   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5725   ins_pipe(ialu_reg_mem);
5726 %}
5727 
5728 instruct loadSSL(rRegL dst, stackSlotL src)
5729 %{
5730   match(Set dst src);
5731 
5732   ins_cost(125);
5733   format %{ "movq    $dst, $src\t# long stk" %}
5734   opcode(0x8B);
5735   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5736   ins_pipe(ialu_reg_mem);
5737 %}
5738 
5739 instruct loadSSP(rRegP dst, stackSlotP src)
5740 %{
5741   match(Set dst src);
5742 
5743   ins_cost(125);
5744   format %{ "movq    $dst, $src\t# ptr stk" %}
5745   opcode(0x8B);
5746   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5747   ins_pipe(ialu_reg_mem);
5748 %}
5749 
5750 instruct loadSSF(regF dst, stackSlotF src)
5751 %{
5752   match(Set dst src);
5753 
5754   ins_cost(125);
5755   format %{ "movss   $dst, $src\t# float stk" %}
5756   ins_encode %{
5757     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5758   %}
5759   ins_pipe(pipe_slow); // XXX
5760 %}
5761 
5762 // Use the same format since predicate() can not be used here.
5763 instruct loadSSD(regD dst, stackSlotD src)
5764 %{
5765   match(Set dst src);
5766 
5767   ins_cost(125);
5768   format %{ "movsd   $dst, $src\t# double stk" %}
5769   ins_encode  %{
5770     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5771   %}
5772   ins_pipe(pipe_slow); // XXX
5773 %}
5774 
5775 // Prefetch instructions for allocation.
5776 // Must be safe to execute with invalid address (cannot fault).
5777 
5778 instruct prefetchAlloc( memory mem ) %{
5779   predicate(AllocatePrefetchInstr==3);
5780   match(PrefetchAllocation mem);
5781   ins_cost(125);
5782 
5783   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5784   ins_encode %{
5785     __ prefetchw($mem$$Address);
5786   %}
5787   ins_pipe(ialu_mem);
5788 %}
5789 
5790 instruct prefetchAllocNTA( memory mem ) %{
5791   predicate(AllocatePrefetchInstr==0);
5792   match(PrefetchAllocation mem);
5793   ins_cost(125);
5794 
5795   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5796   ins_encode %{
5797     __ prefetchnta($mem$$Address);
5798   %}
5799   ins_pipe(ialu_mem);
5800 %}
5801 
5802 instruct prefetchAllocT0( memory mem ) %{
5803   predicate(AllocatePrefetchInstr==1);
5804   match(PrefetchAllocation mem);
5805   ins_cost(125);
5806 
5807   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5808   ins_encode %{
5809     __ prefetcht0($mem$$Address);
5810   %}
5811   ins_pipe(ialu_mem);
5812 %}
5813 
5814 instruct prefetchAllocT2( memory mem ) %{
5815   predicate(AllocatePrefetchInstr==2);
5816   match(PrefetchAllocation mem);
5817   ins_cost(125);
5818 
5819   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5820   ins_encode %{
5821     __ prefetcht2($mem$$Address);
5822   %}
5823   ins_pipe(ialu_mem);
5824 %}
5825 
5826 //----------Store Instructions-------------------------------------------------
5827 
5828 // Store Byte
5829 instruct storeB(memory mem, rRegI src)
5830 %{
5831   match(Set mem (StoreB mem src));
5832 
5833   ins_cost(125); // XXX
5834   format %{ "movb    $mem, $src\t# byte" %}
5835   opcode(0x88);
5836   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5837   ins_pipe(ialu_mem_reg);
5838 %}
5839 
5840 // Store Char/Short
5841 instruct storeC(memory mem, rRegI src)
5842 %{
5843   match(Set mem (StoreC mem src));
5844 
5845   ins_cost(125); // XXX
5846   format %{ "movw    $mem, $src\t# char/short" %}
5847   opcode(0x89);
5848   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5849   ins_pipe(ialu_mem_reg);
5850 %}
5851 
5852 // Store Integer
5853 instruct storeI(memory mem, rRegI src)
5854 %{
5855   match(Set mem (StoreI mem src));
5856 
5857   ins_cost(125); // XXX
5858   format %{ "movl    $mem, $src\t# int" %}
5859   opcode(0x89);
5860   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5861   ins_pipe(ialu_mem_reg);
5862 %}
5863 
5864 // Store Long
5865 instruct storeL(memory mem, rRegL src)
5866 %{
5867   match(Set mem (StoreL mem src));
5868 
5869   ins_cost(125); // XXX
5870   format %{ "movq    $mem, $src\t# long" %}
5871   opcode(0x89);
5872   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5873   ins_pipe(ialu_mem_reg); // XXX
5874 %}
5875 
5876 // Store Pointer
5877 instruct storeP(memory mem, any_RegP src)
5878 %{
5879   match(Set mem (StoreP mem src));
5880 
5881   ins_cost(125); // XXX
5882   format %{ "movq    $mem, $src\t# ptr" %}
5883   opcode(0x89);
5884   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5885   ins_pipe(ialu_mem_reg);
5886 %}
5887 
5888 instruct storeImmP0(memory mem, immP0 zero)
5889 %{
5890   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5891   match(Set mem (StoreP mem zero));
5892 
5893   ins_cost(125); // XXX
5894   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5895   ins_encode %{
5896     __ movq($mem$$Address, r12);
5897   %}
5898   ins_pipe(ialu_mem_reg);
5899 %}
5900 
5901 // Store NULL Pointer, mark word, or other simple pointer constant.
5902 instruct storeImmP(memory mem, immP31 src)
5903 %{
5904   match(Set mem (StoreP mem src));
5905 
5906   ins_cost(150); // XXX
5907   format %{ "movq    $mem, $src\t# ptr" %}
5908   opcode(0xC7); /* C7 /0 */
5909   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5910   ins_pipe(ialu_mem_imm);
5911 %}
5912 
5913 // Store Compressed Pointer
5914 instruct storeN(memory mem, rRegN src)
5915 %{
5916   match(Set mem (StoreN mem src));
5917 
5918   ins_cost(125); // XXX
5919   format %{ "movl    $mem, $src\t# compressed ptr" %}
5920   ins_encode %{
5921     __ movl($mem$$Address, $src$$Register);
5922   %}
5923   ins_pipe(ialu_mem_reg);
5924 %}
5925 
5926 instruct storeNKlass(memory mem, rRegN src)
5927 %{
5928   match(Set mem (StoreNKlass mem src));
5929 
5930   ins_cost(125); // XXX
5931   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5932   ins_encode %{
5933     __ movl($mem$$Address, $src$$Register);
5934   %}
5935   ins_pipe(ialu_mem_reg);
5936 %}
5937 
5938 instruct storeImmN0(memory mem, immN0 zero)
5939 %{
5940   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5941   match(Set mem (StoreN mem zero));
5942 
5943   ins_cost(125); // XXX
5944   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5945   ins_encode %{
5946     __ movl($mem$$Address, r12);
5947   %}
5948   ins_pipe(ialu_mem_reg);
5949 %}
5950 
5951 instruct storeImmN(memory mem, immN src)
5952 %{
5953   match(Set mem (StoreN mem src));
5954 
5955   ins_cost(150); // XXX
5956   format %{ "movl    $mem, $src\t# compressed ptr" %}
5957   ins_encode %{
5958     address con = (address)$src$$constant;
5959     if (con == NULL) {
5960       __ movl($mem$$Address, (int32_t)0);
5961     } else {
5962       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5963     }
5964   %}
5965   ins_pipe(ialu_mem_imm);
5966 %}
5967 
5968 instruct storeImmNKlass(memory mem, immNKlass src)
5969 %{
5970   match(Set mem (StoreNKlass mem src));
5971 
5972   ins_cost(150); // XXX
5973   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5974   ins_encode %{
5975     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5976   %}
5977   ins_pipe(ialu_mem_imm);
5978 %}
5979 
5980 // Store Integer Immediate
5981 instruct storeImmI0(memory mem, immI0 zero)
5982 %{
5983   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5984   match(Set mem (StoreI mem zero));
5985 
5986   ins_cost(125); // XXX
5987   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5988   ins_encode %{
5989     __ movl($mem$$Address, r12);
5990   %}
5991   ins_pipe(ialu_mem_reg);
5992 %}
5993 
5994 instruct storeImmI(memory mem, immI src)
5995 %{
5996   match(Set mem (StoreI mem src));
5997 
5998   ins_cost(150);
5999   format %{ "movl    $mem, $src\t# int" %}
6000   opcode(0xC7); /* C7 /0 */
6001   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6002   ins_pipe(ialu_mem_imm);
6003 %}
6004 
6005 // Store Long Immediate
6006 instruct storeImmL0(memory mem, immL0 zero)
6007 %{
6008   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6009   match(Set mem (StoreL mem zero));
6010 
6011   ins_cost(125); // XXX
6012   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6013   ins_encode %{
6014     __ movq($mem$$Address, r12);
6015   %}
6016   ins_pipe(ialu_mem_reg);
6017 %}
6018 
6019 instruct storeImmL(memory mem, immL32 src)
6020 %{
6021   match(Set mem (StoreL mem src));
6022 
6023   ins_cost(150);
6024   format %{ "movq    $mem, $src\t# long" %}
6025   opcode(0xC7); /* C7 /0 */
6026   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6027   ins_pipe(ialu_mem_imm);
6028 %}
6029 
6030 // Store Short/Char Immediate
6031 instruct storeImmC0(memory mem, immI0 zero)
6032 %{
6033   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6034   match(Set mem (StoreC mem zero));
6035 
6036   ins_cost(125); // XXX
6037   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6038   ins_encode %{
6039     __ movw($mem$$Address, r12);
6040   %}
6041   ins_pipe(ialu_mem_reg);
6042 %}
6043 
6044 instruct storeImmI16(memory mem, immI16 src)
6045 %{
6046   predicate(UseStoreImmI16);
6047   match(Set mem (StoreC mem src));
6048 
6049   ins_cost(150);
6050   format %{ "movw    $mem, $src\t# short/char" %}
6051   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6052   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6053   ins_pipe(ialu_mem_imm);
6054 %}
6055 
6056 // Store Byte Immediate
6057 instruct storeImmB0(memory mem, immI0 zero)
6058 %{
6059   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6060   match(Set mem (StoreB mem zero));
6061 
6062   ins_cost(125); // XXX
6063   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6064   ins_encode %{
6065     __ movb($mem$$Address, r12);
6066   %}
6067   ins_pipe(ialu_mem_reg);
6068 %}
6069 
6070 instruct storeImmB(memory mem, immI8 src)
6071 %{
6072   match(Set mem (StoreB mem src));
6073 
6074   ins_cost(150); // XXX
6075   format %{ "movb    $mem, $src\t# byte" %}
6076   opcode(0xC6); /* C6 /0 */
6077   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6078   ins_pipe(ialu_mem_imm);
6079 %}
6080 
6081 // Store CMS card-mark Immediate
6082 instruct storeImmCM0_reg(memory mem, immI0 zero)
6083 %{
6084   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6085   match(Set mem (StoreCM mem zero));
6086 
6087   ins_cost(125); // XXX
6088   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6089   ins_encode %{
6090     __ movb($mem$$Address, r12);
6091   %}
6092   ins_pipe(ialu_mem_reg);
6093 %}
6094 
6095 instruct storeImmCM0(memory mem, immI0 src)
6096 %{
6097   match(Set mem (StoreCM mem src));
6098 
6099   ins_cost(150); // XXX
6100   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6101   opcode(0xC6); /* C6 /0 */
6102   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6103   ins_pipe(ialu_mem_imm);
6104 %}
6105 
6106 // Store Float
6107 instruct storeF(memory mem, regF src)
6108 %{
6109   match(Set mem (StoreF mem src));
6110 
6111   ins_cost(95); // XXX
6112   format %{ "movss   $mem, $src\t# float" %}
6113   ins_encode %{
6114     __ movflt($mem$$Address, $src$$XMMRegister);
6115   %}
6116   ins_pipe(pipe_slow); // XXX
6117 %}
6118 
6119 // Store immediate Float value (it is faster than store from XMM register)
6120 instruct storeF0(memory mem, immF0 zero)
6121 %{
6122   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6123   match(Set mem (StoreF mem zero));
6124 
6125   ins_cost(25); // XXX
6126   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6127   ins_encode %{
6128     __ movl($mem$$Address, r12);
6129   %}
6130   ins_pipe(ialu_mem_reg);
6131 %}
6132 
6133 instruct storeF_imm(memory mem, immF src)
6134 %{
6135   match(Set mem (StoreF mem src));
6136 
6137   ins_cost(50);
6138   format %{ "movl    $mem, $src\t# float" %}
6139   opcode(0xC7); /* C7 /0 */
6140   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6141   ins_pipe(ialu_mem_imm);
6142 %}
6143 
6144 // Store Double
6145 instruct storeD(memory mem, regD src)
6146 %{
6147   match(Set mem (StoreD mem src));
6148 
6149   ins_cost(95); // XXX
6150   format %{ "movsd   $mem, $src\t# double" %}
6151   ins_encode %{
6152     __ movdbl($mem$$Address, $src$$XMMRegister);
6153   %}
6154   ins_pipe(pipe_slow); // XXX
6155 %}
6156 
6157 // Store immediate double 0.0 (it is faster than store from XMM register)
6158 instruct storeD0_imm(memory mem, immD0 src)
6159 %{
6160   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6161   match(Set mem (StoreD mem src));
6162 
6163   ins_cost(50);
6164   format %{ "movq    $mem, $src\t# double 0." %}
6165   opcode(0xC7); /* C7 /0 */
6166   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6167   ins_pipe(ialu_mem_imm);
6168 %}
6169 
6170 instruct storeD0(memory mem, immD0 zero)
6171 %{
6172   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6173   match(Set mem (StoreD mem zero));
6174 
6175   ins_cost(25); // XXX
6176   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6177   ins_encode %{
6178     __ movq($mem$$Address, r12);
6179   %}
6180   ins_pipe(ialu_mem_reg);
6181 %}
6182 
6183 instruct storeSSI(stackSlotI dst, rRegI src)
6184 %{
6185   match(Set dst src);
6186 
6187   ins_cost(100);
6188   format %{ "movl    $dst, $src\t# int stk" %}
6189   opcode(0x89);
6190   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6191   ins_pipe( ialu_mem_reg );
6192 %}
6193 
6194 instruct storeSSL(stackSlotL dst, rRegL src)
6195 %{
6196   match(Set dst src);
6197 
6198   ins_cost(100);
6199   format %{ "movq    $dst, $src\t# long stk" %}
6200   opcode(0x89);
6201   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6202   ins_pipe(ialu_mem_reg);
6203 %}
6204 
6205 instruct storeSSP(stackSlotP dst, rRegP src)
6206 %{
6207   match(Set dst src);
6208 
6209   ins_cost(100);
6210   format %{ "movq    $dst, $src\t# ptr stk" %}
6211   opcode(0x89);
6212   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6213   ins_pipe(ialu_mem_reg);
6214 %}
6215 
6216 instruct storeSSF(stackSlotF dst, regF src)
6217 %{
6218   match(Set dst src);
6219 
6220   ins_cost(95); // XXX
6221   format %{ "movss   $dst, $src\t# float stk" %}
6222   ins_encode %{
6223     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6224   %}
6225   ins_pipe(pipe_slow); // XXX
6226 %}
6227 
6228 instruct storeSSD(stackSlotD dst, regD src)
6229 %{
6230   match(Set dst src);
6231 
6232   ins_cost(95); // XXX
6233   format %{ "movsd   $dst, $src\t# double stk" %}
6234   ins_encode %{
6235     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6236   %}
6237   ins_pipe(pipe_slow); // XXX
6238 %}
6239 






































6240 //----------BSWAP Instructions-------------------------------------------------
6241 instruct bytes_reverse_int(rRegI dst) %{
6242   match(Set dst (ReverseBytesI dst));
6243 
6244   format %{ "bswapl  $dst" %}
6245   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6246   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6247   ins_pipe( ialu_reg );
6248 %}
6249 
6250 instruct bytes_reverse_long(rRegL dst) %{
6251   match(Set dst (ReverseBytesL dst));
6252 
6253   format %{ "bswapq  $dst" %}
6254   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6255   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6256   ins_pipe( ialu_reg);
6257 %}
6258 
6259 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6260   match(Set dst (ReverseBytesUS dst));
6261   effect(KILL cr);
6262 
6263   format %{ "bswapl  $dst\n\t"
6264             "shrl    $dst,16\n\t" %}
6265   ins_encode %{
6266     __ bswapl($dst$$Register);
6267     __ shrl($dst$$Register, 16);
6268   %}
6269   ins_pipe( ialu_reg );
6270 %}
6271 
6272 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6273   match(Set dst (ReverseBytesS dst));
6274   effect(KILL cr);
6275 
6276   format %{ "bswapl  $dst\n\t"
6277             "sar     $dst,16\n\t" %}
6278   ins_encode %{
6279     __ bswapl($dst$$Register);
6280     __ sarl($dst$$Register, 16);
6281   %}
6282   ins_pipe( ialu_reg );
6283 %}
6284 
6285 //---------- Zeros Count Instructions ------------------------------------------
6286 
6287 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6288   predicate(UseCountLeadingZerosInstruction);
6289   match(Set dst (CountLeadingZerosI src));
6290   effect(KILL cr);
6291 
6292   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6293   ins_encode %{
6294     __ lzcntl($dst$$Register, $src$$Register);
6295   %}
6296   ins_pipe(ialu_reg);
6297 %}
6298 
6299 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6300   predicate(!UseCountLeadingZerosInstruction);
6301   match(Set dst (CountLeadingZerosI src));
6302   effect(KILL cr);
6303 
6304   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6305             "jnz     skip\n\t"
6306             "movl    $dst, -1\n"
6307       "skip:\n\t"
6308             "negl    $dst\n\t"
6309             "addl    $dst, 31" %}
6310   ins_encode %{
6311     Register Rdst = $dst$$Register;
6312     Register Rsrc = $src$$Register;
6313     Label skip;
6314     __ bsrl(Rdst, Rsrc);
6315     __ jccb(Assembler::notZero, skip);
6316     __ movl(Rdst, -1);
6317     __ bind(skip);
6318     __ negl(Rdst);
6319     __ addl(Rdst, BitsPerInt - 1);
6320   %}
6321   ins_pipe(ialu_reg);
6322 %}
6323 
6324 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6325   predicate(UseCountLeadingZerosInstruction);
6326   match(Set dst (CountLeadingZerosL src));
6327   effect(KILL cr);
6328 
6329   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6330   ins_encode %{
6331     __ lzcntq($dst$$Register, $src$$Register);
6332   %}
6333   ins_pipe(ialu_reg);
6334 %}
6335 
6336 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6337   predicate(!UseCountLeadingZerosInstruction);
6338   match(Set dst (CountLeadingZerosL src));
6339   effect(KILL cr);
6340 
6341   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6342             "jnz     skip\n\t"
6343             "movl    $dst, -1\n"
6344       "skip:\n\t"
6345             "negl    $dst\n\t"
6346             "addl    $dst, 63" %}
6347   ins_encode %{
6348     Register Rdst = $dst$$Register;
6349     Register Rsrc = $src$$Register;
6350     Label skip;
6351     __ bsrq(Rdst, Rsrc);
6352     __ jccb(Assembler::notZero, skip);
6353     __ movl(Rdst, -1);
6354     __ bind(skip);
6355     __ negl(Rdst);
6356     __ addl(Rdst, BitsPerLong - 1);
6357   %}
6358   ins_pipe(ialu_reg);
6359 %}
6360 
6361 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6362   predicate(UseCountTrailingZerosInstruction);
6363   match(Set dst (CountTrailingZerosI src));
6364   effect(KILL cr);
6365 
6366   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6367   ins_encode %{
6368     __ tzcntl($dst$$Register, $src$$Register);
6369   %}
6370   ins_pipe(ialu_reg);
6371 %}
6372 
6373 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6374   predicate(!UseCountTrailingZerosInstruction);
6375   match(Set dst (CountTrailingZerosI src));
6376   effect(KILL cr);
6377 
6378   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6379             "jnz     done\n\t"
6380             "movl    $dst, 32\n"
6381       "done:" %}
6382   ins_encode %{
6383     Register Rdst = $dst$$Register;
6384     Label done;
6385     __ bsfl(Rdst, $src$$Register);
6386     __ jccb(Assembler::notZero, done);
6387     __ movl(Rdst, BitsPerInt);
6388     __ bind(done);
6389   %}
6390   ins_pipe(ialu_reg);
6391 %}
6392 
6393 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6394   predicate(UseCountTrailingZerosInstruction);
6395   match(Set dst (CountTrailingZerosL src));
6396   effect(KILL cr);
6397 
6398   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6399   ins_encode %{
6400     __ tzcntq($dst$$Register, $src$$Register);
6401   %}
6402   ins_pipe(ialu_reg);
6403 %}
6404 
6405 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6406   predicate(!UseCountTrailingZerosInstruction);
6407   match(Set dst (CountTrailingZerosL src));
6408   effect(KILL cr);
6409 
6410   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6411             "jnz     done\n\t"
6412             "movl    $dst, 64\n"
6413       "done:" %}
6414   ins_encode %{
6415     Register Rdst = $dst$$Register;
6416     Label done;
6417     __ bsfq(Rdst, $src$$Register);
6418     __ jccb(Assembler::notZero, done);
6419     __ movl(Rdst, BitsPerLong);
6420     __ bind(done);
6421   %}
6422   ins_pipe(ialu_reg);
6423 %}
6424 
6425 
6426 //---------- Population Count Instructions -------------------------------------
6427 
6428 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6429   predicate(UsePopCountInstruction);
6430   match(Set dst (PopCountI src));
6431   effect(KILL cr);
6432 
6433   format %{ "popcnt  $dst, $src" %}
6434   ins_encode %{
6435     __ popcntl($dst$$Register, $src$$Register);
6436   %}
6437   ins_pipe(ialu_reg);
6438 %}
6439 
6440 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6441   predicate(UsePopCountInstruction);
6442   match(Set dst (PopCountI (LoadI mem)));
6443   effect(KILL cr);
6444 
6445   format %{ "popcnt  $dst, $mem" %}
6446   ins_encode %{
6447     __ popcntl($dst$$Register, $mem$$Address);
6448   %}
6449   ins_pipe(ialu_reg);
6450 %}
6451 
6452 // Note: Long.bitCount(long) returns an int.
6453 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6454   predicate(UsePopCountInstruction);
6455   match(Set dst (PopCountL src));
6456   effect(KILL cr);
6457 
6458   format %{ "popcnt  $dst, $src" %}
6459   ins_encode %{
6460     __ popcntq($dst$$Register, $src$$Register);
6461   %}
6462   ins_pipe(ialu_reg);
6463 %}
6464 
6465 // Note: Long.bitCount(long) returns an int.
6466 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6467   predicate(UsePopCountInstruction);
6468   match(Set dst (PopCountL (LoadL mem)));
6469   effect(KILL cr);
6470 
6471   format %{ "popcnt  $dst, $mem" %}
6472   ins_encode %{
6473     __ popcntq($dst$$Register, $mem$$Address);
6474   %}
6475   ins_pipe(ialu_reg);
6476 %}
6477 
6478 
6479 //----------MemBar Instructions-----------------------------------------------
6480 // Memory barrier flavors
6481 
6482 instruct membar_acquire()
6483 %{
6484   match(MemBarAcquire);
6485   match(LoadFence);
6486   ins_cost(0);
6487 
6488   size(0);
6489   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6490   ins_encode();
6491   ins_pipe(empty);
6492 %}
6493 
6494 instruct membar_acquire_lock()
6495 %{
6496   match(MemBarAcquireLock);
6497   ins_cost(0);
6498 
6499   size(0);
6500   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6501   ins_encode();
6502   ins_pipe(empty);
6503 %}
6504 
6505 instruct membar_release()
6506 %{
6507   match(MemBarRelease);
6508   match(StoreFence);
6509   ins_cost(0);
6510 
6511   size(0);
6512   format %{ "MEMBAR-release ! (empty encoding)" %}
6513   ins_encode();
6514   ins_pipe(empty);
6515 %}
6516 
6517 instruct membar_release_lock()
6518 %{
6519   match(MemBarReleaseLock);
6520   ins_cost(0);
6521 
6522   size(0);
6523   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6524   ins_encode();
6525   ins_pipe(empty);
6526 %}
6527 
6528 instruct membar_volatile(rFlagsReg cr) %{
6529   match(MemBarVolatile);
6530   effect(KILL cr);
6531   ins_cost(400);
6532 
6533   format %{
6534     $$template
6535     if (os::is_MP()) {
6536       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6537     } else {
6538       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6539     }
6540   %}
6541   ins_encode %{
6542     __ membar(Assembler::StoreLoad);
6543   %}
6544   ins_pipe(pipe_slow);
6545 %}
6546 
6547 instruct unnecessary_membar_volatile()
6548 %{
6549   match(MemBarVolatile);
6550   predicate(Matcher::post_store_load_barrier(n));
6551   ins_cost(0);
6552 
6553   size(0);
6554   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6555   ins_encode();
6556   ins_pipe(empty);
6557 %}
6558 
6559 instruct membar_storestore() %{
6560   match(MemBarStoreStore);
6561   ins_cost(0);
6562 
6563   size(0);
6564   format %{ "MEMBAR-storestore (empty encoding)" %}
6565   ins_encode( );
6566   ins_pipe(empty);
6567 %}
6568 
6569 //----------Move Instructions--------------------------------------------------
6570 
6571 instruct castX2P(rRegP dst, rRegL src)
6572 %{
6573   match(Set dst (CastX2P src));
6574 
6575   format %{ "movq    $dst, $src\t# long->ptr" %}
6576   ins_encode %{
6577     if ($dst$$reg != $src$$reg) {
6578       __ movptr($dst$$Register, $src$$Register);
6579     }
6580   %}
6581   ins_pipe(ialu_reg_reg); // XXX
6582 %}
6583 
6584 instruct castP2X(rRegL dst, rRegP src)
6585 %{
6586   match(Set dst (CastP2X src));
6587 
6588   format %{ "movq    $dst, $src\t# ptr -> long" %}
6589   ins_encode %{
6590     if ($dst$$reg != $src$$reg) {
6591       __ movptr($dst$$Register, $src$$Register);
6592     }
6593   %}
6594   ins_pipe(ialu_reg_reg); // XXX
6595 %}
6596 
6597 // Convert oop into int for vectors alignment masking
6598 instruct convP2I(rRegI dst, rRegP src)
6599 %{
6600   match(Set dst (ConvL2I (CastP2X src)));
6601 
6602   format %{ "movl    $dst, $src\t# ptr -> int" %}
6603   ins_encode %{
6604     __ movl($dst$$Register, $src$$Register);
6605   %}
6606   ins_pipe(ialu_reg_reg); // XXX
6607 %}
6608 
6609 // Convert compressed oop into int for vectors alignment masking
6610 // in case of 32bit oops (heap < 4Gb).
6611 instruct convN2I(rRegI dst, rRegN src)
6612 %{
6613   predicate(Universe::narrow_oop_shift() == 0);
6614   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6615 
6616   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6617   ins_encode %{
6618     __ movl($dst$$Register, $src$$Register);
6619   %}
6620   ins_pipe(ialu_reg_reg); // XXX
6621 %}
6622 
6623 // Convert oop pointer into compressed form
6624 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6625   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6626   match(Set dst (EncodeP src));
6627   effect(KILL cr);
6628   format %{ "encode_heap_oop $dst,$src" %}
6629   ins_encode %{
6630     Register s = $src$$Register;
6631     Register d = $dst$$Register;
6632     if (s != d) {
6633       __ movq(d, s);
6634     }
6635     __ encode_heap_oop(d);
6636   %}
6637   ins_pipe(ialu_reg_long);
6638 %}
6639 
6640 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6641   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6642   match(Set dst (EncodeP src));
6643   effect(KILL cr);
6644   format %{ "encode_heap_oop_not_null $dst,$src" %}
6645   ins_encode %{
6646     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6647   %}
6648   ins_pipe(ialu_reg_long);
6649 %}
6650 
6651 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6652   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6653             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6654   match(Set dst (DecodeN src));
6655   effect(KILL cr);
6656   format %{ "decode_heap_oop $dst,$src" %}
6657   ins_encode %{
6658     Register s = $src$$Register;
6659     Register d = $dst$$Register;
6660     if (s != d) {
6661       __ movq(d, s);
6662     }
6663     __ decode_heap_oop(d);
6664   %}
6665   ins_pipe(ialu_reg_long);
6666 %}
6667 
6668 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6669   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6670             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6671   match(Set dst (DecodeN src));
6672   effect(KILL cr);
6673   format %{ "decode_heap_oop_not_null $dst,$src" %}
6674   ins_encode %{
6675     Register s = $src$$Register;
6676     Register d = $dst$$Register;
6677     if (s != d) {
6678       __ decode_heap_oop_not_null(d, s);
6679     } else {
6680       __ decode_heap_oop_not_null(d);
6681     }
6682   %}
6683   ins_pipe(ialu_reg_long);
6684 %}
6685 
6686 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6687   match(Set dst (EncodePKlass src));
6688   effect(KILL cr);
6689   format %{ "encode_klass_not_null $dst,$src" %}
6690   ins_encode %{
6691     __ encode_klass_not_null($dst$$Register, $src$$Register);
6692   %}
6693   ins_pipe(ialu_reg_long);
6694 %}
6695 
6696 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6697   match(Set dst (DecodeNKlass src));
6698   effect(KILL cr);
6699   format %{ "decode_klass_not_null $dst,$src" %}
6700   ins_encode %{
6701     Register s = $src$$Register;
6702     Register d = $dst$$Register;
6703     if (s != d) {
6704       __ decode_klass_not_null(d, s);
6705     } else {
6706       __ decode_klass_not_null(d);
6707     }
6708   %}
6709   ins_pipe(ialu_reg_long);
6710 %}
6711 
6712 
6713 //----------Conditional Move---------------------------------------------------
6714 // Jump
6715 // dummy instruction for generating temp registers
6716 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6717   match(Jump (LShiftL switch_val shift));
6718   ins_cost(350);
6719   predicate(false);
6720   effect(TEMP dest);
6721 
6722   format %{ "leaq    $dest, [$constantaddress]\n\t"
6723             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6724   ins_encode %{
6725     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6726     // to do that and the compiler is using that register as one it can allocate.
6727     // So we build it all by hand.
6728     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6729     // ArrayAddress dispatch(table, index);
6730     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6731     __ lea($dest$$Register, $constantaddress);
6732     __ jmp(dispatch);
6733   %}
6734   ins_pipe(pipe_jmp);
6735 %}
6736 
6737 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6738   match(Jump (AddL (LShiftL switch_val shift) offset));
6739   ins_cost(350);
6740   effect(TEMP dest);
6741 
6742   format %{ "leaq    $dest, [$constantaddress]\n\t"
6743             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6744   ins_encode %{
6745     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6746     // to do that and the compiler is using that register as one it can allocate.
6747     // So we build it all by hand.
6748     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6749     // ArrayAddress dispatch(table, index);
6750     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6751     __ lea($dest$$Register, $constantaddress);
6752     __ jmp(dispatch);
6753   %}
6754   ins_pipe(pipe_jmp);
6755 %}
6756 
6757 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6758   match(Jump switch_val);
6759   ins_cost(350);
6760   effect(TEMP dest);
6761 
6762   format %{ "leaq    $dest, [$constantaddress]\n\t"
6763             "jmp     [$dest + $switch_val]\n\t" %}
6764   ins_encode %{
6765     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6766     // to do that and the compiler is using that register as one it can allocate.
6767     // So we build it all by hand.
6768     // Address index(noreg, switch_reg, Address::times_1);
6769     // ArrayAddress dispatch(table, index);
6770     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6771     __ lea($dest$$Register, $constantaddress);
6772     __ jmp(dispatch);
6773   %}
6774   ins_pipe(pipe_jmp);
6775 %}
6776 
6777 // Conditional move
6778 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6779 %{
6780   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6781 
6782   ins_cost(200); // XXX
6783   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6784   opcode(0x0F, 0x40);
6785   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6786   ins_pipe(pipe_cmov_reg);
6787 %}
6788 
6789 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6790   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6791 
6792   ins_cost(200); // XXX
6793   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6794   opcode(0x0F, 0x40);
6795   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6796   ins_pipe(pipe_cmov_reg);
6797 %}
6798 
6799 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6800   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6801   ins_cost(200);
6802   expand %{
6803     cmovI_regU(cop, cr, dst, src);
6804   %}
6805 %}
6806 
6807 // Conditional move
6808 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6809   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6810 
6811   ins_cost(250); // XXX
6812   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6813   opcode(0x0F, 0x40);
6814   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6815   ins_pipe(pipe_cmov_mem);
6816 %}
6817 
6818 // Conditional move
6819 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6820 %{
6821   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6822 
6823   ins_cost(250); // XXX
6824   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6825   opcode(0x0F, 0x40);
6826   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6827   ins_pipe(pipe_cmov_mem);
6828 %}
6829 
6830 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6831   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6832   ins_cost(250);
6833   expand %{
6834     cmovI_memU(cop, cr, dst, src);
6835   %}
6836 %}
6837 
6838 // Conditional move
6839 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6840 %{
6841   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6842 
6843   ins_cost(200); // XXX
6844   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6845   opcode(0x0F, 0x40);
6846   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6847   ins_pipe(pipe_cmov_reg);
6848 %}
6849 
6850 // Conditional move
6851 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6852 %{
6853   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6854 
6855   ins_cost(200); // XXX
6856   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6857   opcode(0x0F, 0x40);
6858   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6859   ins_pipe(pipe_cmov_reg);
6860 %}
6861 
6862 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6863   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6864   ins_cost(200);
6865   expand %{
6866     cmovN_regU(cop, cr, dst, src);
6867   %}
6868 %}
6869 
6870 // Conditional move
6871 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6872 %{
6873   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6874 
6875   ins_cost(200); // XXX
6876   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6877   opcode(0x0F, 0x40);
6878   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6879   ins_pipe(pipe_cmov_reg);  // XXX
6880 %}
6881 
6882 // Conditional move
6883 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6884 %{
6885   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6886 
6887   ins_cost(200); // XXX
6888   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6889   opcode(0x0F, 0x40);
6890   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6891   ins_pipe(pipe_cmov_reg); // XXX
6892 %}
6893 
6894 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6895   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6896   ins_cost(200);
6897   expand %{
6898     cmovP_regU(cop, cr, dst, src);
6899   %}
6900 %}
6901 
6902 // DISABLED: Requires the ADLC to emit a bottom_type call that
6903 // correctly meets the two pointer arguments; one is an incoming
6904 // register but the other is a memory operand.  ALSO appears to
6905 // be buggy with implicit null checks.
6906 //
6907 //// Conditional move
6908 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6909 //%{
6910 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6911 //  ins_cost(250);
6912 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6913 //  opcode(0x0F,0x40);
6914 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6915 //  ins_pipe( pipe_cmov_mem );
6916 //%}
6917 //
6918 //// Conditional move
6919 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6920 //%{
6921 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6922 //  ins_cost(250);
6923 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6924 //  opcode(0x0F,0x40);
6925 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6926 //  ins_pipe( pipe_cmov_mem );
6927 //%}
6928 
6929 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6930 %{
6931   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6932 
6933   ins_cost(200); // XXX
6934   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6935   opcode(0x0F, 0x40);
6936   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6937   ins_pipe(pipe_cmov_reg);  // XXX
6938 %}
6939 
6940 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6941 %{
6942   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6943 
6944   ins_cost(200); // XXX
6945   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6946   opcode(0x0F, 0x40);
6947   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6948   ins_pipe(pipe_cmov_mem);  // XXX
6949 %}
6950 
6951 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6952 %{
6953   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6954 
6955   ins_cost(200); // XXX
6956   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6957   opcode(0x0F, 0x40);
6958   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6959   ins_pipe(pipe_cmov_reg); // XXX
6960 %}
6961 
6962 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6963   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6964   ins_cost(200);
6965   expand %{
6966     cmovL_regU(cop, cr, dst, src);
6967   %}
6968 %}
6969 
6970 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6971 %{
6972   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6973 
6974   ins_cost(200); // XXX
6975   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6976   opcode(0x0F, 0x40);
6977   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6978   ins_pipe(pipe_cmov_mem); // XXX
6979 %}
6980 
6981 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6982   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6983   ins_cost(200);
6984   expand %{
6985     cmovL_memU(cop, cr, dst, src);
6986   %}
6987 %}
6988 
6989 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6990 %{
6991   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6992 
6993   ins_cost(200); // XXX
6994   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6995             "movss     $dst, $src\n"
6996     "skip:" %}
6997   ins_encode %{
6998     Label Lskip;
6999     // Invert sense of branch from sense of CMOV
7000     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7001     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7002     __ bind(Lskip);
7003   %}
7004   ins_pipe(pipe_slow);
7005 %}
7006 
7007 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7008 // %{
7009 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7010 
7011 //   ins_cost(200); // XXX
7012 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7013 //             "movss     $dst, $src\n"
7014 //     "skip:" %}
7015 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7016 //   ins_pipe(pipe_slow);
7017 // %}
7018 
7019 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7020 %{
7021   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7022 
7023   ins_cost(200); // XXX
7024   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7025             "movss     $dst, $src\n"
7026     "skip:" %}
7027   ins_encode %{
7028     Label Lskip;
7029     // Invert sense of branch from sense of CMOV
7030     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7031     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7032     __ bind(Lskip);
7033   %}
7034   ins_pipe(pipe_slow);
7035 %}
7036 
7037 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7038   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7039   ins_cost(200);
7040   expand %{
7041     cmovF_regU(cop, cr, dst, src);
7042   %}
7043 %}
7044 
7045 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7046 %{
7047   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7048 
7049   ins_cost(200); // XXX
7050   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7051             "movsd     $dst, $src\n"
7052     "skip:" %}
7053   ins_encode %{
7054     Label Lskip;
7055     // Invert sense of branch from sense of CMOV
7056     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7057     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7058     __ bind(Lskip);
7059   %}
7060   ins_pipe(pipe_slow);
7061 %}
7062 
7063 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7064 %{
7065   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7066 
7067   ins_cost(200); // XXX
7068   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7069             "movsd     $dst, $src\n"
7070     "skip:" %}
7071   ins_encode %{
7072     Label Lskip;
7073     // Invert sense of branch from sense of CMOV
7074     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7075     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7076     __ bind(Lskip);
7077   %}
7078   ins_pipe(pipe_slow);
7079 %}
7080 
7081 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7082   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7083   ins_cost(200);
7084   expand %{
7085     cmovD_regU(cop, cr, dst, src);
7086   %}
7087 %}
7088 
7089 //----------Arithmetic Instructions--------------------------------------------
7090 //----------Addition Instructions----------------------------------------------
7091 
7092 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7093 %{
7094   match(Set dst (AddI dst src));
7095   effect(KILL cr);
7096 
7097   format %{ "addl    $dst, $src\t# int" %}
7098   opcode(0x03);
7099   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7100   ins_pipe(ialu_reg_reg);
7101 %}
7102 
7103 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7104 %{
7105   match(Set dst (AddI dst src));
7106   effect(KILL cr);
7107 
7108   format %{ "addl    $dst, $src\t# int" %}
7109   opcode(0x81, 0x00); /* /0 id */
7110   ins_encode(OpcSErm(dst, src), Con8or32(src));
7111   ins_pipe( ialu_reg );
7112 %}
7113 
7114 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7115 %{
7116   match(Set dst (AddI dst (LoadI src)));
7117   effect(KILL cr);
7118 
7119   ins_cost(125); // XXX
7120   format %{ "addl    $dst, $src\t# int" %}
7121   opcode(0x03);
7122   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7123   ins_pipe(ialu_reg_mem);
7124 %}
7125 
7126 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7127 %{
7128   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7129   effect(KILL cr);
7130 
7131   ins_cost(150); // XXX
7132   format %{ "addl    $dst, $src\t# int" %}
7133   opcode(0x01); /* Opcode 01 /r */
7134   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7135   ins_pipe(ialu_mem_reg);
7136 %}
7137 
7138 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7139 %{
7140   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7141   effect(KILL cr);
7142 
7143   ins_cost(125); // XXX
7144   format %{ "addl    $dst, $src\t# int" %}
7145   opcode(0x81); /* Opcode 81 /0 id */
7146   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7147   ins_pipe(ialu_mem_imm);
7148 %}
7149 
7150 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7151 %{
7152   predicate(UseIncDec);
7153   match(Set dst (AddI dst src));
7154   effect(KILL cr);
7155 
7156   format %{ "incl    $dst\t# int" %}
7157   opcode(0xFF, 0x00); // FF /0
7158   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7159   ins_pipe(ialu_reg);
7160 %}
7161 
7162 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7163 %{
7164   predicate(UseIncDec);
7165   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7166   effect(KILL cr);
7167 
7168   ins_cost(125); // XXX
7169   format %{ "incl    $dst\t# int" %}
7170   opcode(0xFF); /* Opcode FF /0 */
7171   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7172   ins_pipe(ialu_mem_imm);
7173 %}
7174 
7175 // XXX why does that use AddI
7176 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7177 %{
7178   predicate(UseIncDec);
7179   match(Set dst (AddI dst src));
7180   effect(KILL cr);
7181 
7182   format %{ "decl    $dst\t# int" %}
7183   opcode(0xFF, 0x01); // FF /1
7184   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7185   ins_pipe(ialu_reg);
7186 %}
7187 
7188 // XXX why does that use AddI
7189 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7190 %{
7191   predicate(UseIncDec);
7192   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7193   effect(KILL cr);
7194 
7195   ins_cost(125); // XXX
7196   format %{ "decl    $dst\t# int" %}
7197   opcode(0xFF); /* Opcode FF /1 */
7198   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7199   ins_pipe(ialu_mem_imm);
7200 %}
7201 
7202 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7203 %{
7204   match(Set dst (AddI src0 src1));
7205 
7206   ins_cost(110);
7207   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7208   opcode(0x8D); /* 0x8D /r */
7209   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7210   ins_pipe(ialu_reg_reg);
7211 %}
7212 
7213 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7214 %{
7215   match(Set dst (AddL dst src));
7216   effect(KILL cr);
7217 
7218   format %{ "addq    $dst, $src\t# long" %}
7219   opcode(0x03);
7220   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7221   ins_pipe(ialu_reg_reg);
7222 %}
7223 
7224 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7225 %{
7226   match(Set dst (AddL dst src));
7227   effect(KILL cr);
7228 
7229   format %{ "addq    $dst, $src\t# long" %}
7230   opcode(0x81, 0x00); /* /0 id */
7231   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7232   ins_pipe( ialu_reg );
7233 %}
7234 
7235 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7236 %{
7237   match(Set dst (AddL dst (LoadL src)));
7238   effect(KILL cr);
7239 
7240   ins_cost(125); // XXX
7241   format %{ "addq    $dst, $src\t# long" %}
7242   opcode(0x03);
7243   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7244   ins_pipe(ialu_reg_mem);
7245 %}
7246 
7247 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7248 %{
7249   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7250   effect(KILL cr);
7251 
7252   ins_cost(150); // XXX
7253   format %{ "addq    $dst, $src\t# long" %}
7254   opcode(0x01); /* Opcode 01 /r */
7255   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7256   ins_pipe(ialu_mem_reg);
7257 %}
7258 
7259 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7260 %{
7261   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7262   effect(KILL cr);
7263 
7264   ins_cost(125); // XXX
7265   format %{ "addq    $dst, $src\t# long" %}
7266   opcode(0x81); /* Opcode 81 /0 id */
7267   ins_encode(REX_mem_wide(dst),
7268              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7269   ins_pipe(ialu_mem_imm);
7270 %}
7271 
7272 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7273 %{
7274   predicate(UseIncDec);
7275   match(Set dst (AddL dst src));
7276   effect(KILL cr);
7277 
7278   format %{ "incq    $dst\t# long" %}
7279   opcode(0xFF, 0x00); // FF /0
7280   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7281   ins_pipe(ialu_reg);
7282 %}
7283 
7284 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7285 %{
7286   predicate(UseIncDec);
7287   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7288   effect(KILL cr);
7289 
7290   ins_cost(125); // XXX
7291   format %{ "incq    $dst\t# long" %}
7292   opcode(0xFF); /* Opcode FF /0 */
7293   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7294   ins_pipe(ialu_mem_imm);
7295 %}
7296 
7297 // XXX why does that use AddL
7298 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7299 %{
7300   predicate(UseIncDec);
7301   match(Set dst (AddL dst src));
7302   effect(KILL cr);
7303 
7304   format %{ "decq    $dst\t# long" %}
7305   opcode(0xFF, 0x01); // FF /1
7306   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7307   ins_pipe(ialu_reg);
7308 %}
7309 
7310 // XXX why does that use AddL
7311 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7312 %{
7313   predicate(UseIncDec);
7314   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7315   effect(KILL cr);
7316 
7317   ins_cost(125); // XXX
7318   format %{ "decq    $dst\t# long" %}
7319   opcode(0xFF); /* Opcode FF /1 */
7320   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7321   ins_pipe(ialu_mem_imm);
7322 %}
7323 
7324 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7325 %{
7326   match(Set dst (AddL src0 src1));
7327 
7328   ins_cost(110);
7329   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7330   opcode(0x8D); /* 0x8D /r */
7331   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7332   ins_pipe(ialu_reg_reg);
7333 %}
7334 
7335 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7336 %{
7337   match(Set dst (AddP dst src));
7338   effect(KILL cr);
7339 
7340   format %{ "addq    $dst, $src\t# ptr" %}
7341   opcode(0x03);
7342   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7343   ins_pipe(ialu_reg_reg);
7344 %}
7345 
7346 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7347 %{
7348   match(Set dst (AddP dst src));
7349   effect(KILL cr);
7350 
7351   format %{ "addq    $dst, $src\t# ptr" %}
7352   opcode(0x81, 0x00); /* /0 id */
7353   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7354   ins_pipe( ialu_reg );
7355 %}
7356 
7357 // XXX addP mem ops ????
7358 
7359 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7360 %{
7361   match(Set dst (AddP src0 src1));
7362 
7363   ins_cost(110);
7364   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7365   opcode(0x8D); /* 0x8D /r */
7366   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7367   ins_pipe(ialu_reg_reg);
7368 %}
7369 
7370 instruct checkCastPP(rRegP dst)
7371 %{
7372   match(Set dst (CheckCastPP dst));
7373 
7374   size(0);
7375   format %{ "# checkcastPP of $dst" %}
7376   ins_encode(/* empty encoding */);
7377   ins_pipe(empty);
7378 %}
7379 
7380 instruct castPP(rRegP dst)
7381 %{
7382   match(Set dst (CastPP dst));
7383 
7384   size(0);
7385   format %{ "# castPP of $dst" %}
7386   ins_encode(/* empty encoding */);
7387   ins_pipe(empty);
7388 %}
7389 
7390 instruct castII(rRegI dst)
7391 %{
7392   match(Set dst (CastII dst));
7393 
7394   size(0);
7395   format %{ "# castII of $dst" %}
7396   ins_encode(/* empty encoding */);
7397   ins_cost(0);
7398   ins_pipe(empty);
7399 %}
7400 
7401 // LoadP-locked same as a regular LoadP when used with compare-swap
7402 instruct loadPLocked(rRegP dst, memory mem)
7403 %{
7404   match(Set dst (LoadPLocked mem));
7405 
7406   ins_cost(125); // XXX
7407   format %{ "movq    $dst, $mem\t# ptr locked" %}
7408   opcode(0x8B);
7409   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7410   ins_pipe(ialu_reg_mem); // XXX
7411 %}
7412 
7413 // Conditional-store of the updated heap-top.
7414 // Used during allocation of the shared heap.
7415 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7416 
7417 instruct storePConditional(memory heap_top_ptr,
7418                            rax_RegP oldval, rRegP newval,
7419                            rFlagsReg cr)
7420 %{
7421   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7422 
7423   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7424             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7425   opcode(0x0F, 0xB1);
7426   ins_encode(lock_prefix,
7427              REX_reg_mem_wide(newval, heap_top_ptr),
7428              OpcP, OpcS,
7429              reg_mem(newval, heap_top_ptr));
7430   ins_pipe(pipe_cmpxchg);
7431 %}
7432 
7433 // Conditional-store of an int value.
7434 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7435 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7436 %{
7437   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7438   effect(KILL oldval);
7439 
7440   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7441   opcode(0x0F, 0xB1);
7442   ins_encode(lock_prefix,
7443              REX_reg_mem(newval, mem),
7444              OpcP, OpcS,
7445              reg_mem(newval, mem));
7446   ins_pipe(pipe_cmpxchg);
7447 %}
7448 
7449 // Conditional-store of a long value.
7450 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7451 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7452 %{
7453   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7454   effect(KILL oldval);
7455 
7456   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7457   opcode(0x0F, 0xB1);
7458   ins_encode(lock_prefix,
7459              REX_reg_mem_wide(newval, mem),
7460              OpcP, OpcS,
7461              reg_mem(newval, mem));
7462   ins_pipe(pipe_cmpxchg);
7463 %}
7464 
7465 
7466 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7467 instruct compareAndSwapP(rRegI res,
7468                          memory mem_ptr,
7469                          rax_RegP oldval, rRegP newval,
7470                          rFlagsReg cr)
7471 %{
7472   predicate(VM_Version::supports_cx8());
7473   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7474   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7475   effect(KILL cr, KILL oldval);
7476 
7477   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7478             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7479             "sete    $res\n\t"
7480             "movzbl  $res, $res" %}
7481   opcode(0x0F, 0xB1);
7482   ins_encode(lock_prefix,
7483              REX_reg_mem_wide(newval, mem_ptr),
7484              OpcP, OpcS,
7485              reg_mem(newval, mem_ptr),
7486              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7487              REX_reg_breg(res, res), // movzbl
7488              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7489   ins_pipe( pipe_cmpxchg );
7490 %}
7491 
7492 instruct compareAndSwapL(rRegI res,
7493                          memory mem_ptr,
7494                          rax_RegL oldval, rRegL newval,
7495                          rFlagsReg cr)
7496 %{
7497   predicate(VM_Version::supports_cx8());
7498   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7499   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7500   effect(KILL cr, KILL oldval);
7501 
7502   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7503             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7504             "sete    $res\n\t"
7505             "movzbl  $res, $res" %}
7506   opcode(0x0F, 0xB1);
7507   ins_encode(lock_prefix,
7508              REX_reg_mem_wide(newval, mem_ptr),
7509              OpcP, OpcS,
7510              reg_mem(newval, mem_ptr),
7511              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7512              REX_reg_breg(res, res), // movzbl
7513              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7514   ins_pipe( pipe_cmpxchg );
7515 %}
7516 
7517 instruct compareAndSwapI(rRegI res,
7518                          memory mem_ptr,
7519                          rax_RegI oldval, rRegI newval,
7520                          rFlagsReg cr)
7521 %{
7522   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7523   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7524   effect(KILL cr, KILL oldval);
7525 
7526   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7527             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7528             "sete    $res\n\t"
7529             "movzbl  $res, $res" %}
7530   opcode(0x0F, 0xB1);
7531   ins_encode(lock_prefix,
7532              REX_reg_mem(newval, mem_ptr),
7533              OpcP, OpcS,
7534              reg_mem(newval, mem_ptr),
7535              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7536              REX_reg_breg(res, res), // movzbl
7537              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7538   ins_pipe( pipe_cmpxchg );
7539 %}
7540 
7541 instruct compareAndSwapB(rRegI res,
7542                          memory mem_ptr,
7543                          rax_RegI oldval, rRegI newval,
7544                          rFlagsReg cr)
7545 %{
7546   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7547   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7548   effect(KILL cr, KILL oldval);
7549 
7550   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7551             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7552             "sete    $res\n\t"
7553             "movzbl  $res, $res" %}
7554   opcode(0x0F, 0xB0);
7555   ins_encode(lock_prefix,
7556              REX_breg_mem(newval, mem_ptr),
7557              OpcP, OpcS,
7558              reg_mem(newval, mem_ptr),
7559              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7560              REX_reg_breg(res, res), // movzbl
7561              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7562   ins_pipe( pipe_cmpxchg );
7563 %}
7564 
7565 instruct compareAndSwapS(rRegI res,
7566                          memory mem_ptr,
7567                          rax_RegI oldval, rRegI newval,
7568                          rFlagsReg cr)
7569 %{
7570   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7571   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7572   effect(KILL cr, KILL oldval);
7573 
7574   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7575             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7576             "sete    $res\n\t"
7577             "movzbl  $res, $res" %}
7578   opcode(0x0F, 0xB1);
7579   ins_encode(lock_prefix,
7580              SizePrefix,
7581              REX_reg_mem(newval, mem_ptr),
7582              OpcP, OpcS,
7583              reg_mem(newval, mem_ptr),
7584              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7585              REX_reg_breg(res, res), // movzbl
7586              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7587   ins_pipe( pipe_cmpxchg );
7588 %}
7589 
7590 instruct compareAndSwapN(rRegI res,
7591                           memory mem_ptr,
7592                           rax_RegN oldval, rRegN newval,
7593                           rFlagsReg cr) %{
7594   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7595   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7596   effect(KILL cr, KILL oldval);
7597 
7598   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7599             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7600             "sete    $res\n\t"
7601             "movzbl  $res, $res" %}
7602   opcode(0x0F, 0xB1);
7603   ins_encode(lock_prefix,
7604              REX_reg_mem(newval, mem_ptr),
7605              OpcP, OpcS,
7606              reg_mem(newval, mem_ptr),
7607              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7608              REX_reg_breg(res, res), // movzbl
7609              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7610   ins_pipe( pipe_cmpxchg );
7611 %}
7612 
7613 instruct compareAndExchangeB(
7614                          memory mem_ptr,
7615                          rax_RegI oldval, rRegI newval,
7616                          rFlagsReg cr)
7617 %{
7618   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7619   effect(KILL cr);
7620 
7621   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7622             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7623   opcode(0x0F, 0xB0);
7624   ins_encode(lock_prefix,
7625              REX_breg_mem(newval, mem_ptr),
7626              OpcP, OpcS,
7627              reg_mem(newval, mem_ptr) // lock cmpxchg
7628              );
7629   ins_pipe( pipe_cmpxchg );
7630 %}
7631 
7632 instruct compareAndExchangeS(
7633                          memory mem_ptr,
7634                          rax_RegI oldval, rRegI newval,
7635                          rFlagsReg cr)
7636 %{
7637   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7638   effect(KILL cr);
7639 
7640   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7641             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7642   opcode(0x0F, 0xB1);
7643   ins_encode(lock_prefix,
7644              SizePrefix,
7645              REX_reg_mem(newval, mem_ptr),
7646              OpcP, OpcS,
7647              reg_mem(newval, mem_ptr) // lock cmpxchg
7648              );
7649   ins_pipe( pipe_cmpxchg );
7650 %}
7651 
7652 instruct compareAndExchangeI(
7653                          memory mem_ptr,
7654                          rax_RegI oldval, rRegI newval,
7655                          rFlagsReg cr)
7656 %{
7657   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7658   effect(KILL cr);
7659 
7660   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7661             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7662   opcode(0x0F, 0xB1);
7663   ins_encode(lock_prefix,
7664              REX_reg_mem(newval, mem_ptr),
7665              OpcP, OpcS,
7666              reg_mem(newval, mem_ptr) // lock cmpxchg
7667              );
7668   ins_pipe( pipe_cmpxchg );
7669 %}
7670 
7671 instruct compareAndExchangeL(
7672                          memory mem_ptr,
7673                          rax_RegL oldval, rRegL newval,
7674                          rFlagsReg cr)
7675 %{
7676   predicate(VM_Version::supports_cx8());
7677   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7678   effect(KILL cr);
7679 
7680   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7681             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7682   opcode(0x0F, 0xB1);
7683   ins_encode(lock_prefix,
7684              REX_reg_mem_wide(newval, mem_ptr),
7685              OpcP, OpcS,
7686              reg_mem(newval, mem_ptr)  // lock cmpxchg
7687             );
7688   ins_pipe( pipe_cmpxchg );
7689 %}
7690 
7691 instruct compareAndExchangeN(
7692                           memory mem_ptr,
7693                           rax_RegN oldval, rRegN newval,
7694                           rFlagsReg cr) %{
7695   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7696   effect(KILL cr);
7697 
7698   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7699             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7700   opcode(0x0F, 0xB1);
7701   ins_encode(lock_prefix,
7702              REX_reg_mem(newval, mem_ptr),
7703              OpcP, OpcS,
7704              reg_mem(newval, mem_ptr)  // lock cmpxchg
7705           );
7706   ins_pipe( pipe_cmpxchg );
7707 %}
7708 
7709 instruct compareAndExchangeP(
7710                          memory mem_ptr,
7711                          rax_RegP oldval, rRegP newval,
7712                          rFlagsReg cr)
7713 %{
7714   predicate(VM_Version::supports_cx8());
7715   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7716   effect(KILL cr);
7717 
7718   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7719             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7720   opcode(0x0F, 0xB1);
7721   ins_encode(lock_prefix,
7722              REX_reg_mem_wide(newval, mem_ptr),
7723              OpcP, OpcS,
7724              reg_mem(newval, mem_ptr)  // lock cmpxchg
7725           );
7726   ins_pipe( pipe_cmpxchg );
7727 %}
7728 
7729 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7730   predicate(n->as_LoadStore()->result_not_used());
7731   match(Set dummy (GetAndAddB mem add));
7732   effect(KILL cr);
7733   format %{ "ADDB  [$mem],$add" %}
7734   ins_encode %{
7735     if (os::is_MP()) { __ lock(); }
7736     __ addb($mem$$Address, $add$$constant);
7737   %}
7738   ins_pipe( pipe_cmpxchg );
7739 %}
7740 
7741 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7742   match(Set newval (GetAndAddB mem newval));
7743   effect(KILL cr);
7744   format %{ "XADDB  [$mem],$newval" %}
7745   ins_encode %{
7746     if (os::is_MP()) { __ lock(); }
7747     __ xaddb($mem$$Address, $newval$$Register);
7748   %}
7749   ins_pipe( pipe_cmpxchg );
7750 %}
7751 
7752 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7753   predicate(n->as_LoadStore()->result_not_used());
7754   match(Set dummy (GetAndAddS mem add));
7755   effect(KILL cr);
7756   format %{ "ADDW  [$mem],$add" %}
7757   ins_encode %{
7758     if (os::is_MP()) { __ lock(); }
7759     __ addw($mem$$Address, $add$$constant);
7760   %}
7761   ins_pipe( pipe_cmpxchg );
7762 %}
7763 
7764 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7765   match(Set newval (GetAndAddS mem newval));
7766   effect(KILL cr);
7767   format %{ "XADDW  [$mem],$newval" %}
7768   ins_encode %{
7769     if (os::is_MP()) { __ lock(); }
7770     __ xaddw($mem$$Address, $newval$$Register);
7771   %}
7772   ins_pipe( pipe_cmpxchg );
7773 %}
7774 
7775 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7776   predicate(n->as_LoadStore()->result_not_used());
7777   match(Set dummy (GetAndAddI mem add));
7778   effect(KILL cr);
7779   format %{ "ADDL  [$mem],$add" %}
7780   ins_encode %{
7781     if (os::is_MP()) { __ lock(); }
7782     __ addl($mem$$Address, $add$$constant);
7783   %}
7784   ins_pipe( pipe_cmpxchg );
7785 %}
7786 
7787 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7788   match(Set newval (GetAndAddI mem newval));
7789   effect(KILL cr);
7790   format %{ "XADDL  [$mem],$newval" %}
7791   ins_encode %{
7792     if (os::is_MP()) { __ lock(); }
7793     __ xaddl($mem$$Address, $newval$$Register);
7794   %}
7795   ins_pipe( pipe_cmpxchg );
7796 %}
7797 
7798 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7799   predicate(n->as_LoadStore()->result_not_used());
7800   match(Set dummy (GetAndAddL mem add));
7801   effect(KILL cr);
7802   format %{ "ADDQ  [$mem],$add" %}
7803   ins_encode %{
7804     if (os::is_MP()) { __ lock(); }
7805     __ addq($mem$$Address, $add$$constant);
7806   %}
7807   ins_pipe( pipe_cmpxchg );
7808 %}
7809 
7810 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7811   match(Set newval (GetAndAddL mem newval));
7812   effect(KILL cr);
7813   format %{ "XADDQ  [$mem],$newval" %}
7814   ins_encode %{
7815     if (os::is_MP()) { __ lock(); }
7816     __ xaddq($mem$$Address, $newval$$Register);
7817   %}
7818   ins_pipe( pipe_cmpxchg );
7819 %}
7820 
7821 instruct xchgB( memory mem, rRegI newval) %{
7822   match(Set newval (GetAndSetB mem newval));
7823   format %{ "XCHGB  $newval,[$mem]" %}
7824   ins_encode %{
7825     __ xchgb($newval$$Register, $mem$$Address);
7826   %}
7827   ins_pipe( pipe_cmpxchg );
7828 %}
7829 
7830 instruct xchgS( memory mem, rRegI newval) %{
7831   match(Set newval (GetAndSetS mem newval));
7832   format %{ "XCHGW  $newval,[$mem]" %}
7833   ins_encode %{
7834     __ xchgw($newval$$Register, $mem$$Address);
7835   %}
7836   ins_pipe( pipe_cmpxchg );
7837 %}
7838 
7839 instruct xchgI( memory mem, rRegI newval) %{
7840   match(Set newval (GetAndSetI mem newval));
7841   format %{ "XCHGL  $newval,[$mem]" %}
7842   ins_encode %{
7843     __ xchgl($newval$$Register, $mem$$Address);
7844   %}
7845   ins_pipe( pipe_cmpxchg );
7846 %}
7847 
7848 instruct xchgL( memory mem, rRegL newval) %{
7849   match(Set newval (GetAndSetL mem newval));
7850   format %{ "XCHGL  $newval,[$mem]" %}
7851   ins_encode %{
7852     __ xchgq($newval$$Register, $mem$$Address);
7853   %}
7854   ins_pipe( pipe_cmpxchg );
7855 %}
7856 
7857 instruct xchgP( memory mem, rRegP newval) %{
7858   match(Set newval (GetAndSetP mem newval));
7859   format %{ "XCHGQ  $newval,[$mem]" %}
7860   ins_encode %{
7861     __ xchgq($newval$$Register, $mem$$Address);
7862   %}
7863   ins_pipe( pipe_cmpxchg );
7864 %}
7865 
7866 instruct xchgN( memory mem, rRegN newval) %{
7867   match(Set newval (GetAndSetN mem newval));
7868   format %{ "XCHGL  $newval,$mem]" %}
7869   ins_encode %{
7870     __ xchgl($newval$$Register, $mem$$Address);
7871   %}
7872   ins_pipe( pipe_cmpxchg );
7873 %}
7874 
7875 //----------Subtraction Instructions-------------------------------------------
7876 
7877 // Integer Subtraction Instructions
7878 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7879 %{
7880   match(Set dst (SubI dst src));
7881   effect(KILL cr);
7882 
7883   format %{ "subl    $dst, $src\t# int" %}
7884   opcode(0x2B);
7885   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7886   ins_pipe(ialu_reg_reg);
7887 %}
7888 
7889 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7890 %{
7891   match(Set dst (SubI dst src));
7892   effect(KILL cr);
7893 
7894   format %{ "subl    $dst, $src\t# int" %}
7895   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7896   ins_encode(OpcSErm(dst, src), Con8or32(src));
7897   ins_pipe(ialu_reg);
7898 %}
7899 
7900 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7901 %{
7902   match(Set dst (SubI dst (LoadI src)));
7903   effect(KILL cr);
7904 
7905   ins_cost(125);
7906   format %{ "subl    $dst, $src\t# int" %}
7907   opcode(0x2B);
7908   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7909   ins_pipe(ialu_reg_mem);
7910 %}
7911 
7912 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7913 %{
7914   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7915   effect(KILL cr);
7916 
7917   ins_cost(150);
7918   format %{ "subl    $dst, $src\t# int" %}
7919   opcode(0x29); /* Opcode 29 /r */
7920   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7921   ins_pipe(ialu_mem_reg);
7922 %}
7923 
7924 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7925 %{
7926   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7927   effect(KILL cr);
7928 
7929   ins_cost(125); // XXX
7930   format %{ "subl    $dst, $src\t# int" %}
7931   opcode(0x81); /* Opcode 81 /5 id */
7932   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7933   ins_pipe(ialu_mem_imm);
7934 %}
7935 
7936 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7937 %{
7938   match(Set dst (SubL dst src));
7939   effect(KILL cr);
7940 
7941   format %{ "subq    $dst, $src\t# long" %}
7942   opcode(0x2B);
7943   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7944   ins_pipe(ialu_reg_reg);
7945 %}
7946 
7947 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7948 %{
7949   match(Set dst (SubL dst src));
7950   effect(KILL cr);
7951 
7952   format %{ "subq    $dst, $src\t# long" %}
7953   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7954   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7955   ins_pipe(ialu_reg);
7956 %}
7957 
7958 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7959 %{
7960   match(Set dst (SubL dst (LoadL src)));
7961   effect(KILL cr);
7962 
7963   ins_cost(125);
7964   format %{ "subq    $dst, $src\t# long" %}
7965   opcode(0x2B);
7966   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7967   ins_pipe(ialu_reg_mem);
7968 %}
7969 
7970 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7971 %{
7972   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7973   effect(KILL cr);
7974 
7975   ins_cost(150);
7976   format %{ "subq    $dst, $src\t# long" %}
7977   opcode(0x29); /* Opcode 29 /r */
7978   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7979   ins_pipe(ialu_mem_reg);
7980 %}
7981 
7982 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7983 %{
7984   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7985   effect(KILL cr);
7986 
7987   ins_cost(125); // XXX
7988   format %{ "subq    $dst, $src\t# long" %}
7989   opcode(0x81); /* Opcode 81 /5 id */
7990   ins_encode(REX_mem_wide(dst),
7991              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7992   ins_pipe(ialu_mem_imm);
7993 %}
7994 
7995 // Subtract from a pointer
7996 // XXX hmpf???
7997 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7998 %{
7999   match(Set dst (AddP dst (SubI zero src)));
8000   effect(KILL cr);
8001 
8002   format %{ "subq    $dst, $src\t# ptr - int" %}
8003   opcode(0x2B);
8004   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8005   ins_pipe(ialu_reg_reg);
8006 %}
8007 
8008 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8009 %{
8010   match(Set dst (SubI zero dst));
8011   effect(KILL cr);
8012 
8013   format %{ "negl    $dst\t# int" %}
8014   opcode(0xF7, 0x03);  // Opcode F7 /3
8015   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8016   ins_pipe(ialu_reg);
8017 %}
8018 
8019 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8020 %{
8021   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8022   effect(KILL cr);
8023 
8024   format %{ "negl    $dst\t# int" %}
8025   opcode(0xF7, 0x03);  // Opcode F7 /3
8026   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8027   ins_pipe(ialu_reg);
8028 %}
8029 
8030 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8031 %{
8032   match(Set dst (SubL zero dst));
8033   effect(KILL cr);
8034 
8035   format %{ "negq    $dst\t# long" %}
8036   opcode(0xF7, 0x03);  // Opcode F7 /3
8037   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8038   ins_pipe(ialu_reg);
8039 %}
8040 
8041 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8042 %{
8043   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8044   effect(KILL cr);
8045 
8046   format %{ "negq    $dst\t# long" %}
8047   opcode(0xF7, 0x03);  // Opcode F7 /3
8048   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8049   ins_pipe(ialu_reg);
8050 %}
8051 
8052 //----------Multiplication/Division Instructions-------------------------------
8053 // Integer Multiplication Instructions
8054 // Multiply Register
8055 
8056 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8057 %{
8058   match(Set dst (MulI dst src));
8059   effect(KILL cr);
8060 
8061   ins_cost(300);
8062   format %{ "imull   $dst, $src\t# int" %}
8063   opcode(0x0F, 0xAF);
8064   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8065   ins_pipe(ialu_reg_reg_alu0);
8066 %}
8067 
8068 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8069 %{
8070   match(Set dst (MulI src imm));
8071   effect(KILL cr);
8072 
8073   ins_cost(300);
8074   format %{ "imull   $dst, $src, $imm\t# int" %}
8075   opcode(0x69); /* 69 /r id */
8076   ins_encode(REX_reg_reg(dst, src),
8077              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8078   ins_pipe(ialu_reg_reg_alu0);
8079 %}
8080 
8081 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8082 %{
8083   match(Set dst (MulI dst (LoadI src)));
8084   effect(KILL cr);
8085 
8086   ins_cost(350);
8087   format %{ "imull   $dst, $src\t# int" %}
8088   opcode(0x0F, 0xAF);
8089   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8090   ins_pipe(ialu_reg_mem_alu0);
8091 %}
8092 
8093 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8094 %{
8095   match(Set dst (MulI (LoadI src) imm));
8096   effect(KILL cr);
8097 
8098   ins_cost(300);
8099   format %{ "imull   $dst, $src, $imm\t# int" %}
8100   opcode(0x69); /* 69 /r id */
8101   ins_encode(REX_reg_mem(dst, src),
8102              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8103   ins_pipe(ialu_reg_mem_alu0);
8104 %}
8105 
8106 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8107 %{
8108   match(Set dst (MulL dst src));
8109   effect(KILL cr);
8110 
8111   ins_cost(300);
8112   format %{ "imulq   $dst, $src\t# long" %}
8113   opcode(0x0F, 0xAF);
8114   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8115   ins_pipe(ialu_reg_reg_alu0);
8116 %}
8117 
8118 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8119 %{
8120   match(Set dst (MulL src imm));
8121   effect(KILL cr);
8122 
8123   ins_cost(300);
8124   format %{ "imulq   $dst, $src, $imm\t# long" %}
8125   opcode(0x69); /* 69 /r id */
8126   ins_encode(REX_reg_reg_wide(dst, src),
8127              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8128   ins_pipe(ialu_reg_reg_alu0);
8129 %}
8130 
8131 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8132 %{
8133   match(Set dst (MulL dst (LoadL src)));
8134   effect(KILL cr);
8135 
8136   ins_cost(350);
8137   format %{ "imulq   $dst, $src\t# long" %}
8138   opcode(0x0F, 0xAF);
8139   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8140   ins_pipe(ialu_reg_mem_alu0);
8141 %}
8142 
8143 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8144 %{
8145   match(Set dst (MulL (LoadL src) imm));
8146   effect(KILL cr);
8147 
8148   ins_cost(300);
8149   format %{ "imulq   $dst, $src, $imm\t# long" %}
8150   opcode(0x69); /* 69 /r id */
8151   ins_encode(REX_reg_mem_wide(dst, src),
8152              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8153   ins_pipe(ialu_reg_mem_alu0);
8154 %}
8155 
8156 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8157 %{
8158   match(Set dst (MulHiL src rax));
8159   effect(USE_KILL rax, KILL cr);
8160 
8161   ins_cost(300);
8162   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8163   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8164   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8165   ins_pipe(ialu_reg_reg_alu0);
8166 %}
8167 
8168 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8169                    rFlagsReg cr)
8170 %{
8171   match(Set rax (DivI rax div));
8172   effect(KILL rdx, KILL cr);
8173 
8174   ins_cost(30*100+10*100); // XXX
8175   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8176             "jne,s   normal\n\t"
8177             "xorl    rdx, rdx\n\t"
8178             "cmpl    $div, -1\n\t"
8179             "je,s    done\n"
8180     "normal: cdql\n\t"
8181             "idivl   $div\n"
8182     "done:"        %}
8183   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8184   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8185   ins_pipe(ialu_reg_reg_alu0);
8186 %}
8187 
8188 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8189                    rFlagsReg cr)
8190 %{
8191   match(Set rax (DivL rax div));
8192   effect(KILL rdx, KILL cr);
8193 
8194   ins_cost(30*100+10*100); // XXX
8195   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8196             "cmpq    rax, rdx\n\t"
8197             "jne,s   normal\n\t"
8198             "xorl    rdx, rdx\n\t"
8199             "cmpq    $div, -1\n\t"
8200             "je,s    done\n"
8201     "normal: cdqq\n\t"
8202             "idivq   $div\n"
8203     "done:"        %}
8204   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8205   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8206   ins_pipe(ialu_reg_reg_alu0);
8207 %}
8208 
8209 // Integer DIVMOD with Register, both quotient and mod results
8210 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8211                              rFlagsReg cr)
8212 %{
8213   match(DivModI rax div);
8214   effect(KILL cr);
8215 
8216   ins_cost(30*100+10*100); // XXX
8217   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8218             "jne,s   normal\n\t"
8219             "xorl    rdx, rdx\n\t"
8220             "cmpl    $div, -1\n\t"
8221             "je,s    done\n"
8222     "normal: cdql\n\t"
8223             "idivl   $div\n"
8224     "done:"        %}
8225   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8226   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8227   ins_pipe(pipe_slow);
8228 %}
8229 
8230 // Long DIVMOD with Register, both quotient and mod results
8231 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8232                              rFlagsReg cr)
8233 %{
8234   match(DivModL rax div);
8235   effect(KILL cr);
8236 
8237   ins_cost(30*100+10*100); // XXX
8238   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8239             "cmpq    rax, rdx\n\t"
8240             "jne,s   normal\n\t"
8241             "xorl    rdx, rdx\n\t"
8242             "cmpq    $div, -1\n\t"
8243             "je,s    done\n"
8244     "normal: cdqq\n\t"
8245             "idivq   $div\n"
8246     "done:"        %}
8247   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8248   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8249   ins_pipe(pipe_slow);
8250 %}
8251 
8252 //----------- DivL-By-Constant-Expansions--------------------------------------
8253 // DivI cases are handled by the compiler
8254 
8255 // Magic constant, reciprocal of 10
8256 instruct loadConL_0x6666666666666667(rRegL dst)
8257 %{
8258   effect(DEF dst);
8259 
8260   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8261   ins_encode(load_immL(dst, 0x6666666666666667));
8262   ins_pipe(ialu_reg);
8263 %}
8264 
8265 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8266 %{
8267   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8268 
8269   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8270   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8271   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8272   ins_pipe(ialu_reg_reg_alu0);
8273 %}
8274 
8275 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8276 %{
8277   effect(USE_DEF dst, KILL cr);
8278 
8279   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8280   opcode(0xC1, 0x7); /* C1 /7 ib */
8281   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8282   ins_pipe(ialu_reg);
8283 %}
8284 
8285 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8286 %{
8287   effect(USE_DEF dst, KILL cr);
8288 
8289   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8290   opcode(0xC1, 0x7); /* C1 /7 ib */
8291   ins_encode(reg_opc_imm_wide(dst, 0x2));
8292   ins_pipe(ialu_reg);
8293 %}
8294 
8295 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8296 %{
8297   match(Set dst (DivL src div));
8298 
8299   ins_cost((5+8)*100);
8300   expand %{
8301     rax_RegL rax;                     // Killed temp
8302     rFlagsReg cr;                     // Killed
8303     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8304     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8305     sarL_rReg_63(src, cr);            // sarq  src, 63
8306     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8307     subL_rReg(dst, src, cr);          // subl  rdx, src
8308   %}
8309 %}
8310 
8311 //-----------------------------------------------------------------------------
8312 
8313 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8314                    rFlagsReg cr)
8315 %{
8316   match(Set rdx (ModI rax div));
8317   effect(KILL rax, KILL cr);
8318 
8319   ins_cost(300); // XXX
8320   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8321             "jne,s   normal\n\t"
8322             "xorl    rdx, rdx\n\t"
8323             "cmpl    $div, -1\n\t"
8324             "je,s    done\n"
8325     "normal: cdql\n\t"
8326             "idivl   $div\n"
8327     "done:"        %}
8328   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8329   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8330   ins_pipe(ialu_reg_reg_alu0);
8331 %}
8332 
8333 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8334                    rFlagsReg cr)
8335 %{
8336   match(Set rdx (ModL rax div));
8337   effect(KILL rax, KILL cr);
8338 
8339   ins_cost(300); // XXX
8340   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8341             "cmpq    rax, rdx\n\t"
8342             "jne,s   normal\n\t"
8343             "xorl    rdx, rdx\n\t"
8344             "cmpq    $div, -1\n\t"
8345             "je,s    done\n"
8346     "normal: cdqq\n\t"
8347             "idivq   $div\n"
8348     "done:"        %}
8349   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8350   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8351   ins_pipe(ialu_reg_reg_alu0);
8352 %}
8353 
8354 // Integer Shift Instructions
8355 // Shift Left by one
8356 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8357 %{
8358   match(Set dst (LShiftI dst shift));
8359   effect(KILL cr);
8360 
8361   format %{ "sall    $dst, $shift" %}
8362   opcode(0xD1, 0x4); /* D1 /4 */
8363   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8364   ins_pipe(ialu_reg);
8365 %}
8366 
8367 // Shift Left by one
8368 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8369 %{
8370   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8371   effect(KILL cr);
8372 
8373   format %{ "sall    $dst, $shift\t" %}
8374   opcode(0xD1, 0x4); /* D1 /4 */
8375   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8376   ins_pipe(ialu_mem_imm);
8377 %}
8378 
8379 // Shift Left by 8-bit immediate
8380 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8381 %{
8382   match(Set dst (LShiftI dst shift));
8383   effect(KILL cr);
8384 
8385   format %{ "sall    $dst, $shift" %}
8386   opcode(0xC1, 0x4); /* C1 /4 ib */
8387   ins_encode(reg_opc_imm(dst, shift));
8388   ins_pipe(ialu_reg);
8389 %}
8390 
8391 // Shift Left by 8-bit immediate
8392 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8393 %{
8394   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8395   effect(KILL cr);
8396 
8397   format %{ "sall    $dst, $shift" %}
8398   opcode(0xC1, 0x4); /* C1 /4 ib */
8399   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8400   ins_pipe(ialu_mem_imm);
8401 %}
8402 
8403 // Shift Left by variable
8404 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8405 %{
8406   match(Set dst (LShiftI dst shift));
8407   effect(KILL cr);
8408 
8409   format %{ "sall    $dst, $shift" %}
8410   opcode(0xD3, 0x4); /* D3 /4 */
8411   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8412   ins_pipe(ialu_reg_reg);
8413 %}
8414 
8415 // Shift Left by variable
8416 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8417 %{
8418   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8419   effect(KILL cr);
8420 
8421   format %{ "sall    $dst, $shift" %}
8422   opcode(0xD3, 0x4); /* D3 /4 */
8423   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8424   ins_pipe(ialu_mem_reg);
8425 %}
8426 
8427 // Arithmetic shift right by one
8428 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8429 %{
8430   match(Set dst (RShiftI dst shift));
8431   effect(KILL cr);
8432 
8433   format %{ "sarl    $dst, $shift" %}
8434   opcode(0xD1, 0x7); /* D1 /7 */
8435   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8436   ins_pipe(ialu_reg);
8437 %}
8438 
8439 // Arithmetic shift right by one
8440 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8441 %{
8442   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8443   effect(KILL cr);
8444 
8445   format %{ "sarl    $dst, $shift" %}
8446   opcode(0xD1, 0x7); /* D1 /7 */
8447   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8448   ins_pipe(ialu_mem_imm);
8449 %}
8450 
8451 // Arithmetic Shift Right by 8-bit immediate
8452 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8453 %{
8454   match(Set dst (RShiftI dst shift));
8455   effect(KILL cr);
8456 
8457   format %{ "sarl    $dst, $shift" %}
8458   opcode(0xC1, 0x7); /* C1 /7 ib */
8459   ins_encode(reg_opc_imm(dst, shift));
8460   ins_pipe(ialu_mem_imm);
8461 %}
8462 
8463 // Arithmetic Shift Right by 8-bit immediate
8464 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8465 %{
8466   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8467   effect(KILL cr);
8468 
8469   format %{ "sarl    $dst, $shift" %}
8470   opcode(0xC1, 0x7); /* C1 /7 ib */
8471   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8472   ins_pipe(ialu_mem_imm);
8473 %}
8474 
8475 // Arithmetic Shift Right by variable
8476 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8477 %{
8478   match(Set dst (RShiftI dst shift));
8479   effect(KILL cr);
8480 
8481   format %{ "sarl    $dst, $shift" %}
8482   opcode(0xD3, 0x7); /* D3 /7 */
8483   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8484   ins_pipe(ialu_reg_reg);
8485 %}
8486 
8487 // Arithmetic Shift Right by variable
8488 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8489 %{
8490   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8491   effect(KILL cr);
8492 
8493   format %{ "sarl    $dst, $shift" %}
8494   opcode(0xD3, 0x7); /* D3 /7 */
8495   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8496   ins_pipe(ialu_mem_reg);
8497 %}
8498 
8499 // Logical shift right by one
8500 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8501 %{
8502   match(Set dst (URShiftI dst shift));
8503   effect(KILL cr);
8504 
8505   format %{ "shrl    $dst, $shift" %}
8506   opcode(0xD1, 0x5); /* D1 /5 */
8507   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8508   ins_pipe(ialu_reg);
8509 %}
8510 
8511 // Logical shift right by one
8512 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8513 %{
8514   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8515   effect(KILL cr);
8516 
8517   format %{ "shrl    $dst, $shift" %}
8518   opcode(0xD1, 0x5); /* D1 /5 */
8519   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8520   ins_pipe(ialu_mem_imm);
8521 %}
8522 
8523 // Logical Shift Right by 8-bit immediate
8524 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8525 %{
8526   match(Set dst (URShiftI dst shift));
8527   effect(KILL cr);
8528 
8529   format %{ "shrl    $dst, $shift" %}
8530   opcode(0xC1, 0x5); /* C1 /5 ib */
8531   ins_encode(reg_opc_imm(dst, shift));
8532   ins_pipe(ialu_reg);
8533 %}
8534 
8535 // Logical Shift Right by 8-bit immediate
8536 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8537 %{
8538   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8539   effect(KILL cr);
8540 
8541   format %{ "shrl    $dst, $shift" %}
8542   opcode(0xC1, 0x5); /* C1 /5 ib */
8543   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8544   ins_pipe(ialu_mem_imm);
8545 %}
8546 
8547 // Logical Shift Right by variable
8548 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8549 %{
8550   match(Set dst (URShiftI dst shift));
8551   effect(KILL cr);
8552 
8553   format %{ "shrl    $dst, $shift" %}
8554   opcode(0xD3, 0x5); /* D3 /5 */
8555   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8556   ins_pipe(ialu_reg_reg);
8557 %}
8558 
8559 // Logical Shift Right by variable
8560 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8561 %{
8562   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8563   effect(KILL cr);
8564 
8565   format %{ "shrl    $dst, $shift" %}
8566   opcode(0xD3, 0x5); /* D3 /5 */
8567   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8568   ins_pipe(ialu_mem_reg);
8569 %}
8570 
8571 // Long Shift Instructions
8572 // Shift Left by one
8573 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8574 %{
8575   match(Set dst (LShiftL dst shift));
8576   effect(KILL cr);
8577 
8578   format %{ "salq    $dst, $shift" %}
8579   opcode(0xD1, 0x4); /* D1 /4 */
8580   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8581   ins_pipe(ialu_reg);
8582 %}
8583 
8584 // Shift Left by one
8585 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8586 %{
8587   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8588   effect(KILL cr);
8589 
8590   format %{ "salq    $dst, $shift" %}
8591   opcode(0xD1, 0x4); /* D1 /4 */
8592   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8593   ins_pipe(ialu_mem_imm);
8594 %}
8595 
8596 // Shift Left by 8-bit immediate
8597 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8598 %{
8599   match(Set dst (LShiftL dst shift));
8600   effect(KILL cr);
8601 
8602   format %{ "salq    $dst, $shift" %}
8603   opcode(0xC1, 0x4); /* C1 /4 ib */
8604   ins_encode(reg_opc_imm_wide(dst, shift));
8605   ins_pipe(ialu_reg);
8606 %}
8607 
8608 // Shift Left by 8-bit immediate
8609 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8610 %{
8611   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8612   effect(KILL cr);
8613 
8614   format %{ "salq    $dst, $shift" %}
8615   opcode(0xC1, 0x4); /* C1 /4 ib */
8616   ins_encode(REX_mem_wide(dst), OpcP,
8617              RM_opc_mem(secondary, dst), Con8or32(shift));
8618   ins_pipe(ialu_mem_imm);
8619 %}
8620 
8621 // Shift Left by variable
8622 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8623 %{
8624   match(Set dst (LShiftL dst shift));
8625   effect(KILL cr);
8626 
8627   format %{ "salq    $dst, $shift" %}
8628   opcode(0xD3, 0x4); /* D3 /4 */
8629   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8630   ins_pipe(ialu_reg_reg);
8631 %}
8632 
8633 // Shift Left by variable
8634 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8635 %{
8636   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8637   effect(KILL cr);
8638 
8639   format %{ "salq    $dst, $shift" %}
8640   opcode(0xD3, 0x4); /* D3 /4 */
8641   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8642   ins_pipe(ialu_mem_reg);
8643 %}
8644 
8645 // Arithmetic shift right by one
8646 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8647 %{
8648   match(Set dst (RShiftL dst shift));
8649   effect(KILL cr);
8650 
8651   format %{ "sarq    $dst, $shift" %}
8652   opcode(0xD1, 0x7); /* D1 /7 */
8653   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8654   ins_pipe(ialu_reg);
8655 %}
8656 
8657 // Arithmetic shift right by one
8658 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8659 %{
8660   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8661   effect(KILL cr);
8662 
8663   format %{ "sarq    $dst, $shift" %}
8664   opcode(0xD1, 0x7); /* D1 /7 */
8665   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8666   ins_pipe(ialu_mem_imm);
8667 %}
8668 
8669 // Arithmetic Shift Right by 8-bit immediate
8670 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8671 %{
8672   match(Set dst (RShiftL dst shift));
8673   effect(KILL cr);
8674 
8675   format %{ "sarq    $dst, $shift" %}
8676   opcode(0xC1, 0x7); /* C1 /7 ib */
8677   ins_encode(reg_opc_imm_wide(dst, shift));
8678   ins_pipe(ialu_mem_imm);
8679 %}
8680 
8681 // Arithmetic Shift Right by 8-bit immediate
8682 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8683 %{
8684   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8685   effect(KILL cr);
8686 
8687   format %{ "sarq    $dst, $shift" %}
8688   opcode(0xC1, 0x7); /* C1 /7 ib */
8689   ins_encode(REX_mem_wide(dst), OpcP,
8690              RM_opc_mem(secondary, dst), Con8or32(shift));
8691   ins_pipe(ialu_mem_imm);
8692 %}
8693 
8694 // Arithmetic Shift Right by variable
8695 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8696 %{
8697   match(Set dst (RShiftL dst shift));
8698   effect(KILL cr);
8699 
8700   format %{ "sarq    $dst, $shift" %}
8701   opcode(0xD3, 0x7); /* D3 /7 */
8702   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8703   ins_pipe(ialu_reg_reg);
8704 %}
8705 
8706 // Arithmetic Shift Right by variable
8707 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8708 %{
8709   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8710   effect(KILL cr);
8711 
8712   format %{ "sarq    $dst, $shift" %}
8713   opcode(0xD3, 0x7); /* D3 /7 */
8714   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8715   ins_pipe(ialu_mem_reg);
8716 %}
8717 
8718 // Logical shift right by one
8719 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8720 %{
8721   match(Set dst (URShiftL dst shift));
8722   effect(KILL cr);
8723 
8724   format %{ "shrq    $dst, $shift" %}
8725   opcode(0xD1, 0x5); /* D1 /5 */
8726   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8727   ins_pipe(ialu_reg);
8728 %}
8729 
8730 // Logical shift right by one
8731 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8732 %{
8733   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8734   effect(KILL cr);
8735 
8736   format %{ "shrq    $dst, $shift" %}
8737   opcode(0xD1, 0x5); /* D1 /5 */
8738   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8739   ins_pipe(ialu_mem_imm);
8740 %}
8741 
8742 // Logical Shift Right by 8-bit immediate
8743 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8744 %{
8745   match(Set dst (URShiftL dst shift));
8746   effect(KILL cr);
8747 
8748   format %{ "shrq    $dst, $shift" %}
8749   opcode(0xC1, 0x5); /* C1 /5 ib */
8750   ins_encode(reg_opc_imm_wide(dst, shift));
8751   ins_pipe(ialu_reg);
8752 %}
8753 
8754 
8755 // Logical Shift Right by 8-bit immediate
8756 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8757 %{
8758   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8759   effect(KILL cr);
8760 
8761   format %{ "shrq    $dst, $shift" %}
8762   opcode(0xC1, 0x5); /* C1 /5 ib */
8763   ins_encode(REX_mem_wide(dst), OpcP,
8764              RM_opc_mem(secondary, dst), Con8or32(shift));
8765   ins_pipe(ialu_mem_imm);
8766 %}
8767 
8768 // Logical Shift Right by variable
8769 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8770 %{
8771   match(Set dst (URShiftL dst shift));
8772   effect(KILL cr);
8773 
8774   format %{ "shrq    $dst, $shift" %}
8775   opcode(0xD3, 0x5); /* D3 /5 */
8776   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8777   ins_pipe(ialu_reg_reg);
8778 %}
8779 
8780 // Logical Shift Right by variable
8781 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8782 %{
8783   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8784   effect(KILL cr);
8785 
8786   format %{ "shrq    $dst, $shift" %}
8787   opcode(0xD3, 0x5); /* D3 /5 */
8788   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8789   ins_pipe(ialu_mem_reg);
8790 %}
8791 
8792 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8793 // This idiom is used by the compiler for the i2b bytecode.
8794 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8795 %{
8796   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8797 
8798   format %{ "movsbl  $dst, $src\t# i2b" %}
8799   opcode(0x0F, 0xBE);
8800   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8801   ins_pipe(ialu_reg_reg);
8802 %}
8803 
8804 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8805 // This idiom is used by the compiler the i2s bytecode.
8806 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8807 %{
8808   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8809 
8810   format %{ "movswl  $dst, $src\t# i2s" %}
8811   opcode(0x0F, 0xBF);
8812   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8813   ins_pipe(ialu_reg_reg);
8814 %}
8815 
8816 // ROL/ROR instructions
8817 
8818 // ROL expand
8819 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8820   effect(KILL cr, USE_DEF dst);
8821 
8822   format %{ "roll    $dst" %}
8823   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8824   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8825   ins_pipe(ialu_reg);
8826 %}
8827 
8828 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8829   effect(USE_DEF dst, USE shift, KILL cr);
8830 
8831   format %{ "roll    $dst, $shift" %}
8832   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8833   ins_encode( reg_opc_imm(dst, shift) );
8834   ins_pipe(ialu_reg);
8835 %}
8836 
8837 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8838 %{
8839   effect(USE_DEF dst, USE shift, KILL cr);
8840 
8841   format %{ "roll    $dst, $shift" %}
8842   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8843   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8844   ins_pipe(ialu_reg_reg);
8845 %}
8846 // end of ROL expand
8847 
8848 // Rotate Left by one
8849 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8850 %{
8851   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8852 
8853   expand %{
8854     rolI_rReg_imm1(dst, cr);
8855   %}
8856 %}
8857 
8858 // Rotate Left by 8-bit immediate
8859 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8860 %{
8861   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8862   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8863 
8864   expand %{
8865     rolI_rReg_imm8(dst, lshift, cr);
8866   %}
8867 %}
8868 
8869 // Rotate Left by variable
8870 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8871 %{
8872   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8873 
8874   expand %{
8875     rolI_rReg_CL(dst, shift, cr);
8876   %}
8877 %}
8878 
8879 // Rotate Left by variable
8880 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8881 %{
8882   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8883 
8884   expand %{
8885     rolI_rReg_CL(dst, shift, cr);
8886   %}
8887 %}
8888 
8889 // ROR expand
8890 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8891 %{
8892   effect(USE_DEF dst, KILL cr);
8893 
8894   format %{ "rorl    $dst" %}
8895   opcode(0xD1, 0x1); /* D1 /1 */
8896   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8897   ins_pipe(ialu_reg);
8898 %}
8899 
8900 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8901 %{
8902   effect(USE_DEF dst, USE shift, KILL cr);
8903 
8904   format %{ "rorl    $dst, $shift" %}
8905   opcode(0xC1, 0x1); /* C1 /1 ib */
8906   ins_encode(reg_opc_imm(dst, shift));
8907   ins_pipe(ialu_reg);
8908 %}
8909 
8910 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8911 %{
8912   effect(USE_DEF dst, USE shift, KILL cr);
8913 
8914   format %{ "rorl    $dst, $shift" %}
8915   opcode(0xD3, 0x1); /* D3 /1 */
8916   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8917   ins_pipe(ialu_reg_reg);
8918 %}
8919 // end of ROR expand
8920 
8921 // Rotate Right by one
8922 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8923 %{
8924   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8925 
8926   expand %{
8927     rorI_rReg_imm1(dst, cr);
8928   %}
8929 %}
8930 
8931 // Rotate Right by 8-bit immediate
8932 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8933 %{
8934   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8935   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8936 
8937   expand %{
8938     rorI_rReg_imm8(dst, rshift, cr);
8939   %}
8940 %}
8941 
8942 // Rotate Right by variable
8943 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8944 %{
8945   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8946 
8947   expand %{
8948     rorI_rReg_CL(dst, shift, cr);
8949   %}
8950 %}
8951 
8952 // Rotate Right by variable
8953 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8954 %{
8955   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8956 
8957   expand %{
8958     rorI_rReg_CL(dst, shift, cr);
8959   %}
8960 %}
8961 
8962 // for long rotate
8963 // ROL expand
8964 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8965   effect(USE_DEF dst, KILL cr);
8966 
8967   format %{ "rolq    $dst" %}
8968   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8969   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8970   ins_pipe(ialu_reg);
8971 %}
8972 
8973 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8974   effect(USE_DEF dst, USE shift, KILL cr);
8975 
8976   format %{ "rolq    $dst, $shift" %}
8977   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8978   ins_encode( reg_opc_imm_wide(dst, shift) );
8979   ins_pipe(ialu_reg);
8980 %}
8981 
8982 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8983 %{
8984   effect(USE_DEF dst, USE shift, KILL cr);
8985 
8986   format %{ "rolq    $dst, $shift" %}
8987   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8988   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8989   ins_pipe(ialu_reg_reg);
8990 %}
8991 // end of ROL expand
8992 
8993 // Rotate Left by one
8994 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8995 %{
8996   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8997 
8998   expand %{
8999     rolL_rReg_imm1(dst, cr);
9000   %}
9001 %}
9002 
9003 // Rotate Left by 8-bit immediate
9004 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9005 %{
9006   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9007   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9008 
9009   expand %{
9010     rolL_rReg_imm8(dst, lshift, cr);
9011   %}
9012 %}
9013 
9014 // Rotate Left by variable
9015 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9016 %{
9017   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9018 
9019   expand %{
9020     rolL_rReg_CL(dst, shift, cr);
9021   %}
9022 %}
9023 
9024 // Rotate Left by variable
9025 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9026 %{
9027   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9028 
9029   expand %{
9030     rolL_rReg_CL(dst, shift, cr);
9031   %}
9032 %}
9033 
9034 // ROR expand
9035 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9036 %{
9037   effect(USE_DEF dst, KILL cr);
9038 
9039   format %{ "rorq    $dst" %}
9040   opcode(0xD1, 0x1); /* D1 /1 */
9041   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9042   ins_pipe(ialu_reg);
9043 %}
9044 
9045 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9046 %{
9047   effect(USE_DEF dst, USE shift, KILL cr);
9048 
9049   format %{ "rorq    $dst, $shift" %}
9050   opcode(0xC1, 0x1); /* C1 /1 ib */
9051   ins_encode(reg_opc_imm_wide(dst, shift));
9052   ins_pipe(ialu_reg);
9053 %}
9054 
9055 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9056 %{
9057   effect(USE_DEF dst, USE shift, KILL cr);
9058 
9059   format %{ "rorq    $dst, $shift" %}
9060   opcode(0xD3, 0x1); /* D3 /1 */
9061   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9062   ins_pipe(ialu_reg_reg);
9063 %}
9064 // end of ROR expand
9065 
9066 // Rotate Right by one
9067 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9068 %{
9069   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9070 
9071   expand %{
9072     rorL_rReg_imm1(dst, cr);
9073   %}
9074 %}
9075 
9076 // Rotate Right by 8-bit immediate
9077 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9078 %{
9079   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9080   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9081 
9082   expand %{
9083     rorL_rReg_imm8(dst, rshift, cr);
9084   %}
9085 %}
9086 
9087 // Rotate Right by variable
9088 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9089 %{
9090   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9091 
9092   expand %{
9093     rorL_rReg_CL(dst, shift, cr);
9094   %}
9095 %}
9096 
9097 // Rotate Right by variable
9098 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9099 %{
9100   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9101 
9102   expand %{
9103     rorL_rReg_CL(dst, shift, cr);
9104   %}
9105 %}
9106 
9107 // Logical Instructions
9108 
9109 // Integer Logical Instructions
9110 
9111 // And Instructions
9112 // And Register with Register
9113 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9114 %{
9115   match(Set dst (AndI dst src));
9116   effect(KILL cr);
9117 
9118   format %{ "andl    $dst, $src\t# int" %}
9119   opcode(0x23);
9120   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9121   ins_pipe(ialu_reg_reg);
9122 %}
9123 
9124 // And Register with Immediate 255
9125 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9126 %{
9127   match(Set dst (AndI dst src));
9128 
9129   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9130   opcode(0x0F, 0xB6);
9131   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9132   ins_pipe(ialu_reg);
9133 %}
9134 
9135 // And Register with Immediate 255 and promote to long
9136 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9137 %{
9138   match(Set dst (ConvI2L (AndI src mask)));
9139 
9140   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9141   opcode(0x0F, 0xB6);
9142   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9143   ins_pipe(ialu_reg);
9144 %}
9145 
9146 // And Register with Immediate 65535
9147 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9148 %{
9149   match(Set dst (AndI dst src));
9150 
9151   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9152   opcode(0x0F, 0xB7);
9153   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9154   ins_pipe(ialu_reg);
9155 %}
9156 
9157 // And Register with Immediate 65535 and promote to long
9158 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9159 %{
9160   match(Set dst (ConvI2L (AndI src mask)));
9161 
9162   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9163   opcode(0x0F, 0xB7);
9164   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9165   ins_pipe(ialu_reg);
9166 %}
9167 
9168 // And Register with Immediate
9169 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9170 %{
9171   match(Set dst (AndI dst src));
9172   effect(KILL cr);
9173 
9174   format %{ "andl    $dst, $src\t# int" %}
9175   opcode(0x81, 0x04); /* Opcode 81 /4 */
9176   ins_encode(OpcSErm(dst, src), Con8or32(src));
9177   ins_pipe(ialu_reg);
9178 %}
9179 
9180 // And Register with Memory
9181 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9182 %{
9183   match(Set dst (AndI dst (LoadI src)));
9184   effect(KILL cr);
9185 
9186   ins_cost(125);
9187   format %{ "andl    $dst, $src\t# int" %}
9188   opcode(0x23);
9189   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9190   ins_pipe(ialu_reg_mem);
9191 %}
9192 
9193 // And Memory with Register
9194 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9195 %{
9196   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9197   effect(KILL cr);
9198 
9199   ins_cost(150);
9200   format %{ "andl    $dst, $src\t# int" %}
9201   opcode(0x21); /* Opcode 21 /r */
9202   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9203   ins_pipe(ialu_mem_reg);
9204 %}
9205 
9206 // And Memory with Immediate
9207 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9208 %{
9209   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9210   effect(KILL cr);
9211 
9212   ins_cost(125);
9213   format %{ "andl    $dst, $src\t# int" %}
9214   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9215   ins_encode(REX_mem(dst), OpcSE(src),
9216              RM_opc_mem(secondary, dst), Con8or32(src));
9217   ins_pipe(ialu_mem_imm);
9218 %}
9219 
9220 // BMI1 instructions
9221 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9222   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9223   predicate(UseBMI1Instructions);
9224   effect(KILL cr);
9225 
9226   ins_cost(125);
9227   format %{ "andnl  $dst, $src1, $src2" %}
9228 
9229   ins_encode %{
9230     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9231   %}
9232   ins_pipe(ialu_reg_mem);
9233 %}
9234 
9235 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9236   match(Set dst (AndI (XorI src1 minus_1) src2));
9237   predicate(UseBMI1Instructions);
9238   effect(KILL cr);
9239 
9240   format %{ "andnl  $dst, $src1, $src2" %}
9241 
9242   ins_encode %{
9243     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9244   %}
9245   ins_pipe(ialu_reg);
9246 %}
9247 
9248 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9249   match(Set dst (AndI (SubI imm_zero src) src));
9250   predicate(UseBMI1Instructions);
9251   effect(KILL cr);
9252 
9253   format %{ "blsil  $dst, $src" %}
9254 
9255   ins_encode %{
9256     __ blsil($dst$$Register, $src$$Register);
9257   %}
9258   ins_pipe(ialu_reg);
9259 %}
9260 
9261 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9262   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9263   predicate(UseBMI1Instructions);
9264   effect(KILL cr);
9265 
9266   ins_cost(125);
9267   format %{ "blsil  $dst, $src" %}
9268 
9269   ins_encode %{
9270     __ blsil($dst$$Register, $src$$Address);
9271   %}
9272   ins_pipe(ialu_reg_mem);
9273 %}
9274 
9275 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9276 %{
9277   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9278   predicate(UseBMI1Instructions);
9279   effect(KILL cr);
9280 
9281   ins_cost(125);
9282   format %{ "blsmskl $dst, $src" %}
9283 
9284   ins_encode %{
9285     __ blsmskl($dst$$Register, $src$$Address);
9286   %}
9287   ins_pipe(ialu_reg_mem);
9288 %}
9289 
9290 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9291 %{
9292   match(Set dst (XorI (AddI src minus_1) src));
9293   predicate(UseBMI1Instructions);
9294   effect(KILL cr);
9295 
9296   format %{ "blsmskl $dst, $src" %}
9297 
9298   ins_encode %{
9299     __ blsmskl($dst$$Register, $src$$Register);
9300   %}
9301 
9302   ins_pipe(ialu_reg);
9303 %}
9304 
9305 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9306 %{
9307   match(Set dst (AndI (AddI src minus_1) src) );
9308   predicate(UseBMI1Instructions);
9309   effect(KILL cr);
9310 
9311   format %{ "blsrl  $dst, $src" %}
9312 
9313   ins_encode %{
9314     __ blsrl($dst$$Register, $src$$Register);
9315   %}
9316 
9317   ins_pipe(ialu_reg_mem);
9318 %}
9319 
9320 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9321 %{
9322   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9323   predicate(UseBMI1Instructions);
9324   effect(KILL cr);
9325 
9326   ins_cost(125);
9327   format %{ "blsrl  $dst, $src" %}
9328 
9329   ins_encode %{
9330     __ blsrl($dst$$Register, $src$$Address);
9331   %}
9332 
9333   ins_pipe(ialu_reg);
9334 %}
9335 
9336 // Or Instructions
9337 // Or Register with Register
9338 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9339 %{
9340   match(Set dst (OrI dst src));
9341   effect(KILL cr);
9342 
9343   format %{ "orl     $dst, $src\t# int" %}
9344   opcode(0x0B);
9345   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9346   ins_pipe(ialu_reg_reg);
9347 %}
9348 
9349 // Or Register with Immediate
9350 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9351 %{
9352   match(Set dst (OrI dst src));
9353   effect(KILL cr);
9354 
9355   format %{ "orl     $dst, $src\t# int" %}
9356   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9357   ins_encode(OpcSErm(dst, src), Con8or32(src));
9358   ins_pipe(ialu_reg);
9359 %}
9360 
9361 // Or Register with Memory
9362 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9363 %{
9364   match(Set dst (OrI dst (LoadI src)));
9365   effect(KILL cr);
9366 
9367   ins_cost(125);
9368   format %{ "orl     $dst, $src\t# int" %}
9369   opcode(0x0B);
9370   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9371   ins_pipe(ialu_reg_mem);
9372 %}
9373 
9374 // Or Memory with Register
9375 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9376 %{
9377   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9378   effect(KILL cr);
9379 
9380   ins_cost(150);
9381   format %{ "orl     $dst, $src\t# int" %}
9382   opcode(0x09); /* Opcode 09 /r */
9383   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9384   ins_pipe(ialu_mem_reg);
9385 %}
9386 
9387 // Or Memory with Immediate
9388 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9389 %{
9390   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9391   effect(KILL cr);
9392 
9393   ins_cost(125);
9394   format %{ "orl     $dst, $src\t# int" %}
9395   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9396   ins_encode(REX_mem(dst), OpcSE(src),
9397              RM_opc_mem(secondary, dst), Con8or32(src));
9398   ins_pipe(ialu_mem_imm);
9399 %}
9400 
9401 // Xor Instructions
9402 // Xor Register with Register
9403 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9404 %{
9405   match(Set dst (XorI dst src));
9406   effect(KILL cr);
9407 
9408   format %{ "xorl    $dst, $src\t# int" %}
9409   opcode(0x33);
9410   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9411   ins_pipe(ialu_reg_reg);
9412 %}
9413 
9414 // Xor Register with Immediate -1
9415 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9416   match(Set dst (XorI dst imm));
9417 
9418   format %{ "not    $dst" %}
9419   ins_encode %{
9420      __ notl($dst$$Register);
9421   %}
9422   ins_pipe(ialu_reg);
9423 %}
9424 
9425 // Xor Register with Immediate
9426 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9427 %{
9428   match(Set dst (XorI dst src));
9429   effect(KILL cr);
9430 
9431   format %{ "xorl    $dst, $src\t# int" %}
9432   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9433   ins_encode(OpcSErm(dst, src), Con8or32(src));
9434   ins_pipe(ialu_reg);
9435 %}
9436 
9437 // Xor Register with Memory
9438 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9439 %{
9440   match(Set dst (XorI dst (LoadI src)));
9441   effect(KILL cr);
9442 
9443   ins_cost(125);
9444   format %{ "xorl    $dst, $src\t# int" %}
9445   opcode(0x33);
9446   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9447   ins_pipe(ialu_reg_mem);
9448 %}
9449 
9450 // Xor Memory with Register
9451 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9452 %{
9453   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9454   effect(KILL cr);
9455 
9456   ins_cost(150);
9457   format %{ "xorl    $dst, $src\t# int" %}
9458   opcode(0x31); /* Opcode 31 /r */
9459   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9460   ins_pipe(ialu_mem_reg);
9461 %}
9462 
9463 // Xor Memory with Immediate
9464 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9465 %{
9466   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9467   effect(KILL cr);
9468 
9469   ins_cost(125);
9470   format %{ "xorl    $dst, $src\t# int" %}
9471   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9472   ins_encode(REX_mem(dst), OpcSE(src),
9473              RM_opc_mem(secondary, dst), Con8or32(src));
9474   ins_pipe(ialu_mem_imm);
9475 %}
9476 
9477 
9478 // Long Logical Instructions
9479 
9480 // And Instructions
9481 // And Register with Register
9482 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9483 %{
9484   match(Set dst (AndL dst src));
9485   effect(KILL cr);
9486 
9487   format %{ "andq    $dst, $src\t# long" %}
9488   opcode(0x23);
9489   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9490   ins_pipe(ialu_reg_reg);
9491 %}
9492 
9493 // And Register with Immediate 255
9494 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9495 %{
9496   match(Set dst (AndL dst src));
9497 
9498   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9499   opcode(0x0F, 0xB6);
9500   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9501   ins_pipe(ialu_reg);
9502 %}
9503 
9504 // And Register with Immediate 65535
9505 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9506 %{
9507   match(Set dst (AndL dst src));
9508 
9509   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9510   opcode(0x0F, 0xB7);
9511   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9512   ins_pipe(ialu_reg);
9513 %}
9514 
9515 // And Register with Immediate
9516 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9517 %{
9518   match(Set dst (AndL dst src));
9519   effect(KILL cr);
9520 
9521   format %{ "andq    $dst, $src\t# long" %}
9522   opcode(0x81, 0x04); /* Opcode 81 /4 */
9523   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9524   ins_pipe(ialu_reg);
9525 %}
9526 
9527 // And Register with Memory
9528 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9529 %{
9530   match(Set dst (AndL dst (LoadL src)));
9531   effect(KILL cr);
9532 
9533   ins_cost(125);
9534   format %{ "andq    $dst, $src\t# long" %}
9535   opcode(0x23);
9536   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9537   ins_pipe(ialu_reg_mem);
9538 %}
9539 
9540 // And Memory with Register
9541 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9542 %{
9543   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9544   effect(KILL cr);
9545 
9546   ins_cost(150);
9547   format %{ "andq    $dst, $src\t# long" %}
9548   opcode(0x21); /* Opcode 21 /r */
9549   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9550   ins_pipe(ialu_mem_reg);
9551 %}
9552 
9553 // And Memory with Immediate
9554 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9555 %{
9556   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9557   effect(KILL cr);
9558 
9559   ins_cost(125);
9560   format %{ "andq    $dst, $src\t# long" %}
9561   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9562   ins_encode(REX_mem_wide(dst), OpcSE(src),
9563              RM_opc_mem(secondary, dst), Con8or32(src));
9564   ins_pipe(ialu_mem_imm);
9565 %}
9566 
9567 // BMI1 instructions
9568 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9569   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9570   predicate(UseBMI1Instructions);
9571   effect(KILL cr);
9572 
9573   ins_cost(125);
9574   format %{ "andnq  $dst, $src1, $src2" %}
9575 
9576   ins_encode %{
9577     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9578   %}
9579   ins_pipe(ialu_reg_mem);
9580 %}
9581 
9582 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9583   match(Set dst (AndL (XorL src1 minus_1) src2));
9584   predicate(UseBMI1Instructions);
9585   effect(KILL cr);
9586 
9587   format %{ "andnq  $dst, $src1, $src2" %}
9588 
9589   ins_encode %{
9590   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9591   %}
9592   ins_pipe(ialu_reg_mem);
9593 %}
9594 
9595 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9596   match(Set dst (AndL (SubL imm_zero src) src));
9597   predicate(UseBMI1Instructions);
9598   effect(KILL cr);
9599 
9600   format %{ "blsiq  $dst, $src" %}
9601 
9602   ins_encode %{
9603     __ blsiq($dst$$Register, $src$$Register);
9604   %}
9605   ins_pipe(ialu_reg);
9606 %}
9607 
9608 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9609   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9610   predicate(UseBMI1Instructions);
9611   effect(KILL cr);
9612 
9613   ins_cost(125);
9614   format %{ "blsiq  $dst, $src" %}
9615 
9616   ins_encode %{
9617     __ blsiq($dst$$Register, $src$$Address);
9618   %}
9619   ins_pipe(ialu_reg_mem);
9620 %}
9621 
9622 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9623 %{
9624   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9625   predicate(UseBMI1Instructions);
9626   effect(KILL cr);
9627 
9628   ins_cost(125);
9629   format %{ "blsmskq $dst, $src" %}
9630 
9631   ins_encode %{
9632     __ blsmskq($dst$$Register, $src$$Address);
9633   %}
9634   ins_pipe(ialu_reg_mem);
9635 %}
9636 
9637 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9638 %{
9639   match(Set dst (XorL (AddL src minus_1) src));
9640   predicate(UseBMI1Instructions);
9641   effect(KILL cr);
9642 
9643   format %{ "blsmskq $dst, $src" %}
9644 
9645   ins_encode %{
9646     __ blsmskq($dst$$Register, $src$$Register);
9647   %}
9648 
9649   ins_pipe(ialu_reg);
9650 %}
9651 
9652 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9653 %{
9654   match(Set dst (AndL (AddL src minus_1) src) );
9655   predicate(UseBMI1Instructions);
9656   effect(KILL cr);
9657 
9658   format %{ "blsrq  $dst, $src" %}
9659 
9660   ins_encode %{
9661     __ blsrq($dst$$Register, $src$$Register);
9662   %}
9663 
9664   ins_pipe(ialu_reg);
9665 %}
9666 
9667 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9668 %{
9669   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9670   predicate(UseBMI1Instructions);
9671   effect(KILL cr);
9672 
9673   ins_cost(125);
9674   format %{ "blsrq  $dst, $src" %}
9675 
9676   ins_encode %{
9677     __ blsrq($dst$$Register, $src$$Address);
9678   %}
9679 
9680   ins_pipe(ialu_reg);
9681 %}
9682 
9683 // Or Instructions
9684 // Or Register with Register
9685 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9686 %{
9687   match(Set dst (OrL dst src));
9688   effect(KILL cr);
9689 
9690   format %{ "orq     $dst, $src\t# long" %}
9691   opcode(0x0B);
9692   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9693   ins_pipe(ialu_reg_reg);
9694 %}
9695 
9696 // Use any_RegP to match R15 (TLS register) without spilling.
9697 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9698   match(Set dst (OrL dst (CastP2X src)));
9699   effect(KILL cr);
9700 
9701   format %{ "orq     $dst, $src\t# long" %}
9702   opcode(0x0B);
9703   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9704   ins_pipe(ialu_reg_reg);
9705 %}
9706 
9707 
9708 // Or Register with Immediate
9709 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9710 %{
9711   match(Set dst (OrL dst src));
9712   effect(KILL cr);
9713 
9714   format %{ "orq     $dst, $src\t# long" %}
9715   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9716   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9717   ins_pipe(ialu_reg);
9718 %}
9719 
9720 // Or Register with Memory
9721 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9722 %{
9723   match(Set dst (OrL dst (LoadL src)));
9724   effect(KILL cr);
9725 
9726   ins_cost(125);
9727   format %{ "orq     $dst, $src\t# long" %}
9728   opcode(0x0B);
9729   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9730   ins_pipe(ialu_reg_mem);
9731 %}
9732 
9733 // Or Memory with Register
9734 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9735 %{
9736   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9737   effect(KILL cr);
9738 
9739   ins_cost(150);
9740   format %{ "orq     $dst, $src\t# long" %}
9741   opcode(0x09); /* Opcode 09 /r */
9742   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9743   ins_pipe(ialu_mem_reg);
9744 %}
9745 
9746 // Or Memory with Immediate
9747 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9748 %{
9749   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9750   effect(KILL cr);
9751 
9752   ins_cost(125);
9753   format %{ "orq     $dst, $src\t# long" %}
9754   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9755   ins_encode(REX_mem_wide(dst), OpcSE(src),
9756              RM_opc_mem(secondary, dst), Con8or32(src));
9757   ins_pipe(ialu_mem_imm);
9758 %}
9759 
9760 // Xor Instructions
9761 // Xor Register with Register
9762 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9763 %{
9764   match(Set dst (XorL dst src));
9765   effect(KILL cr);
9766 
9767   format %{ "xorq    $dst, $src\t# long" %}
9768   opcode(0x33);
9769   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9770   ins_pipe(ialu_reg_reg);
9771 %}
9772 
9773 // Xor Register with Immediate -1
9774 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9775   match(Set dst (XorL dst imm));
9776 
9777   format %{ "notq   $dst" %}
9778   ins_encode %{
9779      __ notq($dst$$Register);
9780   %}
9781   ins_pipe(ialu_reg);
9782 %}
9783 
9784 // Xor Register with Immediate
9785 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9786 %{
9787   match(Set dst (XorL dst src));
9788   effect(KILL cr);
9789 
9790   format %{ "xorq    $dst, $src\t# long" %}
9791   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9792   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9793   ins_pipe(ialu_reg);
9794 %}
9795 
9796 // Xor Register with Memory
9797 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9798 %{
9799   match(Set dst (XorL dst (LoadL src)));
9800   effect(KILL cr);
9801 
9802   ins_cost(125);
9803   format %{ "xorq    $dst, $src\t# long" %}
9804   opcode(0x33);
9805   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9806   ins_pipe(ialu_reg_mem);
9807 %}
9808 
9809 // Xor Memory with Register
9810 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9811 %{
9812   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9813   effect(KILL cr);
9814 
9815   ins_cost(150);
9816   format %{ "xorq    $dst, $src\t# long" %}
9817   opcode(0x31); /* Opcode 31 /r */
9818   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9819   ins_pipe(ialu_mem_reg);
9820 %}
9821 
9822 // Xor Memory with Immediate
9823 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9824 %{
9825   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9826   effect(KILL cr);
9827 
9828   ins_cost(125);
9829   format %{ "xorq    $dst, $src\t# long" %}
9830   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9831   ins_encode(REX_mem_wide(dst), OpcSE(src),
9832              RM_opc_mem(secondary, dst), Con8or32(src));
9833   ins_pipe(ialu_mem_imm);
9834 %}
9835 
9836 // Convert Int to Boolean
9837 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9838 %{
9839   match(Set dst (Conv2B src));
9840   effect(KILL cr);
9841 
9842   format %{ "testl   $src, $src\t# ci2b\n\t"
9843             "setnz   $dst\n\t"
9844             "movzbl  $dst, $dst" %}
9845   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9846              setNZ_reg(dst),
9847              REX_reg_breg(dst, dst), // movzbl
9848              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9849   ins_pipe(pipe_slow); // XXX
9850 %}
9851 
9852 // Convert Pointer to Boolean
9853 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9854 %{
9855   match(Set dst (Conv2B src));
9856   effect(KILL cr);
9857 
9858   format %{ "testq   $src, $src\t# cp2b\n\t"
9859             "setnz   $dst\n\t"
9860             "movzbl  $dst, $dst" %}
9861   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9862              setNZ_reg(dst),
9863              REX_reg_breg(dst, dst), // movzbl
9864              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9865   ins_pipe(pipe_slow); // XXX
9866 %}
9867 
9868 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9869 %{
9870   match(Set dst (CmpLTMask p q));
9871   effect(KILL cr);
9872 
9873   ins_cost(400);
9874   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9875             "setlt   $dst\n\t"
9876             "movzbl  $dst, $dst\n\t"
9877             "negl    $dst" %}
9878   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9879              setLT_reg(dst),
9880              REX_reg_breg(dst, dst), // movzbl
9881              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9882              neg_reg(dst));
9883   ins_pipe(pipe_slow);
9884 %}
9885 
9886 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9887 %{
9888   match(Set dst (CmpLTMask dst zero));
9889   effect(KILL cr);
9890 
9891   ins_cost(100);
9892   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9893   ins_encode %{
9894   __ sarl($dst$$Register, 31);
9895   %}
9896   ins_pipe(ialu_reg);
9897 %}
9898 
9899 /* Better to save a register than avoid a branch */
9900 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9901 %{
9902   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9903   effect(KILL cr);
9904   ins_cost(300);
9905   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9906             "jge    done\n\t"
9907             "addl   $p,$y\n"
9908             "done:  " %}
9909   ins_encode %{
9910     Register Rp = $p$$Register;
9911     Register Rq = $q$$Register;
9912     Register Ry = $y$$Register;
9913     Label done;
9914     __ subl(Rp, Rq);
9915     __ jccb(Assembler::greaterEqual, done);
9916     __ addl(Rp, Ry);
9917     __ bind(done);
9918   %}
9919   ins_pipe(pipe_cmplt);
9920 %}
9921 
9922 /* Better to save a register than avoid a branch */
9923 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9924 %{
9925   match(Set y (AndI (CmpLTMask p q) y));
9926   effect(KILL cr);
9927 
9928   ins_cost(300);
9929 
9930   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9931             "jlt      done\n\t"
9932             "xorl     $y, $y\n"
9933             "done:  " %}
9934   ins_encode %{
9935     Register Rp = $p$$Register;
9936     Register Rq = $q$$Register;
9937     Register Ry = $y$$Register;
9938     Label done;
9939     __ cmpl(Rp, Rq);
9940     __ jccb(Assembler::less, done);
9941     __ xorl(Ry, Ry);
9942     __ bind(done);
9943   %}
9944   ins_pipe(pipe_cmplt);
9945 %}
9946 
9947 
9948 //---------- FP Instructions------------------------------------------------
9949 
9950 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9951 %{
9952   match(Set cr (CmpF src1 src2));
9953 
9954   ins_cost(145);
9955   format %{ "ucomiss $src1, $src2\n\t"
9956             "jnp,s   exit\n\t"
9957             "pushfq\t# saw NaN, set CF\n\t"
9958             "andq    [rsp], #0xffffff2b\n\t"
9959             "popfq\n"
9960     "exit:" %}
9961   ins_encode %{
9962     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9963     emit_cmpfp_fixup(_masm);
9964   %}
9965   ins_pipe(pipe_slow);
9966 %}
9967 
9968 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9969   match(Set cr (CmpF src1 src2));
9970 
9971   ins_cost(100);
9972   format %{ "ucomiss $src1, $src2" %}
9973   ins_encode %{
9974     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9975   %}
9976   ins_pipe(pipe_slow);
9977 %}
9978 
9979 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9980 %{
9981   match(Set cr (CmpF src1 (LoadF src2)));
9982 
9983   ins_cost(145);
9984   format %{ "ucomiss $src1, $src2\n\t"
9985             "jnp,s   exit\n\t"
9986             "pushfq\t# saw NaN, set CF\n\t"
9987             "andq    [rsp], #0xffffff2b\n\t"
9988             "popfq\n"
9989     "exit:" %}
9990   ins_encode %{
9991     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9992     emit_cmpfp_fixup(_masm);
9993   %}
9994   ins_pipe(pipe_slow);
9995 %}
9996 
9997 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9998   match(Set cr (CmpF src1 (LoadF src2)));
9999 
10000   ins_cost(100);
10001   format %{ "ucomiss $src1, $src2" %}
10002   ins_encode %{
10003     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10004   %}
10005   ins_pipe(pipe_slow);
10006 %}
10007 
10008 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10009   match(Set cr (CmpF src con));
10010 
10011   ins_cost(145);
10012   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10013             "jnp,s   exit\n\t"
10014             "pushfq\t# saw NaN, set CF\n\t"
10015             "andq    [rsp], #0xffffff2b\n\t"
10016             "popfq\n"
10017     "exit:" %}
10018   ins_encode %{
10019     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10020     emit_cmpfp_fixup(_masm);
10021   %}
10022   ins_pipe(pipe_slow);
10023 %}
10024 
10025 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10026   match(Set cr (CmpF src con));
10027   ins_cost(100);
10028   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10029   ins_encode %{
10030     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10031   %}
10032   ins_pipe(pipe_slow);
10033 %}
10034 
10035 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10036 %{
10037   match(Set cr (CmpD src1 src2));
10038 
10039   ins_cost(145);
10040   format %{ "ucomisd $src1, $src2\n\t"
10041             "jnp,s   exit\n\t"
10042             "pushfq\t# saw NaN, set CF\n\t"
10043             "andq    [rsp], #0xffffff2b\n\t"
10044             "popfq\n"
10045     "exit:" %}
10046   ins_encode %{
10047     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10048     emit_cmpfp_fixup(_masm);
10049   %}
10050   ins_pipe(pipe_slow);
10051 %}
10052 
10053 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10054   match(Set cr (CmpD src1 src2));
10055 
10056   ins_cost(100);
10057   format %{ "ucomisd $src1, $src2 test" %}
10058   ins_encode %{
10059     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10060   %}
10061   ins_pipe(pipe_slow);
10062 %}
10063 
10064 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10065 %{
10066   match(Set cr (CmpD src1 (LoadD src2)));
10067 
10068   ins_cost(145);
10069   format %{ "ucomisd $src1, $src2\n\t"
10070             "jnp,s   exit\n\t"
10071             "pushfq\t# saw NaN, set CF\n\t"
10072             "andq    [rsp], #0xffffff2b\n\t"
10073             "popfq\n"
10074     "exit:" %}
10075   ins_encode %{
10076     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10077     emit_cmpfp_fixup(_masm);
10078   %}
10079   ins_pipe(pipe_slow);
10080 %}
10081 
10082 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10083   match(Set cr (CmpD src1 (LoadD src2)));
10084 
10085   ins_cost(100);
10086   format %{ "ucomisd $src1, $src2" %}
10087   ins_encode %{
10088     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10089   %}
10090   ins_pipe(pipe_slow);
10091 %}
10092 
10093 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10094   match(Set cr (CmpD src con));
10095 
10096   ins_cost(145);
10097   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10098             "jnp,s   exit\n\t"
10099             "pushfq\t# saw NaN, set CF\n\t"
10100             "andq    [rsp], #0xffffff2b\n\t"
10101             "popfq\n"
10102     "exit:" %}
10103   ins_encode %{
10104     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10105     emit_cmpfp_fixup(_masm);
10106   %}
10107   ins_pipe(pipe_slow);
10108 %}
10109 
10110 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10111   match(Set cr (CmpD src con));
10112   ins_cost(100);
10113   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10114   ins_encode %{
10115     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10116   %}
10117   ins_pipe(pipe_slow);
10118 %}
10119 
10120 // Compare into -1,0,1
10121 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10122 %{
10123   match(Set dst (CmpF3 src1 src2));
10124   effect(KILL cr);
10125 
10126   ins_cost(275);
10127   format %{ "ucomiss $src1, $src2\n\t"
10128             "movl    $dst, #-1\n\t"
10129             "jp,s    done\n\t"
10130             "jb,s    done\n\t"
10131             "setne   $dst\n\t"
10132             "movzbl  $dst, $dst\n"
10133     "done:" %}
10134   ins_encode %{
10135     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10136     emit_cmpfp3(_masm, $dst$$Register);
10137   %}
10138   ins_pipe(pipe_slow);
10139 %}
10140 
10141 // Compare into -1,0,1
10142 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10143 %{
10144   match(Set dst (CmpF3 src1 (LoadF src2)));
10145   effect(KILL cr);
10146 
10147   ins_cost(275);
10148   format %{ "ucomiss $src1, $src2\n\t"
10149             "movl    $dst, #-1\n\t"
10150             "jp,s    done\n\t"
10151             "jb,s    done\n\t"
10152             "setne   $dst\n\t"
10153             "movzbl  $dst, $dst\n"
10154     "done:" %}
10155   ins_encode %{
10156     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10157     emit_cmpfp3(_masm, $dst$$Register);
10158   %}
10159   ins_pipe(pipe_slow);
10160 %}
10161 
10162 // Compare into -1,0,1
10163 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10164   match(Set dst (CmpF3 src con));
10165   effect(KILL cr);
10166 
10167   ins_cost(275);
10168   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10169             "movl    $dst, #-1\n\t"
10170             "jp,s    done\n\t"
10171             "jb,s    done\n\t"
10172             "setne   $dst\n\t"
10173             "movzbl  $dst, $dst\n"
10174     "done:" %}
10175   ins_encode %{
10176     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10177     emit_cmpfp3(_masm, $dst$$Register);
10178   %}
10179   ins_pipe(pipe_slow);
10180 %}
10181 
10182 // Compare into -1,0,1
10183 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10184 %{
10185   match(Set dst (CmpD3 src1 src2));
10186   effect(KILL cr);
10187 
10188   ins_cost(275);
10189   format %{ "ucomisd $src1, $src2\n\t"
10190             "movl    $dst, #-1\n\t"
10191             "jp,s    done\n\t"
10192             "jb,s    done\n\t"
10193             "setne   $dst\n\t"
10194             "movzbl  $dst, $dst\n"
10195     "done:" %}
10196   ins_encode %{
10197     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10198     emit_cmpfp3(_masm, $dst$$Register);
10199   %}
10200   ins_pipe(pipe_slow);
10201 %}
10202 
10203 // Compare into -1,0,1
10204 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10205 %{
10206   match(Set dst (CmpD3 src1 (LoadD src2)));
10207   effect(KILL cr);
10208 
10209   ins_cost(275);
10210   format %{ "ucomisd $src1, $src2\n\t"
10211             "movl    $dst, #-1\n\t"
10212             "jp,s    done\n\t"
10213             "jb,s    done\n\t"
10214             "setne   $dst\n\t"
10215             "movzbl  $dst, $dst\n"
10216     "done:" %}
10217   ins_encode %{
10218     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10219     emit_cmpfp3(_masm, $dst$$Register);
10220   %}
10221   ins_pipe(pipe_slow);
10222 %}
10223 
10224 // Compare into -1,0,1
10225 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10226   match(Set dst (CmpD3 src con));
10227   effect(KILL cr);
10228 
10229   ins_cost(275);
10230   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10231             "movl    $dst, #-1\n\t"
10232             "jp,s    done\n\t"
10233             "jb,s    done\n\t"
10234             "setne   $dst\n\t"
10235             "movzbl  $dst, $dst\n"
10236     "done:" %}
10237   ins_encode %{
10238     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10239     emit_cmpfp3(_masm, $dst$$Register);
10240   %}
10241   ins_pipe(pipe_slow);
10242 %}
10243 
10244 //----------Arithmetic Conversion Instructions---------------------------------
10245 
10246 instruct roundFloat_nop(regF dst)
10247 %{
10248   match(Set dst (RoundFloat dst));
10249 
10250   ins_cost(0);
10251   ins_encode();
10252   ins_pipe(empty);
10253 %}
10254 
10255 instruct roundDouble_nop(regD dst)
10256 %{
10257   match(Set dst (RoundDouble dst));
10258 
10259   ins_cost(0);
10260   ins_encode();
10261   ins_pipe(empty);
10262 %}
10263 
10264 instruct convF2D_reg_reg(regD dst, regF src)
10265 %{
10266   match(Set dst (ConvF2D src));
10267 
10268   format %{ "cvtss2sd $dst, $src" %}
10269   ins_encode %{
10270     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10271   %}
10272   ins_pipe(pipe_slow); // XXX
10273 %}
10274 
10275 instruct convF2D_reg_mem(regD dst, memory src)
10276 %{
10277   match(Set dst (ConvF2D (LoadF src)));
10278 
10279   format %{ "cvtss2sd $dst, $src" %}
10280   ins_encode %{
10281     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10282   %}
10283   ins_pipe(pipe_slow); // XXX
10284 %}
10285 
10286 instruct convD2F_reg_reg(regF dst, regD src)
10287 %{
10288   match(Set dst (ConvD2F src));
10289 
10290   format %{ "cvtsd2ss $dst, $src" %}
10291   ins_encode %{
10292     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10293   %}
10294   ins_pipe(pipe_slow); // XXX
10295 %}
10296 
10297 instruct convD2F_reg_mem(regF dst, memory src)
10298 %{
10299   match(Set dst (ConvD2F (LoadD src)));
10300 
10301   format %{ "cvtsd2ss $dst, $src" %}
10302   ins_encode %{
10303     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10304   %}
10305   ins_pipe(pipe_slow); // XXX
10306 %}
10307 
10308 // XXX do mem variants
10309 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10310 %{
10311   match(Set dst (ConvF2I src));
10312   effect(KILL cr);
10313 
10314   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10315             "cmpl    $dst, #0x80000000\n\t"
10316             "jne,s   done\n\t"
10317             "subq    rsp, #8\n\t"
10318             "movss   [rsp], $src\n\t"
10319             "call    f2i_fixup\n\t"
10320             "popq    $dst\n"
10321     "done:   "%}
10322   ins_encode %{
10323     Label done;
10324     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10325     __ cmpl($dst$$Register, 0x80000000);
10326     __ jccb(Assembler::notEqual, done);
10327     __ subptr(rsp, 8);
10328     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10329     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10330     __ pop($dst$$Register);
10331     __ bind(done);
10332   %}
10333   ins_pipe(pipe_slow);
10334 %}
10335 
10336 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10337 %{
10338   match(Set dst (ConvF2L src));
10339   effect(KILL cr);
10340 
10341   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10342             "cmpq    $dst, [0x8000000000000000]\n\t"
10343             "jne,s   done\n\t"
10344             "subq    rsp, #8\n\t"
10345             "movss   [rsp], $src\n\t"
10346             "call    f2l_fixup\n\t"
10347             "popq    $dst\n"
10348     "done:   "%}
10349   ins_encode %{
10350     Label done;
10351     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10352     __ cmp64($dst$$Register,
10353              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10354     __ jccb(Assembler::notEqual, done);
10355     __ subptr(rsp, 8);
10356     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10357     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10358     __ pop($dst$$Register);
10359     __ bind(done);
10360   %}
10361   ins_pipe(pipe_slow);
10362 %}
10363 
10364 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10365 %{
10366   match(Set dst (ConvD2I src));
10367   effect(KILL cr);
10368 
10369   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10370             "cmpl    $dst, #0x80000000\n\t"
10371             "jne,s   done\n\t"
10372             "subq    rsp, #8\n\t"
10373             "movsd   [rsp], $src\n\t"
10374             "call    d2i_fixup\n\t"
10375             "popq    $dst\n"
10376     "done:   "%}
10377   ins_encode %{
10378     Label done;
10379     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10380     __ cmpl($dst$$Register, 0x80000000);
10381     __ jccb(Assembler::notEqual, done);
10382     __ subptr(rsp, 8);
10383     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10384     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10385     __ pop($dst$$Register);
10386     __ bind(done);
10387   %}
10388   ins_pipe(pipe_slow);
10389 %}
10390 
10391 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10392 %{
10393   match(Set dst (ConvD2L src));
10394   effect(KILL cr);
10395 
10396   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10397             "cmpq    $dst, [0x8000000000000000]\n\t"
10398             "jne,s   done\n\t"
10399             "subq    rsp, #8\n\t"
10400             "movsd   [rsp], $src\n\t"
10401             "call    d2l_fixup\n\t"
10402             "popq    $dst\n"
10403     "done:   "%}
10404   ins_encode %{
10405     Label done;
10406     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10407     __ cmp64($dst$$Register,
10408              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10409     __ jccb(Assembler::notEqual, done);
10410     __ subptr(rsp, 8);
10411     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10412     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10413     __ pop($dst$$Register);
10414     __ bind(done);
10415   %}
10416   ins_pipe(pipe_slow);
10417 %}
10418 
10419 instruct convI2F_reg_reg(regF dst, rRegI src)
10420 %{
10421   predicate(!UseXmmI2F);
10422   match(Set dst (ConvI2F src));
10423 
10424   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10425   ins_encode %{
10426     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10427   %}
10428   ins_pipe(pipe_slow); // XXX
10429 %}
10430 
10431 instruct convI2F_reg_mem(regF dst, memory src)
10432 %{
10433   match(Set dst (ConvI2F (LoadI src)));
10434 
10435   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10436   ins_encode %{
10437     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10438   %}
10439   ins_pipe(pipe_slow); // XXX
10440 %}
10441 
10442 instruct convI2D_reg_reg(regD dst, rRegI src)
10443 %{
10444   predicate(!UseXmmI2D);
10445   match(Set dst (ConvI2D src));
10446 
10447   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10448   ins_encode %{
10449     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10450   %}
10451   ins_pipe(pipe_slow); // XXX
10452 %}
10453 
10454 instruct convI2D_reg_mem(regD dst, memory src)
10455 %{
10456   match(Set dst (ConvI2D (LoadI src)));
10457 
10458   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10459   ins_encode %{
10460     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10461   %}
10462   ins_pipe(pipe_slow); // XXX
10463 %}
10464 
10465 instruct convXI2F_reg(regF dst, rRegI src)
10466 %{
10467   predicate(UseXmmI2F);
10468   match(Set dst (ConvI2F src));
10469 
10470   format %{ "movdl $dst, $src\n\t"
10471             "cvtdq2psl $dst, $dst\t# i2f" %}
10472   ins_encode %{
10473     __ movdl($dst$$XMMRegister, $src$$Register);
10474     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10475   %}
10476   ins_pipe(pipe_slow); // XXX
10477 %}
10478 
10479 instruct convXI2D_reg(regD dst, rRegI src)
10480 %{
10481   predicate(UseXmmI2D);
10482   match(Set dst (ConvI2D src));
10483 
10484   format %{ "movdl $dst, $src\n\t"
10485             "cvtdq2pdl $dst, $dst\t# i2d" %}
10486   ins_encode %{
10487     __ movdl($dst$$XMMRegister, $src$$Register);
10488     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10489   %}
10490   ins_pipe(pipe_slow); // XXX
10491 %}
10492 
10493 instruct convL2F_reg_reg(regF dst, rRegL src)
10494 %{
10495   match(Set dst (ConvL2F src));
10496 
10497   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10498   ins_encode %{
10499     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10500   %}
10501   ins_pipe(pipe_slow); // XXX
10502 %}
10503 
10504 instruct convL2F_reg_mem(regF dst, memory src)
10505 %{
10506   match(Set dst (ConvL2F (LoadL src)));
10507 
10508   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10509   ins_encode %{
10510     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10511   %}
10512   ins_pipe(pipe_slow); // XXX
10513 %}
10514 
10515 instruct convL2D_reg_reg(regD dst, rRegL src)
10516 %{
10517   match(Set dst (ConvL2D src));
10518 
10519   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10520   ins_encode %{
10521     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10522   %}
10523   ins_pipe(pipe_slow); // XXX
10524 %}
10525 
10526 instruct convL2D_reg_mem(regD dst, memory src)
10527 %{
10528   match(Set dst (ConvL2D (LoadL src)));
10529 
10530   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10531   ins_encode %{
10532     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10533   %}
10534   ins_pipe(pipe_slow); // XXX
10535 %}
10536 
10537 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10538 %{
10539   match(Set dst (ConvI2L src));
10540 
10541   ins_cost(125);
10542   format %{ "movslq  $dst, $src\t# i2l" %}
10543   ins_encode %{
10544     __ movslq($dst$$Register, $src$$Register);
10545   %}
10546   ins_pipe(ialu_reg_reg);
10547 %}
10548 
10549 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10550 // %{
10551 //   match(Set dst (ConvI2L src));
10552 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10553 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10554 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10555 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10556 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10557 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10558 
10559 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10560 //   ins_encode(enc_copy(dst, src));
10561 // //   opcode(0x63); // needs REX.W
10562 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10563 //   ins_pipe(ialu_reg_reg);
10564 // %}
10565 
10566 // Zero-extend convert int to long
10567 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10568 %{
10569   match(Set dst (AndL (ConvI2L src) mask));
10570 
10571   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10572   ins_encode %{
10573     if ($dst$$reg != $src$$reg) {
10574       __ movl($dst$$Register, $src$$Register);
10575     }
10576   %}
10577   ins_pipe(ialu_reg_reg);
10578 %}
10579 
10580 // Zero-extend convert int to long
10581 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10582 %{
10583   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10584 
10585   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10586   ins_encode %{
10587     __ movl($dst$$Register, $src$$Address);
10588   %}
10589   ins_pipe(ialu_reg_mem);
10590 %}
10591 
10592 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10593 %{
10594   match(Set dst (AndL src mask));
10595 
10596   format %{ "movl    $dst, $src\t# zero-extend long" %}
10597   ins_encode %{
10598     __ movl($dst$$Register, $src$$Register);
10599   %}
10600   ins_pipe(ialu_reg_reg);
10601 %}
10602 
10603 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10604 %{
10605   match(Set dst (ConvL2I src));
10606 
10607   format %{ "movl    $dst, $src\t# l2i" %}
10608   ins_encode %{
10609     __ movl($dst$$Register, $src$$Register);
10610   %}
10611   ins_pipe(ialu_reg_reg);
10612 %}
10613 
10614 
10615 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10616   match(Set dst (MoveF2I src));
10617   effect(DEF dst, USE src);
10618 
10619   ins_cost(125);
10620   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10621   ins_encode %{
10622     __ movl($dst$$Register, Address(rsp, $src$$disp));
10623   %}
10624   ins_pipe(ialu_reg_mem);
10625 %}
10626 
10627 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10628   match(Set dst (MoveI2F src));
10629   effect(DEF dst, USE src);
10630 
10631   ins_cost(125);
10632   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10633   ins_encode %{
10634     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10635   %}
10636   ins_pipe(pipe_slow);
10637 %}
10638 
10639 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10640   match(Set dst (MoveD2L src));
10641   effect(DEF dst, USE src);
10642 
10643   ins_cost(125);
10644   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10645   ins_encode %{
10646     __ movq($dst$$Register, Address(rsp, $src$$disp));
10647   %}
10648   ins_pipe(ialu_reg_mem);
10649 %}
10650 
10651 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10652   predicate(!UseXmmLoadAndClearUpper);
10653   match(Set dst (MoveL2D src));
10654   effect(DEF dst, USE src);
10655 
10656   ins_cost(125);
10657   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10658   ins_encode %{
10659     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10660   %}
10661   ins_pipe(pipe_slow);
10662 %}
10663 
10664 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10665   predicate(UseXmmLoadAndClearUpper);
10666   match(Set dst (MoveL2D src));
10667   effect(DEF dst, USE src);
10668 
10669   ins_cost(125);
10670   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10671   ins_encode %{
10672     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10673   %}
10674   ins_pipe(pipe_slow);
10675 %}
10676 
10677 
10678 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10679   match(Set dst (MoveF2I src));
10680   effect(DEF dst, USE src);
10681 
10682   ins_cost(95); // XXX
10683   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10684   ins_encode %{
10685     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10686   %}
10687   ins_pipe(pipe_slow);
10688 %}
10689 
10690 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10691   match(Set dst (MoveI2F src));
10692   effect(DEF dst, USE src);
10693 
10694   ins_cost(100);
10695   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10696   ins_encode %{
10697     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10698   %}
10699   ins_pipe( ialu_mem_reg );
10700 %}
10701 
10702 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10703   match(Set dst (MoveD2L src));
10704   effect(DEF dst, USE src);
10705 
10706   ins_cost(95); // XXX
10707   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10708   ins_encode %{
10709     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10710   %}
10711   ins_pipe(pipe_slow);
10712 %}
10713 
10714 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10715   match(Set dst (MoveL2D src));
10716   effect(DEF dst, USE src);
10717 
10718   ins_cost(100);
10719   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10720   ins_encode %{
10721     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10722   %}
10723   ins_pipe(ialu_mem_reg);
10724 %}
10725 
10726 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10727   match(Set dst (MoveF2I src));
10728   effect(DEF dst, USE src);
10729   ins_cost(85);
10730   format %{ "movd    $dst,$src\t# MoveF2I" %}
10731   ins_encode %{
10732     __ movdl($dst$$Register, $src$$XMMRegister);
10733   %}
10734   ins_pipe( pipe_slow );
10735 %}
10736 
10737 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10738   match(Set dst (MoveD2L src));
10739   effect(DEF dst, USE src);
10740   ins_cost(85);
10741   format %{ "movd    $dst,$src\t# MoveD2L" %}
10742   ins_encode %{
10743     __ movdq($dst$$Register, $src$$XMMRegister);
10744   %}
10745   ins_pipe( pipe_slow );
10746 %}
10747 
10748 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10749   match(Set dst (MoveI2F src));
10750   effect(DEF dst, USE src);
10751   ins_cost(100);
10752   format %{ "movd    $dst,$src\t# MoveI2F" %}
10753   ins_encode %{
10754     __ movdl($dst$$XMMRegister, $src$$Register);
10755   %}
10756   ins_pipe( pipe_slow );
10757 %}
10758 
10759 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10760   match(Set dst (MoveL2D src));
10761   effect(DEF dst, USE src);
10762   ins_cost(100);
10763   format %{ "movd    $dst,$src\t# MoveL2D" %}
10764   ins_encode %{
10765      __ movdq($dst$$XMMRegister, $src$$Register);
10766   %}
10767   ins_pipe( pipe_slow );
10768 %}
10769 
10770 
10771 // =======================================================================
10772 // fast clearing of an array
10773 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10774                   Universe dummy, rFlagsReg cr)
10775 %{
10776   predicate(!((ClearArrayNode*)n)->is_large());
10777   match(Set dummy (ClearArray cnt base));
10778   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10779 
10780   format %{ $$template
10781     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10782     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10783     $$emit$$"jg      LARGE\n\t"
10784     $$emit$$"dec     rcx\n\t"
10785     $$emit$$"js      DONE\t# Zero length\n\t"
10786     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10787     $$emit$$"dec     rcx\n\t"
10788     $$emit$$"jge     LOOP\n\t"
10789     $$emit$$"jmp     DONE\n\t"
10790     $$emit$$"# LARGE:\n\t"
10791     if (UseFastStosb) {
10792        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10793        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10794     } else if (UseXMMForObjInit) {
10795        $$emit$$"mov     rdi,rax\n\t"
10796        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10797        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10798        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10799        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10800        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10801        $$emit$$"add     0x40,rax\n\t"
10802        $$emit$$"# L_zero_64_bytes:\n\t"
10803        $$emit$$"sub     0x8,rcx\n\t"
10804        $$emit$$"jge     L_loop\n\t"
10805        $$emit$$"add     0x4,rcx\n\t"
10806        $$emit$$"jl      L_tail\n\t"
10807        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10808        $$emit$$"add     0x20,rax\n\t"
10809        $$emit$$"sub     0x4,rcx\n\t"
10810        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10811        $$emit$$"add     0x4,rcx\n\t"
10812        $$emit$$"jle     L_end\n\t"
10813        $$emit$$"dec     rcx\n\t"
10814        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10815        $$emit$$"vmovq   xmm0,(rax)\n\t"
10816        $$emit$$"add     0x8,rax\n\t"
10817        $$emit$$"dec     rcx\n\t"
10818        $$emit$$"jge     L_sloop\n\t"
10819        $$emit$$"# L_end:\n\t"
10820     } else {
10821        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10822     }
10823     $$emit$$"# DONE"
10824   %}
10825   ins_encode %{
10826     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10827                  $tmp$$XMMRegister, false);
10828   %}
10829   ins_pipe(pipe_slow);
10830 %}
10831 
10832 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero, 
10833                         Universe dummy, rFlagsReg cr)
10834 %{
10835   predicate(((ClearArrayNode*)n)->is_large());
10836   match(Set dummy (ClearArray cnt base));
10837   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10838 
10839   format %{ $$template
10840     if (UseFastStosb) {
10841        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10842        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10843        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10844     } else if (UseXMMForObjInit) {
10845        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10846        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10847        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10848        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10849        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10850        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10851        $$emit$$"add     0x40,rax\n\t"
10852        $$emit$$"# L_zero_64_bytes:\n\t"
10853        $$emit$$"sub     0x8,rcx\n\t"
10854        $$emit$$"jge     L_loop\n\t"
10855        $$emit$$"add     0x4,rcx\n\t"
10856        $$emit$$"jl      L_tail\n\t"
10857        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10858        $$emit$$"add     0x20,rax\n\t"
10859        $$emit$$"sub     0x4,rcx\n\t"
10860        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10861        $$emit$$"add     0x4,rcx\n\t"
10862        $$emit$$"jle     L_end\n\t"
10863        $$emit$$"dec     rcx\n\t"
10864        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10865        $$emit$$"vmovq   xmm0,(rax)\n\t"
10866        $$emit$$"add     0x8,rax\n\t"
10867        $$emit$$"dec     rcx\n\t"
10868        $$emit$$"jge     L_sloop\n\t"
10869        $$emit$$"# L_end:\n\t"
10870     } else {
10871        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10872        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10873     }
10874   %}
10875   ins_encode %{
10876     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, 
10877                  $tmp$$XMMRegister, true);
10878   %}
10879   ins_pipe(pipe_slow);
10880 %}
10881 
10882 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10883                          rax_RegI result, regD tmp1, rFlagsReg cr)
10884 %{
10885   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10886   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10887   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10888 
10889   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10890   ins_encode %{
10891     __ string_compare($str1$$Register, $str2$$Register,
10892                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10893                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10894   %}
10895   ins_pipe( pipe_slow );
10896 %}
10897 
10898 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10899                          rax_RegI result, regD tmp1, rFlagsReg cr)
10900 %{
10901   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10902   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10903   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10904 
10905   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10906   ins_encode %{
10907     __ string_compare($str1$$Register, $str2$$Register,
10908                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10909                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10910   %}
10911   ins_pipe( pipe_slow );
10912 %}
10913 
10914 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10915                           rax_RegI result, regD tmp1, rFlagsReg cr)
10916 %{
10917   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10918   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10919   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10920 
10921   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10922   ins_encode %{
10923     __ string_compare($str1$$Register, $str2$$Register,
10924                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10925                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10926   %}
10927   ins_pipe( pipe_slow );
10928 %}
10929 
10930 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10931                           rax_RegI result, regD tmp1, rFlagsReg cr)
10932 %{
10933   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10934   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10935   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10936 
10937   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10938   ins_encode %{
10939     __ string_compare($str2$$Register, $str1$$Register,
10940                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10941                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10942   %}
10943   ins_pipe( pipe_slow );
10944 %}
10945 
10946 // fast search of substring with known size.
10947 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10948                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10949 %{
10950   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10951   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10952   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10953 
10954   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10955   ins_encode %{
10956     int icnt2 = (int)$int_cnt2$$constant;
10957     if (icnt2 >= 16) {
10958       // IndexOf for constant substrings with size >= 16 elements
10959       // which don't need to be loaded through stack.
10960       __ string_indexofC8($str1$$Register, $str2$$Register,
10961                           $cnt1$$Register, $cnt2$$Register,
10962                           icnt2, $result$$Register,
10963                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10964     } else {
10965       // Small strings are loaded through stack if they cross page boundary.
10966       __ string_indexof($str1$$Register, $str2$$Register,
10967                         $cnt1$$Register, $cnt2$$Register,
10968                         icnt2, $result$$Register,
10969                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10970     }
10971   %}
10972   ins_pipe( pipe_slow );
10973 %}
10974 
10975 // fast search of substring with known size.
10976 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10977                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10978 %{
10979   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10980   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10981   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10982 
10983   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10984   ins_encode %{
10985     int icnt2 = (int)$int_cnt2$$constant;
10986     if (icnt2 >= 8) {
10987       // IndexOf for constant substrings with size >= 8 elements
10988       // which don't need to be loaded through stack.
10989       __ string_indexofC8($str1$$Register, $str2$$Register,
10990                           $cnt1$$Register, $cnt2$$Register,
10991                           icnt2, $result$$Register,
10992                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10993     } else {
10994       // Small strings are loaded through stack if they cross page boundary.
10995       __ string_indexof($str1$$Register, $str2$$Register,
10996                         $cnt1$$Register, $cnt2$$Register,
10997                         icnt2, $result$$Register,
10998                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10999     }
11000   %}
11001   ins_pipe( pipe_slow );
11002 %}
11003 
11004 // fast search of substring with known size.
11005 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11006                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11007 %{
11008   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11009   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11010   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11011 
11012   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11013   ins_encode %{
11014     int icnt2 = (int)$int_cnt2$$constant;
11015     if (icnt2 >= 8) {
11016       // IndexOf for constant substrings with size >= 8 elements
11017       // which don't need to be loaded through stack.
11018       __ string_indexofC8($str1$$Register, $str2$$Register,
11019                           $cnt1$$Register, $cnt2$$Register,
11020                           icnt2, $result$$Register,
11021                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11022     } else {
11023       // Small strings are loaded through stack if they cross page boundary.
11024       __ string_indexof($str1$$Register, $str2$$Register,
11025                         $cnt1$$Register, $cnt2$$Register,
11026                         icnt2, $result$$Register,
11027                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11028     }
11029   %}
11030   ins_pipe( pipe_slow );
11031 %}
11032 
11033 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11034                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11035 %{
11036   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11037   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11038   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11039 
11040   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11041   ins_encode %{
11042     __ string_indexof($str1$$Register, $str2$$Register,
11043                       $cnt1$$Register, $cnt2$$Register,
11044                       (-1), $result$$Register,
11045                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11046   %}
11047   ins_pipe( pipe_slow );
11048 %}
11049 
11050 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11051                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11052 %{
11053   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11054   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11055   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11056 
11057   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11058   ins_encode %{
11059     __ string_indexof($str1$$Register, $str2$$Register,
11060                       $cnt1$$Register, $cnt2$$Register,
11061                       (-1), $result$$Register,
11062                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11063   %}
11064   ins_pipe( pipe_slow );
11065 %}
11066 
11067 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11068                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
11069 %{
11070   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11071   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11072   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11073 
11074   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11075   ins_encode %{
11076     __ string_indexof($str1$$Register, $str2$$Register,
11077                       $cnt1$$Register, $cnt2$$Register,
11078                       (-1), $result$$Register,
11079                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11080   %}
11081   ins_pipe( pipe_slow );
11082 %}
11083 
11084 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11085                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
11086 %{
11087   predicate(UseSSE42Intrinsics);
11088   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11089   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11090   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11091   ins_encode %{
11092     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11093                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11094   %}
11095   ins_pipe( pipe_slow );
11096 %}
11097 
11098 // fast string equals
11099 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11100                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11101 %{
11102   match(Set result (StrEquals (Binary str1 str2) cnt));
11103   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11104 
11105   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11106   ins_encode %{
11107     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11108                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11109                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11110   %}
11111   ins_pipe( pipe_slow );
11112 %}
11113 
11114 // fast array equals
11115 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11116                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11117 %{
11118   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11119   match(Set result (AryEq ary1 ary2));
11120   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11121 
11122   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11123   ins_encode %{
11124     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11125                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11126                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11127   %}
11128   ins_pipe( pipe_slow );
11129 %}
11130 
11131 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11132                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11133 %{
11134   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11135   match(Set result (AryEq ary1 ary2));
11136   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11137 
11138   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11139   ins_encode %{
11140     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11141                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11142                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11143   %}
11144   ins_pipe( pipe_slow );
11145 %}
11146 
11147 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11148                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
11149 %{
11150   match(Set result (HasNegatives ary1 len));
11151   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11152 
11153   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11154   ins_encode %{
11155     __ has_negatives($ary1$$Register, $len$$Register,
11156                      $result$$Register, $tmp3$$Register,
11157                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11158   %}
11159   ins_pipe( pipe_slow );
11160 %}
11161 
11162 // fast char[] to byte[] compression
11163 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11164                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11165   match(Set result (StrCompressedCopy src (Binary dst len)));
11166   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11167 
11168   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11169   ins_encode %{
11170     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11171                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11172                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11173   %}
11174   ins_pipe( pipe_slow );
11175 %}
11176 
11177 // fast byte[] to char[] inflation
11178 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11179                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11180   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11181   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11182 
11183   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11184   ins_encode %{
11185     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11186                           $tmp1$$XMMRegister, $tmp2$$Register);
11187   %}
11188   ins_pipe( pipe_slow );
11189 %}
11190 
11191 // encode char[] to byte[] in ISO_8859_1
11192 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11193                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
11194                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11195   match(Set result (EncodeISOArray src (Binary dst len)));
11196   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11197 
11198   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11199   ins_encode %{
11200     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11201                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11202                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11203   %}
11204   ins_pipe( pipe_slow );
11205 %}
11206 
11207 //----------Overflow Math Instructions-----------------------------------------
11208 
11209 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11210 %{
11211   match(Set cr (OverflowAddI op1 op2));
11212   effect(DEF cr, USE_KILL op1, USE op2);
11213 
11214   format %{ "addl    $op1, $op2\t# overflow check int" %}
11215 
11216   ins_encode %{
11217     __ addl($op1$$Register, $op2$$Register);
11218   %}
11219   ins_pipe(ialu_reg_reg);
11220 %}
11221 
11222 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11223 %{
11224   match(Set cr (OverflowAddI op1 op2));
11225   effect(DEF cr, USE_KILL op1, USE op2);
11226 
11227   format %{ "addl    $op1, $op2\t# overflow check int" %}
11228 
11229   ins_encode %{
11230     __ addl($op1$$Register, $op2$$constant);
11231   %}
11232   ins_pipe(ialu_reg_reg);
11233 %}
11234 
11235 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11236 %{
11237   match(Set cr (OverflowAddL op1 op2));
11238   effect(DEF cr, USE_KILL op1, USE op2);
11239 
11240   format %{ "addq    $op1, $op2\t# overflow check long" %}
11241   ins_encode %{
11242     __ addq($op1$$Register, $op2$$Register);
11243   %}
11244   ins_pipe(ialu_reg_reg);
11245 %}
11246 
11247 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11248 %{
11249   match(Set cr (OverflowAddL op1 op2));
11250   effect(DEF cr, USE_KILL op1, USE op2);
11251 
11252   format %{ "addq    $op1, $op2\t# overflow check long" %}
11253   ins_encode %{
11254     __ addq($op1$$Register, $op2$$constant);
11255   %}
11256   ins_pipe(ialu_reg_reg);
11257 %}
11258 
11259 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11260 %{
11261   match(Set cr (OverflowSubI op1 op2));
11262 
11263   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11264   ins_encode %{
11265     __ cmpl($op1$$Register, $op2$$Register);
11266   %}
11267   ins_pipe(ialu_reg_reg);
11268 %}
11269 
11270 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11271 %{
11272   match(Set cr (OverflowSubI op1 op2));
11273 
11274   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11275   ins_encode %{
11276     __ cmpl($op1$$Register, $op2$$constant);
11277   %}
11278   ins_pipe(ialu_reg_reg);
11279 %}
11280 
11281 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11282 %{
11283   match(Set cr (OverflowSubL op1 op2));
11284 
11285   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11286   ins_encode %{
11287     __ cmpq($op1$$Register, $op2$$Register);
11288   %}
11289   ins_pipe(ialu_reg_reg);
11290 %}
11291 
11292 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11293 %{
11294   match(Set cr (OverflowSubL op1 op2));
11295 
11296   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11297   ins_encode %{
11298     __ cmpq($op1$$Register, $op2$$constant);
11299   %}
11300   ins_pipe(ialu_reg_reg);
11301 %}
11302 
11303 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11304 %{
11305   match(Set cr (OverflowSubI zero op2));
11306   effect(DEF cr, USE_KILL op2);
11307 
11308   format %{ "negl    $op2\t# overflow check int" %}
11309   ins_encode %{
11310     __ negl($op2$$Register);
11311   %}
11312   ins_pipe(ialu_reg_reg);
11313 %}
11314 
11315 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11316 %{
11317   match(Set cr (OverflowSubL zero op2));
11318   effect(DEF cr, USE_KILL op2);
11319 
11320   format %{ "negq    $op2\t# overflow check long" %}
11321   ins_encode %{
11322     __ negq($op2$$Register);
11323   %}
11324   ins_pipe(ialu_reg_reg);
11325 %}
11326 
11327 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11328 %{
11329   match(Set cr (OverflowMulI op1 op2));
11330   effect(DEF cr, USE_KILL op1, USE op2);
11331 
11332   format %{ "imull    $op1, $op2\t# overflow check int" %}
11333   ins_encode %{
11334     __ imull($op1$$Register, $op2$$Register);
11335   %}
11336   ins_pipe(ialu_reg_reg_alu0);
11337 %}
11338 
11339 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11340 %{
11341   match(Set cr (OverflowMulI op1 op2));
11342   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11343 
11344   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11345   ins_encode %{
11346     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11347   %}
11348   ins_pipe(ialu_reg_reg_alu0);
11349 %}
11350 
11351 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11352 %{
11353   match(Set cr (OverflowMulL op1 op2));
11354   effect(DEF cr, USE_KILL op1, USE op2);
11355 
11356   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11357   ins_encode %{
11358     __ imulq($op1$$Register, $op2$$Register);
11359   %}
11360   ins_pipe(ialu_reg_reg_alu0);
11361 %}
11362 
11363 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11364 %{
11365   match(Set cr (OverflowMulL op1 op2));
11366   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11367 
11368   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11369   ins_encode %{
11370     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11371   %}
11372   ins_pipe(ialu_reg_reg_alu0);
11373 %}
11374 
11375 
11376 //----------Control Flow Instructions------------------------------------------
11377 // Signed compare Instructions
11378 
11379 // XXX more variants!!
11380 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11381 %{
11382   match(Set cr (CmpI op1 op2));
11383   effect(DEF cr, USE op1, USE op2);
11384 
11385   format %{ "cmpl    $op1, $op2" %}
11386   opcode(0x3B);  /* Opcode 3B /r */
11387   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11388   ins_pipe(ialu_cr_reg_reg);
11389 %}
11390 
11391 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11392 %{
11393   match(Set cr (CmpI op1 op2));
11394 
11395   format %{ "cmpl    $op1, $op2" %}
11396   opcode(0x81, 0x07); /* Opcode 81 /7 */
11397   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11398   ins_pipe(ialu_cr_reg_imm);
11399 %}
11400 
11401 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11402 %{
11403   match(Set cr (CmpI op1 (LoadI op2)));
11404 
11405   ins_cost(500); // XXX
11406   format %{ "cmpl    $op1, $op2" %}
11407   opcode(0x3B); /* Opcode 3B /r */
11408   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11409   ins_pipe(ialu_cr_reg_mem);
11410 %}
11411 
11412 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11413 %{
11414   match(Set cr (CmpI src zero));
11415 
11416   format %{ "testl   $src, $src" %}
11417   opcode(0x85);
11418   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11419   ins_pipe(ialu_cr_reg_imm);
11420 %}
11421 
11422 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11423 %{
11424   match(Set cr (CmpI (AndI src con) zero));
11425 
11426   format %{ "testl   $src, $con" %}
11427   opcode(0xF7, 0x00);
11428   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11429   ins_pipe(ialu_cr_reg_imm);
11430 %}
11431 
11432 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11433 %{
11434   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11435 
11436   format %{ "testl   $src, $mem" %}
11437   opcode(0x85);
11438   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11439   ins_pipe(ialu_cr_reg_mem);
11440 %}
11441 
11442 // Unsigned compare Instructions; really, same as signed except they
11443 // produce an rFlagsRegU instead of rFlagsReg.
11444 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11445 %{
11446   match(Set cr (CmpU op1 op2));
11447 
11448   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11449   opcode(0x3B); /* Opcode 3B /r */
11450   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11451   ins_pipe(ialu_cr_reg_reg);
11452 %}
11453 
11454 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11455 %{
11456   match(Set cr (CmpU op1 op2));
11457 
11458   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11459   opcode(0x81,0x07); /* Opcode 81 /7 */
11460   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11461   ins_pipe(ialu_cr_reg_imm);
11462 %}
11463 
11464 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11465 %{
11466   match(Set cr (CmpU op1 (LoadI op2)));
11467 
11468   ins_cost(500); // XXX
11469   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11470   opcode(0x3B); /* Opcode 3B /r */
11471   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11472   ins_pipe(ialu_cr_reg_mem);
11473 %}
11474 
11475 // // // Cisc-spilled version of cmpU_rReg
11476 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11477 // //%{
11478 // //  match(Set cr (CmpU (LoadI op1) op2));
11479 // //
11480 // //  format %{ "CMPu   $op1,$op2" %}
11481 // //  ins_cost(500);
11482 // //  opcode(0x39);  /* Opcode 39 /r */
11483 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11484 // //%}
11485 
11486 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11487 %{
11488   match(Set cr (CmpU src zero));
11489 
11490   format %{ "testl  $src, $src\t# unsigned" %}
11491   opcode(0x85);
11492   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11493   ins_pipe(ialu_cr_reg_imm);
11494 %}
11495 
11496 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11497 %{
11498   match(Set cr (CmpP op1 op2));
11499 
11500   format %{ "cmpq    $op1, $op2\t# ptr" %}
11501   opcode(0x3B); /* Opcode 3B /r */
11502   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11503   ins_pipe(ialu_cr_reg_reg);
11504 %}
11505 
11506 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11507 %{
11508   match(Set cr (CmpP op1 (LoadP op2)));
11509 
11510   ins_cost(500); // XXX
11511   format %{ "cmpq    $op1, $op2\t# ptr" %}
11512   opcode(0x3B); /* Opcode 3B /r */
11513   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11514   ins_pipe(ialu_cr_reg_mem);
11515 %}
11516 
11517 // // // Cisc-spilled version of cmpP_rReg
11518 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11519 // //%{
11520 // //  match(Set cr (CmpP (LoadP op1) op2));
11521 // //
11522 // //  format %{ "CMPu   $op1,$op2" %}
11523 // //  ins_cost(500);
11524 // //  opcode(0x39);  /* Opcode 39 /r */
11525 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11526 // //%}
11527 
11528 // XXX this is generalized by compP_rReg_mem???
11529 // Compare raw pointer (used in out-of-heap check).
11530 // Only works because non-oop pointers must be raw pointers
11531 // and raw pointers have no anti-dependencies.
11532 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11533 %{
11534   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11535   match(Set cr (CmpP op1 (LoadP op2)));
11536 
11537   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11538   opcode(0x3B); /* Opcode 3B /r */
11539   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11540   ins_pipe(ialu_cr_reg_mem);
11541 %}
11542 
11543 // This will generate a signed flags result. This should be OK since
11544 // any compare to a zero should be eq/neq.
11545 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11546 %{
11547   match(Set cr (CmpP src zero));
11548 
11549   format %{ "testq   $src, $src\t# ptr" %}
11550   opcode(0x85);
11551   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11552   ins_pipe(ialu_cr_reg_imm);
11553 %}
11554 
11555 // This will generate a signed flags result. This should be OK since
11556 // any compare to a zero should be eq/neq.
11557 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11558 %{
11559   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11560   match(Set cr (CmpP (LoadP op) zero));
11561 
11562   ins_cost(500); // XXX
11563   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11564   opcode(0xF7); /* Opcode F7 /0 */
11565   ins_encode(REX_mem_wide(op),
11566              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11567   ins_pipe(ialu_cr_reg_imm);
11568 %}
11569 
11570 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11571 %{
11572   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11573   match(Set cr (CmpP (LoadP mem) zero));
11574 
11575   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11576   ins_encode %{
11577     __ cmpq(r12, $mem$$Address);
11578   %}
11579   ins_pipe(ialu_cr_reg_mem);
11580 %}
11581 
11582 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11583 %{
11584   match(Set cr (CmpN op1 op2));
11585 
11586   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11587   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11588   ins_pipe(ialu_cr_reg_reg);
11589 %}
11590 
11591 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11592 %{
11593   match(Set cr (CmpN src (LoadN mem)));
11594 
11595   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11596   ins_encode %{
11597     __ cmpl($src$$Register, $mem$$Address);
11598   %}
11599   ins_pipe(ialu_cr_reg_mem);
11600 %}
11601 
11602 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11603   match(Set cr (CmpN op1 op2));
11604 
11605   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11606   ins_encode %{
11607     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11608   %}
11609   ins_pipe(ialu_cr_reg_imm);
11610 %}
11611 
11612 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11613 %{
11614   match(Set cr (CmpN src (LoadN mem)));
11615 
11616   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11617   ins_encode %{
11618     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11619   %}
11620   ins_pipe(ialu_cr_reg_mem);
11621 %}
11622 
11623 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11624   match(Set cr (CmpN op1 op2));
11625 
11626   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11627   ins_encode %{
11628     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11629   %}
11630   ins_pipe(ialu_cr_reg_imm);
11631 %}
11632 
11633 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11634 %{
11635   match(Set cr (CmpN src (LoadNKlass mem)));
11636 
11637   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11638   ins_encode %{
11639     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11640   %}
11641   ins_pipe(ialu_cr_reg_mem);
11642 %}
11643 
11644 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11645   match(Set cr (CmpN src zero));
11646 
11647   format %{ "testl   $src, $src\t# compressed ptr" %}
11648   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11649   ins_pipe(ialu_cr_reg_imm);
11650 %}
11651 
11652 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11653 %{
11654   predicate(Universe::narrow_oop_base() != NULL);
11655   match(Set cr (CmpN (LoadN mem) zero));
11656 
11657   ins_cost(500); // XXX
11658   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11659   ins_encode %{
11660     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11661   %}
11662   ins_pipe(ialu_cr_reg_mem);
11663 %}
11664 
11665 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11666 %{
11667   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11668   match(Set cr (CmpN (LoadN mem) zero));
11669 
11670   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11671   ins_encode %{
11672     __ cmpl(r12, $mem$$Address);
11673   %}
11674   ins_pipe(ialu_cr_reg_mem);
11675 %}
11676 
11677 // Yanked all unsigned pointer compare operations.
11678 // Pointer compares are done with CmpP which is already unsigned.
11679 
11680 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11681 %{
11682   match(Set cr (CmpL op1 op2));
11683 
11684   format %{ "cmpq    $op1, $op2" %}
11685   opcode(0x3B);  /* Opcode 3B /r */
11686   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11687   ins_pipe(ialu_cr_reg_reg);
11688 %}
11689 
11690 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11691 %{
11692   match(Set cr (CmpL op1 op2));
11693 
11694   format %{ "cmpq    $op1, $op2" %}
11695   opcode(0x81, 0x07); /* Opcode 81 /7 */
11696   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11697   ins_pipe(ialu_cr_reg_imm);
11698 %}
11699 
11700 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11701 %{
11702   match(Set cr (CmpL op1 (LoadL op2)));
11703 
11704   format %{ "cmpq    $op1, $op2" %}
11705   opcode(0x3B); /* Opcode 3B /r */
11706   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11707   ins_pipe(ialu_cr_reg_mem);
11708 %}
11709 
11710 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11711 %{
11712   match(Set cr (CmpL src zero));
11713 
11714   format %{ "testq   $src, $src" %}
11715   opcode(0x85);
11716   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11717   ins_pipe(ialu_cr_reg_imm);
11718 %}
11719 
11720 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11721 %{
11722   match(Set cr (CmpL (AndL src con) zero));
11723 
11724   format %{ "testq   $src, $con\t# long" %}
11725   opcode(0xF7, 0x00);
11726   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11727   ins_pipe(ialu_cr_reg_imm);
11728 %}
11729 
11730 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11731 %{
11732   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11733 
11734   format %{ "testq   $src, $mem" %}
11735   opcode(0x85);
11736   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11737   ins_pipe(ialu_cr_reg_mem);
11738 %}
11739 
11740 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11741 %{
11742   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11743 
11744   format %{ "testq   $src, $mem" %}
11745   opcode(0x85);
11746   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11747   ins_pipe(ialu_cr_reg_mem);
11748 %}
11749 
11750 // Manifest a CmpL result in an integer register.  Very painful.
11751 // This is the test to avoid.
11752 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11753 %{
11754   match(Set dst (CmpL3 src1 src2));
11755   effect(KILL flags);
11756 
11757   ins_cost(275); // XXX
11758   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11759             "movl    $dst, -1\n\t"
11760             "jl,s    done\n\t"
11761             "setne   $dst\n\t"
11762             "movzbl  $dst, $dst\n\t"
11763     "done:" %}
11764   ins_encode(cmpl3_flag(src1, src2, dst));
11765   ins_pipe(pipe_slow);
11766 %}
11767 
11768 // Unsigned long compare Instructions; really, same as signed long except they
11769 // produce an rFlagsRegU instead of rFlagsReg.
11770 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11771 %{
11772   match(Set cr (CmpUL op1 op2));
11773 
11774   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11775   opcode(0x3B);  /* Opcode 3B /r */
11776   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11777   ins_pipe(ialu_cr_reg_reg);
11778 %}
11779 
11780 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11781 %{
11782   match(Set cr (CmpUL op1 op2));
11783 
11784   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11785   opcode(0x81, 0x07); /* Opcode 81 /7 */
11786   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11787   ins_pipe(ialu_cr_reg_imm);
11788 %}
11789 
11790 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11791 %{
11792   match(Set cr (CmpUL op1 (LoadL op2)));
11793 
11794   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11795   opcode(0x3B); /* Opcode 3B /r */
11796   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11797   ins_pipe(ialu_cr_reg_mem);
11798 %}
11799 
11800 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11801 %{
11802   match(Set cr (CmpUL src zero));
11803 
11804   format %{ "testq   $src, $src\t# unsigned" %}
11805   opcode(0x85);
11806   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11807   ins_pipe(ialu_cr_reg_imm);
11808 %}
11809 
11810 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11811 %{
11812   match(Set cr (CmpI (LoadB mem) imm));
11813 
11814   ins_cost(125);
11815   format %{ "cmpb    $mem, $imm" %}
11816   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11817   ins_pipe(ialu_cr_reg_mem);
11818 %}
11819 
11820 instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI0 zero)
11821 %{
11822   match(Set cr (CmpI (AndI (LoadB mem) imm) zero));
11823 
11824   ins_cost(125);
11825   format %{ "testb   $mem, $imm" %}
11826   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11827   ins_pipe(ialu_cr_reg_mem);
11828 %}
11829 
11830 //----------Max and Min--------------------------------------------------------
11831 // Min Instructions
11832 
11833 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11834 %{
11835   effect(USE_DEF dst, USE src, USE cr);
11836 
11837   format %{ "cmovlgt $dst, $src\t# min" %}
11838   opcode(0x0F, 0x4F);
11839   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11840   ins_pipe(pipe_cmov_reg);
11841 %}
11842 
11843 
11844 instruct minI_rReg(rRegI dst, rRegI src)
11845 %{
11846   match(Set dst (MinI dst src));
11847 
11848   ins_cost(200);
11849   expand %{
11850     rFlagsReg cr;
11851     compI_rReg(cr, dst, src);
11852     cmovI_reg_g(dst, src, cr);
11853   %}
11854 %}
11855 
11856 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11857 %{
11858   effect(USE_DEF dst, USE src, USE cr);
11859 
11860   format %{ "cmovllt $dst, $src\t# max" %}
11861   opcode(0x0F, 0x4C);
11862   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11863   ins_pipe(pipe_cmov_reg);
11864 %}
11865 
11866 
11867 instruct maxI_rReg(rRegI dst, rRegI src)
11868 %{
11869   match(Set dst (MaxI dst src));
11870 
11871   ins_cost(200);
11872   expand %{
11873     rFlagsReg cr;
11874     compI_rReg(cr, dst, src);
11875     cmovI_reg_l(dst, src, cr);
11876   %}
11877 %}
11878 
11879 // ============================================================================
11880 // Branch Instructions
11881 
11882 // Jump Direct - Label defines a relative address from JMP+1
11883 instruct jmpDir(label labl)
11884 %{
11885   match(Goto);
11886   effect(USE labl);
11887 
11888   ins_cost(300);
11889   format %{ "jmp     $labl" %}
11890   size(5);
11891   ins_encode %{
11892     Label* L = $labl$$label;
11893     __ jmp(*L, false); // Always long jump
11894   %}
11895   ins_pipe(pipe_jmp);
11896 %}
11897 
11898 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11899 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11900 %{
11901   match(If cop cr);
11902   effect(USE labl);
11903 
11904   ins_cost(300);
11905   format %{ "j$cop     $labl" %}
11906   size(6);
11907   ins_encode %{
11908     Label* L = $labl$$label;
11909     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11910   %}
11911   ins_pipe(pipe_jcc);
11912 %}
11913 
11914 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11915 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11916 %{
11917   predicate(!n->has_vector_mask_set());
11918   match(CountedLoopEnd cop cr);
11919   effect(USE labl);
11920 
11921   ins_cost(300);
11922   format %{ "j$cop     $labl\t# loop end" %}
11923   size(6);
11924   ins_encode %{
11925     Label* L = $labl$$label;
11926     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11927   %}
11928   ins_pipe(pipe_jcc);
11929 %}
11930 
11931 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11932 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11933   predicate(!n->has_vector_mask_set());
11934   match(CountedLoopEnd cop cmp);
11935   effect(USE labl);
11936 
11937   ins_cost(300);
11938   format %{ "j$cop,u   $labl\t# loop end" %}
11939   size(6);
11940   ins_encode %{
11941     Label* L = $labl$$label;
11942     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11943   %}
11944   ins_pipe(pipe_jcc);
11945 %}
11946 
11947 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11948   predicate(!n->has_vector_mask_set());
11949   match(CountedLoopEnd cop cmp);
11950   effect(USE labl);
11951 
11952   ins_cost(200);
11953   format %{ "j$cop,u   $labl\t# loop end" %}
11954   size(6);
11955   ins_encode %{
11956     Label* L = $labl$$label;
11957     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11958   %}
11959   ins_pipe(pipe_jcc);
11960 %}
11961 
11962 // mask version
11963 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11964 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
11965 %{
11966   predicate(n->has_vector_mask_set());
11967   match(CountedLoopEnd cop cr);
11968   effect(USE labl);
11969 
11970   ins_cost(400);
11971   format %{ "j$cop     $labl\t# loop end\n\t"
11972             "restorevectmask \t# vector mask restore for loops" %}
11973   size(10);
11974   ins_encode %{
11975     Label* L = $labl$$label;
11976     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11977     __ restorevectmask();
11978   %}
11979   ins_pipe(pipe_jcc);
11980 %}
11981 
11982 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11983 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11984   predicate(n->has_vector_mask_set());
11985   match(CountedLoopEnd cop cmp);
11986   effect(USE labl);
11987 
11988   ins_cost(400);
11989   format %{ "j$cop,u   $labl\t# loop end\n\t"
11990             "restorevectmask \t# vector mask restore for loops" %}
11991   size(10);
11992   ins_encode %{
11993     Label* L = $labl$$label;
11994     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11995     __ restorevectmask();
11996   %}
11997   ins_pipe(pipe_jcc);
11998 %}
11999 
12000 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12001   predicate(n->has_vector_mask_set());
12002   match(CountedLoopEnd cop cmp);
12003   effect(USE labl);
12004 
12005   ins_cost(300);
12006   format %{ "j$cop,u   $labl\t# loop end\n\t"
12007             "restorevectmask \t# vector mask restore for loops" %}
12008   size(10);
12009   ins_encode %{
12010     Label* L = $labl$$label;
12011     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12012     __ restorevectmask();
12013   %}
12014   ins_pipe(pipe_jcc);
12015 %}
12016 
12017 // Jump Direct Conditional - using unsigned comparison
12018 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12019   match(If cop cmp);
12020   effect(USE labl);
12021 
12022   ins_cost(300);
12023   format %{ "j$cop,u  $labl" %}
12024   size(6);
12025   ins_encode %{
12026     Label* L = $labl$$label;
12027     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12028   %}
12029   ins_pipe(pipe_jcc);
12030 %}
12031 
12032 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12033   match(If cop cmp);
12034   effect(USE labl);
12035 
12036   ins_cost(200);
12037   format %{ "j$cop,u  $labl" %}
12038   size(6);
12039   ins_encode %{
12040     Label* L = $labl$$label;
12041     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12042   %}
12043   ins_pipe(pipe_jcc);
12044 %}
12045 
12046 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12047   match(If cop cmp);
12048   effect(USE labl);
12049 
12050   ins_cost(200);
12051   format %{ $$template
12052     if ($cop$$cmpcode == Assembler::notEqual) {
12053       $$emit$$"jp,u   $labl\n\t"
12054       $$emit$$"j$cop,u   $labl"
12055     } else {
12056       $$emit$$"jp,u   done\n\t"
12057       $$emit$$"j$cop,u   $labl\n\t"
12058       $$emit$$"done:"
12059     }
12060   %}
12061   ins_encode %{
12062     Label* l = $labl$$label;
12063     if ($cop$$cmpcode == Assembler::notEqual) {
12064       __ jcc(Assembler::parity, *l, false);
12065       __ jcc(Assembler::notEqual, *l, false);
12066     } else if ($cop$$cmpcode == Assembler::equal) {
12067       Label done;
12068       __ jccb(Assembler::parity, done);
12069       __ jcc(Assembler::equal, *l, false);
12070       __ bind(done);
12071     } else {
12072        ShouldNotReachHere();
12073     }
12074   %}
12075   ins_pipe(pipe_jcc);
12076 %}
12077 
12078 // ============================================================================
12079 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12080 // superklass array for an instance of the superklass.  Set a hidden
12081 // internal cache on a hit (cache is checked with exposed code in
12082 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12083 // encoding ALSO sets flags.
12084 
12085 instruct partialSubtypeCheck(rdi_RegP result,
12086                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12087                              rFlagsReg cr)
12088 %{
12089   match(Set result (PartialSubtypeCheck sub super));
12090   effect(KILL rcx, KILL cr);
12091 
12092   ins_cost(1100);  // slightly larger than the next version
12093   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12094             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12095             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12096             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12097             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12098             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12099             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12100     "miss:\t" %}
12101 
12102   opcode(0x1); // Force a XOR of RDI
12103   ins_encode(enc_PartialSubtypeCheck());
12104   ins_pipe(pipe_slow);
12105 %}
12106 
12107 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12108                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12109                                      immP0 zero,
12110                                      rdi_RegP result)
12111 %{
12112   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12113   effect(KILL rcx, KILL result);
12114 
12115   ins_cost(1000);
12116   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12117             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12118             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12119             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12120             "jne,s   miss\t\t# Missed: flags nz\n\t"
12121             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12122     "miss:\t" %}
12123 
12124   opcode(0x0); // No need to XOR RDI
12125   ins_encode(enc_PartialSubtypeCheck());
12126   ins_pipe(pipe_slow);
12127 %}
12128 
12129 // ============================================================================
12130 // Branch Instructions -- short offset versions
12131 //
12132 // These instructions are used to replace jumps of a long offset (the default
12133 // match) with jumps of a shorter offset.  These instructions are all tagged
12134 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12135 // match rules in general matching.  Instead, the ADLC generates a conversion
12136 // method in the MachNode which can be used to do in-place replacement of the
12137 // long variant with the shorter variant.  The compiler will determine if a
12138 // branch can be taken by the is_short_branch_offset() predicate in the machine
12139 // specific code section of the file.
12140 
12141 // Jump Direct - Label defines a relative address from JMP+1
12142 instruct jmpDir_short(label labl) %{
12143   match(Goto);
12144   effect(USE labl);
12145 
12146   ins_cost(300);
12147   format %{ "jmp,s   $labl" %}
12148   size(2);
12149   ins_encode %{
12150     Label* L = $labl$$label;
12151     __ jmpb(*L);
12152   %}
12153   ins_pipe(pipe_jmp);
12154   ins_short_branch(1);
12155 %}
12156 
12157 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12158 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12159   match(If cop cr);
12160   effect(USE labl);
12161 
12162   ins_cost(300);
12163   format %{ "j$cop,s   $labl" %}
12164   size(2);
12165   ins_encode %{
12166     Label* L = $labl$$label;
12167     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12168   %}
12169   ins_pipe(pipe_jcc);
12170   ins_short_branch(1);
12171 %}
12172 
12173 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12174 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12175   match(CountedLoopEnd cop cr);
12176   effect(USE labl);
12177 
12178   ins_cost(300);
12179   format %{ "j$cop,s   $labl\t# loop end" %}
12180   size(2);
12181   ins_encode %{
12182     Label* L = $labl$$label;
12183     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12184   %}
12185   ins_pipe(pipe_jcc);
12186   ins_short_branch(1);
12187 %}
12188 
12189 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12190 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12191   match(CountedLoopEnd cop cmp);
12192   effect(USE labl);
12193 
12194   ins_cost(300);
12195   format %{ "j$cop,us  $labl\t# loop end" %}
12196   size(2);
12197   ins_encode %{
12198     Label* L = $labl$$label;
12199     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12200   %}
12201   ins_pipe(pipe_jcc);
12202   ins_short_branch(1);
12203 %}
12204 
12205 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12206   match(CountedLoopEnd cop cmp);
12207   effect(USE labl);
12208 
12209   ins_cost(300);
12210   format %{ "j$cop,us  $labl\t# loop end" %}
12211   size(2);
12212   ins_encode %{
12213     Label* L = $labl$$label;
12214     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12215   %}
12216   ins_pipe(pipe_jcc);
12217   ins_short_branch(1);
12218 %}
12219 
12220 // Jump Direct Conditional - using unsigned comparison
12221 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12222   match(If cop cmp);
12223   effect(USE labl);
12224 
12225   ins_cost(300);
12226   format %{ "j$cop,us  $labl" %}
12227   size(2);
12228   ins_encode %{
12229     Label* L = $labl$$label;
12230     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12231   %}
12232   ins_pipe(pipe_jcc);
12233   ins_short_branch(1);
12234 %}
12235 
12236 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12237   match(If cop cmp);
12238   effect(USE labl);
12239 
12240   ins_cost(300);
12241   format %{ "j$cop,us  $labl" %}
12242   size(2);
12243   ins_encode %{
12244     Label* L = $labl$$label;
12245     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12246   %}
12247   ins_pipe(pipe_jcc);
12248   ins_short_branch(1);
12249 %}
12250 
12251 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12252   match(If cop cmp);
12253   effect(USE labl);
12254 
12255   ins_cost(300);
12256   format %{ $$template
12257     if ($cop$$cmpcode == Assembler::notEqual) {
12258       $$emit$$"jp,u,s   $labl\n\t"
12259       $$emit$$"j$cop,u,s   $labl"
12260     } else {
12261       $$emit$$"jp,u,s   done\n\t"
12262       $$emit$$"j$cop,u,s  $labl\n\t"
12263       $$emit$$"done:"
12264     }
12265   %}
12266   size(4);
12267   ins_encode %{
12268     Label* l = $labl$$label;
12269     if ($cop$$cmpcode == Assembler::notEqual) {
12270       __ jccb(Assembler::parity, *l);
12271       __ jccb(Assembler::notEqual, *l);
12272     } else if ($cop$$cmpcode == Assembler::equal) {
12273       Label done;
12274       __ jccb(Assembler::parity, done);
12275       __ jccb(Assembler::equal, *l);
12276       __ bind(done);
12277     } else {
12278        ShouldNotReachHere();
12279     }
12280   %}
12281   ins_pipe(pipe_jcc);
12282   ins_short_branch(1);
12283 %}
12284 
12285 // ============================================================================
12286 // inlined locking and unlocking
12287 
12288 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12289   predicate(Compile::current()->use_rtm());
12290   match(Set cr (FastLock object box));
12291   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12292   ins_cost(300);
12293   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12294   ins_encode %{
12295     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12296                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12297                  _counters, _rtm_counters, _stack_rtm_counters,
12298                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12299                  true, ra_->C->profile_rtm());
12300   %}
12301   ins_pipe(pipe_slow);
12302 %}
12303 
12304 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12305   predicate(!Compile::current()->use_rtm());
12306   match(Set cr (FastLock object box));
12307   effect(TEMP tmp, TEMP scr, USE_KILL box);
12308   ins_cost(300);
12309   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12310   ins_encode %{
12311     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12312                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12313   %}
12314   ins_pipe(pipe_slow);
12315 %}
12316 
12317 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12318   match(Set cr (FastUnlock object box));
12319   effect(TEMP tmp, USE_KILL box);
12320   ins_cost(300);
12321   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12322   ins_encode %{
12323     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12324   %}
12325   ins_pipe(pipe_slow);
12326 %}
12327 
12328 
12329 // ============================================================================
12330 // Safepoint Instructions
12331 instruct safePoint_poll(rFlagsReg cr)
12332 %{
12333   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12334   match(SafePoint);
12335   effect(KILL cr);
12336 
12337   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12338             "# Safepoint: poll for GC" %}
12339   ins_cost(125);
12340   ins_encode %{
12341     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12342     __ testl(rax, addr);
12343   %}
12344   ins_pipe(ialu_reg_mem);
12345 %}
12346 
12347 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12348 %{
12349   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12350   match(SafePoint poll);
12351   effect(KILL cr, USE poll);
12352 
12353   format %{ "testl  rax, [$poll]\t"
12354             "# Safepoint: poll for GC" %}
12355   ins_cost(125);
12356   ins_encode %{
12357     __ relocate(relocInfo::poll_type);
12358     __ testl(rax, Address($poll$$Register, 0));
12359   %}
12360   ins_pipe(ialu_reg_mem);
12361 %}
12362 
12363 instruct safePoint_poll_tls(rFlagsReg cr, rex_RegP poll)
12364 %{
12365   predicate(SafepointMechanism::uses_thread_local_poll());
12366   match(SafePoint poll);
12367   effect(KILL cr, USE poll);
12368 
12369   format %{ "testl  rax, [$poll]\t"
12370             "# Safepoint: poll for GC" %}
12371   ins_cost(125);
12372   size(3); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12373   ins_encode %{
12374     __ relocate(relocInfo::poll_type);
12375     address pre_pc = __ pc();
12376     __ testl(rax, Address($poll$$Register, 0));
12377     address post_pc = __ pc();
12378     guarantee(pre_pc[0] == 0x41 && pre_pc[1] == 0x85, "must emit #rex test-ax [reg]");
12379   %}
12380   ins_pipe(ialu_reg_mem);
12381 %}
12382 
12383 // ============================================================================
12384 // Procedure Call/Return Instructions
12385 // Call Java Static Instruction
12386 // Note: If this code changes, the corresponding ret_addr_offset() and
12387 //       compute_padding() functions will have to be adjusted.
12388 instruct CallStaticJavaDirect(method meth) %{
12389   match(CallStaticJava);
12390   effect(USE meth);
12391 
12392   ins_cost(300);
12393   format %{ "call,static " %}
12394   opcode(0xE8); /* E8 cd */
12395   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12396   ins_pipe(pipe_slow);
12397   ins_alignment(4);
12398 %}
12399 
12400 // Call Java Dynamic Instruction
12401 // Note: If this code changes, the corresponding ret_addr_offset() and
12402 //       compute_padding() functions will have to be adjusted.
12403 instruct CallDynamicJavaDirect(method meth)
12404 %{
12405   match(CallDynamicJava);
12406   effect(USE meth);
12407 
12408   ins_cost(300);
12409   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12410             "call,dynamic " %}
12411   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12412   ins_pipe(pipe_slow);
12413   ins_alignment(4);
12414 %}
12415 
12416 // Call Runtime Instruction
12417 instruct CallRuntimeDirect(method meth)
12418 %{
12419   match(CallRuntime);
12420   effect(USE meth);
12421 
12422   ins_cost(300);
12423   format %{ "call,runtime " %}
12424   ins_encode(clear_avx, Java_To_Runtime(meth));
12425   ins_pipe(pipe_slow);
12426 %}
12427 
12428 // Call runtime without safepoint
12429 instruct CallLeafDirect(method meth)
12430 %{
12431   match(CallLeaf);
12432   effect(USE meth);
12433 
12434   ins_cost(300);
12435   format %{ "call_leaf,runtime " %}
12436   ins_encode(clear_avx, Java_To_Runtime(meth));
12437   ins_pipe(pipe_slow);
12438 %}
12439 
12440 // Call runtime without safepoint
12441 instruct CallLeafNoFPDirect(method meth)
12442 %{
12443   match(CallLeafNoFP);
12444   effect(USE meth);
12445 
12446   ins_cost(300);
12447   format %{ "call_leaf_nofp,runtime " %}
12448   ins_encode(clear_avx, Java_To_Runtime(meth));
12449   ins_pipe(pipe_slow);
12450 %}
12451 
12452 // Return Instruction
12453 // Remove the return address & jump to it.
12454 // Notice: We always emit a nop after a ret to make sure there is room
12455 // for safepoint patching
12456 instruct Ret()
12457 %{
12458   match(Return);
12459 
12460   format %{ "ret" %}
12461   opcode(0xC3);
12462   ins_encode(OpcP);
12463   ins_pipe(pipe_jmp);
12464 %}
12465 
12466 // Tail Call; Jump from runtime stub to Java code.
12467 // Also known as an 'interprocedural jump'.
12468 // Target of jump will eventually return to caller.
12469 // TailJump below removes the return address.
12470 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12471 %{
12472   match(TailCall jump_target method_oop);
12473 
12474   ins_cost(300);
12475   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12476   opcode(0xFF, 0x4); /* Opcode FF /4 */
12477   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12478   ins_pipe(pipe_jmp);
12479 %}
12480 
12481 // Tail Jump; remove the return address; jump to target.
12482 // TailCall above leaves the return address around.
12483 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12484 %{
12485   match(TailJump jump_target ex_oop);
12486 
12487   ins_cost(300);
12488   format %{ "popq    rdx\t# pop return address\n\t"
12489             "jmp     $jump_target" %}
12490   opcode(0xFF, 0x4); /* Opcode FF /4 */
12491   ins_encode(Opcode(0x5a), // popq rdx
12492              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12493   ins_pipe(pipe_jmp);
12494 %}
12495 
12496 // Create exception oop: created by stack-crawling runtime code.
12497 // Created exception is now available to this handler, and is setup
12498 // just prior to jumping to this handler.  No code emitted.
12499 instruct CreateException(rax_RegP ex_oop)
12500 %{
12501   match(Set ex_oop (CreateEx));
12502 
12503   size(0);
12504   // use the following format syntax
12505   format %{ "# exception oop is in rax; no code emitted" %}
12506   ins_encode();
12507   ins_pipe(empty);
12508 %}
12509 
12510 // Rethrow exception:
12511 // The exception oop will come in the first argument position.
12512 // Then JUMP (not call) to the rethrow stub code.
12513 instruct RethrowException()
12514 %{
12515   match(Rethrow);
12516 
12517   // use the following format syntax
12518   format %{ "jmp     rethrow_stub" %}
12519   ins_encode(enc_rethrow);
12520   ins_pipe(pipe_jmp);
12521 %}
12522 
12523 //
12524 // Execute ZGC load barrier (strong) slow path
12525 //
12526 
12527 // When running without XMM regs
12528 instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12529 
12530   match(Set dst (LoadBarrierSlowReg mem));
12531   predicate(MaxVectorSize < 16);
12532 
12533   effect(DEF dst, KILL cr);
12534 
12535   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12536   ins_encode %{
12537 #if INCLUDE_ZGC
12538     Register d = $dst$$Register;
12539     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12540 
12541     assert(d != r12, "Can't be R12!");
12542     assert(d != r15, "Can't be R15!");
12543     assert(d != rsp, "Can't be RSP!");
12544 
12545     __ lea(d, $mem$$Address);
12546     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12547 #else
12548     ShouldNotReachHere();
12549 #endif
12550   %}
12551   ins_pipe(pipe_slow);
12552 %}
12553 
12554 // For XMM and YMM enabled processors
12555 instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12556                                      rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12557                                      rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12558                                      rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12559                                      rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12560 
12561   match(Set dst (LoadBarrierSlowReg mem));
12562   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12563 
12564   effect(DEF dst, KILL cr,
12565          KILL x0, KILL x1, KILL x2, KILL x3,
12566          KILL x4, KILL x5, KILL x6, KILL x7,
12567          KILL x8, KILL x9, KILL x10, KILL x11,
12568          KILL x12, KILL x13, KILL x14, KILL x15);
12569 
12570   format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
12571   ins_encode %{
12572 #if INCLUDE_ZGC
12573     Register d = $dst$$Register;
12574     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12575 
12576     assert(d != r12, "Can't be R12!");
12577     assert(d != r15, "Can't be R15!");
12578     assert(d != rsp, "Can't be RSP!");
12579 
12580     __ lea(d, $mem$$Address);
12581     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12582 #else
12583     ShouldNotReachHere();
12584 #endif
12585   %}
12586   ins_pipe(pipe_slow);
12587 %}
12588 
12589 // For ZMM enabled processors
12590 instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12591                                rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12592                                rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12593                                rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12594                                rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12595                                rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12596                                rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12597                                rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12598                                rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12599 
12600   match(Set dst (LoadBarrierSlowReg mem));
12601   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12602 
12603   effect(DEF dst, KILL cr,
12604          KILL x0, KILL x1, KILL x2, KILL x3,
12605          KILL x4, KILL x5, KILL x6, KILL x7,
12606          KILL x8, KILL x9, KILL x10, KILL x11,
12607          KILL x12, KILL x13, KILL x14, KILL x15,
12608          KILL x16, KILL x17, KILL x18, KILL x19,
12609          KILL x20, KILL x21, KILL x22, KILL x23,
12610          KILL x24, KILL x25, KILL x26, KILL x27,
12611          KILL x28, KILL x29, KILL x30, KILL x31);
12612 
12613   format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
12614   ins_encode %{
12615 #if INCLUDE_ZGC
12616     Register d = $dst$$Register;
12617     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12618 
12619     assert(d != r12, "Can't be R12!");
12620     assert(d != r15, "Can't be R15!");
12621     assert(d != rsp, "Can't be RSP!");
12622 
12623     __ lea(d, $mem$$Address);
12624     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12625 #else
12626     ShouldNotReachHere();
12627 #endif
12628   %}
12629   ins_pipe(pipe_slow);
12630 %}
12631 
12632 //
12633 // Execute ZGC load barrier (weak) slow path
12634 //
12635 
12636 // When running without XMM regs
12637 instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12638 
12639   match(Set dst (LoadBarrierSlowReg mem));
12640   predicate(MaxVectorSize < 16);
12641 
12642   effect(DEF dst, KILL cr);
12643 
12644   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12645   ins_encode %{
12646 #if INCLUDE_ZGC
12647     Register d = $dst$$Register;
12648     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12649 
12650     assert(d != r12, "Can't be R12!");
12651     assert(d != r15, "Can't be R15!");
12652     assert(d != rsp, "Can't be RSP!");
12653 
12654     __ lea(d, $mem$$Address);
12655     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12656 #else
12657     ShouldNotReachHere();
12658 #endif
12659   %}
12660   ins_pipe(pipe_slow);
12661 %}
12662 
12663 // For XMM and YMM enabled processors
12664 instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12665                                          rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12666                                          rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12667                                          rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12668                                          rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12669 
12670   match(Set dst (LoadBarrierWeakSlowReg mem));
12671   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12672 
12673   effect(DEF dst, KILL cr,
12674          KILL x0, KILL x1, KILL x2, KILL x3,
12675          KILL x4, KILL x5, KILL x6, KILL x7,
12676          KILL x8, KILL x9, KILL x10, KILL x11,
12677          KILL x12, KILL x13, KILL x14, KILL x15);
12678 
12679   format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
12680   ins_encode %{
12681 #if INCLUDE_ZGC
12682     Register d = $dst$$Register;
12683     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12684 
12685     assert(d != r12, "Can't be R12!");
12686     assert(d != r15, "Can't be R15!");
12687     assert(d != rsp, "Can't be RSP!");
12688 
12689     __ lea(d,$mem$$Address);
12690     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12691 #else
12692     ShouldNotReachHere();
12693 #endif
12694   %}
12695   ins_pipe(pipe_slow);
12696 %}
12697 
12698 // For ZMM enabled processors
12699 instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12700                                    rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12701                                    rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12702                                    rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12703                                    rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12704                                    rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12705                                    rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12706                                    rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12707                                    rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12708 
12709   match(Set dst (LoadBarrierWeakSlowReg mem));
12710   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12711 
12712   effect(DEF dst, KILL cr,
12713          KILL x0, KILL x1, KILL x2, KILL x3,
12714          KILL x4, KILL x5, KILL x6, KILL x7,
12715          KILL x8, KILL x9, KILL x10, KILL x11,
12716          KILL x12, KILL x13, KILL x14, KILL x15,
12717          KILL x16, KILL x17, KILL x18, KILL x19,
12718          KILL x20, KILL x21, KILL x22, KILL x23,
12719          KILL x24, KILL x25, KILL x26, KILL x27,
12720          KILL x28, KILL x29, KILL x30, KILL x31);
12721 
12722   format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
12723   ins_encode %{
12724 #if INCLUDE_ZGC
12725     Register d = $dst$$Register;
12726     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12727 
12728     assert(d != r12, "Can't be R12!");
12729     assert(d != r15, "Can't be R15!");
12730     assert(d != rsp, "Can't be RSP!");
12731 
12732     __ lea(d,$mem$$Address);
12733     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12734 #else
12735     ShouldNotReachHere();
12736 #endif
12737   %}
12738   ins_pipe(pipe_slow);
12739 %}
12740 
12741 // ============================================================================
12742 // This name is KNOWN by the ADLC and cannot be changed.
12743 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12744 // for this guy.
12745 instruct tlsLoadP(r15_RegP dst) %{
12746   match(Set dst (ThreadLocal));
12747   effect(DEF dst);
12748 
12749   size(0);
12750   format %{ "# TLS is in R15" %}
12751   ins_encode( /*empty encoding*/ );
12752   ins_pipe(ialu_reg_reg);
12753 %}
12754 
12755 
12756 //----------PEEPHOLE RULES-----------------------------------------------------
12757 // These must follow all instruction definitions as they use the names
12758 // defined in the instructions definitions.
12759 //
12760 // peepmatch ( root_instr_name [preceding_instruction]* );
12761 //
12762 // peepconstraint %{
12763 // (instruction_number.operand_name relational_op instruction_number.operand_name
12764 //  [, ...] );
12765 // // instruction numbers are zero-based using left to right order in peepmatch
12766 //
12767 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12768 // // provide an instruction_number.operand_name for each operand that appears
12769 // // in the replacement instruction's match rule
12770 //
12771 // ---------VM FLAGS---------------------------------------------------------
12772 //
12773 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12774 //
12775 // Each peephole rule is given an identifying number starting with zero and
12776 // increasing by one in the order seen by the parser.  An individual peephole
12777 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12778 // on the command-line.
12779 //
12780 // ---------CURRENT LIMITATIONS----------------------------------------------
12781 //
12782 // Only match adjacent instructions in same basic block
12783 // Only equality constraints
12784 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12785 // Only one replacement instruction
12786 //
12787 // ---------EXAMPLE----------------------------------------------------------
12788 //
12789 // // pertinent parts of existing instructions in architecture description
12790 // instruct movI(rRegI dst, rRegI src)
12791 // %{
12792 //   match(Set dst (CopyI src));
12793 // %}
12794 //
12795 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12796 // %{
12797 //   match(Set dst (AddI dst src));
12798 //   effect(KILL cr);
12799 // %}
12800 //
12801 // // Change (inc mov) to lea
12802 // peephole %{
12803 //   // increment preceeded by register-register move
12804 //   peepmatch ( incI_rReg movI );
12805 //   // require that the destination register of the increment
12806 //   // match the destination register of the move
12807 //   peepconstraint ( 0.dst == 1.dst );
12808 //   // construct a replacement instruction that sets
12809 //   // the destination to ( move's source register + one )
12810 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12811 // %}
12812 //
12813 
12814 // Implementation no longer uses movX instructions since
12815 // machine-independent system no longer uses CopyX nodes.
12816 //
12817 // peephole
12818 // %{
12819 //   peepmatch (incI_rReg movI);
12820 //   peepconstraint (0.dst == 1.dst);
12821 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12822 // %}
12823 
12824 // peephole
12825 // %{
12826 //   peepmatch (decI_rReg movI);
12827 //   peepconstraint (0.dst == 1.dst);
12828 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12829 // %}
12830 
12831 // peephole
12832 // %{
12833 //   peepmatch (addI_rReg_imm movI);
12834 //   peepconstraint (0.dst == 1.dst);
12835 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12836 // %}
12837 
12838 // peephole
12839 // %{
12840 //   peepmatch (incL_rReg movL);
12841 //   peepconstraint (0.dst == 1.dst);
12842 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12843 // %}
12844 
12845 // peephole
12846 // %{
12847 //   peepmatch (decL_rReg movL);
12848 //   peepconstraint (0.dst == 1.dst);
12849 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12850 // %}
12851 
12852 // peephole
12853 // %{
12854 //   peepmatch (addL_rReg_imm movL);
12855 //   peepconstraint (0.dst == 1.dst);
12856 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12857 // %}
12858 
12859 // peephole
12860 // %{
12861 //   peepmatch (addP_rReg_imm movP);
12862 //   peepconstraint (0.dst == 1.dst);
12863 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12864 // %}
12865 
12866 // // Change load of spilled value to only a spill
12867 // instruct storeI(memory mem, rRegI src)
12868 // %{
12869 //   match(Set mem (StoreI mem src));
12870 // %}
12871 //
12872 // instruct loadI(rRegI dst, memory mem)
12873 // %{
12874 //   match(Set dst (LoadI mem));
12875 // %}
12876 //
12877 
12878 peephole
12879 %{
12880   peepmatch (loadI storeI);
12881   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12882   peepreplace (storeI(1.mem 1.mem 1.src));
12883 %}
12884 
12885 peephole
12886 %{
12887   peepmatch (loadL storeL);
12888   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12889   peepreplace (storeL(1.mem 1.mem 1.src));
12890 %}
12891 
12892 //----------SMARTSPILL RULES---------------------------------------------------
12893 // These must follow all instruction definitions as they use the names
12894 // defined in the instructions definitions.
--- EOF ---