1 //
   2 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 source_hpp %{
 530 #if INCLUDE_ZGC
 531 #include "gc/z/zBarrierSetAssembler.hpp"
 532 #endif
 533 %}
 534 
 535 //----------SOURCE BLOCK-------------------------------------------------------
 536 // This is a block of C++ code which provides values, functions, and
 537 // definitions necessary in the rest of the architecture description
 538 source %{
 539 #define   RELOC_IMM64    Assembler::imm_operand
 540 #define   RELOC_DISP32   Assembler::disp32_operand
 541 
 542 #define __ _masm.
 543 
 544 static bool generate_vzeroupper(Compile* C) {
 545   return (VM_Version::supports_vzeroupper() && (C->max_vector_size() > 16 || C->clear_upper_avx() == true)) ? true: false;  // Generate vzeroupper
 546 }
 547 
 548 static int clear_avx_size() {
 549   return generate_vzeroupper(Compile::current()) ? 3: 0;  // vzeroupper
 550 }
 551 
 552 // !!!!! Special hack to get all types of calls to specify the byte offset
 553 //       from the start of the call to the point where the return address
 554 //       will point.
 555 int MachCallStaticJavaNode::ret_addr_offset()
 556 {
 557   int offset = 5; // 5 bytes from start of call to where return address points
 558   offset += clear_avx_size();
 559   return offset;
 560 }
 561 
 562 int MachCallDynamicJavaNode::ret_addr_offset()
 563 {
 564   int offset = 15; // 15 bytes from start of call to where return address points
 565   offset += clear_avx_size();
 566   return offset;
 567 }
 568 
 569 int MachCallRuntimeNode::ret_addr_offset() {
 570   int offset = 13; // movq r10,#addr; callq (r10)
 571   offset += clear_avx_size();
 572   return offset;
 573 }
 574 
 575 // Indicate if the safepoint node needs the polling page as an input,
 576 // it does if the polling page is more than disp32 away.
 577 bool SafePointNode::needs_polling_address_input()
 578 {
 579   return SafepointMechanism::uses_thread_local_poll() || Assembler::is_polling_page_far();
 580 }
 581 
 582 //
 583 // Compute padding required for nodes which need alignment
 584 //
 585 
 586 // The address of the call instruction needs to be 4-byte aligned to
 587 // ensure that it does not span a cache line so that it can be patched.
 588 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 589 {
 590   current_offset += clear_avx_size(); // skip vzeroupper
 591   current_offset += 1; // skip call opcode byte
 592   return align_up(current_offset, alignment_required()) - current_offset;
 593 }
 594 
 595 // The address of the call instruction needs to be 4-byte aligned to
 596 // ensure that it does not span a cache line so that it can be patched.
 597 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 598 {
 599   current_offset += clear_avx_size(); // skip vzeroupper
 600   current_offset += 11; // skip movq instruction + call opcode byte
 601   return align_up(current_offset, alignment_required()) - current_offset;
 602 }
 603 
 604 // EMIT_RM()
 605 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 606   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 607   cbuf.insts()->emit_int8(c);
 608 }
 609 
 610 // EMIT_CC()
 611 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 612   unsigned char c = (unsigned char) (f1 | f2);
 613   cbuf.insts()->emit_int8(c);
 614 }
 615 
 616 // EMIT_OPCODE()
 617 void emit_opcode(CodeBuffer &cbuf, int code) {
 618   cbuf.insts()->emit_int8((unsigned char) code);
 619 }
 620 
 621 // EMIT_OPCODE() w/ relocation information
 622 void emit_opcode(CodeBuffer &cbuf,
 623                  int code, relocInfo::relocType reloc, int offset, int format)
 624 {
 625   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 626   emit_opcode(cbuf, code);
 627 }
 628 
 629 // EMIT_D8()
 630 void emit_d8(CodeBuffer &cbuf, int d8) {
 631   cbuf.insts()->emit_int8((unsigned char) d8);
 632 }
 633 
 634 // EMIT_D16()
 635 void emit_d16(CodeBuffer &cbuf, int d16) {
 636   cbuf.insts()->emit_int16(d16);
 637 }
 638 
 639 // EMIT_D32()
 640 void emit_d32(CodeBuffer &cbuf, int d32) {
 641   cbuf.insts()->emit_int32(d32);
 642 }
 643 
 644 // EMIT_D64()
 645 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 646   cbuf.insts()->emit_int64(d64);
 647 }
 648 
 649 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 650 void emit_d32_reloc(CodeBuffer& cbuf,
 651                     int d32,
 652                     relocInfo::relocType reloc,
 653                     int format)
 654 {
 655   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 656   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 657   cbuf.insts()->emit_int32(d32);
 658 }
 659 
 660 // emit 32 bit value and construct relocation entry from RelocationHolder
 661 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 662 #ifdef ASSERT
 663   if (rspec.reloc()->type() == relocInfo::oop_type &&
 664       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 665     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 666     assert(oopDesc::is_oop(cast_to_oop((intptr_t)d32)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop((intptr_t)d32))), "cannot embed scavengable oops in code");
 667   }
 668 #endif
 669   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 670   cbuf.insts()->emit_int32(d32);
 671 }
 672 
 673 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 674   address next_ip = cbuf.insts_end() + 4;
 675   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 676                  external_word_Relocation::spec(addr),
 677                  RELOC_DISP32);
 678 }
 679 
 680 
 681 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 682 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 683   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 684   cbuf.insts()->emit_int64(d64);
 685 }
 686 
 687 // emit 64 bit value and construct relocation entry from RelocationHolder
 688 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 689 #ifdef ASSERT
 690   if (rspec.reloc()->type() == relocInfo::oop_type &&
 691       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 692     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 693     assert(oopDesc::is_oop(cast_to_oop(d64)) && (ScavengeRootsInCode || !Universe::heap()->is_scavengable(cast_to_oop(d64))),
 694            "cannot embed scavengable oops in code");
 695   }
 696 #endif
 697   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 698   cbuf.insts()->emit_int64(d64);
 699 }
 700 
 701 // Access stack slot for load or store
 702 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 703 {
 704   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 705   if (-0x80 <= disp && disp < 0x80) {
 706     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 707     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 708     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 709   } else {
 710     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 711     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 712     emit_d32(cbuf, disp);     // Displacement // R/M byte
 713   }
 714 }
 715 
 716    // rRegI ereg, memory mem) %{    // emit_reg_mem
 717 void encode_RegMem(CodeBuffer &cbuf,
 718                    int reg,
 719                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 720 {
 721   assert(disp_reloc == relocInfo::none, "cannot have disp");
 722   int regenc = reg & 7;
 723   int baseenc = base & 7;
 724   int indexenc = index & 7;
 725 
 726   // There is no index & no scale, use form without SIB byte
 727   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 728     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 729     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 730       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 731     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 732       // If 8-bit displacement, mode 0x1
 733       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 734       emit_d8(cbuf, disp);
 735     } else {
 736       // If 32-bit displacement
 737       if (base == -1) { // Special flag for absolute address
 738         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 739         if (disp_reloc != relocInfo::none) {
 740           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 741         } else {
 742           emit_d32(cbuf, disp);
 743         }
 744       } else {
 745         // Normal base + offset
 746         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 747         if (disp_reloc != relocInfo::none) {
 748           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 749         } else {
 750           emit_d32(cbuf, disp);
 751         }
 752       }
 753     }
 754   } else {
 755     // Else, encode with the SIB byte
 756     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 757     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 758       // If no displacement
 759       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 760       emit_rm(cbuf, scale, indexenc, baseenc);
 761     } else {
 762       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 763         // If 8-bit displacement, mode 0x1
 764         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 765         emit_rm(cbuf, scale, indexenc, baseenc);
 766         emit_d8(cbuf, disp);
 767       } else {
 768         // If 32-bit displacement
 769         if (base == 0x04 ) {
 770           emit_rm(cbuf, 0x2, regenc, 0x4);
 771           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 772         } else {
 773           emit_rm(cbuf, 0x2, regenc, 0x4);
 774           emit_rm(cbuf, scale, indexenc, baseenc); // *
 775         }
 776         if (disp_reloc != relocInfo::none) {
 777           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 778         } else {
 779           emit_d32(cbuf, disp);
 780         }
 781       }
 782     }
 783   }
 784 }
 785 
 786 // This could be in MacroAssembler but it's fairly C2 specific
 787 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 788   Label exit;
 789   __ jccb(Assembler::noParity, exit);
 790   __ pushf();
 791   //
 792   // comiss/ucomiss instructions set ZF,PF,CF flags and
 793   // zero OF,AF,SF for NaN values.
 794   // Fixup flags by zeroing ZF,PF so that compare of NaN
 795   // values returns 'less than' result (CF is set).
 796   // Leave the rest of flags unchanged.
 797   //
 798   //    7 6 5 4 3 2 1 0
 799   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 800   //    0 0 1 0 1 0 1 1   (0x2B)
 801   //
 802   __ andq(Address(rsp, 0), 0xffffff2b);
 803   __ popf();
 804   __ bind(exit);
 805 }
 806 
 807 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 808   Label done;
 809   __ movl(dst, -1);
 810   __ jcc(Assembler::parity, done);
 811   __ jcc(Assembler::below, done);
 812   __ setb(Assembler::notEqual, dst);
 813   __ movzbl(dst, dst);
 814   __ bind(done);
 815 }
 816 
 817 
 818 //=============================================================================
 819 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 820 
 821 int Compile::ConstantTable::calculate_table_base_offset() const {
 822   return 0;  // absolute addressing, no offset
 823 }
 824 
 825 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 826 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 827   ShouldNotReachHere();
 828 }
 829 
 830 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 831   // Empty encoding
 832 }
 833 
 834 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 835   return 0;
 836 }
 837 
 838 #ifndef PRODUCT
 839 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 840   st->print("# MachConstantBaseNode (empty encoding)");
 841 }
 842 #endif
 843 
 844 
 845 //=============================================================================
 846 #ifndef PRODUCT
 847 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 848   Compile* C = ra_->C;
 849 
 850   int framesize = C->frame_size_in_bytes();
 851   int bangsize = C->bang_size_in_bytes();
 852   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 853   // Remove wordSize for return addr which is already pushed.
 854   framesize -= wordSize;
 855 
 856   if (C->need_stack_bang(bangsize)) {
 857     framesize -= wordSize;
 858     st->print("# stack bang (%d bytes)", bangsize);
 859     st->print("\n\t");
 860     st->print("pushq   rbp\t# Save rbp");
 861     if (PreserveFramePointer) {
 862         st->print("\n\t");
 863         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 864     }
 865     if (framesize) {
 866       st->print("\n\t");
 867       st->print("subq    rsp, #%d\t# Create frame",framesize);
 868     }
 869   } else {
 870     st->print("subq    rsp, #%d\t# Create frame",framesize);
 871     st->print("\n\t");
 872     framesize -= wordSize;
 873     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 874     if (PreserveFramePointer) {
 875       st->print("\n\t");
 876       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 877       if (framesize > 0) {
 878         st->print("\n\t");
 879         st->print("addq    rbp, #%d", framesize);
 880       }
 881     }
 882   }
 883 
 884   if (VerifyStackAtCalls) {
 885     st->print("\n\t");
 886     framesize -= wordSize;
 887     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 888 #ifdef ASSERT
 889     st->print("\n\t");
 890     st->print("# stack alignment check");
 891 #endif
 892   }
 893   if (C->stub_function() != NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
 894     st->print("\n\t");
 895     st->print("cmpl    [r15_thread + #disarmed_offset], #disarmed_value\t");
 896     st->print("\n\t");
 897     st->print("je      fast_entry\t");
 898     st->print("\n\t");
 899     st->print("call    #nmethod_entry_barrier_stub\t");
 900     st->print("\n\tfast_entry:");
 901   }
 902   st->cr();
 903 }
 904 #endif
 905 
 906 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 907   Compile* C = ra_->C;
 908   MacroAssembler _masm(&cbuf);
 909 
 910   int framesize = C->frame_size_in_bytes();
 911   int bangsize = C->bang_size_in_bytes();
 912 
 913   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false, C->stub_function() != NULL);
 914 
 915   C->set_frame_complete(cbuf.insts_size());
 916 
 917   if (C->has_mach_constant_base_node()) {
 918     // NOTE: We set the table base offset here because users might be
 919     // emitted before MachConstantBaseNode.
 920     Compile::ConstantTable& constant_table = C->constant_table();
 921     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 922   }
 923 }
 924 
 925 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 926 {
 927   return MachNode::size(ra_); // too many variables; just compute it
 928                               // the hard way
 929 }
 930 
 931 int MachPrologNode::reloc() const
 932 {
 933   return 0; // a large enough number
 934 }
 935 
 936 //=============================================================================
 937 #ifndef PRODUCT
 938 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 939 {
 940   Compile* C = ra_->C;
 941   if (generate_vzeroupper(C)) {
 942     st->print("vzeroupper");
 943     st->cr(); st->print("\t");
 944   }
 945 
 946   int framesize = C->frame_size_in_bytes();
 947   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 948   // Remove word for return adr already pushed
 949   // and RBP
 950   framesize -= 2*wordSize;
 951 
 952   if (framesize) {
 953     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 954     st->print("\t");
 955   }
 956 
 957   st->print_cr("popq   rbp");
 958   if (do_polling() && C->is_method_compilation()) {
 959     st->print("\t");
 960     if (SafepointMechanism::uses_thread_local_poll()) {
 961       st->print_cr("movq   rscratch1, poll_offset[r15_thread] #polling_page_address\n\t"
 962                    "testl  rax, [rscratch1]\t"
 963                    "# Safepoint: poll for GC");
 964     } else if (Assembler::is_polling_page_far()) {
 965       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 966                    "testl  rax, [rscratch1]\t"
 967                    "# Safepoint: poll for GC");
 968     } else {
 969       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 970                    "# Safepoint: poll for GC");
 971     }
 972   }
 973 }
 974 #endif
 975 
 976 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 977 {
 978   Compile* C = ra_->C;
 979   MacroAssembler _masm(&cbuf);
 980 
 981   if (generate_vzeroupper(C)) {
 982     // Clear upper bits of YMM registers when current compiled code uses
 983     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 984     __ vzeroupper();
 985   }
 986 
 987   int framesize = C->frame_size_in_bytes();
 988   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 989   // Remove word for return adr already pushed
 990   // and RBP
 991   framesize -= 2*wordSize;
 992 
 993   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 994 
 995   if (framesize) {
 996     emit_opcode(cbuf, Assembler::REX_W);
 997     if (framesize < 0x80) {
 998       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 999       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1000       emit_d8(cbuf, framesize);
1001     } else {
1002       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
1003       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
1004       emit_d32(cbuf, framesize);
1005     }
1006   }
1007 
1008   // popq rbp
1009   emit_opcode(cbuf, 0x58 | RBP_enc);
1010 
1011   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1012     __ reserved_stack_check();
1013   }
1014 
1015   if (do_polling() && C->is_method_compilation()) {
1016     MacroAssembler _masm(&cbuf);
1017     if (SafepointMechanism::uses_thread_local_poll()) {
1018       __ movq(rscratch1, Address(r15_thread, Thread::polling_page_offset()));
1019       __ relocate(relocInfo::poll_return_type);
1020       __ testl(rax, Address(rscratch1, 0));
1021     } else {
1022       AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
1023       if (Assembler::is_polling_page_far()) {
1024         __ lea(rscratch1, polling_page);
1025         __ relocate(relocInfo::poll_return_type);
1026         __ testl(rax, Address(rscratch1, 0));
1027       } else {
1028         __ testl(rax, polling_page);
1029       }
1030     }
1031   }
1032 }
1033 
1034 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1035 {
1036   return MachNode::size(ra_); // too many variables; just compute it
1037                               // the hard way
1038 }
1039 
1040 int MachEpilogNode::reloc() const
1041 {
1042   return 2; // a large enough number
1043 }
1044 
1045 const Pipeline* MachEpilogNode::pipeline() const
1046 {
1047   return MachNode::pipeline_class();
1048 }
1049 
1050 int MachEpilogNode::safepoint_offset() const
1051 {
1052   return 0;
1053 }
1054 
1055 //=============================================================================
1056 
1057 enum RC {
1058   rc_bad,
1059   rc_int,
1060   rc_float,
1061   rc_stack
1062 };
1063 
1064 static enum RC rc_class(OptoReg::Name reg)
1065 {
1066   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1067 
1068   if (OptoReg::is_stack(reg)) return rc_stack;
1069 
1070   VMReg r = OptoReg::as_VMReg(reg);
1071 
1072   if (r->is_Register()) return rc_int;
1073 
1074   assert(r->is_XMMRegister(), "must be");
1075   return rc_float;
1076 }
1077 
1078 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1079 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1080                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1081 
1082 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1083                             int stack_offset, int reg, uint ireg, outputStream* st);
1084 
1085 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1086                                       int dst_offset, uint ireg, outputStream* st) {
1087   if (cbuf) {
1088     MacroAssembler _masm(cbuf);
1089     switch (ireg) {
1090     case Op_VecS:
1091       __ movq(Address(rsp, -8), rax);
1092       __ movl(rax, Address(rsp, src_offset));
1093       __ movl(Address(rsp, dst_offset), rax);
1094       __ movq(rax, Address(rsp, -8));
1095       break;
1096     case Op_VecD:
1097       __ pushq(Address(rsp, src_offset));
1098       __ popq (Address(rsp, dst_offset));
1099       break;
1100     case Op_VecX:
1101       __ pushq(Address(rsp, src_offset));
1102       __ popq (Address(rsp, dst_offset));
1103       __ pushq(Address(rsp, src_offset+8));
1104       __ popq (Address(rsp, dst_offset+8));
1105       break;
1106     case Op_VecY:
1107       __ vmovdqu(Address(rsp, -32), xmm0);
1108       __ vmovdqu(xmm0, Address(rsp, src_offset));
1109       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1110       __ vmovdqu(xmm0, Address(rsp, -32));
1111       break;
1112     case Op_VecZ:
1113       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1114       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1115       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1116       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1117       break;
1118     default:
1119       ShouldNotReachHere();
1120     }
1121 #ifndef PRODUCT
1122   } else {
1123     switch (ireg) {
1124     case Op_VecS:
1125       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1126                 "movl    rax, [rsp + #%d]\n\t"
1127                 "movl    [rsp + #%d], rax\n\t"
1128                 "movq    rax, [rsp - #8]",
1129                 src_offset, dst_offset);
1130       break;
1131     case Op_VecD:
1132       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1133                 "popq    [rsp + #%d]",
1134                 src_offset, dst_offset);
1135       break;
1136      case Op_VecX:
1137       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1138                 "popq    [rsp + #%d]\n\t"
1139                 "pushq   [rsp + #%d]\n\t"
1140                 "popq    [rsp + #%d]",
1141                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1142       break;
1143     case Op_VecY:
1144       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1145                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1146                 "vmovdqu [rsp + #%d], xmm0\n\t"
1147                 "vmovdqu xmm0, [rsp - #32]",
1148                 src_offset, dst_offset);
1149       break;
1150     case Op_VecZ:
1151       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1152                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1153                 "vmovdqu [rsp + #%d], xmm0\n\t"
1154                 "vmovdqu xmm0, [rsp - #64]",
1155                 src_offset, dst_offset);
1156       break;
1157     default:
1158       ShouldNotReachHere();
1159     }
1160 #endif
1161   }
1162 }
1163 
1164 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1165                                        PhaseRegAlloc* ra_,
1166                                        bool do_size,
1167                                        outputStream* st) const {
1168   assert(cbuf != NULL || st  != NULL, "sanity");
1169   // Get registers to move
1170   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1171   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1172   OptoReg::Name dst_second = ra_->get_reg_second(this);
1173   OptoReg::Name dst_first = ra_->get_reg_first(this);
1174 
1175   enum RC src_second_rc = rc_class(src_second);
1176   enum RC src_first_rc = rc_class(src_first);
1177   enum RC dst_second_rc = rc_class(dst_second);
1178   enum RC dst_first_rc = rc_class(dst_first);
1179 
1180   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1181          "must move at least 1 register" );
1182 
1183   if (src_first == dst_first && src_second == dst_second) {
1184     // Self copy, no move
1185     return 0;
1186   }
1187   if (bottom_type()->isa_vect() != NULL) {
1188     uint ireg = ideal_reg();
1189     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1190     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1191     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1192       // mem -> mem
1193       int src_offset = ra_->reg2offset(src_first);
1194       int dst_offset = ra_->reg2offset(dst_first);
1195       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1196     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1197       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1198     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1199       int stack_offset = ra_->reg2offset(dst_first);
1200       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1201     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1202       int stack_offset = ra_->reg2offset(src_first);
1203       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1204     } else {
1205       ShouldNotReachHere();
1206     }
1207     return 0;
1208   }
1209   if (src_first_rc == rc_stack) {
1210     // mem ->
1211     if (dst_first_rc == rc_stack) {
1212       // mem -> mem
1213       assert(src_second != dst_first, "overlap");
1214       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1215           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1216         // 64-bit
1217         int src_offset = ra_->reg2offset(src_first);
1218         int dst_offset = ra_->reg2offset(dst_first);
1219         if (cbuf) {
1220           MacroAssembler _masm(cbuf);
1221           __ pushq(Address(rsp, src_offset));
1222           __ popq (Address(rsp, dst_offset));
1223 #ifndef PRODUCT
1224         } else {
1225           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1226                     "popq    [rsp + #%d]",
1227                      src_offset, dst_offset);
1228 #endif
1229         }
1230       } else {
1231         // 32-bit
1232         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1233         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1234         // No pushl/popl, so:
1235         int src_offset = ra_->reg2offset(src_first);
1236         int dst_offset = ra_->reg2offset(dst_first);
1237         if (cbuf) {
1238           MacroAssembler _masm(cbuf);
1239           __ movq(Address(rsp, -8), rax);
1240           __ movl(rax, Address(rsp, src_offset));
1241           __ movl(Address(rsp, dst_offset), rax);
1242           __ movq(rax, Address(rsp, -8));
1243 #ifndef PRODUCT
1244         } else {
1245           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1246                     "movl    rax, [rsp + #%d]\n\t"
1247                     "movl    [rsp + #%d], rax\n\t"
1248                     "movq    rax, [rsp - #8]",
1249                      src_offset, dst_offset);
1250 #endif
1251         }
1252       }
1253       return 0;
1254     } else if (dst_first_rc == rc_int) {
1255       // mem -> gpr
1256       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1257           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1258         // 64-bit
1259         int offset = ra_->reg2offset(src_first);
1260         if (cbuf) {
1261           MacroAssembler _masm(cbuf);
1262           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1263 #ifndef PRODUCT
1264         } else {
1265           st->print("movq    %s, [rsp + #%d]\t# spill",
1266                      Matcher::regName[dst_first],
1267                      offset);
1268 #endif
1269         }
1270       } else {
1271         // 32-bit
1272         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1273         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1274         int offset = ra_->reg2offset(src_first);
1275         if (cbuf) {
1276           MacroAssembler _masm(cbuf);
1277           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1278 #ifndef PRODUCT
1279         } else {
1280           st->print("movl    %s, [rsp + #%d]\t# spill",
1281                      Matcher::regName[dst_first],
1282                      offset);
1283 #endif
1284         }
1285       }
1286       return 0;
1287     } else if (dst_first_rc == rc_float) {
1288       // mem-> xmm
1289       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1290           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1291         // 64-bit
1292         int offset = ra_->reg2offset(src_first);
1293         if (cbuf) {
1294           MacroAssembler _masm(cbuf);
1295           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1296 #ifndef PRODUCT
1297         } else {
1298           st->print("%s  %s, [rsp + #%d]\t# spill",
1299                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1300                      Matcher::regName[dst_first],
1301                      offset);
1302 #endif
1303         }
1304       } else {
1305         // 32-bit
1306         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1307         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1308         int offset = ra_->reg2offset(src_first);
1309         if (cbuf) {
1310           MacroAssembler _masm(cbuf);
1311           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1312 #ifndef PRODUCT
1313         } else {
1314           st->print("movss   %s, [rsp + #%d]\t# spill",
1315                      Matcher::regName[dst_first],
1316                      offset);
1317 #endif
1318         }
1319       }
1320       return 0;
1321     }
1322   } else if (src_first_rc == rc_int) {
1323     // gpr ->
1324     if (dst_first_rc == rc_stack) {
1325       // gpr -> mem
1326       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1327           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1328         // 64-bit
1329         int offset = ra_->reg2offset(dst_first);
1330         if (cbuf) {
1331           MacroAssembler _masm(cbuf);
1332           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1333 #ifndef PRODUCT
1334         } else {
1335           st->print("movq    [rsp + #%d], %s\t# spill",
1336                      offset,
1337                      Matcher::regName[src_first]);
1338 #endif
1339         }
1340       } else {
1341         // 32-bit
1342         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1343         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1344         int offset = ra_->reg2offset(dst_first);
1345         if (cbuf) {
1346           MacroAssembler _masm(cbuf);
1347           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1348 #ifndef PRODUCT
1349         } else {
1350           st->print("movl    [rsp + #%d], %s\t# spill",
1351                      offset,
1352                      Matcher::regName[src_first]);
1353 #endif
1354         }
1355       }
1356       return 0;
1357     } else if (dst_first_rc == rc_int) {
1358       // gpr -> gpr
1359       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1360           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1361         // 64-bit
1362         if (cbuf) {
1363           MacroAssembler _masm(cbuf);
1364           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1365                   as_Register(Matcher::_regEncode[src_first]));
1366 #ifndef PRODUCT
1367         } else {
1368           st->print("movq    %s, %s\t# spill",
1369                      Matcher::regName[dst_first],
1370                      Matcher::regName[src_first]);
1371 #endif
1372         }
1373         return 0;
1374       } else {
1375         // 32-bit
1376         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1377         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1378         if (cbuf) {
1379           MacroAssembler _masm(cbuf);
1380           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1381                   as_Register(Matcher::_regEncode[src_first]));
1382 #ifndef PRODUCT
1383         } else {
1384           st->print("movl    %s, %s\t# spill",
1385                      Matcher::regName[dst_first],
1386                      Matcher::regName[src_first]);
1387 #endif
1388         }
1389         return 0;
1390       }
1391     } else if (dst_first_rc == rc_float) {
1392       // gpr -> xmm
1393       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1394           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1395         // 64-bit
1396         if (cbuf) {
1397           MacroAssembler _masm(cbuf);
1398           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1399 #ifndef PRODUCT
1400         } else {
1401           st->print("movdq   %s, %s\t# spill",
1402                      Matcher::regName[dst_first],
1403                      Matcher::regName[src_first]);
1404 #endif
1405         }
1406       } else {
1407         // 32-bit
1408         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1409         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1410         if (cbuf) {
1411           MacroAssembler _masm(cbuf);
1412           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1413 #ifndef PRODUCT
1414         } else {
1415           st->print("movdl   %s, %s\t# spill",
1416                      Matcher::regName[dst_first],
1417                      Matcher::regName[src_first]);
1418 #endif
1419         }
1420       }
1421       return 0;
1422     }
1423   } else if (src_first_rc == rc_float) {
1424     // xmm ->
1425     if (dst_first_rc == rc_stack) {
1426       // xmm -> mem
1427       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1428           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1429         // 64-bit
1430         int offset = ra_->reg2offset(dst_first);
1431         if (cbuf) {
1432           MacroAssembler _masm(cbuf);
1433           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1434 #ifndef PRODUCT
1435         } else {
1436           st->print("movsd   [rsp + #%d], %s\t# spill",
1437                      offset,
1438                      Matcher::regName[src_first]);
1439 #endif
1440         }
1441       } else {
1442         // 32-bit
1443         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1444         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1445         int offset = ra_->reg2offset(dst_first);
1446         if (cbuf) {
1447           MacroAssembler _masm(cbuf);
1448           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1449 #ifndef PRODUCT
1450         } else {
1451           st->print("movss   [rsp + #%d], %s\t# spill",
1452                      offset,
1453                      Matcher::regName[src_first]);
1454 #endif
1455         }
1456       }
1457       return 0;
1458     } else if (dst_first_rc == rc_int) {
1459       // xmm -> gpr
1460       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1461           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1462         // 64-bit
1463         if (cbuf) {
1464           MacroAssembler _masm(cbuf);
1465           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1466 #ifndef PRODUCT
1467         } else {
1468           st->print("movdq   %s, %s\t# spill",
1469                      Matcher::regName[dst_first],
1470                      Matcher::regName[src_first]);
1471 #endif
1472         }
1473       } else {
1474         // 32-bit
1475         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1476         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1477         if (cbuf) {
1478           MacroAssembler _masm(cbuf);
1479           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1480 #ifndef PRODUCT
1481         } else {
1482           st->print("movdl   %s, %s\t# spill",
1483                      Matcher::regName[dst_first],
1484                      Matcher::regName[src_first]);
1485 #endif
1486         }
1487       }
1488       return 0;
1489     } else if (dst_first_rc == rc_float) {
1490       // xmm -> xmm
1491       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1492           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1493         // 64-bit
1494         if (cbuf) {
1495           MacroAssembler _masm(cbuf);
1496           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1497 #ifndef PRODUCT
1498         } else {
1499           st->print("%s  %s, %s\t# spill",
1500                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1501                      Matcher::regName[dst_first],
1502                      Matcher::regName[src_first]);
1503 #endif
1504         }
1505       } else {
1506         // 32-bit
1507         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1508         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1509         if (cbuf) {
1510           MacroAssembler _masm(cbuf);
1511           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1512 #ifndef PRODUCT
1513         } else {
1514           st->print("%s  %s, %s\t# spill",
1515                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1516                      Matcher::regName[dst_first],
1517                      Matcher::regName[src_first]);
1518 #endif
1519         }
1520       }
1521       return 0;
1522     }
1523   }
1524 
1525   assert(0," foo ");
1526   Unimplemented();
1527   return 0;
1528 }
1529 
1530 #ifndef PRODUCT
1531 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1532   implementation(NULL, ra_, false, st);
1533 }
1534 #endif
1535 
1536 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1537   implementation(&cbuf, ra_, false, NULL);
1538 }
1539 
1540 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1541   return MachNode::size(ra_);
1542 }
1543 
1544 //=============================================================================
1545 #ifndef PRODUCT
1546 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1547 {
1548   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1549   int reg = ra_->get_reg_first(this);
1550   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1551             Matcher::regName[reg], offset);
1552 }
1553 #endif
1554 
1555 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1556 {
1557   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1558   int reg = ra_->get_encode(this);
1559   if (offset >= 0x80) {
1560     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1561     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1562     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1563     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1564     emit_d32(cbuf, offset);
1565   } else {
1566     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1567     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1568     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1569     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1570     emit_d8(cbuf, offset);
1571   }
1572 }
1573 
1574 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1575 {
1576   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1577   return (offset < 0x80) ? 5 : 8; // REX
1578 }
1579 
1580 //=============================================================================
1581 #ifndef PRODUCT
1582 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1583 {
1584   if (UseCompressedClassPointers) {
1585     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1586     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1587     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1588   } else {
1589     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1590                  "# Inline cache check");
1591   }
1592   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1593   st->print_cr("\tnop\t# nops to align entry point");
1594 }
1595 #endif
1596 
1597 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1598 {
1599   MacroAssembler masm(&cbuf);
1600   uint insts_size = cbuf.insts_size();
1601   if (UseCompressedClassPointers) {
1602     masm.load_klass(rscratch1, j_rarg0);
1603     masm.cmpptr(rax, rscratch1);
1604   } else {
1605     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1606   }
1607 
1608   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1609 
1610   /* WARNING these NOPs are critical so that verified entry point is properly
1611      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1612   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1613   if (OptoBreakpoint) {
1614     // Leave space for int3
1615     nops_cnt -= 1;
1616   }
1617   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1618   if (nops_cnt > 0)
1619     masm.nop(nops_cnt);
1620 }
1621 
1622 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1623 {
1624   return MachNode::size(ra_); // too many variables; just compute it
1625                               // the hard way
1626 }
1627 
1628 
1629 //=============================================================================
1630 
1631 int Matcher::regnum_to_fpu_offset(int regnum)
1632 {
1633   return regnum - 32; // The FP registers are in the second chunk
1634 }
1635 
1636 // This is UltraSparc specific, true just means we have fast l2f conversion
1637 const bool Matcher::convL2FSupported(void) {
1638   return true;
1639 }
1640 
1641 // Is this branch offset short enough that a short branch can be used?
1642 //
1643 // NOTE: If the platform does not provide any short branch variants, then
1644 //       this method should return false for offset 0.
1645 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1646   // The passed offset is relative to address of the branch.
1647   // On 86 a branch displacement is calculated relative to address
1648   // of a next instruction.
1649   offset -= br_size;
1650 
1651   // the short version of jmpConUCF2 contains multiple branches,
1652   // making the reach slightly less
1653   if (rule == jmpConUCF2_rule)
1654     return (-126 <= offset && offset <= 125);
1655   return (-128 <= offset && offset <= 127);
1656 }
1657 
1658 const bool Matcher::isSimpleConstant64(jlong value) {
1659   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1660   //return value == (int) value;  // Cf. storeImmL and immL32.
1661 
1662   // Probably always true, even if a temp register is required.
1663   return true;
1664 }
1665 
1666 // The ecx parameter to rep stosq for the ClearArray node is in words.
1667 const bool Matcher::init_array_count_is_in_bytes = false;
1668 
1669 // No additional cost for CMOVL.
1670 const int Matcher::long_cmove_cost() { return 0; }
1671 
1672 // No CMOVF/CMOVD with SSE2
1673 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1674 
1675 // Does the CPU require late expand (see block.cpp for description of late expand)?
1676 const bool Matcher::require_postalloc_expand = false;
1677 
1678 // Do we need to mask the count passed to shift instructions or does
1679 // the cpu only look at the lower 5/6 bits anyway?
1680 const bool Matcher::need_masked_shift_count = false;
1681 
1682 bool Matcher::narrow_oop_use_complex_address() {
1683   assert(UseCompressedOops, "only for compressed oops code");
1684   return (LogMinObjAlignmentInBytes <= 3);
1685 }
1686 
1687 bool Matcher::narrow_klass_use_complex_address() {
1688   assert(UseCompressedClassPointers, "only for compressed klass code");
1689   return (LogKlassAlignmentInBytes <= 3);
1690 }
1691 
1692 bool Matcher::const_oop_prefer_decode() {
1693   // Prefer ConN+DecodeN over ConP.
1694   return true;
1695 }
1696 
1697 bool Matcher::const_klass_prefer_decode() {
1698   // TODO: Either support matching DecodeNKlass (heap-based) in operand
1699   //       or condisider the following:
1700   // Prefer ConNKlass+DecodeNKlass over ConP in simple compressed klass mode.
1701   //return Universe::narrow_klass_base() == NULL;
1702   return true;
1703 }
1704 
1705 // Is it better to copy float constants, or load them directly from
1706 // memory?  Intel can load a float constant from a direct address,
1707 // requiring no extra registers.  Most RISCs will have to materialize
1708 // an address into a register first, so they would do better to copy
1709 // the constant from stack.
1710 const bool Matcher::rematerialize_float_constants = true; // XXX
1711 
1712 // If CPU can load and store mis-aligned doubles directly then no
1713 // fixup is needed.  Else we split the double into 2 integer pieces
1714 // and move it piece-by-piece.  Only happens when passing doubles into
1715 // C code as the Java calling convention forces doubles to be aligned.
1716 const bool Matcher::misaligned_doubles_ok = true;
1717 
1718 // No-op on amd64
1719 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1720 
1721 // Advertise here if the CPU requires explicit rounding operations to
1722 // implement the UseStrictFP mode.
1723 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1724 
1725 // Are floats conerted to double when stored to stack during deoptimization?
1726 // On x64 it is stored without convertion so we can use normal access.
1727 bool Matcher::float_in_double() { return false; }
1728 
1729 // Do ints take an entire long register or just half?
1730 const bool Matcher::int_in_long = true;
1731 
1732 // Return whether or not this register is ever used as an argument.
1733 // This function is used on startup to build the trampoline stubs in
1734 // generateOptoStub.  Registers not mentioned will be killed by the VM
1735 // call in the trampoline, and arguments in those registers not be
1736 // available to the callee.
1737 bool Matcher::can_be_java_arg(int reg)
1738 {
1739   return
1740     reg ==  RDI_num || reg == RDI_H_num ||
1741     reg ==  RSI_num || reg == RSI_H_num ||
1742     reg ==  RDX_num || reg == RDX_H_num ||
1743     reg ==  RCX_num || reg == RCX_H_num ||
1744     reg ==   R8_num || reg ==  R8_H_num ||
1745     reg ==   R9_num || reg ==  R9_H_num ||
1746     reg ==  R12_num || reg == R12_H_num ||
1747     reg == XMM0_num || reg == XMM0b_num ||
1748     reg == XMM1_num || reg == XMM1b_num ||
1749     reg == XMM2_num || reg == XMM2b_num ||
1750     reg == XMM3_num || reg == XMM3b_num ||
1751     reg == XMM4_num || reg == XMM4b_num ||
1752     reg == XMM5_num || reg == XMM5b_num ||
1753     reg == XMM6_num || reg == XMM6b_num ||
1754     reg == XMM7_num || reg == XMM7b_num;
1755 }
1756 
1757 bool Matcher::is_spillable_arg(int reg)
1758 {
1759   return can_be_java_arg(reg);
1760 }
1761 
1762 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1763   // In 64 bit mode a code which use multiply when
1764   // devisor is constant is faster than hardware
1765   // DIV instruction (it uses MulHiL).
1766   return false;
1767 }
1768 
1769 // Register for DIVI projection of divmodI
1770 RegMask Matcher::divI_proj_mask() {
1771   return INT_RAX_REG_mask();
1772 }
1773 
1774 // Register for MODI projection of divmodI
1775 RegMask Matcher::modI_proj_mask() {
1776   return INT_RDX_REG_mask();
1777 }
1778 
1779 // Register for DIVL projection of divmodL
1780 RegMask Matcher::divL_proj_mask() {
1781   return LONG_RAX_REG_mask();
1782 }
1783 
1784 // Register for MODL projection of divmodL
1785 RegMask Matcher::modL_proj_mask() {
1786   return LONG_RDX_REG_mask();
1787 }
1788 
1789 // Register for saving SP into on method handle invokes. Not used on x86_64.
1790 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1791     return NO_REG_mask();
1792 }
1793 
1794 %}
1795 
1796 //----------ENCODING BLOCK-----------------------------------------------------
1797 // This block specifies the encoding classes used by the compiler to
1798 // output byte streams.  Encoding classes are parameterized macros
1799 // used by Machine Instruction Nodes in order to generate the bit
1800 // encoding of the instruction.  Operands specify their base encoding
1801 // interface with the interface keyword.  There are currently
1802 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1803 // COND_INTER.  REG_INTER causes an operand to generate a function
1804 // which returns its register number when queried.  CONST_INTER causes
1805 // an operand to generate a function which returns the value of the
1806 // constant when queried.  MEMORY_INTER causes an operand to generate
1807 // four functions which return the Base Register, the Index Register,
1808 // the Scale Value, and the Offset Value of the operand when queried.
1809 // COND_INTER causes an operand to generate six functions which return
1810 // the encoding code (ie - encoding bits for the instruction)
1811 // associated with each basic boolean condition for a conditional
1812 // instruction.
1813 //
1814 // Instructions specify two basic values for encoding.  Again, a
1815 // function is available to check if the constant displacement is an
1816 // oop. They use the ins_encode keyword to specify their encoding
1817 // classes (which must be a sequence of enc_class names, and their
1818 // parameters, specified in the encoding block), and they use the
1819 // opcode keyword to specify, in order, their primary, secondary, and
1820 // tertiary opcode.  Only the opcode sections which a particular
1821 // instruction needs for encoding need to be specified.
1822 encode %{
1823   // Build emit functions for each basic byte or larger field in the
1824   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1825   // from C++ code in the enc_class source block.  Emit functions will
1826   // live in the main source block for now.  In future, we can
1827   // generalize this by adding a syntax that specifies the sizes of
1828   // fields in an order, so that the adlc can build the emit functions
1829   // automagically
1830 
1831   // Emit primary opcode
1832   enc_class OpcP
1833   %{
1834     emit_opcode(cbuf, $primary);
1835   %}
1836 
1837   // Emit secondary opcode
1838   enc_class OpcS
1839   %{
1840     emit_opcode(cbuf, $secondary);
1841   %}
1842 
1843   // Emit tertiary opcode
1844   enc_class OpcT
1845   %{
1846     emit_opcode(cbuf, $tertiary);
1847   %}
1848 
1849   // Emit opcode directly
1850   enc_class Opcode(immI d8)
1851   %{
1852     emit_opcode(cbuf, $d8$$constant);
1853   %}
1854 
1855   // Emit size prefix
1856   enc_class SizePrefix
1857   %{
1858     emit_opcode(cbuf, 0x66);
1859   %}
1860 
1861   enc_class reg(rRegI reg)
1862   %{
1863     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1864   %}
1865 
1866   enc_class reg_reg(rRegI dst, rRegI src)
1867   %{
1868     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1869   %}
1870 
1871   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1872   %{
1873     emit_opcode(cbuf, $opcode$$constant);
1874     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1875   %}
1876 
1877   enc_class cdql_enc(no_rax_rdx_RegI div)
1878   %{
1879     // Full implementation of Java idiv and irem; checks for
1880     // special case as described in JVM spec., p.243 & p.271.
1881     //
1882     //         normal case                           special case
1883     //
1884     // input : rax: dividend                         min_int
1885     //         reg: divisor                          -1
1886     //
1887     // output: rax: quotient  (= rax idiv reg)       min_int
1888     //         rdx: remainder (= rax irem reg)       0
1889     //
1890     //  Code sequnce:
1891     //
1892     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1893     //    5:   75 07/08                jne    e <normal>
1894     //    7:   33 d2                   xor    %edx,%edx
1895     //  [div >= 8 -> offset + 1]
1896     //  [REX_B]
1897     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1898     //    c:   74 03/04                je     11 <done>
1899     // 000000000000000e <normal>:
1900     //    e:   99                      cltd
1901     //  [div >= 8 -> offset + 1]
1902     //  [REX_B]
1903     //    f:   f7 f9                   idiv   $div
1904     // 0000000000000011 <done>:
1905 
1906     // cmp    $0x80000000,%eax
1907     emit_opcode(cbuf, 0x3d);
1908     emit_d8(cbuf, 0x00);
1909     emit_d8(cbuf, 0x00);
1910     emit_d8(cbuf, 0x00);
1911     emit_d8(cbuf, 0x80);
1912 
1913     // jne    e <normal>
1914     emit_opcode(cbuf, 0x75);
1915     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1916 
1917     // xor    %edx,%edx
1918     emit_opcode(cbuf, 0x33);
1919     emit_d8(cbuf, 0xD2);
1920 
1921     // cmp    $0xffffffffffffffff,%ecx
1922     if ($div$$reg >= 8) {
1923       emit_opcode(cbuf, Assembler::REX_B);
1924     }
1925     emit_opcode(cbuf, 0x83);
1926     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1927     emit_d8(cbuf, 0xFF);
1928 
1929     // je     11 <done>
1930     emit_opcode(cbuf, 0x74);
1931     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1932 
1933     // <normal>
1934     // cltd
1935     emit_opcode(cbuf, 0x99);
1936 
1937     // idivl (note: must be emitted by the user of this rule)
1938     // <done>
1939   %}
1940 
1941   enc_class cdqq_enc(no_rax_rdx_RegL div)
1942   %{
1943     // Full implementation of Java ldiv and lrem; checks for
1944     // special case as described in JVM spec., p.243 & p.271.
1945     //
1946     //         normal case                           special case
1947     //
1948     // input : rax: dividend                         min_long
1949     //         reg: divisor                          -1
1950     //
1951     // output: rax: quotient  (= rax idiv reg)       min_long
1952     //         rdx: remainder (= rax irem reg)       0
1953     //
1954     //  Code sequnce:
1955     //
1956     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1957     //    7:   00 00 80
1958     //    a:   48 39 d0                cmp    %rdx,%rax
1959     //    d:   75 08                   jne    17 <normal>
1960     //    f:   33 d2                   xor    %edx,%edx
1961     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1962     //   15:   74 05                   je     1c <done>
1963     // 0000000000000017 <normal>:
1964     //   17:   48 99                   cqto
1965     //   19:   48 f7 f9                idiv   $div
1966     // 000000000000001c <done>:
1967 
1968     // mov    $0x8000000000000000,%rdx
1969     emit_opcode(cbuf, Assembler::REX_W);
1970     emit_opcode(cbuf, 0xBA);
1971     emit_d8(cbuf, 0x00);
1972     emit_d8(cbuf, 0x00);
1973     emit_d8(cbuf, 0x00);
1974     emit_d8(cbuf, 0x00);
1975     emit_d8(cbuf, 0x00);
1976     emit_d8(cbuf, 0x00);
1977     emit_d8(cbuf, 0x00);
1978     emit_d8(cbuf, 0x80);
1979 
1980     // cmp    %rdx,%rax
1981     emit_opcode(cbuf, Assembler::REX_W);
1982     emit_opcode(cbuf, 0x39);
1983     emit_d8(cbuf, 0xD0);
1984 
1985     // jne    17 <normal>
1986     emit_opcode(cbuf, 0x75);
1987     emit_d8(cbuf, 0x08);
1988 
1989     // xor    %edx,%edx
1990     emit_opcode(cbuf, 0x33);
1991     emit_d8(cbuf, 0xD2);
1992 
1993     // cmp    $0xffffffffffffffff,$div
1994     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1995     emit_opcode(cbuf, 0x83);
1996     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1997     emit_d8(cbuf, 0xFF);
1998 
1999     // je     1e <done>
2000     emit_opcode(cbuf, 0x74);
2001     emit_d8(cbuf, 0x05);
2002 
2003     // <normal>
2004     // cqto
2005     emit_opcode(cbuf, Assembler::REX_W);
2006     emit_opcode(cbuf, 0x99);
2007 
2008     // idivq (note: must be emitted by the user of this rule)
2009     // <done>
2010   %}
2011 
2012   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
2013   enc_class OpcSE(immI imm)
2014   %{
2015     // Emit primary opcode and set sign-extend bit
2016     // Check for 8-bit immediate, and set sign extend bit in opcode
2017     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2018       emit_opcode(cbuf, $primary | 0x02);
2019     } else {
2020       // 32-bit immediate
2021       emit_opcode(cbuf, $primary);
2022     }
2023   %}
2024 
2025   enc_class OpcSErm(rRegI dst, immI imm)
2026   %{
2027     // OpcSEr/m
2028     int dstenc = $dst$$reg;
2029     if (dstenc >= 8) {
2030       emit_opcode(cbuf, Assembler::REX_B);
2031       dstenc -= 8;
2032     }
2033     // Emit primary opcode and set sign-extend bit
2034     // Check for 8-bit immediate, and set sign extend bit in opcode
2035     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2036       emit_opcode(cbuf, $primary | 0x02);
2037     } else {
2038       // 32-bit immediate
2039       emit_opcode(cbuf, $primary);
2040     }
2041     // Emit r/m byte with secondary opcode, after primary opcode.
2042     emit_rm(cbuf, 0x3, $secondary, dstenc);
2043   %}
2044 
2045   enc_class OpcSErm_wide(rRegL dst, immI imm)
2046   %{
2047     // OpcSEr/m
2048     int dstenc = $dst$$reg;
2049     if (dstenc < 8) {
2050       emit_opcode(cbuf, Assembler::REX_W);
2051     } else {
2052       emit_opcode(cbuf, Assembler::REX_WB);
2053       dstenc -= 8;
2054     }
2055     // Emit primary opcode and set sign-extend bit
2056     // Check for 8-bit immediate, and set sign extend bit in opcode
2057     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2058       emit_opcode(cbuf, $primary | 0x02);
2059     } else {
2060       // 32-bit immediate
2061       emit_opcode(cbuf, $primary);
2062     }
2063     // Emit r/m byte with secondary opcode, after primary opcode.
2064     emit_rm(cbuf, 0x3, $secondary, dstenc);
2065   %}
2066 
2067   enc_class Con8or32(immI imm)
2068   %{
2069     // Check for 8-bit immediate, and set sign extend bit in opcode
2070     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2071       $$$emit8$imm$$constant;
2072     } else {
2073       // 32-bit immediate
2074       $$$emit32$imm$$constant;
2075     }
2076   %}
2077 
2078   enc_class opc2_reg(rRegI dst)
2079   %{
2080     // BSWAP
2081     emit_cc(cbuf, $secondary, $dst$$reg);
2082   %}
2083 
2084   enc_class opc3_reg(rRegI dst)
2085   %{
2086     // BSWAP
2087     emit_cc(cbuf, $tertiary, $dst$$reg);
2088   %}
2089 
2090   enc_class reg_opc(rRegI div)
2091   %{
2092     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2093     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2094   %}
2095 
2096   enc_class enc_cmov(cmpOp cop)
2097   %{
2098     // CMOV
2099     $$$emit8$primary;
2100     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2101   %}
2102 
2103   enc_class enc_PartialSubtypeCheck()
2104   %{
2105     Register Rrdi = as_Register(RDI_enc); // result register
2106     Register Rrax = as_Register(RAX_enc); // super class
2107     Register Rrcx = as_Register(RCX_enc); // killed
2108     Register Rrsi = as_Register(RSI_enc); // sub class
2109     Label miss;
2110     const bool set_cond_codes = true;
2111 
2112     MacroAssembler _masm(&cbuf);
2113     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2114                                      NULL, &miss,
2115                                      /*set_cond_codes:*/ true);
2116     if ($primary) {
2117       __ xorptr(Rrdi, Rrdi);
2118     }
2119     __ bind(miss);
2120   %}
2121 
2122   enc_class clear_avx %{
2123     debug_only(int off0 = cbuf.insts_size());
2124     if (generate_vzeroupper(Compile::current())) {
2125       // Clear upper bits of YMM registers to avoid AVX <-> SSE transition penalty
2126       // Clear upper bits of YMM registers when current compiled code uses
2127       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2128       MacroAssembler _masm(&cbuf);
2129       __ vzeroupper();
2130     }
2131     debug_only(int off1 = cbuf.insts_size());
2132     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2133   %}
2134 
2135   enc_class Java_To_Runtime(method meth) %{
2136     // No relocation needed
2137     MacroAssembler _masm(&cbuf);
2138     __ mov64(r10, (int64_t) $meth$$method);
2139     __ call(r10);
2140   %}
2141 
2142   enc_class Java_To_Interpreter(method meth)
2143   %{
2144     // CALL Java_To_Interpreter
2145     // This is the instruction starting address for relocation info.
2146     cbuf.set_insts_mark();
2147     $$$emit8$primary;
2148     // CALL directly to the runtime
2149     emit_d32_reloc(cbuf,
2150                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2151                    runtime_call_Relocation::spec(),
2152                    RELOC_DISP32);
2153   %}
2154 
2155   enc_class Java_Static_Call(method meth)
2156   %{
2157     // JAVA STATIC CALL
2158     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2159     // determine who we intended to call.
2160     cbuf.set_insts_mark();
2161     $$$emit8$primary;
2162 
2163     if (!_method) {
2164       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2165                      runtime_call_Relocation::spec(),
2166                      RELOC_DISP32);
2167     } else {
2168       int method_index = resolved_method_index(cbuf);
2169       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2170                                                   : static_call_Relocation::spec(method_index);
2171       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2172                      rspec, RELOC_DISP32);
2173       // Emit stubs for static call.
2174       address mark = cbuf.insts_mark();
2175       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2176       if (stub == NULL) {
2177         ciEnv::current()->record_failure("CodeCache is full");
2178         return;
2179       }
2180 #if INCLUDE_AOT
2181       CompiledStaticCall::emit_to_aot_stub(cbuf, mark);
2182 #endif
2183     }
2184   %}
2185 
2186   enc_class Java_Dynamic_Call(method meth) %{
2187     MacroAssembler _masm(&cbuf);
2188     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2189   %}
2190 
2191   enc_class Java_Compiled_Call(method meth)
2192   %{
2193     // JAVA COMPILED CALL
2194     int disp = in_bytes(Method:: from_compiled_offset());
2195 
2196     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2197     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2198 
2199     // callq *disp(%rax)
2200     cbuf.set_insts_mark();
2201     $$$emit8$primary;
2202     if (disp < 0x80) {
2203       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2204       emit_d8(cbuf, disp); // Displacement
2205     } else {
2206       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2207       emit_d32(cbuf, disp); // Displacement
2208     }
2209   %}
2210 
2211   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2212   %{
2213     // SAL, SAR, SHR
2214     int dstenc = $dst$$reg;
2215     if (dstenc >= 8) {
2216       emit_opcode(cbuf, Assembler::REX_B);
2217       dstenc -= 8;
2218     }
2219     $$$emit8$primary;
2220     emit_rm(cbuf, 0x3, $secondary, dstenc);
2221     $$$emit8$shift$$constant;
2222   %}
2223 
2224   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2225   %{
2226     // SAL, SAR, SHR
2227     int dstenc = $dst$$reg;
2228     if (dstenc < 8) {
2229       emit_opcode(cbuf, Assembler::REX_W);
2230     } else {
2231       emit_opcode(cbuf, Assembler::REX_WB);
2232       dstenc -= 8;
2233     }
2234     $$$emit8$primary;
2235     emit_rm(cbuf, 0x3, $secondary, dstenc);
2236     $$$emit8$shift$$constant;
2237   %}
2238 
2239   enc_class load_immI(rRegI dst, immI src)
2240   %{
2241     int dstenc = $dst$$reg;
2242     if (dstenc >= 8) {
2243       emit_opcode(cbuf, Assembler::REX_B);
2244       dstenc -= 8;
2245     }
2246     emit_opcode(cbuf, 0xB8 | dstenc);
2247     $$$emit32$src$$constant;
2248   %}
2249 
2250   enc_class load_immL(rRegL dst, immL src)
2251   %{
2252     int dstenc = $dst$$reg;
2253     if (dstenc < 8) {
2254       emit_opcode(cbuf, Assembler::REX_W);
2255     } else {
2256       emit_opcode(cbuf, Assembler::REX_WB);
2257       dstenc -= 8;
2258     }
2259     emit_opcode(cbuf, 0xB8 | dstenc);
2260     emit_d64(cbuf, $src$$constant);
2261   %}
2262 
2263   enc_class load_immUL32(rRegL dst, immUL32 src)
2264   %{
2265     // same as load_immI, but this time we care about zeroes in the high word
2266     int dstenc = $dst$$reg;
2267     if (dstenc >= 8) {
2268       emit_opcode(cbuf, Assembler::REX_B);
2269       dstenc -= 8;
2270     }
2271     emit_opcode(cbuf, 0xB8 | dstenc);
2272     $$$emit32$src$$constant;
2273   %}
2274 
2275   enc_class load_immL32(rRegL dst, immL32 src)
2276   %{
2277     int dstenc = $dst$$reg;
2278     if (dstenc < 8) {
2279       emit_opcode(cbuf, Assembler::REX_W);
2280     } else {
2281       emit_opcode(cbuf, Assembler::REX_WB);
2282       dstenc -= 8;
2283     }
2284     emit_opcode(cbuf, 0xC7);
2285     emit_rm(cbuf, 0x03, 0x00, dstenc);
2286     $$$emit32$src$$constant;
2287   %}
2288 
2289   enc_class load_immP31(rRegP dst, immP32 src)
2290   %{
2291     // same as load_immI, but this time we care about zeroes in the high word
2292     int dstenc = $dst$$reg;
2293     if (dstenc >= 8) {
2294       emit_opcode(cbuf, Assembler::REX_B);
2295       dstenc -= 8;
2296     }
2297     emit_opcode(cbuf, 0xB8 | dstenc);
2298     $$$emit32$src$$constant;
2299   %}
2300 
2301   enc_class load_immP(rRegP dst, immP src)
2302   %{
2303     int dstenc = $dst$$reg;
2304     if (dstenc < 8) {
2305       emit_opcode(cbuf, Assembler::REX_W);
2306     } else {
2307       emit_opcode(cbuf, Assembler::REX_WB);
2308       dstenc -= 8;
2309     }
2310     emit_opcode(cbuf, 0xB8 | dstenc);
2311     // This next line should be generated from ADLC
2312     if ($src->constant_reloc() != relocInfo::none) {
2313       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2314     } else {
2315       emit_d64(cbuf, $src$$constant);
2316     }
2317   %}
2318 
2319   enc_class Con32(immI src)
2320   %{
2321     // Output immediate
2322     $$$emit32$src$$constant;
2323   %}
2324 
2325   enc_class Con32F_as_bits(immF src)
2326   %{
2327     // Output Float immediate bits
2328     jfloat jf = $src$$constant;
2329     jint jf_as_bits = jint_cast(jf);
2330     emit_d32(cbuf, jf_as_bits);
2331   %}
2332 
2333   enc_class Con16(immI src)
2334   %{
2335     // Output immediate
2336     $$$emit16$src$$constant;
2337   %}
2338 
2339   // How is this different from Con32??? XXX
2340   enc_class Con_d32(immI src)
2341   %{
2342     emit_d32(cbuf,$src$$constant);
2343   %}
2344 
2345   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2346     // Output immediate memory reference
2347     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2348     emit_d32(cbuf, 0x00);
2349   %}
2350 
2351   enc_class lock_prefix()
2352   %{
2353     emit_opcode(cbuf, 0xF0); // lock
2354   %}
2355 
2356   enc_class REX_mem(memory mem)
2357   %{
2358     if ($mem$$base >= 8) {
2359       if ($mem$$index < 8) {
2360         emit_opcode(cbuf, Assembler::REX_B);
2361       } else {
2362         emit_opcode(cbuf, Assembler::REX_XB);
2363       }
2364     } else {
2365       if ($mem$$index >= 8) {
2366         emit_opcode(cbuf, Assembler::REX_X);
2367       }
2368     }
2369   %}
2370 
2371   enc_class REX_mem_wide(memory mem)
2372   %{
2373     if ($mem$$base >= 8) {
2374       if ($mem$$index < 8) {
2375         emit_opcode(cbuf, Assembler::REX_WB);
2376       } else {
2377         emit_opcode(cbuf, Assembler::REX_WXB);
2378       }
2379     } else {
2380       if ($mem$$index < 8) {
2381         emit_opcode(cbuf, Assembler::REX_W);
2382       } else {
2383         emit_opcode(cbuf, Assembler::REX_WX);
2384       }
2385     }
2386   %}
2387 
2388   // for byte regs
2389   enc_class REX_breg(rRegI reg)
2390   %{
2391     if ($reg$$reg >= 4) {
2392       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2393     }
2394   %}
2395 
2396   // for byte regs
2397   enc_class REX_reg_breg(rRegI dst, rRegI src)
2398   %{
2399     if ($dst$$reg < 8) {
2400       if ($src$$reg >= 4) {
2401         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2402       }
2403     } else {
2404       if ($src$$reg < 8) {
2405         emit_opcode(cbuf, Assembler::REX_R);
2406       } else {
2407         emit_opcode(cbuf, Assembler::REX_RB);
2408       }
2409     }
2410   %}
2411 
2412   // for byte regs
2413   enc_class REX_breg_mem(rRegI reg, memory mem)
2414   %{
2415     if ($reg$$reg < 8) {
2416       if ($mem$$base < 8) {
2417         if ($mem$$index >= 8) {
2418           emit_opcode(cbuf, Assembler::REX_X);
2419         } else if ($reg$$reg >= 4) {
2420           emit_opcode(cbuf, Assembler::REX);
2421         }
2422       } else {
2423         if ($mem$$index < 8) {
2424           emit_opcode(cbuf, Assembler::REX_B);
2425         } else {
2426           emit_opcode(cbuf, Assembler::REX_XB);
2427         }
2428       }
2429     } else {
2430       if ($mem$$base < 8) {
2431         if ($mem$$index < 8) {
2432           emit_opcode(cbuf, Assembler::REX_R);
2433         } else {
2434           emit_opcode(cbuf, Assembler::REX_RX);
2435         }
2436       } else {
2437         if ($mem$$index < 8) {
2438           emit_opcode(cbuf, Assembler::REX_RB);
2439         } else {
2440           emit_opcode(cbuf, Assembler::REX_RXB);
2441         }
2442       }
2443     }
2444   %}
2445 
2446   enc_class REX_reg(rRegI reg)
2447   %{
2448     if ($reg$$reg >= 8) {
2449       emit_opcode(cbuf, Assembler::REX_B);
2450     }
2451   %}
2452 
2453   enc_class REX_reg_wide(rRegI reg)
2454   %{
2455     if ($reg$$reg < 8) {
2456       emit_opcode(cbuf, Assembler::REX_W);
2457     } else {
2458       emit_opcode(cbuf, Assembler::REX_WB);
2459     }
2460   %}
2461 
2462   enc_class REX_reg_reg(rRegI dst, rRegI src)
2463   %{
2464     if ($dst$$reg < 8) {
2465       if ($src$$reg >= 8) {
2466         emit_opcode(cbuf, Assembler::REX_B);
2467       }
2468     } else {
2469       if ($src$$reg < 8) {
2470         emit_opcode(cbuf, Assembler::REX_R);
2471       } else {
2472         emit_opcode(cbuf, Assembler::REX_RB);
2473       }
2474     }
2475   %}
2476 
2477   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2478   %{
2479     if ($dst$$reg < 8) {
2480       if ($src$$reg < 8) {
2481         emit_opcode(cbuf, Assembler::REX_W);
2482       } else {
2483         emit_opcode(cbuf, Assembler::REX_WB);
2484       }
2485     } else {
2486       if ($src$$reg < 8) {
2487         emit_opcode(cbuf, Assembler::REX_WR);
2488       } else {
2489         emit_opcode(cbuf, Assembler::REX_WRB);
2490       }
2491     }
2492   %}
2493 
2494   enc_class REX_reg_mem(rRegI reg, memory mem)
2495   %{
2496     if ($reg$$reg < 8) {
2497       if ($mem$$base < 8) {
2498         if ($mem$$index >= 8) {
2499           emit_opcode(cbuf, Assembler::REX_X);
2500         }
2501       } else {
2502         if ($mem$$index < 8) {
2503           emit_opcode(cbuf, Assembler::REX_B);
2504         } else {
2505           emit_opcode(cbuf, Assembler::REX_XB);
2506         }
2507       }
2508     } else {
2509       if ($mem$$base < 8) {
2510         if ($mem$$index < 8) {
2511           emit_opcode(cbuf, Assembler::REX_R);
2512         } else {
2513           emit_opcode(cbuf, Assembler::REX_RX);
2514         }
2515       } else {
2516         if ($mem$$index < 8) {
2517           emit_opcode(cbuf, Assembler::REX_RB);
2518         } else {
2519           emit_opcode(cbuf, Assembler::REX_RXB);
2520         }
2521       }
2522     }
2523   %}
2524 
2525   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2526   %{
2527     if ($reg$$reg < 8) {
2528       if ($mem$$base < 8) {
2529         if ($mem$$index < 8) {
2530           emit_opcode(cbuf, Assembler::REX_W);
2531         } else {
2532           emit_opcode(cbuf, Assembler::REX_WX);
2533         }
2534       } else {
2535         if ($mem$$index < 8) {
2536           emit_opcode(cbuf, Assembler::REX_WB);
2537         } else {
2538           emit_opcode(cbuf, Assembler::REX_WXB);
2539         }
2540       }
2541     } else {
2542       if ($mem$$base < 8) {
2543         if ($mem$$index < 8) {
2544           emit_opcode(cbuf, Assembler::REX_WR);
2545         } else {
2546           emit_opcode(cbuf, Assembler::REX_WRX);
2547         }
2548       } else {
2549         if ($mem$$index < 8) {
2550           emit_opcode(cbuf, Assembler::REX_WRB);
2551         } else {
2552           emit_opcode(cbuf, Assembler::REX_WRXB);
2553         }
2554       }
2555     }
2556   %}
2557 
2558   enc_class reg_mem(rRegI ereg, memory mem)
2559   %{
2560     // High registers handle in encode_RegMem
2561     int reg = $ereg$$reg;
2562     int base = $mem$$base;
2563     int index = $mem$$index;
2564     int scale = $mem$$scale;
2565     int disp = $mem$$disp;
2566     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2567 
2568     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2569   %}
2570 
2571   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2572   %{
2573     int rm_byte_opcode = $rm_opcode$$constant;
2574 
2575     // High registers handle in encode_RegMem
2576     int base = $mem$$base;
2577     int index = $mem$$index;
2578     int scale = $mem$$scale;
2579     int displace = $mem$$disp;
2580 
2581     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2582                                             // working with static
2583                                             // globals
2584     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2585                   disp_reloc);
2586   %}
2587 
2588   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2589   %{
2590     int reg_encoding = $dst$$reg;
2591     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2592     int index        = 0x04;            // 0x04 indicates no index
2593     int scale        = 0x00;            // 0x00 indicates no scale
2594     int displace     = $src1$$constant; // 0x00 indicates no displacement
2595     relocInfo::relocType disp_reloc = relocInfo::none;
2596     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2597                   disp_reloc);
2598   %}
2599 
2600   enc_class neg_reg(rRegI dst)
2601   %{
2602     int dstenc = $dst$$reg;
2603     if (dstenc >= 8) {
2604       emit_opcode(cbuf, Assembler::REX_B);
2605       dstenc -= 8;
2606     }
2607     // NEG $dst
2608     emit_opcode(cbuf, 0xF7);
2609     emit_rm(cbuf, 0x3, 0x03, dstenc);
2610   %}
2611 
2612   enc_class neg_reg_wide(rRegI dst)
2613   %{
2614     int dstenc = $dst$$reg;
2615     if (dstenc < 8) {
2616       emit_opcode(cbuf, Assembler::REX_W);
2617     } else {
2618       emit_opcode(cbuf, Assembler::REX_WB);
2619       dstenc -= 8;
2620     }
2621     // NEG $dst
2622     emit_opcode(cbuf, 0xF7);
2623     emit_rm(cbuf, 0x3, 0x03, dstenc);
2624   %}
2625 
2626   enc_class setLT_reg(rRegI dst)
2627   %{
2628     int dstenc = $dst$$reg;
2629     if (dstenc >= 8) {
2630       emit_opcode(cbuf, Assembler::REX_B);
2631       dstenc -= 8;
2632     } else if (dstenc >= 4) {
2633       emit_opcode(cbuf, Assembler::REX);
2634     }
2635     // SETLT $dst
2636     emit_opcode(cbuf, 0x0F);
2637     emit_opcode(cbuf, 0x9C);
2638     emit_rm(cbuf, 0x3, 0x0, dstenc);
2639   %}
2640 
2641   enc_class setNZ_reg(rRegI dst)
2642   %{
2643     int dstenc = $dst$$reg;
2644     if (dstenc >= 8) {
2645       emit_opcode(cbuf, Assembler::REX_B);
2646       dstenc -= 8;
2647     } else if (dstenc >= 4) {
2648       emit_opcode(cbuf, Assembler::REX);
2649     }
2650     // SETNZ $dst
2651     emit_opcode(cbuf, 0x0F);
2652     emit_opcode(cbuf, 0x95);
2653     emit_rm(cbuf, 0x3, 0x0, dstenc);
2654   %}
2655 
2656 
2657   // Compare the lonogs and set -1, 0, or 1 into dst
2658   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2659   %{
2660     int src1enc = $src1$$reg;
2661     int src2enc = $src2$$reg;
2662     int dstenc = $dst$$reg;
2663 
2664     // cmpq $src1, $src2
2665     if (src1enc < 8) {
2666       if (src2enc < 8) {
2667         emit_opcode(cbuf, Assembler::REX_W);
2668       } else {
2669         emit_opcode(cbuf, Assembler::REX_WB);
2670       }
2671     } else {
2672       if (src2enc < 8) {
2673         emit_opcode(cbuf, Assembler::REX_WR);
2674       } else {
2675         emit_opcode(cbuf, Assembler::REX_WRB);
2676       }
2677     }
2678     emit_opcode(cbuf, 0x3B);
2679     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2680 
2681     // movl $dst, -1
2682     if (dstenc >= 8) {
2683       emit_opcode(cbuf, Assembler::REX_B);
2684     }
2685     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2686     emit_d32(cbuf, -1);
2687 
2688     // jl,s done
2689     emit_opcode(cbuf, 0x7C);
2690     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2691 
2692     // setne $dst
2693     if (dstenc >= 4) {
2694       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2695     }
2696     emit_opcode(cbuf, 0x0F);
2697     emit_opcode(cbuf, 0x95);
2698     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2699 
2700     // movzbl $dst, $dst
2701     if (dstenc >= 4) {
2702       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2703     }
2704     emit_opcode(cbuf, 0x0F);
2705     emit_opcode(cbuf, 0xB6);
2706     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2707   %}
2708 
2709   enc_class Push_ResultXD(regD dst) %{
2710     MacroAssembler _masm(&cbuf);
2711     __ fstp_d(Address(rsp, 0));
2712     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2713     __ addptr(rsp, 8);
2714   %}
2715 
2716   enc_class Push_SrcXD(regD src) %{
2717     MacroAssembler _masm(&cbuf);
2718     __ subptr(rsp, 8);
2719     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2720     __ fld_d(Address(rsp, 0));
2721   %}
2722 
2723 
2724   enc_class enc_rethrow()
2725   %{
2726     cbuf.set_insts_mark();
2727     emit_opcode(cbuf, 0xE9); // jmp entry
2728     emit_d32_reloc(cbuf,
2729                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2730                    runtime_call_Relocation::spec(),
2731                    RELOC_DISP32);
2732   %}
2733 
2734 %}
2735 
2736 
2737 
2738 //----------FRAME--------------------------------------------------------------
2739 // Definition of frame structure and management information.
2740 //
2741 //  S T A C K   L A Y O U T    Allocators stack-slot number
2742 //                             |   (to get allocators register number
2743 //  G  Owned by    |        |  v    add OptoReg::stack0())
2744 //  r   CALLER     |        |
2745 //  o     |        +--------+      pad to even-align allocators stack-slot
2746 //  w     V        |  pad0  |        numbers; owned by CALLER
2747 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2748 //  h     ^        |   in   |  5
2749 //        |        |  args  |  4   Holes in incoming args owned by SELF
2750 //  |     |        |        |  3
2751 //  |     |        +--------+
2752 //  V     |        | old out|      Empty on Intel, window on Sparc
2753 //        |    old |preserve|      Must be even aligned.
2754 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2755 //        |        |   in   |  3   area for Intel ret address
2756 //     Owned by    |preserve|      Empty on Sparc.
2757 //       SELF      +--------+
2758 //        |        |  pad2  |  2   pad to align old SP
2759 //        |        +--------+  1
2760 //        |        | locks  |  0
2761 //        |        +--------+----> OptoReg::stack0(), even aligned
2762 //        |        |  pad1  | 11   pad to align new SP
2763 //        |        +--------+
2764 //        |        |        | 10
2765 //        |        | spills |  9   spills
2766 //        V        |        |  8   (pad0 slot for callee)
2767 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2768 //        ^        |  out   |  7
2769 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2770 //     Owned by    +--------+
2771 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2772 //        |    new |preserve|      Must be even-aligned.
2773 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2774 //        |        |        |
2775 //
2776 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2777 //         known from SELF's arguments and the Java calling convention.
2778 //         Region 6-7 is determined per call site.
2779 // Note 2: If the calling convention leaves holes in the incoming argument
2780 //         area, those holes are owned by SELF.  Holes in the outgoing area
2781 //         are owned by the CALLEE.  Holes should not be nessecary in the
2782 //         incoming area, as the Java calling convention is completely under
2783 //         the control of the AD file.  Doubles can be sorted and packed to
2784 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2785 //         varargs C calling conventions.
2786 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2787 //         even aligned with pad0 as needed.
2788 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2789 //         region 6-11 is even aligned; it may be padded out more so that
2790 //         the region from SP to FP meets the minimum stack alignment.
2791 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2792 //         alignment.  Region 11, pad1, may be dynamically extended so that
2793 //         SP meets the minimum alignment.
2794 
2795 frame
2796 %{
2797   // What direction does stack grow in (assumed to be same for C & Java)
2798   stack_direction(TOWARDS_LOW);
2799 
2800   // These three registers define part of the calling convention
2801   // between compiled code and the interpreter.
2802   inline_cache_reg(RAX);                // Inline Cache Register
2803   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2804                                         // calling interpreter
2805 
2806   // Optional: name the operand used by cisc-spilling to access
2807   // [stack_pointer + offset]
2808   cisc_spilling_operand_name(indOffset32);
2809 
2810   // Number of stack slots consumed by locking an object
2811   sync_stack_slots(2);
2812 
2813   // Compiled code's Frame Pointer
2814   frame_pointer(RSP);
2815 
2816   // Interpreter stores its frame pointer in a register which is
2817   // stored to the stack by I2CAdaptors.
2818   // I2CAdaptors convert from interpreted java to compiled java.
2819   interpreter_frame_pointer(RBP);
2820 
2821   // Stack alignment requirement
2822   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2823 
2824   // Number of stack slots between incoming argument block and the start of
2825   // a new frame.  The PROLOG must add this many slots to the stack.  The
2826   // EPILOG must remove this many slots.  amd64 needs two slots for
2827   // return address.
2828   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2829 
2830   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2831   // for calls to C.  Supports the var-args backing area for register parms.
2832   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2833 
2834   // The after-PROLOG location of the return address.  Location of
2835   // return address specifies a type (REG or STACK) and a number
2836   // representing the register number (i.e. - use a register name) or
2837   // stack slot.
2838   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2839   // Otherwise, it is above the locks and verification slot and alignment word
2840   return_addr(STACK - 2 +
2841               align_up((Compile::current()->in_preserve_stack_slots() +
2842                         Compile::current()->fixed_slots()),
2843                        stack_alignment_in_slots()));
2844 
2845   // Body of function which returns an integer array locating
2846   // arguments either in registers or in stack slots.  Passed an array
2847   // of ideal registers called "sig" and a "length" count.  Stack-slot
2848   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2849   // arguments for a CALLEE.  Incoming stack arguments are
2850   // automatically biased by the preserve_stack_slots field above.
2851 
2852   calling_convention
2853   %{
2854     // No difference between ingoing/outgoing just pass false
2855     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2856   %}
2857 
2858   c_calling_convention
2859   %{
2860     // This is obviously always outgoing
2861     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2862   %}
2863 
2864   // Location of compiled Java return values.  Same as C for now.
2865   return_value
2866   %{
2867     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2868            "only return normal values");
2869 
2870     static const int lo[Op_RegL + 1] = {
2871       0,
2872       0,
2873       RAX_num,  // Op_RegN
2874       RAX_num,  // Op_RegI
2875       RAX_num,  // Op_RegP
2876       XMM0_num, // Op_RegF
2877       XMM0_num, // Op_RegD
2878       RAX_num   // Op_RegL
2879     };
2880     static const int hi[Op_RegL + 1] = {
2881       0,
2882       0,
2883       OptoReg::Bad, // Op_RegN
2884       OptoReg::Bad, // Op_RegI
2885       RAX_H_num,    // Op_RegP
2886       OptoReg::Bad, // Op_RegF
2887       XMM0b_num,    // Op_RegD
2888       RAX_H_num     // Op_RegL
2889     };
2890     // Excluded flags and vector registers.
2891     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2892     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2893   %}
2894 %}
2895 
2896 //----------ATTRIBUTES---------------------------------------------------------
2897 //----------Operand Attributes-------------------------------------------------
2898 op_attrib op_cost(0);        // Required cost attribute
2899 
2900 //----------Instruction Attributes---------------------------------------------
2901 ins_attrib ins_cost(100);       // Required cost attribute
2902 ins_attrib ins_size(8);         // Required size attribute (in bits)
2903 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2904                                 // a non-matching short branch variant
2905                                 // of some long branch?
2906 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2907                                 // be a power of 2) specifies the
2908                                 // alignment that some part of the
2909                                 // instruction (not necessarily the
2910                                 // start) requires.  If > 1, a
2911                                 // compute_padding() function must be
2912                                 // provided for the instruction
2913 
2914 //----------OPERANDS-----------------------------------------------------------
2915 // Operand definitions must precede instruction definitions for correct parsing
2916 // in the ADLC because operands constitute user defined types which are used in
2917 // instruction definitions.
2918 
2919 //----------Simple Operands----------------------------------------------------
2920 // Immediate Operands
2921 // Integer Immediate
2922 operand immI()
2923 %{
2924   match(ConI);
2925 
2926   op_cost(10);
2927   format %{ %}
2928   interface(CONST_INTER);
2929 %}
2930 
2931 // Constant for test vs zero
2932 operand immI0()
2933 %{
2934   predicate(n->get_int() == 0);
2935   match(ConI);
2936 
2937   op_cost(0);
2938   format %{ %}
2939   interface(CONST_INTER);
2940 %}
2941 
2942 // Constant for increment
2943 operand immI1()
2944 %{
2945   predicate(n->get_int() == 1);
2946   match(ConI);
2947 
2948   op_cost(0);
2949   format %{ %}
2950   interface(CONST_INTER);
2951 %}
2952 
2953 // Constant for decrement
2954 operand immI_M1()
2955 %{
2956   predicate(n->get_int() == -1);
2957   match(ConI);
2958 
2959   op_cost(0);
2960   format %{ %}
2961   interface(CONST_INTER);
2962 %}
2963 
2964 // Valid scale values for addressing modes
2965 operand immI2()
2966 %{
2967   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2968   match(ConI);
2969 
2970   format %{ %}
2971   interface(CONST_INTER);
2972 %}
2973 
2974 operand immI8()
2975 %{
2976   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2977   match(ConI);
2978 
2979   op_cost(5);
2980   format %{ %}
2981   interface(CONST_INTER);
2982 %}
2983 
2984 operand immU8()
2985 %{
2986   predicate((0 <= n->get_int()) && (n->get_int() <= 255));
2987   match(ConI);
2988 
2989   op_cost(5);
2990   format %{ %}
2991   interface(CONST_INTER);
2992 %}
2993 
2994 operand immI16()
2995 %{
2996   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2997   match(ConI);
2998 
2999   op_cost(10);
3000   format %{ %}
3001   interface(CONST_INTER);
3002 %}
3003 
3004 // Int Immediate non-negative
3005 operand immU31()
3006 %{
3007   predicate(n->get_int() >= 0);
3008   match(ConI);
3009 
3010   op_cost(0);
3011   format %{ %}
3012   interface(CONST_INTER);
3013 %}
3014 
3015 // Constant for long shifts
3016 operand immI_32()
3017 %{
3018   predicate( n->get_int() == 32 );
3019   match(ConI);
3020 
3021   op_cost(0);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // Constant for long shifts
3027 operand immI_64()
3028 %{
3029   predicate( n->get_int() == 64 );
3030   match(ConI);
3031 
3032   op_cost(0);
3033   format %{ %}
3034   interface(CONST_INTER);
3035 %}
3036 
3037 // Pointer Immediate
3038 operand immP()
3039 %{
3040   match(ConP);
3041 
3042   op_cost(10);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 // NULL Pointer Immediate
3048 operand immP0()
3049 %{
3050   predicate(n->get_ptr() == 0);
3051   match(ConP);
3052 
3053   op_cost(5);
3054   format %{ %}
3055   interface(CONST_INTER);
3056 %}
3057 
3058 // Pointer Immediate
3059 operand immN() %{
3060   match(ConN);
3061 
3062   op_cost(10);
3063   format %{ %}
3064   interface(CONST_INTER);
3065 %}
3066 
3067 operand immNKlass() %{
3068   match(ConNKlass);
3069 
3070   op_cost(10);
3071   format %{ %}
3072   interface(CONST_INTER);
3073 %}
3074 
3075 // NULL Pointer Immediate
3076 operand immN0() %{
3077   predicate(n->get_narrowcon() == 0);
3078   match(ConN);
3079 
3080   op_cost(5);
3081   format %{ %}
3082   interface(CONST_INTER);
3083 %}
3084 
3085 operand immP31()
3086 %{
3087   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3088             && (n->get_ptr() >> 31) == 0);
3089   match(ConP);
3090 
3091   op_cost(5);
3092   format %{ %}
3093   interface(CONST_INTER);
3094 %}
3095 
3096 
3097 // Long Immediate
3098 operand immL()
3099 %{
3100   match(ConL);
3101 
3102   op_cost(20);
3103   format %{ %}
3104   interface(CONST_INTER);
3105 %}
3106 
3107 // Long Immediate 8-bit
3108 operand immL8()
3109 %{
3110   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3111   match(ConL);
3112 
3113   op_cost(5);
3114   format %{ %}
3115   interface(CONST_INTER);
3116 %}
3117 
3118 // Long Immediate 32-bit unsigned
3119 operand immUL32()
3120 %{
3121   predicate(n->get_long() == (unsigned int) (n->get_long()));
3122   match(ConL);
3123 
3124   op_cost(10);
3125   format %{ %}
3126   interface(CONST_INTER);
3127 %}
3128 
3129 // Long Immediate 32-bit signed
3130 operand immL32()
3131 %{
3132   predicate(n->get_long() == (int) (n->get_long()));
3133   match(ConL);
3134 
3135   op_cost(15);
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // Long Immediate zero
3141 operand immL0()
3142 %{
3143   predicate(n->get_long() == 0L);
3144   match(ConL);
3145 
3146   op_cost(10);
3147   format %{ %}
3148   interface(CONST_INTER);
3149 %}
3150 
3151 // Constant for increment
3152 operand immL1()
3153 %{
3154   predicate(n->get_long() == 1);
3155   match(ConL);
3156 
3157   format %{ %}
3158   interface(CONST_INTER);
3159 %}
3160 
3161 // Constant for decrement
3162 operand immL_M1()
3163 %{
3164   predicate(n->get_long() == -1);
3165   match(ConL);
3166 
3167   format %{ %}
3168   interface(CONST_INTER);
3169 %}
3170 
3171 // Long Immediate: the value 10
3172 operand immL10()
3173 %{
3174   predicate(n->get_long() == 10);
3175   match(ConL);
3176 
3177   format %{ %}
3178   interface(CONST_INTER);
3179 %}
3180 
3181 // Long immediate from 0 to 127.
3182 // Used for a shorter form of long mul by 10.
3183 operand immL_127()
3184 %{
3185   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3186   match(ConL);
3187 
3188   op_cost(10);
3189   format %{ %}
3190   interface(CONST_INTER);
3191 %}
3192 
3193 // Long Immediate: low 32-bit mask
3194 operand immL_32bits()
3195 %{
3196   predicate(n->get_long() == 0xFFFFFFFFL);
3197   match(ConL);
3198   op_cost(20);
3199 
3200   format %{ %}
3201   interface(CONST_INTER);
3202 %}
3203 
3204 // Float Immediate zero
3205 operand immF0()
3206 %{
3207   predicate(jint_cast(n->getf()) == 0);
3208   match(ConF);
3209 
3210   op_cost(5);
3211   format %{ %}
3212   interface(CONST_INTER);
3213 %}
3214 
3215 // Float Immediate
3216 operand immF()
3217 %{
3218   match(ConF);
3219 
3220   op_cost(15);
3221   format %{ %}
3222   interface(CONST_INTER);
3223 %}
3224 
3225 // Double Immediate zero
3226 operand immD0()
3227 %{
3228   predicate(jlong_cast(n->getd()) == 0);
3229   match(ConD);
3230 
3231   op_cost(5);
3232   format %{ %}
3233   interface(CONST_INTER);
3234 %}
3235 
3236 // Double Immediate
3237 operand immD()
3238 %{
3239   match(ConD);
3240 
3241   op_cost(15);
3242   format %{ %}
3243   interface(CONST_INTER);
3244 %}
3245 
3246 // Immediates for special shifts (sign extend)
3247 
3248 // Constants for increment
3249 operand immI_16()
3250 %{
3251   predicate(n->get_int() == 16);
3252   match(ConI);
3253 
3254   format %{ %}
3255   interface(CONST_INTER);
3256 %}
3257 
3258 operand immI_24()
3259 %{
3260   predicate(n->get_int() == 24);
3261   match(ConI);
3262 
3263   format %{ %}
3264   interface(CONST_INTER);
3265 %}
3266 
3267 // Constant for byte-wide masking
3268 operand immI_255()
3269 %{
3270   predicate(n->get_int() == 255);
3271   match(ConI);
3272 
3273   format %{ %}
3274   interface(CONST_INTER);
3275 %}
3276 
3277 // Constant for short-wide masking
3278 operand immI_65535()
3279 %{
3280   predicate(n->get_int() == 65535);
3281   match(ConI);
3282 
3283   format %{ %}
3284   interface(CONST_INTER);
3285 %}
3286 
3287 // Constant for byte-wide masking
3288 operand immL_255()
3289 %{
3290   predicate(n->get_long() == 255);
3291   match(ConL);
3292 
3293   format %{ %}
3294   interface(CONST_INTER);
3295 %}
3296 
3297 // Constant for short-wide masking
3298 operand immL_65535()
3299 %{
3300   predicate(n->get_long() == 65535);
3301   match(ConL);
3302 
3303   format %{ %}
3304   interface(CONST_INTER);
3305 %}
3306 
3307 // Register Operands
3308 // Integer Register
3309 operand rRegI()
3310 %{
3311   constraint(ALLOC_IN_RC(int_reg));
3312   match(RegI);
3313 
3314   match(rax_RegI);
3315   match(rbx_RegI);
3316   match(rcx_RegI);
3317   match(rdx_RegI);
3318   match(rdi_RegI);
3319 
3320   format %{ %}
3321   interface(REG_INTER);
3322 %}
3323 
3324 // Special Registers
3325 operand rax_RegI()
3326 %{
3327   constraint(ALLOC_IN_RC(int_rax_reg));
3328   match(RegI);
3329   match(rRegI);
3330 
3331   format %{ "RAX" %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 // Special Registers
3336 operand rbx_RegI()
3337 %{
3338   constraint(ALLOC_IN_RC(int_rbx_reg));
3339   match(RegI);
3340   match(rRegI);
3341 
3342   format %{ "RBX" %}
3343   interface(REG_INTER);
3344 %}
3345 
3346 operand rcx_RegI()
3347 %{
3348   constraint(ALLOC_IN_RC(int_rcx_reg));
3349   match(RegI);
3350   match(rRegI);
3351 
3352   format %{ "RCX" %}
3353   interface(REG_INTER);
3354 %}
3355 
3356 operand rdx_RegI()
3357 %{
3358   constraint(ALLOC_IN_RC(int_rdx_reg));
3359   match(RegI);
3360   match(rRegI);
3361 
3362   format %{ "RDX" %}
3363   interface(REG_INTER);
3364 %}
3365 
3366 operand rdi_RegI()
3367 %{
3368   constraint(ALLOC_IN_RC(int_rdi_reg));
3369   match(RegI);
3370   match(rRegI);
3371 
3372   format %{ "RDI" %}
3373   interface(REG_INTER);
3374 %}
3375 
3376 operand no_rcx_RegI()
3377 %{
3378   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3379   match(RegI);
3380   match(rax_RegI);
3381   match(rbx_RegI);
3382   match(rdx_RegI);
3383   match(rdi_RegI);
3384 
3385   format %{ %}
3386   interface(REG_INTER);
3387 %}
3388 
3389 operand no_rax_rdx_RegI()
3390 %{
3391   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3392   match(RegI);
3393   match(rbx_RegI);
3394   match(rcx_RegI);
3395   match(rdi_RegI);
3396 
3397   format %{ %}
3398   interface(REG_INTER);
3399 %}
3400 
3401 // Pointer Register
3402 operand any_RegP()
3403 %{
3404   constraint(ALLOC_IN_RC(any_reg));
3405   match(RegP);
3406   match(rax_RegP);
3407   match(rbx_RegP);
3408   match(rdi_RegP);
3409   match(rsi_RegP);
3410   match(rbp_RegP);
3411   match(r15_RegP);
3412   match(rRegP);
3413 
3414   format %{ %}
3415   interface(REG_INTER);
3416 %}
3417 
3418 operand rRegP()
3419 %{
3420   constraint(ALLOC_IN_RC(ptr_reg));
3421   match(RegP);
3422   match(rax_RegP);
3423   match(rbx_RegP);
3424   match(rdi_RegP);
3425   match(rsi_RegP);
3426   match(rbp_RegP);  // See Q&A below about
3427   match(r15_RegP);  // r15_RegP and rbp_RegP.
3428 
3429   format %{ %}
3430   interface(REG_INTER);
3431 %}
3432 
3433 operand rRegN() %{
3434   constraint(ALLOC_IN_RC(int_reg));
3435   match(RegN);
3436 
3437   format %{ %}
3438   interface(REG_INTER);
3439 %}
3440 
3441 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3442 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3443 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3444 // The output of an instruction is controlled by the allocator, which respects
3445 // register class masks, not match rules.  Unless an instruction mentions
3446 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3447 // by the allocator as an input.
3448 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3449 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3450 // result, RBP is not included in the output of the instruction either.
3451 
3452 operand no_rax_RegP()
3453 %{
3454   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3455   match(RegP);
3456   match(rbx_RegP);
3457   match(rsi_RegP);
3458   match(rdi_RegP);
3459 
3460   format %{ %}
3461   interface(REG_INTER);
3462 %}
3463 
3464 // This operand is not allowed to use RBP even if
3465 // RBP is not used to hold the frame pointer.
3466 operand no_rbp_RegP()
3467 %{
3468   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3469   match(RegP);
3470   match(rbx_RegP);
3471   match(rsi_RegP);
3472   match(rdi_RegP);
3473 
3474   format %{ %}
3475   interface(REG_INTER);
3476 %}
3477 
3478 operand no_rax_rbx_RegP()
3479 %{
3480   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3481   match(RegP);
3482   match(rsi_RegP);
3483   match(rdi_RegP);
3484 
3485   format %{ %}
3486   interface(REG_INTER);
3487 %}
3488 
3489 // Special Registers
3490 // Return a pointer value
3491 operand rax_RegP()
3492 %{
3493   constraint(ALLOC_IN_RC(ptr_rax_reg));
3494   match(RegP);
3495   match(rRegP);
3496 
3497   format %{ %}
3498   interface(REG_INTER);
3499 %}
3500 
3501 // Special Registers
3502 // Return a compressed pointer value
3503 operand rax_RegN()
3504 %{
3505   constraint(ALLOC_IN_RC(int_rax_reg));
3506   match(RegN);
3507   match(rRegN);
3508 
3509   format %{ %}
3510   interface(REG_INTER);
3511 %}
3512 
3513 // Used in AtomicAdd
3514 operand rbx_RegP()
3515 %{
3516   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3517   match(RegP);
3518   match(rRegP);
3519 
3520   format %{ %}
3521   interface(REG_INTER);
3522 %}
3523 
3524 operand rsi_RegP()
3525 %{
3526   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3527   match(RegP);
3528   match(rRegP);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 // Used in rep stosq
3535 operand rdi_RegP()
3536 %{
3537   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3538   match(RegP);
3539   match(rRegP);
3540 
3541   format %{ %}
3542   interface(REG_INTER);
3543 %}
3544 
3545 operand r15_RegP()
3546 %{
3547   constraint(ALLOC_IN_RC(ptr_r15_reg));
3548   match(RegP);
3549   match(rRegP);
3550 
3551   format %{ %}
3552   interface(REG_INTER);
3553 %}
3554 
3555 operand rRegL()
3556 %{
3557   constraint(ALLOC_IN_RC(long_reg));
3558   match(RegL);
3559   match(rax_RegL);
3560   match(rdx_RegL);
3561 
3562   format %{ %}
3563   interface(REG_INTER);
3564 %}
3565 
3566 // Special Registers
3567 operand no_rax_rdx_RegL()
3568 %{
3569   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3570   match(RegL);
3571   match(rRegL);
3572 
3573   format %{ %}
3574   interface(REG_INTER);
3575 %}
3576 
3577 operand no_rax_RegL()
3578 %{
3579   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3580   match(RegL);
3581   match(rRegL);
3582   match(rdx_RegL);
3583 
3584   format %{ %}
3585   interface(REG_INTER);
3586 %}
3587 
3588 operand no_rcx_RegL()
3589 %{
3590   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3591   match(RegL);
3592   match(rRegL);
3593 
3594   format %{ %}
3595   interface(REG_INTER);
3596 %}
3597 
3598 operand rax_RegL()
3599 %{
3600   constraint(ALLOC_IN_RC(long_rax_reg));
3601   match(RegL);
3602   match(rRegL);
3603 
3604   format %{ "RAX" %}
3605   interface(REG_INTER);
3606 %}
3607 
3608 operand rcx_RegL()
3609 %{
3610   constraint(ALLOC_IN_RC(long_rcx_reg));
3611   match(RegL);
3612   match(rRegL);
3613 
3614   format %{ %}
3615   interface(REG_INTER);
3616 %}
3617 
3618 operand rdx_RegL()
3619 %{
3620   constraint(ALLOC_IN_RC(long_rdx_reg));
3621   match(RegL);
3622   match(rRegL);
3623 
3624   format %{ %}
3625   interface(REG_INTER);
3626 %}
3627 
3628 // Flags register, used as output of compare instructions
3629 operand rFlagsReg()
3630 %{
3631   constraint(ALLOC_IN_RC(int_flags));
3632   match(RegFlags);
3633 
3634   format %{ "RFLAGS" %}
3635   interface(REG_INTER);
3636 %}
3637 
3638 // Flags register, used as output of FLOATING POINT compare instructions
3639 operand rFlagsRegU()
3640 %{
3641   constraint(ALLOC_IN_RC(int_flags));
3642   match(RegFlags);
3643 
3644   format %{ "RFLAGS_U" %}
3645   interface(REG_INTER);
3646 %}
3647 
3648 operand rFlagsRegUCF() %{
3649   constraint(ALLOC_IN_RC(int_flags));
3650   match(RegFlags);
3651   predicate(false);
3652 
3653   format %{ "RFLAGS_U_CF" %}
3654   interface(REG_INTER);
3655 %}
3656 
3657 // Float register operands
3658 operand regF() %{
3659    constraint(ALLOC_IN_RC(float_reg));
3660    match(RegF);
3661 
3662    format %{ %}
3663    interface(REG_INTER);
3664 %}
3665 
3666 // Float register operands
3667 operand vlRegF() %{
3668    constraint(ALLOC_IN_RC(float_reg_vl));
3669    match(RegF);
3670 
3671    format %{ %}
3672    interface(REG_INTER);
3673 %}
3674 
3675 // Double register operands
3676 operand regD() %{
3677    constraint(ALLOC_IN_RC(double_reg));
3678    match(RegD);
3679 
3680    format %{ %}
3681    interface(REG_INTER);
3682 %}
3683 
3684 // Double register operands
3685 operand vlRegD() %{
3686    constraint(ALLOC_IN_RC(double_reg_vl));
3687    match(RegD);
3688 
3689    format %{ %}
3690    interface(REG_INTER);
3691 %}
3692 
3693 // Vectors
3694 operand vecS() %{
3695   constraint(ALLOC_IN_RC(vectors_reg_vlbwdq));
3696   match(VecS);
3697 
3698   format %{ %}
3699   interface(REG_INTER);
3700 %}
3701 
3702 // Vectors
3703 operand legVecS() %{
3704   constraint(ALLOC_IN_RC(vectors_reg_legacy));
3705   match(VecS);
3706 
3707   format %{ %}
3708   interface(REG_INTER);
3709 %}
3710 
3711 operand vecD() %{
3712   constraint(ALLOC_IN_RC(vectord_reg_vlbwdq));
3713   match(VecD);
3714 
3715   format %{ %}
3716   interface(REG_INTER);
3717 %}
3718 
3719 operand legVecD() %{
3720   constraint(ALLOC_IN_RC(vectord_reg_legacy));
3721   match(VecD);
3722 
3723   format %{ %}
3724   interface(REG_INTER);
3725 %}
3726 
3727 operand vecX() %{
3728   constraint(ALLOC_IN_RC(vectorx_reg_vlbwdq));
3729   match(VecX);
3730 
3731   format %{ %}
3732   interface(REG_INTER);
3733 %}
3734 
3735 operand legVecX() %{
3736   constraint(ALLOC_IN_RC(vectorx_reg_legacy));
3737   match(VecX);
3738 
3739   format %{ %}
3740   interface(REG_INTER);
3741 %}
3742 
3743 operand vecY() %{
3744   constraint(ALLOC_IN_RC(vectory_reg_vlbwdq));
3745   match(VecY);
3746 
3747   format %{ %}
3748   interface(REG_INTER);
3749 %}
3750 
3751 operand legVecY() %{
3752   constraint(ALLOC_IN_RC(vectory_reg_legacy));
3753   match(VecY);
3754 
3755   format %{ %}
3756   interface(REG_INTER);
3757 %}
3758 
3759 //----------Memory Operands----------------------------------------------------
3760 // Direct Memory Operand
3761 // operand direct(immP addr)
3762 // %{
3763 //   match(addr);
3764 
3765 //   format %{ "[$addr]" %}
3766 //   interface(MEMORY_INTER) %{
3767 //     base(0xFFFFFFFF);
3768 //     index(0x4);
3769 //     scale(0x0);
3770 //     disp($addr);
3771 //   %}
3772 // %}
3773 
3774 // Indirect Memory Operand
3775 operand indirect(any_RegP reg)
3776 %{
3777   constraint(ALLOC_IN_RC(ptr_reg));
3778   match(reg);
3779 
3780   format %{ "[$reg]" %}
3781   interface(MEMORY_INTER) %{
3782     base($reg);
3783     index(0x4);
3784     scale(0x0);
3785     disp(0x0);
3786   %}
3787 %}
3788 
3789 // Indirect Memory Plus Short Offset Operand
3790 operand indOffset8(any_RegP reg, immL8 off)
3791 %{
3792   constraint(ALLOC_IN_RC(ptr_reg));
3793   match(AddP reg off);
3794 
3795   format %{ "[$reg + $off (8-bit)]" %}
3796   interface(MEMORY_INTER) %{
3797     base($reg);
3798     index(0x4);
3799     scale(0x0);
3800     disp($off);
3801   %}
3802 %}
3803 
3804 // Indirect Memory Plus Long Offset Operand
3805 operand indOffset32(any_RegP reg, immL32 off)
3806 %{
3807   constraint(ALLOC_IN_RC(ptr_reg));
3808   match(AddP reg off);
3809 
3810   format %{ "[$reg + $off (32-bit)]" %}
3811   interface(MEMORY_INTER) %{
3812     base($reg);
3813     index(0x4);
3814     scale(0x0);
3815     disp($off);
3816   %}
3817 %}
3818 
3819 // Indirect Memory Plus Index Register Plus Offset Operand
3820 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3821 %{
3822   constraint(ALLOC_IN_RC(ptr_reg));
3823   match(AddP (AddP reg lreg) off);
3824 
3825   op_cost(10);
3826   format %{"[$reg + $off + $lreg]" %}
3827   interface(MEMORY_INTER) %{
3828     base($reg);
3829     index($lreg);
3830     scale(0x0);
3831     disp($off);
3832   %}
3833 %}
3834 
3835 // Indirect Memory Plus Index Register Plus Offset Operand
3836 operand indIndex(any_RegP reg, rRegL lreg)
3837 %{
3838   constraint(ALLOC_IN_RC(ptr_reg));
3839   match(AddP reg lreg);
3840 
3841   op_cost(10);
3842   format %{"[$reg + $lreg]" %}
3843   interface(MEMORY_INTER) %{
3844     base($reg);
3845     index($lreg);
3846     scale(0x0);
3847     disp(0x0);
3848   %}
3849 %}
3850 
3851 // Indirect Memory Times Scale Plus Index Register
3852 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3853 %{
3854   constraint(ALLOC_IN_RC(ptr_reg));
3855   match(AddP reg (LShiftL lreg scale));
3856 
3857   op_cost(10);
3858   format %{"[$reg + $lreg << $scale]" %}
3859   interface(MEMORY_INTER) %{
3860     base($reg);
3861     index($lreg);
3862     scale($scale);
3863     disp(0x0);
3864   %}
3865 %}
3866 
3867 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3868 %{
3869   constraint(ALLOC_IN_RC(ptr_reg));
3870   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3871   match(AddP reg (LShiftL (ConvI2L idx) scale));
3872 
3873   op_cost(10);
3874   format %{"[$reg + pos $idx << $scale]" %}
3875   interface(MEMORY_INTER) %{
3876     base($reg);
3877     index($idx);
3878     scale($scale);
3879     disp(0x0);
3880   %}
3881 %}
3882 
3883 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3884 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3885 %{
3886   constraint(ALLOC_IN_RC(ptr_reg));
3887   match(AddP (AddP reg (LShiftL lreg scale)) off);
3888 
3889   op_cost(10);
3890   format %{"[$reg + $off + $lreg << $scale]" %}
3891   interface(MEMORY_INTER) %{
3892     base($reg);
3893     index($lreg);
3894     scale($scale);
3895     disp($off);
3896   %}
3897 %}
3898 
3899 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3900 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3901 %{
3902   constraint(ALLOC_IN_RC(ptr_reg));
3903   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3904   match(AddP (AddP reg (ConvI2L idx)) off);
3905 
3906   op_cost(10);
3907   format %{"[$reg + $off + $idx]" %}
3908   interface(MEMORY_INTER) %{
3909     base($reg);
3910     index($idx);
3911     scale(0x0);
3912     disp($off);
3913   %}
3914 %}
3915 
3916 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3917 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3918 %{
3919   constraint(ALLOC_IN_RC(ptr_reg));
3920   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3921   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3922 
3923   op_cost(10);
3924   format %{"[$reg + $off + $idx << $scale]" %}
3925   interface(MEMORY_INTER) %{
3926     base($reg);
3927     index($idx);
3928     scale($scale);
3929     disp($off);
3930   %}
3931 %}
3932 
3933 // Indirect Narrow Oop Plus Offset Operand
3934 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3935 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3936 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3937   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3938   constraint(ALLOC_IN_RC(ptr_reg));
3939   match(AddP (DecodeN reg) off);
3940 
3941   op_cost(10);
3942   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3943   interface(MEMORY_INTER) %{
3944     base(0xc); // R12
3945     index($reg);
3946     scale(0x3);
3947     disp($off);
3948   %}
3949 %}
3950 
3951 // Indirect Memory Operand
3952 operand indirectNarrow(rRegN reg)
3953 %{
3954   predicate(Universe::narrow_oop_shift() == 0);
3955   constraint(ALLOC_IN_RC(ptr_reg));
3956   match(DecodeN reg);
3957 
3958   format %{ "[$reg]" %}
3959   interface(MEMORY_INTER) %{
3960     base($reg);
3961     index(0x4);
3962     scale(0x0);
3963     disp(0x0);
3964   %}
3965 %}
3966 
3967 // Indirect Memory Plus Short Offset Operand
3968 operand indOffset8Narrow(rRegN reg, immL8 off)
3969 %{
3970   predicate(Universe::narrow_oop_shift() == 0);
3971   constraint(ALLOC_IN_RC(ptr_reg));
3972   match(AddP (DecodeN reg) off);
3973 
3974   format %{ "[$reg + $off (8-bit)]" %}
3975   interface(MEMORY_INTER) %{
3976     base($reg);
3977     index(0x4);
3978     scale(0x0);
3979     disp($off);
3980   %}
3981 %}
3982 
3983 // Indirect Memory Plus Long Offset Operand
3984 operand indOffset32Narrow(rRegN reg, immL32 off)
3985 %{
3986   predicate(Universe::narrow_oop_shift() == 0);
3987   constraint(ALLOC_IN_RC(ptr_reg));
3988   match(AddP (DecodeN reg) off);
3989 
3990   format %{ "[$reg + $off (32-bit)]" %}
3991   interface(MEMORY_INTER) %{
3992     base($reg);
3993     index(0x4);
3994     scale(0x0);
3995     disp($off);
3996   %}
3997 %}
3998 
3999 // Indirect Memory Plus Index Register Plus Offset Operand
4000 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
4001 %{
4002   predicate(Universe::narrow_oop_shift() == 0);
4003   constraint(ALLOC_IN_RC(ptr_reg));
4004   match(AddP (AddP (DecodeN reg) lreg) off);
4005 
4006   op_cost(10);
4007   format %{"[$reg + $off + $lreg]" %}
4008   interface(MEMORY_INTER) %{
4009     base($reg);
4010     index($lreg);
4011     scale(0x0);
4012     disp($off);
4013   %}
4014 %}
4015 
4016 // Indirect Memory Plus Index Register Plus Offset Operand
4017 operand indIndexNarrow(rRegN reg, rRegL lreg)
4018 %{
4019   predicate(Universe::narrow_oop_shift() == 0);
4020   constraint(ALLOC_IN_RC(ptr_reg));
4021   match(AddP (DecodeN reg) lreg);
4022 
4023   op_cost(10);
4024   format %{"[$reg + $lreg]" %}
4025   interface(MEMORY_INTER) %{
4026     base($reg);
4027     index($lreg);
4028     scale(0x0);
4029     disp(0x0);
4030   %}
4031 %}
4032 
4033 // Indirect Memory Times Scale Plus Index Register
4034 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
4035 %{
4036   predicate(Universe::narrow_oop_shift() == 0);
4037   constraint(ALLOC_IN_RC(ptr_reg));
4038   match(AddP (DecodeN reg) (LShiftL lreg scale));
4039 
4040   op_cost(10);
4041   format %{"[$reg + $lreg << $scale]" %}
4042   interface(MEMORY_INTER) %{
4043     base($reg);
4044     index($lreg);
4045     scale($scale);
4046     disp(0x0);
4047   %}
4048 %}
4049 
4050 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
4051 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
4052 %{
4053   predicate(Universe::narrow_oop_shift() == 0);
4054   constraint(ALLOC_IN_RC(ptr_reg));
4055   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
4056 
4057   op_cost(10);
4058   format %{"[$reg + $off + $lreg << $scale]" %}
4059   interface(MEMORY_INTER) %{
4060     base($reg);
4061     index($lreg);
4062     scale($scale);
4063     disp($off);
4064   %}
4065 %}
4066 
4067 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
4068 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
4069 %{
4070   constraint(ALLOC_IN_RC(ptr_reg));
4071   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
4072   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
4073 
4074   op_cost(10);
4075   format %{"[$reg + $off + $idx]" %}
4076   interface(MEMORY_INTER) %{
4077     base($reg);
4078     index($idx);
4079     scale(0x0);
4080     disp($off);
4081   %}
4082 %}
4083 
4084 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
4085 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
4086 %{
4087   constraint(ALLOC_IN_RC(ptr_reg));
4088   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
4089   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
4090 
4091   op_cost(10);
4092   format %{"[$reg + $off + $idx << $scale]" %}
4093   interface(MEMORY_INTER) %{
4094     base($reg);
4095     index($idx);
4096     scale($scale);
4097     disp($off);
4098   %}
4099 %}
4100 
4101 //----------Special Memory Operands--------------------------------------------
4102 // Stack Slot Operand - This operand is used for loading and storing temporary
4103 //                      values on the stack where a match requires a value to
4104 //                      flow through memory.
4105 operand stackSlotP(sRegP reg)
4106 %{
4107   constraint(ALLOC_IN_RC(stack_slots));
4108   // No match rule because this operand is only generated in matching
4109 
4110   format %{ "[$reg]" %}
4111   interface(MEMORY_INTER) %{
4112     base(0x4);   // RSP
4113     index(0x4);  // No Index
4114     scale(0x0);  // No Scale
4115     disp($reg);  // Stack Offset
4116   %}
4117 %}
4118 
4119 operand stackSlotI(sRegI reg)
4120 %{
4121   constraint(ALLOC_IN_RC(stack_slots));
4122   // No match rule because this operand is only generated in matching
4123 
4124   format %{ "[$reg]" %}
4125   interface(MEMORY_INTER) %{
4126     base(0x4);   // RSP
4127     index(0x4);  // No Index
4128     scale(0x0);  // No Scale
4129     disp($reg);  // Stack Offset
4130   %}
4131 %}
4132 
4133 operand stackSlotF(sRegF reg)
4134 %{
4135   constraint(ALLOC_IN_RC(stack_slots));
4136   // No match rule because this operand is only generated in matching
4137 
4138   format %{ "[$reg]" %}
4139   interface(MEMORY_INTER) %{
4140     base(0x4);   // RSP
4141     index(0x4);  // No Index
4142     scale(0x0);  // No Scale
4143     disp($reg);  // Stack Offset
4144   %}
4145 %}
4146 
4147 operand stackSlotD(sRegD reg)
4148 %{
4149   constraint(ALLOC_IN_RC(stack_slots));
4150   // No match rule because this operand is only generated in matching
4151 
4152   format %{ "[$reg]" %}
4153   interface(MEMORY_INTER) %{
4154     base(0x4);   // RSP
4155     index(0x4);  // No Index
4156     scale(0x0);  // No Scale
4157     disp($reg);  // Stack Offset
4158   %}
4159 %}
4160 operand stackSlotL(sRegL reg)
4161 %{
4162   constraint(ALLOC_IN_RC(stack_slots));
4163   // No match rule because this operand is only generated in matching
4164 
4165   format %{ "[$reg]" %}
4166   interface(MEMORY_INTER) %{
4167     base(0x4);   // RSP
4168     index(0x4);  // No Index
4169     scale(0x0);  // No Scale
4170     disp($reg);  // Stack Offset
4171   %}
4172 %}
4173 
4174 //----------Conditional Branch Operands----------------------------------------
4175 // Comparison Op  - This is the operation of the comparison, and is limited to
4176 //                  the following set of codes:
4177 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4178 //
4179 // Other attributes of the comparison, such as unsignedness, are specified
4180 // by the comparison instruction that sets a condition code flags register.
4181 // That result is represented by a flags operand whose subtype is appropriate
4182 // to the unsignedness (etc.) of the comparison.
4183 //
4184 // Later, the instruction which matches both the Comparison Op (a Bool) and
4185 // the flags (produced by the Cmp) specifies the coding of the comparison op
4186 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4187 
4188 // Comparision Code
4189 operand cmpOp()
4190 %{
4191   match(Bool);
4192 
4193   format %{ "" %}
4194   interface(COND_INTER) %{
4195     equal(0x4, "e");
4196     not_equal(0x5, "ne");
4197     less(0xC, "l");
4198     greater_equal(0xD, "ge");
4199     less_equal(0xE, "le");
4200     greater(0xF, "g");
4201     overflow(0x0, "o");
4202     no_overflow(0x1, "no");
4203   %}
4204 %}
4205 
4206 // Comparison Code, unsigned compare.  Used by FP also, with
4207 // C2 (unordered) turned into GT or LT already.  The other bits
4208 // C0 and C3 are turned into Carry & Zero flags.
4209 operand cmpOpU()
4210 %{
4211   match(Bool);
4212 
4213   format %{ "" %}
4214   interface(COND_INTER) %{
4215     equal(0x4, "e");
4216     not_equal(0x5, "ne");
4217     less(0x2, "b");
4218     greater_equal(0x3, "nb");
4219     less_equal(0x6, "be");
4220     greater(0x7, "nbe");
4221     overflow(0x0, "o");
4222     no_overflow(0x1, "no");
4223   %}
4224 %}
4225 
4226 
4227 // Floating comparisons that don't require any fixup for the unordered case
4228 operand cmpOpUCF() %{
4229   match(Bool);
4230   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4231             n->as_Bool()->_test._test == BoolTest::ge ||
4232             n->as_Bool()->_test._test == BoolTest::le ||
4233             n->as_Bool()->_test._test == BoolTest::gt);
4234   format %{ "" %}
4235   interface(COND_INTER) %{
4236     equal(0x4, "e");
4237     not_equal(0x5, "ne");
4238     less(0x2, "b");
4239     greater_equal(0x3, "nb");
4240     less_equal(0x6, "be");
4241     greater(0x7, "nbe");
4242     overflow(0x0, "o");
4243     no_overflow(0x1, "no");
4244   %}
4245 %}
4246 
4247 
4248 // Floating comparisons that can be fixed up with extra conditional jumps
4249 operand cmpOpUCF2() %{
4250   match(Bool);
4251   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4252             n->as_Bool()->_test._test == BoolTest::eq);
4253   format %{ "" %}
4254   interface(COND_INTER) %{
4255     equal(0x4, "e");
4256     not_equal(0x5, "ne");
4257     less(0x2, "b");
4258     greater_equal(0x3, "nb");
4259     less_equal(0x6, "be");
4260     greater(0x7, "nbe");
4261     overflow(0x0, "o");
4262     no_overflow(0x1, "no");
4263   %}
4264 %}
4265 
4266 // Operands for bound floating pointer register arguments
4267 operand rxmm0() %{
4268   constraint(ALLOC_IN_RC(xmm0_reg));  match(VecX);
4269   predicate((UseSSE > 0) && (UseAVX<= 2));  format%{%}  interface(REG_INTER);
4270 %}
4271 operand rxmm1() %{
4272   constraint(ALLOC_IN_RC(xmm1_reg));  match(VecX);
4273   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4274 %}
4275 operand rxmm2() %{
4276   constraint(ALLOC_IN_RC(xmm2_reg));  match(VecX);
4277   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4278 %}
4279 operand rxmm3() %{
4280   constraint(ALLOC_IN_RC(xmm3_reg));  match(VecX);
4281   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4282 %}
4283 operand rxmm4() %{
4284   constraint(ALLOC_IN_RC(xmm4_reg));  match(VecX);
4285   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4286 %}
4287 operand rxmm5() %{
4288   constraint(ALLOC_IN_RC(xmm5_reg));  match(VecX);
4289   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4290 %}
4291 operand rxmm6() %{
4292   constraint(ALLOC_IN_RC(xmm6_reg));  match(VecX);
4293   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4294 %}
4295 operand rxmm7() %{
4296   constraint(ALLOC_IN_RC(xmm7_reg));  match(VecX);
4297   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4298 %}
4299 operand rxmm8() %{
4300   constraint(ALLOC_IN_RC(xmm8_reg));  match(VecX);
4301   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4302 %}
4303 operand rxmm9() %{
4304   constraint(ALLOC_IN_RC(xmm9_reg));  match(VecX);
4305   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4306 %}
4307 operand rxmm10() %{
4308   constraint(ALLOC_IN_RC(xmm10_reg));  match(VecX);
4309   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4310 %}
4311 operand rxmm11() %{
4312   constraint(ALLOC_IN_RC(xmm11_reg));  match(VecX);
4313   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4314 %}
4315 operand rxmm12() %{
4316   constraint(ALLOC_IN_RC(xmm12_reg));  match(VecX);
4317   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4318 %}
4319 operand rxmm13() %{
4320   constraint(ALLOC_IN_RC(xmm13_reg));  match(VecX);
4321   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4322 %}
4323 operand rxmm14() %{
4324   constraint(ALLOC_IN_RC(xmm14_reg));  match(VecX);
4325   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4326 %}
4327 operand rxmm15() %{
4328   constraint(ALLOC_IN_RC(xmm15_reg));  match(VecX);
4329   predicate((UseSSE > 0) && (UseAVX <= 2));  format%{%}  interface(REG_INTER);
4330 %}
4331 operand rxmm16() %{
4332   constraint(ALLOC_IN_RC(xmm16_reg));  match(VecX);
4333   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4334 %}
4335 operand rxmm17() %{
4336   constraint(ALLOC_IN_RC(xmm17_reg));  match(VecX);
4337   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4338 %}
4339 operand rxmm18() %{
4340   constraint(ALLOC_IN_RC(xmm18_reg));  match(VecX);
4341   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4342 %}
4343 operand rxmm19() %{
4344   constraint(ALLOC_IN_RC(xmm19_reg));  match(VecX);
4345   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4346 %}
4347 operand rxmm20() %{
4348   constraint(ALLOC_IN_RC(xmm20_reg));  match(VecX);
4349   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4350 %}
4351 operand rxmm21() %{
4352   constraint(ALLOC_IN_RC(xmm21_reg));  match(VecX);
4353   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4354 %}
4355 operand rxmm22() %{
4356   constraint(ALLOC_IN_RC(xmm22_reg));  match(VecX);
4357   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4358 %}
4359 operand rxmm23() %{
4360   constraint(ALLOC_IN_RC(xmm23_reg));  match(VecX);
4361   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4362 %}
4363 operand rxmm24() %{
4364   constraint(ALLOC_IN_RC(xmm24_reg));  match(VecX);
4365   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4366 %}
4367 operand rxmm25() %{
4368   constraint(ALLOC_IN_RC(xmm25_reg));  match(VecX);
4369   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4370 %}
4371 operand rxmm26() %{
4372   constraint(ALLOC_IN_RC(xmm26_reg));  match(VecX);
4373   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4374 %}
4375 operand rxmm27() %{
4376   constraint(ALLOC_IN_RC(xmm27_reg));  match(VecX);
4377   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4378 %}
4379 operand rxmm28() %{
4380   constraint(ALLOC_IN_RC(xmm28_reg));  match(VecX);
4381   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4382 %}
4383 operand rxmm29() %{
4384   constraint(ALLOC_IN_RC(xmm29_reg));  match(VecX);
4385   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4386 %}
4387 operand rxmm30() %{
4388   constraint(ALLOC_IN_RC(xmm30_reg));  match(VecX);
4389   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4390 %}
4391 operand rxmm31() %{
4392   constraint(ALLOC_IN_RC(xmm31_reg));  match(VecX);
4393   predicate(UseAVX == 3);  format%{%}  interface(REG_INTER);
4394 %}
4395 
4396 //----------OPERAND CLASSES----------------------------------------------------
4397 // Operand Classes are groups of operands that are used as to simplify
4398 // instruction definitions by not requiring the AD writer to specify separate
4399 // instructions for every form of operand when the instruction accepts
4400 // multiple operand types with the same basic encoding and format.  The classic
4401 // case of this is memory operands.
4402 
4403 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4404                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4405                indCompressedOopOffset,
4406                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4407                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4408                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4409 
4410 //----------PIPELINE-----------------------------------------------------------
4411 // Rules which define the behavior of the target architectures pipeline.
4412 pipeline %{
4413 
4414 //----------ATTRIBUTES---------------------------------------------------------
4415 attributes %{
4416   variable_size_instructions;        // Fixed size instructions
4417   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4418   instruction_unit_size = 1;         // An instruction is 1 bytes long
4419   instruction_fetch_unit_size = 16;  // The processor fetches one line
4420   instruction_fetch_units = 1;       // of 16 bytes
4421 
4422   // List of nop instructions
4423   nops( MachNop );
4424 %}
4425 
4426 //----------RESOURCES----------------------------------------------------------
4427 // Resources are the functional units available to the machine
4428 
4429 // Generic P2/P3 pipeline
4430 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4431 // 3 instructions decoded per cycle.
4432 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4433 // 3 ALU op, only ALU0 handles mul instructions.
4434 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4435            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4436            BR, FPU,
4437            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4438 
4439 //----------PIPELINE DESCRIPTION-----------------------------------------------
4440 // Pipeline Description specifies the stages in the machine's pipeline
4441 
4442 // Generic P2/P3 pipeline
4443 pipe_desc(S0, S1, S2, S3, S4, S5);
4444 
4445 //----------PIPELINE CLASSES---------------------------------------------------
4446 // Pipeline Classes describe the stages in which input and output are
4447 // referenced by the hardware pipeline.
4448 
4449 // Naming convention: ialu or fpu
4450 // Then: _reg
4451 // Then: _reg if there is a 2nd register
4452 // Then: _long if it's a pair of instructions implementing a long
4453 // Then: _fat if it requires the big decoder
4454 //   Or: _mem if it requires the big decoder and a memory unit.
4455 
4456 // Integer ALU reg operation
4457 pipe_class ialu_reg(rRegI dst)
4458 %{
4459     single_instruction;
4460     dst    : S4(write);
4461     dst    : S3(read);
4462     DECODE : S0;        // any decoder
4463     ALU    : S3;        // any alu
4464 %}
4465 
4466 // Long ALU reg operation
4467 pipe_class ialu_reg_long(rRegL dst)
4468 %{
4469     instruction_count(2);
4470     dst    : S4(write);
4471     dst    : S3(read);
4472     DECODE : S0(2);     // any 2 decoders
4473     ALU    : S3(2);     // both alus
4474 %}
4475 
4476 // Integer ALU reg operation using big decoder
4477 pipe_class ialu_reg_fat(rRegI dst)
4478 %{
4479     single_instruction;
4480     dst    : S4(write);
4481     dst    : S3(read);
4482     D0     : S0;        // big decoder only
4483     ALU    : S3;        // any alu
4484 %}
4485 
4486 // Long ALU reg operation using big decoder
4487 pipe_class ialu_reg_long_fat(rRegL dst)
4488 %{
4489     instruction_count(2);
4490     dst    : S4(write);
4491     dst    : S3(read);
4492     D0     : S0(2);     // big decoder only; twice
4493     ALU    : S3(2);     // any 2 alus
4494 %}
4495 
4496 // Integer ALU reg-reg operation
4497 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4498 %{
4499     single_instruction;
4500     dst    : S4(write);
4501     src    : S3(read);
4502     DECODE : S0;        // any decoder
4503     ALU    : S3;        // any alu
4504 %}
4505 
4506 // Long ALU reg-reg operation
4507 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4508 %{
4509     instruction_count(2);
4510     dst    : S4(write);
4511     src    : S3(read);
4512     DECODE : S0(2);     // any 2 decoders
4513     ALU    : S3(2);     // both alus
4514 %}
4515 
4516 // Integer ALU reg-reg operation
4517 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4518 %{
4519     single_instruction;
4520     dst    : S4(write);
4521     src    : S3(read);
4522     D0     : S0;        // big decoder only
4523     ALU    : S3;        // any alu
4524 %}
4525 
4526 // Long ALU reg-reg operation
4527 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4528 %{
4529     instruction_count(2);
4530     dst    : S4(write);
4531     src    : S3(read);
4532     D0     : S0(2);     // big decoder only; twice
4533     ALU    : S3(2);     // both alus
4534 %}
4535 
4536 // Integer ALU reg-mem operation
4537 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4538 %{
4539     single_instruction;
4540     dst    : S5(write);
4541     mem    : S3(read);
4542     D0     : S0;        // big decoder only
4543     ALU    : S4;        // any alu
4544     MEM    : S3;        // any mem
4545 %}
4546 
4547 // Integer mem operation (prefetch)
4548 pipe_class ialu_mem(memory mem)
4549 %{
4550     single_instruction;
4551     mem    : S3(read);
4552     D0     : S0;        // big decoder only
4553     MEM    : S3;        // any mem
4554 %}
4555 
4556 // Integer Store to Memory
4557 pipe_class ialu_mem_reg(memory mem, rRegI src)
4558 %{
4559     single_instruction;
4560     mem    : S3(read);
4561     src    : S5(read);
4562     D0     : S0;        // big decoder only
4563     ALU    : S4;        // any alu
4564     MEM    : S3;
4565 %}
4566 
4567 // // Long Store to Memory
4568 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4569 // %{
4570 //     instruction_count(2);
4571 //     mem    : S3(read);
4572 //     src    : S5(read);
4573 //     D0     : S0(2);          // big decoder only; twice
4574 //     ALU    : S4(2);     // any 2 alus
4575 //     MEM    : S3(2);  // Both mems
4576 // %}
4577 
4578 // Integer Store to Memory
4579 pipe_class ialu_mem_imm(memory mem)
4580 %{
4581     single_instruction;
4582     mem    : S3(read);
4583     D0     : S0;        // big decoder only
4584     ALU    : S4;        // any alu
4585     MEM    : S3;
4586 %}
4587 
4588 // Integer ALU0 reg-reg operation
4589 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4590 %{
4591     single_instruction;
4592     dst    : S4(write);
4593     src    : S3(read);
4594     D0     : S0;        // Big decoder only
4595     ALU0   : S3;        // only alu0
4596 %}
4597 
4598 // Integer ALU0 reg-mem operation
4599 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4600 %{
4601     single_instruction;
4602     dst    : S5(write);
4603     mem    : S3(read);
4604     D0     : S0;        // big decoder only
4605     ALU0   : S4;        // ALU0 only
4606     MEM    : S3;        // any mem
4607 %}
4608 
4609 // Integer ALU reg-reg operation
4610 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4611 %{
4612     single_instruction;
4613     cr     : S4(write);
4614     src1   : S3(read);
4615     src2   : S3(read);
4616     DECODE : S0;        // any decoder
4617     ALU    : S3;        // any alu
4618 %}
4619 
4620 // Integer ALU reg-imm operation
4621 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4622 %{
4623     single_instruction;
4624     cr     : S4(write);
4625     src1   : S3(read);
4626     DECODE : S0;        // any decoder
4627     ALU    : S3;        // any alu
4628 %}
4629 
4630 // Integer ALU reg-mem operation
4631 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4632 %{
4633     single_instruction;
4634     cr     : S4(write);
4635     src1   : S3(read);
4636     src2   : S3(read);
4637     D0     : S0;        // big decoder only
4638     ALU    : S4;        // any alu
4639     MEM    : S3;
4640 %}
4641 
4642 // Conditional move reg-reg
4643 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4644 %{
4645     instruction_count(4);
4646     y      : S4(read);
4647     q      : S3(read);
4648     p      : S3(read);
4649     DECODE : S0(4);     // any decoder
4650 %}
4651 
4652 // Conditional move reg-reg
4653 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4654 %{
4655     single_instruction;
4656     dst    : S4(write);
4657     src    : S3(read);
4658     cr     : S3(read);
4659     DECODE : S0;        // any decoder
4660 %}
4661 
4662 // Conditional move reg-mem
4663 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4664 %{
4665     single_instruction;
4666     dst    : S4(write);
4667     src    : S3(read);
4668     cr     : S3(read);
4669     DECODE : S0;        // any decoder
4670     MEM    : S3;
4671 %}
4672 
4673 // Conditional move reg-reg long
4674 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4675 %{
4676     single_instruction;
4677     dst    : S4(write);
4678     src    : S3(read);
4679     cr     : S3(read);
4680     DECODE : S0(2);     // any 2 decoders
4681 %}
4682 
4683 // XXX
4684 // // Conditional move double reg-reg
4685 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4686 // %{
4687 //     single_instruction;
4688 //     dst    : S4(write);
4689 //     src    : S3(read);
4690 //     cr     : S3(read);
4691 //     DECODE : S0;     // any decoder
4692 // %}
4693 
4694 // Float reg-reg operation
4695 pipe_class fpu_reg(regD dst)
4696 %{
4697     instruction_count(2);
4698     dst    : S3(read);
4699     DECODE : S0(2);     // any 2 decoders
4700     FPU    : S3;
4701 %}
4702 
4703 // Float reg-reg operation
4704 pipe_class fpu_reg_reg(regD dst, regD src)
4705 %{
4706     instruction_count(2);
4707     dst    : S4(write);
4708     src    : S3(read);
4709     DECODE : S0(2);     // any 2 decoders
4710     FPU    : S3;
4711 %}
4712 
4713 // Float reg-reg operation
4714 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4715 %{
4716     instruction_count(3);
4717     dst    : S4(write);
4718     src1   : S3(read);
4719     src2   : S3(read);
4720     DECODE : S0(3);     // any 3 decoders
4721     FPU    : S3(2);
4722 %}
4723 
4724 // Float reg-reg operation
4725 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4726 %{
4727     instruction_count(4);
4728     dst    : S4(write);
4729     src1   : S3(read);
4730     src2   : S3(read);
4731     src3   : S3(read);
4732     DECODE : S0(4);     // any 3 decoders
4733     FPU    : S3(2);
4734 %}
4735 
4736 // Float reg-reg operation
4737 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4738 %{
4739     instruction_count(4);
4740     dst    : S4(write);
4741     src1   : S3(read);
4742     src2   : S3(read);
4743     src3   : S3(read);
4744     DECODE : S1(3);     // any 3 decoders
4745     D0     : S0;        // Big decoder only
4746     FPU    : S3(2);
4747     MEM    : S3;
4748 %}
4749 
4750 // Float reg-mem operation
4751 pipe_class fpu_reg_mem(regD dst, memory mem)
4752 %{
4753     instruction_count(2);
4754     dst    : S5(write);
4755     mem    : S3(read);
4756     D0     : S0;        // big decoder only
4757     DECODE : S1;        // any decoder for FPU POP
4758     FPU    : S4;
4759     MEM    : S3;        // any mem
4760 %}
4761 
4762 // Float reg-mem operation
4763 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4764 %{
4765     instruction_count(3);
4766     dst    : S5(write);
4767     src1   : S3(read);
4768     mem    : S3(read);
4769     D0     : S0;        // big decoder only
4770     DECODE : S1(2);     // any decoder for FPU POP
4771     FPU    : S4;
4772     MEM    : S3;        // any mem
4773 %}
4774 
4775 // Float mem-reg operation
4776 pipe_class fpu_mem_reg(memory mem, regD src)
4777 %{
4778     instruction_count(2);
4779     src    : S5(read);
4780     mem    : S3(read);
4781     DECODE : S0;        // any decoder for FPU PUSH
4782     D0     : S1;        // big decoder only
4783     FPU    : S4;
4784     MEM    : S3;        // any mem
4785 %}
4786 
4787 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4788 %{
4789     instruction_count(3);
4790     src1   : S3(read);
4791     src2   : S3(read);
4792     mem    : S3(read);
4793     DECODE : S0(2);     // any decoder for FPU PUSH
4794     D0     : S1;        // big decoder only
4795     FPU    : S4;
4796     MEM    : S3;        // any mem
4797 %}
4798 
4799 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4800 %{
4801     instruction_count(3);
4802     src1   : S3(read);
4803     src2   : S3(read);
4804     mem    : S4(read);
4805     DECODE : S0;        // any decoder for FPU PUSH
4806     D0     : S0(2);     // big decoder only
4807     FPU    : S4;
4808     MEM    : S3(2);     // any mem
4809 %}
4810 
4811 pipe_class fpu_mem_mem(memory dst, memory src1)
4812 %{
4813     instruction_count(2);
4814     src1   : S3(read);
4815     dst    : S4(read);
4816     D0     : S0(2);     // big decoder only
4817     MEM    : S3(2);     // any mem
4818 %}
4819 
4820 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4821 %{
4822     instruction_count(3);
4823     src1   : S3(read);
4824     src2   : S3(read);
4825     dst    : S4(read);
4826     D0     : S0(3);     // big decoder only
4827     FPU    : S4;
4828     MEM    : S3(3);     // any mem
4829 %}
4830 
4831 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4832 %{
4833     instruction_count(3);
4834     src1   : S4(read);
4835     mem    : S4(read);
4836     DECODE : S0;        // any decoder for FPU PUSH
4837     D0     : S0(2);     // big decoder only
4838     FPU    : S4;
4839     MEM    : S3(2);     // any mem
4840 %}
4841 
4842 // Float load constant
4843 pipe_class fpu_reg_con(regD dst)
4844 %{
4845     instruction_count(2);
4846     dst    : S5(write);
4847     D0     : S0;        // big decoder only for the load
4848     DECODE : S1;        // any decoder for FPU POP
4849     FPU    : S4;
4850     MEM    : S3;        // any mem
4851 %}
4852 
4853 // Float load constant
4854 pipe_class fpu_reg_reg_con(regD dst, regD src)
4855 %{
4856     instruction_count(3);
4857     dst    : S5(write);
4858     src    : S3(read);
4859     D0     : S0;        // big decoder only for the load
4860     DECODE : S1(2);     // any decoder for FPU POP
4861     FPU    : S4;
4862     MEM    : S3;        // any mem
4863 %}
4864 
4865 // UnConditional branch
4866 pipe_class pipe_jmp(label labl)
4867 %{
4868     single_instruction;
4869     BR   : S3;
4870 %}
4871 
4872 // Conditional branch
4873 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4874 %{
4875     single_instruction;
4876     cr    : S1(read);
4877     BR    : S3;
4878 %}
4879 
4880 // Allocation idiom
4881 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4882 %{
4883     instruction_count(1); force_serialization;
4884     fixed_latency(6);
4885     heap_ptr : S3(read);
4886     DECODE   : S0(3);
4887     D0       : S2;
4888     MEM      : S3;
4889     ALU      : S3(2);
4890     dst      : S5(write);
4891     BR       : S5;
4892 %}
4893 
4894 // Generic big/slow expanded idiom
4895 pipe_class pipe_slow()
4896 %{
4897     instruction_count(10); multiple_bundles; force_serialization;
4898     fixed_latency(100);
4899     D0  : S0(2);
4900     MEM : S3(2);
4901 %}
4902 
4903 // The real do-nothing guy
4904 pipe_class empty()
4905 %{
4906     instruction_count(0);
4907 %}
4908 
4909 // Define the class for the Nop node
4910 define
4911 %{
4912    MachNop = empty;
4913 %}
4914 
4915 %}
4916 
4917 //----------INSTRUCTIONS-------------------------------------------------------
4918 //
4919 // match      -- States which machine-independent subtree may be replaced
4920 //               by this instruction.
4921 // ins_cost   -- The estimated cost of this instruction is used by instruction
4922 //               selection to identify a minimum cost tree of machine
4923 //               instructions that matches a tree of machine-independent
4924 //               instructions.
4925 // format     -- A string providing the disassembly for this instruction.
4926 //               The value of an instruction's operand may be inserted
4927 //               by referring to it with a '$' prefix.
4928 // opcode     -- Three instruction opcodes may be provided.  These are referred
4929 //               to within an encode class as $primary, $secondary, and $tertiary
4930 //               rrspectively.  The primary opcode is commonly used to
4931 //               indicate the type of machine instruction, while secondary
4932 //               and tertiary are often used for prefix options or addressing
4933 //               modes.
4934 // ins_encode -- A list of encode classes with parameters. The encode class
4935 //               name must have been defined in an 'enc_class' specification
4936 //               in the encode section of the architecture description.
4937 
4938 
4939 //----------Load/Store/Move Instructions---------------------------------------
4940 //----------Load Instructions--------------------------------------------------
4941 
4942 // Load Byte (8 bit signed)
4943 instruct loadB(rRegI dst, memory mem)
4944 %{
4945   match(Set dst (LoadB mem));
4946 
4947   ins_cost(125);
4948   format %{ "movsbl  $dst, $mem\t# byte" %}
4949 
4950   ins_encode %{
4951     __ movsbl($dst$$Register, $mem$$Address);
4952   %}
4953 
4954   ins_pipe(ialu_reg_mem);
4955 %}
4956 
4957 // Load Byte (8 bit signed) into Long Register
4958 instruct loadB2L(rRegL dst, memory mem)
4959 %{
4960   match(Set dst (ConvI2L (LoadB mem)));
4961 
4962   ins_cost(125);
4963   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4964 
4965   ins_encode %{
4966     __ movsbq($dst$$Register, $mem$$Address);
4967   %}
4968 
4969   ins_pipe(ialu_reg_mem);
4970 %}
4971 
4972 // Load Unsigned Byte (8 bit UNsigned)
4973 instruct loadUB(rRegI dst, memory mem)
4974 %{
4975   match(Set dst (LoadUB mem));
4976 
4977   ins_cost(125);
4978   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4979 
4980   ins_encode %{
4981     __ movzbl($dst$$Register, $mem$$Address);
4982   %}
4983 
4984   ins_pipe(ialu_reg_mem);
4985 %}
4986 
4987 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4988 instruct loadUB2L(rRegL dst, memory mem)
4989 %{
4990   match(Set dst (ConvI2L (LoadUB mem)));
4991 
4992   ins_cost(125);
4993   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4994 
4995   ins_encode %{
4996     __ movzbq($dst$$Register, $mem$$Address);
4997   %}
4998 
4999   ins_pipe(ialu_reg_mem);
5000 %}
5001 
5002 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
5003 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5004   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
5005   effect(KILL cr);
5006 
5007   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
5008             "andl    $dst, right_n_bits($mask, 8)" %}
5009   ins_encode %{
5010     Register Rdst = $dst$$Register;
5011     __ movzbq(Rdst, $mem$$Address);
5012     __ andl(Rdst, $mask$$constant & right_n_bits(8));
5013   %}
5014   ins_pipe(ialu_reg_mem);
5015 %}
5016 
5017 // Load Short (16 bit signed)
5018 instruct loadS(rRegI dst, memory mem)
5019 %{
5020   match(Set dst (LoadS mem));
5021 
5022   ins_cost(125);
5023   format %{ "movswl $dst, $mem\t# short" %}
5024 
5025   ins_encode %{
5026     __ movswl($dst$$Register, $mem$$Address);
5027   %}
5028 
5029   ins_pipe(ialu_reg_mem);
5030 %}
5031 
5032 // Load Short (16 bit signed) to Byte (8 bit signed)
5033 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5034   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
5035 
5036   ins_cost(125);
5037   format %{ "movsbl $dst, $mem\t# short -> byte" %}
5038   ins_encode %{
5039     __ movsbl($dst$$Register, $mem$$Address);
5040   %}
5041   ins_pipe(ialu_reg_mem);
5042 %}
5043 
5044 // Load Short (16 bit signed) into Long Register
5045 instruct loadS2L(rRegL dst, memory mem)
5046 %{
5047   match(Set dst (ConvI2L (LoadS mem)));
5048 
5049   ins_cost(125);
5050   format %{ "movswq $dst, $mem\t# short -> long" %}
5051 
5052   ins_encode %{
5053     __ movswq($dst$$Register, $mem$$Address);
5054   %}
5055 
5056   ins_pipe(ialu_reg_mem);
5057 %}
5058 
5059 // Load Unsigned Short/Char (16 bit UNsigned)
5060 instruct loadUS(rRegI dst, memory mem)
5061 %{
5062   match(Set dst (LoadUS mem));
5063 
5064   ins_cost(125);
5065   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
5066 
5067   ins_encode %{
5068     __ movzwl($dst$$Register, $mem$$Address);
5069   %}
5070 
5071   ins_pipe(ialu_reg_mem);
5072 %}
5073 
5074 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
5075 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5076   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
5077 
5078   ins_cost(125);
5079   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
5080   ins_encode %{
5081     __ movsbl($dst$$Register, $mem$$Address);
5082   %}
5083   ins_pipe(ialu_reg_mem);
5084 %}
5085 
5086 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
5087 instruct loadUS2L(rRegL dst, memory mem)
5088 %{
5089   match(Set dst (ConvI2L (LoadUS mem)));
5090 
5091   ins_cost(125);
5092   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
5093 
5094   ins_encode %{
5095     __ movzwq($dst$$Register, $mem$$Address);
5096   %}
5097 
5098   ins_pipe(ialu_reg_mem);
5099 %}
5100 
5101 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
5102 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5103   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5104 
5105   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
5106   ins_encode %{
5107     __ movzbq($dst$$Register, $mem$$Address);
5108   %}
5109   ins_pipe(ialu_reg_mem);
5110 %}
5111 
5112 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
5113 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
5114   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
5115   effect(KILL cr);
5116 
5117   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
5118             "andl    $dst, right_n_bits($mask, 16)" %}
5119   ins_encode %{
5120     Register Rdst = $dst$$Register;
5121     __ movzwq(Rdst, $mem$$Address);
5122     __ andl(Rdst, $mask$$constant & right_n_bits(16));
5123   %}
5124   ins_pipe(ialu_reg_mem);
5125 %}
5126 
5127 // Load Integer
5128 instruct loadI(rRegI dst, memory mem)
5129 %{
5130   match(Set dst (LoadI mem));
5131 
5132   ins_cost(125);
5133   format %{ "movl    $dst, $mem\t# int" %}
5134 
5135   ins_encode %{
5136     __ movl($dst$$Register, $mem$$Address);
5137   %}
5138 
5139   ins_pipe(ialu_reg_mem);
5140 %}
5141 
5142 // Load Integer (32 bit signed) to Byte (8 bit signed)
5143 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
5144   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
5145 
5146   ins_cost(125);
5147   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
5148   ins_encode %{
5149     __ movsbl($dst$$Register, $mem$$Address);
5150   %}
5151   ins_pipe(ialu_reg_mem);
5152 %}
5153 
5154 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
5155 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
5156   match(Set dst (AndI (LoadI mem) mask));
5157 
5158   ins_cost(125);
5159   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
5160   ins_encode %{
5161     __ movzbl($dst$$Register, $mem$$Address);
5162   %}
5163   ins_pipe(ialu_reg_mem);
5164 %}
5165 
5166 // Load Integer (32 bit signed) to Short (16 bit signed)
5167 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
5168   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
5169 
5170   ins_cost(125);
5171   format %{ "movswl  $dst, $mem\t# int -> short" %}
5172   ins_encode %{
5173     __ movswl($dst$$Register, $mem$$Address);
5174   %}
5175   ins_pipe(ialu_reg_mem);
5176 %}
5177 
5178 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
5179 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
5180   match(Set dst (AndI (LoadI mem) mask));
5181 
5182   ins_cost(125);
5183   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
5184   ins_encode %{
5185     __ movzwl($dst$$Register, $mem$$Address);
5186   %}
5187   ins_pipe(ialu_reg_mem);
5188 %}
5189 
5190 // Load Integer into Long Register
5191 instruct loadI2L(rRegL dst, memory mem)
5192 %{
5193   match(Set dst (ConvI2L (LoadI mem)));
5194 
5195   ins_cost(125);
5196   format %{ "movslq  $dst, $mem\t# int -> long" %}
5197 
5198   ins_encode %{
5199     __ movslq($dst$$Register, $mem$$Address);
5200   %}
5201 
5202   ins_pipe(ialu_reg_mem);
5203 %}
5204 
5205 // Load Integer with mask 0xFF into Long Register
5206 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
5207   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5208 
5209   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
5210   ins_encode %{
5211     __ movzbq($dst$$Register, $mem$$Address);
5212   %}
5213   ins_pipe(ialu_reg_mem);
5214 %}
5215 
5216 // Load Integer with mask 0xFFFF into Long Register
5217 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
5218   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5219 
5220   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5221   ins_encode %{
5222     __ movzwq($dst$$Register, $mem$$Address);
5223   %}
5224   ins_pipe(ialu_reg_mem);
5225 %}
5226 
5227 // Load Integer with a 31-bit mask into Long Register
5228 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5229   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5230   effect(KILL cr);
5231 
5232   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5233             "andl    $dst, $mask" %}
5234   ins_encode %{
5235     Register Rdst = $dst$$Register;
5236     __ movl(Rdst, $mem$$Address);
5237     __ andl(Rdst, $mask$$constant);
5238   %}
5239   ins_pipe(ialu_reg_mem);
5240 %}
5241 
5242 // Load Unsigned Integer into Long Register
5243 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5244 %{
5245   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5246 
5247   ins_cost(125);
5248   format %{ "movl    $dst, $mem\t# uint -> long" %}
5249 
5250   ins_encode %{
5251     __ movl($dst$$Register, $mem$$Address);
5252   %}
5253 
5254   ins_pipe(ialu_reg_mem);
5255 %}
5256 
5257 // Load Long
5258 instruct loadL(rRegL dst, memory mem)
5259 %{
5260   match(Set dst (LoadL mem));
5261 
5262   ins_cost(125);
5263   format %{ "movq    $dst, $mem\t# long" %}
5264 
5265   ins_encode %{
5266     __ movq($dst$$Register, $mem$$Address);
5267   %}
5268 
5269   ins_pipe(ialu_reg_mem); // XXX
5270 %}
5271 
5272 // Load Range
5273 instruct loadRange(rRegI dst, memory mem)
5274 %{
5275   match(Set dst (LoadRange mem));
5276 
5277   ins_cost(125); // XXX
5278   format %{ "movl    $dst, $mem\t# range" %}
5279   opcode(0x8B);
5280   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5281   ins_pipe(ialu_reg_mem);
5282 %}
5283 
5284 // Load Pointer
5285 instruct loadP(rRegP dst, memory mem)
5286 %{
5287   match(Set dst (LoadP mem));
5288 
5289   ins_cost(125); // XXX
5290   format %{ "movq    $dst, $mem\t# ptr" %}
5291   opcode(0x8B);
5292   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5293   ins_pipe(ialu_reg_mem); // XXX
5294 %}
5295 
5296 // Load Compressed Pointer
5297 instruct loadN(rRegN dst, memory mem)
5298 %{
5299    match(Set dst (LoadN mem));
5300 
5301    ins_cost(125); // XXX
5302    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5303    ins_encode %{
5304      __ movl($dst$$Register, $mem$$Address);
5305    %}
5306    ins_pipe(ialu_reg_mem); // XXX
5307 %}
5308 
5309 
5310 // Load Klass Pointer
5311 instruct loadKlass(rRegP dst, memory mem)
5312 %{
5313   match(Set dst (LoadKlass mem));
5314 
5315   ins_cost(125); // XXX
5316   format %{ "movq    $dst, $mem\t# class" %}
5317   opcode(0x8B);
5318   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5319   ins_pipe(ialu_reg_mem); // XXX
5320 %}
5321 
5322 // Load narrow Klass Pointer
5323 instruct loadNKlass(rRegN dst, memory mem)
5324 %{
5325   match(Set dst (LoadNKlass mem));
5326 
5327   ins_cost(125); // XXX
5328   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5329   ins_encode %{
5330     __ movl($dst$$Register, $mem$$Address);
5331   %}
5332   ins_pipe(ialu_reg_mem); // XXX
5333 %}
5334 
5335 // Load Float
5336 instruct loadF(regF dst, memory mem)
5337 %{
5338   match(Set dst (LoadF mem));
5339 
5340   ins_cost(145); // XXX
5341   format %{ "movss   $dst, $mem\t# float" %}
5342   ins_encode %{
5343     __ movflt($dst$$XMMRegister, $mem$$Address);
5344   %}
5345   ins_pipe(pipe_slow); // XXX
5346 %}
5347 
5348 // Load Float
5349 instruct MoveF2VL(vlRegF dst, regF src) %{
5350   match(Set dst src);
5351   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5352   ins_encode %{
5353     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5354   %}
5355   ins_pipe( fpu_reg_reg );
5356 %}
5357 
5358 // Load Float
5359 instruct MoveVL2F(regF dst, vlRegF src) %{
5360   match(Set dst src);
5361   format %{ "movss $dst,$src\t! load float (4 bytes)" %}
5362   ins_encode %{
5363     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
5364   %}
5365   ins_pipe( fpu_reg_reg );
5366 %}
5367 
5368 // Load Double
5369 instruct loadD_partial(regD dst, memory mem)
5370 %{
5371   predicate(!UseXmmLoadAndClearUpper);
5372   match(Set dst (LoadD mem));
5373 
5374   ins_cost(145); // XXX
5375   format %{ "movlpd  $dst, $mem\t# double" %}
5376   ins_encode %{
5377     __ movdbl($dst$$XMMRegister, $mem$$Address);
5378   %}
5379   ins_pipe(pipe_slow); // XXX
5380 %}
5381 
5382 instruct loadD(regD dst, memory mem)
5383 %{
5384   predicate(UseXmmLoadAndClearUpper);
5385   match(Set dst (LoadD mem));
5386 
5387   ins_cost(145); // XXX
5388   format %{ "movsd   $dst, $mem\t# double" %}
5389   ins_encode %{
5390     __ movdbl($dst$$XMMRegister, $mem$$Address);
5391   %}
5392   ins_pipe(pipe_slow); // XXX
5393 %}
5394 
5395 // Load Double
5396 instruct MoveD2VL(vlRegD dst, regD src) %{
5397   match(Set dst src);
5398   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5399   ins_encode %{
5400     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5401   %}
5402   ins_pipe( fpu_reg_reg );
5403 %}
5404 
5405 // Load Double
5406 instruct MoveVL2D(regD dst, vlRegD src) %{
5407   match(Set dst src);
5408   format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
5409   ins_encode %{
5410     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
5411   %}
5412   ins_pipe( fpu_reg_reg );
5413 %}
5414 
5415 // Load Effective Address
5416 instruct leaP8(rRegP dst, indOffset8 mem)
5417 %{
5418   match(Set dst mem);
5419 
5420   ins_cost(110); // XXX
5421   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5422   opcode(0x8D);
5423   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5424   ins_pipe(ialu_reg_reg_fat);
5425 %}
5426 
5427 instruct leaP32(rRegP dst, indOffset32 mem)
5428 %{
5429   match(Set dst mem);
5430 
5431   ins_cost(110);
5432   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5433   opcode(0x8D);
5434   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5435   ins_pipe(ialu_reg_reg_fat);
5436 %}
5437 
5438 // instruct leaPIdx(rRegP dst, indIndex mem)
5439 // %{
5440 //   match(Set dst mem);
5441 
5442 //   ins_cost(110);
5443 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5444 //   opcode(0x8D);
5445 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5446 //   ins_pipe(ialu_reg_reg_fat);
5447 // %}
5448 
5449 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5450 %{
5451   match(Set dst mem);
5452 
5453   ins_cost(110);
5454   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5455   opcode(0x8D);
5456   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5457   ins_pipe(ialu_reg_reg_fat);
5458 %}
5459 
5460 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5461 %{
5462   match(Set dst mem);
5463 
5464   ins_cost(110);
5465   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5466   opcode(0x8D);
5467   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5468   ins_pipe(ialu_reg_reg_fat);
5469 %}
5470 
5471 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5472 %{
5473   match(Set dst mem);
5474 
5475   ins_cost(110);
5476   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5477   opcode(0x8D);
5478   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5479   ins_pipe(ialu_reg_reg_fat);
5480 %}
5481 
5482 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5483 %{
5484   match(Set dst mem);
5485 
5486   ins_cost(110);
5487   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5488   opcode(0x8D);
5489   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5490   ins_pipe(ialu_reg_reg_fat);
5491 %}
5492 
5493 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5494 %{
5495   match(Set dst mem);
5496 
5497   ins_cost(110);
5498   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5499   opcode(0x8D);
5500   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5501   ins_pipe(ialu_reg_reg_fat);
5502 %}
5503 
5504 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5505 %{
5506   match(Set dst mem);
5507 
5508   ins_cost(110);
5509   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5510   opcode(0x8D);
5511   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5512   ins_pipe(ialu_reg_reg_fat);
5513 %}
5514 
5515 // Load Effective Address which uses Narrow (32-bits) oop
5516 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5517 %{
5518   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5519   match(Set dst mem);
5520 
5521   ins_cost(110);
5522   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5523   opcode(0x8D);
5524   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5525   ins_pipe(ialu_reg_reg_fat);
5526 %}
5527 
5528 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5529 %{
5530   predicate(Universe::narrow_oop_shift() == 0);
5531   match(Set dst mem);
5532 
5533   ins_cost(110); // XXX
5534   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5535   opcode(0x8D);
5536   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5537   ins_pipe(ialu_reg_reg_fat);
5538 %}
5539 
5540 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5541 %{
5542   predicate(Universe::narrow_oop_shift() == 0);
5543   match(Set dst mem);
5544 
5545   ins_cost(110);
5546   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5547   opcode(0x8D);
5548   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5549   ins_pipe(ialu_reg_reg_fat);
5550 %}
5551 
5552 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5553 %{
5554   predicate(Universe::narrow_oop_shift() == 0);
5555   match(Set dst mem);
5556 
5557   ins_cost(110);
5558   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5559   opcode(0x8D);
5560   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5561   ins_pipe(ialu_reg_reg_fat);
5562 %}
5563 
5564 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5565 %{
5566   predicate(Universe::narrow_oop_shift() == 0);
5567   match(Set dst mem);
5568 
5569   ins_cost(110);
5570   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5571   opcode(0x8D);
5572   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5573   ins_pipe(ialu_reg_reg_fat);
5574 %}
5575 
5576 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5577 %{
5578   predicate(Universe::narrow_oop_shift() == 0);
5579   match(Set dst mem);
5580 
5581   ins_cost(110);
5582   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5583   opcode(0x8D);
5584   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5585   ins_pipe(ialu_reg_reg_fat);
5586 %}
5587 
5588 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5589 %{
5590   predicate(Universe::narrow_oop_shift() == 0);
5591   match(Set dst mem);
5592 
5593   ins_cost(110);
5594   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5595   opcode(0x8D);
5596   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5597   ins_pipe(ialu_reg_reg_fat);
5598 %}
5599 
5600 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5601 %{
5602   predicate(Universe::narrow_oop_shift() == 0);
5603   match(Set dst mem);
5604 
5605   ins_cost(110);
5606   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5607   opcode(0x8D);
5608   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5609   ins_pipe(ialu_reg_reg_fat);
5610 %}
5611 
5612 instruct loadConI(rRegI dst, immI src)
5613 %{
5614   match(Set dst src);
5615 
5616   format %{ "movl    $dst, $src\t# int" %}
5617   ins_encode(load_immI(dst, src));
5618   ins_pipe(ialu_reg_fat); // XXX
5619 %}
5620 
5621 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5622 %{
5623   match(Set dst src);
5624   effect(KILL cr);
5625 
5626   ins_cost(50);
5627   format %{ "xorl    $dst, $dst\t# int" %}
5628   opcode(0x33); /* + rd */
5629   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5630   ins_pipe(ialu_reg);
5631 %}
5632 
5633 instruct loadConL(rRegL dst, immL src)
5634 %{
5635   match(Set dst src);
5636 
5637   ins_cost(150);
5638   format %{ "movq    $dst, $src\t# long" %}
5639   ins_encode(load_immL(dst, src));
5640   ins_pipe(ialu_reg);
5641 %}
5642 
5643 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5644 %{
5645   match(Set dst src);
5646   effect(KILL cr);
5647 
5648   ins_cost(50);
5649   format %{ "xorl    $dst, $dst\t# long" %}
5650   opcode(0x33); /* + rd */
5651   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5652   ins_pipe(ialu_reg); // XXX
5653 %}
5654 
5655 instruct loadConUL32(rRegL dst, immUL32 src)
5656 %{
5657   match(Set dst src);
5658 
5659   ins_cost(60);
5660   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5661   ins_encode(load_immUL32(dst, src));
5662   ins_pipe(ialu_reg);
5663 %}
5664 
5665 instruct loadConL32(rRegL dst, immL32 src)
5666 %{
5667   match(Set dst src);
5668 
5669   ins_cost(70);
5670   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5671   ins_encode(load_immL32(dst, src));
5672   ins_pipe(ialu_reg);
5673 %}
5674 
5675 instruct loadConP(rRegP dst, immP con) %{
5676   match(Set dst con);
5677 
5678   format %{ "movq    $dst, $con\t# ptr" %}
5679   ins_encode(load_immP(dst, con));
5680   ins_pipe(ialu_reg_fat); // XXX
5681 %}
5682 
5683 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5684 %{
5685   match(Set dst src);
5686   effect(KILL cr);
5687 
5688   ins_cost(50);
5689   format %{ "xorl    $dst, $dst\t# ptr" %}
5690   opcode(0x33); /* + rd */
5691   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5692   ins_pipe(ialu_reg);
5693 %}
5694 
5695 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5696 %{
5697   match(Set dst src);
5698   effect(KILL cr);
5699 
5700   ins_cost(60);
5701   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5702   ins_encode(load_immP31(dst, src));
5703   ins_pipe(ialu_reg);
5704 %}
5705 
5706 instruct loadConF(regF dst, immF con) %{
5707   match(Set dst con);
5708   ins_cost(125);
5709   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5710   ins_encode %{
5711     __ movflt($dst$$XMMRegister, $constantaddress($con));
5712   %}
5713   ins_pipe(pipe_slow);
5714 %}
5715 
5716 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5717   match(Set dst src);
5718   effect(KILL cr);
5719   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5720   ins_encode %{
5721     __ xorq($dst$$Register, $dst$$Register);
5722   %}
5723   ins_pipe(ialu_reg);
5724 %}
5725 
5726 instruct loadConN(rRegN dst, immN src) %{
5727   match(Set dst src);
5728 
5729   ins_cost(125);
5730   format %{ "movl    $dst, $src\t# compressed ptr" %}
5731   ins_encode %{
5732     address con = (address)$src$$constant;
5733     if (con == NULL) {
5734       ShouldNotReachHere();
5735     } else {
5736       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5737     }
5738   %}
5739   ins_pipe(ialu_reg_fat); // XXX
5740 %}
5741 
5742 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5743   match(Set dst src);
5744 
5745   ins_cost(125);
5746   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5747   ins_encode %{
5748     address con = (address)$src$$constant;
5749     if (con == NULL) {
5750       ShouldNotReachHere();
5751     } else {
5752       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5753     }
5754   %}
5755   ins_pipe(ialu_reg_fat); // XXX
5756 %}
5757 
5758 instruct loadConF0(regF dst, immF0 src)
5759 %{
5760   match(Set dst src);
5761   ins_cost(100);
5762 
5763   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5764   ins_encode %{
5765     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5766   %}
5767   ins_pipe(pipe_slow);
5768 %}
5769 
5770 // Use the same format since predicate() can not be used here.
5771 instruct loadConD(regD dst, immD con) %{
5772   match(Set dst con);
5773   ins_cost(125);
5774   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5775   ins_encode %{
5776     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5777   %}
5778   ins_pipe(pipe_slow);
5779 %}
5780 
5781 instruct loadConD0(regD dst, immD0 src)
5782 %{
5783   match(Set dst src);
5784   ins_cost(100);
5785 
5786   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5787   ins_encode %{
5788     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5789   %}
5790   ins_pipe(pipe_slow);
5791 %}
5792 
5793 instruct loadSSI(rRegI dst, stackSlotI src)
5794 %{
5795   match(Set dst src);
5796 
5797   ins_cost(125);
5798   format %{ "movl    $dst, $src\t# int stk" %}
5799   opcode(0x8B);
5800   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5801   ins_pipe(ialu_reg_mem);
5802 %}
5803 
5804 instruct loadSSL(rRegL dst, stackSlotL src)
5805 %{
5806   match(Set dst src);
5807 
5808   ins_cost(125);
5809   format %{ "movq    $dst, $src\t# long stk" %}
5810   opcode(0x8B);
5811   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5812   ins_pipe(ialu_reg_mem);
5813 %}
5814 
5815 instruct loadSSP(rRegP dst, stackSlotP src)
5816 %{
5817   match(Set dst src);
5818 
5819   ins_cost(125);
5820   format %{ "movq    $dst, $src\t# ptr stk" %}
5821   opcode(0x8B);
5822   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5823   ins_pipe(ialu_reg_mem);
5824 %}
5825 
5826 instruct loadSSF(regF dst, stackSlotF src)
5827 %{
5828   match(Set dst src);
5829 
5830   ins_cost(125);
5831   format %{ "movss   $dst, $src\t# float stk" %}
5832   ins_encode %{
5833     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5834   %}
5835   ins_pipe(pipe_slow); // XXX
5836 %}
5837 
5838 // Use the same format since predicate() can not be used here.
5839 instruct loadSSD(regD dst, stackSlotD src)
5840 %{
5841   match(Set dst src);
5842 
5843   ins_cost(125);
5844   format %{ "movsd   $dst, $src\t# double stk" %}
5845   ins_encode  %{
5846     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5847   %}
5848   ins_pipe(pipe_slow); // XXX
5849 %}
5850 
5851 // Prefetch instructions for allocation.
5852 // Must be safe to execute with invalid address (cannot fault).
5853 
5854 instruct prefetchAlloc( memory mem ) %{
5855   predicate(AllocatePrefetchInstr==3);
5856   match(PrefetchAllocation mem);
5857   ins_cost(125);
5858 
5859   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5860   ins_encode %{
5861     __ prefetchw($mem$$Address);
5862   %}
5863   ins_pipe(ialu_mem);
5864 %}
5865 
5866 instruct prefetchAllocNTA( memory mem ) %{
5867   predicate(AllocatePrefetchInstr==0);
5868   match(PrefetchAllocation mem);
5869   ins_cost(125);
5870 
5871   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5872   ins_encode %{
5873     __ prefetchnta($mem$$Address);
5874   %}
5875   ins_pipe(ialu_mem);
5876 %}
5877 
5878 instruct prefetchAllocT0( memory mem ) %{
5879   predicate(AllocatePrefetchInstr==1);
5880   match(PrefetchAllocation mem);
5881   ins_cost(125);
5882 
5883   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5884   ins_encode %{
5885     __ prefetcht0($mem$$Address);
5886   %}
5887   ins_pipe(ialu_mem);
5888 %}
5889 
5890 instruct prefetchAllocT2( memory mem ) %{
5891   predicate(AllocatePrefetchInstr==2);
5892   match(PrefetchAllocation mem);
5893   ins_cost(125);
5894 
5895   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5896   ins_encode %{
5897     __ prefetcht2($mem$$Address);
5898   %}
5899   ins_pipe(ialu_mem);
5900 %}
5901 
5902 //----------Store Instructions-------------------------------------------------
5903 
5904 // Store Byte
5905 instruct storeB(memory mem, rRegI src)
5906 %{
5907   match(Set mem (StoreB mem src));
5908 
5909   ins_cost(125); // XXX
5910   format %{ "movb    $mem, $src\t# byte" %}
5911   opcode(0x88);
5912   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5913   ins_pipe(ialu_mem_reg);
5914 %}
5915 
5916 // Store Char/Short
5917 instruct storeC(memory mem, rRegI src)
5918 %{
5919   match(Set mem (StoreC mem src));
5920 
5921   ins_cost(125); // XXX
5922   format %{ "movw    $mem, $src\t# char/short" %}
5923   opcode(0x89);
5924   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5925   ins_pipe(ialu_mem_reg);
5926 %}
5927 
5928 // Store Integer
5929 instruct storeI(memory mem, rRegI src)
5930 %{
5931   match(Set mem (StoreI mem src));
5932 
5933   ins_cost(125); // XXX
5934   format %{ "movl    $mem, $src\t# int" %}
5935   opcode(0x89);
5936   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5937   ins_pipe(ialu_mem_reg);
5938 %}
5939 
5940 // Store Long
5941 instruct storeL(memory mem, rRegL src)
5942 %{
5943   match(Set mem (StoreL mem src));
5944 
5945   ins_cost(125); // XXX
5946   format %{ "movq    $mem, $src\t# long" %}
5947   opcode(0x89);
5948   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5949   ins_pipe(ialu_mem_reg); // XXX
5950 %}
5951 
5952 // Store Pointer
5953 instruct storeP(memory mem, any_RegP src)
5954 %{
5955   match(Set mem (StoreP mem src));
5956 
5957   ins_cost(125); // XXX
5958   format %{ "movq    $mem, $src\t# ptr" %}
5959   opcode(0x89);
5960   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5961   ins_pipe(ialu_mem_reg);
5962 %}
5963 
5964 instruct storeImmP0(memory mem, immP0 zero)
5965 %{
5966   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5967   match(Set mem (StoreP mem zero));
5968 
5969   ins_cost(125); // XXX
5970   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5971   ins_encode %{
5972     __ movq($mem$$Address, r12);
5973   %}
5974   ins_pipe(ialu_mem_reg);
5975 %}
5976 
5977 // Store NULL Pointer, mark word, or other simple pointer constant.
5978 instruct storeImmP(memory mem, immP31 src)
5979 %{
5980   match(Set mem (StoreP mem src));
5981 
5982   ins_cost(150); // XXX
5983   format %{ "movq    $mem, $src\t# ptr" %}
5984   opcode(0xC7); /* C7 /0 */
5985   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5986   ins_pipe(ialu_mem_imm);
5987 %}
5988 
5989 // Store Compressed Pointer
5990 instruct storeN(memory mem, rRegN src)
5991 %{
5992   match(Set mem (StoreN mem src));
5993 
5994   ins_cost(125); // XXX
5995   format %{ "movl    $mem, $src\t# compressed ptr" %}
5996   ins_encode %{
5997     __ movl($mem$$Address, $src$$Register);
5998   %}
5999   ins_pipe(ialu_mem_reg);
6000 %}
6001 
6002 instruct storeNKlass(memory mem, rRegN src)
6003 %{
6004   match(Set mem (StoreNKlass mem src));
6005 
6006   ins_cost(125); // XXX
6007   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6008   ins_encode %{
6009     __ movl($mem$$Address, $src$$Register);
6010   %}
6011   ins_pipe(ialu_mem_reg);
6012 %}
6013 
6014 instruct storeImmN0(memory mem, immN0 zero)
6015 %{
6016   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
6017   match(Set mem (StoreN mem zero));
6018 
6019   ins_cost(125); // XXX
6020   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
6021   ins_encode %{
6022     __ movl($mem$$Address, r12);
6023   %}
6024   ins_pipe(ialu_mem_reg);
6025 %}
6026 
6027 instruct storeImmN(memory mem, immN src)
6028 %{
6029   match(Set mem (StoreN mem src));
6030 
6031   ins_cost(150); // XXX
6032   format %{ "movl    $mem, $src\t# compressed ptr" %}
6033   ins_encode %{
6034     address con = (address)$src$$constant;
6035     if (con == NULL) {
6036       __ movl($mem$$Address, (int32_t)0);
6037     } else {
6038       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
6039     }
6040   %}
6041   ins_pipe(ialu_mem_imm);
6042 %}
6043 
6044 instruct storeImmNKlass(memory mem, immNKlass src)
6045 %{
6046   match(Set mem (StoreNKlass mem src));
6047 
6048   ins_cost(150); // XXX
6049   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
6050   ins_encode %{
6051     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
6052   %}
6053   ins_pipe(ialu_mem_imm);
6054 %}
6055 
6056 // Store Integer Immediate
6057 instruct storeImmI0(memory mem, immI0 zero)
6058 %{
6059   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6060   match(Set mem (StoreI mem zero));
6061 
6062   ins_cost(125); // XXX
6063   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
6064   ins_encode %{
6065     __ movl($mem$$Address, r12);
6066   %}
6067   ins_pipe(ialu_mem_reg);
6068 %}
6069 
6070 instruct storeImmI(memory mem, immI src)
6071 %{
6072   match(Set mem (StoreI mem src));
6073 
6074   ins_cost(150);
6075   format %{ "movl    $mem, $src\t# int" %}
6076   opcode(0xC7); /* C7 /0 */
6077   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6078   ins_pipe(ialu_mem_imm);
6079 %}
6080 
6081 // Store Long Immediate
6082 instruct storeImmL0(memory mem, immL0 zero)
6083 %{
6084   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6085   match(Set mem (StoreL mem zero));
6086 
6087   ins_cost(125); // XXX
6088   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
6089   ins_encode %{
6090     __ movq($mem$$Address, r12);
6091   %}
6092   ins_pipe(ialu_mem_reg);
6093 %}
6094 
6095 instruct storeImmL(memory mem, immL32 src)
6096 %{
6097   match(Set mem (StoreL mem src));
6098 
6099   ins_cost(150);
6100   format %{ "movq    $mem, $src\t# long" %}
6101   opcode(0xC7); /* C7 /0 */
6102   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
6103   ins_pipe(ialu_mem_imm);
6104 %}
6105 
6106 // Store Short/Char Immediate
6107 instruct storeImmC0(memory mem, immI0 zero)
6108 %{
6109   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6110   match(Set mem (StoreC mem zero));
6111 
6112   ins_cost(125); // XXX
6113   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
6114   ins_encode %{
6115     __ movw($mem$$Address, r12);
6116   %}
6117   ins_pipe(ialu_mem_reg);
6118 %}
6119 
6120 instruct storeImmI16(memory mem, immI16 src)
6121 %{
6122   predicate(UseStoreImmI16);
6123   match(Set mem (StoreC mem src));
6124 
6125   ins_cost(150);
6126   format %{ "movw    $mem, $src\t# short/char" %}
6127   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
6128   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
6129   ins_pipe(ialu_mem_imm);
6130 %}
6131 
6132 // Store Byte Immediate
6133 instruct storeImmB0(memory mem, immI0 zero)
6134 %{
6135   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6136   match(Set mem (StoreB mem zero));
6137 
6138   ins_cost(125); // XXX
6139   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
6140   ins_encode %{
6141     __ movb($mem$$Address, r12);
6142   %}
6143   ins_pipe(ialu_mem_reg);
6144 %}
6145 
6146 instruct storeImmB(memory mem, immI8 src)
6147 %{
6148   match(Set mem (StoreB mem src));
6149 
6150   ins_cost(150); // XXX
6151   format %{ "movb    $mem, $src\t# byte" %}
6152   opcode(0xC6); /* C6 /0 */
6153   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6154   ins_pipe(ialu_mem_imm);
6155 %}
6156 
6157 // Store CMS card-mark Immediate
6158 instruct storeImmCM0_reg(memory mem, immI0 zero)
6159 %{
6160   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6161   match(Set mem (StoreCM mem zero));
6162 
6163   ins_cost(125); // XXX
6164   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
6165   ins_encode %{
6166     __ movb($mem$$Address, r12);
6167   %}
6168   ins_pipe(ialu_mem_reg);
6169 %}
6170 
6171 instruct storeImmCM0(memory mem, immI0 src)
6172 %{
6173   match(Set mem (StoreCM mem src));
6174 
6175   ins_cost(150); // XXX
6176   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
6177   opcode(0xC6); /* C6 /0 */
6178   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
6179   ins_pipe(ialu_mem_imm);
6180 %}
6181 
6182 // Store Float
6183 instruct storeF(memory mem, regF src)
6184 %{
6185   match(Set mem (StoreF mem src));
6186 
6187   ins_cost(95); // XXX
6188   format %{ "movss   $mem, $src\t# float" %}
6189   ins_encode %{
6190     __ movflt($mem$$Address, $src$$XMMRegister);
6191   %}
6192   ins_pipe(pipe_slow); // XXX
6193 %}
6194 
6195 // Store immediate Float value (it is faster than store from XMM register)
6196 instruct storeF0(memory mem, immF0 zero)
6197 %{
6198   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6199   match(Set mem (StoreF mem zero));
6200 
6201   ins_cost(25); // XXX
6202   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
6203   ins_encode %{
6204     __ movl($mem$$Address, r12);
6205   %}
6206   ins_pipe(ialu_mem_reg);
6207 %}
6208 
6209 instruct storeF_imm(memory mem, immF src)
6210 %{
6211   match(Set mem (StoreF mem src));
6212 
6213   ins_cost(50);
6214   format %{ "movl    $mem, $src\t# float" %}
6215   opcode(0xC7); /* C7 /0 */
6216   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6217   ins_pipe(ialu_mem_imm);
6218 %}
6219 
6220 // Store Double
6221 instruct storeD(memory mem, regD src)
6222 %{
6223   match(Set mem (StoreD mem src));
6224 
6225   ins_cost(95); // XXX
6226   format %{ "movsd   $mem, $src\t# double" %}
6227   ins_encode %{
6228     __ movdbl($mem$$Address, $src$$XMMRegister);
6229   %}
6230   ins_pipe(pipe_slow); // XXX
6231 %}
6232 
6233 // Store immediate double 0.0 (it is faster than store from XMM register)
6234 instruct storeD0_imm(memory mem, immD0 src)
6235 %{
6236   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
6237   match(Set mem (StoreD mem src));
6238 
6239   ins_cost(50);
6240   format %{ "movq    $mem, $src\t# double 0." %}
6241   opcode(0xC7); /* C7 /0 */
6242   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
6243   ins_pipe(ialu_mem_imm);
6244 %}
6245 
6246 instruct storeD0(memory mem, immD0 zero)
6247 %{
6248   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
6249   match(Set mem (StoreD mem zero));
6250 
6251   ins_cost(25); // XXX
6252   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
6253   ins_encode %{
6254     __ movq($mem$$Address, r12);
6255   %}
6256   ins_pipe(ialu_mem_reg);
6257 %}
6258 
6259 instruct storeSSI(stackSlotI dst, rRegI src)
6260 %{
6261   match(Set dst src);
6262 
6263   ins_cost(100);
6264   format %{ "movl    $dst, $src\t# int stk" %}
6265   opcode(0x89);
6266   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6267   ins_pipe( ialu_mem_reg );
6268 %}
6269 
6270 instruct storeSSL(stackSlotL dst, rRegL src)
6271 %{
6272   match(Set dst src);
6273 
6274   ins_cost(100);
6275   format %{ "movq    $dst, $src\t# long stk" %}
6276   opcode(0x89);
6277   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6278   ins_pipe(ialu_mem_reg);
6279 %}
6280 
6281 instruct storeSSP(stackSlotP dst, rRegP src)
6282 %{
6283   match(Set dst src);
6284 
6285   ins_cost(100);
6286   format %{ "movq    $dst, $src\t# ptr stk" %}
6287   opcode(0x89);
6288   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6289   ins_pipe(ialu_mem_reg);
6290 %}
6291 
6292 instruct storeSSF(stackSlotF dst, regF src)
6293 %{
6294   match(Set dst src);
6295 
6296   ins_cost(95); // XXX
6297   format %{ "movss   $dst, $src\t# float stk" %}
6298   ins_encode %{
6299     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6300   %}
6301   ins_pipe(pipe_slow); // XXX
6302 %}
6303 
6304 instruct storeSSD(stackSlotD dst, regD src)
6305 %{
6306   match(Set dst src);
6307 
6308   ins_cost(95); // XXX
6309   format %{ "movsd   $dst, $src\t# double stk" %}
6310   ins_encode %{
6311     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6312   %}
6313   ins_pipe(pipe_slow); // XXX
6314 %}
6315 
6316 //----------BSWAP Instructions-------------------------------------------------
6317 instruct bytes_reverse_int(rRegI dst) %{
6318   match(Set dst (ReverseBytesI dst));
6319 
6320   format %{ "bswapl  $dst" %}
6321   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6322   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6323   ins_pipe( ialu_reg );
6324 %}
6325 
6326 instruct bytes_reverse_long(rRegL dst) %{
6327   match(Set dst (ReverseBytesL dst));
6328 
6329   format %{ "bswapq  $dst" %}
6330   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6331   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6332   ins_pipe( ialu_reg);
6333 %}
6334 
6335 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6336   match(Set dst (ReverseBytesUS dst));
6337   effect(KILL cr);
6338 
6339   format %{ "bswapl  $dst\n\t"
6340             "shrl    $dst,16\n\t" %}
6341   ins_encode %{
6342     __ bswapl($dst$$Register);
6343     __ shrl($dst$$Register, 16);
6344   %}
6345   ins_pipe( ialu_reg );
6346 %}
6347 
6348 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6349   match(Set dst (ReverseBytesS dst));
6350   effect(KILL cr);
6351 
6352   format %{ "bswapl  $dst\n\t"
6353             "sar     $dst,16\n\t" %}
6354   ins_encode %{
6355     __ bswapl($dst$$Register);
6356     __ sarl($dst$$Register, 16);
6357   %}
6358   ins_pipe( ialu_reg );
6359 %}
6360 
6361 //---------- Zeros Count Instructions ------------------------------------------
6362 
6363 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6364   predicate(UseCountLeadingZerosInstruction);
6365   match(Set dst (CountLeadingZerosI src));
6366   effect(KILL cr);
6367 
6368   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6369   ins_encode %{
6370     __ lzcntl($dst$$Register, $src$$Register);
6371   %}
6372   ins_pipe(ialu_reg);
6373 %}
6374 
6375 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6376   predicate(!UseCountLeadingZerosInstruction);
6377   match(Set dst (CountLeadingZerosI src));
6378   effect(KILL cr);
6379 
6380   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6381             "jnz     skip\n\t"
6382             "movl    $dst, -1\n"
6383       "skip:\n\t"
6384             "negl    $dst\n\t"
6385             "addl    $dst, 31" %}
6386   ins_encode %{
6387     Register Rdst = $dst$$Register;
6388     Register Rsrc = $src$$Register;
6389     Label skip;
6390     __ bsrl(Rdst, Rsrc);
6391     __ jccb(Assembler::notZero, skip);
6392     __ movl(Rdst, -1);
6393     __ bind(skip);
6394     __ negl(Rdst);
6395     __ addl(Rdst, BitsPerInt - 1);
6396   %}
6397   ins_pipe(ialu_reg);
6398 %}
6399 
6400 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6401   predicate(UseCountLeadingZerosInstruction);
6402   match(Set dst (CountLeadingZerosL src));
6403   effect(KILL cr);
6404 
6405   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6406   ins_encode %{
6407     __ lzcntq($dst$$Register, $src$$Register);
6408   %}
6409   ins_pipe(ialu_reg);
6410 %}
6411 
6412 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6413   predicate(!UseCountLeadingZerosInstruction);
6414   match(Set dst (CountLeadingZerosL src));
6415   effect(KILL cr);
6416 
6417   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6418             "jnz     skip\n\t"
6419             "movl    $dst, -1\n"
6420       "skip:\n\t"
6421             "negl    $dst\n\t"
6422             "addl    $dst, 63" %}
6423   ins_encode %{
6424     Register Rdst = $dst$$Register;
6425     Register Rsrc = $src$$Register;
6426     Label skip;
6427     __ bsrq(Rdst, Rsrc);
6428     __ jccb(Assembler::notZero, skip);
6429     __ movl(Rdst, -1);
6430     __ bind(skip);
6431     __ negl(Rdst);
6432     __ addl(Rdst, BitsPerLong - 1);
6433   %}
6434   ins_pipe(ialu_reg);
6435 %}
6436 
6437 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6438   predicate(UseCountTrailingZerosInstruction);
6439   match(Set dst (CountTrailingZerosI src));
6440   effect(KILL cr);
6441 
6442   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6443   ins_encode %{
6444     __ tzcntl($dst$$Register, $src$$Register);
6445   %}
6446   ins_pipe(ialu_reg);
6447 %}
6448 
6449 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6450   predicate(!UseCountTrailingZerosInstruction);
6451   match(Set dst (CountTrailingZerosI src));
6452   effect(KILL cr);
6453 
6454   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6455             "jnz     done\n\t"
6456             "movl    $dst, 32\n"
6457       "done:" %}
6458   ins_encode %{
6459     Register Rdst = $dst$$Register;
6460     Label done;
6461     __ bsfl(Rdst, $src$$Register);
6462     __ jccb(Assembler::notZero, done);
6463     __ movl(Rdst, BitsPerInt);
6464     __ bind(done);
6465   %}
6466   ins_pipe(ialu_reg);
6467 %}
6468 
6469 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6470   predicate(UseCountTrailingZerosInstruction);
6471   match(Set dst (CountTrailingZerosL src));
6472   effect(KILL cr);
6473 
6474   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6475   ins_encode %{
6476     __ tzcntq($dst$$Register, $src$$Register);
6477   %}
6478   ins_pipe(ialu_reg);
6479 %}
6480 
6481 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6482   predicate(!UseCountTrailingZerosInstruction);
6483   match(Set dst (CountTrailingZerosL src));
6484   effect(KILL cr);
6485 
6486   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6487             "jnz     done\n\t"
6488             "movl    $dst, 64\n"
6489       "done:" %}
6490   ins_encode %{
6491     Register Rdst = $dst$$Register;
6492     Label done;
6493     __ bsfq(Rdst, $src$$Register);
6494     __ jccb(Assembler::notZero, done);
6495     __ movl(Rdst, BitsPerLong);
6496     __ bind(done);
6497   %}
6498   ins_pipe(ialu_reg);
6499 %}
6500 
6501 
6502 //---------- Population Count Instructions -------------------------------------
6503 
6504 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6505   predicate(UsePopCountInstruction);
6506   match(Set dst (PopCountI src));
6507   effect(KILL cr);
6508 
6509   format %{ "popcnt  $dst, $src" %}
6510   ins_encode %{
6511     __ popcntl($dst$$Register, $src$$Register);
6512   %}
6513   ins_pipe(ialu_reg);
6514 %}
6515 
6516 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6517   predicate(UsePopCountInstruction);
6518   match(Set dst (PopCountI (LoadI mem)));
6519   effect(KILL cr);
6520 
6521   format %{ "popcnt  $dst, $mem" %}
6522   ins_encode %{
6523     __ popcntl($dst$$Register, $mem$$Address);
6524   %}
6525   ins_pipe(ialu_reg);
6526 %}
6527 
6528 // Note: Long.bitCount(long) returns an int.
6529 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6530   predicate(UsePopCountInstruction);
6531   match(Set dst (PopCountL src));
6532   effect(KILL cr);
6533 
6534   format %{ "popcnt  $dst, $src" %}
6535   ins_encode %{
6536     __ popcntq($dst$$Register, $src$$Register);
6537   %}
6538   ins_pipe(ialu_reg);
6539 %}
6540 
6541 // Note: Long.bitCount(long) returns an int.
6542 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6543   predicate(UsePopCountInstruction);
6544   match(Set dst (PopCountL (LoadL mem)));
6545   effect(KILL cr);
6546 
6547   format %{ "popcnt  $dst, $mem" %}
6548   ins_encode %{
6549     __ popcntq($dst$$Register, $mem$$Address);
6550   %}
6551   ins_pipe(ialu_reg);
6552 %}
6553 
6554 
6555 //----------MemBar Instructions-----------------------------------------------
6556 // Memory barrier flavors
6557 
6558 instruct membar_acquire()
6559 %{
6560   match(MemBarAcquire);
6561   match(LoadFence);
6562   ins_cost(0);
6563 
6564   size(0);
6565   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6566   ins_encode();
6567   ins_pipe(empty);
6568 %}
6569 
6570 instruct membar_acquire_lock()
6571 %{
6572   match(MemBarAcquireLock);
6573   ins_cost(0);
6574 
6575   size(0);
6576   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6577   ins_encode();
6578   ins_pipe(empty);
6579 %}
6580 
6581 instruct membar_release()
6582 %{
6583   match(MemBarRelease);
6584   match(StoreFence);
6585   ins_cost(0);
6586 
6587   size(0);
6588   format %{ "MEMBAR-release ! (empty encoding)" %}
6589   ins_encode();
6590   ins_pipe(empty);
6591 %}
6592 
6593 instruct membar_release_lock()
6594 %{
6595   match(MemBarReleaseLock);
6596   ins_cost(0);
6597 
6598   size(0);
6599   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6600   ins_encode();
6601   ins_pipe(empty);
6602 %}
6603 
6604 instruct membar_volatile(rFlagsReg cr) %{
6605   match(MemBarVolatile);
6606   effect(KILL cr);
6607   ins_cost(400);
6608 
6609   format %{
6610     $$template
6611     $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6612   %}
6613   ins_encode %{
6614     __ membar(Assembler::StoreLoad);
6615   %}
6616   ins_pipe(pipe_slow);
6617 %}
6618 
6619 instruct unnecessary_membar_volatile()
6620 %{
6621   match(MemBarVolatile);
6622   predicate(Matcher::post_store_load_barrier(n));
6623   ins_cost(0);
6624 
6625   size(0);
6626   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6627   ins_encode();
6628   ins_pipe(empty);
6629 %}
6630 
6631 instruct membar_storestore() %{
6632   match(MemBarStoreStore);
6633   ins_cost(0);
6634 
6635   size(0);
6636   format %{ "MEMBAR-storestore (empty encoding)" %}
6637   ins_encode( );
6638   ins_pipe(empty);
6639 %}
6640 
6641 //----------Move Instructions--------------------------------------------------
6642 
6643 instruct castX2P(rRegP dst, rRegL src)
6644 %{
6645   match(Set dst (CastX2P src));
6646 
6647   format %{ "movq    $dst, $src\t# long->ptr" %}
6648   ins_encode %{
6649     if ($dst$$reg != $src$$reg) {
6650       __ movptr($dst$$Register, $src$$Register);
6651     }
6652   %}
6653   ins_pipe(ialu_reg_reg); // XXX
6654 %}
6655 
6656 instruct castP2X(rRegL dst, rRegP src)
6657 %{
6658   match(Set dst (CastP2X src));
6659 
6660   format %{ "movq    $dst, $src\t# ptr -> long" %}
6661   ins_encode %{
6662     if ($dst$$reg != $src$$reg) {
6663       __ movptr($dst$$Register, $src$$Register);
6664     }
6665   %}
6666   ins_pipe(ialu_reg_reg); // XXX
6667 %}
6668 
6669 // Convert oop into int for vectors alignment masking
6670 instruct convP2I(rRegI dst, rRegP src)
6671 %{
6672   match(Set dst (ConvL2I (CastP2X src)));
6673 
6674   format %{ "movl    $dst, $src\t# ptr -> int" %}
6675   ins_encode %{
6676     __ movl($dst$$Register, $src$$Register);
6677   %}
6678   ins_pipe(ialu_reg_reg); // XXX
6679 %}
6680 
6681 // Convert compressed oop into int for vectors alignment masking
6682 // in case of 32bit oops (heap < 4Gb).
6683 instruct convN2I(rRegI dst, rRegN src)
6684 %{
6685   predicate(Universe::narrow_oop_shift() == 0);
6686   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6687 
6688   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6689   ins_encode %{
6690     __ movl($dst$$Register, $src$$Register);
6691   %}
6692   ins_pipe(ialu_reg_reg); // XXX
6693 %}
6694 
6695 // Convert oop pointer into compressed form
6696 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6697   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6698   match(Set dst (EncodeP src));
6699   effect(KILL cr);
6700   format %{ "encode_heap_oop $dst,$src" %}
6701   ins_encode %{
6702     Register s = $src$$Register;
6703     Register d = $dst$$Register;
6704     if (s != d) {
6705       __ movq(d, s);
6706     }
6707     __ encode_heap_oop(d);
6708   %}
6709   ins_pipe(ialu_reg_long);
6710 %}
6711 
6712 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6713   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6714   match(Set dst (EncodeP src));
6715   effect(KILL cr);
6716   format %{ "encode_heap_oop_not_null $dst,$src" %}
6717   ins_encode %{
6718     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6719   %}
6720   ins_pipe(ialu_reg_long);
6721 %}
6722 
6723 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6724   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6725             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6726   match(Set dst (DecodeN src));
6727   effect(KILL cr);
6728   format %{ "decode_heap_oop $dst,$src" %}
6729   ins_encode %{
6730     Register s = $src$$Register;
6731     Register d = $dst$$Register;
6732     if (s != d) {
6733       __ movq(d, s);
6734     }
6735     __ decode_heap_oop(d);
6736   %}
6737   ins_pipe(ialu_reg_long);
6738 %}
6739 
6740 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6741   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6742             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6743   match(Set dst (DecodeN src));
6744   effect(KILL cr);
6745   format %{ "decode_heap_oop_not_null $dst,$src" %}
6746   ins_encode %{
6747     Register s = $src$$Register;
6748     Register d = $dst$$Register;
6749     if (s != d) {
6750       __ decode_heap_oop_not_null(d, s);
6751     } else {
6752       __ decode_heap_oop_not_null(d);
6753     }
6754   %}
6755   ins_pipe(ialu_reg_long);
6756 %}
6757 
6758 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6759   match(Set dst (EncodePKlass src));
6760   effect(KILL cr);
6761   format %{ "encode_klass_not_null $dst,$src" %}
6762   ins_encode %{
6763     __ encode_klass_not_null($dst$$Register, $src$$Register);
6764   %}
6765   ins_pipe(ialu_reg_long);
6766 %}
6767 
6768 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6769   match(Set dst (DecodeNKlass src));
6770   effect(KILL cr);
6771   format %{ "decode_klass_not_null $dst,$src" %}
6772   ins_encode %{
6773     Register s = $src$$Register;
6774     Register d = $dst$$Register;
6775     if (s != d) {
6776       __ decode_klass_not_null(d, s);
6777     } else {
6778       __ decode_klass_not_null(d);
6779     }
6780   %}
6781   ins_pipe(ialu_reg_long);
6782 %}
6783 
6784 
6785 //----------Conditional Move---------------------------------------------------
6786 // Jump
6787 // dummy instruction for generating temp registers
6788 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6789   match(Jump (LShiftL switch_val shift));
6790   ins_cost(350);
6791   predicate(false);
6792   effect(TEMP dest);
6793 
6794   format %{ "leaq    $dest, [$constantaddress]\n\t"
6795             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6796   ins_encode %{
6797     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6798     // to do that and the compiler is using that register as one it can allocate.
6799     // So we build it all by hand.
6800     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6801     // ArrayAddress dispatch(table, index);
6802     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6803     __ lea($dest$$Register, $constantaddress);
6804     __ jmp(dispatch);
6805   %}
6806   ins_pipe(pipe_jmp);
6807 %}
6808 
6809 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6810   match(Jump (AddL (LShiftL switch_val shift) offset));
6811   ins_cost(350);
6812   effect(TEMP dest);
6813 
6814   format %{ "leaq    $dest, [$constantaddress]\n\t"
6815             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6816   ins_encode %{
6817     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6818     // to do that and the compiler is using that register as one it can allocate.
6819     // So we build it all by hand.
6820     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6821     // ArrayAddress dispatch(table, index);
6822     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6823     __ lea($dest$$Register, $constantaddress);
6824     __ jmp(dispatch);
6825   %}
6826   ins_pipe(pipe_jmp);
6827 %}
6828 
6829 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6830   match(Jump switch_val);
6831   ins_cost(350);
6832   effect(TEMP dest);
6833 
6834   format %{ "leaq    $dest, [$constantaddress]\n\t"
6835             "jmp     [$dest + $switch_val]\n\t" %}
6836   ins_encode %{
6837     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6838     // to do that and the compiler is using that register as one it can allocate.
6839     // So we build it all by hand.
6840     // Address index(noreg, switch_reg, Address::times_1);
6841     // ArrayAddress dispatch(table, index);
6842     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6843     __ lea($dest$$Register, $constantaddress);
6844     __ jmp(dispatch);
6845   %}
6846   ins_pipe(pipe_jmp);
6847 %}
6848 
6849 // Conditional move
6850 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6851 %{
6852   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6853 
6854   ins_cost(200); // XXX
6855   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6856   opcode(0x0F, 0x40);
6857   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6858   ins_pipe(pipe_cmov_reg);
6859 %}
6860 
6861 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6862   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6863 
6864   ins_cost(200); // XXX
6865   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6866   opcode(0x0F, 0x40);
6867   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6868   ins_pipe(pipe_cmov_reg);
6869 %}
6870 
6871 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6872   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6873   ins_cost(200);
6874   expand %{
6875     cmovI_regU(cop, cr, dst, src);
6876   %}
6877 %}
6878 
6879 // Conditional move
6880 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6881   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6882 
6883   ins_cost(250); // XXX
6884   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6885   opcode(0x0F, 0x40);
6886   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6887   ins_pipe(pipe_cmov_mem);
6888 %}
6889 
6890 // Conditional move
6891 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6892 %{
6893   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6894 
6895   ins_cost(250); // XXX
6896   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6897   opcode(0x0F, 0x40);
6898   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6899   ins_pipe(pipe_cmov_mem);
6900 %}
6901 
6902 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6903   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6904   ins_cost(250);
6905   expand %{
6906     cmovI_memU(cop, cr, dst, src);
6907   %}
6908 %}
6909 
6910 // Conditional move
6911 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6912 %{
6913   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6914 
6915   ins_cost(200); // XXX
6916   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6917   opcode(0x0F, 0x40);
6918   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6919   ins_pipe(pipe_cmov_reg);
6920 %}
6921 
6922 // Conditional move
6923 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6924 %{
6925   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6926 
6927   ins_cost(200); // XXX
6928   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6929   opcode(0x0F, 0x40);
6930   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6931   ins_pipe(pipe_cmov_reg);
6932 %}
6933 
6934 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6935   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6936   ins_cost(200);
6937   expand %{
6938     cmovN_regU(cop, cr, dst, src);
6939   %}
6940 %}
6941 
6942 // Conditional move
6943 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6944 %{
6945   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6946 
6947   ins_cost(200); // XXX
6948   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6949   opcode(0x0F, 0x40);
6950   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6951   ins_pipe(pipe_cmov_reg);  // XXX
6952 %}
6953 
6954 // Conditional move
6955 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6956 %{
6957   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6958 
6959   ins_cost(200); // XXX
6960   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6961   opcode(0x0F, 0x40);
6962   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6963   ins_pipe(pipe_cmov_reg); // XXX
6964 %}
6965 
6966 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6967   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6968   ins_cost(200);
6969   expand %{
6970     cmovP_regU(cop, cr, dst, src);
6971   %}
6972 %}
6973 
6974 // DISABLED: Requires the ADLC to emit a bottom_type call that
6975 // correctly meets the two pointer arguments; one is an incoming
6976 // register but the other is a memory operand.  ALSO appears to
6977 // be buggy with implicit null checks.
6978 //
6979 //// Conditional move
6980 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6981 //%{
6982 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6983 //  ins_cost(250);
6984 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6985 //  opcode(0x0F,0x40);
6986 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6987 //  ins_pipe( pipe_cmov_mem );
6988 //%}
6989 //
6990 //// Conditional move
6991 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6992 //%{
6993 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6994 //  ins_cost(250);
6995 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6996 //  opcode(0x0F,0x40);
6997 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6998 //  ins_pipe( pipe_cmov_mem );
6999 //%}
7000 
7001 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
7002 %{
7003   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7004 
7005   ins_cost(200); // XXX
7006   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7007   opcode(0x0F, 0x40);
7008   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7009   ins_pipe(pipe_cmov_reg);  // XXX
7010 %}
7011 
7012 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
7013 %{
7014   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7015 
7016   ins_cost(200); // XXX
7017   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
7018   opcode(0x0F, 0x40);
7019   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7020   ins_pipe(pipe_cmov_mem);  // XXX
7021 %}
7022 
7023 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
7024 %{
7025   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7026 
7027   ins_cost(200); // XXX
7028   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7029   opcode(0x0F, 0x40);
7030   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
7031   ins_pipe(pipe_cmov_reg); // XXX
7032 %}
7033 
7034 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
7035   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
7036   ins_cost(200);
7037   expand %{
7038     cmovL_regU(cop, cr, dst, src);
7039   %}
7040 %}
7041 
7042 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
7043 %{
7044   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7045 
7046   ins_cost(200); // XXX
7047   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
7048   opcode(0x0F, 0x40);
7049   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
7050   ins_pipe(pipe_cmov_mem); // XXX
7051 %}
7052 
7053 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
7054   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
7055   ins_cost(200);
7056   expand %{
7057     cmovL_memU(cop, cr, dst, src);
7058   %}
7059 %}
7060 
7061 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
7062 %{
7063   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7064 
7065   ins_cost(200); // XXX
7066   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7067             "movss     $dst, $src\n"
7068     "skip:" %}
7069   ins_encode %{
7070     Label Lskip;
7071     // Invert sense of branch from sense of CMOV
7072     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7073     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7074     __ bind(Lskip);
7075   %}
7076   ins_pipe(pipe_slow);
7077 %}
7078 
7079 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
7080 // %{
7081 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
7082 
7083 //   ins_cost(200); // XXX
7084 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
7085 //             "movss     $dst, $src\n"
7086 //     "skip:" %}
7087 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
7088 //   ins_pipe(pipe_slow);
7089 // %}
7090 
7091 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
7092 %{
7093   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7094 
7095   ins_cost(200); // XXX
7096   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
7097             "movss     $dst, $src\n"
7098     "skip:" %}
7099   ins_encode %{
7100     Label Lskip;
7101     // Invert sense of branch from sense of CMOV
7102     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7103     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
7104     __ bind(Lskip);
7105   %}
7106   ins_pipe(pipe_slow);
7107 %}
7108 
7109 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
7110   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
7111   ins_cost(200);
7112   expand %{
7113     cmovF_regU(cop, cr, dst, src);
7114   %}
7115 %}
7116 
7117 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
7118 %{
7119   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7120 
7121   ins_cost(200); // XXX
7122   format %{ "jn$cop    skip\t# signed cmove double\n\t"
7123             "movsd     $dst, $src\n"
7124     "skip:" %}
7125   ins_encode %{
7126     Label Lskip;
7127     // Invert sense of branch from sense of CMOV
7128     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7129     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7130     __ bind(Lskip);
7131   %}
7132   ins_pipe(pipe_slow);
7133 %}
7134 
7135 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
7136 %{
7137   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7138 
7139   ins_cost(200); // XXX
7140   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
7141             "movsd     $dst, $src\n"
7142     "skip:" %}
7143   ins_encode %{
7144     Label Lskip;
7145     // Invert sense of branch from sense of CMOV
7146     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
7147     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
7148     __ bind(Lskip);
7149   %}
7150   ins_pipe(pipe_slow);
7151 %}
7152 
7153 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
7154   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
7155   ins_cost(200);
7156   expand %{
7157     cmovD_regU(cop, cr, dst, src);
7158   %}
7159 %}
7160 
7161 //----------Arithmetic Instructions--------------------------------------------
7162 //----------Addition Instructions----------------------------------------------
7163 
7164 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7165 %{
7166   match(Set dst (AddI dst src));
7167   effect(KILL cr);
7168 
7169   format %{ "addl    $dst, $src\t# int" %}
7170   opcode(0x03);
7171   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7172   ins_pipe(ialu_reg_reg);
7173 %}
7174 
7175 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7176 %{
7177   match(Set dst (AddI dst src));
7178   effect(KILL cr);
7179 
7180   format %{ "addl    $dst, $src\t# int" %}
7181   opcode(0x81, 0x00); /* /0 id */
7182   ins_encode(OpcSErm(dst, src), Con8or32(src));
7183   ins_pipe( ialu_reg );
7184 %}
7185 
7186 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7187 %{
7188   match(Set dst (AddI dst (LoadI src)));
7189   effect(KILL cr);
7190 
7191   ins_cost(125); // XXX
7192   format %{ "addl    $dst, $src\t# int" %}
7193   opcode(0x03);
7194   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7195   ins_pipe(ialu_reg_mem);
7196 %}
7197 
7198 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7199 %{
7200   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7201   effect(KILL cr);
7202 
7203   ins_cost(150); // XXX
7204   format %{ "addl    $dst, $src\t# int" %}
7205   opcode(0x01); /* Opcode 01 /r */
7206   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7207   ins_pipe(ialu_mem_reg);
7208 %}
7209 
7210 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
7211 %{
7212   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7213   effect(KILL cr);
7214 
7215   ins_cost(125); // XXX
7216   format %{ "addl    $dst, $src\t# int" %}
7217   opcode(0x81); /* Opcode 81 /0 id */
7218   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7219   ins_pipe(ialu_mem_imm);
7220 %}
7221 
7222 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
7223 %{
7224   predicate(UseIncDec);
7225   match(Set dst (AddI dst src));
7226   effect(KILL cr);
7227 
7228   format %{ "incl    $dst\t# int" %}
7229   opcode(0xFF, 0x00); // FF /0
7230   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7231   ins_pipe(ialu_reg);
7232 %}
7233 
7234 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
7235 %{
7236   predicate(UseIncDec);
7237   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7238   effect(KILL cr);
7239 
7240   ins_cost(125); // XXX
7241   format %{ "incl    $dst\t# int" %}
7242   opcode(0xFF); /* Opcode FF /0 */
7243   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7244   ins_pipe(ialu_mem_imm);
7245 %}
7246 
7247 // XXX why does that use AddI
7248 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7249 %{
7250   predicate(UseIncDec);
7251   match(Set dst (AddI dst src));
7252   effect(KILL cr);
7253 
7254   format %{ "decl    $dst\t# int" %}
7255   opcode(0xFF, 0x01); // FF /1
7256   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7257   ins_pipe(ialu_reg);
7258 %}
7259 
7260 // XXX why does that use AddI
7261 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7262 %{
7263   predicate(UseIncDec);
7264   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7265   effect(KILL cr);
7266 
7267   ins_cost(125); // XXX
7268   format %{ "decl    $dst\t# int" %}
7269   opcode(0xFF); /* Opcode FF /1 */
7270   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7271   ins_pipe(ialu_mem_imm);
7272 %}
7273 
7274 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7275 %{
7276   match(Set dst (AddI src0 src1));
7277 
7278   ins_cost(110);
7279   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7280   opcode(0x8D); /* 0x8D /r */
7281   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7282   ins_pipe(ialu_reg_reg);
7283 %}
7284 
7285 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7286 %{
7287   match(Set dst (AddL dst src));
7288   effect(KILL cr);
7289 
7290   format %{ "addq    $dst, $src\t# long" %}
7291   opcode(0x03);
7292   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7293   ins_pipe(ialu_reg_reg);
7294 %}
7295 
7296 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7297 %{
7298   match(Set dst (AddL dst src));
7299   effect(KILL cr);
7300 
7301   format %{ "addq    $dst, $src\t# long" %}
7302   opcode(0x81, 0x00); /* /0 id */
7303   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7304   ins_pipe( ialu_reg );
7305 %}
7306 
7307 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7308 %{
7309   match(Set dst (AddL dst (LoadL src)));
7310   effect(KILL cr);
7311 
7312   ins_cost(125); // XXX
7313   format %{ "addq    $dst, $src\t# long" %}
7314   opcode(0x03);
7315   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7316   ins_pipe(ialu_reg_mem);
7317 %}
7318 
7319 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7320 %{
7321   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7322   effect(KILL cr);
7323 
7324   ins_cost(150); // XXX
7325   format %{ "addq    $dst, $src\t# long" %}
7326   opcode(0x01); /* Opcode 01 /r */
7327   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7328   ins_pipe(ialu_mem_reg);
7329 %}
7330 
7331 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7332 %{
7333   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7334   effect(KILL cr);
7335 
7336   ins_cost(125); // XXX
7337   format %{ "addq    $dst, $src\t# long" %}
7338   opcode(0x81); /* Opcode 81 /0 id */
7339   ins_encode(REX_mem_wide(dst),
7340              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7341   ins_pipe(ialu_mem_imm);
7342 %}
7343 
7344 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7345 %{
7346   predicate(UseIncDec);
7347   match(Set dst (AddL dst src));
7348   effect(KILL cr);
7349 
7350   format %{ "incq    $dst\t# long" %}
7351   opcode(0xFF, 0x00); // FF /0
7352   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7353   ins_pipe(ialu_reg);
7354 %}
7355 
7356 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7357 %{
7358   predicate(UseIncDec);
7359   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7360   effect(KILL cr);
7361 
7362   ins_cost(125); // XXX
7363   format %{ "incq    $dst\t# long" %}
7364   opcode(0xFF); /* Opcode FF /0 */
7365   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7366   ins_pipe(ialu_mem_imm);
7367 %}
7368 
7369 // XXX why does that use AddL
7370 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7371 %{
7372   predicate(UseIncDec);
7373   match(Set dst (AddL dst src));
7374   effect(KILL cr);
7375 
7376   format %{ "decq    $dst\t# long" %}
7377   opcode(0xFF, 0x01); // FF /1
7378   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7379   ins_pipe(ialu_reg);
7380 %}
7381 
7382 // XXX why does that use AddL
7383 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7384 %{
7385   predicate(UseIncDec);
7386   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7387   effect(KILL cr);
7388 
7389   ins_cost(125); // XXX
7390   format %{ "decq    $dst\t# long" %}
7391   opcode(0xFF); /* Opcode FF /1 */
7392   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7393   ins_pipe(ialu_mem_imm);
7394 %}
7395 
7396 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7397 %{
7398   match(Set dst (AddL src0 src1));
7399 
7400   ins_cost(110);
7401   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7402   opcode(0x8D); /* 0x8D /r */
7403   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7404   ins_pipe(ialu_reg_reg);
7405 %}
7406 
7407 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7408 %{
7409   match(Set dst (AddP dst src));
7410   effect(KILL cr);
7411 
7412   format %{ "addq    $dst, $src\t# ptr" %}
7413   opcode(0x03);
7414   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7415   ins_pipe(ialu_reg_reg);
7416 %}
7417 
7418 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7419 %{
7420   match(Set dst (AddP dst src));
7421   effect(KILL cr);
7422 
7423   format %{ "addq    $dst, $src\t# ptr" %}
7424   opcode(0x81, 0x00); /* /0 id */
7425   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7426   ins_pipe( ialu_reg );
7427 %}
7428 
7429 // XXX addP mem ops ????
7430 
7431 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7432 %{
7433   match(Set dst (AddP src0 src1));
7434 
7435   ins_cost(110);
7436   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7437   opcode(0x8D); /* 0x8D /r */
7438   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7439   ins_pipe(ialu_reg_reg);
7440 %}
7441 
7442 instruct checkCastPP(rRegP dst)
7443 %{
7444   match(Set dst (CheckCastPP dst));
7445 
7446   size(0);
7447   format %{ "# checkcastPP of $dst" %}
7448   ins_encode(/* empty encoding */);
7449   ins_pipe(empty);
7450 %}
7451 
7452 instruct castPP(rRegP dst)
7453 %{
7454   match(Set dst (CastPP dst));
7455 
7456   size(0);
7457   format %{ "# castPP of $dst" %}
7458   ins_encode(/* empty encoding */);
7459   ins_pipe(empty);
7460 %}
7461 
7462 instruct castII(rRegI dst)
7463 %{
7464   match(Set dst (CastII dst));
7465 
7466   size(0);
7467   format %{ "# castII of $dst" %}
7468   ins_encode(/* empty encoding */);
7469   ins_cost(0);
7470   ins_pipe(empty);
7471 %}
7472 
7473 // LoadP-locked same as a regular LoadP when used with compare-swap
7474 instruct loadPLocked(rRegP dst, memory mem)
7475 %{
7476   match(Set dst (LoadPLocked mem));
7477 
7478   ins_cost(125); // XXX
7479   format %{ "movq    $dst, $mem\t# ptr locked" %}
7480   opcode(0x8B);
7481   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7482   ins_pipe(ialu_reg_mem); // XXX
7483 %}
7484 
7485 // Conditional-store of the updated heap-top.
7486 // Used during allocation of the shared heap.
7487 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7488 
7489 instruct storePConditional(memory heap_top_ptr,
7490                            rax_RegP oldval, rRegP newval,
7491                            rFlagsReg cr)
7492 %{
7493   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7494 
7495   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7496             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7497   opcode(0x0F, 0xB1);
7498   ins_encode(lock_prefix,
7499              REX_reg_mem_wide(newval, heap_top_ptr),
7500              OpcP, OpcS,
7501              reg_mem(newval, heap_top_ptr));
7502   ins_pipe(pipe_cmpxchg);
7503 %}
7504 
7505 // Conditional-store of an int value.
7506 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7507 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7508 %{
7509   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7510   effect(KILL oldval);
7511 
7512   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7513   opcode(0x0F, 0xB1);
7514   ins_encode(lock_prefix,
7515              REX_reg_mem(newval, mem),
7516              OpcP, OpcS,
7517              reg_mem(newval, mem));
7518   ins_pipe(pipe_cmpxchg);
7519 %}
7520 
7521 // Conditional-store of a long value.
7522 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7523 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7524 %{
7525   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7526   effect(KILL oldval);
7527 
7528   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7529   opcode(0x0F, 0xB1);
7530   ins_encode(lock_prefix,
7531              REX_reg_mem_wide(newval, mem),
7532              OpcP, OpcS,
7533              reg_mem(newval, mem));
7534   ins_pipe(pipe_cmpxchg);
7535 %}
7536 
7537 
7538 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7539 instruct compareAndSwapP(rRegI res,
7540                          memory mem_ptr,
7541                          rax_RegP oldval, rRegP newval,
7542                          rFlagsReg cr)
7543 %{
7544   predicate(VM_Version::supports_cx8());
7545   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7546   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7547   effect(KILL cr, KILL oldval);
7548 
7549   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7550             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7551             "sete    $res\n\t"
7552             "movzbl  $res, $res" %}
7553   opcode(0x0F, 0xB1);
7554   ins_encode(lock_prefix,
7555              REX_reg_mem_wide(newval, mem_ptr),
7556              OpcP, OpcS,
7557              reg_mem(newval, mem_ptr),
7558              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7559              REX_reg_breg(res, res), // movzbl
7560              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7561   ins_pipe( pipe_cmpxchg );
7562 %}
7563 
7564 instruct compareAndSwapL(rRegI res,
7565                          memory mem_ptr,
7566                          rax_RegL oldval, rRegL newval,
7567                          rFlagsReg cr)
7568 %{
7569   predicate(VM_Version::supports_cx8());
7570   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7571   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7572   effect(KILL cr, KILL oldval);
7573 
7574   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7575             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7576             "sete    $res\n\t"
7577             "movzbl  $res, $res" %}
7578   opcode(0x0F, 0xB1);
7579   ins_encode(lock_prefix,
7580              REX_reg_mem_wide(newval, mem_ptr),
7581              OpcP, OpcS,
7582              reg_mem(newval, mem_ptr),
7583              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7584              REX_reg_breg(res, res), // movzbl
7585              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7586   ins_pipe( pipe_cmpxchg );
7587 %}
7588 
7589 instruct compareAndSwapI(rRegI res,
7590                          memory mem_ptr,
7591                          rax_RegI oldval, rRegI newval,
7592                          rFlagsReg cr)
7593 %{
7594   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7595   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7596   effect(KILL cr, KILL oldval);
7597 
7598   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7599             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7600             "sete    $res\n\t"
7601             "movzbl  $res, $res" %}
7602   opcode(0x0F, 0xB1);
7603   ins_encode(lock_prefix,
7604              REX_reg_mem(newval, mem_ptr),
7605              OpcP, OpcS,
7606              reg_mem(newval, mem_ptr),
7607              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7608              REX_reg_breg(res, res), // movzbl
7609              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7610   ins_pipe( pipe_cmpxchg );
7611 %}
7612 
7613 instruct compareAndSwapB(rRegI res,
7614                          memory mem_ptr,
7615                          rax_RegI oldval, rRegI newval,
7616                          rFlagsReg cr)
7617 %{
7618   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7619   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7620   effect(KILL cr, KILL oldval);
7621 
7622   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7623             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7624             "sete    $res\n\t"
7625             "movzbl  $res, $res" %}
7626   opcode(0x0F, 0xB0);
7627   ins_encode(lock_prefix,
7628              REX_breg_mem(newval, mem_ptr),
7629              OpcP, OpcS,
7630              reg_mem(newval, mem_ptr),
7631              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7632              REX_reg_breg(res, res), // movzbl
7633              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7634   ins_pipe( pipe_cmpxchg );
7635 %}
7636 
7637 instruct compareAndSwapS(rRegI res,
7638                          memory mem_ptr,
7639                          rax_RegI oldval, rRegI newval,
7640                          rFlagsReg cr)
7641 %{
7642   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7643   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7644   effect(KILL cr, KILL oldval);
7645 
7646   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7647             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7648             "sete    $res\n\t"
7649             "movzbl  $res, $res" %}
7650   opcode(0x0F, 0xB1);
7651   ins_encode(lock_prefix,
7652              SizePrefix,
7653              REX_reg_mem(newval, mem_ptr),
7654              OpcP, OpcS,
7655              reg_mem(newval, mem_ptr),
7656              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7657              REX_reg_breg(res, res), // movzbl
7658              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7659   ins_pipe( pipe_cmpxchg );
7660 %}
7661 
7662 instruct compareAndSwapN(rRegI res,
7663                           memory mem_ptr,
7664                           rax_RegN oldval, rRegN newval,
7665                           rFlagsReg cr) %{
7666   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7667   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7668   effect(KILL cr, KILL oldval);
7669 
7670   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7671             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7672             "sete    $res\n\t"
7673             "movzbl  $res, $res" %}
7674   opcode(0x0F, 0xB1);
7675   ins_encode(lock_prefix,
7676              REX_reg_mem(newval, mem_ptr),
7677              OpcP, OpcS,
7678              reg_mem(newval, mem_ptr),
7679              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7680              REX_reg_breg(res, res), // movzbl
7681              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7682   ins_pipe( pipe_cmpxchg );
7683 %}
7684 
7685 instruct compareAndExchangeB(
7686                          memory mem_ptr,
7687                          rax_RegI oldval, rRegI newval,
7688                          rFlagsReg cr)
7689 %{
7690   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7691   effect(KILL cr);
7692 
7693   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7694             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7695   opcode(0x0F, 0xB0);
7696   ins_encode(lock_prefix,
7697              REX_breg_mem(newval, mem_ptr),
7698              OpcP, OpcS,
7699              reg_mem(newval, mem_ptr) // lock cmpxchg
7700              );
7701   ins_pipe( pipe_cmpxchg );
7702 %}
7703 
7704 instruct compareAndExchangeS(
7705                          memory mem_ptr,
7706                          rax_RegI oldval, rRegI newval,
7707                          rFlagsReg cr)
7708 %{
7709   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7710   effect(KILL cr);
7711 
7712   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7713             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7714   opcode(0x0F, 0xB1);
7715   ins_encode(lock_prefix,
7716              SizePrefix,
7717              REX_reg_mem(newval, mem_ptr),
7718              OpcP, OpcS,
7719              reg_mem(newval, mem_ptr) // lock cmpxchg
7720              );
7721   ins_pipe( pipe_cmpxchg );
7722 %}
7723 
7724 instruct compareAndExchangeI(
7725                          memory mem_ptr,
7726                          rax_RegI oldval, rRegI newval,
7727                          rFlagsReg cr)
7728 %{
7729   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7730   effect(KILL cr);
7731 
7732   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7733             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7734   opcode(0x0F, 0xB1);
7735   ins_encode(lock_prefix,
7736              REX_reg_mem(newval, mem_ptr),
7737              OpcP, OpcS,
7738              reg_mem(newval, mem_ptr) // lock cmpxchg
7739              );
7740   ins_pipe( pipe_cmpxchg );
7741 %}
7742 
7743 instruct compareAndExchangeL(
7744                          memory mem_ptr,
7745                          rax_RegL oldval, rRegL newval,
7746                          rFlagsReg cr)
7747 %{
7748   predicate(VM_Version::supports_cx8());
7749   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7750   effect(KILL cr);
7751 
7752   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7753             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7754   opcode(0x0F, 0xB1);
7755   ins_encode(lock_prefix,
7756              REX_reg_mem_wide(newval, mem_ptr),
7757              OpcP, OpcS,
7758              reg_mem(newval, mem_ptr)  // lock cmpxchg
7759             );
7760   ins_pipe( pipe_cmpxchg );
7761 %}
7762 
7763 instruct compareAndExchangeN(
7764                           memory mem_ptr,
7765                           rax_RegN oldval, rRegN newval,
7766                           rFlagsReg cr) %{
7767   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7768   effect(KILL cr);
7769 
7770   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7771             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7772   opcode(0x0F, 0xB1);
7773   ins_encode(lock_prefix,
7774              REX_reg_mem(newval, mem_ptr),
7775              OpcP, OpcS,
7776              reg_mem(newval, mem_ptr)  // lock cmpxchg
7777           );
7778   ins_pipe( pipe_cmpxchg );
7779 %}
7780 
7781 instruct compareAndExchangeP(
7782                          memory mem_ptr,
7783                          rax_RegP oldval, rRegP newval,
7784                          rFlagsReg cr)
7785 %{
7786   predicate(VM_Version::supports_cx8());
7787   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7788   effect(KILL cr);
7789 
7790   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7791             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7792   opcode(0x0F, 0xB1);
7793   ins_encode(lock_prefix,
7794              REX_reg_mem_wide(newval, mem_ptr),
7795              OpcP, OpcS,
7796              reg_mem(newval, mem_ptr)  // lock cmpxchg
7797           );
7798   ins_pipe( pipe_cmpxchg );
7799 %}
7800 
7801 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7802   predicate(n->as_LoadStore()->result_not_used());
7803   match(Set dummy (GetAndAddB mem add));
7804   effect(KILL cr);
7805   format %{ "ADDB  [$mem],$add" %}
7806   ins_encode %{
7807     __ lock();
7808     __ addb($mem$$Address, $add$$constant);
7809   %}
7810   ins_pipe( pipe_cmpxchg );
7811 %}
7812 
7813 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7814   match(Set newval (GetAndAddB mem newval));
7815   effect(KILL cr);
7816   format %{ "XADDB  [$mem],$newval" %}
7817   ins_encode %{
7818     __ lock();
7819     __ xaddb($mem$$Address, $newval$$Register);
7820   %}
7821   ins_pipe( pipe_cmpxchg );
7822 %}
7823 
7824 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7825   predicate(n->as_LoadStore()->result_not_used());
7826   match(Set dummy (GetAndAddS mem add));
7827   effect(KILL cr);
7828   format %{ "ADDW  [$mem],$add" %}
7829   ins_encode %{
7830     __ lock();
7831     __ addw($mem$$Address, $add$$constant);
7832   %}
7833   ins_pipe( pipe_cmpxchg );
7834 %}
7835 
7836 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7837   match(Set newval (GetAndAddS mem newval));
7838   effect(KILL cr);
7839   format %{ "XADDW  [$mem],$newval" %}
7840   ins_encode %{
7841     __ lock();
7842     __ xaddw($mem$$Address, $newval$$Register);
7843   %}
7844   ins_pipe( pipe_cmpxchg );
7845 %}
7846 
7847 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7848   predicate(n->as_LoadStore()->result_not_used());
7849   match(Set dummy (GetAndAddI mem add));
7850   effect(KILL cr);
7851   format %{ "ADDL  [$mem],$add" %}
7852   ins_encode %{
7853     __ lock();
7854     __ addl($mem$$Address, $add$$constant);
7855   %}
7856   ins_pipe( pipe_cmpxchg );
7857 %}
7858 
7859 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7860   match(Set newval (GetAndAddI mem newval));
7861   effect(KILL cr);
7862   format %{ "XADDL  [$mem],$newval" %}
7863   ins_encode %{
7864     __ lock();
7865     __ xaddl($mem$$Address, $newval$$Register);
7866   %}
7867   ins_pipe( pipe_cmpxchg );
7868 %}
7869 
7870 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7871   predicate(n->as_LoadStore()->result_not_used());
7872   match(Set dummy (GetAndAddL mem add));
7873   effect(KILL cr);
7874   format %{ "ADDQ  [$mem],$add" %}
7875   ins_encode %{
7876     __ lock();
7877     __ addq($mem$$Address, $add$$constant);
7878   %}
7879   ins_pipe( pipe_cmpxchg );
7880 %}
7881 
7882 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7883   match(Set newval (GetAndAddL mem newval));
7884   effect(KILL cr);
7885   format %{ "XADDQ  [$mem],$newval" %}
7886   ins_encode %{
7887     __ lock();
7888     __ xaddq($mem$$Address, $newval$$Register);
7889   %}
7890   ins_pipe( pipe_cmpxchg );
7891 %}
7892 
7893 instruct xchgB( memory mem, rRegI newval) %{
7894   match(Set newval (GetAndSetB mem newval));
7895   format %{ "XCHGB  $newval,[$mem]" %}
7896   ins_encode %{
7897     __ xchgb($newval$$Register, $mem$$Address);
7898   %}
7899   ins_pipe( pipe_cmpxchg );
7900 %}
7901 
7902 instruct xchgS( memory mem, rRegI newval) %{
7903   match(Set newval (GetAndSetS mem newval));
7904   format %{ "XCHGW  $newval,[$mem]" %}
7905   ins_encode %{
7906     __ xchgw($newval$$Register, $mem$$Address);
7907   %}
7908   ins_pipe( pipe_cmpxchg );
7909 %}
7910 
7911 instruct xchgI( memory mem, rRegI newval) %{
7912   match(Set newval (GetAndSetI mem newval));
7913   format %{ "XCHGL  $newval,[$mem]" %}
7914   ins_encode %{
7915     __ xchgl($newval$$Register, $mem$$Address);
7916   %}
7917   ins_pipe( pipe_cmpxchg );
7918 %}
7919 
7920 instruct xchgL( memory mem, rRegL newval) %{
7921   match(Set newval (GetAndSetL mem newval));
7922   format %{ "XCHGL  $newval,[$mem]" %}
7923   ins_encode %{
7924     __ xchgq($newval$$Register, $mem$$Address);
7925   %}
7926   ins_pipe( pipe_cmpxchg );
7927 %}
7928 
7929 instruct xchgP( memory mem, rRegP newval) %{
7930   match(Set newval (GetAndSetP mem newval));
7931   format %{ "XCHGQ  $newval,[$mem]" %}
7932   ins_encode %{
7933     __ xchgq($newval$$Register, $mem$$Address);
7934   %}
7935   ins_pipe( pipe_cmpxchg );
7936 %}
7937 
7938 instruct xchgN( memory mem, rRegN newval) %{
7939   match(Set newval (GetAndSetN mem newval));
7940   format %{ "XCHGL  $newval,$mem]" %}
7941   ins_encode %{
7942     __ xchgl($newval$$Register, $mem$$Address);
7943   %}
7944   ins_pipe( pipe_cmpxchg );
7945 %}
7946 
7947 //----------Subtraction Instructions-------------------------------------------
7948 
7949 // Integer Subtraction Instructions
7950 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7951 %{
7952   match(Set dst (SubI dst src));
7953   effect(KILL cr);
7954 
7955   format %{ "subl    $dst, $src\t# int" %}
7956   opcode(0x2B);
7957   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7958   ins_pipe(ialu_reg_reg);
7959 %}
7960 
7961 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7962 %{
7963   match(Set dst (SubI dst src));
7964   effect(KILL cr);
7965 
7966   format %{ "subl    $dst, $src\t# int" %}
7967   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7968   ins_encode(OpcSErm(dst, src), Con8or32(src));
7969   ins_pipe(ialu_reg);
7970 %}
7971 
7972 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7973 %{
7974   match(Set dst (SubI dst (LoadI src)));
7975   effect(KILL cr);
7976 
7977   ins_cost(125);
7978   format %{ "subl    $dst, $src\t# int" %}
7979   opcode(0x2B);
7980   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7981   ins_pipe(ialu_reg_mem);
7982 %}
7983 
7984 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7985 %{
7986   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7987   effect(KILL cr);
7988 
7989   ins_cost(150);
7990   format %{ "subl    $dst, $src\t# int" %}
7991   opcode(0x29); /* Opcode 29 /r */
7992   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7993   ins_pipe(ialu_mem_reg);
7994 %}
7995 
7996 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7997 %{
7998   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7999   effect(KILL cr);
8000 
8001   ins_cost(125); // XXX
8002   format %{ "subl    $dst, $src\t# int" %}
8003   opcode(0x81); /* Opcode 81 /5 id */
8004   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8005   ins_pipe(ialu_mem_imm);
8006 %}
8007 
8008 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8009 %{
8010   match(Set dst (SubL dst src));
8011   effect(KILL cr);
8012 
8013   format %{ "subq    $dst, $src\t# long" %}
8014   opcode(0x2B);
8015   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8016   ins_pipe(ialu_reg_reg);
8017 %}
8018 
8019 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
8020 %{
8021   match(Set dst (SubL dst src));
8022   effect(KILL cr);
8023 
8024   format %{ "subq    $dst, $src\t# long" %}
8025   opcode(0x81, 0x05);  /* Opcode 81 /5 */
8026   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
8027   ins_pipe(ialu_reg);
8028 %}
8029 
8030 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
8031 %{
8032   match(Set dst (SubL dst (LoadL src)));
8033   effect(KILL cr);
8034 
8035   ins_cost(125);
8036   format %{ "subq    $dst, $src\t# long" %}
8037   opcode(0x2B);
8038   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
8039   ins_pipe(ialu_reg_mem);
8040 %}
8041 
8042 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
8043 %{
8044   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8045   effect(KILL cr);
8046 
8047   ins_cost(150);
8048   format %{ "subq    $dst, $src\t# long" %}
8049   opcode(0x29); /* Opcode 29 /r */
8050   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
8051   ins_pipe(ialu_mem_reg);
8052 %}
8053 
8054 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
8055 %{
8056   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
8057   effect(KILL cr);
8058 
8059   ins_cost(125); // XXX
8060   format %{ "subq    $dst, $src\t# long" %}
8061   opcode(0x81); /* Opcode 81 /5 id */
8062   ins_encode(REX_mem_wide(dst),
8063              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
8064   ins_pipe(ialu_mem_imm);
8065 %}
8066 
8067 // Subtract from a pointer
8068 // XXX hmpf???
8069 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
8070 %{
8071   match(Set dst (AddP dst (SubI zero src)));
8072   effect(KILL cr);
8073 
8074   format %{ "subq    $dst, $src\t# ptr - int" %}
8075   opcode(0x2B);
8076   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
8077   ins_pipe(ialu_reg_reg);
8078 %}
8079 
8080 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
8081 %{
8082   match(Set dst (SubI zero dst));
8083   effect(KILL cr);
8084 
8085   format %{ "negl    $dst\t# int" %}
8086   opcode(0xF7, 0x03);  // Opcode F7 /3
8087   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8088   ins_pipe(ialu_reg);
8089 %}
8090 
8091 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
8092 %{
8093   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
8094   effect(KILL cr);
8095 
8096   format %{ "negl    $dst\t# int" %}
8097   opcode(0xF7, 0x03);  // Opcode F7 /3
8098   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8099   ins_pipe(ialu_reg);
8100 %}
8101 
8102 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
8103 %{
8104   match(Set dst (SubL zero dst));
8105   effect(KILL cr);
8106 
8107   format %{ "negq    $dst\t# long" %}
8108   opcode(0xF7, 0x03);  // Opcode F7 /3
8109   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8110   ins_pipe(ialu_reg);
8111 %}
8112 
8113 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
8114 %{
8115   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
8116   effect(KILL cr);
8117 
8118   format %{ "negq    $dst\t# long" %}
8119   opcode(0xF7, 0x03);  // Opcode F7 /3
8120   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8121   ins_pipe(ialu_reg);
8122 %}
8123 
8124 //----------Multiplication/Division Instructions-------------------------------
8125 // Integer Multiplication Instructions
8126 // Multiply Register
8127 
8128 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8129 %{
8130   match(Set dst (MulI dst src));
8131   effect(KILL cr);
8132 
8133   ins_cost(300);
8134   format %{ "imull   $dst, $src\t# int" %}
8135   opcode(0x0F, 0xAF);
8136   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8137   ins_pipe(ialu_reg_reg_alu0);
8138 %}
8139 
8140 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
8141 %{
8142   match(Set dst (MulI src imm));
8143   effect(KILL cr);
8144 
8145   ins_cost(300);
8146   format %{ "imull   $dst, $src, $imm\t# int" %}
8147   opcode(0x69); /* 69 /r id */
8148   ins_encode(REX_reg_reg(dst, src),
8149              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8150   ins_pipe(ialu_reg_reg_alu0);
8151 %}
8152 
8153 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
8154 %{
8155   match(Set dst (MulI dst (LoadI src)));
8156   effect(KILL cr);
8157 
8158   ins_cost(350);
8159   format %{ "imull   $dst, $src\t# int" %}
8160   opcode(0x0F, 0xAF);
8161   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
8162   ins_pipe(ialu_reg_mem_alu0);
8163 %}
8164 
8165 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
8166 %{
8167   match(Set dst (MulI (LoadI src) imm));
8168   effect(KILL cr);
8169 
8170   ins_cost(300);
8171   format %{ "imull   $dst, $src, $imm\t# int" %}
8172   opcode(0x69); /* 69 /r id */
8173   ins_encode(REX_reg_mem(dst, src),
8174              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8175   ins_pipe(ialu_reg_mem_alu0);
8176 %}
8177 
8178 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
8179 %{
8180   match(Set dst (MulL dst src));
8181   effect(KILL cr);
8182 
8183   ins_cost(300);
8184   format %{ "imulq   $dst, $src\t# long" %}
8185   opcode(0x0F, 0xAF);
8186   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
8187   ins_pipe(ialu_reg_reg_alu0);
8188 %}
8189 
8190 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
8191 %{
8192   match(Set dst (MulL src imm));
8193   effect(KILL cr);
8194 
8195   ins_cost(300);
8196   format %{ "imulq   $dst, $src, $imm\t# long" %}
8197   opcode(0x69); /* 69 /r id */
8198   ins_encode(REX_reg_reg_wide(dst, src),
8199              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
8200   ins_pipe(ialu_reg_reg_alu0);
8201 %}
8202 
8203 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
8204 %{
8205   match(Set dst (MulL dst (LoadL src)));
8206   effect(KILL cr);
8207 
8208   ins_cost(350);
8209   format %{ "imulq   $dst, $src\t# long" %}
8210   opcode(0x0F, 0xAF);
8211   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
8212   ins_pipe(ialu_reg_mem_alu0);
8213 %}
8214 
8215 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
8216 %{
8217   match(Set dst (MulL (LoadL src) imm));
8218   effect(KILL cr);
8219 
8220   ins_cost(300);
8221   format %{ "imulq   $dst, $src, $imm\t# long" %}
8222   opcode(0x69); /* 69 /r id */
8223   ins_encode(REX_reg_mem_wide(dst, src),
8224              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
8225   ins_pipe(ialu_reg_mem_alu0);
8226 %}
8227 
8228 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8229 %{
8230   match(Set dst (MulHiL src rax));
8231   effect(USE_KILL rax, KILL cr);
8232 
8233   ins_cost(300);
8234   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
8235   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8236   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8237   ins_pipe(ialu_reg_reg_alu0);
8238 %}
8239 
8240 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8241                    rFlagsReg cr)
8242 %{
8243   match(Set rax (DivI rax div));
8244   effect(KILL rdx, KILL cr);
8245 
8246   ins_cost(30*100+10*100); // XXX
8247   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8248             "jne,s   normal\n\t"
8249             "xorl    rdx, rdx\n\t"
8250             "cmpl    $div, -1\n\t"
8251             "je,s    done\n"
8252     "normal: cdql\n\t"
8253             "idivl   $div\n"
8254     "done:"        %}
8255   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8256   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8257   ins_pipe(ialu_reg_reg_alu0);
8258 %}
8259 
8260 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8261                    rFlagsReg cr)
8262 %{
8263   match(Set rax (DivL rax div));
8264   effect(KILL rdx, KILL cr);
8265 
8266   ins_cost(30*100+10*100); // XXX
8267   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8268             "cmpq    rax, rdx\n\t"
8269             "jne,s   normal\n\t"
8270             "xorl    rdx, rdx\n\t"
8271             "cmpq    $div, -1\n\t"
8272             "je,s    done\n"
8273     "normal: cdqq\n\t"
8274             "idivq   $div\n"
8275     "done:"        %}
8276   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8277   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8278   ins_pipe(ialu_reg_reg_alu0);
8279 %}
8280 
8281 // Integer DIVMOD with Register, both quotient and mod results
8282 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8283                              rFlagsReg cr)
8284 %{
8285   match(DivModI rax div);
8286   effect(KILL cr);
8287 
8288   ins_cost(30*100+10*100); // XXX
8289   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8290             "jne,s   normal\n\t"
8291             "xorl    rdx, rdx\n\t"
8292             "cmpl    $div, -1\n\t"
8293             "je,s    done\n"
8294     "normal: cdql\n\t"
8295             "idivl   $div\n"
8296     "done:"        %}
8297   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8298   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8299   ins_pipe(pipe_slow);
8300 %}
8301 
8302 // Long DIVMOD with Register, both quotient and mod results
8303 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8304                              rFlagsReg cr)
8305 %{
8306   match(DivModL rax div);
8307   effect(KILL cr);
8308 
8309   ins_cost(30*100+10*100); // XXX
8310   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8311             "cmpq    rax, rdx\n\t"
8312             "jne,s   normal\n\t"
8313             "xorl    rdx, rdx\n\t"
8314             "cmpq    $div, -1\n\t"
8315             "je,s    done\n"
8316     "normal: cdqq\n\t"
8317             "idivq   $div\n"
8318     "done:"        %}
8319   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8320   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8321   ins_pipe(pipe_slow);
8322 %}
8323 
8324 //----------- DivL-By-Constant-Expansions--------------------------------------
8325 // DivI cases are handled by the compiler
8326 
8327 // Magic constant, reciprocal of 10
8328 instruct loadConL_0x6666666666666667(rRegL dst)
8329 %{
8330   effect(DEF dst);
8331 
8332   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8333   ins_encode(load_immL(dst, 0x6666666666666667));
8334   ins_pipe(ialu_reg);
8335 %}
8336 
8337 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8338 %{
8339   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8340 
8341   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8342   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8343   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8344   ins_pipe(ialu_reg_reg_alu0);
8345 %}
8346 
8347 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8348 %{
8349   effect(USE_DEF dst, KILL cr);
8350 
8351   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8352   opcode(0xC1, 0x7); /* C1 /7 ib */
8353   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8354   ins_pipe(ialu_reg);
8355 %}
8356 
8357 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8358 %{
8359   effect(USE_DEF dst, KILL cr);
8360 
8361   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8362   opcode(0xC1, 0x7); /* C1 /7 ib */
8363   ins_encode(reg_opc_imm_wide(dst, 0x2));
8364   ins_pipe(ialu_reg);
8365 %}
8366 
8367 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8368 %{
8369   match(Set dst (DivL src div));
8370 
8371   ins_cost((5+8)*100);
8372   expand %{
8373     rax_RegL rax;                     // Killed temp
8374     rFlagsReg cr;                     // Killed
8375     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8376     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8377     sarL_rReg_63(src, cr);            // sarq  src, 63
8378     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8379     subL_rReg(dst, src, cr);          // subl  rdx, src
8380   %}
8381 %}
8382 
8383 //-----------------------------------------------------------------------------
8384 
8385 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8386                    rFlagsReg cr)
8387 %{
8388   match(Set rdx (ModI rax div));
8389   effect(KILL rax, KILL cr);
8390 
8391   ins_cost(300); // XXX
8392   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8393             "jne,s   normal\n\t"
8394             "xorl    rdx, rdx\n\t"
8395             "cmpl    $div, -1\n\t"
8396             "je,s    done\n"
8397     "normal: cdql\n\t"
8398             "idivl   $div\n"
8399     "done:"        %}
8400   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8401   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8402   ins_pipe(ialu_reg_reg_alu0);
8403 %}
8404 
8405 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8406                    rFlagsReg cr)
8407 %{
8408   match(Set rdx (ModL rax div));
8409   effect(KILL rax, KILL cr);
8410 
8411   ins_cost(300); // XXX
8412   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8413             "cmpq    rax, rdx\n\t"
8414             "jne,s   normal\n\t"
8415             "xorl    rdx, rdx\n\t"
8416             "cmpq    $div, -1\n\t"
8417             "je,s    done\n"
8418     "normal: cdqq\n\t"
8419             "idivq   $div\n"
8420     "done:"        %}
8421   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8422   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8423   ins_pipe(ialu_reg_reg_alu0);
8424 %}
8425 
8426 // Integer Shift Instructions
8427 // Shift Left by one
8428 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8429 %{
8430   match(Set dst (LShiftI dst shift));
8431   effect(KILL cr);
8432 
8433   format %{ "sall    $dst, $shift" %}
8434   opcode(0xD1, 0x4); /* D1 /4 */
8435   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8436   ins_pipe(ialu_reg);
8437 %}
8438 
8439 // Shift Left by one
8440 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8441 %{
8442   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8443   effect(KILL cr);
8444 
8445   format %{ "sall    $dst, $shift\t" %}
8446   opcode(0xD1, 0x4); /* D1 /4 */
8447   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8448   ins_pipe(ialu_mem_imm);
8449 %}
8450 
8451 // Shift Left by 8-bit immediate
8452 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8453 %{
8454   match(Set dst (LShiftI dst shift));
8455   effect(KILL cr);
8456 
8457   format %{ "sall    $dst, $shift" %}
8458   opcode(0xC1, 0x4); /* C1 /4 ib */
8459   ins_encode(reg_opc_imm(dst, shift));
8460   ins_pipe(ialu_reg);
8461 %}
8462 
8463 // Shift Left by 8-bit immediate
8464 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8465 %{
8466   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8467   effect(KILL cr);
8468 
8469   format %{ "sall    $dst, $shift" %}
8470   opcode(0xC1, 0x4); /* C1 /4 ib */
8471   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8472   ins_pipe(ialu_mem_imm);
8473 %}
8474 
8475 // Shift Left by variable
8476 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8477 %{
8478   match(Set dst (LShiftI dst shift));
8479   effect(KILL cr);
8480 
8481   format %{ "sall    $dst, $shift" %}
8482   opcode(0xD3, 0x4); /* D3 /4 */
8483   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8484   ins_pipe(ialu_reg_reg);
8485 %}
8486 
8487 // Shift Left by variable
8488 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8489 %{
8490   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8491   effect(KILL cr);
8492 
8493   format %{ "sall    $dst, $shift" %}
8494   opcode(0xD3, 0x4); /* D3 /4 */
8495   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8496   ins_pipe(ialu_mem_reg);
8497 %}
8498 
8499 // Arithmetic shift right by one
8500 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8501 %{
8502   match(Set dst (RShiftI dst shift));
8503   effect(KILL cr);
8504 
8505   format %{ "sarl    $dst, $shift" %}
8506   opcode(0xD1, 0x7); /* D1 /7 */
8507   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8508   ins_pipe(ialu_reg);
8509 %}
8510 
8511 // Arithmetic shift right by one
8512 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8513 %{
8514   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8515   effect(KILL cr);
8516 
8517   format %{ "sarl    $dst, $shift" %}
8518   opcode(0xD1, 0x7); /* D1 /7 */
8519   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8520   ins_pipe(ialu_mem_imm);
8521 %}
8522 
8523 // Arithmetic Shift Right by 8-bit immediate
8524 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8525 %{
8526   match(Set dst (RShiftI dst shift));
8527   effect(KILL cr);
8528 
8529   format %{ "sarl    $dst, $shift" %}
8530   opcode(0xC1, 0x7); /* C1 /7 ib */
8531   ins_encode(reg_opc_imm(dst, shift));
8532   ins_pipe(ialu_mem_imm);
8533 %}
8534 
8535 // Arithmetic Shift Right by 8-bit immediate
8536 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8537 %{
8538   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8539   effect(KILL cr);
8540 
8541   format %{ "sarl    $dst, $shift" %}
8542   opcode(0xC1, 0x7); /* C1 /7 ib */
8543   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8544   ins_pipe(ialu_mem_imm);
8545 %}
8546 
8547 // Arithmetic Shift Right by variable
8548 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8549 %{
8550   match(Set dst (RShiftI dst shift));
8551   effect(KILL cr);
8552 
8553   format %{ "sarl    $dst, $shift" %}
8554   opcode(0xD3, 0x7); /* D3 /7 */
8555   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8556   ins_pipe(ialu_reg_reg);
8557 %}
8558 
8559 // Arithmetic Shift Right by variable
8560 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8561 %{
8562   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8563   effect(KILL cr);
8564 
8565   format %{ "sarl    $dst, $shift" %}
8566   opcode(0xD3, 0x7); /* D3 /7 */
8567   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8568   ins_pipe(ialu_mem_reg);
8569 %}
8570 
8571 // Logical shift right by one
8572 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8573 %{
8574   match(Set dst (URShiftI dst shift));
8575   effect(KILL cr);
8576 
8577   format %{ "shrl    $dst, $shift" %}
8578   opcode(0xD1, 0x5); /* D1 /5 */
8579   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8580   ins_pipe(ialu_reg);
8581 %}
8582 
8583 // Logical shift right by one
8584 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8585 %{
8586   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8587   effect(KILL cr);
8588 
8589   format %{ "shrl    $dst, $shift" %}
8590   opcode(0xD1, 0x5); /* D1 /5 */
8591   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8592   ins_pipe(ialu_mem_imm);
8593 %}
8594 
8595 // Logical Shift Right by 8-bit immediate
8596 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8597 %{
8598   match(Set dst (URShiftI dst shift));
8599   effect(KILL cr);
8600 
8601   format %{ "shrl    $dst, $shift" %}
8602   opcode(0xC1, 0x5); /* C1 /5 ib */
8603   ins_encode(reg_opc_imm(dst, shift));
8604   ins_pipe(ialu_reg);
8605 %}
8606 
8607 // Logical Shift Right by 8-bit immediate
8608 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8609 %{
8610   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8611   effect(KILL cr);
8612 
8613   format %{ "shrl    $dst, $shift" %}
8614   opcode(0xC1, 0x5); /* C1 /5 ib */
8615   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8616   ins_pipe(ialu_mem_imm);
8617 %}
8618 
8619 // Logical Shift Right by variable
8620 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8621 %{
8622   match(Set dst (URShiftI dst shift));
8623   effect(KILL cr);
8624 
8625   format %{ "shrl    $dst, $shift" %}
8626   opcode(0xD3, 0x5); /* D3 /5 */
8627   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8628   ins_pipe(ialu_reg_reg);
8629 %}
8630 
8631 // Logical Shift Right by variable
8632 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8633 %{
8634   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8635   effect(KILL cr);
8636 
8637   format %{ "shrl    $dst, $shift" %}
8638   opcode(0xD3, 0x5); /* D3 /5 */
8639   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8640   ins_pipe(ialu_mem_reg);
8641 %}
8642 
8643 // Long Shift Instructions
8644 // Shift Left by one
8645 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8646 %{
8647   match(Set dst (LShiftL dst shift));
8648   effect(KILL cr);
8649 
8650   format %{ "salq    $dst, $shift" %}
8651   opcode(0xD1, 0x4); /* D1 /4 */
8652   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8653   ins_pipe(ialu_reg);
8654 %}
8655 
8656 // Shift Left by one
8657 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8658 %{
8659   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8660   effect(KILL cr);
8661 
8662   format %{ "salq    $dst, $shift" %}
8663   opcode(0xD1, 0x4); /* D1 /4 */
8664   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8665   ins_pipe(ialu_mem_imm);
8666 %}
8667 
8668 // Shift Left by 8-bit immediate
8669 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8670 %{
8671   match(Set dst (LShiftL dst shift));
8672   effect(KILL cr);
8673 
8674   format %{ "salq    $dst, $shift" %}
8675   opcode(0xC1, 0x4); /* C1 /4 ib */
8676   ins_encode(reg_opc_imm_wide(dst, shift));
8677   ins_pipe(ialu_reg);
8678 %}
8679 
8680 // Shift Left by 8-bit immediate
8681 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8682 %{
8683   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8684   effect(KILL cr);
8685 
8686   format %{ "salq    $dst, $shift" %}
8687   opcode(0xC1, 0x4); /* C1 /4 ib */
8688   ins_encode(REX_mem_wide(dst), OpcP,
8689              RM_opc_mem(secondary, dst), Con8or32(shift));
8690   ins_pipe(ialu_mem_imm);
8691 %}
8692 
8693 // Shift Left by variable
8694 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8695 %{
8696   match(Set dst (LShiftL dst shift));
8697   effect(KILL cr);
8698 
8699   format %{ "salq    $dst, $shift" %}
8700   opcode(0xD3, 0x4); /* D3 /4 */
8701   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8702   ins_pipe(ialu_reg_reg);
8703 %}
8704 
8705 // Shift Left by variable
8706 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8707 %{
8708   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8709   effect(KILL cr);
8710 
8711   format %{ "salq    $dst, $shift" %}
8712   opcode(0xD3, 0x4); /* D3 /4 */
8713   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8714   ins_pipe(ialu_mem_reg);
8715 %}
8716 
8717 // Arithmetic shift right by one
8718 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8719 %{
8720   match(Set dst (RShiftL dst shift));
8721   effect(KILL cr);
8722 
8723   format %{ "sarq    $dst, $shift" %}
8724   opcode(0xD1, 0x7); /* D1 /7 */
8725   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8726   ins_pipe(ialu_reg);
8727 %}
8728 
8729 // Arithmetic shift right by one
8730 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8731 %{
8732   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8733   effect(KILL cr);
8734 
8735   format %{ "sarq    $dst, $shift" %}
8736   opcode(0xD1, 0x7); /* D1 /7 */
8737   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8738   ins_pipe(ialu_mem_imm);
8739 %}
8740 
8741 // Arithmetic Shift Right by 8-bit immediate
8742 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8743 %{
8744   match(Set dst (RShiftL dst shift));
8745   effect(KILL cr);
8746 
8747   format %{ "sarq    $dst, $shift" %}
8748   opcode(0xC1, 0x7); /* C1 /7 ib */
8749   ins_encode(reg_opc_imm_wide(dst, shift));
8750   ins_pipe(ialu_mem_imm);
8751 %}
8752 
8753 // Arithmetic Shift Right by 8-bit immediate
8754 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8755 %{
8756   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8757   effect(KILL cr);
8758 
8759   format %{ "sarq    $dst, $shift" %}
8760   opcode(0xC1, 0x7); /* C1 /7 ib */
8761   ins_encode(REX_mem_wide(dst), OpcP,
8762              RM_opc_mem(secondary, dst), Con8or32(shift));
8763   ins_pipe(ialu_mem_imm);
8764 %}
8765 
8766 // Arithmetic Shift Right by variable
8767 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8768 %{
8769   match(Set dst (RShiftL dst shift));
8770   effect(KILL cr);
8771 
8772   format %{ "sarq    $dst, $shift" %}
8773   opcode(0xD3, 0x7); /* D3 /7 */
8774   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8775   ins_pipe(ialu_reg_reg);
8776 %}
8777 
8778 // Arithmetic Shift Right by variable
8779 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8780 %{
8781   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8782   effect(KILL cr);
8783 
8784   format %{ "sarq    $dst, $shift" %}
8785   opcode(0xD3, 0x7); /* D3 /7 */
8786   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8787   ins_pipe(ialu_mem_reg);
8788 %}
8789 
8790 // Logical shift right by one
8791 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8792 %{
8793   match(Set dst (URShiftL dst shift));
8794   effect(KILL cr);
8795 
8796   format %{ "shrq    $dst, $shift" %}
8797   opcode(0xD1, 0x5); /* D1 /5 */
8798   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8799   ins_pipe(ialu_reg);
8800 %}
8801 
8802 // Logical shift right by one
8803 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8804 %{
8805   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8806   effect(KILL cr);
8807 
8808   format %{ "shrq    $dst, $shift" %}
8809   opcode(0xD1, 0x5); /* D1 /5 */
8810   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8811   ins_pipe(ialu_mem_imm);
8812 %}
8813 
8814 // Logical Shift Right by 8-bit immediate
8815 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8816 %{
8817   match(Set dst (URShiftL dst shift));
8818   effect(KILL cr);
8819 
8820   format %{ "shrq    $dst, $shift" %}
8821   opcode(0xC1, 0x5); /* C1 /5 ib */
8822   ins_encode(reg_opc_imm_wide(dst, shift));
8823   ins_pipe(ialu_reg);
8824 %}
8825 
8826 
8827 // Logical Shift Right by 8-bit immediate
8828 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8829 %{
8830   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8831   effect(KILL cr);
8832 
8833   format %{ "shrq    $dst, $shift" %}
8834   opcode(0xC1, 0x5); /* C1 /5 ib */
8835   ins_encode(REX_mem_wide(dst), OpcP,
8836              RM_opc_mem(secondary, dst), Con8or32(shift));
8837   ins_pipe(ialu_mem_imm);
8838 %}
8839 
8840 // Logical Shift Right by variable
8841 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8842 %{
8843   match(Set dst (URShiftL dst shift));
8844   effect(KILL cr);
8845 
8846   format %{ "shrq    $dst, $shift" %}
8847   opcode(0xD3, 0x5); /* D3 /5 */
8848   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8849   ins_pipe(ialu_reg_reg);
8850 %}
8851 
8852 // Logical Shift Right by variable
8853 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8854 %{
8855   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8856   effect(KILL cr);
8857 
8858   format %{ "shrq    $dst, $shift" %}
8859   opcode(0xD3, 0x5); /* D3 /5 */
8860   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8861   ins_pipe(ialu_mem_reg);
8862 %}
8863 
8864 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8865 // This idiom is used by the compiler for the i2b bytecode.
8866 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8867 %{
8868   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8869 
8870   format %{ "movsbl  $dst, $src\t# i2b" %}
8871   opcode(0x0F, 0xBE);
8872   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8873   ins_pipe(ialu_reg_reg);
8874 %}
8875 
8876 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8877 // This idiom is used by the compiler the i2s bytecode.
8878 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8879 %{
8880   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8881 
8882   format %{ "movswl  $dst, $src\t# i2s" %}
8883   opcode(0x0F, 0xBF);
8884   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8885   ins_pipe(ialu_reg_reg);
8886 %}
8887 
8888 // ROL/ROR instructions
8889 
8890 // ROL expand
8891 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8892   effect(KILL cr, USE_DEF dst);
8893 
8894   format %{ "roll    $dst" %}
8895   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8896   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8897   ins_pipe(ialu_reg);
8898 %}
8899 
8900 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8901   effect(USE_DEF dst, USE shift, KILL cr);
8902 
8903   format %{ "roll    $dst, $shift" %}
8904   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8905   ins_encode( reg_opc_imm(dst, shift) );
8906   ins_pipe(ialu_reg);
8907 %}
8908 
8909 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8910 %{
8911   effect(USE_DEF dst, USE shift, KILL cr);
8912 
8913   format %{ "roll    $dst, $shift" %}
8914   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8915   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8916   ins_pipe(ialu_reg_reg);
8917 %}
8918 // end of ROL expand
8919 
8920 // Rotate Left by one
8921 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8922 %{
8923   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8924 
8925   expand %{
8926     rolI_rReg_imm1(dst, cr);
8927   %}
8928 %}
8929 
8930 // Rotate Left by 8-bit immediate
8931 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8932 %{
8933   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8934   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8935 
8936   expand %{
8937     rolI_rReg_imm8(dst, lshift, cr);
8938   %}
8939 %}
8940 
8941 // Rotate Left by variable
8942 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8943 %{
8944   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8945 
8946   expand %{
8947     rolI_rReg_CL(dst, shift, cr);
8948   %}
8949 %}
8950 
8951 // Rotate Left by variable
8952 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8953 %{
8954   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8955 
8956   expand %{
8957     rolI_rReg_CL(dst, shift, cr);
8958   %}
8959 %}
8960 
8961 // ROR expand
8962 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8963 %{
8964   effect(USE_DEF dst, KILL cr);
8965 
8966   format %{ "rorl    $dst" %}
8967   opcode(0xD1, 0x1); /* D1 /1 */
8968   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8969   ins_pipe(ialu_reg);
8970 %}
8971 
8972 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8973 %{
8974   effect(USE_DEF dst, USE shift, KILL cr);
8975 
8976   format %{ "rorl    $dst, $shift" %}
8977   opcode(0xC1, 0x1); /* C1 /1 ib */
8978   ins_encode(reg_opc_imm(dst, shift));
8979   ins_pipe(ialu_reg);
8980 %}
8981 
8982 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8983 %{
8984   effect(USE_DEF dst, USE shift, KILL cr);
8985 
8986   format %{ "rorl    $dst, $shift" %}
8987   opcode(0xD3, 0x1); /* D3 /1 */
8988   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8989   ins_pipe(ialu_reg_reg);
8990 %}
8991 // end of ROR expand
8992 
8993 // Rotate Right by one
8994 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8995 %{
8996   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8997 
8998   expand %{
8999     rorI_rReg_imm1(dst, cr);
9000   %}
9001 %}
9002 
9003 // Rotate Right by 8-bit immediate
9004 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9005 %{
9006   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
9007   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
9008 
9009   expand %{
9010     rorI_rReg_imm8(dst, rshift, cr);
9011   %}
9012 %}
9013 
9014 // Rotate Right by variable
9015 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9016 %{
9017   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
9018 
9019   expand %{
9020     rorI_rReg_CL(dst, shift, cr);
9021   %}
9022 %}
9023 
9024 // Rotate Right by variable
9025 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
9026 %{
9027   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
9028 
9029   expand %{
9030     rorI_rReg_CL(dst, shift, cr);
9031   %}
9032 %}
9033 
9034 // for long rotate
9035 // ROL expand
9036 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
9037   effect(USE_DEF dst, KILL cr);
9038 
9039   format %{ "rolq    $dst" %}
9040   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
9041   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9042   ins_pipe(ialu_reg);
9043 %}
9044 
9045 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
9046   effect(USE_DEF dst, USE shift, KILL cr);
9047 
9048   format %{ "rolq    $dst, $shift" %}
9049   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
9050   ins_encode( reg_opc_imm_wide(dst, shift) );
9051   ins_pipe(ialu_reg);
9052 %}
9053 
9054 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9055 %{
9056   effect(USE_DEF dst, USE shift, KILL cr);
9057 
9058   format %{ "rolq    $dst, $shift" %}
9059   opcode(0xD3, 0x0); /* Opcode D3 /0 */
9060   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9061   ins_pipe(ialu_reg_reg);
9062 %}
9063 // end of ROL expand
9064 
9065 // Rotate Left by one
9066 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
9067 %{
9068   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9069 
9070   expand %{
9071     rolL_rReg_imm1(dst, cr);
9072   %}
9073 %}
9074 
9075 // Rotate Left by 8-bit immediate
9076 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
9077 %{
9078   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9079   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
9080 
9081   expand %{
9082     rolL_rReg_imm8(dst, lshift, cr);
9083   %}
9084 %}
9085 
9086 // Rotate Left by variable
9087 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9088 %{
9089   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
9090 
9091   expand %{
9092     rolL_rReg_CL(dst, shift, cr);
9093   %}
9094 %}
9095 
9096 // Rotate Left by variable
9097 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9098 %{
9099   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
9100 
9101   expand %{
9102     rolL_rReg_CL(dst, shift, cr);
9103   %}
9104 %}
9105 
9106 // ROR expand
9107 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
9108 %{
9109   effect(USE_DEF dst, KILL cr);
9110 
9111   format %{ "rorq    $dst" %}
9112   opcode(0xD1, 0x1); /* D1 /1 */
9113   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9114   ins_pipe(ialu_reg);
9115 %}
9116 
9117 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
9118 %{
9119   effect(USE_DEF dst, USE shift, KILL cr);
9120 
9121   format %{ "rorq    $dst, $shift" %}
9122   opcode(0xC1, 0x1); /* C1 /1 ib */
9123   ins_encode(reg_opc_imm_wide(dst, shift));
9124   ins_pipe(ialu_reg);
9125 %}
9126 
9127 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
9128 %{
9129   effect(USE_DEF dst, USE shift, KILL cr);
9130 
9131   format %{ "rorq    $dst, $shift" %}
9132   opcode(0xD3, 0x1); /* D3 /1 */
9133   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
9134   ins_pipe(ialu_reg_reg);
9135 %}
9136 // end of ROR expand
9137 
9138 // Rotate Right by one
9139 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
9140 %{
9141   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9142 
9143   expand %{
9144     rorL_rReg_imm1(dst, cr);
9145   %}
9146 %}
9147 
9148 // Rotate Right by 8-bit immediate
9149 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
9150 %{
9151   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
9152   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
9153 
9154   expand %{
9155     rorL_rReg_imm8(dst, rshift, cr);
9156   %}
9157 %}
9158 
9159 // Rotate Right by variable
9160 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
9161 %{
9162   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
9163 
9164   expand %{
9165     rorL_rReg_CL(dst, shift, cr);
9166   %}
9167 %}
9168 
9169 // Rotate Right by variable
9170 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
9171 %{
9172   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
9173 
9174   expand %{
9175     rorL_rReg_CL(dst, shift, cr);
9176   %}
9177 %}
9178 
9179 // Logical Instructions
9180 
9181 // Integer Logical Instructions
9182 
9183 // And Instructions
9184 // And Register with Register
9185 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9186 %{
9187   match(Set dst (AndI dst src));
9188   effect(KILL cr);
9189 
9190   format %{ "andl    $dst, $src\t# int" %}
9191   opcode(0x23);
9192   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9193   ins_pipe(ialu_reg_reg);
9194 %}
9195 
9196 // And Register with Immediate 255
9197 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
9198 %{
9199   match(Set dst (AndI dst src));
9200 
9201   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
9202   opcode(0x0F, 0xB6);
9203   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9204   ins_pipe(ialu_reg);
9205 %}
9206 
9207 // And Register with Immediate 255 and promote to long
9208 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
9209 %{
9210   match(Set dst (ConvI2L (AndI src mask)));
9211 
9212   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
9213   opcode(0x0F, 0xB6);
9214   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9215   ins_pipe(ialu_reg);
9216 %}
9217 
9218 // And Register with Immediate 65535
9219 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
9220 %{
9221   match(Set dst (AndI dst src));
9222 
9223   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
9224   opcode(0x0F, 0xB7);
9225   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9226   ins_pipe(ialu_reg);
9227 %}
9228 
9229 // And Register with Immediate 65535 and promote to long
9230 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
9231 %{
9232   match(Set dst (ConvI2L (AndI src mask)));
9233 
9234   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
9235   opcode(0x0F, 0xB7);
9236   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
9237   ins_pipe(ialu_reg);
9238 %}
9239 
9240 // And Register with Immediate
9241 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9242 %{
9243   match(Set dst (AndI dst src));
9244   effect(KILL cr);
9245 
9246   format %{ "andl    $dst, $src\t# int" %}
9247   opcode(0x81, 0x04); /* Opcode 81 /4 */
9248   ins_encode(OpcSErm(dst, src), Con8or32(src));
9249   ins_pipe(ialu_reg);
9250 %}
9251 
9252 // And Register with Memory
9253 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9254 %{
9255   match(Set dst (AndI dst (LoadI src)));
9256   effect(KILL cr);
9257 
9258   ins_cost(125);
9259   format %{ "andl    $dst, $src\t# int" %}
9260   opcode(0x23);
9261   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9262   ins_pipe(ialu_reg_mem);
9263 %}
9264 
9265 // And Memory with Register
9266 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9267 %{
9268   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9269   effect(KILL cr);
9270 
9271   ins_cost(150);
9272   format %{ "andl    $dst, $src\t# int" %}
9273   opcode(0x21); /* Opcode 21 /r */
9274   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9275   ins_pipe(ialu_mem_reg);
9276 %}
9277 
9278 // And Memory with Immediate
9279 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9280 %{
9281   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9282   effect(KILL cr);
9283 
9284   ins_cost(125);
9285   format %{ "andl    $dst, $src\t# int" %}
9286   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9287   ins_encode(REX_mem(dst), OpcSE(src),
9288              RM_opc_mem(secondary, dst), Con8or32(src));
9289   ins_pipe(ialu_mem_imm);
9290 %}
9291 
9292 // BMI1 instructions
9293 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9294   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9295   predicate(UseBMI1Instructions);
9296   effect(KILL cr);
9297 
9298   ins_cost(125);
9299   format %{ "andnl  $dst, $src1, $src2" %}
9300 
9301   ins_encode %{
9302     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9303   %}
9304   ins_pipe(ialu_reg_mem);
9305 %}
9306 
9307 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9308   match(Set dst (AndI (XorI src1 minus_1) src2));
9309   predicate(UseBMI1Instructions);
9310   effect(KILL cr);
9311 
9312   format %{ "andnl  $dst, $src1, $src2" %}
9313 
9314   ins_encode %{
9315     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9316   %}
9317   ins_pipe(ialu_reg);
9318 %}
9319 
9320 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9321   match(Set dst (AndI (SubI imm_zero src) src));
9322   predicate(UseBMI1Instructions);
9323   effect(KILL cr);
9324 
9325   format %{ "blsil  $dst, $src" %}
9326 
9327   ins_encode %{
9328     __ blsil($dst$$Register, $src$$Register);
9329   %}
9330   ins_pipe(ialu_reg);
9331 %}
9332 
9333 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9334   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9335   predicate(UseBMI1Instructions);
9336   effect(KILL cr);
9337 
9338   ins_cost(125);
9339   format %{ "blsil  $dst, $src" %}
9340 
9341   ins_encode %{
9342     __ blsil($dst$$Register, $src$$Address);
9343   %}
9344   ins_pipe(ialu_reg_mem);
9345 %}
9346 
9347 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9348 %{
9349   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9350   predicate(UseBMI1Instructions);
9351   effect(KILL cr);
9352 
9353   ins_cost(125);
9354   format %{ "blsmskl $dst, $src" %}
9355 
9356   ins_encode %{
9357     __ blsmskl($dst$$Register, $src$$Address);
9358   %}
9359   ins_pipe(ialu_reg_mem);
9360 %}
9361 
9362 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9363 %{
9364   match(Set dst (XorI (AddI src minus_1) src));
9365   predicate(UseBMI1Instructions);
9366   effect(KILL cr);
9367 
9368   format %{ "blsmskl $dst, $src" %}
9369 
9370   ins_encode %{
9371     __ blsmskl($dst$$Register, $src$$Register);
9372   %}
9373 
9374   ins_pipe(ialu_reg);
9375 %}
9376 
9377 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9378 %{
9379   match(Set dst (AndI (AddI src minus_1) src) );
9380   predicate(UseBMI1Instructions);
9381   effect(KILL cr);
9382 
9383   format %{ "blsrl  $dst, $src" %}
9384 
9385   ins_encode %{
9386     __ blsrl($dst$$Register, $src$$Register);
9387   %}
9388 
9389   ins_pipe(ialu_reg_mem);
9390 %}
9391 
9392 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9393 %{
9394   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9395   predicate(UseBMI1Instructions);
9396   effect(KILL cr);
9397 
9398   ins_cost(125);
9399   format %{ "blsrl  $dst, $src" %}
9400 
9401   ins_encode %{
9402     __ blsrl($dst$$Register, $src$$Address);
9403   %}
9404 
9405   ins_pipe(ialu_reg);
9406 %}
9407 
9408 // Or Instructions
9409 // Or Register with Register
9410 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9411 %{
9412   match(Set dst (OrI dst src));
9413   effect(KILL cr);
9414 
9415   format %{ "orl     $dst, $src\t# int" %}
9416   opcode(0x0B);
9417   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9418   ins_pipe(ialu_reg_reg);
9419 %}
9420 
9421 // Or Register with Immediate
9422 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9423 %{
9424   match(Set dst (OrI dst src));
9425   effect(KILL cr);
9426 
9427   format %{ "orl     $dst, $src\t# int" %}
9428   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9429   ins_encode(OpcSErm(dst, src), Con8or32(src));
9430   ins_pipe(ialu_reg);
9431 %}
9432 
9433 // Or Register with Memory
9434 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9435 %{
9436   match(Set dst (OrI dst (LoadI src)));
9437   effect(KILL cr);
9438 
9439   ins_cost(125);
9440   format %{ "orl     $dst, $src\t# int" %}
9441   opcode(0x0B);
9442   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9443   ins_pipe(ialu_reg_mem);
9444 %}
9445 
9446 // Or Memory with Register
9447 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9448 %{
9449   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9450   effect(KILL cr);
9451 
9452   ins_cost(150);
9453   format %{ "orl     $dst, $src\t# int" %}
9454   opcode(0x09); /* Opcode 09 /r */
9455   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9456   ins_pipe(ialu_mem_reg);
9457 %}
9458 
9459 // Or Memory with Immediate
9460 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9461 %{
9462   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9463   effect(KILL cr);
9464 
9465   ins_cost(125);
9466   format %{ "orl     $dst, $src\t# int" %}
9467   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9468   ins_encode(REX_mem(dst), OpcSE(src),
9469              RM_opc_mem(secondary, dst), Con8or32(src));
9470   ins_pipe(ialu_mem_imm);
9471 %}
9472 
9473 // Xor Instructions
9474 // Xor Register with Register
9475 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9476 %{
9477   match(Set dst (XorI dst src));
9478   effect(KILL cr);
9479 
9480   format %{ "xorl    $dst, $src\t# int" %}
9481   opcode(0x33);
9482   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9483   ins_pipe(ialu_reg_reg);
9484 %}
9485 
9486 // Xor Register with Immediate -1
9487 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9488   match(Set dst (XorI dst imm));
9489 
9490   format %{ "not    $dst" %}
9491   ins_encode %{
9492      __ notl($dst$$Register);
9493   %}
9494   ins_pipe(ialu_reg);
9495 %}
9496 
9497 // Xor Register with Immediate
9498 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9499 %{
9500   match(Set dst (XorI dst src));
9501   effect(KILL cr);
9502 
9503   format %{ "xorl    $dst, $src\t# int" %}
9504   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9505   ins_encode(OpcSErm(dst, src), Con8or32(src));
9506   ins_pipe(ialu_reg);
9507 %}
9508 
9509 // Xor Register with Memory
9510 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9511 %{
9512   match(Set dst (XorI dst (LoadI src)));
9513   effect(KILL cr);
9514 
9515   ins_cost(125);
9516   format %{ "xorl    $dst, $src\t# int" %}
9517   opcode(0x33);
9518   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9519   ins_pipe(ialu_reg_mem);
9520 %}
9521 
9522 // Xor Memory with Register
9523 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9524 %{
9525   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9526   effect(KILL cr);
9527 
9528   ins_cost(150);
9529   format %{ "xorl    $dst, $src\t# int" %}
9530   opcode(0x31); /* Opcode 31 /r */
9531   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9532   ins_pipe(ialu_mem_reg);
9533 %}
9534 
9535 // Xor Memory with Immediate
9536 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9537 %{
9538   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9539   effect(KILL cr);
9540 
9541   ins_cost(125);
9542   format %{ "xorl    $dst, $src\t# int" %}
9543   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9544   ins_encode(REX_mem(dst), OpcSE(src),
9545              RM_opc_mem(secondary, dst), Con8or32(src));
9546   ins_pipe(ialu_mem_imm);
9547 %}
9548 
9549 
9550 // Long Logical Instructions
9551 
9552 // And Instructions
9553 // And Register with Register
9554 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9555 %{
9556   match(Set dst (AndL dst src));
9557   effect(KILL cr);
9558 
9559   format %{ "andq    $dst, $src\t# long" %}
9560   opcode(0x23);
9561   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9562   ins_pipe(ialu_reg_reg);
9563 %}
9564 
9565 // And Register with Immediate 255
9566 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9567 %{
9568   match(Set dst (AndL dst src));
9569 
9570   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9571   opcode(0x0F, 0xB6);
9572   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9573   ins_pipe(ialu_reg);
9574 %}
9575 
9576 // And Register with Immediate 65535
9577 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9578 %{
9579   match(Set dst (AndL dst src));
9580 
9581   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9582   opcode(0x0F, 0xB7);
9583   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9584   ins_pipe(ialu_reg);
9585 %}
9586 
9587 // And Register with Immediate
9588 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9589 %{
9590   match(Set dst (AndL dst src));
9591   effect(KILL cr);
9592 
9593   format %{ "andq    $dst, $src\t# long" %}
9594   opcode(0x81, 0x04); /* Opcode 81 /4 */
9595   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9596   ins_pipe(ialu_reg);
9597 %}
9598 
9599 // And Register with Memory
9600 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9601 %{
9602   match(Set dst (AndL dst (LoadL src)));
9603   effect(KILL cr);
9604 
9605   ins_cost(125);
9606   format %{ "andq    $dst, $src\t# long" %}
9607   opcode(0x23);
9608   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9609   ins_pipe(ialu_reg_mem);
9610 %}
9611 
9612 // And Memory with Register
9613 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9614 %{
9615   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9616   effect(KILL cr);
9617 
9618   ins_cost(150);
9619   format %{ "andq    $dst, $src\t# long" %}
9620   opcode(0x21); /* Opcode 21 /r */
9621   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9622   ins_pipe(ialu_mem_reg);
9623 %}
9624 
9625 // And Memory with Immediate
9626 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9627 %{
9628   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9629   effect(KILL cr);
9630 
9631   ins_cost(125);
9632   format %{ "andq    $dst, $src\t# long" %}
9633   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9634   ins_encode(REX_mem_wide(dst), OpcSE(src),
9635              RM_opc_mem(secondary, dst), Con8or32(src));
9636   ins_pipe(ialu_mem_imm);
9637 %}
9638 
9639 // BMI1 instructions
9640 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9641   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9642   predicate(UseBMI1Instructions);
9643   effect(KILL cr);
9644 
9645   ins_cost(125);
9646   format %{ "andnq  $dst, $src1, $src2" %}
9647 
9648   ins_encode %{
9649     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9650   %}
9651   ins_pipe(ialu_reg_mem);
9652 %}
9653 
9654 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9655   match(Set dst (AndL (XorL src1 minus_1) src2));
9656   predicate(UseBMI1Instructions);
9657   effect(KILL cr);
9658 
9659   format %{ "andnq  $dst, $src1, $src2" %}
9660 
9661   ins_encode %{
9662   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9663   %}
9664   ins_pipe(ialu_reg_mem);
9665 %}
9666 
9667 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9668   match(Set dst (AndL (SubL imm_zero src) src));
9669   predicate(UseBMI1Instructions);
9670   effect(KILL cr);
9671 
9672   format %{ "blsiq  $dst, $src" %}
9673 
9674   ins_encode %{
9675     __ blsiq($dst$$Register, $src$$Register);
9676   %}
9677   ins_pipe(ialu_reg);
9678 %}
9679 
9680 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9681   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9682   predicate(UseBMI1Instructions);
9683   effect(KILL cr);
9684 
9685   ins_cost(125);
9686   format %{ "blsiq  $dst, $src" %}
9687 
9688   ins_encode %{
9689     __ blsiq($dst$$Register, $src$$Address);
9690   %}
9691   ins_pipe(ialu_reg_mem);
9692 %}
9693 
9694 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9695 %{
9696   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9697   predicate(UseBMI1Instructions);
9698   effect(KILL cr);
9699 
9700   ins_cost(125);
9701   format %{ "blsmskq $dst, $src" %}
9702 
9703   ins_encode %{
9704     __ blsmskq($dst$$Register, $src$$Address);
9705   %}
9706   ins_pipe(ialu_reg_mem);
9707 %}
9708 
9709 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9710 %{
9711   match(Set dst (XorL (AddL src minus_1) src));
9712   predicate(UseBMI1Instructions);
9713   effect(KILL cr);
9714 
9715   format %{ "blsmskq $dst, $src" %}
9716 
9717   ins_encode %{
9718     __ blsmskq($dst$$Register, $src$$Register);
9719   %}
9720 
9721   ins_pipe(ialu_reg);
9722 %}
9723 
9724 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9725 %{
9726   match(Set dst (AndL (AddL src minus_1) src) );
9727   predicate(UseBMI1Instructions);
9728   effect(KILL cr);
9729 
9730   format %{ "blsrq  $dst, $src" %}
9731 
9732   ins_encode %{
9733     __ blsrq($dst$$Register, $src$$Register);
9734   %}
9735 
9736   ins_pipe(ialu_reg);
9737 %}
9738 
9739 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9740 %{
9741   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9742   predicate(UseBMI1Instructions);
9743   effect(KILL cr);
9744 
9745   ins_cost(125);
9746   format %{ "blsrq  $dst, $src" %}
9747 
9748   ins_encode %{
9749     __ blsrq($dst$$Register, $src$$Address);
9750   %}
9751 
9752   ins_pipe(ialu_reg);
9753 %}
9754 
9755 // Or Instructions
9756 // Or Register with Register
9757 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9758 %{
9759   match(Set dst (OrL dst src));
9760   effect(KILL cr);
9761 
9762   format %{ "orq     $dst, $src\t# long" %}
9763   opcode(0x0B);
9764   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9765   ins_pipe(ialu_reg_reg);
9766 %}
9767 
9768 // Use any_RegP to match R15 (TLS register) without spilling.
9769 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9770   match(Set dst (OrL dst (CastP2X src)));
9771   effect(KILL cr);
9772 
9773   format %{ "orq     $dst, $src\t# long" %}
9774   opcode(0x0B);
9775   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9776   ins_pipe(ialu_reg_reg);
9777 %}
9778 
9779 
9780 // Or Register with Immediate
9781 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9782 %{
9783   match(Set dst (OrL dst src));
9784   effect(KILL cr);
9785 
9786   format %{ "orq     $dst, $src\t# long" %}
9787   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9788   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9789   ins_pipe(ialu_reg);
9790 %}
9791 
9792 // Or Register with Memory
9793 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9794 %{
9795   match(Set dst (OrL dst (LoadL src)));
9796   effect(KILL cr);
9797 
9798   ins_cost(125);
9799   format %{ "orq     $dst, $src\t# long" %}
9800   opcode(0x0B);
9801   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9802   ins_pipe(ialu_reg_mem);
9803 %}
9804 
9805 // Or Memory with Register
9806 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9807 %{
9808   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9809   effect(KILL cr);
9810 
9811   ins_cost(150);
9812   format %{ "orq     $dst, $src\t# long" %}
9813   opcode(0x09); /* Opcode 09 /r */
9814   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9815   ins_pipe(ialu_mem_reg);
9816 %}
9817 
9818 // Or Memory with Immediate
9819 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9820 %{
9821   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9822   effect(KILL cr);
9823 
9824   ins_cost(125);
9825   format %{ "orq     $dst, $src\t# long" %}
9826   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9827   ins_encode(REX_mem_wide(dst), OpcSE(src),
9828              RM_opc_mem(secondary, dst), Con8or32(src));
9829   ins_pipe(ialu_mem_imm);
9830 %}
9831 
9832 // Xor Instructions
9833 // Xor Register with Register
9834 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9835 %{
9836   match(Set dst (XorL dst src));
9837   effect(KILL cr);
9838 
9839   format %{ "xorq    $dst, $src\t# long" %}
9840   opcode(0x33);
9841   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9842   ins_pipe(ialu_reg_reg);
9843 %}
9844 
9845 // Xor Register with Immediate -1
9846 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9847   match(Set dst (XorL dst imm));
9848 
9849   format %{ "notq   $dst" %}
9850   ins_encode %{
9851      __ notq($dst$$Register);
9852   %}
9853   ins_pipe(ialu_reg);
9854 %}
9855 
9856 // Xor Register with Immediate
9857 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9858 %{
9859   match(Set dst (XorL dst src));
9860   effect(KILL cr);
9861 
9862   format %{ "xorq    $dst, $src\t# long" %}
9863   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9864   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9865   ins_pipe(ialu_reg);
9866 %}
9867 
9868 // Xor Register with Memory
9869 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9870 %{
9871   match(Set dst (XorL dst (LoadL src)));
9872   effect(KILL cr);
9873 
9874   ins_cost(125);
9875   format %{ "xorq    $dst, $src\t# long" %}
9876   opcode(0x33);
9877   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9878   ins_pipe(ialu_reg_mem);
9879 %}
9880 
9881 // Xor Memory with Register
9882 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9883 %{
9884   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9885   effect(KILL cr);
9886 
9887   ins_cost(150);
9888   format %{ "xorq    $dst, $src\t# long" %}
9889   opcode(0x31); /* Opcode 31 /r */
9890   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9891   ins_pipe(ialu_mem_reg);
9892 %}
9893 
9894 // Xor Memory with Immediate
9895 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9896 %{
9897   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9898   effect(KILL cr);
9899 
9900   ins_cost(125);
9901   format %{ "xorq    $dst, $src\t# long" %}
9902   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9903   ins_encode(REX_mem_wide(dst), OpcSE(src),
9904              RM_opc_mem(secondary, dst), Con8or32(src));
9905   ins_pipe(ialu_mem_imm);
9906 %}
9907 
9908 // Convert Int to Boolean
9909 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9910 %{
9911   match(Set dst (Conv2B src));
9912   effect(KILL cr);
9913 
9914   format %{ "testl   $src, $src\t# ci2b\n\t"
9915             "setnz   $dst\n\t"
9916             "movzbl  $dst, $dst" %}
9917   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9918              setNZ_reg(dst),
9919              REX_reg_breg(dst, dst), // movzbl
9920              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9921   ins_pipe(pipe_slow); // XXX
9922 %}
9923 
9924 // Convert Pointer to Boolean
9925 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9926 %{
9927   match(Set dst (Conv2B src));
9928   effect(KILL cr);
9929 
9930   format %{ "testq   $src, $src\t# cp2b\n\t"
9931             "setnz   $dst\n\t"
9932             "movzbl  $dst, $dst" %}
9933   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9934              setNZ_reg(dst),
9935              REX_reg_breg(dst, dst), // movzbl
9936              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9937   ins_pipe(pipe_slow); // XXX
9938 %}
9939 
9940 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9941 %{
9942   match(Set dst (CmpLTMask p q));
9943   effect(KILL cr);
9944 
9945   ins_cost(400);
9946   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9947             "setlt   $dst\n\t"
9948             "movzbl  $dst, $dst\n\t"
9949             "negl    $dst" %}
9950   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9951              setLT_reg(dst),
9952              REX_reg_breg(dst, dst), // movzbl
9953              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9954              neg_reg(dst));
9955   ins_pipe(pipe_slow);
9956 %}
9957 
9958 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9959 %{
9960   match(Set dst (CmpLTMask dst zero));
9961   effect(KILL cr);
9962 
9963   ins_cost(100);
9964   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9965   ins_encode %{
9966   __ sarl($dst$$Register, 31);
9967   %}
9968   ins_pipe(ialu_reg);
9969 %}
9970 
9971 /* Better to save a register than avoid a branch */
9972 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9973 %{
9974   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9975   effect(KILL cr);
9976   ins_cost(300);
9977   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9978             "jge    done\n\t"
9979             "addl   $p,$y\n"
9980             "done:  " %}
9981   ins_encode %{
9982     Register Rp = $p$$Register;
9983     Register Rq = $q$$Register;
9984     Register Ry = $y$$Register;
9985     Label done;
9986     __ subl(Rp, Rq);
9987     __ jccb(Assembler::greaterEqual, done);
9988     __ addl(Rp, Ry);
9989     __ bind(done);
9990   %}
9991   ins_pipe(pipe_cmplt);
9992 %}
9993 
9994 /* Better to save a register than avoid a branch */
9995 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9996 %{
9997   match(Set y (AndI (CmpLTMask p q) y));
9998   effect(KILL cr);
9999 
10000   ins_cost(300);
10001 
10002   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
10003             "jlt      done\n\t"
10004             "xorl     $y, $y\n"
10005             "done:  " %}
10006   ins_encode %{
10007     Register Rp = $p$$Register;
10008     Register Rq = $q$$Register;
10009     Register Ry = $y$$Register;
10010     Label done;
10011     __ cmpl(Rp, Rq);
10012     __ jccb(Assembler::less, done);
10013     __ xorl(Ry, Ry);
10014     __ bind(done);
10015   %}
10016   ins_pipe(pipe_cmplt);
10017 %}
10018 
10019 
10020 //---------- FP Instructions------------------------------------------------
10021 
10022 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
10023 %{
10024   match(Set cr (CmpF src1 src2));
10025 
10026   ins_cost(145);
10027   format %{ "ucomiss $src1, $src2\n\t"
10028             "jnp,s   exit\n\t"
10029             "pushfq\t# saw NaN, set CF\n\t"
10030             "andq    [rsp], #0xffffff2b\n\t"
10031             "popfq\n"
10032     "exit:" %}
10033   ins_encode %{
10034     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10035     emit_cmpfp_fixup(_masm);
10036   %}
10037   ins_pipe(pipe_slow);
10038 %}
10039 
10040 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
10041   match(Set cr (CmpF src1 src2));
10042 
10043   ins_cost(100);
10044   format %{ "ucomiss $src1, $src2" %}
10045   ins_encode %{
10046     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10047   %}
10048   ins_pipe(pipe_slow);
10049 %}
10050 
10051 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
10052 %{
10053   match(Set cr (CmpF src1 (LoadF src2)));
10054 
10055   ins_cost(145);
10056   format %{ "ucomiss $src1, $src2\n\t"
10057             "jnp,s   exit\n\t"
10058             "pushfq\t# saw NaN, set CF\n\t"
10059             "andq    [rsp], #0xffffff2b\n\t"
10060             "popfq\n"
10061     "exit:" %}
10062   ins_encode %{
10063     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10064     emit_cmpfp_fixup(_masm);
10065   %}
10066   ins_pipe(pipe_slow);
10067 %}
10068 
10069 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
10070   match(Set cr (CmpF src1 (LoadF src2)));
10071 
10072   ins_cost(100);
10073   format %{ "ucomiss $src1, $src2" %}
10074   ins_encode %{
10075     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10076   %}
10077   ins_pipe(pipe_slow);
10078 %}
10079 
10080 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
10081   match(Set cr (CmpF src con));
10082 
10083   ins_cost(145);
10084   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10085             "jnp,s   exit\n\t"
10086             "pushfq\t# saw NaN, set CF\n\t"
10087             "andq    [rsp], #0xffffff2b\n\t"
10088             "popfq\n"
10089     "exit:" %}
10090   ins_encode %{
10091     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10092     emit_cmpfp_fixup(_masm);
10093   %}
10094   ins_pipe(pipe_slow);
10095 %}
10096 
10097 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
10098   match(Set cr (CmpF src con));
10099   ins_cost(100);
10100   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
10101   ins_encode %{
10102     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10103   %}
10104   ins_pipe(pipe_slow);
10105 %}
10106 
10107 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
10108 %{
10109   match(Set cr (CmpD src1 src2));
10110 
10111   ins_cost(145);
10112   format %{ "ucomisd $src1, $src2\n\t"
10113             "jnp,s   exit\n\t"
10114             "pushfq\t# saw NaN, set CF\n\t"
10115             "andq    [rsp], #0xffffff2b\n\t"
10116             "popfq\n"
10117     "exit:" %}
10118   ins_encode %{
10119     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10120     emit_cmpfp_fixup(_masm);
10121   %}
10122   ins_pipe(pipe_slow);
10123 %}
10124 
10125 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
10126   match(Set cr (CmpD src1 src2));
10127 
10128   ins_cost(100);
10129   format %{ "ucomisd $src1, $src2 test" %}
10130   ins_encode %{
10131     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10132   %}
10133   ins_pipe(pipe_slow);
10134 %}
10135 
10136 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
10137 %{
10138   match(Set cr (CmpD src1 (LoadD src2)));
10139 
10140   ins_cost(145);
10141   format %{ "ucomisd $src1, $src2\n\t"
10142             "jnp,s   exit\n\t"
10143             "pushfq\t# saw NaN, set CF\n\t"
10144             "andq    [rsp], #0xffffff2b\n\t"
10145             "popfq\n"
10146     "exit:" %}
10147   ins_encode %{
10148     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10149     emit_cmpfp_fixup(_masm);
10150   %}
10151   ins_pipe(pipe_slow);
10152 %}
10153 
10154 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
10155   match(Set cr (CmpD src1 (LoadD src2)));
10156 
10157   ins_cost(100);
10158   format %{ "ucomisd $src1, $src2" %}
10159   ins_encode %{
10160     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10161   %}
10162   ins_pipe(pipe_slow);
10163 %}
10164 
10165 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
10166   match(Set cr (CmpD src con));
10167 
10168   ins_cost(145);
10169   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10170             "jnp,s   exit\n\t"
10171             "pushfq\t# saw NaN, set CF\n\t"
10172             "andq    [rsp], #0xffffff2b\n\t"
10173             "popfq\n"
10174     "exit:" %}
10175   ins_encode %{
10176     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10177     emit_cmpfp_fixup(_masm);
10178   %}
10179   ins_pipe(pipe_slow);
10180 %}
10181 
10182 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
10183   match(Set cr (CmpD src con));
10184   ins_cost(100);
10185   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
10186   ins_encode %{
10187     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10188   %}
10189   ins_pipe(pipe_slow);
10190 %}
10191 
10192 // Compare into -1,0,1
10193 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
10194 %{
10195   match(Set dst (CmpF3 src1 src2));
10196   effect(KILL cr);
10197 
10198   ins_cost(275);
10199   format %{ "ucomiss $src1, $src2\n\t"
10200             "movl    $dst, #-1\n\t"
10201             "jp,s    done\n\t"
10202             "jb,s    done\n\t"
10203             "setne   $dst\n\t"
10204             "movzbl  $dst, $dst\n"
10205     "done:" %}
10206   ins_encode %{
10207     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
10208     emit_cmpfp3(_masm, $dst$$Register);
10209   %}
10210   ins_pipe(pipe_slow);
10211 %}
10212 
10213 // Compare into -1,0,1
10214 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
10215 %{
10216   match(Set dst (CmpF3 src1 (LoadF src2)));
10217   effect(KILL cr);
10218 
10219   ins_cost(275);
10220   format %{ "ucomiss $src1, $src2\n\t"
10221             "movl    $dst, #-1\n\t"
10222             "jp,s    done\n\t"
10223             "jb,s    done\n\t"
10224             "setne   $dst\n\t"
10225             "movzbl  $dst, $dst\n"
10226     "done:" %}
10227   ins_encode %{
10228     __ ucomiss($src1$$XMMRegister, $src2$$Address);
10229     emit_cmpfp3(_masm, $dst$$Register);
10230   %}
10231   ins_pipe(pipe_slow);
10232 %}
10233 
10234 // Compare into -1,0,1
10235 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
10236   match(Set dst (CmpF3 src con));
10237   effect(KILL cr);
10238 
10239   ins_cost(275);
10240   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
10241             "movl    $dst, #-1\n\t"
10242             "jp,s    done\n\t"
10243             "jb,s    done\n\t"
10244             "setne   $dst\n\t"
10245             "movzbl  $dst, $dst\n"
10246     "done:" %}
10247   ins_encode %{
10248     __ ucomiss($src$$XMMRegister, $constantaddress($con));
10249     emit_cmpfp3(_masm, $dst$$Register);
10250   %}
10251   ins_pipe(pipe_slow);
10252 %}
10253 
10254 // Compare into -1,0,1
10255 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
10256 %{
10257   match(Set dst (CmpD3 src1 src2));
10258   effect(KILL cr);
10259 
10260   ins_cost(275);
10261   format %{ "ucomisd $src1, $src2\n\t"
10262             "movl    $dst, #-1\n\t"
10263             "jp,s    done\n\t"
10264             "jb,s    done\n\t"
10265             "setne   $dst\n\t"
10266             "movzbl  $dst, $dst\n"
10267     "done:" %}
10268   ins_encode %{
10269     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10270     emit_cmpfp3(_masm, $dst$$Register);
10271   %}
10272   ins_pipe(pipe_slow);
10273 %}
10274 
10275 // Compare into -1,0,1
10276 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10277 %{
10278   match(Set dst (CmpD3 src1 (LoadD src2)));
10279   effect(KILL cr);
10280 
10281   ins_cost(275);
10282   format %{ "ucomisd $src1, $src2\n\t"
10283             "movl    $dst, #-1\n\t"
10284             "jp,s    done\n\t"
10285             "jb,s    done\n\t"
10286             "setne   $dst\n\t"
10287             "movzbl  $dst, $dst\n"
10288     "done:" %}
10289   ins_encode %{
10290     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10291     emit_cmpfp3(_masm, $dst$$Register);
10292   %}
10293   ins_pipe(pipe_slow);
10294 %}
10295 
10296 // Compare into -1,0,1
10297 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10298   match(Set dst (CmpD3 src con));
10299   effect(KILL cr);
10300 
10301   ins_cost(275);
10302   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10303             "movl    $dst, #-1\n\t"
10304             "jp,s    done\n\t"
10305             "jb,s    done\n\t"
10306             "setne   $dst\n\t"
10307             "movzbl  $dst, $dst\n"
10308     "done:" %}
10309   ins_encode %{
10310     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10311     emit_cmpfp3(_masm, $dst$$Register);
10312   %}
10313   ins_pipe(pipe_slow);
10314 %}
10315 
10316 //----------Arithmetic Conversion Instructions---------------------------------
10317 
10318 instruct roundFloat_nop(regF dst)
10319 %{
10320   match(Set dst (RoundFloat dst));
10321 
10322   ins_cost(0);
10323   ins_encode();
10324   ins_pipe(empty);
10325 %}
10326 
10327 instruct roundDouble_nop(regD dst)
10328 %{
10329   match(Set dst (RoundDouble dst));
10330 
10331   ins_cost(0);
10332   ins_encode();
10333   ins_pipe(empty);
10334 %}
10335 
10336 instruct convF2D_reg_reg(regD dst, regF src)
10337 %{
10338   match(Set dst (ConvF2D src));
10339 
10340   format %{ "cvtss2sd $dst, $src" %}
10341   ins_encode %{
10342     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10343   %}
10344   ins_pipe(pipe_slow); // XXX
10345 %}
10346 
10347 instruct convF2D_reg_mem(regD dst, memory src)
10348 %{
10349   match(Set dst (ConvF2D (LoadF src)));
10350 
10351   format %{ "cvtss2sd $dst, $src" %}
10352   ins_encode %{
10353     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10354   %}
10355   ins_pipe(pipe_slow); // XXX
10356 %}
10357 
10358 instruct convD2F_reg_reg(regF dst, regD src)
10359 %{
10360   match(Set dst (ConvD2F src));
10361 
10362   format %{ "cvtsd2ss $dst, $src" %}
10363   ins_encode %{
10364     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10365   %}
10366   ins_pipe(pipe_slow); // XXX
10367 %}
10368 
10369 instruct convD2F_reg_mem(regF dst, memory src)
10370 %{
10371   match(Set dst (ConvD2F (LoadD src)));
10372 
10373   format %{ "cvtsd2ss $dst, $src" %}
10374   ins_encode %{
10375     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10376   %}
10377   ins_pipe(pipe_slow); // XXX
10378 %}
10379 
10380 // XXX do mem variants
10381 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10382 %{
10383   match(Set dst (ConvF2I src));
10384   effect(KILL cr);
10385 
10386   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10387             "cmpl    $dst, #0x80000000\n\t"
10388             "jne,s   done\n\t"
10389             "subq    rsp, #8\n\t"
10390             "movss   [rsp], $src\n\t"
10391             "call    f2i_fixup\n\t"
10392             "popq    $dst\n"
10393     "done:   "%}
10394   ins_encode %{
10395     Label done;
10396     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10397     __ cmpl($dst$$Register, 0x80000000);
10398     __ jccb(Assembler::notEqual, done);
10399     __ subptr(rsp, 8);
10400     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10401     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10402     __ pop($dst$$Register);
10403     __ bind(done);
10404   %}
10405   ins_pipe(pipe_slow);
10406 %}
10407 
10408 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10409 %{
10410   match(Set dst (ConvF2L src));
10411   effect(KILL cr);
10412 
10413   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10414             "cmpq    $dst, [0x8000000000000000]\n\t"
10415             "jne,s   done\n\t"
10416             "subq    rsp, #8\n\t"
10417             "movss   [rsp], $src\n\t"
10418             "call    f2l_fixup\n\t"
10419             "popq    $dst\n"
10420     "done:   "%}
10421   ins_encode %{
10422     Label done;
10423     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10424     __ cmp64($dst$$Register,
10425              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10426     __ jccb(Assembler::notEqual, done);
10427     __ subptr(rsp, 8);
10428     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10429     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10430     __ pop($dst$$Register);
10431     __ bind(done);
10432   %}
10433   ins_pipe(pipe_slow);
10434 %}
10435 
10436 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10437 %{
10438   match(Set dst (ConvD2I src));
10439   effect(KILL cr);
10440 
10441   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10442             "cmpl    $dst, #0x80000000\n\t"
10443             "jne,s   done\n\t"
10444             "subq    rsp, #8\n\t"
10445             "movsd   [rsp], $src\n\t"
10446             "call    d2i_fixup\n\t"
10447             "popq    $dst\n"
10448     "done:   "%}
10449   ins_encode %{
10450     Label done;
10451     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10452     __ cmpl($dst$$Register, 0x80000000);
10453     __ jccb(Assembler::notEqual, done);
10454     __ subptr(rsp, 8);
10455     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10456     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10457     __ pop($dst$$Register);
10458     __ bind(done);
10459   %}
10460   ins_pipe(pipe_slow);
10461 %}
10462 
10463 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10464 %{
10465   match(Set dst (ConvD2L src));
10466   effect(KILL cr);
10467 
10468   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10469             "cmpq    $dst, [0x8000000000000000]\n\t"
10470             "jne,s   done\n\t"
10471             "subq    rsp, #8\n\t"
10472             "movsd   [rsp], $src\n\t"
10473             "call    d2l_fixup\n\t"
10474             "popq    $dst\n"
10475     "done:   "%}
10476   ins_encode %{
10477     Label done;
10478     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10479     __ cmp64($dst$$Register,
10480              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10481     __ jccb(Assembler::notEqual, done);
10482     __ subptr(rsp, 8);
10483     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10484     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10485     __ pop($dst$$Register);
10486     __ bind(done);
10487   %}
10488   ins_pipe(pipe_slow);
10489 %}
10490 
10491 instruct convI2F_reg_reg(regF dst, rRegI src)
10492 %{
10493   predicate(!UseXmmI2F);
10494   match(Set dst (ConvI2F src));
10495 
10496   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10497   ins_encode %{
10498     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10499   %}
10500   ins_pipe(pipe_slow); // XXX
10501 %}
10502 
10503 instruct convI2F_reg_mem(regF dst, memory src)
10504 %{
10505   match(Set dst (ConvI2F (LoadI src)));
10506 
10507   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10508   ins_encode %{
10509     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10510   %}
10511   ins_pipe(pipe_slow); // XXX
10512 %}
10513 
10514 instruct convI2D_reg_reg(regD dst, rRegI src)
10515 %{
10516   predicate(!UseXmmI2D);
10517   match(Set dst (ConvI2D src));
10518 
10519   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10520   ins_encode %{
10521     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10522   %}
10523   ins_pipe(pipe_slow); // XXX
10524 %}
10525 
10526 instruct convI2D_reg_mem(regD dst, memory src)
10527 %{
10528   match(Set dst (ConvI2D (LoadI src)));
10529 
10530   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10531   ins_encode %{
10532     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10533   %}
10534   ins_pipe(pipe_slow); // XXX
10535 %}
10536 
10537 instruct convXI2F_reg(regF dst, rRegI src)
10538 %{
10539   predicate(UseXmmI2F);
10540   match(Set dst (ConvI2F src));
10541 
10542   format %{ "movdl $dst, $src\n\t"
10543             "cvtdq2psl $dst, $dst\t# i2f" %}
10544   ins_encode %{
10545     __ movdl($dst$$XMMRegister, $src$$Register);
10546     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10547   %}
10548   ins_pipe(pipe_slow); // XXX
10549 %}
10550 
10551 instruct convXI2D_reg(regD dst, rRegI src)
10552 %{
10553   predicate(UseXmmI2D);
10554   match(Set dst (ConvI2D src));
10555 
10556   format %{ "movdl $dst, $src\n\t"
10557             "cvtdq2pdl $dst, $dst\t# i2d" %}
10558   ins_encode %{
10559     __ movdl($dst$$XMMRegister, $src$$Register);
10560     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10561   %}
10562   ins_pipe(pipe_slow); // XXX
10563 %}
10564 
10565 instruct convL2F_reg_reg(regF dst, rRegL src)
10566 %{
10567   match(Set dst (ConvL2F src));
10568 
10569   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10570   ins_encode %{
10571     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10572   %}
10573   ins_pipe(pipe_slow); // XXX
10574 %}
10575 
10576 instruct convL2F_reg_mem(regF dst, memory src)
10577 %{
10578   match(Set dst (ConvL2F (LoadL src)));
10579 
10580   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10581   ins_encode %{
10582     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10583   %}
10584   ins_pipe(pipe_slow); // XXX
10585 %}
10586 
10587 instruct convL2D_reg_reg(regD dst, rRegL src)
10588 %{
10589   match(Set dst (ConvL2D src));
10590 
10591   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10592   ins_encode %{
10593     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10594   %}
10595   ins_pipe(pipe_slow); // XXX
10596 %}
10597 
10598 instruct convL2D_reg_mem(regD dst, memory src)
10599 %{
10600   match(Set dst (ConvL2D (LoadL src)));
10601 
10602   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10603   ins_encode %{
10604     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10605   %}
10606   ins_pipe(pipe_slow); // XXX
10607 %}
10608 
10609 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10610 %{
10611   match(Set dst (ConvI2L src));
10612 
10613   ins_cost(125);
10614   format %{ "movslq  $dst, $src\t# i2l" %}
10615   ins_encode %{
10616     __ movslq($dst$$Register, $src$$Register);
10617   %}
10618   ins_pipe(ialu_reg_reg);
10619 %}
10620 
10621 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10622 // %{
10623 //   match(Set dst (ConvI2L src));
10624 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10625 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10626 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10627 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10628 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10629 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10630 
10631 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10632 //   ins_encode(enc_copy(dst, src));
10633 // //   opcode(0x63); // needs REX.W
10634 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10635 //   ins_pipe(ialu_reg_reg);
10636 // %}
10637 
10638 // Zero-extend convert int to long
10639 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10640 %{
10641   match(Set dst (AndL (ConvI2L src) mask));
10642 
10643   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10644   ins_encode %{
10645     if ($dst$$reg != $src$$reg) {
10646       __ movl($dst$$Register, $src$$Register);
10647     }
10648   %}
10649   ins_pipe(ialu_reg_reg);
10650 %}
10651 
10652 // Zero-extend convert int to long
10653 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10654 %{
10655   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10656 
10657   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10658   ins_encode %{
10659     __ movl($dst$$Register, $src$$Address);
10660   %}
10661   ins_pipe(ialu_reg_mem);
10662 %}
10663 
10664 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10665 %{
10666   match(Set dst (AndL src mask));
10667 
10668   format %{ "movl    $dst, $src\t# zero-extend long" %}
10669   ins_encode %{
10670     __ movl($dst$$Register, $src$$Register);
10671   %}
10672   ins_pipe(ialu_reg_reg);
10673 %}
10674 
10675 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10676 %{
10677   match(Set dst (ConvL2I src));
10678 
10679   format %{ "movl    $dst, $src\t# l2i" %}
10680   ins_encode %{
10681     __ movl($dst$$Register, $src$$Register);
10682   %}
10683   ins_pipe(ialu_reg_reg);
10684 %}
10685 
10686 
10687 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10688   match(Set dst (MoveF2I src));
10689   effect(DEF dst, USE src);
10690 
10691   ins_cost(125);
10692   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10693   ins_encode %{
10694     __ movl($dst$$Register, Address(rsp, $src$$disp));
10695   %}
10696   ins_pipe(ialu_reg_mem);
10697 %}
10698 
10699 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10700   match(Set dst (MoveI2F src));
10701   effect(DEF dst, USE src);
10702 
10703   ins_cost(125);
10704   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10705   ins_encode %{
10706     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10707   %}
10708   ins_pipe(pipe_slow);
10709 %}
10710 
10711 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10712   match(Set dst (MoveD2L src));
10713   effect(DEF dst, USE src);
10714 
10715   ins_cost(125);
10716   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10717   ins_encode %{
10718     __ movq($dst$$Register, Address(rsp, $src$$disp));
10719   %}
10720   ins_pipe(ialu_reg_mem);
10721 %}
10722 
10723 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10724   predicate(!UseXmmLoadAndClearUpper);
10725   match(Set dst (MoveL2D src));
10726   effect(DEF dst, USE src);
10727 
10728   ins_cost(125);
10729   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10730   ins_encode %{
10731     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10732   %}
10733   ins_pipe(pipe_slow);
10734 %}
10735 
10736 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10737   predicate(UseXmmLoadAndClearUpper);
10738   match(Set dst (MoveL2D src));
10739   effect(DEF dst, USE src);
10740 
10741   ins_cost(125);
10742   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10743   ins_encode %{
10744     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10745   %}
10746   ins_pipe(pipe_slow);
10747 %}
10748 
10749 
10750 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10751   match(Set dst (MoveF2I src));
10752   effect(DEF dst, USE src);
10753 
10754   ins_cost(95); // XXX
10755   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10756   ins_encode %{
10757     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10758   %}
10759   ins_pipe(pipe_slow);
10760 %}
10761 
10762 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10763   match(Set dst (MoveI2F src));
10764   effect(DEF dst, USE src);
10765 
10766   ins_cost(100);
10767   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10768   ins_encode %{
10769     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10770   %}
10771   ins_pipe( ialu_mem_reg );
10772 %}
10773 
10774 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10775   match(Set dst (MoveD2L src));
10776   effect(DEF dst, USE src);
10777 
10778   ins_cost(95); // XXX
10779   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10780   ins_encode %{
10781     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10782   %}
10783   ins_pipe(pipe_slow);
10784 %}
10785 
10786 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10787   match(Set dst (MoveL2D src));
10788   effect(DEF dst, USE src);
10789 
10790   ins_cost(100);
10791   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10792   ins_encode %{
10793     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10794   %}
10795   ins_pipe(ialu_mem_reg);
10796 %}
10797 
10798 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10799   match(Set dst (MoveF2I src));
10800   effect(DEF dst, USE src);
10801   ins_cost(85);
10802   format %{ "movd    $dst,$src\t# MoveF2I" %}
10803   ins_encode %{
10804     __ movdl($dst$$Register, $src$$XMMRegister);
10805   %}
10806   ins_pipe( pipe_slow );
10807 %}
10808 
10809 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10810   match(Set dst (MoveD2L src));
10811   effect(DEF dst, USE src);
10812   ins_cost(85);
10813   format %{ "movd    $dst,$src\t# MoveD2L" %}
10814   ins_encode %{
10815     __ movdq($dst$$Register, $src$$XMMRegister);
10816   %}
10817   ins_pipe( pipe_slow );
10818 %}
10819 
10820 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10821   match(Set dst (MoveI2F src));
10822   effect(DEF dst, USE src);
10823   ins_cost(100);
10824   format %{ "movd    $dst,$src\t# MoveI2F" %}
10825   ins_encode %{
10826     __ movdl($dst$$XMMRegister, $src$$Register);
10827   %}
10828   ins_pipe( pipe_slow );
10829 %}
10830 
10831 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10832   match(Set dst (MoveL2D src));
10833   effect(DEF dst, USE src);
10834   ins_cost(100);
10835   format %{ "movd    $dst,$src\t# MoveL2D" %}
10836   ins_encode %{
10837      __ movdq($dst$$XMMRegister, $src$$Register);
10838   %}
10839   ins_pipe( pipe_slow );
10840 %}
10841 
10842 
10843 // =======================================================================
10844 // fast clearing of an array
10845 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10846                   Universe dummy, rFlagsReg cr)
10847 %{
10848   predicate(!((ClearArrayNode*)n)->is_large());
10849   match(Set dummy (ClearArray cnt base));
10850   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10851 
10852   format %{ $$template
10853     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10854     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10855     $$emit$$"jg      LARGE\n\t"
10856     $$emit$$"dec     rcx\n\t"
10857     $$emit$$"js      DONE\t# Zero length\n\t"
10858     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10859     $$emit$$"dec     rcx\n\t"
10860     $$emit$$"jge     LOOP\n\t"
10861     $$emit$$"jmp     DONE\n\t"
10862     $$emit$$"# LARGE:\n\t"
10863     if (UseFastStosb) {
10864        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10865        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10866     } else if (UseXMMForObjInit) {
10867        $$emit$$"mov     rdi,rax\n\t"
10868        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10869        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10870        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10871        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10872        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10873        $$emit$$"add     0x40,rax\n\t"
10874        $$emit$$"# L_zero_64_bytes:\n\t"
10875        $$emit$$"sub     0x8,rcx\n\t"
10876        $$emit$$"jge     L_loop\n\t"
10877        $$emit$$"add     0x4,rcx\n\t"
10878        $$emit$$"jl      L_tail\n\t"
10879        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10880        $$emit$$"add     0x20,rax\n\t"
10881        $$emit$$"sub     0x4,rcx\n\t"
10882        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10883        $$emit$$"add     0x4,rcx\n\t"
10884        $$emit$$"jle     L_end\n\t"
10885        $$emit$$"dec     rcx\n\t"
10886        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10887        $$emit$$"vmovq   xmm0,(rax)\n\t"
10888        $$emit$$"add     0x8,rax\n\t"
10889        $$emit$$"dec     rcx\n\t"
10890        $$emit$$"jge     L_sloop\n\t"
10891        $$emit$$"# L_end:\n\t"
10892     } else {
10893        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10894     }
10895     $$emit$$"# DONE"
10896   %}
10897   ins_encode %{
10898     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10899                  $tmp$$XMMRegister, false);
10900   %}
10901   ins_pipe(pipe_slow);
10902 %}
10903 
10904 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, regD tmp, rax_RegI zero,
10905                         Universe dummy, rFlagsReg cr)
10906 %{
10907   predicate(((ClearArrayNode*)n)->is_large());
10908   match(Set dummy (ClearArray cnt base));
10909   effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
10910 
10911   format %{ $$template
10912     if (UseFastStosb) {
10913        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10914        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10915        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10916     } else if (UseXMMForObjInit) {
10917        $$emit$$"mov     rdi,rax\t# ClearArray:\n\t"
10918        $$emit$$"vpxor   ymm0,ymm0,ymm0\n\t"
10919        $$emit$$"jmpq    L_zero_64_bytes\n\t"
10920        $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
10921        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10922        $$emit$$"vmovdqu ymm0,0x20(rax)\n\t"
10923        $$emit$$"add     0x40,rax\n\t"
10924        $$emit$$"# L_zero_64_bytes:\n\t"
10925        $$emit$$"sub     0x8,rcx\n\t"
10926        $$emit$$"jge     L_loop\n\t"
10927        $$emit$$"add     0x4,rcx\n\t"
10928        $$emit$$"jl      L_tail\n\t"
10929        $$emit$$"vmovdqu ymm0,(rax)\n\t"
10930        $$emit$$"add     0x20,rax\n\t"
10931        $$emit$$"sub     0x4,rcx\n\t"
10932        $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
10933        $$emit$$"add     0x4,rcx\n\t"
10934        $$emit$$"jle     L_end\n\t"
10935        $$emit$$"dec     rcx\n\t"
10936        $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
10937        $$emit$$"vmovq   xmm0,(rax)\n\t"
10938        $$emit$$"add     0x8,rax\n\t"
10939        $$emit$$"dec     rcx\n\t"
10940        $$emit$$"jge     L_sloop\n\t"
10941        $$emit$$"# L_end:\n\t"
10942     } else {
10943        $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10944        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10945     }
10946   %}
10947   ins_encode %{
10948     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
10949                  $tmp$$XMMRegister, true);
10950   %}
10951   ins_pipe(pipe_slow);
10952 %}
10953 
10954 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10955                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
10956 %{
10957   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10958   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10959   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10960 
10961   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10962   ins_encode %{
10963     __ string_compare($str1$$Register, $str2$$Register,
10964                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10965                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10966   %}
10967   ins_pipe( pipe_slow );
10968 %}
10969 
10970 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10971                          rax_RegI result, legVecS tmp1, rFlagsReg cr)
10972 %{
10973   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10974   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10975   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10976 
10977   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10978   ins_encode %{
10979     __ string_compare($str1$$Register, $str2$$Register,
10980                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10981                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10982   %}
10983   ins_pipe( pipe_slow );
10984 %}
10985 
10986 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10987                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
10988 %{
10989   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10990   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10991   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10992 
10993   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10994   ins_encode %{
10995     __ string_compare($str1$$Register, $str2$$Register,
10996                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10997                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10998   %}
10999   ins_pipe( pipe_slow );
11000 %}
11001 
11002 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
11003                           rax_RegI result, legVecS tmp1, rFlagsReg cr)
11004 %{
11005   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
11006   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
11007   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
11008 
11009   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
11010   ins_encode %{
11011     __ string_compare($str2$$Register, $str1$$Register,
11012                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
11013                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
11014   %}
11015   ins_pipe( pipe_slow );
11016 %}
11017 
11018 // fast search of substring with known size.
11019 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11020                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11021 %{
11022   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11023   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11024   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11025 
11026   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11027   ins_encode %{
11028     int icnt2 = (int)$int_cnt2$$constant;
11029     if (icnt2 >= 16) {
11030       // IndexOf for constant substrings with size >= 16 elements
11031       // which don't need to be loaded through stack.
11032       __ string_indexofC8($str1$$Register, $str2$$Register,
11033                           $cnt1$$Register, $cnt2$$Register,
11034                           icnt2, $result$$Register,
11035                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11036     } else {
11037       // Small strings are loaded through stack if they cross page boundary.
11038       __ string_indexof($str1$$Register, $str2$$Register,
11039                         $cnt1$$Register, $cnt2$$Register,
11040                         icnt2, $result$$Register,
11041                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11042     }
11043   %}
11044   ins_pipe( pipe_slow );
11045 %}
11046 
11047 // fast search of substring with known size.
11048 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11049                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11050 %{
11051   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11052   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11053   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11054 
11055   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11056   ins_encode %{
11057     int icnt2 = (int)$int_cnt2$$constant;
11058     if (icnt2 >= 8) {
11059       // IndexOf for constant substrings with size >= 8 elements
11060       // which don't need to be loaded through stack.
11061       __ string_indexofC8($str1$$Register, $str2$$Register,
11062                           $cnt1$$Register, $cnt2$$Register,
11063                           icnt2, $result$$Register,
11064                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11065     } else {
11066       // Small strings are loaded through stack if they cross page boundary.
11067       __ string_indexof($str1$$Register, $str2$$Register,
11068                         $cnt1$$Register, $cnt2$$Register,
11069                         icnt2, $result$$Register,
11070                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11071     }
11072   %}
11073   ins_pipe( pipe_slow );
11074 %}
11075 
11076 // fast search of substring with known size.
11077 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
11078                              rbx_RegI result, legVecS vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
11079 %{
11080   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11081   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
11082   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
11083 
11084   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
11085   ins_encode %{
11086     int icnt2 = (int)$int_cnt2$$constant;
11087     if (icnt2 >= 8) {
11088       // IndexOf for constant substrings with size >= 8 elements
11089       // which don't need to be loaded through stack.
11090       __ string_indexofC8($str1$$Register, $str2$$Register,
11091                           $cnt1$$Register, $cnt2$$Register,
11092                           icnt2, $result$$Register,
11093                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11094     } else {
11095       // Small strings are loaded through stack if they cross page boundary.
11096       __ string_indexof($str1$$Register, $str2$$Register,
11097                         $cnt1$$Register, $cnt2$$Register,
11098                         icnt2, $result$$Register,
11099                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11100     }
11101   %}
11102   ins_pipe( pipe_slow );
11103 %}
11104 
11105 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11106                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11107 %{
11108   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
11109   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11110   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11111 
11112   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11113   ins_encode %{
11114     __ string_indexof($str1$$Register, $str2$$Register,
11115                       $cnt1$$Register, $cnt2$$Register,
11116                       (-1), $result$$Register,
11117                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
11118   %}
11119   ins_pipe( pipe_slow );
11120 %}
11121 
11122 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11123                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11124 %{
11125   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
11126   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11127   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11128 
11129   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11130   ins_encode %{
11131     __ string_indexof($str1$$Register, $str2$$Register,
11132                       $cnt1$$Register, $cnt2$$Register,
11133                       (-1), $result$$Register,
11134                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
11135   %}
11136   ins_pipe( pipe_slow );
11137 %}
11138 
11139 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
11140                          rbx_RegI result, legVecS vec, rcx_RegI tmp, rFlagsReg cr)
11141 %{
11142   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
11143   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
11144   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
11145 
11146   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
11147   ins_encode %{
11148     __ string_indexof($str1$$Register, $str2$$Register,
11149                       $cnt1$$Register, $cnt2$$Register,
11150                       (-1), $result$$Register,
11151                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
11152   %}
11153   ins_pipe( pipe_slow );
11154 %}
11155 
11156 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
11157                               rbx_RegI result, legVecS vec1, legVecS vec2, legVecS vec3, rcx_RegI tmp, rFlagsReg cr)
11158 %{
11159   predicate(UseSSE42Intrinsics);
11160   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
11161   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
11162   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
11163   ins_encode %{
11164     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
11165                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
11166   %}
11167   ins_pipe( pipe_slow );
11168 %}
11169 
11170 // fast string equals
11171 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
11172                        legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11173 %{
11174   match(Set result (StrEquals (Binary str1 str2) cnt));
11175   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
11176 
11177   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
11178   ins_encode %{
11179     __ arrays_equals(false, $str1$$Register, $str2$$Register,
11180                      $cnt$$Register, $result$$Register, $tmp3$$Register,
11181                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11182   %}
11183   ins_pipe( pipe_slow );
11184 %}
11185 
11186 // fast array equals
11187 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11188                        legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11189 %{
11190   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
11191   match(Set result (AryEq ary1 ary2));
11192   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11193 
11194   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11195   ins_encode %{
11196     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11197                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11198                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
11199   %}
11200   ins_pipe( pipe_slow );
11201 %}
11202 
11203 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
11204                       legVecS tmp1, legVecS tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
11205 %{
11206   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
11207   match(Set result (AryEq ary1 ary2));
11208   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
11209 
11210   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
11211   ins_encode %{
11212     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
11213                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
11214                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
11215   %}
11216   ins_pipe( pipe_slow );
11217 %}
11218 
11219 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
11220                       legVecS tmp1, legVecS tmp2, rbx_RegI tmp3, rFlagsReg cr)
11221 %{
11222   match(Set result (HasNegatives ary1 len));
11223   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
11224 
11225   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
11226   ins_encode %{
11227     __ has_negatives($ary1$$Register, $len$$Register,
11228                      $result$$Register, $tmp3$$Register,
11229                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
11230   %}
11231   ins_pipe( pipe_slow );
11232 %}
11233 
11234 // fast char[] to byte[] compression
11235 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11236                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11237   match(Set result (StrCompressedCopy src (Binary dst len)));
11238   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11239 
11240   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
11241   ins_encode %{
11242     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
11243                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11244                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11245   %}
11246   ins_pipe( pipe_slow );
11247 %}
11248 
11249 // fast byte[] to char[] inflation
11250 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11251                         legVecS tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
11252   match(Set dummy (StrInflatedCopy src (Binary dst len)));
11253   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
11254 
11255   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
11256   ins_encode %{
11257     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
11258                           $tmp1$$XMMRegister, $tmp2$$Register);
11259   %}
11260   ins_pipe( pipe_slow );
11261 %}
11262 
11263 // encode char[] to byte[] in ISO_8859_1
11264 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
11265                           legVecS tmp1, legVecS tmp2, legVecS tmp3, legVecS tmp4,
11266                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
11267   match(Set result (EncodeISOArray src (Binary dst len)));
11268   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
11269 
11270   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
11271   ins_encode %{
11272     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
11273                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
11274                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
11275   %}
11276   ins_pipe( pipe_slow );
11277 %}
11278 
11279 //----------Overflow Math Instructions-----------------------------------------
11280 
11281 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11282 %{
11283   match(Set cr (OverflowAddI op1 op2));
11284   effect(DEF cr, USE_KILL op1, USE op2);
11285 
11286   format %{ "addl    $op1, $op2\t# overflow check int" %}
11287 
11288   ins_encode %{
11289     __ addl($op1$$Register, $op2$$Register);
11290   %}
11291   ins_pipe(ialu_reg_reg);
11292 %}
11293 
11294 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
11295 %{
11296   match(Set cr (OverflowAddI op1 op2));
11297   effect(DEF cr, USE_KILL op1, USE op2);
11298 
11299   format %{ "addl    $op1, $op2\t# overflow check int" %}
11300 
11301   ins_encode %{
11302     __ addl($op1$$Register, $op2$$constant);
11303   %}
11304   ins_pipe(ialu_reg_reg);
11305 %}
11306 
11307 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11308 %{
11309   match(Set cr (OverflowAddL op1 op2));
11310   effect(DEF cr, USE_KILL op1, USE op2);
11311 
11312   format %{ "addq    $op1, $op2\t# overflow check long" %}
11313   ins_encode %{
11314     __ addq($op1$$Register, $op2$$Register);
11315   %}
11316   ins_pipe(ialu_reg_reg);
11317 %}
11318 
11319 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11320 %{
11321   match(Set cr (OverflowAddL op1 op2));
11322   effect(DEF cr, USE_KILL op1, USE op2);
11323 
11324   format %{ "addq    $op1, $op2\t# overflow check long" %}
11325   ins_encode %{
11326     __ addq($op1$$Register, $op2$$constant);
11327   %}
11328   ins_pipe(ialu_reg_reg);
11329 %}
11330 
11331 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11332 %{
11333   match(Set cr (OverflowSubI op1 op2));
11334 
11335   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11336   ins_encode %{
11337     __ cmpl($op1$$Register, $op2$$Register);
11338   %}
11339   ins_pipe(ialu_reg_reg);
11340 %}
11341 
11342 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11343 %{
11344   match(Set cr (OverflowSubI op1 op2));
11345 
11346   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11347   ins_encode %{
11348     __ cmpl($op1$$Register, $op2$$constant);
11349   %}
11350   ins_pipe(ialu_reg_reg);
11351 %}
11352 
11353 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11354 %{
11355   match(Set cr (OverflowSubL op1 op2));
11356 
11357   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11358   ins_encode %{
11359     __ cmpq($op1$$Register, $op2$$Register);
11360   %}
11361   ins_pipe(ialu_reg_reg);
11362 %}
11363 
11364 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11365 %{
11366   match(Set cr (OverflowSubL op1 op2));
11367 
11368   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11369   ins_encode %{
11370     __ cmpq($op1$$Register, $op2$$constant);
11371   %}
11372   ins_pipe(ialu_reg_reg);
11373 %}
11374 
11375 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11376 %{
11377   match(Set cr (OverflowSubI zero op2));
11378   effect(DEF cr, USE_KILL op2);
11379 
11380   format %{ "negl    $op2\t# overflow check int" %}
11381   ins_encode %{
11382     __ negl($op2$$Register);
11383   %}
11384   ins_pipe(ialu_reg_reg);
11385 %}
11386 
11387 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11388 %{
11389   match(Set cr (OverflowSubL zero op2));
11390   effect(DEF cr, USE_KILL op2);
11391 
11392   format %{ "negq    $op2\t# overflow check long" %}
11393   ins_encode %{
11394     __ negq($op2$$Register);
11395   %}
11396   ins_pipe(ialu_reg_reg);
11397 %}
11398 
11399 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11400 %{
11401   match(Set cr (OverflowMulI op1 op2));
11402   effect(DEF cr, USE_KILL op1, USE op2);
11403 
11404   format %{ "imull    $op1, $op2\t# overflow check int" %}
11405   ins_encode %{
11406     __ imull($op1$$Register, $op2$$Register);
11407   %}
11408   ins_pipe(ialu_reg_reg_alu0);
11409 %}
11410 
11411 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11412 %{
11413   match(Set cr (OverflowMulI op1 op2));
11414   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11415 
11416   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11417   ins_encode %{
11418     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11419   %}
11420   ins_pipe(ialu_reg_reg_alu0);
11421 %}
11422 
11423 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11424 %{
11425   match(Set cr (OverflowMulL op1 op2));
11426   effect(DEF cr, USE_KILL op1, USE op2);
11427 
11428   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11429   ins_encode %{
11430     __ imulq($op1$$Register, $op2$$Register);
11431   %}
11432   ins_pipe(ialu_reg_reg_alu0);
11433 %}
11434 
11435 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11436 %{
11437   match(Set cr (OverflowMulL op1 op2));
11438   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11439 
11440   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11441   ins_encode %{
11442     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11443   %}
11444   ins_pipe(ialu_reg_reg_alu0);
11445 %}
11446 
11447 
11448 //----------Control Flow Instructions------------------------------------------
11449 // Signed compare Instructions
11450 
11451 // XXX more variants!!
11452 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11453 %{
11454   match(Set cr (CmpI op1 op2));
11455   effect(DEF cr, USE op1, USE op2);
11456 
11457   format %{ "cmpl    $op1, $op2" %}
11458   opcode(0x3B);  /* Opcode 3B /r */
11459   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11460   ins_pipe(ialu_cr_reg_reg);
11461 %}
11462 
11463 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11464 %{
11465   match(Set cr (CmpI op1 op2));
11466 
11467   format %{ "cmpl    $op1, $op2" %}
11468   opcode(0x81, 0x07); /* Opcode 81 /7 */
11469   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11470   ins_pipe(ialu_cr_reg_imm);
11471 %}
11472 
11473 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11474 %{
11475   match(Set cr (CmpI op1 (LoadI op2)));
11476 
11477   ins_cost(500); // XXX
11478   format %{ "cmpl    $op1, $op2" %}
11479   opcode(0x3B); /* Opcode 3B /r */
11480   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11481   ins_pipe(ialu_cr_reg_mem);
11482 %}
11483 
11484 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11485 %{
11486   match(Set cr (CmpI src zero));
11487 
11488   format %{ "testl   $src, $src" %}
11489   opcode(0x85);
11490   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11491   ins_pipe(ialu_cr_reg_imm);
11492 %}
11493 
11494 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11495 %{
11496   match(Set cr (CmpI (AndI src con) zero));
11497 
11498   format %{ "testl   $src, $con" %}
11499   opcode(0xF7, 0x00);
11500   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11501   ins_pipe(ialu_cr_reg_imm);
11502 %}
11503 
11504 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11505 %{
11506   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11507 
11508   format %{ "testl   $src, $mem" %}
11509   opcode(0x85);
11510   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11511   ins_pipe(ialu_cr_reg_mem);
11512 %}
11513 
11514 // Unsigned compare Instructions; really, same as signed except they
11515 // produce an rFlagsRegU instead of rFlagsReg.
11516 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11517 %{
11518   match(Set cr (CmpU op1 op2));
11519 
11520   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11521   opcode(0x3B); /* Opcode 3B /r */
11522   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11523   ins_pipe(ialu_cr_reg_reg);
11524 %}
11525 
11526 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11527 %{
11528   match(Set cr (CmpU op1 op2));
11529 
11530   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11531   opcode(0x81,0x07); /* Opcode 81 /7 */
11532   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11533   ins_pipe(ialu_cr_reg_imm);
11534 %}
11535 
11536 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11537 %{
11538   match(Set cr (CmpU op1 (LoadI op2)));
11539 
11540   ins_cost(500); // XXX
11541   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11542   opcode(0x3B); /* Opcode 3B /r */
11543   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11544   ins_pipe(ialu_cr_reg_mem);
11545 %}
11546 
11547 // // // Cisc-spilled version of cmpU_rReg
11548 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11549 // //%{
11550 // //  match(Set cr (CmpU (LoadI op1) op2));
11551 // //
11552 // //  format %{ "CMPu   $op1,$op2" %}
11553 // //  ins_cost(500);
11554 // //  opcode(0x39);  /* Opcode 39 /r */
11555 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11556 // //%}
11557 
11558 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11559 %{
11560   match(Set cr (CmpU src zero));
11561 
11562   format %{ "testl  $src, $src\t# unsigned" %}
11563   opcode(0x85);
11564   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11565   ins_pipe(ialu_cr_reg_imm);
11566 %}
11567 
11568 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11569 %{
11570   match(Set cr (CmpP op1 op2));
11571 
11572   format %{ "cmpq    $op1, $op2\t# ptr" %}
11573   opcode(0x3B); /* Opcode 3B /r */
11574   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11575   ins_pipe(ialu_cr_reg_reg);
11576 %}
11577 
11578 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11579 %{
11580   match(Set cr (CmpP op1 (LoadP op2)));
11581 
11582   ins_cost(500); // XXX
11583   format %{ "cmpq    $op1, $op2\t# ptr" %}
11584   opcode(0x3B); /* Opcode 3B /r */
11585   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11586   ins_pipe(ialu_cr_reg_mem);
11587 %}
11588 
11589 // // // Cisc-spilled version of cmpP_rReg
11590 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11591 // //%{
11592 // //  match(Set cr (CmpP (LoadP op1) op2));
11593 // //
11594 // //  format %{ "CMPu   $op1,$op2" %}
11595 // //  ins_cost(500);
11596 // //  opcode(0x39);  /* Opcode 39 /r */
11597 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11598 // //%}
11599 
11600 // XXX this is generalized by compP_rReg_mem???
11601 // Compare raw pointer (used in out-of-heap check).
11602 // Only works because non-oop pointers must be raw pointers
11603 // and raw pointers have no anti-dependencies.
11604 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11605 %{
11606   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11607   match(Set cr (CmpP op1 (LoadP op2)));
11608 
11609   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11610   opcode(0x3B); /* Opcode 3B /r */
11611   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11612   ins_pipe(ialu_cr_reg_mem);
11613 %}
11614 
11615 // This will generate a signed flags result. This should be OK since
11616 // any compare to a zero should be eq/neq.
11617 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11618 %{
11619   match(Set cr (CmpP src zero));
11620 
11621   format %{ "testq   $src, $src\t# ptr" %}
11622   opcode(0x85);
11623   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11624   ins_pipe(ialu_cr_reg_imm);
11625 %}
11626 
11627 // This will generate a signed flags result. This should be OK since
11628 // any compare to a zero should be eq/neq.
11629 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11630 %{
11631   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11632   match(Set cr (CmpP (LoadP op) zero));
11633 
11634   ins_cost(500); // XXX
11635   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11636   opcode(0xF7); /* Opcode F7 /0 */
11637   ins_encode(REX_mem_wide(op),
11638              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11639   ins_pipe(ialu_cr_reg_imm);
11640 %}
11641 
11642 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11643 %{
11644   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11645   match(Set cr (CmpP (LoadP mem) zero));
11646 
11647   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11648   ins_encode %{
11649     __ cmpq(r12, $mem$$Address);
11650   %}
11651   ins_pipe(ialu_cr_reg_mem);
11652 %}
11653 
11654 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11655 %{
11656   match(Set cr (CmpN op1 op2));
11657 
11658   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11659   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11660   ins_pipe(ialu_cr_reg_reg);
11661 %}
11662 
11663 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11664 %{
11665   match(Set cr (CmpN src (LoadN mem)));
11666 
11667   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11668   ins_encode %{
11669     __ cmpl($src$$Register, $mem$$Address);
11670   %}
11671   ins_pipe(ialu_cr_reg_mem);
11672 %}
11673 
11674 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11675   match(Set cr (CmpN op1 op2));
11676 
11677   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11678   ins_encode %{
11679     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11680   %}
11681   ins_pipe(ialu_cr_reg_imm);
11682 %}
11683 
11684 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11685 %{
11686   match(Set cr (CmpN src (LoadN mem)));
11687 
11688   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11689   ins_encode %{
11690     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11691   %}
11692   ins_pipe(ialu_cr_reg_mem);
11693 %}
11694 
11695 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11696   match(Set cr (CmpN op1 op2));
11697 
11698   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11699   ins_encode %{
11700     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11701   %}
11702   ins_pipe(ialu_cr_reg_imm);
11703 %}
11704 
11705 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11706 %{
11707   match(Set cr (CmpN src (LoadNKlass mem)));
11708 
11709   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11710   ins_encode %{
11711     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11712   %}
11713   ins_pipe(ialu_cr_reg_mem);
11714 %}
11715 
11716 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11717   match(Set cr (CmpN src zero));
11718 
11719   format %{ "testl   $src, $src\t# compressed ptr" %}
11720   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11721   ins_pipe(ialu_cr_reg_imm);
11722 %}
11723 
11724 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11725 %{
11726   predicate(Universe::narrow_oop_base() != NULL);
11727   match(Set cr (CmpN (LoadN mem) zero));
11728 
11729   ins_cost(500); // XXX
11730   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11731   ins_encode %{
11732     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11733   %}
11734   ins_pipe(ialu_cr_reg_mem);
11735 %}
11736 
11737 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11738 %{
11739   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11740   match(Set cr (CmpN (LoadN mem) zero));
11741 
11742   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11743   ins_encode %{
11744     __ cmpl(r12, $mem$$Address);
11745   %}
11746   ins_pipe(ialu_cr_reg_mem);
11747 %}
11748 
11749 // Yanked all unsigned pointer compare operations.
11750 // Pointer compares are done with CmpP which is already unsigned.
11751 
11752 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11753 %{
11754   match(Set cr (CmpL op1 op2));
11755 
11756   format %{ "cmpq    $op1, $op2" %}
11757   opcode(0x3B);  /* Opcode 3B /r */
11758   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11759   ins_pipe(ialu_cr_reg_reg);
11760 %}
11761 
11762 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11763 %{
11764   match(Set cr (CmpL op1 op2));
11765 
11766   format %{ "cmpq    $op1, $op2" %}
11767   opcode(0x81, 0x07); /* Opcode 81 /7 */
11768   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11769   ins_pipe(ialu_cr_reg_imm);
11770 %}
11771 
11772 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11773 %{
11774   match(Set cr (CmpL op1 (LoadL op2)));
11775 
11776   format %{ "cmpq    $op1, $op2" %}
11777   opcode(0x3B); /* Opcode 3B /r */
11778   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11779   ins_pipe(ialu_cr_reg_mem);
11780 %}
11781 
11782 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11783 %{
11784   match(Set cr (CmpL src zero));
11785 
11786   format %{ "testq   $src, $src" %}
11787   opcode(0x85);
11788   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11789   ins_pipe(ialu_cr_reg_imm);
11790 %}
11791 
11792 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11793 %{
11794   match(Set cr (CmpL (AndL src con) zero));
11795 
11796   format %{ "testq   $src, $con\t# long" %}
11797   opcode(0xF7, 0x00);
11798   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11799   ins_pipe(ialu_cr_reg_imm);
11800 %}
11801 
11802 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11803 %{
11804   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11805 
11806   format %{ "testq   $src, $mem" %}
11807   opcode(0x85);
11808   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11809   ins_pipe(ialu_cr_reg_mem);
11810 %}
11811 
11812 instruct testL_reg_mem2(rFlagsReg cr, rRegP src, memory mem, immL0 zero)
11813 %{
11814   match(Set cr (CmpL (AndL (CastP2X src) (LoadL mem)) zero));
11815 
11816   format %{ "testq   $src, $mem" %}
11817   opcode(0x85);
11818   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11819   ins_pipe(ialu_cr_reg_mem);
11820 %}
11821 
11822 // Manifest a CmpL result in an integer register.  Very painful.
11823 // This is the test to avoid.
11824 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11825 %{
11826   match(Set dst (CmpL3 src1 src2));
11827   effect(KILL flags);
11828 
11829   ins_cost(275); // XXX
11830   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11831             "movl    $dst, -1\n\t"
11832             "jl,s    done\n\t"
11833             "setne   $dst\n\t"
11834             "movzbl  $dst, $dst\n\t"
11835     "done:" %}
11836   ins_encode(cmpl3_flag(src1, src2, dst));
11837   ins_pipe(pipe_slow);
11838 %}
11839 
11840 // Unsigned long compare Instructions; really, same as signed long except they
11841 // produce an rFlagsRegU instead of rFlagsReg.
11842 instruct compUL_rReg(rFlagsRegU cr, rRegL op1, rRegL op2)
11843 %{
11844   match(Set cr (CmpUL op1 op2));
11845 
11846   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11847   opcode(0x3B);  /* Opcode 3B /r */
11848   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11849   ins_pipe(ialu_cr_reg_reg);
11850 %}
11851 
11852 instruct compUL_rReg_imm(rFlagsRegU cr, rRegL op1, immL32 op2)
11853 %{
11854   match(Set cr (CmpUL op1 op2));
11855 
11856   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11857   opcode(0x81, 0x07); /* Opcode 81 /7 */
11858   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11859   ins_pipe(ialu_cr_reg_imm);
11860 %}
11861 
11862 instruct compUL_rReg_mem(rFlagsRegU cr, rRegL op1, memory op2)
11863 %{
11864   match(Set cr (CmpUL op1 (LoadL op2)));
11865 
11866   format %{ "cmpq    $op1, $op2\t# unsigned" %}
11867   opcode(0x3B); /* Opcode 3B /r */
11868   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11869   ins_pipe(ialu_cr_reg_mem);
11870 %}
11871 
11872 instruct testUL_reg(rFlagsRegU cr, rRegL src, immL0 zero)
11873 %{
11874   match(Set cr (CmpUL src zero));
11875 
11876   format %{ "testq   $src, $src\t# unsigned" %}
11877   opcode(0x85);
11878   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11879   ins_pipe(ialu_cr_reg_imm);
11880 %}
11881 
11882 instruct compB_mem_imm(rFlagsReg cr, memory mem, immI8 imm)
11883 %{
11884   match(Set cr (CmpI (LoadB mem) imm));
11885 
11886   ins_cost(125);
11887   format %{ "cmpb    $mem, $imm" %}
11888   ins_encode %{ __ cmpb($mem$$Address, $imm$$constant); %}
11889   ins_pipe(ialu_cr_reg_mem);
11890 %}
11891 
11892 instruct testUB_mem_imm(rFlagsReg cr, memory mem, immU8 imm, immI0 zero)
11893 %{
11894   match(Set cr (CmpI (AndI (LoadUB mem) imm) zero));
11895 
11896   ins_cost(125);
11897   format %{ "testb   $mem, $imm" %}
11898   ins_encode %{ __ testb($mem$$Address, $imm$$constant); %}
11899   ins_pipe(ialu_cr_reg_mem);
11900 %}
11901 
11902 //----------Max and Min--------------------------------------------------------
11903 // Min Instructions
11904 
11905 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11906 %{
11907   effect(USE_DEF dst, USE src, USE cr);
11908 
11909   format %{ "cmovlgt $dst, $src\t# min" %}
11910   opcode(0x0F, 0x4F);
11911   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11912   ins_pipe(pipe_cmov_reg);
11913 %}
11914 
11915 
11916 instruct minI_rReg(rRegI dst, rRegI src)
11917 %{
11918   match(Set dst (MinI dst src));
11919 
11920   ins_cost(200);
11921   expand %{
11922     rFlagsReg cr;
11923     compI_rReg(cr, dst, src);
11924     cmovI_reg_g(dst, src, cr);
11925   %}
11926 %}
11927 
11928 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11929 %{
11930   effect(USE_DEF dst, USE src, USE cr);
11931 
11932   format %{ "cmovllt $dst, $src\t# max" %}
11933   opcode(0x0F, 0x4C);
11934   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11935   ins_pipe(pipe_cmov_reg);
11936 %}
11937 
11938 
11939 instruct maxI_rReg(rRegI dst, rRegI src)
11940 %{
11941   match(Set dst (MaxI dst src));
11942 
11943   ins_cost(200);
11944   expand %{
11945     rFlagsReg cr;
11946     compI_rReg(cr, dst, src);
11947     cmovI_reg_l(dst, src, cr);
11948   %}
11949 %}
11950 
11951 // ============================================================================
11952 // Branch Instructions
11953 
11954 // Jump Direct - Label defines a relative address from JMP+1
11955 instruct jmpDir(label labl)
11956 %{
11957   match(Goto);
11958   effect(USE labl);
11959 
11960   ins_cost(300);
11961   format %{ "jmp     $labl" %}
11962   size(5);
11963   ins_encode %{
11964     Label* L = $labl$$label;
11965     __ jmp(*L, false); // Always long jump
11966   %}
11967   ins_pipe(pipe_jmp);
11968 %}
11969 
11970 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11971 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11972 %{
11973   match(If cop cr);
11974   effect(USE labl);
11975 
11976   ins_cost(300);
11977   format %{ "j$cop     $labl" %}
11978   size(6);
11979   ins_encode %{
11980     Label* L = $labl$$label;
11981     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11982   %}
11983   ins_pipe(pipe_jcc);
11984 %}
11985 
11986 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11987 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11988 %{
11989   predicate(!n->has_vector_mask_set());
11990   match(CountedLoopEnd cop cr);
11991   effect(USE labl);
11992 
11993   ins_cost(300);
11994   format %{ "j$cop     $labl\t# loop end" %}
11995   size(6);
11996   ins_encode %{
11997     Label* L = $labl$$label;
11998     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11999   %}
12000   ins_pipe(pipe_jcc);
12001 %}
12002 
12003 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12004 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12005   predicate(!n->has_vector_mask_set());
12006   match(CountedLoopEnd cop cmp);
12007   effect(USE labl);
12008 
12009   ins_cost(300);
12010   format %{ "j$cop,u   $labl\t# loop end" %}
12011   size(6);
12012   ins_encode %{
12013     Label* L = $labl$$label;
12014     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12015   %}
12016   ins_pipe(pipe_jcc);
12017 %}
12018 
12019 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12020   predicate(!n->has_vector_mask_set());
12021   match(CountedLoopEnd cop cmp);
12022   effect(USE labl);
12023 
12024   ins_cost(200);
12025   format %{ "j$cop,u   $labl\t# loop end" %}
12026   size(6);
12027   ins_encode %{
12028     Label* L = $labl$$label;
12029     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12030   %}
12031   ins_pipe(pipe_jcc);
12032 %}
12033 
12034 // mask version
12035 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12036 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
12037 %{
12038   predicate(n->has_vector_mask_set());
12039   match(CountedLoopEnd cop cr);
12040   effect(USE labl);
12041 
12042   ins_cost(400);
12043   format %{ "j$cop     $labl\t# loop end\n\t"
12044             "restorevectmask \t# vector mask restore for loops" %}
12045   size(10);
12046   ins_encode %{
12047     Label* L = $labl$$label;
12048     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12049     __ restorevectmask();
12050   %}
12051   ins_pipe(pipe_jcc);
12052 %}
12053 
12054 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12055 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12056   predicate(n->has_vector_mask_set());
12057   match(CountedLoopEnd cop cmp);
12058   effect(USE labl);
12059 
12060   ins_cost(400);
12061   format %{ "j$cop,u   $labl\t# loop end\n\t"
12062             "restorevectmask \t# vector mask restore for loops" %}
12063   size(10);
12064   ins_encode %{
12065     Label* L = $labl$$label;
12066     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12067     __ restorevectmask();
12068   %}
12069   ins_pipe(pipe_jcc);
12070 %}
12071 
12072 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12073   predicate(n->has_vector_mask_set());
12074   match(CountedLoopEnd cop cmp);
12075   effect(USE labl);
12076 
12077   ins_cost(300);
12078   format %{ "j$cop,u   $labl\t# loop end\n\t"
12079             "restorevectmask \t# vector mask restore for loops" %}
12080   size(10);
12081   ins_encode %{
12082     Label* L = $labl$$label;
12083     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12084     __ restorevectmask();
12085   %}
12086   ins_pipe(pipe_jcc);
12087 %}
12088 
12089 // Jump Direct Conditional - using unsigned comparison
12090 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12091   match(If cop cmp);
12092   effect(USE labl);
12093 
12094   ins_cost(300);
12095   format %{ "j$cop,u  $labl" %}
12096   size(6);
12097   ins_encode %{
12098     Label* L = $labl$$label;
12099     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12100   %}
12101   ins_pipe(pipe_jcc);
12102 %}
12103 
12104 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12105   match(If cop cmp);
12106   effect(USE labl);
12107 
12108   ins_cost(200);
12109   format %{ "j$cop,u  $labl" %}
12110   size(6);
12111   ins_encode %{
12112     Label* L = $labl$$label;
12113     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
12114   %}
12115   ins_pipe(pipe_jcc);
12116 %}
12117 
12118 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12119   match(If cop cmp);
12120   effect(USE labl);
12121 
12122   ins_cost(200);
12123   format %{ $$template
12124     if ($cop$$cmpcode == Assembler::notEqual) {
12125       $$emit$$"jp,u   $labl\n\t"
12126       $$emit$$"j$cop,u   $labl"
12127     } else {
12128       $$emit$$"jp,u   done\n\t"
12129       $$emit$$"j$cop,u   $labl\n\t"
12130       $$emit$$"done:"
12131     }
12132   %}
12133   ins_encode %{
12134     Label* l = $labl$$label;
12135     if ($cop$$cmpcode == Assembler::notEqual) {
12136       __ jcc(Assembler::parity, *l, false);
12137       __ jcc(Assembler::notEqual, *l, false);
12138     } else if ($cop$$cmpcode == Assembler::equal) {
12139       Label done;
12140       __ jccb(Assembler::parity, done);
12141       __ jcc(Assembler::equal, *l, false);
12142       __ bind(done);
12143     } else {
12144        ShouldNotReachHere();
12145     }
12146   %}
12147   ins_pipe(pipe_jcc);
12148 %}
12149 
12150 // ============================================================================
12151 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
12152 // superklass array for an instance of the superklass.  Set a hidden
12153 // internal cache on a hit (cache is checked with exposed code in
12154 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
12155 // encoding ALSO sets flags.
12156 
12157 instruct partialSubtypeCheck(rdi_RegP result,
12158                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12159                              rFlagsReg cr)
12160 %{
12161   match(Set result (PartialSubtypeCheck sub super));
12162   effect(KILL rcx, KILL cr);
12163 
12164   ins_cost(1100);  // slightly larger than the next version
12165   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12166             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12167             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12168             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
12169             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
12170             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12171             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
12172     "miss:\t" %}
12173 
12174   opcode(0x1); // Force a XOR of RDI
12175   ins_encode(enc_PartialSubtypeCheck());
12176   ins_pipe(pipe_slow);
12177 %}
12178 
12179 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
12180                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
12181                                      immP0 zero,
12182                                      rdi_RegP result)
12183 %{
12184   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
12185   effect(KILL rcx, KILL result);
12186 
12187   ins_cost(1000);
12188   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
12189             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
12190             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
12191             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
12192             "jne,s   miss\t\t# Missed: flags nz\n\t"
12193             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
12194     "miss:\t" %}
12195 
12196   opcode(0x0); // No need to XOR RDI
12197   ins_encode(enc_PartialSubtypeCheck());
12198   ins_pipe(pipe_slow);
12199 %}
12200 
12201 // ============================================================================
12202 // Branch Instructions -- short offset versions
12203 //
12204 // These instructions are used to replace jumps of a long offset (the default
12205 // match) with jumps of a shorter offset.  These instructions are all tagged
12206 // with the ins_short_branch attribute, which causes the ADLC to suppress the
12207 // match rules in general matching.  Instead, the ADLC generates a conversion
12208 // method in the MachNode which can be used to do in-place replacement of the
12209 // long variant with the shorter variant.  The compiler will determine if a
12210 // branch can be taken by the is_short_branch_offset() predicate in the machine
12211 // specific code section of the file.
12212 
12213 // Jump Direct - Label defines a relative address from JMP+1
12214 instruct jmpDir_short(label labl) %{
12215   match(Goto);
12216   effect(USE labl);
12217 
12218   ins_cost(300);
12219   format %{ "jmp,s   $labl" %}
12220   size(2);
12221   ins_encode %{
12222     Label* L = $labl$$label;
12223     __ jmpb(*L);
12224   %}
12225   ins_pipe(pipe_jmp);
12226   ins_short_branch(1);
12227 %}
12228 
12229 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12230 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
12231   match(If cop cr);
12232   effect(USE labl);
12233 
12234   ins_cost(300);
12235   format %{ "j$cop,s   $labl" %}
12236   size(2);
12237   ins_encode %{
12238     Label* L = $labl$$label;
12239     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12240   %}
12241   ins_pipe(pipe_jcc);
12242   ins_short_branch(1);
12243 %}
12244 
12245 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12246 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
12247   match(CountedLoopEnd cop cr);
12248   effect(USE labl);
12249 
12250   ins_cost(300);
12251   format %{ "j$cop,s   $labl\t# loop end" %}
12252   size(2);
12253   ins_encode %{
12254     Label* L = $labl$$label;
12255     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12256   %}
12257   ins_pipe(pipe_jcc);
12258   ins_short_branch(1);
12259 %}
12260 
12261 // Jump Direct Conditional - Label defines a relative address from Jcc+1
12262 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12263   match(CountedLoopEnd cop cmp);
12264   effect(USE labl);
12265 
12266   ins_cost(300);
12267   format %{ "j$cop,us  $labl\t# loop end" %}
12268   size(2);
12269   ins_encode %{
12270     Label* L = $labl$$label;
12271     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12272   %}
12273   ins_pipe(pipe_jcc);
12274   ins_short_branch(1);
12275 %}
12276 
12277 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12278   match(CountedLoopEnd cop cmp);
12279   effect(USE labl);
12280 
12281   ins_cost(300);
12282   format %{ "j$cop,us  $labl\t# loop end" %}
12283   size(2);
12284   ins_encode %{
12285     Label* L = $labl$$label;
12286     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12287   %}
12288   ins_pipe(pipe_jcc);
12289   ins_short_branch(1);
12290 %}
12291 
12292 // Jump Direct Conditional - using unsigned comparison
12293 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
12294   match(If cop cmp);
12295   effect(USE labl);
12296 
12297   ins_cost(300);
12298   format %{ "j$cop,us  $labl" %}
12299   size(2);
12300   ins_encode %{
12301     Label* L = $labl$$label;
12302     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12303   %}
12304   ins_pipe(pipe_jcc);
12305   ins_short_branch(1);
12306 %}
12307 
12308 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
12309   match(If cop cmp);
12310   effect(USE labl);
12311 
12312   ins_cost(300);
12313   format %{ "j$cop,us  $labl" %}
12314   size(2);
12315   ins_encode %{
12316     Label* L = $labl$$label;
12317     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
12318   %}
12319   ins_pipe(pipe_jcc);
12320   ins_short_branch(1);
12321 %}
12322 
12323 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
12324   match(If cop cmp);
12325   effect(USE labl);
12326 
12327   ins_cost(300);
12328   format %{ $$template
12329     if ($cop$$cmpcode == Assembler::notEqual) {
12330       $$emit$$"jp,u,s   $labl\n\t"
12331       $$emit$$"j$cop,u,s   $labl"
12332     } else {
12333       $$emit$$"jp,u,s   done\n\t"
12334       $$emit$$"j$cop,u,s  $labl\n\t"
12335       $$emit$$"done:"
12336     }
12337   %}
12338   size(4);
12339   ins_encode %{
12340     Label* l = $labl$$label;
12341     if ($cop$$cmpcode == Assembler::notEqual) {
12342       __ jccb(Assembler::parity, *l);
12343       __ jccb(Assembler::notEqual, *l);
12344     } else if ($cop$$cmpcode == Assembler::equal) {
12345       Label done;
12346       __ jccb(Assembler::parity, done);
12347       __ jccb(Assembler::equal, *l);
12348       __ bind(done);
12349     } else {
12350        ShouldNotReachHere();
12351     }
12352   %}
12353   ins_pipe(pipe_jcc);
12354   ins_short_branch(1);
12355 %}
12356 
12357 // ============================================================================
12358 // inlined locking and unlocking
12359 
12360 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
12361   predicate(Compile::current()->use_rtm());
12362   match(Set cr (FastLock object box));
12363   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
12364   ins_cost(300);
12365   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
12366   ins_encode %{
12367     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12368                  $scr$$Register, $cx1$$Register, $cx2$$Register,
12369                  _counters, _rtm_counters, _stack_rtm_counters,
12370                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
12371                  true, ra_->C->profile_rtm());
12372   %}
12373   ins_pipe(pipe_slow);
12374 %}
12375 
12376 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12377   predicate(!Compile::current()->use_rtm());
12378   match(Set cr (FastLock object box));
12379   effect(TEMP tmp, TEMP scr, USE_KILL box);
12380   ins_cost(300);
12381   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12382   ins_encode %{
12383     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12384                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12385   %}
12386   ins_pipe(pipe_slow);
12387 %}
12388 
12389 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12390   match(Set cr (FastUnlock object box));
12391   effect(TEMP tmp, USE_KILL box);
12392   ins_cost(300);
12393   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12394   ins_encode %{
12395     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12396   %}
12397   ins_pipe(pipe_slow);
12398 %}
12399 
12400 
12401 // ============================================================================
12402 // Safepoint Instructions
12403 instruct safePoint_poll(rFlagsReg cr)
12404 %{
12405   predicate(!Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12406   match(SafePoint);
12407   effect(KILL cr);
12408 
12409   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12410             "# Safepoint: poll for GC" %}
12411   ins_cost(125);
12412   ins_encode %{
12413     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12414     __ testl(rax, addr);
12415   %}
12416   ins_pipe(ialu_reg_mem);
12417 %}
12418 
12419 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12420 %{
12421   predicate(Assembler::is_polling_page_far() && SafepointMechanism::uses_global_page_poll());
12422   match(SafePoint poll);
12423   effect(KILL cr, USE poll);
12424 
12425   format %{ "testl  rax, [$poll]\t"
12426             "# Safepoint: poll for GC" %}
12427   ins_cost(125);
12428   ins_encode %{
12429     __ relocate(relocInfo::poll_type);
12430     __ testl(rax, Address($poll$$Register, 0));
12431   %}
12432   ins_pipe(ialu_reg_mem);
12433 %}
12434 
12435 instruct safePoint_poll_tls(rFlagsReg cr, rRegP poll)
12436 %{
12437   predicate(SafepointMechanism::uses_thread_local_poll());
12438   match(SafePoint poll);
12439   effect(KILL cr, USE poll);
12440 
12441   format %{ "testl  rax, [$poll]\t"
12442             "# Safepoint: poll for GC" %}
12443   ins_cost(125);
12444   size(4); /* setting an explicit size will cause debug builds to assert if size is incorrect */
12445   ins_encode %{
12446     __ relocate(relocInfo::poll_type);
12447     address pre_pc = __ pc();
12448     __ testl(rax, Address($poll$$Register, 0));
12449     assert(nativeInstruction_at(pre_pc)->is_safepoint_poll(), "must emit test %%eax [reg]");
12450   %}
12451   ins_pipe(ialu_reg_mem);
12452 %}
12453 
12454 // ============================================================================
12455 // Procedure Call/Return Instructions
12456 // Call Java Static Instruction
12457 // Note: If this code changes, the corresponding ret_addr_offset() and
12458 //       compute_padding() functions will have to be adjusted.
12459 instruct CallStaticJavaDirect(method meth) %{
12460   match(CallStaticJava);
12461   effect(USE meth);
12462 
12463   ins_cost(300);
12464   format %{ "call,static " %}
12465   opcode(0xE8); /* E8 cd */
12466   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12467   ins_pipe(pipe_slow);
12468   ins_alignment(4);
12469 %}
12470 
12471 // Call Java Dynamic Instruction
12472 // Note: If this code changes, the corresponding ret_addr_offset() and
12473 //       compute_padding() functions will have to be adjusted.
12474 instruct CallDynamicJavaDirect(method meth)
12475 %{
12476   match(CallDynamicJava);
12477   effect(USE meth);
12478 
12479   ins_cost(300);
12480   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12481             "call,dynamic " %}
12482   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12483   ins_pipe(pipe_slow);
12484   ins_alignment(4);
12485 %}
12486 
12487 // Call Runtime Instruction
12488 instruct CallRuntimeDirect(method meth)
12489 %{
12490   match(CallRuntime);
12491   effect(USE meth);
12492 
12493   ins_cost(300);
12494   format %{ "call,runtime " %}
12495   ins_encode(clear_avx, Java_To_Runtime(meth));
12496   ins_pipe(pipe_slow);
12497 %}
12498 
12499 // Call runtime without safepoint
12500 instruct CallLeafDirect(method meth)
12501 %{
12502   match(CallLeaf);
12503   effect(USE meth);
12504 
12505   ins_cost(300);
12506   format %{ "call_leaf,runtime " %}
12507   ins_encode(clear_avx, Java_To_Runtime(meth));
12508   ins_pipe(pipe_slow);
12509 %}
12510 
12511 // Call runtime without safepoint
12512 instruct CallLeafNoFPDirect(method meth)
12513 %{
12514   match(CallLeafNoFP);
12515   effect(USE meth);
12516 
12517   ins_cost(300);
12518   format %{ "call_leaf_nofp,runtime " %}
12519   ins_encode(clear_avx, Java_To_Runtime(meth));
12520   ins_pipe(pipe_slow);
12521 %}
12522 
12523 // Return Instruction
12524 // Remove the return address & jump to it.
12525 // Notice: We always emit a nop after a ret to make sure there is room
12526 // for safepoint patching
12527 instruct Ret()
12528 %{
12529   match(Return);
12530 
12531   format %{ "ret" %}
12532   opcode(0xC3);
12533   ins_encode(OpcP);
12534   ins_pipe(pipe_jmp);
12535 %}
12536 
12537 // Tail Call; Jump from runtime stub to Java code.
12538 // Also known as an 'interprocedural jump'.
12539 // Target of jump will eventually return to caller.
12540 // TailJump below removes the return address.
12541 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12542 %{
12543   match(TailCall jump_target method_oop);
12544 
12545   ins_cost(300);
12546   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12547   opcode(0xFF, 0x4); /* Opcode FF /4 */
12548   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12549   ins_pipe(pipe_jmp);
12550 %}
12551 
12552 // Tail Jump; remove the return address; jump to target.
12553 // TailCall above leaves the return address around.
12554 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12555 %{
12556   match(TailJump jump_target ex_oop);
12557 
12558   ins_cost(300);
12559   format %{ "popq    rdx\t# pop return address\n\t"
12560             "jmp     $jump_target" %}
12561   opcode(0xFF, 0x4); /* Opcode FF /4 */
12562   ins_encode(Opcode(0x5a), // popq rdx
12563              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12564   ins_pipe(pipe_jmp);
12565 %}
12566 
12567 // Create exception oop: created by stack-crawling runtime code.
12568 // Created exception is now available to this handler, and is setup
12569 // just prior to jumping to this handler.  No code emitted.
12570 instruct CreateException(rax_RegP ex_oop)
12571 %{
12572   match(Set ex_oop (CreateEx));
12573 
12574   size(0);
12575   // use the following format syntax
12576   format %{ "# exception oop is in rax; no code emitted" %}
12577   ins_encode();
12578   ins_pipe(empty);
12579 %}
12580 
12581 // Rethrow exception:
12582 // The exception oop will come in the first argument position.
12583 // Then JUMP (not call) to the rethrow stub code.
12584 instruct RethrowException()
12585 %{
12586   match(Rethrow);
12587 
12588   // use the following format syntax
12589   format %{ "jmp     rethrow_stub" %}
12590   ins_encode(enc_rethrow);
12591   ins_pipe(pipe_jmp);
12592 %}
12593 
12594 //
12595 // Execute ZGC load barrier (strong) slow path
12596 //
12597 
12598 // When running without XMM regs
12599 instruct loadBarrierSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12600 
12601   match(Set dst (LoadBarrierSlowReg mem));
12602   predicate(MaxVectorSize < 16);
12603 
12604   effect(DEF dst, KILL cr);
12605 
12606   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12607   ins_encode %{
12608 #if INCLUDE_ZGC
12609     Register d = $dst$$Register;
12610     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12611 
12612     assert(d != r12, "Can't be R12!");
12613     assert(d != r15, "Can't be R15!");
12614     assert(d != rsp, "Can't be RSP!");
12615 
12616     __ lea(d, $mem$$Address);
12617     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12618 #else
12619     ShouldNotReachHere();
12620 #endif
12621   %}
12622   ins_pipe(pipe_slow);
12623 %}
12624 
12625 // For XMM and YMM enabled processors
12626 instruct loadBarrierSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12627                                      rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12628                                      rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12629                                      rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12630                                      rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12631 
12632   match(Set dst (LoadBarrierSlowReg mem));
12633   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12634 
12635   effect(DEF dst, KILL cr,
12636          KILL x0, KILL x1, KILL x2, KILL x3,
12637          KILL x4, KILL x5, KILL x6, KILL x7,
12638          KILL x8, KILL x9, KILL x10, KILL x11,
12639          KILL x12, KILL x13, KILL x14, KILL x15);
12640 
12641   format %{"LoadBarrierSlowRegXmm $dst, $mem" %}
12642   ins_encode %{
12643 #if INCLUDE_ZGC
12644     Register d = $dst$$Register;
12645     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12646 
12647     assert(d != r12, "Can't be R12!");
12648     assert(d != r15, "Can't be R15!");
12649     assert(d != rsp, "Can't be RSP!");
12650 
12651     __ lea(d, $mem$$Address);
12652     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12653 #else
12654     ShouldNotReachHere();
12655 #endif
12656   %}
12657   ins_pipe(pipe_slow);
12658 %}
12659 
12660 // For ZMM enabled processors
12661 instruct loadBarrierSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12662                                rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12663                                rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12664                                rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12665                                rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12666                                rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12667                                rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12668                                rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12669                                rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12670 
12671   match(Set dst (LoadBarrierSlowReg mem));
12672   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12673 
12674   effect(DEF dst, KILL cr,
12675          KILL x0, KILL x1, KILL x2, KILL x3,
12676          KILL x4, KILL x5, KILL x6, KILL x7,
12677          KILL x8, KILL x9, KILL x10, KILL x11,
12678          KILL x12, KILL x13, KILL x14, KILL x15,
12679          KILL x16, KILL x17, KILL x18, KILL x19,
12680          KILL x20, KILL x21, KILL x22, KILL x23,
12681          KILL x24, KILL x25, KILL x26, KILL x27,
12682          KILL x28, KILL x29, KILL x30, KILL x31);
12683 
12684   format %{"LoadBarrierSlowRegZmm $dst, $mem" %}
12685   ins_encode %{
12686 #if INCLUDE_ZGC
12687     Register d = $dst$$Register;
12688     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12689 
12690     assert(d != r12, "Can't be R12!");
12691     assert(d != r15, "Can't be R15!");
12692     assert(d != rsp, "Can't be RSP!");
12693 
12694     __ lea(d, $mem$$Address);
12695     __ call(RuntimeAddress(bs->load_barrier_slow_stub(d)));
12696 #else
12697     ShouldNotReachHere();
12698 #endif
12699   %}
12700   ins_pipe(pipe_slow);
12701 %}
12702 
12703 //
12704 // Execute ZGC load barrier (weak) slow path
12705 //
12706 
12707 // When running without XMM regs
12708 instruct loadBarrierWeakSlowRegNoVec(rRegP dst, memory mem, rFlagsReg cr) %{
12709 
12710   match(Set dst (LoadBarrierSlowReg mem));
12711   predicate(MaxVectorSize < 16);
12712 
12713   effect(DEF dst, KILL cr);
12714 
12715   format %{"LoadBarrierSlowRegNoVec $dst, $mem" %}
12716   ins_encode %{
12717 #if INCLUDE_ZGC
12718     Register d = $dst$$Register;
12719     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12720 
12721     assert(d != r12, "Can't be R12!");
12722     assert(d != r15, "Can't be R15!");
12723     assert(d != rsp, "Can't be RSP!");
12724 
12725     __ lea(d, $mem$$Address);
12726     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12727 #else
12728     ShouldNotReachHere();
12729 #endif
12730   %}
12731   ins_pipe(pipe_slow);
12732 %}
12733 
12734 // For XMM and YMM enabled processors
12735 instruct loadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory mem, rFlagsReg cr,
12736                                          rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12737                                          rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12738                                          rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12739                                          rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
12740 
12741   match(Set dst (LoadBarrierWeakSlowReg mem));
12742   predicate((UseSSE > 0) && (UseAVX <= 2) && (MaxVectorSize >= 16));
12743 
12744   effect(DEF dst, KILL cr,
12745          KILL x0, KILL x1, KILL x2, KILL x3,
12746          KILL x4, KILL x5, KILL x6, KILL x7,
12747          KILL x8, KILL x9, KILL x10, KILL x11,
12748          KILL x12, KILL x13, KILL x14, KILL x15);
12749 
12750   format %{"LoadBarrierWeakSlowRegXmm $dst, $mem" %}
12751   ins_encode %{
12752 #if INCLUDE_ZGC
12753     Register d = $dst$$Register;
12754     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12755 
12756     assert(d != r12, "Can't be R12!");
12757     assert(d != r15, "Can't be R15!");
12758     assert(d != rsp, "Can't be RSP!");
12759 
12760     __ lea(d,$mem$$Address);
12761     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12762 #else
12763     ShouldNotReachHere();
12764 #endif
12765   %}
12766   ins_pipe(pipe_slow);
12767 %}
12768 
12769 // For ZMM enabled processors
12770 instruct loadBarrierWeakSlowRegZmm(rRegP dst, memory mem, rFlagsReg cr,
12771                                    rxmm0 x0, rxmm1 x1, rxmm2 x2,rxmm3 x3,
12772                                    rxmm4 x4, rxmm5 x5, rxmm6 x6, rxmm7 x7,
12773                                    rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
12774                                    rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15,
12775                                    rxmm16 x16, rxmm17 x17, rxmm18 x18, rxmm19 x19,
12776                                    rxmm20 x20, rxmm21 x21, rxmm22 x22, rxmm23 x23,
12777                                    rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
12778                                    rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
12779 
12780   match(Set dst (LoadBarrierWeakSlowReg mem));
12781   predicate((UseAVX == 3) && (MaxVectorSize >= 16));
12782 
12783   effect(DEF dst, KILL cr,
12784          KILL x0, KILL x1, KILL x2, KILL x3,
12785          KILL x4, KILL x5, KILL x6, KILL x7,
12786          KILL x8, KILL x9, KILL x10, KILL x11,
12787          KILL x12, KILL x13, KILL x14, KILL x15,
12788          KILL x16, KILL x17, KILL x18, KILL x19,
12789          KILL x20, KILL x21, KILL x22, KILL x23,
12790          KILL x24, KILL x25, KILL x26, KILL x27,
12791          KILL x28, KILL x29, KILL x30, KILL x31);
12792 
12793   format %{"LoadBarrierWeakSlowRegZmm $dst, $mem" %}
12794   ins_encode %{
12795 #if INCLUDE_ZGC
12796     Register d = $dst$$Register;
12797     ZBarrierSetAssembler* bs = (ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
12798 
12799     assert(d != r12, "Can't be R12!");
12800     assert(d != r15, "Can't be R15!");
12801     assert(d != rsp, "Can't be RSP!");
12802 
12803     __ lea(d,$mem$$Address);
12804     __ call(RuntimeAddress(bs->load_barrier_weak_slow_stub(d)));
12805 #else
12806     ShouldNotReachHere();
12807 #endif
12808   %}
12809   ins_pipe(pipe_slow);
12810 %}
12811 
12812 // ============================================================================
12813 // This name is KNOWN by the ADLC and cannot be changed.
12814 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12815 // for this guy.
12816 instruct tlsLoadP(r15_RegP dst) %{
12817   match(Set dst (ThreadLocal));
12818   effect(DEF dst);
12819 
12820   size(0);
12821   format %{ "# TLS is in R15" %}
12822   ins_encode( /*empty encoding*/ );
12823   ins_pipe(ialu_reg_reg);
12824 %}
12825 
12826 
12827 //----------PEEPHOLE RULES-----------------------------------------------------
12828 // These must follow all instruction definitions as they use the names
12829 // defined in the instructions definitions.
12830 //
12831 // peepmatch ( root_instr_name [preceding_instruction]* );
12832 //
12833 // peepconstraint %{
12834 // (instruction_number.operand_name relational_op instruction_number.operand_name
12835 //  [, ...] );
12836 // // instruction numbers are zero-based using left to right order in peepmatch
12837 //
12838 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12839 // // provide an instruction_number.operand_name for each operand that appears
12840 // // in the replacement instruction's match rule
12841 //
12842 // ---------VM FLAGS---------------------------------------------------------
12843 //
12844 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12845 //
12846 // Each peephole rule is given an identifying number starting with zero and
12847 // increasing by one in the order seen by the parser.  An individual peephole
12848 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12849 // on the command-line.
12850 //
12851 // ---------CURRENT LIMITATIONS----------------------------------------------
12852 //
12853 // Only match adjacent instructions in same basic block
12854 // Only equality constraints
12855 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12856 // Only one replacement instruction
12857 //
12858 // ---------EXAMPLE----------------------------------------------------------
12859 //
12860 // // pertinent parts of existing instructions in architecture description
12861 // instruct movI(rRegI dst, rRegI src)
12862 // %{
12863 //   match(Set dst (CopyI src));
12864 // %}
12865 //
12866 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12867 // %{
12868 //   match(Set dst (AddI dst src));
12869 //   effect(KILL cr);
12870 // %}
12871 //
12872 // // Change (inc mov) to lea
12873 // peephole %{
12874 //   // increment preceeded by register-register move
12875 //   peepmatch ( incI_rReg movI );
12876 //   // require that the destination register of the increment
12877 //   // match the destination register of the move
12878 //   peepconstraint ( 0.dst == 1.dst );
12879 //   // construct a replacement instruction that sets
12880 //   // the destination to ( move's source register + one )
12881 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12882 // %}
12883 //
12884 
12885 // Implementation no longer uses movX instructions since
12886 // machine-independent system no longer uses CopyX nodes.
12887 //
12888 // peephole
12889 // %{
12890 //   peepmatch (incI_rReg movI);
12891 //   peepconstraint (0.dst == 1.dst);
12892 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12893 // %}
12894 
12895 // peephole
12896 // %{
12897 //   peepmatch (decI_rReg movI);
12898 //   peepconstraint (0.dst == 1.dst);
12899 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12900 // %}
12901 
12902 // peephole
12903 // %{
12904 //   peepmatch (addI_rReg_imm movI);
12905 //   peepconstraint (0.dst == 1.dst);
12906 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12907 // %}
12908 
12909 // peephole
12910 // %{
12911 //   peepmatch (incL_rReg movL);
12912 //   peepconstraint (0.dst == 1.dst);
12913 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12914 // %}
12915 
12916 // peephole
12917 // %{
12918 //   peepmatch (decL_rReg movL);
12919 //   peepconstraint (0.dst == 1.dst);
12920 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12921 // %}
12922 
12923 // peephole
12924 // %{
12925 //   peepmatch (addL_rReg_imm movL);
12926 //   peepconstraint (0.dst == 1.dst);
12927 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12928 // %}
12929 
12930 // peephole
12931 // %{
12932 //   peepmatch (addP_rReg_imm movP);
12933 //   peepconstraint (0.dst == 1.dst);
12934 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12935 // %}
12936 
12937 // // Change load of spilled value to only a spill
12938 // instruct storeI(memory mem, rRegI src)
12939 // %{
12940 //   match(Set mem (StoreI mem src));
12941 // %}
12942 //
12943 // instruct loadI(rRegI dst, memory mem)
12944 // %{
12945 //   match(Set dst (LoadI mem));
12946 // %}
12947 //
12948 
12949 peephole
12950 %{
12951   peepmatch (loadI storeI);
12952   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12953   peepreplace (storeI(1.mem 1.mem 1.src));
12954 %}
12955 
12956 peephole
12957 %{
12958   peepmatch (loadL storeL);
12959   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12960   peepreplace (storeL(1.mem 1.mem 1.src));
12961 %}
12962 
12963 //----------SMARTSPILL RULES---------------------------------------------------
12964 // These must follow all instruction definitions as they use the names
12965 // defined in the instructions definitions.