1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   MacroAssembler _masm(&cbuf);
 957 
 958   if (C->max_vector_size() > 16) {
 959     // Clear upper bits of YMM registers when current compiled code uses
 960     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 961     __ vzeroupper();
 962   }
 963 
 964   int framesize = C->frame_size_in_bytes();
 965   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 966   // Remove word for return adr already pushed
 967   // and RBP
 968   framesize -= 2*wordSize;
 969 
 970   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 971 
 972   if (framesize) {
 973     emit_opcode(cbuf, Assembler::REX_W);
 974     if (framesize < 0x80) {
 975       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 976       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 977       emit_d8(cbuf, framesize);
 978     } else {
 979       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 980       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 981       emit_d32(cbuf, framesize);
 982     }
 983   }
 984 
 985   // popq rbp
 986   emit_opcode(cbuf, 0x58 | RBP_enc);
 987 
 988   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 989     __ reserved_stack_check(); 
 990   }
 991 
 992   if (do_polling() && C->is_method_compilation()) {
 993     MacroAssembler _masm(&cbuf);
 994     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 995     if (Assembler::is_polling_page_far()) {
 996       __ lea(rscratch1, polling_page);
 997       __ relocate(relocInfo::poll_return_type);
 998       __ testl(rax, Address(rscratch1, 0));
 999     } else {
1000       __ testl(rax, polling_page);
1001     }
1002   }
1003 }
1004 
1005 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1006 {
1007   return MachNode::size(ra_); // too many variables; just compute it
1008                               // the hard way
1009 }
1010 
1011 int MachEpilogNode::reloc() const
1012 {
1013   return 2; // a large enough number
1014 }
1015 
1016 const Pipeline* MachEpilogNode::pipeline() const
1017 {
1018   return MachNode::pipeline_class();
1019 }
1020 
1021 int MachEpilogNode::safepoint_offset() const
1022 {
1023   return 0;
1024 }
1025 
1026 //=============================================================================
1027 
1028 enum RC {
1029   rc_bad,
1030   rc_int,
1031   rc_float,
1032   rc_stack
1033 };
1034 
1035 static enum RC rc_class(OptoReg::Name reg)
1036 {
1037   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1038 
1039   if (OptoReg::is_stack(reg)) return rc_stack;
1040 
1041   VMReg r = OptoReg::as_VMReg(reg);
1042 
1043   if (r->is_Register()) return rc_int;
1044 
1045   assert(r->is_XMMRegister(), "must be");
1046   return rc_float;
1047 }
1048 
1049 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1050 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1051                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1052 
1053 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1054                             int stack_offset, int reg, uint ireg, outputStream* st);
1055 
1056 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1057                                       int dst_offset, uint ireg, outputStream* st) {
1058   if (cbuf) {
1059     MacroAssembler _masm(cbuf);
1060     switch (ireg) {
1061     case Op_VecS:
1062       __ movq(Address(rsp, -8), rax);
1063       __ movl(rax, Address(rsp, src_offset));
1064       __ movl(Address(rsp, dst_offset), rax);
1065       __ movq(rax, Address(rsp, -8));
1066       break;
1067     case Op_VecD:
1068       __ pushq(Address(rsp, src_offset));
1069       __ popq (Address(rsp, dst_offset));
1070       break;
1071     case Op_VecX:
1072       __ pushq(Address(rsp, src_offset));
1073       __ popq (Address(rsp, dst_offset));
1074       __ pushq(Address(rsp, src_offset+8));
1075       __ popq (Address(rsp, dst_offset+8));
1076       break;
1077     case Op_VecY:
1078       __ vmovdqu(Address(rsp, -32), xmm0);
1079       __ vmovdqu(xmm0, Address(rsp, src_offset));
1080       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1081       __ vmovdqu(xmm0, Address(rsp, -32));
1082     case Op_VecZ:
1083       __ evmovdqul(Address(rsp, -64), xmm0, 2);
1084       __ evmovdqul(xmm0, Address(rsp, src_offset), 2);
1085       __ evmovdqul(Address(rsp, dst_offset), xmm0, 2);
1086       __ evmovdqul(xmm0, Address(rsp, -64), 2);
1087       break;
1088     default:
1089       ShouldNotReachHere();
1090     }
1091 #ifndef PRODUCT
1092   } else {
1093     switch (ireg) {
1094     case Op_VecS:
1095       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1096                 "movl    rax, [rsp + #%d]\n\t"
1097                 "movl    [rsp + #%d], rax\n\t"
1098                 "movq    rax, [rsp - #8]",
1099                 src_offset, dst_offset);
1100       break;
1101     case Op_VecD:
1102       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1103                 "popq    [rsp + #%d]",
1104                 src_offset, dst_offset);
1105       break;
1106      case Op_VecX:
1107       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1108                 "popq    [rsp + #%d]\n\t"
1109                 "pushq   [rsp + #%d]\n\t"
1110                 "popq    [rsp + #%d]",
1111                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1112       break;
1113     case Op_VecY:
1114       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1115                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1116                 "vmovdqu [rsp + #%d], xmm0\n\t"
1117                 "vmovdqu xmm0, [rsp - #32]",
1118                 src_offset, dst_offset);
1119       break;
1120     case Op_VecZ:
1121       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1122                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1123                 "vmovdqu [rsp + #%d], xmm0\n\t"
1124                 "vmovdqu xmm0, [rsp - #64]",
1125                 src_offset, dst_offset);
1126       break;
1127     default:
1128       ShouldNotReachHere();
1129     }
1130 #endif
1131   }
1132 }
1133 
1134 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1135                                        PhaseRegAlloc* ra_,
1136                                        bool do_size,
1137                                        outputStream* st) const {
1138   assert(cbuf != NULL || st  != NULL, "sanity");
1139   // Get registers to move
1140   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1141   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1142   OptoReg::Name dst_second = ra_->get_reg_second(this);
1143   OptoReg::Name dst_first = ra_->get_reg_first(this);
1144 
1145   enum RC src_second_rc = rc_class(src_second);
1146   enum RC src_first_rc = rc_class(src_first);
1147   enum RC dst_second_rc = rc_class(dst_second);
1148   enum RC dst_first_rc = rc_class(dst_first);
1149 
1150   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1151          "must move at least 1 register" );
1152 
1153   if (src_first == dst_first && src_second == dst_second) {
1154     // Self copy, no move
1155     return 0;
1156   }
1157   if (bottom_type()->isa_vect() != NULL) {
1158     uint ireg = ideal_reg();
1159     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1160     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1161     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1162       // mem -> mem
1163       int src_offset = ra_->reg2offset(src_first);
1164       int dst_offset = ra_->reg2offset(dst_first);
1165       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1166     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1167       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1168     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1169       int stack_offset = ra_->reg2offset(dst_first);
1170       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1171     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1172       int stack_offset = ra_->reg2offset(src_first);
1173       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1174     } else {
1175       ShouldNotReachHere();
1176     }
1177     return 0;
1178   }
1179   if (src_first_rc == rc_stack) {
1180     // mem ->
1181     if (dst_first_rc == rc_stack) {
1182       // mem -> mem
1183       assert(src_second != dst_first, "overlap");
1184       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1185           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1186         // 64-bit
1187         int src_offset = ra_->reg2offset(src_first);
1188         int dst_offset = ra_->reg2offset(dst_first);
1189         if (cbuf) {
1190           MacroAssembler _masm(cbuf);
1191           __ pushq(Address(rsp, src_offset));
1192           __ popq (Address(rsp, dst_offset));
1193 #ifndef PRODUCT
1194         } else {
1195           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1196                     "popq    [rsp + #%d]",
1197                      src_offset, dst_offset);
1198 #endif
1199         }
1200       } else {
1201         // 32-bit
1202         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1203         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1204         // No pushl/popl, so:
1205         int src_offset = ra_->reg2offset(src_first);
1206         int dst_offset = ra_->reg2offset(dst_first);
1207         if (cbuf) {
1208           MacroAssembler _masm(cbuf);
1209           __ movq(Address(rsp, -8), rax);
1210           __ movl(rax, Address(rsp, src_offset));
1211           __ movl(Address(rsp, dst_offset), rax);
1212           __ movq(rax, Address(rsp, -8));
1213 #ifndef PRODUCT
1214         } else {
1215           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1216                     "movl    rax, [rsp + #%d]\n\t"
1217                     "movl    [rsp + #%d], rax\n\t"
1218                     "movq    rax, [rsp - #8]",
1219                      src_offset, dst_offset);
1220 #endif
1221         }
1222       }
1223       return 0;
1224     } else if (dst_first_rc == rc_int) {
1225       // mem -> gpr
1226       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1227           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1228         // 64-bit
1229         int offset = ra_->reg2offset(src_first);
1230         if (cbuf) {
1231           MacroAssembler _masm(cbuf);
1232           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1233 #ifndef PRODUCT
1234         } else {
1235           st->print("movq    %s, [rsp + #%d]\t# spill",
1236                      Matcher::regName[dst_first],
1237                      offset);
1238 #endif
1239         }
1240       } else {
1241         // 32-bit
1242         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1243         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1244         int offset = ra_->reg2offset(src_first);
1245         if (cbuf) {
1246           MacroAssembler _masm(cbuf);
1247           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1248 #ifndef PRODUCT
1249         } else {
1250           st->print("movl    %s, [rsp + #%d]\t# spill",
1251                      Matcher::regName[dst_first],
1252                      offset);
1253 #endif
1254         }
1255       }
1256       return 0;
1257     } else if (dst_first_rc == rc_float) {
1258       // mem-> xmm
1259       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1260           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1261         // 64-bit
1262         int offset = ra_->reg2offset(src_first);
1263         if (cbuf) {
1264           MacroAssembler _masm(cbuf);
1265           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1266 #ifndef PRODUCT
1267         } else {
1268           st->print("%s  %s, [rsp + #%d]\t# spill",
1269                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1270                      Matcher::regName[dst_first],
1271                      offset);
1272 #endif
1273         }
1274       } else {
1275         // 32-bit
1276         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1277         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1278         int offset = ra_->reg2offset(src_first);
1279         if (cbuf) {
1280           MacroAssembler _masm(cbuf);
1281           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1282 #ifndef PRODUCT
1283         } else {
1284           st->print("movss   %s, [rsp + #%d]\t# spill",
1285                      Matcher::regName[dst_first],
1286                      offset);
1287 #endif
1288         }
1289       }
1290       return 0;
1291     }
1292   } else if (src_first_rc == rc_int) {
1293     // gpr ->
1294     if (dst_first_rc == rc_stack) {
1295       // gpr -> mem
1296       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1297           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1298         // 64-bit
1299         int offset = ra_->reg2offset(dst_first);
1300         if (cbuf) {
1301           MacroAssembler _masm(cbuf);
1302           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1303 #ifndef PRODUCT
1304         } else {
1305           st->print("movq    [rsp + #%d], %s\t# spill",
1306                      offset,
1307                      Matcher::regName[src_first]);
1308 #endif
1309         }
1310       } else {
1311         // 32-bit
1312         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1313         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1314         int offset = ra_->reg2offset(dst_first);
1315         if (cbuf) {
1316           MacroAssembler _masm(cbuf);
1317           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1318 #ifndef PRODUCT
1319         } else {
1320           st->print("movl    [rsp + #%d], %s\t# spill",
1321                      offset,
1322                      Matcher::regName[src_first]);
1323 #endif
1324         }
1325       }
1326       return 0;
1327     } else if (dst_first_rc == rc_int) {
1328       // gpr -> gpr
1329       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1330           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1331         // 64-bit
1332         if (cbuf) {
1333           MacroAssembler _masm(cbuf);
1334           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1335                   as_Register(Matcher::_regEncode[src_first]));
1336 #ifndef PRODUCT
1337         } else {
1338           st->print("movq    %s, %s\t# spill",
1339                      Matcher::regName[dst_first],
1340                      Matcher::regName[src_first]);
1341 #endif
1342         }
1343         return 0;
1344       } else {
1345         // 32-bit
1346         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1347         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1348         if (cbuf) {
1349           MacroAssembler _masm(cbuf);
1350           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1351                   as_Register(Matcher::_regEncode[src_first]));
1352 #ifndef PRODUCT
1353         } else {
1354           st->print("movl    %s, %s\t# spill",
1355                      Matcher::regName[dst_first],
1356                      Matcher::regName[src_first]);
1357 #endif
1358         }
1359         return 0;
1360       }
1361     } else if (dst_first_rc == rc_float) {
1362       // gpr -> xmm
1363       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1364           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1365         // 64-bit
1366         if (cbuf) {
1367           MacroAssembler _masm(cbuf);
1368           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1369 #ifndef PRODUCT
1370         } else {
1371           st->print("movdq   %s, %s\t# spill",
1372                      Matcher::regName[dst_first],
1373                      Matcher::regName[src_first]);
1374 #endif
1375         }
1376       } else {
1377         // 32-bit
1378         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1379         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1380         if (cbuf) {
1381           MacroAssembler _masm(cbuf);
1382           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1383 #ifndef PRODUCT
1384         } else {
1385           st->print("movdl   %s, %s\t# spill",
1386                      Matcher::regName[dst_first],
1387                      Matcher::regName[src_first]);
1388 #endif
1389         }
1390       }
1391       return 0;
1392     }
1393   } else if (src_first_rc == rc_float) {
1394     // xmm ->
1395     if (dst_first_rc == rc_stack) {
1396       // xmm -> mem
1397       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1398           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1399         // 64-bit
1400         int offset = ra_->reg2offset(dst_first);
1401         if (cbuf) {
1402           MacroAssembler _masm(cbuf);
1403           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1404 #ifndef PRODUCT
1405         } else {
1406           st->print("movsd   [rsp + #%d], %s\t# spill",
1407                      offset,
1408                      Matcher::regName[src_first]);
1409 #endif
1410         }
1411       } else {
1412         // 32-bit
1413         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1414         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1415         int offset = ra_->reg2offset(dst_first);
1416         if (cbuf) {
1417           MacroAssembler _masm(cbuf);
1418           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1419 #ifndef PRODUCT
1420         } else {
1421           st->print("movss   [rsp + #%d], %s\t# spill",
1422                      offset,
1423                      Matcher::regName[src_first]);
1424 #endif
1425         }
1426       }
1427       return 0;
1428     } else if (dst_first_rc == rc_int) {
1429       // xmm -> gpr
1430       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1431           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1432         // 64-bit
1433         if (cbuf) {
1434           MacroAssembler _masm(cbuf);
1435           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1436 #ifndef PRODUCT
1437         } else {
1438           st->print("movdq   %s, %s\t# spill",
1439                      Matcher::regName[dst_first],
1440                      Matcher::regName[src_first]);
1441 #endif
1442         }
1443       } else {
1444         // 32-bit
1445         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1446         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1447         if (cbuf) {
1448           MacroAssembler _masm(cbuf);
1449           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1450 #ifndef PRODUCT
1451         } else {
1452           st->print("movdl   %s, %s\t# spill",
1453                      Matcher::regName[dst_first],
1454                      Matcher::regName[src_first]);
1455 #endif
1456         }
1457       }
1458       return 0;
1459     } else if (dst_first_rc == rc_float) {
1460       // xmm -> xmm
1461       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1462           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1463         // 64-bit
1464         if (cbuf) {
1465           MacroAssembler _masm(cbuf);
1466           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1467 #ifndef PRODUCT
1468         } else {
1469           st->print("%s  %s, %s\t# spill",
1470                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1471                      Matcher::regName[dst_first],
1472                      Matcher::regName[src_first]);
1473 #endif
1474         }
1475       } else {
1476         // 32-bit
1477         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1478         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1479         if (cbuf) {
1480           MacroAssembler _masm(cbuf);
1481           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1482 #ifndef PRODUCT
1483         } else {
1484           st->print("%s  %s, %s\t# spill",
1485                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1486                      Matcher::regName[dst_first],
1487                      Matcher::regName[src_first]);
1488 #endif
1489         }
1490       }
1491       return 0;
1492     }
1493   }
1494 
1495   assert(0," foo ");
1496   Unimplemented();
1497   return 0;
1498 }
1499 
1500 #ifndef PRODUCT
1501 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1502   implementation(NULL, ra_, false, st);
1503 }
1504 #endif
1505 
1506 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1507   implementation(&cbuf, ra_, false, NULL);
1508 }
1509 
1510 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1511   return MachNode::size(ra_);
1512 }
1513 
1514 //=============================================================================
1515 #ifndef PRODUCT
1516 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1517 {
1518   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1519   int reg = ra_->get_reg_first(this);
1520   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1521             Matcher::regName[reg], offset);
1522 }
1523 #endif
1524 
1525 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1526 {
1527   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1528   int reg = ra_->get_encode(this);
1529   if (offset >= 0x80) {
1530     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1531     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1532     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1533     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1534     emit_d32(cbuf, offset);
1535   } else {
1536     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1537     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1538     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1539     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1540     emit_d8(cbuf, offset);
1541   }
1542 }
1543 
1544 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1545 {
1546   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1547   return (offset < 0x80) ? 5 : 8; // REX
1548 }
1549 
1550 //=============================================================================
1551 #ifndef PRODUCT
1552 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1553 {
1554   if (UseCompressedClassPointers) {
1555     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1556     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1557     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1558   } else {
1559     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1560                  "# Inline cache check");
1561   }
1562   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1563   st->print_cr("\tnop\t# nops to align entry point");
1564 }
1565 #endif
1566 
1567 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1568 {
1569   MacroAssembler masm(&cbuf);
1570   uint insts_size = cbuf.insts_size();
1571   if (UseCompressedClassPointers) {
1572     masm.load_klass(rscratch1, j_rarg0);
1573     masm.cmpptr(rax, rscratch1);
1574   } else {
1575     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1576   }
1577 
1578   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1579 
1580   /* WARNING these NOPs are critical so that verified entry point is properly
1581      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1582   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1583   if (OptoBreakpoint) {
1584     // Leave space for int3
1585     nops_cnt -= 1;
1586   }
1587   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1588   if (nops_cnt > 0)
1589     masm.nop(nops_cnt);
1590 }
1591 
1592 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1593 {
1594   return MachNode::size(ra_); // too many variables; just compute it
1595                               // the hard way
1596 }
1597 
1598 
1599 //=============================================================================
1600 
1601 int Matcher::regnum_to_fpu_offset(int regnum)
1602 {
1603   return regnum - 32; // The FP registers are in the second chunk
1604 }
1605 
1606 // This is UltraSparc specific, true just means we have fast l2f conversion
1607 const bool Matcher::convL2FSupported(void) {
1608   return true;
1609 }
1610 
1611 // Is this branch offset short enough that a short branch can be used?
1612 //
1613 // NOTE: If the platform does not provide any short branch variants, then
1614 //       this method should return false for offset 0.
1615 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1616   // The passed offset is relative to address of the branch.
1617   // On 86 a branch displacement is calculated relative to address
1618   // of a next instruction.
1619   offset -= br_size;
1620 
1621   // the short version of jmpConUCF2 contains multiple branches,
1622   // making the reach slightly less
1623   if (rule == jmpConUCF2_rule)
1624     return (-126 <= offset && offset <= 125);
1625   return (-128 <= offset && offset <= 127);
1626 }
1627 
1628 const bool Matcher::isSimpleConstant64(jlong value) {
1629   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1630   //return value == (int) value;  // Cf. storeImmL and immL32.
1631 
1632   // Probably always true, even if a temp register is required.
1633   return true;
1634 }
1635 
1636 // The ecx parameter to rep stosq for the ClearArray node is in words.
1637 const bool Matcher::init_array_count_is_in_bytes = false;
1638 
1639 // Threshold size for cleararray.
1640 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1641 
1642 // No additional cost for CMOVL.
1643 const int Matcher::long_cmove_cost() { return 0; }
1644 
1645 // No CMOVF/CMOVD with SSE2
1646 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1647 
1648 // Does the CPU require late expand (see block.cpp for description of late expand)?
1649 const bool Matcher::require_postalloc_expand = false;
1650 
1651 // Should the Matcher clone shifts on addressing modes, expecting them
1652 // to be subsumed into complex addressing expressions or compute them
1653 // into registers?  True for Intel but false for most RISCs
1654 const bool Matcher::clone_shift_expressions = true;
1655 
1656 // Do we need to mask the count passed to shift instructions or does
1657 // the cpu only look at the lower 5/6 bits anyway?
1658 const bool Matcher::need_masked_shift_count = false;
1659 
1660 bool Matcher::narrow_oop_use_complex_address() {
1661   assert(UseCompressedOops, "only for compressed oops code");
1662   return (LogMinObjAlignmentInBytes <= 3);
1663 }
1664 
1665 bool Matcher::narrow_klass_use_complex_address() {
1666   assert(UseCompressedClassPointers, "only for compressed klass code");
1667   return (LogKlassAlignmentInBytes <= 3);
1668 }
1669 
1670 // Is it better to copy float constants, or load them directly from
1671 // memory?  Intel can load a float constant from a direct address,
1672 // requiring no extra registers.  Most RISCs will have to materialize
1673 // an address into a register first, so they would do better to copy
1674 // the constant from stack.
1675 const bool Matcher::rematerialize_float_constants = true; // XXX
1676 
1677 // If CPU can load and store mis-aligned doubles directly then no
1678 // fixup is needed.  Else we split the double into 2 integer pieces
1679 // and move it piece-by-piece.  Only happens when passing doubles into
1680 // C code as the Java calling convention forces doubles to be aligned.
1681 const bool Matcher::misaligned_doubles_ok = true;
1682 
1683 // No-op on amd64
1684 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1685 
1686 // Advertise here if the CPU requires explicit rounding operations to
1687 // implement the UseStrictFP mode.
1688 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1689 
1690 // Are floats conerted to double when stored to stack during deoptimization?
1691 // On x64 it is stored without convertion so we can use normal access.
1692 bool Matcher::float_in_double() { return false; }
1693 
1694 // Do ints take an entire long register or just half?
1695 const bool Matcher::int_in_long = true;
1696 
1697 // Return whether or not this register is ever used as an argument.
1698 // This function is used on startup to build the trampoline stubs in
1699 // generateOptoStub.  Registers not mentioned will be killed by the VM
1700 // call in the trampoline, and arguments in those registers not be
1701 // available to the callee.
1702 bool Matcher::can_be_java_arg(int reg)
1703 {
1704   return
1705     reg ==  RDI_num || reg == RDI_H_num ||
1706     reg ==  RSI_num || reg == RSI_H_num ||
1707     reg ==  RDX_num || reg == RDX_H_num ||
1708     reg ==  RCX_num || reg == RCX_H_num ||
1709     reg ==   R8_num || reg ==  R8_H_num ||
1710     reg ==   R9_num || reg ==  R9_H_num ||
1711     reg ==  R12_num || reg == R12_H_num ||
1712     reg == XMM0_num || reg == XMM0b_num ||
1713     reg == XMM1_num || reg == XMM1b_num ||
1714     reg == XMM2_num || reg == XMM2b_num ||
1715     reg == XMM3_num || reg == XMM3b_num ||
1716     reg == XMM4_num || reg == XMM4b_num ||
1717     reg == XMM5_num || reg == XMM5b_num ||
1718     reg == XMM6_num || reg == XMM6b_num ||
1719     reg == XMM7_num || reg == XMM7b_num;
1720 }
1721 
1722 bool Matcher::is_spillable_arg(int reg)
1723 {
1724   return can_be_java_arg(reg);
1725 }
1726 
1727 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1728   // In 64 bit mode a code which use multiply when
1729   // devisor is constant is faster than hardware
1730   // DIV instruction (it uses MulHiL).
1731   return false;
1732 }
1733 
1734 // Register for DIVI projection of divmodI
1735 RegMask Matcher::divI_proj_mask() {
1736   return INT_RAX_REG_mask();
1737 }
1738 
1739 // Register for MODI projection of divmodI
1740 RegMask Matcher::modI_proj_mask() {
1741   return INT_RDX_REG_mask();
1742 }
1743 
1744 // Register for DIVL projection of divmodL
1745 RegMask Matcher::divL_proj_mask() {
1746   return LONG_RAX_REG_mask();
1747 }
1748 
1749 // Register for MODL projection of divmodL
1750 RegMask Matcher::modL_proj_mask() {
1751   return LONG_RDX_REG_mask();
1752 }
1753 
1754 // Register for saving SP into on method handle invokes. Not used on x86_64.
1755 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1756     return NO_REG_mask();
1757 }
1758 
1759 %}
1760 
1761 //----------ENCODING BLOCK-----------------------------------------------------
1762 // This block specifies the encoding classes used by the compiler to
1763 // output byte streams.  Encoding classes are parameterized macros
1764 // used by Machine Instruction Nodes in order to generate the bit
1765 // encoding of the instruction.  Operands specify their base encoding
1766 // interface with the interface keyword.  There are currently
1767 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1768 // COND_INTER.  REG_INTER causes an operand to generate a function
1769 // which returns its register number when queried.  CONST_INTER causes
1770 // an operand to generate a function which returns the value of the
1771 // constant when queried.  MEMORY_INTER causes an operand to generate
1772 // four functions which return the Base Register, the Index Register,
1773 // the Scale Value, and the Offset Value of the operand when queried.
1774 // COND_INTER causes an operand to generate six functions which return
1775 // the encoding code (ie - encoding bits for the instruction)
1776 // associated with each basic boolean condition for a conditional
1777 // instruction.
1778 //
1779 // Instructions specify two basic values for encoding.  Again, a
1780 // function is available to check if the constant displacement is an
1781 // oop. They use the ins_encode keyword to specify their encoding
1782 // classes (which must be a sequence of enc_class names, and their
1783 // parameters, specified in the encoding block), and they use the
1784 // opcode keyword to specify, in order, their primary, secondary, and
1785 // tertiary opcode.  Only the opcode sections which a particular
1786 // instruction needs for encoding need to be specified.
1787 encode %{
1788   // Build emit functions for each basic byte or larger field in the
1789   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1790   // from C++ code in the enc_class source block.  Emit functions will
1791   // live in the main source block for now.  In future, we can
1792   // generalize this by adding a syntax that specifies the sizes of
1793   // fields in an order, so that the adlc can build the emit functions
1794   // automagically
1795 
1796   // Emit primary opcode
1797   enc_class OpcP
1798   %{
1799     emit_opcode(cbuf, $primary);
1800   %}
1801 
1802   // Emit secondary opcode
1803   enc_class OpcS
1804   %{
1805     emit_opcode(cbuf, $secondary);
1806   %}
1807 
1808   // Emit tertiary opcode
1809   enc_class OpcT
1810   %{
1811     emit_opcode(cbuf, $tertiary);
1812   %}
1813 
1814   // Emit opcode directly
1815   enc_class Opcode(immI d8)
1816   %{
1817     emit_opcode(cbuf, $d8$$constant);
1818   %}
1819 
1820   // Emit size prefix
1821   enc_class SizePrefix
1822   %{
1823     emit_opcode(cbuf, 0x66);
1824   %}
1825 
1826   enc_class reg(rRegI reg)
1827   %{
1828     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1829   %}
1830 
1831   enc_class reg_reg(rRegI dst, rRegI src)
1832   %{
1833     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1834   %}
1835 
1836   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1837   %{
1838     emit_opcode(cbuf, $opcode$$constant);
1839     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1840   %}
1841 
1842   enc_class cdql_enc(no_rax_rdx_RegI div)
1843   %{
1844     // Full implementation of Java idiv and irem; checks for
1845     // special case as described in JVM spec., p.243 & p.271.
1846     //
1847     //         normal case                           special case
1848     //
1849     // input : rax: dividend                         min_int
1850     //         reg: divisor                          -1
1851     //
1852     // output: rax: quotient  (= rax idiv reg)       min_int
1853     //         rdx: remainder (= rax irem reg)       0
1854     //
1855     //  Code sequnce:
1856     //
1857     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1858     //    5:   75 07/08                jne    e <normal>
1859     //    7:   33 d2                   xor    %edx,%edx
1860     //  [div >= 8 -> offset + 1]
1861     //  [REX_B]
1862     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1863     //    c:   74 03/04                je     11 <done>
1864     // 000000000000000e <normal>:
1865     //    e:   99                      cltd
1866     //  [div >= 8 -> offset + 1]
1867     //  [REX_B]
1868     //    f:   f7 f9                   idiv   $div
1869     // 0000000000000011 <done>:
1870 
1871     // cmp    $0x80000000,%eax
1872     emit_opcode(cbuf, 0x3d);
1873     emit_d8(cbuf, 0x00);
1874     emit_d8(cbuf, 0x00);
1875     emit_d8(cbuf, 0x00);
1876     emit_d8(cbuf, 0x80);
1877 
1878     // jne    e <normal>
1879     emit_opcode(cbuf, 0x75);
1880     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1881 
1882     // xor    %edx,%edx
1883     emit_opcode(cbuf, 0x33);
1884     emit_d8(cbuf, 0xD2);
1885 
1886     // cmp    $0xffffffffffffffff,%ecx
1887     if ($div$$reg >= 8) {
1888       emit_opcode(cbuf, Assembler::REX_B);
1889     }
1890     emit_opcode(cbuf, 0x83);
1891     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1892     emit_d8(cbuf, 0xFF);
1893 
1894     // je     11 <done>
1895     emit_opcode(cbuf, 0x74);
1896     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1897 
1898     // <normal>
1899     // cltd
1900     emit_opcode(cbuf, 0x99);
1901 
1902     // idivl (note: must be emitted by the user of this rule)
1903     // <done>
1904   %}
1905 
1906   enc_class cdqq_enc(no_rax_rdx_RegL div)
1907   %{
1908     // Full implementation of Java ldiv and lrem; checks for
1909     // special case as described in JVM spec., p.243 & p.271.
1910     //
1911     //         normal case                           special case
1912     //
1913     // input : rax: dividend                         min_long
1914     //         reg: divisor                          -1
1915     //
1916     // output: rax: quotient  (= rax idiv reg)       min_long
1917     //         rdx: remainder (= rax irem reg)       0
1918     //
1919     //  Code sequnce:
1920     //
1921     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1922     //    7:   00 00 80
1923     //    a:   48 39 d0                cmp    %rdx,%rax
1924     //    d:   75 08                   jne    17 <normal>
1925     //    f:   33 d2                   xor    %edx,%edx
1926     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1927     //   15:   74 05                   je     1c <done>
1928     // 0000000000000017 <normal>:
1929     //   17:   48 99                   cqto
1930     //   19:   48 f7 f9                idiv   $div
1931     // 000000000000001c <done>:
1932 
1933     // mov    $0x8000000000000000,%rdx
1934     emit_opcode(cbuf, Assembler::REX_W);
1935     emit_opcode(cbuf, 0xBA);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x00);
1939     emit_d8(cbuf, 0x00);
1940     emit_d8(cbuf, 0x00);
1941     emit_d8(cbuf, 0x00);
1942     emit_d8(cbuf, 0x00);
1943     emit_d8(cbuf, 0x80);
1944 
1945     // cmp    %rdx,%rax
1946     emit_opcode(cbuf, Assembler::REX_W);
1947     emit_opcode(cbuf, 0x39);
1948     emit_d8(cbuf, 0xD0);
1949 
1950     // jne    17 <normal>
1951     emit_opcode(cbuf, 0x75);
1952     emit_d8(cbuf, 0x08);
1953 
1954     // xor    %edx,%edx
1955     emit_opcode(cbuf, 0x33);
1956     emit_d8(cbuf, 0xD2);
1957 
1958     // cmp    $0xffffffffffffffff,$div
1959     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1960     emit_opcode(cbuf, 0x83);
1961     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1962     emit_d8(cbuf, 0xFF);
1963 
1964     // je     1e <done>
1965     emit_opcode(cbuf, 0x74);
1966     emit_d8(cbuf, 0x05);
1967 
1968     // <normal>
1969     // cqto
1970     emit_opcode(cbuf, Assembler::REX_W);
1971     emit_opcode(cbuf, 0x99);
1972 
1973     // idivq (note: must be emitted by the user of this rule)
1974     // <done>
1975   %}
1976 
1977   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1978   enc_class OpcSE(immI imm)
1979   %{
1980     // Emit primary opcode and set sign-extend bit
1981     // Check for 8-bit immediate, and set sign extend bit in opcode
1982     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1983       emit_opcode(cbuf, $primary | 0x02);
1984     } else {
1985       // 32-bit immediate
1986       emit_opcode(cbuf, $primary);
1987     }
1988   %}
1989 
1990   enc_class OpcSErm(rRegI dst, immI imm)
1991   %{
1992     // OpcSEr/m
1993     int dstenc = $dst$$reg;
1994     if (dstenc >= 8) {
1995       emit_opcode(cbuf, Assembler::REX_B);
1996       dstenc -= 8;
1997     }
1998     // Emit primary opcode and set sign-extend bit
1999     // Check for 8-bit immediate, and set sign extend bit in opcode
2000     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2001       emit_opcode(cbuf, $primary | 0x02);
2002     } else {
2003       // 32-bit immediate
2004       emit_opcode(cbuf, $primary);
2005     }
2006     // Emit r/m byte with secondary opcode, after primary opcode.
2007     emit_rm(cbuf, 0x3, $secondary, dstenc);
2008   %}
2009 
2010   enc_class OpcSErm_wide(rRegL dst, immI imm)
2011   %{
2012     // OpcSEr/m
2013     int dstenc = $dst$$reg;
2014     if (dstenc < 8) {
2015       emit_opcode(cbuf, Assembler::REX_W);
2016     } else {
2017       emit_opcode(cbuf, Assembler::REX_WB);
2018       dstenc -= 8;
2019     }
2020     // Emit primary opcode and set sign-extend bit
2021     // Check for 8-bit immediate, and set sign extend bit in opcode
2022     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2023       emit_opcode(cbuf, $primary | 0x02);
2024     } else {
2025       // 32-bit immediate
2026       emit_opcode(cbuf, $primary);
2027     }
2028     // Emit r/m byte with secondary opcode, after primary opcode.
2029     emit_rm(cbuf, 0x3, $secondary, dstenc);
2030   %}
2031 
2032   enc_class Con8or32(immI imm)
2033   %{
2034     // Check for 8-bit immediate, and set sign extend bit in opcode
2035     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2036       $$$emit8$imm$$constant;
2037     } else {
2038       // 32-bit immediate
2039       $$$emit32$imm$$constant;
2040     }
2041   %}
2042 
2043   enc_class opc2_reg(rRegI dst)
2044   %{
2045     // BSWAP
2046     emit_cc(cbuf, $secondary, $dst$$reg);
2047   %}
2048 
2049   enc_class opc3_reg(rRegI dst)
2050   %{
2051     // BSWAP
2052     emit_cc(cbuf, $tertiary, $dst$$reg);
2053   %}
2054 
2055   enc_class reg_opc(rRegI div)
2056   %{
2057     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2058     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2059   %}
2060 
2061   enc_class enc_cmov(cmpOp cop)
2062   %{
2063     // CMOV
2064     $$$emit8$primary;
2065     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2066   %}
2067 
2068   enc_class enc_PartialSubtypeCheck()
2069   %{
2070     Register Rrdi = as_Register(RDI_enc); // result register
2071     Register Rrax = as_Register(RAX_enc); // super class
2072     Register Rrcx = as_Register(RCX_enc); // killed
2073     Register Rrsi = as_Register(RSI_enc); // sub class
2074     Label miss;
2075     const bool set_cond_codes = true;
2076 
2077     MacroAssembler _masm(&cbuf);
2078     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2079                                      NULL, &miss,
2080                                      /*set_cond_codes:*/ true);
2081     if ($primary) {
2082       __ xorptr(Rrdi, Rrdi);
2083     }
2084     __ bind(miss);
2085   %}
2086 
2087   enc_class clear_avx %{
2088     debug_only(int off0 = cbuf.insts_size());
2089     if (ra_->C->max_vector_size() > 16) {
2090       // Clear upper bits of YMM registers when current compiled code uses
2091       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2092       MacroAssembler _masm(&cbuf);
2093       __ vzeroupper();
2094     }
2095     debug_only(int off1 = cbuf.insts_size());
2096     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2097   %}
2098 
2099   enc_class Java_To_Runtime(method meth) %{
2100     // No relocation needed
2101     MacroAssembler _masm(&cbuf);
2102     __ mov64(r10, (int64_t) $meth$$method);
2103     __ call(r10);
2104   %}
2105 
2106   enc_class Java_To_Interpreter(method meth)
2107   %{
2108     // CALL Java_To_Interpreter
2109     // This is the instruction starting address for relocation info.
2110     cbuf.set_insts_mark();
2111     $$$emit8$primary;
2112     // CALL directly to the runtime
2113     emit_d32_reloc(cbuf,
2114                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2115                    runtime_call_Relocation::spec(),
2116                    RELOC_DISP32);
2117   %}
2118 
2119   enc_class Java_Static_Call(method meth)
2120   %{
2121     // JAVA STATIC CALL
2122     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2123     // determine who we intended to call.
2124     cbuf.set_insts_mark();
2125     $$$emit8$primary;
2126 
2127     if (!_method) {
2128       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2129                      runtime_call_Relocation::spec(),
2130                      RELOC_DISP32);
2131     } else {
2132       int method_index = resolved_method_index(cbuf);
2133       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2134                                                   : static_call_Relocation::spec(method_index);
2135       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2136                      rspec, RELOC_DISP32);
2137       // Emit stubs for static call.
2138       address mark = cbuf.insts_mark();
2139       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2140       if (stub == NULL) {
2141         ciEnv::current()->record_failure("CodeCache is full");
2142         return;
2143       }
2144     }
2145   %}
2146 
2147   enc_class Java_Dynamic_Call(method meth) %{
2148     MacroAssembler _masm(&cbuf);
2149     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2150   %}
2151 
2152   enc_class Java_Compiled_Call(method meth)
2153   %{
2154     // JAVA COMPILED CALL
2155     int disp = in_bytes(Method:: from_compiled_offset());
2156 
2157     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2158     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2159 
2160     // callq *disp(%rax)
2161     cbuf.set_insts_mark();
2162     $$$emit8$primary;
2163     if (disp < 0x80) {
2164       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2165       emit_d8(cbuf, disp); // Displacement
2166     } else {
2167       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2168       emit_d32(cbuf, disp); // Displacement
2169     }
2170   %}
2171 
2172   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2173   %{
2174     // SAL, SAR, SHR
2175     int dstenc = $dst$$reg;
2176     if (dstenc >= 8) {
2177       emit_opcode(cbuf, Assembler::REX_B);
2178       dstenc -= 8;
2179     }
2180     $$$emit8$primary;
2181     emit_rm(cbuf, 0x3, $secondary, dstenc);
2182     $$$emit8$shift$$constant;
2183   %}
2184 
2185   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2186   %{
2187     // SAL, SAR, SHR
2188     int dstenc = $dst$$reg;
2189     if (dstenc < 8) {
2190       emit_opcode(cbuf, Assembler::REX_W);
2191     } else {
2192       emit_opcode(cbuf, Assembler::REX_WB);
2193       dstenc -= 8;
2194     }
2195     $$$emit8$primary;
2196     emit_rm(cbuf, 0x3, $secondary, dstenc);
2197     $$$emit8$shift$$constant;
2198   %}
2199 
2200   enc_class load_immI(rRegI dst, immI src)
2201   %{
2202     int dstenc = $dst$$reg;
2203     if (dstenc >= 8) {
2204       emit_opcode(cbuf, Assembler::REX_B);
2205       dstenc -= 8;
2206     }
2207     emit_opcode(cbuf, 0xB8 | dstenc);
2208     $$$emit32$src$$constant;
2209   %}
2210 
2211   enc_class load_immL(rRegL dst, immL src)
2212   %{
2213     int dstenc = $dst$$reg;
2214     if (dstenc < 8) {
2215       emit_opcode(cbuf, Assembler::REX_W);
2216     } else {
2217       emit_opcode(cbuf, Assembler::REX_WB);
2218       dstenc -= 8;
2219     }
2220     emit_opcode(cbuf, 0xB8 | dstenc);
2221     emit_d64(cbuf, $src$$constant);
2222   %}
2223 
2224   enc_class load_immUL32(rRegL dst, immUL32 src)
2225   %{
2226     // same as load_immI, but this time we care about zeroes in the high word
2227     int dstenc = $dst$$reg;
2228     if (dstenc >= 8) {
2229       emit_opcode(cbuf, Assembler::REX_B);
2230       dstenc -= 8;
2231     }
2232     emit_opcode(cbuf, 0xB8 | dstenc);
2233     $$$emit32$src$$constant;
2234   %}
2235 
2236   enc_class load_immL32(rRegL dst, immL32 src)
2237   %{
2238     int dstenc = $dst$$reg;
2239     if (dstenc < 8) {
2240       emit_opcode(cbuf, Assembler::REX_W);
2241     } else {
2242       emit_opcode(cbuf, Assembler::REX_WB);
2243       dstenc -= 8;
2244     }
2245     emit_opcode(cbuf, 0xC7);
2246     emit_rm(cbuf, 0x03, 0x00, dstenc);
2247     $$$emit32$src$$constant;
2248   %}
2249 
2250   enc_class load_immP31(rRegP dst, immP32 src)
2251   %{
2252     // same as load_immI, but this time we care about zeroes in the high word
2253     int dstenc = $dst$$reg;
2254     if (dstenc >= 8) {
2255       emit_opcode(cbuf, Assembler::REX_B);
2256       dstenc -= 8;
2257     }
2258     emit_opcode(cbuf, 0xB8 | dstenc);
2259     $$$emit32$src$$constant;
2260   %}
2261 
2262   enc_class load_immP(rRegP dst, immP src)
2263   %{
2264     int dstenc = $dst$$reg;
2265     if (dstenc < 8) {
2266       emit_opcode(cbuf, Assembler::REX_W);
2267     } else {
2268       emit_opcode(cbuf, Assembler::REX_WB);
2269       dstenc -= 8;
2270     }
2271     emit_opcode(cbuf, 0xB8 | dstenc);
2272     // This next line should be generated from ADLC
2273     if ($src->constant_reloc() != relocInfo::none) {
2274       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2275     } else {
2276       emit_d64(cbuf, $src$$constant);
2277     }
2278   %}
2279 
2280   enc_class Con32(immI src)
2281   %{
2282     // Output immediate
2283     $$$emit32$src$$constant;
2284   %}
2285 
2286   enc_class Con32F_as_bits(immF src)
2287   %{
2288     // Output Float immediate bits
2289     jfloat jf = $src$$constant;
2290     jint jf_as_bits = jint_cast(jf);
2291     emit_d32(cbuf, jf_as_bits);
2292   %}
2293 
2294   enc_class Con16(immI src)
2295   %{
2296     // Output immediate
2297     $$$emit16$src$$constant;
2298   %}
2299 
2300   // How is this different from Con32??? XXX
2301   enc_class Con_d32(immI src)
2302   %{
2303     emit_d32(cbuf,$src$$constant);
2304   %}
2305 
2306   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2307     // Output immediate memory reference
2308     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2309     emit_d32(cbuf, 0x00);
2310   %}
2311 
2312   enc_class lock_prefix()
2313   %{
2314     if (os::is_MP()) {
2315       emit_opcode(cbuf, 0xF0); // lock
2316     }
2317   %}
2318 
2319   enc_class REX_mem(memory mem)
2320   %{
2321     if ($mem$$base >= 8) {
2322       if ($mem$$index < 8) {
2323         emit_opcode(cbuf, Assembler::REX_B);
2324       } else {
2325         emit_opcode(cbuf, Assembler::REX_XB);
2326       }
2327     } else {
2328       if ($mem$$index >= 8) {
2329         emit_opcode(cbuf, Assembler::REX_X);
2330       }
2331     }
2332   %}
2333 
2334   enc_class REX_mem_wide(memory mem)
2335   %{
2336     if ($mem$$base >= 8) {
2337       if ($mem$$index < 8) {
2338         emit_opcode(cbuf, Assembler::REX_WB);
2339       } else {
2340         emit_opcode(cbuf, Assembler::REX_WXB);
2341       }
2342     } else {
2343       if ($mem$$index < 8) {
2344         emit_opcode(cbuf, Assembler::REX_W);
2345       } else {
2346         emit_opcode(cbuf, Assembler::REX_WX);
2347       }
2348     }
2349   %}
2350 
2351   // for byte regs
2352   enc_class REX_breg(rRegI reg)
2353   %{
2354     if ($reg$$reg >= 4) {
2355       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2356     }
2357   %}
2358 
2359   // for byte regs
2360   enc_class REX_reg_breg(rRegI dst, rRegI src)
2361   %{
2362     if ($dst$$reg < 8) {
2363       if ($src$$reg >= 4) {
2364         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2365       }
2366     } else {
2367       if ($src$$reg < 8) {
2368         emit_opcode(cbuf, Assembler::REX_R);
2369       } else {
2370         emit_opcode(cbuf, Assembler::REX_RB);
2371       }
2372     }
2373   %}
2374 
2375   // for byte regs
2376   enc_class REX_breg_mem(rRegI reg, memory mem)
2377   %{
2378     if ($reg$$reg < 8) {
2379       if ($mem$$base < 8) {
2380         if ($mem$$index >= 8) {
2381           emit_opcode(cbuf, Assembler::REX_X);
2382         } else if ($reg$$reg >= 4) {
2383           emit_opcode(cbuf, Assembler::REX);
2384         }
2385       } else {
2386         if ($mem$$index < 8) {
2387           emit_opcode(cbuf, Assembler::REX_B);
2388         } else {
2389           emit_opcode(cbuf, Assembler::REX_XB);
2390         }
2391       }
2392     } else {
2393       if ($mem$$base < 8) {
2394         if ($mem$$index < 8) {
2395           emit_opcode(cbuf, Assembler::REX_R);
2396         } else {
2397           emit_opcode(cbuf, Assembler::REX_RX);
2398         }
2399       } else {
2400         if ($mem$$index < 8) {
2401           emit_opcode(cbuf, Assembler::REX_RB);
2402         } else {
2403           emit_opcode(cbuf, Assembler::REX_RXB);
2404         }
2405       }
2406     }
2407   %}
2408 
2409   enc_class REX_reg(rRegI reg)
2410   %{
2411     if ($reg$$reg >= 8) {
2412       emit_opcode(cbuf, Assembler::REX_B);
2413     }
2414   %}
2415 
2416   enc_class REX_reg_wide(rRegI reg)
2417   %{
2418     if ($reg$$reg < 8) {
2419       emit_opcode(cbuf, Assembler::REX_W);
2420     } else {
2421       emit_opcode(cbuf, Assembler::REX_WB);
2422     }
2423   %}
2424 
2425   enc_class REX_reg_reg(rRegI dst, rRegI src)
2426   %{
2427     if ($dst$$reg < 8) {
2428       if ($src$$reg >= 8) {
2429         emit_opcode(cbuf, Assembler::REX_B);
2430       }
2431     } else {
2432       if ($src$$reg < 8) {
2433         emit_opcode(cbuf, Assembler::REX_R);
2434       } else {
2435         emit_opcode(cbuf, Assembler::REX_RB);
2436       }
2437     }
2438   %}
2439 
2440   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2441   %{
2442     if ($dst$$reg < 8) {
2443       if ($src$$reg < 8) {
2444         emit_opcode(cbuf, Assembler::REX_W);
2445       } else {
2446         emit_opcode(cbuf, Assembler::REX_WB);
2447       }
2448     } else {
2449       if ($src$$reg < 8) {
2450         emit_opcode(cbuf, Assembler::REX_WR);
2451       } else {
2452         emit_opcode(cbuf, Assembler::REX_WRB);
2453       }
2454     }
2455   %}
2456 
2457   enc_class REX_reg_mem(rRegI reg, memory mem)
2458   %{
2459     if ($reg$$reg < 8) {
2460       if ($mem$$base < 8) {
2461         if ($mem$$index >= 8) {
2462           emit_opcode(cbuf, Assembler::REX_X);
2463         }
2464       } else {
2465         if ($mem$$index < 8) {
2466           emit_opcode(cbuf, Assembler::REX_B);
2467         } else {
2468           emit_opcode(cbuf, Assembler::REX_XB);
2469         }
2470       }
2471     } else {
2472       if ($mem$$base < 8) {
2473         if ($mem$$index < 8) {
2474           emit_opcode(cbuf, Assembler::REX_R);
2475         } else {
2476           emit_opcode(cbuf, Assembler::REX_RX);
2477         }
2478       } else {
2479         if ($mem$$index < 8) {
2480           emit_opcode(cbuf, Assembler::REX_RB);
2481         } else {
2482           emit_opcode(cbuf, Assembler::REX_RXB);
2483         }
2484       }
2485     }
2486   %}
2487 
2488   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2489   %{
2490     if ($reg$$reg < 8) {
2491       if ($mem$$base < 8) {
2492         if ($mem$$index < 8) {
2493           emit_opcode(cbuf, Assembler::REX_W);
2494         } else {
2495           emit_opcode(cbuf, Assembler::REX_WX);
2496         }
2497       } else {
2498         if ($mem$$index < 8) {
2499           emit_opcode(cbuf, Assembler::REX_WB);
2500         } else {
2501           emit_opcode(cbuf, Assembler::REX_WXB);
2502         }
2503       }
2504     } else {
2505       if ($mem$$base < 8) {
2506         if ($mem$$index < 8) {
2507           emit_opcode(cbuf, Assembler::REX_WR);
2508         } else {
2509           emit_opcode(cbuf, Assembler::REX_WRX);
2510         }
2511       } else {
2512         if ($mem$$index < 8) {
2513           emit_opcode(cbuf, Assembler::REX_WRB);
2514         } else {
2515           emit_opcode(cbuf, Assembler::REX_WRXB);
2516         }
2517       }
2518     }
2519   %}
2520 
2521   enc_class reg_mem(rRegI ereg, memory mem)
2522   %{
2523     // High registers handle in encode_RegMem
2524     int reg = $ereg$$reg;
2525     int base = $mem$$base;
2526     int index = $mem$$index;
2527     int scale = $mem$$scale;
2528     int disp = $mem$$disp;
2529     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2530 
2531     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2532   %}
2533 
2534   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2535   %{
2536     int rm_byte_opcode = $rm_opcode$$constant;
2537 
2538     // High registers handle in encode_RegMem
2539     int base = $mem$$base;
2540     int index = $mem$$index;
2541     int scale = $mem$$scale;
2542     int displace = $mem$$disp;
2543 
2544     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2545                                             // working with static
2546                                             // globals
2547     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2548                   disp_reloc);
2549   %}
2550 
2551   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2552   %{
2553     int reg_encoding = $dst$$reg;
2554     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2555     int index        = 0x04;            // 0x04 indicates no index
2556     int scale        = 0x00;            // 0x00 indicates no scale
2557     int displace     = $src1$$constant; // 0x00 indicates no displacement
2558     relocInfo::relocType disp_reloc = relocInfo::none;
2559     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2560                   disp_reloc);
2561   %}
2562 
2563   enc_class neg_reg(rRegI dst)
2564   %{
2565     int dstenc = $dst$$reg;
2566     if (dstenc >= 8) {
2567       emit_opcode(cbuf, Assembler::REX_B);
2568       dstenc -= 8;
2569     }
2570     // NEG $dst
2571     emit_opcode(cbuf, 0xF7);
2572     emit_rm(cbuf, 0x3, 0x03, dstenc);
2573   %}
2574 
2575   enc_class neg_reg_wide(rRegI dst)
2576   %{
2577     int dstenc = $dst$$reg;
2578     if (dstenc < 8) {
2579       emit_opcode(cbuf, Assembler::REX_W);
2580     } else {
2581       emit_opcode(cbuf, Assembler::REX_WB);
2582       dstenc -= 8;
2583     }
2584     // NEG $dst
2585     emit_opcode(cbuf, 0xF7);
2586     emit_rm(cbuf, 0x3, 0x03, dstenc);
2587   %}
2588 
2589   enc_class setLT_reg(rRegI dst)
2590   %{
2591     int dstenc = $dst$$reg;
2592     if (dstenc >= 8) {
2593       emit_opcode(cbuf, Assembler::REX_B);
2594       dstenc -= 8;
2595     } else if (dstenc >= 4) {
2596       emit_opcode(cbuf, Assembler::REX);
2597     }
2598     // SETLT $dst
2599     emit_opcode(cbuf, 0x0F);
2600     emit_opcode(cbuf, 0x9C);
2601     emit_rm(cbuf, 0x3, 0x0, dstenc);
2602   %}
2603 
2604   enc_class setNZ_reg(rRegI dst)
2605   %{
2606     int dstenc = $dst$$reg;
2607     if (dstenc >= 8) {
2608       emit_opcode(cbuf, Assembler::REX_B);
2609       dstenc -= 8;
2610     } else if (dstenc >= 4) {
2611       emit_opcode(cbuf, Assembler::REX);
2612     }
2613     // SETNZ $dst
2614     emit_opcode(cbuf, 0x0F);
2615     emit_opcode(cbuf, 0x95);
2616     emit_rm(cbuf, 0x3, 0x0, dstenc);
2617   %}
2618 
2619 
2620   // Compare the lonogs and set -1, 0, or 1 into dst
2621   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2622   %{
2623     int src1enc = $src1$$reg;
2624     int src2enc = $src2$$reg;
2625     int dstenc = $dst$$reg;
2626 
2627     // cmpq $src1, $src2
2628     if (src1enc < 8) {
2629       if (src2enc < 8) {
2630         emit_opcode(cbuf, Assembler::REX_W);
2631       } else {
2632         emit_opcode(cbuf, Assembler::REX_WB);
2633       }
2634     } else {
2635       if (src2enc < 8) {
2636         emit_opcode(cbuf, Assembler::REX_WR);
2637       } else {
2638         emit_opcode(cbuf, Assembler::REX_WRB);
2639       }
2640     }
2641     emit_opcode(cbuf, 0x3B);
2642     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2643 
2644     // movl $dst, -1
2645     if (dstenc >= 8) {
2646       emit_opcode(cbuf, Assembler::REX_B);
2647     }
2648     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2649     emit_d32(cbuf, -1);
2650 
2651     // jl,s done
2652     emit_opcode(cbuf, 0x7C);
2653     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2654 
2655     // setne $dst
2656     if (dstenc >= 4) {
2657       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2658     }
2659     emit_opcode(cbuf, 0x0F);
2660     emit_opcode(cbuf, 0x95);
2661     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2662 
2663     // movzbl $dst, $dst
2664     if (dstenc >= 4) {
2665       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2666     }
2667     emit_opcode(cbuf, 0x0F);
2668     emit_opcode(cbuf, 0xB6);
2669     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2670   %}
2671 
2672   enc_class Push_ResultXD(regD dst) %{
2673     MacroAssembler _masm(&cbuf);
2674     __ fstp_d(Address(rsp, 0));
2675     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2676     __ addptr(rsp, 8);
2677   %}
2678 
2679   enc_class Push_SrcXD(regD src) %{
2680     MacroAssembler _masm(&cbuf);
2681     __ subptr(rsp, 8);
2682     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2683     __ fld_d(Address(rsp, 0));
2684   %}
2685 
2686 
2687   enc_class enc_rethrow()
2688   %{
2689     cbuf.set_insts_mark();
2690     emit_opcode(cbuf, 0xE9); // jmp entry
2691     emit_d32_reloc(cbuf,
2692                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2693                    runtime_call_Relocation::spec(),
2694                    RELOC_DISP32);
2695   %}
2696 
2697 %}
2698 
2699 
2700 
2701 //----------FRAME--------------------------------------------------------------
2702 // Definition of frame structure and management information.
2703 //
2704 //  S T A C K   L A Y O U T    Allocators stack-slot number
2705 //                             |   (to get allocators register number
2706 //  G  Owned by    |        |  v    add OptoReg::stack0())
2707 //  r   CALLER     |        |
2708 //  o     |        +--------+      pad to even-align allocators stack-slot
2709 //  w     V        |  pad0  |        numbers; owned by CALLER
2710 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2711 //  h     ^        |   in   |  5
2712 //        |        |  args  |  4   Holes in incoming args owned by SELF
2713 //  |     |        |        |  3
2714 //  |     |        +--------+
2715 //  V     |        | old out|      Empty on Intel, window on Sparc
2716 //        |    old |preserve|      Must be even aligned.
2717 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2718 //        |        |   in   |  3   area for Intel ret address
2719 //     Owned by    |preserve|      Empty on Sparc.
2720 //       SELF      +--------+
2721 //        |        |  pad2  |  2   pad to align old SP
2722 //        |        +--------+  1
2723 //        |        | locks  |  0
2724 //        |        +--------+----> OptoReg::stack0(), even aligned
2725 //        |        |  pad1  | 11   pad to align new SP
2726 //        |        +--------+
2727 //        |        |        | 10
2728 //        |        | spills |  9   spills
2729 //        V        |        |  8   (pad0 slot for callee)
2730 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2731 //        ^        |  out   |  7
2732 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2733 //     Owned by    +--------+
2734 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2735 //        |    new |preserve|      Must be even-aligned.
2736 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2737 //        |        |        |
2738 //
2739 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2740 //         known from SELF's arguments and the Java calling convention.
2741 //         Region 6-7 is determined per call site.
2742 // Note 2: If the calling convention leaves holes in the incoming argument
2743 //         area, those holes are owned by SELF.  Holes in the outgoing area
2744 //         are owned by the CALLEE.  Holes should not be nessecary in the
2745 //         incoming area, as the Java calling convention is completely under
2746 //         the control of the AD file.  Doubles can be sorted and packed to
2747 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2748 //         varargs C calling conventions.
2749 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2750 //         even aligned with pad0 as needed.
2751 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2752 //         region 6-11 is even aligned; it may be padded out more so that
2753 //         the region from SP to FP meets the minimum stack alignment.
2754 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2755 //         alignment.  Region 11, pad1, may be dynamically extended so that
2756 //         SP meets the minimum alignment.
2757 
2758 frame
2759 %{
2760   // What direction does stack grow in (assumed to be same for C & Java)
2761   stack_direction(TOWARDS_LOW);
2762 
2763   // These three registers define part of the calling convention
2764   // between compiled code and the interpreter.
2765   inline_cache_reg(RAX);                // Inline Cache Register
2766   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2767                                         // calling interpreter
2768 
2769   // Optional: name the operand used by cisc-spilling to access
2770   // [stack_pointer + offset]
2771   cisc_spilling_operand_name(indOffset32);
2772 
2773   // Number of stack slots consumed by locking an object
2774   sync_stack_slots(2);
2775 
2776   // Compiled code's Frame Pointer
2777   frame_pointer(RSP);
2778 
2779   // Interpreter stores its frame pointer in a register which is
2780   // stored to the stack by I2CAdaptors.
2781   // I2CAdaptors convert from interpreted java to compiled java.
2782   interpreter_frame_pointer(RBP);
2783 
2784   // Stack alignment requirement
2785   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2786 
2787   // Number of stack slots between incoming argument block and the start of
2788   // a new frame.  The PROLOG must add this many slots to the stack.  The
2789   // EPILOG must remove this many slots.  amd64 needs two slots for
2790   // return address.
2791   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2792 
2793   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2794   // for calls to C.  Supports the var-args backing area for register parms.
2795   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2796 
2797   // The after-PROLOG location of the return address.  Location of
2798   // return address specifies a type (REG or STACK) and a number
2799   // representing the register number (i.e. - use a register name) or
2800   // stack slot.
2801   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2802   // Otherwise, it is above the locks and verification slot and alignment word
2803   return_addr(STACK - 2 +
2804               round_to((Compile::current()->in_preserve_stack_slots() +
2805                         Compile::current()->fixed_slots()),
2806                        stack_alignment_in_slots()));
2807 
2808   // Body of function which returns an integer array locating
2809   // arguments either in registers or in stack slots.  Passed an array
2810   // of ideal registers called "sig" and a "length" count.  Stack-slot
2811   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2812   // arguments for a CALLEE.  Incoming stack arguments are
2813   // automatically biased by the preserve_stack_slots field above.
2814 
2815   calling_convention
2816   %{
2817     // No difference between ingoing/outgoing just pass false
2818     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2819   %}
2820 
2821   c_calling_convention
2822   %{
2823     // This is obviously always outgoing
2824     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2825   %}
2826 
2827   // Location of compiled Java return values.  Same as C for now.
2828   return_value
2829   %{
2830     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2831            "only return normal values");
2832 
2833     static const int lo[Op_RegL + 1] = {
2834       0,
2835       0,
2836       RAX_num,  // Op_RegN
2837       RAX_num,  // Op_RegI
2838       RAX_num,  // Op_RegP
2839       XMM0_num, // Op_RegF
2840       XMM0_num, // Op_RegD
2841       RAX_num   // Op_RegL
2842     };
2843     static const int hi[Op_RegL + 1] = {
2844       0,
2845       0,
2846       OptoReg::Bad, // Op_RegN
2847       OptoReg::Bad, // Op_RegI
2848       RAX_H_num,    // Op_RegP
2849       OptoReg::Bad, // Op_RegF
2850       XMM0b_num,    // Op_RegD
2851       RAX_H_num     // Op_RegL
2852     };
2853     // Excluded flags and vector registers.
2854     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2855     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2856   %}
2857 %}
2858 
2859 //----------ATTRIBUTES---------------------------------------------------------
2860 //----------Operand Attributes-------------------------------------------------
2861 op_attrib op_cost(0);        // Required cost attribute
2862 
2863 //----------Instruction Attributes---------------------------------------------
2864 ins_attrib ins_cost(100);       // Required cost attribute
2865 ins_attrib ins_size(8);         // Required size attribute (in bits)
2866 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2867                                 // a non-matching short branch variant
2868                                 // of some long branch?
2869 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2870                                 // be a power of 2) specifies the
2871                                 // alignment that some part of the
2872                                 // instruction (not necessarily the
2873                                 // start) requires.  If > 1, a
2874                                 // compute_padding() function must be
2875                                 // provided for the instruction
2876 
2877 //----------OPERANDS-----------------------------------------------------------
2878 // Operand definitions must precede instruction definitions for correct parsing
2879 // in the ADLC because operands constitute user defined types which are used in
2880 // instruction definitions.
2881 
2882 //----------Simple Operands----------------------------------------------------
2883 // Immediate Operands
2884 // Integer Immediate
2885 operand immI()
2886 %{
2887   match(ConI);
2888 
2889   op_cost(10);
2890   format %{ %}
2891   interface(CONST_INTER);
2892 %}
2893 
2894 // Constant for test vs zero
2895 operand immI0()
2896 %{
2897   predicate(n->get_int() == 0);
2898   match(ConI);
2899 
2900   op_cost(0);
2901   format %{ %}
2902   interface(CONST_INTER);
2903 %}
2904 
2905 // Constant for increment
2906 operand immI1()
2907 %{
2908   predicate(n->get_int() == 1);
2909   match(ConI);
2910 
2911   op_cost(0);
2912   format %{ %}
2913   interface(CONST_INTER);
2914 %}
2915 
2916 // Constant for decrement
2917 operand immI_M1()
2918 %{
2919   predicate(n->get_int() == -1);
2920   match(ConI);
2921 
2922   op_cost(0);
2923   format %{ %}
2924   interface(CONST_INTER);
2925 %}
2926 
2927 // Valid scale values for addressing modes
2928 operand immI2()
2929 %{
2930   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2931   match(ConI);
2932 
2933   format %{ %}
2934   interface(CONST_INTER);
2935 %}
2936 
2937 operand immI8()
2938 %{
2939   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2940   match(ConI);
2941 
2942   op_cost(5);
2943   format %{ %}
2944   interface(CONST_INTER);
2945 %}
2946 
2947 operand immI16()
2948 %{
2949   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2950   match(ConI);
2951 
2952   op_cost(10);
2953   format %{ %}
2954   interface(CONST_INTER);
2955 %}
2956 
2957 // Int Immediate non-negative
2958 operand immU31()
2959 %{
2960   predicate(n->get_int() >= 0);
2961   match(ConI);
2962 
2963   op_cost(0);
2964   format %{ %}
2965   interface(CONST_INTER);
2966 %}
2967 
2968 // Constant for long shifts
2969 operand immI_32()
2970 %{
2971   predicate( n->get_int() == 32 );
2972   match(ConI);
2973 
2974   op_cost(0);
2975   format %{ %}
2976   interface(CONST_INTER);
2977 %}
2978 
2979 // Constant for long shifts
2980 operand immI_64()
2981 %{
2982   predicate( n->get_int() == 64 );
2983   match(ConI);
2984 
2985   op_cost(0);
2986   format %{ %}
2987   interface(CONST_INTER);
2988 %}
2989 
2990 // Pointer Immediate
2991 operand immP()
2992 %{
2993   match(ConP);
2994 
2995   op_cost(10);
2996   format %{ %}
2997   interface(CONST_INTER);
2998 %}
2999 
3000 // NULL Pointer Immediate
3001 operand immP0()
3002 %{
3003   predicate(n->get_ptr() == 0);
3004   match(ConP);
3005 
3006   op_cost(5);
3007   format %{ %}
3008   interface(CONST_INTER);
3009 %}
3010 
3011 // Pointer Immediate
3012 operand immN() %{
3013   match(ConN);
3014 
3015   op_cost(10);
3016   format %{ %}
3017   interface(CONST_INTER);
3018 %}
3019 
3020 operand immNKlass() %{
3021   match(ConNKlass);
3022 
3023   op_cost(10);
3024   format %{ %}
3025   interface(CONST_INTER);
3026 %}
3027 
3028 // NULL Pointer Immediate
3029 operand immN0() %{
3030   predicate(n->get_narrowcon() == 0);
3031   match(ConN);
3032 
3033   op_cost(5);
3034   format %{ %}
3035   interface(CONST_INTER);
3036 %}
3037 
3038 operand immP31()
3039 %{
3040   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3041             && (n->get_ptr() >> 31) == 0);
3042   match(ConP);
3043 
3044   op_cost(5);
3045   format %{ %}
3046   interface(CONST_INTER);
3047 %}
3048 
3049 
3050 // Long Immediate
3051 operand immL()
3052 %{
3053   match(ConL);
3054 
3055   op_cost(20);
3056   format %{ %}
3057   interface(CONST_INTER);
3058 %}
3059 
3060 // Long Immediate 8-bit
3061 operand immL8()
3062 %{
3063   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3064   match(ConL);
3065 
3066   op_cost(5);
3067   format %{ %}
3068   interface(CONST_INTER);
3069 %}
3070 
3071 // Long Immediate 32-bit unsigned
3072 operand immUL32()
3073 %{
3074   predicate(n->get_long() == (unsigned int) (n->get_long()));
3075   match(ConL);
3076 
3077   op_cost(10);
3078   format %{ %}
3079   interface(CONST_INTER);
3080 %}
3081 
3082 // Long Immediate 32-bit signed
3083 operand immL32()
3084 %{
3085   predicate(n->get_long() == (int) (n->get_long()));
3086   match(ConL);
3087 
3088   op_cost(15);
3089   format %{ %}
3090   interface(CONST_INTER);
3091 %}
3092 
3093 // Long Immediate zero
3094 operand immL0()
3095 %{
3096   predicate(n->get_long() == 0L);
3097   match(ConL);
3098 
3099   op_cost(10);
3100   format %{ %}
3101   interface(CONST_INTER);
3102 %}
3103 
3104 // Constant for increment
3105 operand immL1()
3106 %{
3107   predicate(n->get_long() == 1);
3108   match(ConL);
3109 
3110   format %{ %}
3111   interface(CONST_INTER);
3112 %}
3113 
3114 // Constant for decrement
3115 operand immL_M1()
3116 %{
3117   predicate(n->get_long() == -1);
3118   match(ConL);
3119 
3120   format %{ %}
3121   interface(CONST_INTER);
3122 %}
3123 
3124 // Long Immediate: the value 10
3125 operand immL10()
3126 %{
3127   predicate(n->get_long() == 10);
3128   match(ConL);
3129 
3130   format %{ %}
3131   interface(CONST_INTER);
3132 %}
3133 
3134 // Long immediate from 0 to 127.
3135 // Used for a shorter form of long mul by 10.
3136 operand immL_127()
3137 %{
3138   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3139   match(ConL);
3140 
3141   op_cost(10);
3142   format %{ %}
3143   interface(CONST_INTER);
3144 %}
3145 
3146 // Long Immediate: low 32-bit mask
3147 operand immL_32bits()
3148 %{
3149   predicate(n->get_long() == 0xFFFFFFFFL);
3150   match(ConL);
3151   op_cost(20);
3152 
3153   format %{ %}
3154   interface(CONST_INTER);
3155 %}
3156 
3157 // Float Immediate zero
3158 operand immF0()
3159 %{
3160   predicate(jint_cast(n->getf()) == 0);
3161   match(ConF);
3162 
3163   op_cost(5);
3164   format %{ %}
3165   interface(CONST_INTER);
3166 %}
3167 
3168 // Float Immediate
3169 operand immF()
3170 %{
3171   match(ConF);
3172 
3173   op_cost(15);
3174   format %{ %}
3175   interface(CONST_INTER);
3176 %}
3177 
3178 // Double Immediate zero
3179 operand immD0()
3180 %{
3181   predicate(jlong_cast(n->getd()) == 0);
3182   match(ConD);
3183 
3184   op_cost(5);
3185   format %{ %}
3186   interface(CONST_INTER);
3187 %}
3188 
3189 // Double Immediate
3190 operand immD()
3191 %{
3192   match(ConD);
3193 
3194   op_cost(15);
3195   format %{ %}
3196   interface(CONST_INTER);
3197 %}
3198 
3199 // Immediates for special shifts (sign extend)
3200 
3201 // Constants for increment
3202 operand immI_16()
3203 %{
3204   predicate(n->get_int() == 16);
3205   match(ConI);
3206 
3207   format %{ %}
3208   interface(CONST_INTER);
3209 %}
3210 
3211 operand immI_24()
3212 %{
3213   predicate(n->get_int() == 24);
3214   match(ConI);
3215 
3216   format %{ %}
3217   interface(CONST_INTER);
3218 %}
3219 
3220 // Constant for byte-wide masking
3221 operand immI_255()
3222 %{
3223   predicate(n->get_int() == 255);
3224   match(ConI);
3225 
3226   format %{ %}
3227   interface(CONST_INTER);
3228 %}
3229 
3230 // Constant for short-wide masking
3231 operand immI_65535()
3232 %{
3233   predicate(n->get_int() == 65535);
3234   match(ConI);
3235 
3236   format %{ %}
3237   interface(CONST_INTER);
3238 %}
3239 
3240 // Constant for byte-wide masking
3241 operand immL_255()
3242 %{
3243   predicate(n->get_long() == 255);
3244   match(ConL);
3245 
3246   format %{ %}
3247   interface(CONST_INTER);
3248 %}
3249 
3250 // Constant for short-wide masking
3251 operand immL_65535()
3252 %{
3253   predicate(n->get_long() == 65535);
3254   match(ConL);
3255 
3256   format %{ %}
3257   interface(CONST_INTER);
3258 %}
3259 
3260 // Register Operands
3261 // Integer Register
3262 operand rRegI()
3263 %{
3264   constraint(ALLOC_IN_RC(int_reg));
3265   match(RegI);
3266 
3267   match(rax_RegI);
3268   match(rbx_RegI);
3269   match(rcx_RegI);
3270   match(rdx_RegI);
3271   match(rdi_RegI);
3272 
3273   format %{ %}
3274   interface(REG_INTER);
3275 %}
3276 
3277 // Special Registers
3278 operand rax_RegI()
3279 %{
3280   constraint(ALLOC_IN_RC(int_rax_reg));
3281   match(RegI);
3282   match(rRegI);
3283 
3284   format %{ "RAX" %}
3285   interface(REG_INTER);
3286 %}
3287 
3288 // Special Registers
3289 operand rbx_RegI()
3290 %{
3291   constraint(ALLOC_IN_RC(int_rbx_reg));
3292   match(RegI);
3293   match(rRegI);
3294 
3295   format %{ "RBX" %}
3296   interface(REG_INTER);
3297 %}
3298 
3299 operand rcx_RegI()
3300 %{
3301   constraint(ALLOC_IN_RC(int_rcx_reg));
3302   match(RegI);
3303   match(rRegI);
3304 
3305   format %{ "RCX" %}
3306   interface(REG_INTER);
3307 %}
3308 
3309 operand rdx_RegI()
3310 %{
3311   constraint(ALLOC_IN_RC(int_rdx_reg));
3312   match(RegI);
3313   match(rRegI);
3314 
3315   format %{ "RDX" %}
3316   interface(REG_INTER);
3317 %}
3318 
3319 operand rdi_RegI()
3320 %{
3321   constraint(ALLOC_IN_RC(int_rdi_reg));
3322   match(RegI);
3323   match(rRegI);
3324 
3325   format %{ "RDI" %}
3326   interface(REG_INTER);
3327 %}
3328 
3329 operand no_rcx_RegI()
3330 %{
3331   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3332   match(RegI);
3333   match(rax_RegI);
3334   match(rbx_RegI);
3335   match(rdx_RegI);
3336   match(rdi_RegI);
3337 
3338   format %{ %}
3339   interface(REG_INTER);
3340 %}
3341 
3342 operand no_rax_rdx_RegI()
3343 %{
3344   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3345   match(RegI);
3346   match(rbx_RegI);
3347   match(rcx_RegI);
3348   match(rdi_RegI);
3349 
3350   format %{ %}
3351   interface(REG_INTER);
3352 %}
3353 
3354 // Pointer Register
3355 operand any_RegP()
3356 %{
3357   constraint(ALLOC_IN_RC(any_reg));
3358   match(RegP);
3359   match(rax_RegP);
3360   match(rbx_RegP);
3361   match(rdi_RegP);
3362   match(rsi_RegP);
3363   match(rbp_RegP);
3364   match(r15_RegP);
3365   match(rRegP);
3366 
3367   format %{ %}
3368   interface(REG_INTER);
3369 %}
3370 
3371 operand rRegP()
3372 %{
3373   constraint(ALLOC_IN_RC(ptr_reg));
3374   match(RegP);
3375   match(rax_RegP);
3376   match(rbx_RegP);
3377   match(rdi_RegP);
3378   match(rsi_RegP);
3379   match(rbp_RegP);  // See Q&A below about
3380   match(r15_RegP);  // r15_RegP and rbp_RegP.
3381 
3382   format %{ %}
3383   interface(REG_INTER);
3384 %}
3385 
3386 operand rRegN() %{
3387   constraint(ALLOC_IN_RC(int_reg));
3388   match(RegN);
3389 
3390   format %{ %}
3391   interface(REG_INTER);
3392 %}
3393 
3394 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3395 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3396 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3397 // The output of an instruction is controlled by the allocator, which respects
3398 // register class masks, not match rules.  Unless an instruction mentions
3399 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3400 // by the allocator as an input.
3401 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3402 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3403 // result, RBP is not included in the output of the instruction either.
3404 
3405 operand no_rax_RegP()
3406 %{
3407   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3408   match(RegP);
3409   match(rbx_RegP);
3410   match(rsi_RegP);
3411   match(rdi_RegP);
3412 
3413   format %{ %}
3414   interface(REG_INTER);
3415 %}
3416 
3417 // This operand is not allowed to use RBP even if
3418 // RBP is not used to hold the frame pointer.
3419 operand no_rbp_RegP()
3420 %{
3421   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3422   match(RegP);
3423   match(rbx_RegP);
3424   match(rsi_RegP);
3425   match(rdi_RegP);
3426 
3427   format %{ %}
3428   interface(REG_INTER);
3429 %}
3430 
3431 operand no_rax_rbx_RegP()
3432 %{
3433   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3434   match(RegP);
3435   match(rsi_RegP);
3436   match(rdi_RegP);
3437 
3438   format %{ %}
3439   interface(REG_INTER);
3440 %}
3441 
3442 // Special Registers
3443 // Return a pointer value
3444 operand rax_RegP()
3445 %{
3446   constraint(ALLOC_IN_RC(ptr_rax_reg));
3447   match(RegP);
3448   match(rRegP);
3449 
3450   format %{ %}
3451   interface(REG_INTER);
3452 %}
3453 
3454 // Special Registers
3455 // Return a compressed pointer value
3456 operand rax_RegN()
3457 %{
3458   constraint(ALLOC_IN_RC(int_rax_reg));
3459   match(RegN);
3460   match(rRegN);
3461 
3462   format %{ %}
3463   interface(REG_INTER);
3464 %}
3465 
3466 // Used in AtomicAdd
3467 operand rbx_RegP()
3468 %{
3469   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3470   match(RegP);
3471   match(rRegP);
3472 
3473   format %{ %}
3474   interface(REG_INTER);
3475 %}
3476 
3477 operand rsi_RegP()
3478 %{
3479   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3480   match(RegP);
3481   match(rRegP);
3482 
3483   format %{ %}
3484   interface(REG_INTER);
3485 %}
3486 
3487 // Used in rep stosq
3488 operand rdi_RegP()
3489 %{
3490   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3491   match(RegP);
3492   match(rRegP);
3493 
3494   format %{ %}
3495   interface(REG_INTER);
3496 %}
3497 
3498 operand r15_RegP()
3499 %{
3500   constraint(ALLOC_IN_RC(ptr_r15_reg));
3501   match(RegP);
3502   match(rRegP);
3503 
3504   format %{ %}
3505   interface(REG_INTER);
3506 %}
3507 
3508 operand rRegL()
3509 %{
3510   constraint(ALLOC_IN_RC(long_reg));
3511   match(RegL);
3512   match(rax_RegL);
3513   match(rdx_RegL);
3514 
3515   format %{ %}
3516   interface(REG_INTER);
3517 %}
3518 
3519 // Special Registers
3520 operand no_rax_rdx_RegL()
3521 %{
3522   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3523   match(RegL);
3524   match(rRegL);
3525 
3526   format %{ %}
3527   interface(REG_INTER);
3528 %}
3529 
3530 operand no_rax_RegL()
3531 %{
3532   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3533   match(RegL);
3534   match(rRegL);
3535   match(rdx_RegL);
3536 
3537   format %{ %}
3538   interface(REG_INTER);
3539 %}
3540 
3541 operand no_rcx_RegL()
3542 %{
3543   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3544   match(RegL);
3545   match(rRegL);
3546 
3547   format %{ %}
3548   interface(REG_INTER);
3549 %}
3550 
3551 operand rax_RegL()
3552 %{
3553   constraint(ALLOC_IN_RC(long_rax_reg));
3554   match(RegL);
3555   match(rRegL);
3556 
3557   format %{ "RAX" %}
3558   interface(REG_INTER);
3559 %}
3560 
3561 operand rcx_RegL()
3562 %{
3563   constraint(ALLOC_IN_RC(long_rcx_reg));
3564   match(RegL);
3565   match(rRegL);
3566 
3567   format %{ %}
3568   interface(REG_INTER);
3569 %}
3570 
3571 operand rdx_RegL()
3572 %{
3573   constraint(ALLOC_IN_RC(long_rdx_reg));
3574   match(RegL);
3575   match(rRegL);
3576 
3577   format %{ %}
3578   interface(REG_INTER);
3579 %}
3580 
3581 // Flags register, used as output of compare instructions
3582 operand rFlagsReg()
3583 %{
3584   constraint(ALLOC_IN_RC(int_flags));
3585   match(RegFlags);
3586 
3587   format %{ "RFLAGS" %}
3588   interface(REG_INTER);
3589 %}
3590 
3591 // Flags register, used as output of FLOATING POINT compare instructions
3592 operand rFlagsRegU()
3593 %{
3594   constraint(ALLOC_IN_RC(int_flags));
3595   match(RegFlags);
3596 
3597   format %{ "RFLAGS_U" %}
3598   interface(REG_INTER);
3599 %}
3600 
3601 operand rFlagsRegUCF() %{
3602   constraint(ALLOC_IN_RC(int_flags));
3603   match(RegFlags);
3604   predicate(false);
3605 
3606   format %{ "RFLAGS_U_CF" %}
3607   interface(REG_INTER);
3608 %}
3609 
3610 // Float register operands
3611 operand regF() %{
3612    constraint(ALLOC_IN_RC(float_reg));
3613    match(RegF);
3614 
3615    format %{ %}
3616    interface(REG_INTER);
3617 %}
3618 
3619 // Double register operands
3620 operand regD() %{
3621    constraint(ALLOC_IN_RC(double_reg));
3622    match(RegD);
3623 
3624    format %{ %}
3625    interface(REG_INTER);
3626 %}
3627 
3628 // Vectors
3629 operand vecS() %{
3630   constraint(ALLOC_IN_RC(vectors_reg));
3631   match(VecS);
3632 
3633   format %{ %}
3634   interface(REG_INTER);
3635 %}
3636 
3637 operand vecD() %{
3638   constraint(ALLOC_IN_RC(vectord_reg));
3639   match(VecD);
3640 
3641   format %{ %}
3642   interface(REG_INTER);
3643 %}
3644 
3645 operand vecX() %{
3646   constraint(ALLOC_IN_RC(vectorx_reg));
3647   match(VecX);
3648 
3649   format %{ %}
3650   interface(REG_INTER);
3651 %}
3652 
3653 operand vecY() %{
3654   constraint(ALLOC_IN_RC(vectory_reg));
3655   match(VecY);
3656 
3657   format %{ %}
3658   interface(REG_INTER);
3659 %}
3660 
3661 //----------Memory Operands----------------------------------------------------
3662 // Direct Memory Operand
3663 // operand direct(immP addr)
3664 // %{
3665 //   match(addr);
3666 
3667 //   format %{ "[$addr]" %}
3668 //   interface(MEMORY_INTER) %{
3669 //     base(0xFFFFFFFF);
3670 //     index(0x4);
3671 //     scale(0x0);
3672 //     disp($addr);
3673 //   %}
3674 // %}
3675 
3676 // Indirect Memory Operand
3677 operand indirect(any_RegP reg)
3678 %{
3679   constraint(ALLOC_IN_RC(ptr_reg));
3680   match(reg);
3681 
3682   format %{ "[$reg]" %}
3683   interface(MEMORY_INTER) %{
3684     base($reg);
3685     index(0x4);
3686     scale(0x0);
3687     disp(0x0);
3688   %}
3689 %}
3690 
3691 // Indirect Memory Plus Short Offset Operand
3692 operand indOffset8(any_RegP reg, immL8 off)
3693 %{
3694   constraint(ALLOC_IN_RC(ptr_reg));
3695   match(AddP reg off);
3696 
3697   format %{ "[$reg + $off (8-bit)]" %}
3698   interface(MEMORY_INTER) %{
3699     base($reg);
3700     index(0x4);
3701     scale(0x0);
3702     disp($off);
3703   %}
3704 %}
3705 
3706 // Indirect Memory Plus Long Offset Operand
3707 operand indOffset32(any_RegP reg, immL32 off)
3708 %{
3709   constraint(ALLOC_IN_RC(ptr_reg));
3710   match(AddP reg off);
3711 
3712   format %{ "[$reg + $off (32-bit)]" %}
3713   interface(MEMORY_INTER) %{
3714     base($reg);
3715     index(0x4);
3716     scale(0x0);
3717     disp($off);
3718   %}
3719 %}
3720 
3721 // Indirect Memory Plus Index Register Plus Offset Operand
3722 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3723 %{
3724   constraint(ALLOC_IN_RC(ptr_reg));
3725   match(AddP (AddP reg lreg) off);
3726 
3727   op_cost(10);
3728   format %{"[$reg + $off + $lreg]" %}
3729   interface(MEMORY_INTER) %{
3730     base($reg);
3731     index($lreg);
3732     scale(0x0);
3733     disp($off);
3734   %}
3735 %}
3736 
3737 // Indirect Memory Plus Index Register Plus Offset Operand
3738 operand indIndex(any_RegP reg, rRegL lreg)
3739 %{
3740   constraint(ALLOC_IN_RC(ptr_reg));
3741   match(AddP reg lreg);
3742 
3743   op_cost(10);
3744   format %{"[$reg + $lreg]" %}
3745   interface(MEMORY_INTER) %{
3746     base($reg);
3747     index($lreg);
3748     scale(0x0);
3749     disp(0x0);
3750   %}
3751 %}
3752 
3753 // Indirect Memory Times Scale Plus Index Register
3754 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3755 %{
3756   constraint(ALLOC_IN_RC(ptr_reg));
3757   match(AddP reg (LShiftL lreg scale));
3758 
3759   op_cost(10);
3760   format %{"[$reg + $lreg << $scale]" %}
3761   interface(MEMORY_INTER) %{
3762     base($reg);
3763     index($lreg);
3764     scale($scale);
3765     disp(0x0);
3766   %}
3767 %}
3768 
3769 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3770 %{
3771   constraint(ALLOC_IN_RC(ptr_reg));
3772   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3773   match(AddP reg (LShiftL (ConvI2L idx) scale));
3774 
3775   op_cost(10);
3776   format %{"[$reg + pos $idx << $scale]" %}
3777   interface(MEMORY_INTER) %{
3778     base($reg);
3779     index($idx);
3780     scale($scale);
3781     disp(0x0);
3782   %}
3783 %}
3784 
3785 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3786 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3787 %{
3788   constraint(ALLOC_IN_RC(ptr_reg));
3789   match(AddP (AddP reg (LShiftL lreg scale)) off);
3790 
3791   op_cost(10);
3792   format %{"[$reg + $off + $lreg << $scale]" %}
3793   interface(MEMORY_INTER) %{
3794     base($reg);
3795     index($lreg);
3796     scale($scale);
3797     disp($off);
3798   %}
3799 %}
3800 
3801 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3802 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3803 %{
3804   constraint(ALLOC_IN_RC(ptr_reg));
3805   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3806   match(AddP (AddP reg (ConvI2L idx)) off);
3807 
3808   op_cost(10);
3809   format %{"[$reg + $off + $idx]" %}
3810   interface(MEMORY_INTER) %{
3811     base($reg);
3812     index($idx);
3813     scale(0x0);
3814     disp($off);
3815   %}
3816 %}
3817 
3818 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3819 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3820 %{
3821   constraint(ALLOC_IN_RC(ptr_reg));
3822   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3823   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3824 
3825   op_cost(10);
3826   format %{"[$reg + $off + $idx << $scale]" %}
3827   interface(MEMORY_INTER) %{
3828     base($reg);
3829     index($idx);
3830     scale($scale);
3831     disp($off);
3832   %}
3833 %}
3834 
3835 // Indirect Narrow Oop Plus Offset Operand
3836 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3837 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3838 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3839   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3840   constraint(ALLOC_IN_RC(ptr_reg));
3841   match(AddP (DecodeN reg) off);
3842 
3843   op_cost(10);
3844   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3845   interface(MEMORY_INTER) %{
3846     base(0xc); // R12
3847     index($reg);
3848     scale(0x3);
3849     disp($off);
3850   %}
3851 %}
3852 
3853 // Indirect Memory Operand
3854 operand indirectNarrow(rRegN reg)
3855 %{
3856   predicate(Universe::narrow_oop_shift() == 0);
3857   constraint(ALLOC_IN_RC(ptr_reg));
3858   match(DecodeN reg);
3859 
3860   format %{ "[$reg]" %}
3861   interface(MEMORY_INTER) %{
3862     base($reg);
3863     index(0x4);
3864     scale(0x0);
3865     disp(0x0);
3866   %}
3867 %}
3868 
3869 // Indirect Memory Plus Short Offset Operand
3870 operand indOffset8Narrow(rRegN reg, immL8 off)
3871 %{
3872   predicate(Universe::narrow_oop_shift() == 0);
3873   constraint(ALLOC_IN_RC(ptr_reg));
3874   match(AddP (DecodeN reg) off);
3875 
3876   format %{ "[$reg + $off (8-bit)]" %}
3877   interface(MEMORY_INTER) %{
3878     base($reg);
3879     index(0x4);
3880     scale(0x0);
3881     disp($off);
3882   %}
3883 %}
3884 
3885 // Indirect Memory Plus Long Offset Operand
3886 operand indOffset32Narrow(rRegN reg, immL32 off)
3887 %{
3888   predicate(Universe::narrow_oop_shift() == 0);
3889   constraint(ALLOC_IN_RC(ptr_reg));
3890   match(AddP (DecodeN reg) off);
3891 
3892   format %{ "[$reg + $off (32-bit)]" %}
3893   interface(MEMORY_INTER) %{
3894     base($reg);
3895     index(0x4);
3896     scale(0x0);
3897     disp($off);
3898   %}
3899 %}
3900 
3901 // Indirect Memory Plus Index Register Plus Offset Operand
3902 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3903 %{
3904   predicate(Universe::narrow_oop_shift() == 0);
3905   constraint(ALLOC_IN_RC(ptr_reg));
3906   match(AddP (AddP (DecodeN reg) lreg) off);
3907 
3908   op_cost(10);
3909   format %{"[$reg + $off + $lreg]" %}
3910   interface(MEMORY_INTER) %{
3911     base($reg);
3912     index($lreg);
3913     scale(0x0);
3914     disp($off);
3915   %}
3916 %}
3917 
3918 // Indirect Memory Plus Index Register Plus Offset Operand
3919 operand indIndexNarrow(rRegN reg, rRegL lreg)
3920 %{
3921   predicate(Universe::narrow_oop_shift() == 0);
3922   constraint(ALLOC_IN_RC(ptr_reg));
3923   match(AddP (DecodeN reg) lreg);
3924 
3925   op_cost(10);
3926   format %{"[$reg + $lreg]" %}
3927   interface(MEMORY_INTER) %{
3928     base($reg);
3929     index($lreg);
3930     scale(0x0);
3931     disp(0x0);
3932   %}
3933 %}
3934 
3935 // Indirect Memory Times Scale Plus Index Register
3936 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3937 %{
3938   predicate(Universe::narrow_oop_shift() == 0);
3939   constraint(ALLOC_IN_RC(ptr_reg));
3940   match(AddP (DecodeN reg) (LShiftL lreg scale));
3941 
3942   op_cost(10);
3943   format %{"[$reg + $lreg << $scale]" %}
3944   interface(MEMORY_INTER) %{
3945     base($reg);
3946     index($lreg);
3947     scale($scale);
3948     disp(0x0);
3949   %}
3950 %}
3951 
3952 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3953 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3954 %{
3955   predicate(Universe::narrow_oop_shift() == 0);
3956   constraint(ALLOC_IN_RC(ptr_reg));
3957   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3958 
3959   op_cost(10);
3960   format %{"[$reg + $off + $lreg << $scale]" %}
3961   interface(MEMORY_INTER) %{
3962     base($reg);
3963     index($lreg);
3964     scale($scale);
3965     disp($off);
3966   %}
3967 %}
3968 
3969 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3970 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3971 %{
3972   constraint(ALLOC_IN_RC(ptr_reg));
3973   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3974   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3975 
3976   op_cost(10);
3977   format %{"[$reg + $off + $idx]" %}
3978   interface(MEMORY_INTER) %{
3979     base($reg);
3980     index($idx);
3981     scale(0x0);
3982     disp($off);
3983   %}
3984 %}
3985 
3986 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3987 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3988 %{
3989   constraint(ALLOC_IN_RC(ptr_reg));
3990   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3991   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3992 
3993   op_cost(10);
3994   format %{"[$reg + $off + $idx << $scale]" %}
3995   interface(MEMORY_INTER) %{
3996     base($reg);
3997     index($idx);
3998     scale($scale);
3999     disp($off);
4000   %}
4001 %}
4002 
4003 //----------Special Memory Operands--------------------------------------------
4004 // Stack Slot Operand - This operand is used for loading and storing temporary
4005 //                      values on the stack where a match requires a value to
4006 //                      flow through memory.
4007 operand stackSlotP(sRegP reg)
4008 %{
4009   constraint(ALLOC_IN_RC(stack_slots));
4010   // No match rule because this operand is only generated in matching
4011 
4012   format %{ "[$reg]" %}
4013   interface(MEMORY_INTER) %{
4014     base(0x4);   // RSP
4015     index(0x4);  // No Index
4016     scale(0x0);  // No Scale
4017     disp($reg);  // Stack Offset
4018   %}
4019 %}
4020 
4021 operand stackSlotI(sRegI reg)
4022 %{
4023   constraint(ALLOC_IN_RC(stack_slots));
4024   // No match rule because this operand is only generated in matching
4025 
4026   format %{ "[$reg]" %}
4027   interface(MEMORY_INTER) %{
4028     base(0x4);   // RSP
4029     index(0x4);  // No Index
4030     scale(0x0);  // No Scale
4031     disp($reg);  // Stack Offset
4032   %}
4033 %}
4034 
4035 operand stackSlotF(sRegF reg)
4036 %{
4037   constraint(ALLOC_IN_RC(stack_slots));
4038   // No match rule because this operand is only generated in matching
4039 
4040   format %{ "[$reg]" %}
4041   interface(MEMORY_INTER) %{
4042     base(0x4);   // RSP
4043     index(0x4);  // No Index
4044     scale(0x0);  // No Scale
4045     disp($reg);  // Stack Offset
4046   %}
4047 %}
4048 
4049 operand stackSlotD(sRegD reg)
4050 %{
4051   constraint(ALLOC_IN_RC(stack_slots));
4052   // No match rule because this operand is only generated in matching
4053 
4054   format %{ "[$reg]" %}
4055   interface(MEMORY_INTER) %{
4056     base(0x4);   // RSP
4057     index(0x4);  // No Index
4058     scale(0x0);  // No Scale
4059     disp($reg);  // Stack Offset
4060   %}
4061 %}
4062 operand stackSlotL(sRegL reg)
4063 %{
4064   constraint(ALLOC_IN_RC(stack_slots));
4065   // No match rule because this operand is only generated in matching
4066 
4067   format %{ "[$reg]" %}
4068   interface(MEMORY_INTER) %{
4069     base(0x4);   // RSP
4070     index(0x4);  // No Index
4071     scale(0x0);  // No Scale
4072     disp($reg);  // Stack Offset
4073   %}
4074 %}
4075 
4076 //----------Conditional Branch Operands----------------------------------------
4077 // Comparison Op  - This is the operation of the comparison, and is limited to
4078 //                  the following set of codes:
4079 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4080 //
4081 // Other attributes of the comparison, such as unsignedness, are specified
4082 // by the comparison instruction that sets a condition code flags register.
4083 // That result is represented by a flags operand whose subtype is appropriate
4084 // to the unsignedness (etc.) of the comparison.
4085 //
4086 // Later, the instruction which matches both the Comparison Op (a Bool) and
4087 // the flags (produced by the Cmp) specifies the coding of the comparison op
4088 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4089 
4090 // Comparision Code
4091 operand cmpOp()
4092 %{
4093   match(Bool);
4094 
4095   format %{ "" %}
4096   interface(COND_INTER) %{
4097     equal(0x4, "e");
4098     not_equal(0x5, "ne");
4099     less(0xC, "l");
4100     greater_equal(0xD, "ge");
4101     less_equal(0xE, "le");
4102     greater(0xF, "g");
4103     overflow(0x0, "o");
4104     no_overflow(0x1, "no");
4105   %}
4106 %}
4107 
4108 // Comparison Code, unsigned compare.  Used by FP also, with
4109 // C2 (unordered) turned into GT or LT already.  The other bits
4110 // C0 and C3 are turned into Carry & Zero flags.
4111 operand cmpOpU()
4112 %{
4113   match(Bool);
4114 
4115   format %{ "" %}
4116   interface(COND_INTER) %{
4117     equal(0x4, "e");
4118     not_equal(0x5, "ne");
4119     less(0x2, "b");
4120     greater_equal(0x3, "nb");
4121     less_equal(0x6, "be");
4122     greater(0x7, "nbe");
4123     overflow(0x0, "o");
4124     no_overflow(0x1, "no");
4125   %}
4126 %}
4127 
4128 
4129 // Floating comparisons that don't require any fixup for the unordered case
4130 operand cmpOpUCF() %{
4131   match(Bool);
4132   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4133             n->as_Bool()->_test._test == BoolTest::ge ||
4134             n->as_Bool()->_test._test == BoolTest::le ||
4135             n->as_Bool()->_test._test == BoolTest::gt);
4136   format %{ "" %}
4137   interface(COND_INTER) %{
4138     equal(0x4, "e");
4139     not_equal(0x5, "ne");
4140     less(0x2, "b");
4141     greater_equal(0x3, "nb");
4142     less_equal(0x6, "be");
4143     greater(0x7, "nbe");
4144     overflow(0x0, "o");
4145     no_overflow(0x1, "no");
4146   %}
4147 %}
4148 
4149 
4150 // Floating comparisons that can be fixed up with extra conditional jumps
4151 operand cmpOpUCF2() %{
4152   match(Bool);
4153   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4154             n->as_Bool()->_test._test == BoolTest::eq);
4155   format %{ "" %}
4156   interface(COND_INTER) %{
4157     equal(0x4, "e");
4158     not_equal(0x5, "ne");
4159     less(0x2, "b");
4160     greater_equal(0x3, "nb");
4161     less_equal(0x6, "be");
4162     greater(0x7, "nbe");
4163     overflow(0x0, "o");
4164     no_overflow(0x1, "no");
4165   %}
4166 %}
4167 
4168 
4169 //----------OPERAND CLASSES----------------------------------------------------
4170 // Operand Classes are groups of operands that are used as to simplify
4171 // instruction definitions by not requiring the AD writer to specify separate
4172 // instructions for every form of operand when the instruction accepts
4173 // multiple operand types with the same basic encoding and format.  The classic
4174 // case of this is memory operands.
4175 
4176 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4177                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4178                indCompressedOopOffset,
4179                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4180                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4181                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4182 
4183 //----------PIPELINE-----------------------------------------------------------
4184 // Rules which define the behavior of the target architectures pipeline.
4185 pipeline %{
4186 
4187 //----------ATTRIBUTES---------------------------------------------------------
4188 attributes %{
4189   variable_size_instructions;        // Fixed size instructions
4190   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4191   instruction_unit_size = 1;         // An instruction is 1 bytes long
4192   instruction_fetch_unit_size = 16;  // The processor fetches one line
4193   instruction_fetch_units = 1;       // of 16 bytes
4194 
4195   // List of nop instructions
4196   nops( MachNop );
4197 %}
4198 
4199 //----------RESOURCES----------------------------------------------------------
4200 // Resources are the functional units available to the machine
4201 
4202 // Generic P2/P3 pipeline
4203 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4204 // 3 instructions decoded per cycle.
4205 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4206 // 3 ALU op, only ALU0 handles mul instructions.
4207 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4208            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4209            BR, FPU,
4210            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4211 
4212 //----------PIPELINE DESCRIPTION-----------------------------------------------
4213 // Pipeline Description specifies the stages in the machine's pipeline
4214 
4215 // Generic P2/P3 pipeline
4216 pipe_desc(S0, S1, S2, S3, S4, S5);
4217 
4218 //----------PIPELINE CLASSES---------------------------------------------------
4219 // Pipeline Classes describe the stages in which input and output are
4220 // referenced by the hardware pipeline.
4221 
4222 // Naming convention: ialu or fpu
4223 // Then: _reg
4224 // Then: _reg if there is a 2nd register
4225 // Then: _long if it's a pair of instructions implementing a long
4226 // Then: _fat if it requires the big decoder
4227 //   Or: _mem if it requires the big decoder and a memory unit.
4228 
4229 // Integer ALU reg operation
4230 pipe_class ialu_reg(rRegI dst)
4231 %{
4232     single_instruction;
4233     dst    : S4(write);
4234     dst    : S3(read);
4235     DECODE : S0;        // any decoder
4236     ALU    : S3;        // any alu
4237 %}
4238 
4239 // Long ALU reg operation
4240 pipe_class ialu_reg_long(rRegL dst)
4241 %{
4242     instruction_count(2);
4243     dst    : S4(write);
4244     dst    : S3(read);
4245     DECODE : S0(2);     // any 2 decoders
4246     ALU    : S3(2);     // both alus
4247 %}
4248 
4249 // Integer ALU reg operation using big decoder
4250 pipe_class ialu_reg_fat(rRegI dst)
4251 %{
4252     single_instruction;
4253     dst    : S4(write);
4254     dst    : S3(read);
4255     D0     : S0;        // big decoder only
4256     ALU    : S3;        // any alu
4257 %}
4258 
4259 // Long ALU reg operation using big decoder
4260 pipe_class ialu_reg_long_fat(rRegL dst)
4261 %{
4262     instruction_count(2);
4263     dst    : S4(write);
4264     dst    : S3(read);
4265     D0     : S0(2);     // big decoder only; twice
4266     ALU    : S3(2);     // any 2 alus
4267 %}
4268 
4269 // Integer ALU reg-reg operation
4270 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4271 %{
4272     single_instruction;
4273     dst    : S4(write);
4274     src    : S3(read);
4275     DECODE : S0;        // any decoder
4276     ALU    : S3;        // any alu
4277 %}
4278 
4279 // Long ALU reg-reg operation
4280 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4281 %{
4282     instruction_count(2);
4283     dst    : S4(write);
4284     src    : S3(read);
4285     DECODE : S0(2);     // any 2 decoders
4286     ALU    : S3(2);     // both alus
4287 %}
4288 
4289 // Integer ALU reg-reg operation
4290 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4291 %{
4292     single_instruction;
4293     dst    : S4(write);
4294     src    : S3(read);
4295     D0     : S0;        // big decoder only
4296     ALU    : S3;        // any alu
4297 %}
4298 
4299 // Long ALU reg-reg operation
4300 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4301 %{
4302     instruction_count(2);
4303     dst    : S4(write);
4304     src    : S3(read);
4305     D0     : S0(2);     // big decoder only; twice
4306     ALU    : S3(2);     // both alus
4307 %}
4308 
4309 // Integer ALU reg-mem operation
4310 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4311 %{
4312     single_instruction;
4313     dst    : S5(write);
4314     mem    : S3(read);
4315     D0     : S0;        // big decoder only
4316     ALU    : S4;        // any alu
4317     MEM    : S3;        // any mem
4318 %}
4319 
4320 // Integer mem operation (prefetch)
4321 pipe_class ialu_mem(memory mem)
4322 %{
4323     single_instruction;
4324     mem    : S3(read);
4325     D0     : S0;        // big decoder only
4326     MEM    : S3;        // any mem
4327 %}
4328 
4329 // Integer Store to Memory
4330 pipe_class ialu_mem_reg(memory mem, rRegI src)
4331 %{
4332     single_instruction;
4333     mem    : S3(read);
4334     src    : S5(read);
4335     D0     : S0;        // big decoder only
4336     ALU    : S4;        // any alu
4337     MEM    : S3;
4338 %}
4339 
4340 // // Long Store to Memory
4341 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4342 // %{
4343 //     instruction_count(2);
4344 //     mem    : S3(read);
4345 //     src    : S5(read);
4346 //     D0     : S0(2);          // big decoder only; twice
4347 //     ALU    : S4(2);     // any 2 alus
4348 //     MEM    : S3(2);  // Both mems
4349 // %}
4350 
4351 // Integer Store to Memory
4352 pipe_class ialu_mem_imm(memory mem)
4353 %{
4354     single_instruction;
4355     mem    : S3(read);
4356     D0     : S0;        // big decoder only
4357     ALU    : S4;        // any alu
4358     MEM    : S3;
4359 %}
4360 
4361 // Integer ALU0 reg-reg operation
4362 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4363 %{
4364     single_instruction;
4365     dst    : S4(write);
4366     src    : S3(read);
4367     D0     : S0;        // Big decoder only
4368     ALU0   : S3;        // only alu0
4369 %}
4370 
4371 // Integer ALU0 reg-mem operation
4372 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4373 %{
4374     single_instruction;
4375     dst    : S5(write);
4376     mem    : S3(read);
4377     D0     : S0;        // big decoder only
4378     ALU0   : S4;        // ALU0 only
4379     MEM    : S3;        // any mem
4380 %}
4381 
4382 // Integer ALU reg-reg operation
4383 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4384 %{
4385     single_instruction;
4386     cr     : S4(write);
4387     src1   : S3(read);
4388     src2   : S3(read);
4389     DECODE : S0;        // any decoder
4390     ALU    : S3;        // any alu
4391 %}
4392 
4393 // Integer ALU reg-imm operation
4394 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4395 %{
4396     single_instruction;
4397     cr     : S4(write);
4398     src1   : S3(read);
4399     DECODE : S0;        // any decoder
4400     ALU    : S3;        // any alu
4401 %}
4402 
4403 // Integer ALU reg-mem operation
4404 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4405 %{
4406     single_instruction;
4407     cr     : S4(write);
4408     src1   : S3(read);
4409     src2   : S3(read);
4410     D0     : S0;        // big decoder only
4411     ALU    : S4;        // any alu
4412     MEM    : S3;
4413 %}
4414 
4415 // Conditional move reg-reg
4416 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4417 %{
4418     instruction_count(4);
4419     y      : S4(read);
4420     q      : S3(read);
4421     p      : S3(read);
4422     DECODE : S0(4);     // any decoder
4423 %}
4424 
4425 // Conditional move reg-reg
4426 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4427 %{
4428     single_instruction;
4429     dst    : S4(write);
4430     src    : S3(read);
4431     cr     : S3(read);
4432     DECODE : S0;        // any decoder
4433 %}
4434 
4435 // Conditional move reg-mem
4436 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4437 %{
4438     single_instruction;
4439     dst    : S4(write);
4440     src    : S3(read);
4441     cr     : S3(read);
4442     DECODE : S0;        // any decoder
4443     MEM    : S3;
4444 %}
4445 
4446 // Conditional move reg-reg long
4447 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4448 %{
4449     single_instruction;
4450     dst    : S4(write);
4451     src    : S3(read);
4452     cr     : S3(read);
4453     DECODE : S0(2);     // any 2 decoders
4454 %}
4455 
4456 // XXX
4457 // // Conditional move double reg-reg
4458 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4459 // %{
4460 //     single_instruction;
4461 //     dst    : S4(write);
4462 //     src    : S3(read);
4463 //     cr     : S3(read);
4464 //     DECODE : S0;     // any decoder
4465 // %}
4466 
4467 // Float reg-reg operation
4468 pipe_class fpu_reg(regD dst)
4469 %{
4470     instruction_count(2);
4471     dst    : S3(read);
4472     DECODE : S0(2);     // any 2 decoders
4473     FPU    : S3;
4474 %}
4475 
4476 // Float reg-reg operation
4477 pipe_class fpu_reg_reg(regD dst, regD src)
4478 %{
4479     instruction_count(2);
4480     dst    : S4(write);
4481     src    : S3(read);
4482     DECODE : S0(2);     // any 2 decoders
4483     FPU    : S3;
4484 %}
4485 
4486 // Float reg-reg operation
4487 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4488 %{
4489     instruction_count(3);
4490     dst    : S4(write);
4491     src1   : S3(read);
4492     src2   : S3(read);
4493     DECODE : S0(3);     // any 3 decoders
4494     FPU    : S3(2);
4495 %}
4496 
4497 // Float reg-reg operation
4498 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4499 %{
4500     instruction_count(4);
4501     dst    : S4(write);
4502     src1   : S3(read);
4503     src2   : S3(read);
4504     src3   : S3(read);
4505     DECODE : S0(4);     // any 3 decoders
4506     FPU    : S3(2);
4507 %}
4508 
4509 // Float reg-reg operation
4510 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4511 %{
4512     instruction_count(4);
4513     dst    : S4(write);
4514     src1   : S3(read);
4515     src2   : S3(read);
4516     src3   : S3(read);
4517     DECODE : S1(3);     // any 3 decoders
4518     D0     : S0;        // Big decoder only
4519     FPU    : S3(2);
4520     MEM    : S3;
4521 %}
4522 
4523 // Float reg-mem operation
4524 pipe_class fpu_reg_mem(regD dst, memory mem)
4525 %{
4526     instruction_count(2);
4527     dst    : S5(write);
4528     mem    : S3(read);
4529     D0     : S0;        // big decoder only
4530     DECODE : S1;        // any decoder for FPU POP
4531     FPU    : S4;
4532     MEM    : S3;        // any mem
4533 %}
4534 
4535 // Float reg-mem operation
4536 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4537 %{
4538     instruction_count(3);
4539     dst    : S5(write);
4540     src1   : S3(read);
4541     mem    : S3(read);
4542     D0     : S0;        // big decoder only
4543     DECODE : S1(2);     // any decoder for FPU POP
4544     FPU    : S4;
4545     MEM    : S3;        // any mem
4546 %}
4547 
4548 // Float mem-reg operation
4549 pipe_class fpu_mem_reg(memory mem, regD src)
4550 %{
4551     instruction_count(2);
4552     src    : S5(read);
4553     mem    : S3(read);
4554     DECODE : S0;        // any decoder for FPU PUSH
4555     D0     : S1;        // big decoder only
4556     FPU    : S4;
4557     MEM    : S3;        // any mem
4558 %}
4559 
4560 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4561 %{
4562     instruction_count(3);
4563     src1   : S3(read);
4564     src2   : S3(read);
4565     mem    : S3(read);
4566     DECODE : S0(2);     // any decoder for FPU PUSH
4567     D0     : S1;        // big decoder only
4568     FPU    : S4;
4569     MEM    : S3;        // any mem
4570 %}
4571 
4572 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4573 %{
4574     instruction_count(3);
4575     src1   : S3(read);
4576     src2   : S3(read);
4577     mem    : S4(read);
4578     DECODE : S0;        // any decoder for FPU PUSH
4579     D0     : S0(2);     // big decoder only
4580     FPU    : S4;
4581     MEM    : S3(2);     // any mem
4582 %}
4583 
4584 pipe_class fpu_mem_mem(memory dst, memory src1)
4585 %{
4586     instruction_count(2);
4587     src1   : S3(read);
4588     dst    : S4(read);
4589     D0     : S0(2);     // big decoder only
4590     MEM    : S3(2);     // any mem
4591 %}
4592 
4593 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4594 %{
4595     instruction_count(3);
4596     src1   : S3(read);
4597     src2   : S3(read);
4598     dst    : S4(read);
4599     D0     : S0(3);     // big decoder only
4600     FPU    : S4;
4601     MEM    : S3(3);     // any mem
4602 %}
4603 
4604 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4605 %{
4606     instruction_count(3);
4607     src1   : S4(read);
4608     mem    : S4(read);
4609     DECODE : S0;        // any decoder for FPU PUSH
4610     D0     : S0(2);     // big decoder only
4611     FPU    : S4;
4612     MEM    : S3(2);     // any mem
4613 %}
4614 
4615 // Float load constant
4616 pipe_class fpu_reg_con(regD dst)
4617 %{
4618     instruction_count(2);
4619     dst    : S5(write);
4620     D0     : S0;        // big decoder only for the load
4621     DECODE : S1;        // any decoder for FPU POP
4622     FPU    : S4;
4623     MEM    : S3;        // any mem
4624 %}
4625 
4626 // Float load constant
4627 pipe_class fpu_reg_reg_con(regD dst, regD src)
4628 %{
4629     instruction_count(3);
4630     dst    : S5(write);
4631     src    : S3(read);
4632     D0     : S0;        // big decoder only for the load
4633     DECODE : S1(2);     // any decoder for FPU POP
4634     FPU    : S4;
4635     MEM    : S3;        // any mem
4636 %}
4637 
4638 // UnConditional branch
4639 pipe_class pipe_jmp(label labl)
4640 %{
4641     single_instruction;
4642     BR   : S3;
4643 %}
4644 
4645 // Conditional branch
4646 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4647 %{
4648     single_instruction;
4649     cr    : S1(read);
4650     BR    : S3;
4651 %}
4652 
4653 // Allocation idiom
4654 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4655 %{
4656     instruction_count(1); force_serialization;
4657     fixed_latency(6);
4658     heap_ptr : S3(read);
4659     DECODE   : S0(3);
4660     D0       : S2;
4661     MEM      : S3;
4662     ALU      : S3(2);
4663     dst      : S5(write);
4664     BR       : S5;
4665 %}
4666 
4667 // Generic big/slow expanded idiom
4668 pipe_class pipe_slow()
4669 %{
4670     instruction_count(10); multiple_bundles; force_serialization;
4671     fixed_latency(100);
4672     D0  : S0(2);
4673     MEM : S3(2);
4674 %}
4675 
4676 // The real do-nothing guy
4677 pipe_class empty()
4678 %{
4679     instruction_count(0);
4680 %}
4681 
4682 // Define the class for the Nop node
4683 define
4684 %{
4685    MachNop = empty;
4686 %}
4687 
4688 %}
4689 
4690 //----------INSTRUCTIONS-------------------------------------------------------
4691 //
4692 // match      -- States which machine-independent subtree may be replaced
4693 //               by this instruction.
4694 // ins_cost   -- The estimated cost of this instruction is used by instruction
4695 //               selection to identify a minimum cost tree of machine
4696 //               instructions that matches a tree of machine-independent
4697 //               instructions.
4698 // format     -- A string providing the disassembly for this instruction.
4699 //               The value of an instruction's operand may be inserted
4700 //               by referring to it with a '$' prefix.
4701 // opcode     -- Three instruction opcodes may be provided.  These are referred
4702 //               to within an encode class as $primary, $secondary, and $tertiary
4703 //               rrspectively.  The primary opcode is commonly used to
4704 //               indicate the type of machine instruction, while secondary
4705 //               and tertiary are often used for prefix options or addressing
4706 //               modes.
4707 // ins_encode -- A list of encode classes with parameters. The encode class
4708 //               name must have been defined in an 'enc_class' specification
4709 //               in the encode section of the architecture description.
4710 
4711 
4712 //----------Load/Store/Move Instructions---------------------------------------
4713 //----------Load Instructions--------------------------------------------------
4714 
4715 // Load Byte (8 bit signed)
4716 instruct loadB(rRegI dst, memory mem)
4717 %{
4718   match(Set dst (LoadB mem));
4719 
4720   ins_cost(125);
4721   format %{ "movsbl  $dst, $mem\t# byte" %}
4722 
4723   ins_encode %{
4724     __ movsbl($dst$$Register, $mem$$Address);
4725   %}
4726 
4727   ins_pipe(ialu_reg_mem);
4728 %}
4729 
4730 // Load Byte (8 bit signed) into Long Register
4731 instruct loadB2L(rRegL dst, memory mem)
4732 %{
4733   match(Set dst (ConvI2L (LoadB mem)));
4734 
4735   ins_cost(125);
4736   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4737 
4738   ins_encode %{
4739     __ movsbq($dst$$Register, $mem$$Address);
4740   %}
4741 
4742   ins_pipe(ialu_reg_mem);
4743 %}
4744 
4745 // Load Unsigned Byte (8 bit UNsigned)
4746 instruct loadUB(rRegI dst, memory mem)
4747 %{
4748   match(Set dst (LoadUB mem));
4749 
4750   ins_cost(125);
4751   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4752 
4753   ins_encode %{
4754     __ movzbl($dst$$Register, $mem$$Address);
4755   %}
4756 
4757   ins_pipe(ialu_reg_mem);
4758 %}
4759 
4760 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4761 instruct loadUB2L(rRegL dst, memory mem)
4762 %{
4763   match(Set dst (ConvI2L (LoadUB mem)));
4764 
4765   ins_cost(125);
4766   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4767 
4768   ins_encode %{
4769     __ movzbq($dst$$Register, $mem$$Address);
4770   %}
4771 
4772   ins_pipe(ialu_reg_mem);
4773 %}
4774 
4775 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4776 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4777   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4778   effect(KILL cr);
4779 
4780   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4781             "andl    $dst, right_n_bits($mask, 8)" %}
4782   ins_encode %{
4783     Register Rdst = $dst$$Register;
4784     __ movzbq(Rdst, $mem$$Address);
4785     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4786   %}
4787   ins_pipe(ialu_reg_mem);
4788 %}
4789 
4790 // Load Short (16 bit signed)
4791 instruct loadS(rRegI dst, memory mem)
4792 %{
4793   match(Set dst (LoadS mem));
4794 
4795   ins_cost(125);
4796   format %{ "movswl $dst, $mem\t# short" %}
4797 
4798   ins_encode %{
4799     __ movswl($dst$$Register, $mem$$Address);
4800   %}
4801 
4802   ins_pipe(ialu_reg_mem);
4803 %}
4804 
4805 // Load Short (16 bit signed) to Byte (8 bit signed)
4806 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4807   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4808 
4809   ins_cost(125);
4810   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4811   ins_encode %{
4812     __ movsbl($dst$$Register, $mem$$Address);
4813   %}
4814   ins_pipe(ialu_reg_mem);
4815 %}
4816 
4817 // Load Short (16 bit signed) into Long Register
4818 instruct loadS2L(rRegL dst, memory mem)
4819 %{
4820   match(Set dst (ConvI2L (LoadS mem)));
4821 
4822   ins_cost(125);
4823   format %{ "movswq $dst, $mem\t# short -> long" %}
4824 
4825   ins_encode %{
4826     __ movswq($dst$$Register, $mem$$Address);
4827   %}
4828 
4829   ins_pipe(ialu_reg_mem);
4830 %}
4831 
4832 // Load Unsigned Short/Char (16 bit UNsigned)
4833 instruct loadUS(rRegI dst, memory mem)
4834 %{
4835   match(Set dst (LoadUS mem));
4836 
4837   ins_cost(125);
4838   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4839 
4840   ins_encode %{
4841     __ movzwl($dst$$Register, $mem$$Address);
4842   %}
4843 
4844   ins_pipe(ialu_reg_mem);
4845 %}
4846 
4847 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4848 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4849   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4850 
4851   ins_cost(125);
4852   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4853   ins_encode %{
4854     __ movsbl($dst$$Register, $mem$$Address);
4855   %}
4856   ins_pipe(ialu_reg_mem);
4857 %}
4858 
4859 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4860 instruct loadUS2L(rRegL dst, memory mem)
4861 %{
4862   match(Set dst (ConvI2L (LoadUS mem)));
4863 
4864   ins_cost(125);
4865   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4866 
4867   ins_encode %{
4868     __ movzwq($dst$$Register, $mem$$Address);
4869   %}
4870 
4871   ins_pipe(ialu_reg_mem);
4872 %}
4873 
4874 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4875 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4876   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4877 
4878   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4879   ins_encode %{
4880     __ movzbq($dst$$Register, $mem$$Address);
4881   %}
4882   ins_pipe(ialu_reg_mem);
4883 %}
4884 
4885 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4886 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4887   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4888   effect(KILL cr);
4889 
4890   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4891             "andl    $dst, right_n_bits($mask, 16)" %}
4892   ins_encode %{
4893     Register Rdst = $dst$$Register;
4894     __ movzwq(Rdst, $mem$$Address);
4895     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4896   %}
4897   ins_pipe(ialu_reg_mem);
4898 %}
4899 
4900 // Load Integer
4901 instruct loadI(rRegI dst, memory mem)
4902 %{
4903   match(Set dst (LoadI mem));
4904 
4905   ins_cost(125);
4906   format %{ "movl    $dst, $mem\t# int" %}
4907 
4908   ins_encode %{
4909     __ movl($dst$$Register, $mem$$Address);
4910   %}
4911 
4912   ins_pipe(ialu_reg_mem);
4913 %}
4914 
4915 // Load Integer (32 bit signed) to Byte (8 bit signed)
4916 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4917   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4918 
4919   ins_cost(125);
4920   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4921   ins_encode %{
4922     __ movsbl($dst$$Register, $mem$$Address);
4923   %}
4924   ins_pipe(ialu_reg_mem);
4925 %}
4926 
4927 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4928 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4929   match(Set dst (AndI (LoadI mem) mask));
4930 
4931   ins_cost(125);
4932   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4933   ins_encode %{
4934     __ movzbl($dst$$Register, $mem$$Address);
4935   %}
4936   ins_pipe(ialu_reg_mem);
4937 %}
4938 
4939 // Load Integer (32 bit signed) to Short (16 bit signed)
4940 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4941   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4942 
4943   ins_cost(125);
4944   format %{ "movswl  $dst, $mem\t# int -> short" %}
4945   ins_encode %{
4946     __ movswl($dst$$Register, $mem$$Address);
4947   %}
4948   ins_pipe(ialu_reg_mem);
4949 %}
4950 
4951 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4952 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4953   match(Set dst (AndI (LoadI mem) mask));
4954 
4955   ins_cost(125);
4956   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4957   ins_encode %{
4958     __ movzwl($dst$$Register, $mem$$Address);
4959   %}
4960   ins_pipe(ialu_reg_mem);
4961 %}
4962 
4963 // Load Integer into Long Register
4964 instruct loadI2L(rRegL dst, memory mem)
4965 %{
4966   match(Set dst (ConvI2L (LoadI mem)));
4967 
4968   ins_cost(125);
4969   format %{ "movslq  $dst, $mem\t# int -> long" %}
4970 
4971   ins_encode %{
4972     __ movslq($dst$$Register, $mem$$Address);
4973   %}
4974 
4975   ins_pipe(ialu_reg_mem);
4976 %}
4977 
4978 // Load Integer with mask 0xFF into Long Register
4979 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4980   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4981 
4982   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4983   ins_encode %{
4984     __ movzbq($dst$$Register, $mem$$Address);
4985   %}
4986   ins_pipe(ialu_reg_mem);
4987 %}
4988 
4989 // Load Integer with mask 0xFFFF into Long Register
4990 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4991   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4992 
4993   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4994   ins_encode %{
4995     __ movzwq($dst$$Register, $mem$$Address);
4996   %}
4997   ins_pipe(ialu_reg_mem);
4998 %}
4999 
5000 // Load Integer with a 31-bit mask into Long Register
5001 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5002   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5003   effect(KILL cr);
5004 
5005   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5006             "andl    $dst, $mask" %}
5007   ins_encode %{
5008     Register Rdst = $dst$$Register;
5009     __ movl(Rdst, $mem$$Address);
5010     __ andl(Rdst, $mask$$constant);
5011   %}
5012   ins_pipe(ialu_reg_mem);
5013 %}
5014 
5015 // Load Unsigned Integer into Long Register
5016 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5017 %{
5018   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5019 
5020   ins_cost(125);
5021   format %{ "movl    $dst, $mem\t# uint -> long" %}
5022 
5023   ins_encode %{
5024     __ movl($dst$$Register, $mem$$Address);
5025   %}
5026 
5027   ins_pipe(ialu_reg_mem);
5028 %}
5029 
5030 // Load Long
5031 instruct loadL(rRegL dst, memory mem)
5032 %{
5033   match(Set dst (LoadL mem));
5034 
5035   ins_cost(125);
5036   format %{ "movq    $dst, $mem\t# long" %}
5037 
5038   ins_encode %{
5039     __ movq($dst$$Register, $mem$$Address);
5040   %}
5041 
5042   ins_pipe(ialu_reg_mem); // XXX
5043 %}
5044 
5045 // Load Range
5046 instruct loadRange(rRegI dst, memory mem)
5047 %{
5048   match(Set dst (LoadRange mem));
5049 
5050   ins_cost(125); // XXX
5051   format %{ "movl    $dst, $mem\t# range" %}
5052   opcode(0x8B);
5053   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5054   ins_pipe(ialu_reg_mem);
5055 %}
5056 
5057 // Load Pointer
5058 instruct loadP(rRegP dst, memory mem)
5059 %{
5060   match(Set dst (LoadP mem));
5061 
5062   ins_cost(125); // XXX
5063   format %{ "movq    $dst, $mem\t# ptr" %}
5064   opcode(0x8B);
5065   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5066   ins_pipe(ialu_reg_mem); // XXX
5067 %}
5068 
5069 // Load Compressed Pointer
5070 instruct loadN(rRegN dst, memory mem)
5071 %{
5072    match(Set dst (LoadN mem));
5073 
5074    ins_cost(125); // XXX
5075    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5076    ins_encode %{
5077      __ movl($dst$$Register, $mem$$Address);
5078    %}
5079    ins_pipe(ialu_reg_mem); // XXX
5080 %}
5081 
5082 
5083 // Load Klass Pointer
5084 instruct loadKlass(rRegP dst, memory mem)
5085 %{
5086   match(Set dst (LoadKlass mem));
5087 
5088   ins_cost(125); // XXX
5089   format %{ "movq    $dst, $mem\t# class" %}
5090   opcode(0x8B);
5091   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5092   ins_pipe(ialu_reg_mem); // XXX
5093 %}
5094 
5095 // Load narrow Klass Pointer
5096 instruct loadNKlass(rRegN dst, memory mem)
5097 %{
5098   match(Set dst (LoadNKlass mem));
5099 
5100   ins_cost(125); // XXX
5101   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5102   ins_encode %{
5103     __ movl($dst$$Register, $mem$$Address);
5104   %}
5105   ins_pipe(ialu_reg_mem); // XXX
5106 %}
5107 
5108 // Load Float
5109 instruct loadF(regF dst, memory mem)
5110 %{
5111   match(Set dst (LoadF mem));
5112 
5113   ins_cost(145); // XXX
5114   format %{ "movss   $dst, $mem\t# float" %}
5115   ins_encode %{
5116     __ movflt($dst$$XMMRegister, $mem$$Address);
5117   %}
5118   ins_pipe(pipe_slow); // XXX
5119 %}
5120 
5121 // Load Double
5122 instruct loadD_partial(regD dst, memory mem)
5123 %{
5124   predicate(!UseXmmLoadAndClearUpper);
5125   match(Set dst (LoadD mem));
5126 
5127   ins_cost(145); // XXX
5128   format %{ "movlpd  $dst, $mem\t# double" %}
5129   ins_encode %{
5130     __ movdbl($dst$$XMMRegister, $mem$$Address);
5131   %}
5132   ins_pipe(pipe_slow); // XXX
5133 %}
5134 
5135 instruct loadD(regD dst, memory mem)
5136 %{
5137   predicate(UseXmmLoadAndClearUpper);
5138   match(Set dst (LoadD mem));
5139 
5140   ins_cost(145); // XXX
5141   format %{ "movsd   $dst, $mem\t# double" %}
5142   ins_encode %{
5143     __ movdbl($dst$$XMMRegister, $mem$$Address);
5144   %}
5145   ins_pipe(pipe_slow); // XXX
5146 %}
5147 
5148 // Load Effective Address
5149 instruct leaP8(rRegP dst, indOffset8 mem)
5150 %{
5151   match(Set dst mem);
5152 
5153   ins_cost(110); // XXX
5154   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5155   opcode(0x8D);
5156   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5157   ins_pipe(ialu_reg_reg_fat);
5158 %}
5159 
5160 instruct leaP32(rRegP dst, indOffset32 mem)
5161 %{
5162   match(Set dst mem);
5163 
5164   ins_cost(110);
5165   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5166   opcode(0x8D);
5167   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5168   ins_pipe(ialu_reg_reg_fat);
5169 %}
5170 
5171 // instruct leaPIdx(rRegP dst, indIndex mem)
5172 // %{
5173 //   match(Set dst mem);
5174 
5175 //   ins_cost(110);
5176 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5177 //   opcode(0x8D);
5178 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5179 //   ins_pipe(ialu_reg_reg_fat);
5180 // %}
5181 
5182 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5183 %{
5184   match(Set dst mem);
5185 
5186   ins_cost(110);
5187   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5188   opcode(0x8D);
5189   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5190   ins_pipe(ialu_reg_reg_fat);
5191 %}
5192 
5193 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5194 %{
5195   match(Set dst mem);
5196 
5197   ins_cost(110);
5198   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5199   opcode(0x8D);
5200   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5201   ins_pipe(ialu_reg_reg_fat);
5202 %}
5203 
5204 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5205 %{
5206   match(Set dst mem);
5207 
5208   ins_cost(110);
5209   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5210   opcode(0x8D);
5211   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5212   ins_pipe(ialu_reg_reg_fat);
5213 %}
5214 
5215 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5216 %{
5217   match(Set dst mem);
5218 
5219   ins_cost(110);
5220   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5221   opcode(0x8D);
5222   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5223   ins_pipe(ialu_reg_reg_fat);
5224 %}
5225 
5226 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5227 %{
5228   match(Set dst mem);
5229 
5230   ins_cost(110);
5231   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5232   opcode(0x8D);
5233   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5234   ins_pipe(ialu_reg_reg_fat);
5235 %}
5236 
5237 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5238 %{
5239   match(Set dst mem);
5240 
5241   ins_cost(110);
5242   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5243   opcode(0x8D);
5244   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5245   ins_pipe(ialu_reg_reg_fat);
5246 %}
5247 
5248 // Load Effective Address which uses Narrow (32-bits) oop
5249 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5250 %{
5251   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5252   match(Set dst mem);
5253 
5254   ins_cost(110);
5255   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5256   opcode(0x8D);
5257   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5258   ins_pipe(ialu_reg_reg_fat);
5259 %}
5260 
5261 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5262 %{
5263   predicate(Universe::narrow_oop_shift() == 0);
5264   match(Set dst mem);
5265 
5266   ins_cost(110); // XXX
5267   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5268   opcode(0x8D);
5269   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5270   ins_pipe(ialu_reg_reg_fat);
5271 %}
5272 
5273 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5274 %{
5275   predicate(Universe::narrow_oop_shift() == 0);
5276   match(Set dst mem);
5277 
5278   ins_cost(110);
5279   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5280   opcode(0x8D);
5281   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5282   ins_pipe(ialu_reg_reg_fat);
5283 %}
5284 
5285 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5286 %{
5287   predicate(Universe::narrow_oop_shift() == 0);
5288   match(Set dst mem);
5289 
5290   ins_cost(110);
5291   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5292   opcode(0x8D);
5293   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5294   ins_pipe(ialu_reg_reg_fat);
5295 %}
5296 
5297 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5298 %{
5299   predicate(Universe::narrow_oop_shift() == 0);
5300   match(Set dst mem);
5301 
5302   ins_cost(110);
5303   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5304   opcode(0x8D);
5305   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5306   ins_pipe(ialu_reg_reg_fat);
5307 %}
5308 
5309 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5310 %{
5311   predicate(Universe::narrow_oop_shift() == 0);
5312   match(Set dst mem);
5313 
5314   ins_cost(110);
5315   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5316   opcode(0x8D);
5317   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5318   ins_pipe(ialu_reg_reg_fat);
5319 %}
5320 
5321 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5322 %{
5323   predicate(Universe::narrow_oop_shift() == 0);
5324   match(Set dst mem);
5325 
5326   ins_cost(110);
5327   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5328   opcode(0x8D);
5329   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5330   ins_pipe(ialu_reg_reg_fat);
5331 %}
5332 
5333 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5334 %{
5335   predicate(Universe::narrow_oop_shift() == 0);
5336   match(Set dst mem);
5337 
5338   ins_cost(110);
5339   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5340   opcode(0x8D);
5341   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5342   ins_pipe(ialu_reg_reg_fat);
5343 %}
5344 
5345 instruct loadConI(rRegI dst, immI src)
5346 %{
5347   match(Set dst src);
5348 
5349   format %{ "movl    $dst, $src\t# int" %}
5350   ins_encode(load_immI(dst, src));
5351   ins_pipe(ialu_reg_fat); // XXX
5352 %}
5353 
5354 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5355 %{
5356   match(Set dst src);
5357   effect(KILL cr);
5358 
5359   ins_cost(50);
5360   format %{ "xorl    $dst, $dst\t# int" %}
5361   opcode(0x33); /* + rd */
5362   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5363   ins_pipe(ialu_reg);
5364 %}
5365 
5366 instruct loadConL(rRegL dst, immL src)
5367 %{
5368   match(Set dst src);
5369 
5370   ins_cost(150);
5371   format %{ "movq    $dst, $src\t# long" %}
5372   ins_encode(load_immL(dst, src));
5373   ins_pipe(ialu_reg);
5374 %}
5375 
5376 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5377 %{
5378   match(Set dst src);
5379   effect(KILL cr);
5380 
5381   ins_cost(50);
5382   format %{ "xorl    $dst, $dst\t# long" %}
5383   opcode(0x33); /* + rd */
5384   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5385   ins_pipe(ialu_reg); // XXX
5386 %}
5387 
5388 instruct loadConUL32(rRegL dst, immUL32 src)
5389 %{
5390   match(Set dst src);
5391 
5392   ins_cost(60);
5393   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5394   ins_encode(load_immUL32(dst, src));
5395   ins_pipe(ialu_reg);
5396 %}
5397 
5398 instruct loadConL32(rRegL dst, immL32 src)
5399 %{
5400   match(Set dst src);
5401 
5402   ins_cost(70);
5403   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5404   ins_encode(load_immL32(dst, src));
5405   ins_pipe(ialu_reg);
5406 %}
5407 
5408 instruct loadConP(rRegP dst, immP con) %{
5409   match(Set dst con);
5410 
5411   format %{ "movq    $dst, $con\t# ptr" %}
5412   ins_encode(load_immP(dst, con));
5413   ins_pipe(ialu_reg_fat); // XXX
5414 %}
5415 
5416 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5417 %{
5418   match(Set dst src);
5419   effect(KILL cr);
5420 
5421   ins_cost(50);
5422   format %{ "xorl    $dst, $dst\t# ptr" %}
5423   opcode(0x33); /* + rd */
5424   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5425   ins_pipe(ialu_reg);
5426 %}
5427 
5428 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5429 %{
5430   match(Set dst src);
5431   effect(KILL cr);
5432 
5433   ins_cost(60);
5434   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5435   ins_encode(load_immP31(dst, src));
5436   ins_pipe(ialu_reg);
5437 %}
5438 
5439 instruct loadConF(regF dst, immF con) %{
5440   match(Set dst con);
5441   ins_cost(125);
5442   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5443   ins_encode %{
5444     __ movflt($dst$$XMMRegister, $constantaddress($con));
5445   %}
5446   ins_pipe(pipe_slow);
5447 %}
5448 
5449 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5450   match(Set dst src);
5451   effect(KILL cr);
5452   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5453   ins_encode %{
5454     __ xorq($dst$$Register, $dst$$Register);
5455   %}
5456   ins_pipe(ialu_reg);
5457 %}
5458 
5459 instruct loadConN(rRegN dst, immN src) %{
5460   match(Set dst src);
5461 
5462   ins_cost(125);
5463   format %{ "movl    $dst, $src\t# compressed ptr" %}
5464   ins_encode %{
5465     address con = (address)$src$$constant;
5466     if (con == NULL) {
5467       ShouldNotReachHere();
5468     } else {
5469       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5470     }
5471   %}
5472   ins_pipe(ialu_reg_fat); // XXX
5473 %}
5474 
5475 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5476   match(Set dst src);
5477 
5478   ins_cost(125);
5479   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5480   ins_encode %{
5481     address con = (address)$src$$constant;
5482     if (con == NULL) {
5483       ShouldNotReachHere();
5484     } else {
5485       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5486     }
5487   %}
5488   ins_pipe(ialu_reg_fat); // XXX
5489 %}
5490 
5491 instruct loadConF0(regF dst, immF0 src)
5492 %{
5493   match(Set dst src);
5494   ins_cost(100);
5495 
5496   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5497   ins_encode %{
5498     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5499   %}
5500   ins_pipe(pipe_slow);
5501 %}
5502 
5503 // Use the same format since predicate() can not be used here.
5504 instruct loadConD(regD dst, immD con) %{
5505   match(Set dst con);
5506   ins_cost(125);
5507   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5508   ins_encode %{
5509     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5510   %}
5511   ins_pipe(pipe_slow);
5512 %}
5513 
5514 instruct loadConD0(regD dst, immD0 src)
5515 %{
5516   match(Set dst src);
5517   ins_cost(100);
5518 
5519   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5520   ins_encode %{
5521     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5522   %}
5523   ins_pipe(pipe_slow);
5524 %}
5525 
5526 instruct loadSSI(rRegI dst, stackSlotI src)
5527 %{
5528   match(Set dst src);
5529 
5530   ins_cost(125);
5531   format %{ "movl    $dst, $src\t# int stk" %}
5532   opcode(0x8B);
5533   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5534   ins_pipe(ialu_reg_mem);
5535 %}
5536 
5537 instruct loadSSL(rRegL dst, stackSlotL src)
5538 %{
5539   match(Set dst src);
5540 
5541   ins_cost(125);
5542   format %{ "movq    $dst, $src\t# long stk" %}
5543   opcode(0x8B);
5544   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5545   ins_pipe(ialu_reg_mem);
5546 %}
5547 
5548 instruct loadSSP(rRegP dst, stackSlotP src)
5549 %{
5550   match(Set dst src);
5551 
5552   ins_cost(125);
5553   format %{ "movq    $dst, $src\t# ptr stk" %}
5554   opcode(0x8B);
5555   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5556   ins_pipe(ialu_reg_mem);
5557 %}
5558 
5559 instruct loadSSF(regF dst, stackSlotF src)
5560 %{
5561   match(Set dst src);
5562 
5563   ins_cost(125);
5564   format %{ "movss   $dst, $src\t# float stk" %}
5565   ins_encode %{
5566     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5567   %}
5568   ins_pipe(pipe_slow); // XXX
5569 %}
5570 
5571 // Use the same format since predicate() can not be used here.
5572 instruct loadSSD(regD dst, stackSlotD src)
5573 %{
5574   match(Set dst src);
5575 
5576   ins_cost(125);
5577   format %{ "movsd   $dst, $src\t# double stk" %}
5578   ins_encode  %{
5579     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5580   %}
5581   ins_pipe(pipe_slow); // XXX
5582 %}
5583 
5584 // Prefetch instructions for allocation.
5585 // Must be safe to execute with invalid address (cannot fault).
5586 
5587 instruct prefetchAlloc( memory mem ) %{
5588   predicate(AllocatePrefetchInstr==3);
5589   match(PrefetchAllocation mem);
5590   ins_cost(125);
5591 
5592   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5593   ins_encode %{
5594     __ prefetchw($mem$$Address);
5595   %}
5596   ins_pipe(ialu_mem);
5597 %}
5598 
5599 instruct prefetchAllocNTA( memory mem ) %{
5600   predicate(AllocatePrefetchInstr==0);
5601   match(PrefetchAllocation mem);
5602   ins_cost(125);
5603 
5604   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5605   ins_encode %{
5606     __ prefetchnta($mem$$Address);
5607   %}
5608   ins_pipe(ialu_mem);
5609 %}
5610 
5611 instruct prefetchAllocT0( memory mem ) %{
5612   predicate(AllocatePrefetchInstr==1);
5613   match(PrefetchAllocation mem);
5614   ins_cost(125);
5615 
5616   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5617   ins_encode %{
5618     __ prefetcht0($mem$$Address);
5619   %}
5620   ins_pipe(ialu_mem);
5621 %}
5622 
5623 instruct prefetchAllocT2( memory mem ) %{
5624   predicate(AllocatePrefetchInstr==2);
5625   match(PrefetchAllocation mem);
5626   ins_cost(125);
5627 
5628   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5629   ins_encode %{
5630     __ prefetcht2($mem$$Address);
5631   %}
5632   ins_pipe(ialu_mem);
5633 %}
5634 
5635 //----------Store Instructions-------------------------------------------------
5636 
5637 // Store Byte
5638 instruct storeB(memory mem, rRegI src)
5639 %{
5640   match(Set mem (StoreB mem src));
5641 
5642   ins_cost(125); // XXX
5643   format %{ "movb    $mem, $src\t# byte" %}
5644   opcode(0x88);
5645   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5646   ins_pipe(ialu_mem_reg);
5647 %}
5648 
5649 // Store Char/Short
5650 instruct storeC(memory mem, rRegI src)
5651 %{
5652   match(Set mem (StoreC mem src));
5653 
5654   ins_cost(125); // XXX
5655   format %{ "movw    $mem, $src\t# char/short" %}
5656   opcode(0x89);
5657   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5658   ins_pipe(ialu_mem_reg);
5659 %}
5660 
5661 // Store Integer
5662 instruct storeI(memory mem, rRegI src)
5663 %{
5664   match(Set mem (StoreI mem src));
5665 
5666   ins_cost(125); // XXX
5667   format %{ "movl    $mem, $src\t# int" %}
5668   opcode(0x89);
5669   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5670   ins_pipe(ialu_mem_reg);
5671 %}
5672 
5673 // Store Long
5674 instruct storeL(memory mem, rRegL src)
5675 %{
5676   match(Set mem (StoreL mem src));
5677 
5678   ins_cost(125); // XXX
5679   format %{ "movq    $mem, $src\t# long" %}
5680   opcode(0x89);
5681   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5682   ins_pipe(ialu_mem_reg); // XXX
5683 %}
5684 
5685 // Store Pointer
5686 instruct storeP(memory mem, any_RegP src)
5687 %{
5688   match(Set mem (StoreP mem src));
5689 
5690   ins_cost(125); // XXX
5691   format %{ "movq    $mem, $src\t# ptr" %}
5692   opcode(0x89);
5693   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5694   ins_pipe(ialu_mem_reg);
5695 %}
5696 
5697 instruct storeImmP0(memory mem, immP0 zero)
5698 %{
5699   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5700   match(Set mem (StoreP mem zero));
5701 
5702   ins_cost(125); // XXX
5703   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5704   ins_encode %{
5705     __ movq($mem$$Address, r12);
5706   %}
5707   ins_pipe(ialu_mem_reg);
5708 %}
5709 
5710 // Store NULL Pointer, mark word, or other simple pointer constant.
5711 instruct storeImmP(memory mem, immP31 src)
5712 %{
5713   match(Set mem (StoreP mem src));
5714 
5715   ins_cost(150); // XXX
5716   format %{ "movq    $mem, $src\t# ptr" %}
5717   opcode(0xC7); /* C7 /0 */
5718   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5719   ins_pipe(ialu_mem_imm);
5720 %}
5721 
5722 // Store Compressed Pointer
5723 instruct storeN(memory mem, rRegN src)
5724 %{
5725   match(Set mem (StoreN mem src));
5726 
5727   ins_cost(125); // XXX
5728   format %{ "movl    $mem, $src\t# compressed ptr" %}
5729   ins_encode %{
5730     __ movl($mem$$Address, $src$$Register);
5731   %}
5732   ins_pipe(ialu_mem_reg);
5733 %}
5734 
5735 instruct storeNKlass(memory mem, rRegN src)
5736 %{
5737   match(Set mem (StoreNKlass mem src));
5738 
5739   ins_cost(125); // XXX
5740   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5741   ins_encode %{
5742     __ movl($mem$$Address, $src$$Register);
5743   %}
5744   ins_pipe(ialu_mem_reg);
5745 %}
5746 
5747 instruct storeImmN0(memory mem, immN0 zero)
5748 %{
5749   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5750   match(Set mem (StoreN mem zero));
5751 
5752   ins_cost(125); // XXX
5753   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5754   ins_encode %{
5755     __ movl($mem$$Address, r12);
5756   %}
5757   ins_pipe(ialu_mem_reg);
5758 %}
5759 
5760 instruct storeImmN(memory mem, immN src)
5761 %{
5762   match(Set mem (StoreN mem src));
5763 
5764   ins_cost(150); // XXX
5765   format %{ "movl    $mem, $src\t# compressed ptr" %}
5766   ins_encode %{
5767     address con = (address)$src$$constant;
5768     if (con == NULL) {
5769       __ movl($mem$$Address, (int32_t)0);
5770     } else {
5771       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5772     }
5773   %}
5774   ins_pipe(ialu_mem_imm);
5775 %}
5776 
5777 instruct storeImmNKlass(memory mem, immNKlass src)
5778 %{
5779   match(Set mem (StoreNKlass mem src));
5780 
5781   ins_cost(150); // XXX
5782   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5783   ins_encode %{
5784     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5785   %}
5786   ins_pipe(ialu_mem_imm);
5787 %}
5788 
5789 // Store Integer Immediate
5790 instruct storeImmI0(memory mem, immI0 zero)
5791 %{
5792   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5793   match(Set mem (StoreI mem zero));
5794 
5795   ins_cost(125); // XXX
5796   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5797   ins_encode %{
5798     __ movl($mem$$Address, r12);
5799   %}
5800   ins_pipe(ialu_mem_reg);
5801 %}
5802 
5803 instruct storeImmI(memory mem, immI src)
5804 %{
5805   match(Set mem (StoreI mem src));
5806 
5807   ins_cost(150);
5808   format %{ "movl    $mem, $src\t# int" %}
5809   opcode(0xC7); /* C7 /0 */
5810   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5811   ins_pipe(ialu_mem_imm);
5812 %}
5813 
5814 // Store Long Immediate
5815 instruct storeImmL0(memory mem, immL0 zero)
5816 %{
5817   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5818   match(Set mem (StoreL mem zero));
5819 
5820   ins_cost(125); // XXX
5821   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5822   ins_encode %{
5823     __ movq($mem$$Address, r12);
5824   %}
5825   ins_pipe(ialu_mem_reg);
5826 %}
5827 
5828 instruct storeImmL(memory mem, immL32 src)
5829 %{
5830   match(Set mem (StoreL mem src));
5831 
5832   ins_cost(150);
5833   format %{ "movq    $mem, $src\t# long" %}
5834   opcode(0xC7); /* C7 /0 */
5835   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5836   ins_pipe(ialu_mem_imm);
5837 %}
5838 
5839 // Store Short/Char Immediate
5840 instruct storeImmC0(memory mem, immI0 zero)
5841 %{
5842   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5843   match(Set mem (StoreC mem zero));
5844 
5845   ins_cost(125); // XXX
5846   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5847   ins_encode %{
5848     __ movw($mem$$Address, r12);
5849   %}
5850   ins_pipe(ialu_mem_reg);
5851 %}
5852 
5853 instruct storeImmI16(memory mem, immI16 src)
5854 %{
5855   predicate(UseStoreImmI16);
5856   match(Set mem (StoreC mem src));
5857 
5858   ins_cost(150);
5859   format %{ "movw    $mem, $src\t# short/char" %}
5860   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5861   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5862   ins_pipe(ialu_mem_imm);
5863 %}
5864 
5865 // Store Byte Immediate
5866 instruct storeImmB0(memory mem, immI0 zero)
5867 %{
5868   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5869   match(Set mem (StoreB mem zero));
5870 
5871   ins_cost(125); // XXX
5872   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5873   ins_encode %{
5874     __ movb($mem$$Address, r12);
5875   %}
5876   ins_pipe(ialu_mem_reg);
5877 %}
5878 
5879 instruct storeImmB(memory mem, immI8 src)
5880 %{
5881   match(Set mem (StoreB mem src));
5882 
5883   ins_cost(150); // XXX
5884   format %{ "movb    $mem, $src\t# byte" %}
5885   opcode(0xC6); /* C6 /0 */
5886   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5887   ins_pipe(ialu_mem_imm);
5888 %}
5889 
5890 // Store CMS card-mark Immediate
5891 instruct storeImmCM0_reg(memory mem, immI0 zero)
5892 %{
5893   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5894   match(Set mem (StoreCM mem zero));
5895 
5896   ins_cost(125); // XXX
5897   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5898   ins_encode %{
5899     __ movb($mem$$Address, r12);
5900   %}
5901   ins_pipe(ialu_mem_reg);
5902 %}
5903 
5904 instruct storeImmCM0(memory mem, immI0 src)
5905 %{
5906   match(Set mem (StoreCM mem src));
5907 
5908   ins_cost(150); // XXX
5909   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5910   opcode(0xC6); /* C6 /0 */
5911   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5912   ins_pipe(ialu_mem_imm);
5913 %}
5914 
5915 // Store Float
5916 instruct storeF(memory mem, regF src)
5917 %{
5918   match(Set mem (StoreF mem src));
5919 
5920   ins_cost(95); // XXX
5921   format %{ "movss   $mem, $src\t# float" %}
5922   ins_encode %{
5923     __ movflt($mem$$Address, $src$$XMMRegister);
5924   %}
5925   ins_pipe(pipe_slow); // XXX
5926 %}
5927 
5928 // Store immediate Float value (it is faster than store from XMM register)
5929 instruct storeF0(memory mem, immF0 zero)
5930 %{
5931   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5932   match(Set mem (StoreF mem zero));
5933 
5934   ins_cost(25); // XXX
5935   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5936   ins_encode %{
5937     __ movl($mem$$Address, r12);
5938   %}
5939   ins_pipe(ialu_mem_reg);
5940 %}
5941 
5942 instruct storeF_imm(memory mem, immF src)
5943 %{
5944   match(Set mem (StoreF mem src));
5945 
5946   ins_cost(50);
5947   format %{ "movl    $mem, $src\t# float" %}
5948   opcode(0xC7); /* C7 /0 */
5949   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5950   ins_pipe(ialu_mem_imm);
5951 %}
5952 
5953 // Store Double
5954 instruct storeD(memory mem, regD src)
5955 %{
5956   match(Set mem (StoreD mem src));
5957 
5958   ins_cost(95); // XXX
5959   format %{ "movsd   $mem, $src\t# double" %}
5960   ins_encode %{
5961     __ movdbl($mem$$Address, $src$$XMMRegister);
5962   %}
5963   ins_pipe(pipe_slow); // XXX
5964 %}
5965 
5966 // Store immediate double 0.0 (it is faster than store from XMM register)
5967 instruct storeD0_imm(memory mem, immD0 src)
5968 %{
5969   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5970   match(Set mem (StoreD mem src));
5971 
5972   ins_cost(50);
5973   format %{ "movq    $mem, $src\t# double 0." %}
5974   opcode(0xC7); /* C7 /0 */
5975   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5976   ins_pipe(ialu_mem_imm);
5977 %}
5978 
5979 instruct storeD0(memory mem, immD0 zero)
5980 %{
5981   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5982   match(Set mem (StoreD mem zero));
5983 
5984   ins_cost(25); // XXX
5985   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5986   ins_encode %{
5987     __ movq($mem$$Address, r12);
5988   %}
5989   ins_pipe(ialu_mem_reg);
5990 %}
5991 
5992 instruct storeSSI(stackSlotI dst, rRegI src)
5993 %{
5994   match(Set dst src);
5995 
5996   ins_cost(100);
5997   format %{ "movl    $dst, $src\t# int stk" %}
5998   opcode(0x89);
5999   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6000   ins_pipe( ialu_mem_reg );
6001 %}
6002 
6003 instruct storeSSL(stackSlotL dst, rRegL src)
6004 %{
6005   match(Set dst src);
6006 
6007   ins_cost(100);
6008   format %{ "movq    $dst, $src\t# long stk" %}
6009   opcode(0x89);
6010   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6011   ins_pipe(ialu_mem_reg);
6012 %}
6013 
6014 instruct storeSSP(stackSlotP dst, rRegP src)
6015 %{
6016   match(Set dst src);
6017 
6018   ins_cost(100);
6019   format %{ "movq    $dst, $src\t# ptr stk" %}
6020   opcode(0x89);
6021   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6022   ins_pipe(ialu_mem_reg);
6023 %}
6024 
6025 instruct storeSSF(stackSlotF dst, regF src)
6026 %{
6027   match(Set dst src);
6028 
6029   ins_cost(95); // XXX
6030   format %{ "movss   $dst, $src\t# float stk" %}
6031   ins_encode %{
6032     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6033   %}
6034   ins_pipe(pipe_slow); // XXX
6035 %}
6036 
6037 instruct storeSSD(stackSlotD dst, regD src)
6038 %{
6039   match(Set dst src);
6040 
6041   ins_cost(95); // XXX
6042   format %{ "movsd   $dst, $src\t# double stk" %}
6043   ins_encode %{
6044     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6045   %}
6046   ins_pipe(pipe_slow); // XXX
6047 %}
6048 
6049 //----------BSWAP Instructions-------------------------------------------------
6050 instruct bytes_reverse_int(rRegI dst) %{
6051   match(Set dst (ReverseBytesI dst));
6052 
6053   format %{ "bswapl  $dst" %}
6054   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6055   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6056   ins_pipe( ialu_reg );
6057 %}
6058 
6059 instruct bytes_reverse_long(rRegL dst) %{
6060   match(Set dst (ReverseBytesL dst));
6061 
6062   format %{ "bswapq  $dst" %}
6063   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6064   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6065   ins_pipe( ialu_reg);
6066 %}
6067 
6068 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6069   match(Set dst (ReverseBytesUS dst));
6070   effect(KILL cr);
6071 
6072   format %{ "bswapl  $dst\n\t"
6073             "shrl    $dst,16\n\t" %}
6074   ins_encode %{
6075     __ bswapl($dst$$Register);
6076     __ shrl($dst$$Register, 16);
6077   %}
6078   ins_pipe( ialu_reg );
6079 %}
6080 
6081 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6082   match(Set dst (ReverseBytesS dst));
6083   effect(KILL cr);
6084 
6085   format %{ "bswapl  $dst\n\t"
6086             "sar     $dst,16\n\t" %}
6087   ins_encode %{
6088     __ bswapl($dst$$Register);
6089     __ sarl($dst$$Register, 16);
6090   %}
6091   ins_pipe( ialu_reg );
6092 %}
6093 
6094 //---------- Zeros Count Instructions ------------------------------------------
6095 
6096 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6097   predicate(UseCountLeadingZerosInstruction);
6098   match(Set dst (CountLeadingZerosI src));
6099   effect(KILL cr);
6100 
6101   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6102   ins_encode %{
6103     __ lzcntl($dst$$Register, $src$$Register);
6104   %}
6105   ins_pipe(ialu_reg);
6106 %}
6107 
6108 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6109   predicate(!UseCountLeadingZerosInstruction);
6110   match(Set dst (CountLeadingZerosI src));
6111   effect(KILL cr);
6112 
6113   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6114             "jnz     skip\n\t"
6115             "movl    $dst, -1\n"
6116       "skip:\n\t"
6117             "negl    $dst\n\t"
6118             "addl    $dst, 31" %}
6119   ins_encode %{
6120     Register Rdst = $dst$$Register;
6121     Register Rsrc = $src$$Register;
6122     Label skip;
6123     __ bsrl(Rdst, Rsrc);
6124     __ jccb(Assembler::notZero, skip);
6125     __ movl(Rdst, -1);
6126     __ bind(skip);
6127     __ negl(Rdst);
6128     __ addl(Rdst, BitsPerInt - 1);
6129   %}
6130   ins_pipe(ialu_reg);
6131 %}
6132 
6133 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6134   predicate(UseCountLeadingZerosInstruction);
6135   match(Set dst (CountLeadingZerosL src));
6136   effect(KILL cr);
6137 
6138   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6139   ins_encode %{
6140     __ lzcntq($dst$$Register, $src$$Register);
6141   %}
6142   ins_pipe(ialu_reg);
6143 %}
6144 
6145 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6146   predicate(!UseCountLeadingZerosInstruction);
6147   match(Set dst (CountLeadingZerosL src));
6148   effect(KILL cr);
6149 
6150   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6151             "jnz     skip\n\t"
6152             "movl    $dst, -1\n"
6153       "skip:\n\t"
6154             "negl    $dst\n\t"
6155             "addl    $dst, 63" %}
6156   ins_encode %{
6157     Register Rdst = $dst$$Register;
6158     Register Rsrc = $src$$Register;
6159     Label skip;
6160     __ bsrq(Rdst, Rsrc);
6161     __ jccb(Assembler::notZero, skip);
6162     __ movl(Rdst, -1);
6163     __ bind(skip);
6164     __ negl(Rdst);
6165     __ addl(Rdst, BitsPerLong - 1);
6166   %}
6167   ins_pipe(ialu_reg);
6168 %}
6169 
6170 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6171   predicate(UseCountTrailingZerosInstruction);
6172   match(Set dst (CountTrailingZerosI src));
6173   effect(KILL cr);
6174 
6175   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6176   ins_encode %{
6177     __ tzcntl($dst$$Register, $src$$Register);
6178   %}
6179   ins_pipe(ialu_reg);
6180 %}
6181 
6182 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6183   predicate(!UseCountTrailingZerosInstruction);
6184   match(Set dst (CountTrailingZerosI src));
6185   effect(KILL cr);
6186 
6187   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6188             "jnz     done\n\t"
6189             "movl    $dst, 32\n"
6190       "done:" %}
6191   ins_encode %{
6192     Register Rdst = $dst$$Register;
6193     Label done;
6194     __ bsfl(Rdst, $src$$Register);
6195     __ jccb(Assembler::notZero, done);
6196     __ movl(Rdst, BitsPerInt);
6197     __ bind(done);
6198   %}
6199   ins_pipe(ialu_reg);
6200 %}
6201 
6202 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6203   predicate(UseCountTrailingZerosInstruction);
6204   match(Set dst (CountTrailingZerosL src));
6205   effect(KILL cr);
6206 
6207   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6208   ins_encode %{
6209     __ tzcntq($dst$$Register, $src$$Register);
6210   %}
6211   ins_pipe(ialu_reg);
6212 %}
6213 
6214 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6215   predicate(!UseCountTrailingZerosInstruction);
6216   match(Set dst (CountTrailingZerosL src));
6217   effect(KILL cr);
6218 
6219   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6220             "jnz     done\n\t"
6221             "movl    $dst, 64\n"
6222       "done:" %}
6223   ins_encode %{
6224     Register Rdst = $dst$$Register;
6225     Label done;
6226     __ bsfq(Rdst, $src$$Register);
6227     __ jccb(Assembler::notZero, done);
6228     __ movl(Rdst, BitsPerLong);
6229     __ bind(done);
6230   %}
6231   ins_pipe(ialu_reg);
6232 %}
6233 
6234 
6235 //---------- Population Count Instructions -------------------------------------
6236 
6237 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6238   predicate(UsePopCountInstruction);
6239   match(Set dst (PopCountI src));
6240   effect(KILL cr);
6241 
6242   format %{ "popcnt  $dst, $src" %}
6243   ins_encode %{
6244     __ popcntl($dst$$Register, $src$$Register);
6245   %}
6246   ins_pipe(ialu_reg);
6247 %}
6248 
6249 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6250   predicate(UsePopCountInstruction);
6251   match(Set dst (PopCountI (LoadI mem)));
6252   effect(KILL cr);
6253 
6254   format %{ "popcnt  $dst, $mem" %}
6255   ins_encode %{
6256     __ popcntl($dst$$Register, $mem$$Address);
6257   %}
6258   ins_pipe(ialu_reg);
6259 %}
6260 
6261 // Note: Long.bitCount(long) returns an int.
6262 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6263   predicate(UsePopCountInstruction);
6264   match(Set dst (PopCountL src));
6265   effect(KILL cr);
6266 
6267   format %{ "popcnt  $dst, $src" %}
6268   ins_encode %{
6269     __ popcntq($dst$$Register, $src$$Register);
6270   %}
6271   ins_pipe(ialu_reg);
6272 %}
6273 
6274 // Note: Long.bitCount(long) returns an int.
6275 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6276   predicate(UsePopCountInstruction);
6277   match(Set dst (PopCountL (LoadL mem)));
6278   effect(KILL cr);
6279 
6280   format %{ "popcnt  $dst, $mem" %}
6281   ins_encode %{
6282     __ popcntq($dst$$Register, $mem$$Address);
6283   %}
6284   ins_pipe(ialu_reg);
6285 %}
6286 
6287 
6288 //----------MemBar Instructions-----------------------------------------------
6289 // Memory barrier flavors
6290 
6291 instruct membar_acquire()
6292 %{
6293   match(MemBarAcquire);
6294   match(LoadFence);
6295   ins_cost(0);
6296 
6297   size(0);
6298   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6299   ins_encode();
6300   ins_pipe(empty);
6301 %}
6302 
6303 instruct membar_acquire_lock()
6304 %{
6305   match(MemBarAcquireLock);
6306   ins_cost(0);
6307 
6308   size(0);
6309   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6310   ins_encode();
6311   ins_pipe(empty);
6312 %}
6313 
6314 instruct membar_release()
6315 %{
6316   match(MemBarRelease);
6317   match(StoreFence);
6318   ins_cost(0);
6319 
6320   size(0);
6321   format %{ "MEMBAR-release ! (empty encoding)" %}
6322   ins_encode();
6323   ins_pipe(empty);
6324 %}
6325 
6326 instruct membar_release_lock()
6327 %{
6328   match(MemBarReleaseLock);
6329   ins_cost(0);
6330 
6331   size(0);
6332   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6333   ins_encode();
6334   ins_pipe(empty);
6335 %}
6336 
6337 instruct membar_volatile(rFlagsReg cr) %{
6338   match(MemBarVolatile);
6339   effect(KILL cr);
6340   ins_cost(400);
6341 
6342   format %{
6343     $$template
6344     if (os::is_MP()) {
6345       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6346     } else {
6347       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6348     }
6349   %}
6350   ins_encode %{
6351     __ membar(Assembler::StoreLoad);
6352   %}
6353   ins_pipe(pipe_slow);
6354 %}
6355 
6356 instruct unnecessary_membar_volatile()
6357 %{
6358   match(MemBarVolatile);
6359   predicate(Matcher::post_store_load_barrier(n));
6360   ins_cost(0);
6361 
6362   size(0);
6363   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6364   ins_encode();
6365   ins_pipe(empty);
6366 %}
6367 
6368 instruct membar_storestore() %{
6369   match(MemBarStoreStore);
6370   ins_cost(0);
6371 
6372   size(0);
6373   format %{ "MEMBAR-storestore (empty encoding)" %}
6374   ins_encode( );
6375   ins_pipe(empty);
6376 %}
6377 
6378 //----------Move Instructions--------------------------------------------------
6379 
6380 instruct castX2P(rRegP dst, rRegL src)
6381 %{
6382   match(Set dst (CastX2P src));
6383 
6384   format %{ "movq    $dst, $src\t# long->ptr" %}
6385   ins_encode %{
6386     if ($dst$$reg != $src$$reg) {
6387       __ movptr($dst$$Register, $src$$Register);
6388     }
6389   %}
6390   ins_pipe(ialu_reg_reg); // XXX
6391 %}
6392 
6393 instruct castP2X(rRegL dst, rRegP src)
6394 %{
6395   match(Set dst (CastP2X src));
6396 
6397   format %{ "movq    $dst, $src\t# ptr -> long" %}
6398   ins_encode %{
6399     if ($dst$$reg != $src$$reg) {
6400       __ movptr($dst$$Register, $src$$Register);
6401     }
6402   %}
6403   ins_pipe(ialu_reg_reg); // XXX
6404 %}
6405 
6406 // Convert oop into int for vectors alignment masking
6407 instruct convP2I(rRegI dst, rRegP src)
6408 %{
6409   match(Set dst (ConvL2I (CastP2X src)));
6410 
6411   format %{ "movl    $dst, $src\t# ptr -> int" %}
6412   ins_encode %{
6413     __ movl($dst$$Register, $src$$Register);
6414   %}
6415   ins_pipe(ialu_reg_reg); // XXX
6416 %}
6417 
6418 // Convert compressed oop into int for vectors alignment masking
6419 // in case of 32bit oops (heap < 4Gb).
6420 instruct convN2I(rRegI dst, rRegN src)
6421 %{
6422   predicate(Universe::narrow_oop_shift() == 0);
6423   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6424 
6425   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6426   ins_encode %{
6427     __ movl($dst$$Register, $src$$Register);
6428   %}
6429   ins_pipe(ialu_reg_reg); // XXX
6430 %}
6431 
6432 // Convert oop pointer into compressed form
6433 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6434   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6435   match(Set dst (EncodeP src));
6436   effect(KILL cr);
6437   format %{ "encode_heap_oop $dst,$src" %}
6438   ins_encode %{
6439     Register s = $src$$Register;
6440     Register d = $dst$$Register;
6441     if (s != d) {
6442       __ movq(d, s);
6443     }
6444     __ encode_heap_oop(d);
6445   %}
6446   ins_pipe(ialu_reg_long);
6447 %}
6448 
6449 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6450   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6451   match(Set dst (EncodeP src));
6452   effect(KILL cr);
6453   format %{ "encode_heap_oop_not_null $dst,$src" %}
6454   ins_encode %{
6455     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6456   %}
6457   ins_pipe(ialu_reg_long);
6458 %}
6459 
6460 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6461   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6462             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6463   match(Set dst (DecodeN src));
6464   effect(KILL cr);
6465   format %{ "decode_heap_oop $dst,$src" %}
6466   ins_encode %{
6467     Register s = $src$$Register;
6468     Register d = $dst$$Register;
6469     if (s != d) {
6470       __ movq(d, s);
6471     }
6472     __ decode_heap_oop(d);
6473   %}
6474   ins_pipe(ialu_reg_long);
6475 %}
6476 
6477 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6478   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6479             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6480   match(Set dst (DecodeN src));
6481   effect(KILL cr);
6482   format %{ "decode_heap_oop_not_null $dst,$src" %}
6483   ins_encode %{
6484     Register s = $src$$Register;
6485     Register d = $dst$$Register;
6486     if (s != d) {
6487       __ decode_heap_oop_not_null(d, s);
6488     } else {
6489       __ decode_heap_oop_not_null(d);
6490     }
6491   %}
6492   ins_pipe(ialu_reg_long);
6493 %}
6494 
6495 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6496   match(Set dst (EncodePKlass src));
6497   effect(KILL cr);
6498   format %{ "encode_klass_not_null $dst,$src" %}
6499   ins_encode %{
6500     __ encode_klass_not_null($dst$$Register, $src$$Register);
6501   %}
6502   ins_pipe(ialu_reg_long);
6503 %}
6504 
6505 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6506   match(Set dst (DecodeNKlass src));
6507   effect(KILL cr);
6508   format %{ "decode_klass_not_null $dst,$src" %}
6509   ins_encode %{
6510     Register s = $src$$Register;
6511     Register d = $dst$$Register;
6512     if (s != d) {
6513       __ decode_klass_not_null(d, s);
6514     } else {
6515       __ decode_klass_not_null(d);
6516     }
6517   %}
6518   ins_pipe(ialu_reg_long);
6519 %}
6520 
6521 
6522 //----------Conditional Move---------------------------------------------------
6523 // Jump
6524 // dummy instruction for generating temp registers
6525 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6526   match(Jump (LShiftL switch_val shift));
6527   ins_cost(350);
6528   predicate(false);
6529   effect(TEMP dest);
6530 
6531   format %{ "leaq    $dest, [$constantaddress]\n\t"
6532             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6533   ins_encode %{
6534     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6535     // to do that and the compiler is using that register as one it can allocate.
6536     // So we build it all by hand.
6537     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6538     // ArrayAddress dispatch(table, index);
6539     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6540     __ lea($dest$$Register, $constantaddress);
6541     __ jmp(dispatch);
6542   %}
6543   ins_pipe(pipe_jmp);
6544 %}
6545 
6546 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6547   match(Jump (AddL (LShiftL switch_val shift) offset));
6548   ins_cost(350);
6549   effect(TEMP dest);
6550 
6551   format %{ "leaq    $dest, [$constantaddress]\n\t"
6552             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6553   ins_encode %{
6554     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6555     // to do that and the compiler is using that register as one it can allocate.
6556     // So we build it all by hand.
6557     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6558     // ArrayAddress dispatch(table, index);
6559     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6560     __ lea($dest$$Register, $constantaddress);
6561     __ jmp(dispatch);
6562   %}
6563   ins_pipe(pipe_jmp);
6564 %}
6565 
6566 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6567   match(Jump switch_val);
6568   ins_cost(350);
6569   effect(TEMP dest);
6570 
6571   format %{ "leaq    $dest, [$constantaddress]\n\t"
6572             "jmp     [$dest + $switch_val]\n\t" %}
6573   ins_encode %{
6574     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6575     // to do that and the compiler is using that register as one it can allocate.
6576     // So we build it all by hand.
6577     // Address index(noreg, switch_reg, Address::times_1);
6578     // ArrayAddress dispatch(table, index);
6579     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6580     __ lea($dest$$Register, $constantaddress);
6581     __ jmp(dispatch);
6582   %}
6583   ins_pipe(pipe_jmp);
6584 %}
6585 
6586 // Conditional move
6587 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6588 %{
6589   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6590 
6591   ins_cost(200); // XXX
6592   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6593   opcode(0x0F, 0x40);
6594   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6595   ins_pipe(pipe_cmov_reg);
6596 %}
6597 
6598 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6599   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6600 
6601   ins_cost(200); // XXX
6602   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6603   opcode(0x0F, 0x40);
6604   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6605   ins_pipe(pipe_cmov_reg);
6606 %}
6607 
6608 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6609   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6610   ins_cost(200);
6611   expand %{
6612     cmovI_regU(cop, cr, dst, src);
6613   %}
6614 %}
6615 
6616 // Conditional move
6617 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6618   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6619 
6620   ins_cost(250); // XXX
6621   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6622   opcode(0x0F, 0x40);
6623   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6624   ins_pipe(pipe_cmov_mem);
6625 %}
6626 
6627 // Conditional move
6628 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6629 %{
6630   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6631 
6632   ins_cost(250); // XXX
6633   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6634   opcode(0x0F, 0x40);
6635   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6636   ins_pipe(pipe_cmov_mem);
6637 %}
6638 
6639 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6640   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6641   ins_cost(250);
6642   expand %{
6643     cmovI_memU(cop, cr, dst, src);
6644   %}
6645 %}
6646 
6647 // Conditional move
6648 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6649 %{
6650   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6651 
6652   ins_cost(200); // XXX
6653   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6654   opcode(0x0F, 0x40);
6655   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6656   ins_pipe(pipe_cmov_reg);
6657 %}
6658 
6659 // Conditional move
6660 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6661 %{
6662   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6663 
6664   ins_cost(200); // XXX
6665   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6666   opcode(0x0F, 0x40);
6667   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6668   ins_pipe(pipe_cmov_reg);
6669 %}
6670 
6671 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6672   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6673   ins_cost(200);
6674   expand %{
6675     cmovN_regU(cop, cr, dst, src);
6676   %}
6677 %}
6678 
6679 // Conditional move
6680 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6681 %{
6682   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6683 
6684   ins_cost(200); // XXX
6685   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6686   opcode(0x0F, 0x40);
6687   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6688   ins_pipe(pipe_cmov_reg);  // XXX
6689 %}
6690 
6691 // Conditional move
6692 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6693 %{
6694   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6695 
6696   ins_cost(200); // XXX
6697   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6698   opcode(0x0F, 0x40);
6699   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6700   ins_pipe(pipe_cmov_reg); // XXX
6701 %}
6702 
6703 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6704   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6705   ins_cost(200);
6706   expand %{
6707     cmovP_regU(cop, cr, dst, src);
6708   %}
6709 %}
6710 
6711 // DISABLED: Requires the ADLC to emit a bottom_type call that
6712 // correctly meets the two pointer arguments; one is an incoming
6713 // register but the other is a memory operand.  ALSO appears to
6714 // be buggy with implicit null checks.
6715 //
6716 //// Conditional move
6717 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6718 //%{
6719 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6720 //  ins_cost(250);
6721 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6722 //  opcode(0x0F,0x40);
6723 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6724 //  ins_pipe( pipe_cmov_mem );
6725 //%}
6726 //
6727 //// Conditional move
6728 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6729 //%{
6730 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6731 //  ins_cost(250);
6732 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6733 //  opcode(0x0F,0x40);
6734 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6735 //  ins_pipe( pipe_cmov_mem );
6736 //%}
6737 
6738 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6739 %{
6740   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6741 
6742   ins_cost(200); // XXX
6743   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6744   opcode(0x0F, 0x40);
6745   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6746   ins_pipe(pipe_cmov_reg);  // XXX
6747 %}
6748 
6749 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6750 %{
6751   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6752 
6753   ins_cost(200); // XXX
6754   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6755   opcode(0x0F, 0x40);
6756   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6757   ins_pipe(pipe_cmov_mem);  // XXX
6758 %}
6759 
6760 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6761 %{
6762   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6763 
6764   ins_cost(200); // XXX
6765   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6766   opcode(0x0F, 0x40);
6767   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6768   ins_pipe(pipe_cmov_reg); // XXX
6769 %}
6770 
6771 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6772   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6773   ins_cost(200);
6774   expand %{
6775     cmovL_regU(cop, cr, dst, src);
6776   %}
6777 %}
6778 
6779 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6780 %{
6781   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6782 
6783   ins_cost(200); // XXX
6784   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6785   opcode(0x0F, 0x40);
6786   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6787   ins_pipe(pipe_cmov_mem); // XXX
6788 %}
6789 
6790 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6791   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6792   ins_cost(200);
6793   expand %{
6794     cmovL_memU(cop, cr, dst, src);
6795   %}
6796 %}
6797 
6798 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6799 %{
6800   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6801 
6802   ins_cost(200); // XXX
6803   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6804             "movss     $dst, $src\n"
6805     "skip:" %}
6806   ins_encode %{
6807     Label Lskip;
6808     // Invert sense of branch from sense of CMOV
6809     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6810     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6811     __ bind(Lskip);
6812   %}
6813   ins_pipe(pipe_slow);
6814 %}
6815 
6816 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6817 // %{
6818 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6819 
6820 //   ins_cost(200); // XXX
6821 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6822 //             "movss     $dst, $src\n"
6823 //     "skip:" %}
6824 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6825 //   ins_pipe(pipe_slow);
6826 // %}
6827 
6828 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6829 %{
6830   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6831 
6832   ins_cost(200); // XXX
6833   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6834             "movss     $dst, $src\n"
6835     "skip:" %}
6836   ins_encode %{
6837     Label Lskip;
6838     // Invert sense of branch from sense of CMOV
6839     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6840     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6841     __ bind(Lskip);
6842   %}
6843   ins_pipe(pipe_slow);
6844 %}
6845 
6846 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6847   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6848   ins_cost(200);
6849   expand %{
6850     cmovF_regU(cop, cr, dst, src);
6851   %}
6852 %}
6853 
6854 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6855 %{
6856   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6857 
6858   ins_cost(200); // XXX
6859   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6860             "movsd     $dst, $src\n"
6861     "skip:" %}
6862   ins_encode %{
6863     Label Lskip;
6864     // Invert sense of branch from sense of CMOV
6865     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6866     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6867     __ bind(Lskip);
6868   %}
6869   ins_pipe(pipe_slow);
6870 %}
6871 
6872 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6873 %{
6874   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6875 
6876   ins_cost(200); // XXX
6877   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6878             "movsd     $dst, $src\n"
6879     "skip:" %}
6880   ins_encode %{
6881     Label Lskip;
6882     // Invert sense of branch from sense of CMOV
6883     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6884     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6885     __ bind(Lskip);
6886   %}
6887   ins_pipe(pipe_slow);
6888 %}
6889 
6890 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6891   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6892   ins_cost(200);
6893   expand %{
6894     cmovD_regU(cop, cr, dst, src);
6895   %}
6896 %}
6897 
6898 //----------Arithmetic Instructions--------------------------------------------
6899 //----------Addition Instructions----------------------------------------------
6900 
6901 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6902 %{
6903   match(Set dst (AddI dst src));
6904   effect(KILL cr);
6905 
6906   format %{ "addl    $dst, $src\t# int" %}
6907   opcode(0x03);
6908   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6909   ins_pipe(ialu_reg_reg);
6910 %}
6911 
6912 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6913 %{
6914   match(Set dst (AddI dst src));
6915   effect(KILL cr);
6916 
6917   format %{ "addl    $dst, $src\t# int" %}
6918   opcode(0x81, 0x00); /* /0 id */
6919   ins_encode(OpcSErm(dst, src), Con8or32(src));
6920   ins_pipe( ialu_reg );
6921 %}
6922 
6923 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6924 %{
6925   match(Set dst (AddI dst (LoadI src)));
6926   effect(KILL cr);
6927 
6928   ins_cost(125); // XXX
6929   format %{ "addl    $dst, $src\t# int" %}
6930   opcode(0x03);
6931   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6932   ins_pipe(ialu_reg_mem);
6933 %}
6934 
6935 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6936 %{
6937   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6938   effect(KILL cr);
6939 
6940   ins_cost(150); // XXX
6941   format %{ "addl    $dst, $src\t# int" %}
6942   opcode(0x01); /* Opcode 01 /r */
6943   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6944   ins_pipe(ialu_mem_reg);
6945 %}
6946 
6947 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6948 %{
6949   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6950   effect(KILL cr);
6951 
6952   ins_cost(125); // XXX
6953   format %{ "addl    $dst, $src\t# int" %}
6954   opcode(0x81); /* Opcode 81 /0 id */
6955   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6956   ins_pipe(ialu_mem_imm);
6957 %}
6958 
6959 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6960 %{
6961   predicate(UseIncDec);
6962   match(Set dst (AddI dst src));
6963   effect(KILL cr);
6964 
6965   format %{ "incl    $dst\t# int" %}
6966   opcode(0xFF, 0x00); // FF /0
6967   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6968   ins_pipe(ialu_reg);
6969 %}
6970 
6971 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6972 %{
6973   predicate(UseIncDec);
6974   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6975   effect(KILL cr);
6976 
6977   ins_cost(125); // XXX
6978   format %{ "incl    $dst\t# int" %}
6979   opcode(0xFF); /* Opcode FF /0 */
6980   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6981   ins_pipe(ialu_mem_imm);
6982 %}
6983 
6984 // XXX why does that use AddI
6985 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6986 %{
6987   predicate(UseIncDec);
6988   match(Set dst (AddI dst src));
6989   effect(KILL cr);
6990 
6991   format %{ "decl    $dst\t# int" %}
6992   opcode(0xFF, 0x01); // FF /1
6993   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6994   ins_pipe(ialu_reg);
6995 %}
6996 
6997 // XXX why does that use AddI
6998 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6999 %{
7000   predicate(UseIncDec);
7001   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7002   effect(KILL cr);
7003 
7004   ins_cost(125); // XXX
7005   format %{ "decl    $dst\t# int" %}
7006   opcode(0xFF); /* Opcode FF /1 */
7007   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7008   ins_pipe(ialu_mem_imm);
7009 %}
7010 
7011 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7012 %{
7013   match(Set dst (AddI src0 src1));
7014 
7015   ins_cost(110);
7016   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7017   opcode(0x8D); /* 0x8D /r */
7018   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7019   ins_pipe(ialu_reg_reg);
7020 %}
7021 
7022 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7023 %{
7024   match(Set dst (AddL dst src));
7025   effect(KILL cr);
7026 
7027   format %{ "addq    $dst, $src\t# long" %}
7028   opcode(0x03);
7029   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7030   ins_pipe(ialu_reg_reg);
7031 %}
7032 
7033 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7034 %{
7035   match(Set dst (AddL dst src));
7036   effect(KILL cr);
7037 
7038   format %{ "addq    $dst, $src\t# long" %}
7039   opcode(0x81, 0x00); /* /0 id */
7040   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7041   ins_pipe( ialu_reg );
7042 %}
7043 
7044 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7045 %{
7046   match(Set dst (AddL dst (LoadL src)));
7047   effect(KILL cr);
7048 
7049   ins_cost(125); // XXX
7050   format %{ "addq    $dst, $src\t# long" %}
7051   opcode(0x03);
7052   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7053   ins_pipe(ialu_reg_mem);
7054 %}
7055 
7056 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7057 %{
7058   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7059   effect(KILL cr);
7060 
7061   ins_cost(150); // XXX
7062   format %{ "addq    $dst, $src\t# long" %}
7063   opcode(0x01); /* Opcode 01 /r */
7064   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7065   ins_pipe(ialu_mem_reg);
7066 %}
7067 
7068 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7069 %{
7070   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7071   effect(KILL cr);
7072 
7073   ins_cost(125); // XXX
7074   format %{ "addq    $dst, $src\t# long" %}
7075   opcode(0x81); /* Opcode 81 /0 id */
7076   ins_encode(REX_mem_wide(dst),
7077              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7078   ins_pipe(ialu_mem_imm);
7079 %}
7080 
7081 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7082 %{
7083   predicate(UseIncDec);
7084   match(Set dst (AddL dst src));
7085   effect(KILL cr);
7086 
7087   format %{ "incq    $dst\t# long" %}
7088   opcode(0xFF, 0x00); // FF /0
7089   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7090   ins_pipe(ialu_reg);
7091 %}
7092 
7093 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7094 %{
7095   predicate(UseIncDec);
7096   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7097   effect(KILL cr);
7098 
7099   ins_cost(125); // XXX
7100   format %{ "incq    $dst\t# long" %}
7101   opcode(0xFF); /* Opcode FF /0 */
7102   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7103   ins_pipe(ialu_mem_imm);
7104 %}
7105 
7106 // XXX why does that use AddL
7107 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7108 %{
7109   predicate(UseIncDec);
7110   match(Set dst (AddL dst src));
7111   effect(KILL cr);
7112 
7113   format %{ "decq    $dst\t# long" %}
7114   opcode(0xFF, 0x01); // FF /1
7115   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7116   ins_pipe(ialu_reg);
7117 %}
7118 
7119 // XXX why does that use AddL
7120 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7121 %{
7122   predicate(UseIncDec);
7123   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7124   effect(KILL cr);
7125 
7126   ins_cost(125); // XXX
7127   format %{ "decq    $dst\t# long" %}
7128   opcode(0xFF); /* Opcode FF /1 */
7129   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7130   ins_pipe(ialu_mem_imm);
7131 %}
7132 
7133 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7134 %{
7135   match(Set dst (AddL src0 src1));
7136 
7137   ins_cost(110);
7138   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7139   opcode(0x8D); /* 0x8D /r */
7140   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7141   ins_pipe(ialu_reg_reg);
7142 %}
7143 
7144 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7145 %{
7146   match(Set dst (AddP dst src));
7147   effect(KILL cr);
7148 
7149   format %{ "addq    $dst, $src\t# ptr" %}
7150   opcode(0x03);
7151   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7152   ins_pipe(ialu_reg_reg);
7153 %}
7154 
7155 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7156 %{
7157   match(Set dst (AddP dst src));
7158   effect(KILL cr);
7159 
7160   format %{ "addq    $dst, $src\t# ptr" %}
7161   opcode(0x81, 0x00); /* /0 id */
7162   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7163   ins_pipe( ialu_reg );
7164 %}
7165 
7166 // XXX addP mem ops ????
7167 
7168 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7169 %{
7170   match(Set dst (AddP src0 src1));
7171 
7172   ins_cost(110);
7173   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7174   opcode(0x8D); /* 0x8D /r */
7175   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7176   ins_pipe(ialu_reg_reg);
7177 %}
7178 
7179 instruct checkCastPP(rRegP dst)
7180 %{
7181   match(Set dst (CheckCastPP dst));
7182 
7183   size(0);
7184   format %{ "# checkcastPP of $dst" %}
7185   ins_encode(/* empty encoding */);
7186   ins_pipe(empty);
7187 %}
7188 
7189 instruct castPP(rRegP dst)
7190 %{
7191   match(Set dst (CastPP dst));
7192 
7193   size(0);
7194   format %{ "# castPP of $dst" %}
7195   ins_encode(/* empty encoding */);
7196   ins_pipe(empty);
7197 %}
7198 
7199 instruct castII(rRegI dst)
7200 %{
7201   match(Set dst (CastII dst));
7202 
7203   size(0);
7204   format %{ "# castII of $dst" %}
7205   ins_encode(/* empty encoding */);
7206   ins_cost(0);
7207   ins_pipe(empty);
7208 %}
7209 
7210 // LoadP-locked same as a regular LoadP when used with compare-swap
7211 instruct loadPLocked(rRegP dst, memory mem)
7212 %{
7213   match(Set dst (LoadPLocked mem));
7214 
7215   ins_cost(125); // XXX
7216   format %{ "movq    $dst, $mem\t# ptr locked" %}
7217   opcode(0x8B);
7218   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7219   ins_pipe(ialu_reg_mem); // XXX
7220 %}
7221 
7222 // Conditional-store of the updated heap-top.
7223 // Used during allocation of the shared heap.
7224 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7225 
7226 instruct storePConditional(memory heap_top_ptr,
7227                            rax_RegP oldval, rRegP newval,
7228                            rFlagsReg cr)
7229 %{
7230   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7231 
7232   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7233             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7234   opcode(0x0F, 0xB1);
7235   ins_encode(lock_prefix,
7236              REX_reg_mem_wide(newval, heap_top_ptr),
7237              OpcP, OpcS,
7238              reg_mem(newval, heap_top_ptr));
7239   ins_pipe(pipe_cmpxchg);
7240 %}
7241 
7242 // Conditional-store of an int value.
7243 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7244 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7245 %{
7246   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7247   effect(KILL oldval);
7248 
7249   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7250   opcode(0x0F, 0xB1);
7251   ins_encode(lock_prefix,
7252              REX_reg_mem(newval, mem),
7253              OpcP, OpcS,
7254              reg_mem(newval, mem));
7255   ins_pipe(pipe_cmpxchg);
7256 %}
7257 
7258 // Conditional-store of a long value.
7259 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7260 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7261 %{
7262   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7263   effect(KILL oldval);
7264 
7265   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7266   opcode(0x0F, 0xB1);
7267   ins_encode(lock_prefix,
7268              REX_reg_mem_wide(newval, mem),
7269              OpcP, OpcS,
7270              reg_mem(newval, mem));
7271   ins_pipe(pipe_cmpxchg);
7272 %}
7273 
7274 
7275 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7276 instruct compareAndSwapP(rRegI res,
7277                          memory mem_ptr,
7278                          rax_RegP oldval, rRegP newval,
7279                          rFlagsReg cr)
7280 %{
7281   predicate(VM_Version::supports_cx8());
7282   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7283   effect(KILL cr, KILL oldval);
7284 
7285   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7286             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7287             "sete    $res\n\t"
7288             "movzbl  $res, $res" %}
7289   opcode(0x0F, 0xB1);
7290   ins_encode(lock_prefix,
7291              REX_reg_mem_wide(newval, mem_ptr),
7292              OpcP, OpcS,
7293              reg_mem(newval, mem_ptr),
7294              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7295              REX_reg_breg(res, res), // movzbl
7296              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7297   ins_pipe( pipe_cmpxchg );
7298 %}
7299 
7300 instruct compareAndSwapL(rRegI res,
7301                          memory mem_ptr,
7302                          rax_RegL oldval, rRegL newval,
7303                          rFlagsReg cr)
7304 %{
7305   predicate(VM_Version::supports_cx8());
7306   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7307   effect(KILL cr, KILL oldval);
7308 
7309   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7310             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7311             "sete    $res\n\t"
7312             "movzbl  $res, $res" %}
7313   opcode(0x0F, 0xB1);
7314   ins_encode(lock_prefix,
7315              REX_reg_mem_wide(newval, mem_ptr),
7316              OpcP, OpcS,
7317              reg_mem(newval, mem_ptr),
7318              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7319              REX_reg_breg(res, res), // movzbl
7320              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7321   ins_pipe( pipe_cmpxchg );
7322 %}
7323 
7324 instruct compareAndSwapI(rRegI res,
7325                          memory mem_ptr,
7326                          rax_RegI oldval, rRegI newval,
7327                          rFlagsReg cr)
7328 %{
7329   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7330   effect(KILL cr, KILL oldval);
7331 
7332   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7333             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7334             "sete    $res\n\t"
7335             "movzbl  $res, $res" %}
7336   opcode(0x0F, 0xB1);
7337   ins_encode(lock_prefix,
7338              REX_reg_mem(newval, mem_ptr),
7339              OpcP, OpcS,
7340              reg_mem(newval, mem_ptr),
7341              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7342              REX_reg_breg(res, res), // movzbl
7343              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7344   ins_pipe( pipe_cmpxchg );
7345 %}
7346 
7347 
7348 instruct compareAndSwapN(rRegI res,
7349                           memory mem_ptr,
7350                           rax_RegN oldval, rRegN newval,
7351                           rFlagsReg cr) %{
7352   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7353   effect(KILL cr, KILL oldval);
7354 
7355   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7356             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7357             "sete    $res\n\t"
7358             "movzbl  $res, $res" %}
7359   opcode(0x0F, 0xB1);
7360   ins_encode(lock_prefix,
7361              REX_reg_mem(newval, mem_ptr),
7362              OpcP, OpcS,
7363              reg_mem(newval, mem_ptr),
7364              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7365              REX_reg_breg(res, res), // movzbl
7366              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7367   ins_pipe( pipe_cmpxchg );
7368 %}
7369 
7370 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7371   predicate(n->as_LoadStore()->result_not_used());
7372   match(Set dummy (GetAndAddI mem add));
7373   effect(KILL cr);
7374   format %{ "ADDL  [$mem],$add" %}
7375   ins_encode %{
7376     if (os::is_MP()) { __ lock(); }
7377     __ addl($mem$$Address, $add$$constant);
7378   %}
7379   ins_pipe( pipe_cmpxchg );
7380 %}
7381 
7382 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7383   match(Set newval (GetAndAddI mem newval));
7384   effect(KILL cr);
7385   format %{ "XADDL  [$mem],$newval" %}
7386   ins_encode %{
7387     if (os::is_MP()) { __ lock(); }
7388     __ xaddl($mem$$Address, $newval$$Register);
7389   %}
7390   ins_pipe( pipe_cmpxchg );
7391 %}
7392 
7393 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7394   predicate(n->as_LoadStore()->result_not_used());
7395   match(Set dummy (GetAndAddL mem add));
7396   effect(KILL cr);
7397   format %{ "ADDQ  [$mem],$add" %}
7398   ins_encode %{
7399     if (os::is_MP()) { __ lock(); }
7400     __ addq($mem$$Address, $add$$constant);
7401   %}
7402   ins_pipe( pipe_cmpxchg );
7403 %}
7404 
7405 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7406   match(Set newval (GetAndAddL mem newval));
7407   effect(KILL cr);
7408   format %{ "XADDQ  [$mem],$newval" %}
7409   ins_encode %{
7410     if (os::is_MP()) { __ lock(); }
7411     __ xaddq($mem$$Address, $newval$$Register);
7412   %}
7413   ins_pipe( pipe_cmpxchg );
7414 %}
7415 
7416 instruct xchgI( memory mem, rRegI newval) %{
7417   match(Set newval (GetAndSetI mem newval));
7418   format %{ "XCHGL  $newval,[$mem]" %}
7419   ins_encode %{
7420     __ xchgl($newval$$Register, $mem$$Address);
7421   %}
7422   ins_pipe( pipe_cmpxchg );
7423 %}
7424 
7425 instruct xchgL( memory mem, rRegL newval) %{
7426   match(Set newval (GetAndSetL mem newval));
7427   format %{ "XCHGL  $newval,[$mem]" %}
7428   ins_encode %{
7429     __ xchgq($newval$$Register, $mem$$Address);
7430   %}
7431   ins_pipe( pipe_cmpxchg );
7432 %}
7433 
7434 instruct xchgP( memory mem, rRegP newval) %{
7435   match(Set newval (GetAndSetP mem newval));
7436   format %{ "XCHGQ  $newval,[$mem]" %}
7437   ins_encode %{
7438     __ xchgq($newval$$Register, $mem$$Address);
7439   %}
7440   ins_pipe( pipe_cmpxchg );
7441 %}
7442 
7443 instruct xchgN( memory mem, rRegN newval) %{
7444   match(Set newval (GetAndSetN mem newval));
7445   format %{ "XCHGL  $newval,$mem]" %}
7446   ins_encode %{
7447     __ xchgl($newval$$Register, $mem$$Address);
7448   %}
7449   ins_pipe( pipe_cmpxchg );
7450 %}
7451 
7452 //----------Subtraction Instructions-------------------------------------------
7453 
7454 // Integer Subtraction Instructions
7455 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7456 %{
7457   match(Set dst (SubI dst src));
7458   effect(KILL cr);
7459 
7460   format %{ "subl    $dst, $src\t# int" %}
7461   opcode(0x2B);
7462   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7463   ins_pipe(ialu_reg_reg);
7464 %}
7465 
7466 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7467 %{
7468   match(Set dst (SubI dst src));
7469   effect(KILL cr);
7470 
7471   format %{ "subl    $dst, $src\t# int" %}
7472   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7473   ins_encode(OpcSErm(dst, src), Con8or32(src));
7474   ins_pipe(ialu_reg);
7475 %}
7476 
7477 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7478 %{
7479   match(Set dst (SubI dst (LoadI src)));
7480   effect(KILL cr);
7481 
7482   ins_cost(125);
7483   format %{ "subl    $dst, $src\t# int" %}
7484   opcode(0x2B);
7485   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7486   ins_pipe(ialu_reg_mem);
7487 %}
7488 
7489 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7490 %{
7491   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7492   effect(KILL cr);
7493 
7494   ins_cost(150);
7495   format %{ "subl    $dst, $src\t# int" %}
7496   opcode(0x29); /* Opcode 29 /r */
7497   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7498   ins_pipe(ialu_mem_reg);
7499 %}
7500 
7501 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7502 %{
7503   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7504   effect(KILL cr);
7505 
7506   ins_cost(125); // XXX
7507   format %{ "subl    $dst, $src\t# int" %}
7508   opcode(0x81); /* Opcode 81 /5 id */
7509   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7510   ins_pipe(ialu_mem_imm);
7511 %}
7512 
7513 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7514 %{
7515   match(Set dst (SubL dst src));
7516   effect(KILL cr);
7517 
7518   format %{ "subq    $dst, $src\t# long" %}
7519   opcode(0x2B);
7520   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7521   ins_pipe(ialu_reg_reg);
7522 %}
7523 
7524 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7525 %{
7526   match(Set dst (SubL dst src));
7527   effect(KILL cr);
7528 
7529   format %{ "subq    $dst, $src\t# long" %}
7530   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7531   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7532   ins_pipe(ialu_reg);
7533 %}
7534 
7535 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7536 %{
7537   match(Set dst (SubL dst (LoadL src)));
7538   effect(KILL cr);
7539 
7540   ins_cost(125);
7541   format %{ "subq    $dst, $src\t# long" %}
7542   opcode(0x2B);
7543   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7544   ins_pipe(ialu_reg_mem);
7545 %}
7546 
7547 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7548 %{
7549   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7550   effect(KILL cr);
7551 
7552   ins_cost(150);
7553   format %{ "subq    $dst, $src\t# long" %}
7554   opcode(0x29); /* Opcode 29 /r */
7555   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7556   ins_pipe(ialu_mem_reg);
7557 %}
7558 
7559 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7560 %{
7561   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7562   effect(KILL cr);
7563 
7564   ins_cost(125); // XXX
7565   format %{ "subq    $dst, $src\t# long" %}
7566   opcode(0x81); /* Opcode 81 /5 id */
7567   ins_encode(REX_mem_wide(dst),
7568              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7569   ins_pipe(ialu_mem_imm);
7570 %}
7571 
7572 // Subtract from a pointer
7573 // XXX hmpf???
7574 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7575 %{
7576   match(Set dst (AddP dst (SubI zero src)));
7577   effect(KILL cr);
7578 
7579   format %{ "subq    $dst, $src\t# ptr - int" %}
7580   opcode(0x2B);
7581   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7582   ins_pipe(ialu_reg_reg);
7583 %}
7584 
7585 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7586 %{
7587   match(Set dst (SubI zero dst));
7588   effect(KILL cr);
7589 
7590   format %{ "negl    $dst\t# int" %}
7591   opcode(0xF7, 0x03);  // Opcode F7 /3
7592   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7593   ins_pipe(ialu_reg);
7594 %}
7595 
7596 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7597 %{
7598   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7599   effect(KILL cr);
7600 
7601   format %{ "negl    $dst\t# int" %}
7602   opcode(0xF7, 0x03);  // Opcode F7 /3
7603   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7604   ins_pipe(ialu_reg);
7605 %}
7606 
7607 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7608 %{
7609   match(Set dst (SubL zero dst));
7610   effect(KILL cr);
7611 
7612   format %{ "negq    $dst\t# long" %}
7613   opcode(0xF7, 0x03);  // Opcode F7 /3
7614   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7615   ins_pipe(ialu_reg);
7616 %}
7617 
7618 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7619 %{
7620   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7621   effect(KILL cr);
7622 
7623   format %{ "negq    $dst\t# long" %}
7624   opcode(0xF7, 0x03);  // Opcode F7 /3
7625   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7626   ins_pipe(ialu_reg);
7627 %}
7628 
7629 //----------Multiplication/Division Instructions-------------------------------
7630 // Integer Multiplication Instructions
7631 // Multiply Register
7632 
7633 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7634 %{
7635   match(Set dst (MulI dst src));
7636   effect(KILL cr);
7637 
7638   ins_cost(300);
7639   format %{ "imull   $dst, $src\t# int" %}
7640   opcode(0x0F, 0xAF);
7641   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7642   ins_pipe(ialu_reg_reg_alu0);
7643 %}
7644 
7645 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7646 %{
7647   match(Set dst (MulI src imm));
7648   effect(KILL cr);
7649 
7650   ins_cost(300);
7651   format %{ "imull   $dst, $src, $imm\t# int" %}
7652   opcode(0x69); /* 69 /r id */
7653   ins_encode(REX_reg_reg(dst, src),
7654              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7655   ins_pipe(ialu_reg_reg_alu0);
7656 %}
7657 
7658 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7659 %{
7660   match(Set dst (MulI dst (LoadI src)));
7661   effect(KILL cr);
7662 
7663   ins_cost(350);
7664   format %{ "imull   $dst, $src\t# int" %}
7665   opcode(0x0F, 0xAF);
7666   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7667   ins_pipe(ialu_reg_mem_alu0);
7668 %}
7669 
7670 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7671 %{
7672   match(Set dst (MulI (LoadI src) imm));
7673   effect(KILL cr);
7674 
7675   ins_cost(300);
7676   format %{ "imull   $dst, $src, $imm\t# int" %}
7677   opcode(0x69); /* 69 /r id */
7678   ins_encode(REX_reg_mem(dst, src),
7679              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7680   ins_pipe(ialu_reg_mem_alu0);
7681 %}
7682 
7683 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7684 %{
7685   match(Set dst (MulL dst src));
7686   effect(KILL cr);
7687 
7688   ins_cost(300);
7689   format %{ "imulq   $dst, $src\t# long" %}
7690   opcode(0x0F, 0xAF);
7691   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7692   ins_pipe(ialu_reg_reg_alu0);
7693 %}
7694 
7695 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7696 %{
7697   match(Set dst (MulL src imm));
7698   effect(KILL cr);
7699 
7700   ins_cost(300);
7701   format %{ "imulq   $dst, $src, $imm\t# long" %}
7702   opcode(0x69); /* 69 /r id */
7703   ins_encode(REX_reg_reg_wide(dst, src),
7704              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7705   ins_pipe(ialu_reg_reg_alu0);
7706 %}
7707 
7708 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7709 %{
7710   match(Set dst (MulL dst (LoadL src)));
7711   effect(KILL cr);
7712 
7713   ins_cost(350);
7714   format %{ "imulq   $dst, $src\t# long" %}
7715   opcode(0x0F, 0xAF);
7716   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7717   ins_pipe(ialu_reg_mem_alu0);
7718 %}
7719 
7720 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7721 %{
7722   match(Set dst (MulL (LoadL src) imm));
7723   effect(KILL cr);
7724 
7725   ins_cost(300);
7726   format %{ "imulq   $dst, $src, $imm\t# long" %}
7727   opcode(0x69); /* 69 /r id */
7728   ins_encode(REX_reg_mem_wide(dst, src),
7729              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7730   ins_pipe(ialu_reg_mem_alu0);
7731 %}
7732 
7733 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7734 %{
7735   match(Set dst (MulHiL src rax));
7736   effect(USE_KILL rax, KILL cr);
7737 
7738   ins_cost(300);
7739   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7740   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7741   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7742   ins_pipe(ialu_reg_reg_alu0);
7743 %}
7744 
7745 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7746                    rFlagsReg cr)
7747 %{
7748   match(Set rax (DivI rax div));
7749   effect(KILL rdx, KILL cr);
7750 
7751   ins_cost(30*100+10*100); // XXX
7752   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7753             "jne,s   normal\n\t"
7754             "xorl    rdx, rdx\n\t"
7755             "cmpl    $div, -1\n\t"
7756             "je,s    done\n"
7757     "normal: cdql\n\t"
7758             "idivl   $div\n"
7759     "done:"        %}
7760   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7761   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7762   ins_pipe(ialu_reg_reg_alu0);
7763 %}
7764 
7765 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7766                    rFlagsReg cr)
7767 %{
7768   match(Set rax (DivL rax div));
7769   effect(KILL rdx, KILL cr);
7770 
7771   ins_cost(30*100+10*100); // XXX
7772   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7773             "cmpq    rax, rdx\n\t"
7774             "jne,s   normal\n\t"
7775             "xorl    rdx, rdx\n\t"
7776             "cmpq    $div, -1\n\t"
7777             "je,s    done\n"
7778     "normal: cdqq\n\t"
7779             "idivq   $div\n"
7780     "done:"        %}
7781   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7782   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7783   ins_pipe(ialu_reg_reg_alu0);
7784 %}
7785 
7786 // Integer DIVMOD with Register, both quotient and mod results
7787 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7788                              rFlagsReg cr)
7789 %{
7790   match(DivModI rax div);
7791   effect(KILL cr);
7792 
7793   ins_cost(30*100+10*100); // XXX
7794   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7795             "jne,s   normal\n\t"
7796             "xorl    rdx, rdx\n\t"
7797             "cmpl    $div, -1\n\t"
7798             "je,s    done\n"
7799     "normal: cdql\n\t"
7800             "idivl   $div\n"
7801     "done:"        %}
7802   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7803   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7804   ins_pipe(pipe_slow);
7805 %}
7806 
7807 // Long DIVMOD with Register, both quotient and mod results
7808 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7809                              rFlagsReg cr)
7810 %{
7811   match(DivModL rax div);
7812   effect(KILL cr);
7813 
7814   ins_cost(30*100+10*100); // XXX
7815   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7816             "cmpq    rax, rdx\n\t"
7817             "jne,s   normal\n\t"
7818             "xorl    rdx, rdx\n\t"
7819             "cmpq    $div, -1\n\t"
7820             "je,s    done\n"
7821     "normal: cdqq\n\t"
7822             "idivq   $div\n"
7823     "done:"        %}
7824   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7825   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7826   ins_pipe(pipe_slow);
7827 %}
7828 
7829 //----------- DivL-By-Constant-Expansions--------------------------------------
7830 // DivI cases are handled by the compiler
7831 
7832 // Magic constant, reciprocal of 10
7833 instruct loadConL_0x6666666666666667(rRegL dst)
7834 %{
7835   effect(DEF dst);
7836 
7837   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7838   ins_encode(load_immL(dst, 0x6666666666666667));
7839   ins_pipe(ialu_reg);
7840 %}
7841 
7842 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7843 %{
7844   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7845 
7846   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7847   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7848   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7849   ins_pipe(ialu_reg_reg_alu0);
7850 %}
7851 
7852 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7853 %{
7854   effect(USE_DEF dst, KILL cr);
7855 
7856   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7857   opcode(0xC1, 0x7); /* C1 /7 ib */
7858   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7859   ins_pipe(ialu_reg);
7860 %}
7861 
7862 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7863 %{
7864   effect(USE_DEF dst, KILL cr);
7865 
7866   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7867   opcode(0xC1, 0x7); /* C1 /7 ib */
7868   ins_encode(reg_opc_imm_wide(dst, 0x2));
7869   ins_pipe(ialu_reg);
7870 %}
7871 
7872 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7873 %{
7874   match(Set dst (DivL src div));
7875 
7876   ins_cost((5+8)*100);
7877   expand %{
7878     rax_RegL rax;                     // Killed temp
7879     rFlagsReg cr;                     // Killed
7880     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7881     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7882     sarL_rReg_63(src, cr);            // sarq  src, 63
7883     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7884     subL_rReg(dst, src, cr);          // subl  rdx, src
7885   %}
7886 %}
7887 
7888 //-----------------------------------------------------------------------------
7889 
7890 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7891                    rFlagsReg cr)
7892 %{
7893   match(Set rdx (ModI rax div));
7894   effect(KILL rax, KILL cr);
7895 
7896   ins_cost(300); // XXX
7897   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7898             "jne,s   normal\n\t"
7899             "xorl    rdx, rdx\n\t"
7900             "cmpl    $div, -1\n\t"
7901             "je,s    done\n"
7902     "normal: cdql\n\t"
7903             "idivl   $div\n"
7904     "done:"        %}
7905   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7906   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7907   ins_pipe(ialu_reg_reg_alu0);
7908 %}
7909 
7910 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7911                    rFlagsReg cr)
7912 %{
7913   match(Set rdx (ModL rax div));
7914   effect(KILL rax, KILL cr);
7915 
7916   ins_cost(300); // XXX
7917   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7918             "cmpq    rax, rdx\n\t"
7919             "jne,s   normal\n\t"
7920             "xorl    rdx, rdx\n\t"
7921             "cmpq    $div, -1\n\t"
7922             "je,s    done\n"
7923     "normal: cdqq\n\t"
7924             "idivq   $div\n"
7925     "done:"        %}
7926   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7927   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7928   ins_pipe(ialu_reg_reg_alu0);
7929 %}
7930 
7931 // Integer Shift Instructions
7932 // Shift Left by one
7933 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7934 %{
7935   match(Set dst (LShiftI dst shift));
7936   effect(KILL cr);
7937 
7938   format %{ "sall    $dst, $shift" %}
7939   opcode(0xD1, 0x4); /* D1 /4 */
7940   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7941   ins_pipe(ialu_reg);
7942 %}
7943 
7944 // Shift Left by one
7945 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7946 %{
7947   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7948   effect(KILL cr);
7949 
7950   format %{ "sall    $dst, $shift\t" %}
7951   opcode(0xD1, 0x4); /* D1 /4 */
7952   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7953   ins_pipe(ialu_mem_imm);
7954 %}
7955 
7956 // Shift Left by 8-bit immediate
7957 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7958 %{
7959   match(Set dst (LShiftI dst shift));
7960   effect(KILL cr);
7961 
7962   format %{ "sall    $dst, $shift" %}
7963   opcode(0xC1, 0x4); /* C1 /4 ib */
7964   ins_encode(reg_opc_imm(dst, shift));
7965   ins_pipe(ialu_reg);
7966 %}
7967 
7968 // Shift Left by 8-bit immediate
7969 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7970 %{
7971   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7972   effect(KILL cr);
7973 
7974   format %{ "sall    $dst, $shift" %}
7975   opcode(0xC1, 0x4); /* C1 /4 ib */
7976   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7977   ins_pipe(ialu_mem_imm);
7978 %}
7979 
7980 // Shift Left by variable
7981 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7982 %{
7983   match(Set dst (LShiftI dst shift));
7984   effect(KILL cr);
7985 
7986   format %{ "sall    $dst, $shift" %}
7987   opcode(0xD3, 0x4); /* D3 /4 */
7988   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7989   ins_pipe(ialu_reg_reg);
7990 %}
7991 
7992 // Shift Left by variable
7993 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7994 %{
7995   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7996   effect(KILL cr);
7997 
7998   format %{ "sall    $dst, $shift" %}
7999   opcode(0xD3, 0x4); /* D3 /4 */
8000   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8001   ins_pipe(ialu_mem_reg);
8002 %}
8003 
8004 // Arithmetic shift right by one
8005 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8006 %{
8007   match(Set dst (RShiftI dst shift));
8008   effect(KILL cr);
8009 
8010   format %{ "sarl    $dst, $shift" %}
8011   opcode(0xD1, 0x7); /* D1 /7 */
8012   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8013   ins_pipe(ialu_reg);
8014 %}
8015 
8016 // Arithmetic shift right by one
8017 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8018 %{
8019   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8020   effect(KILL cr);
8021 
8022   format %{ "sarl    $dst, $shift" %}
8023   opcode(0xD1, 0x7); /* D1 /7 */
8024   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8025   ins_pipe(ialu_mem_imm);
8026 %}
8027 
8028 // Arithmetic Shift Right by 8-bit immediate
8029 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8030 %{
8031   match(Set dst (RShiftI dst shift));
8032   effect(KILL cr);
8033 
8034   format %{ "sarl    $dst, $shift" %}
8035   opcode(0xC1, 0x7); /* C1 /7 ib */
8036   ins_encode(reg_opc_imm(dst, shift));
8037   ins_pipe(ialu_mem_imm);
8038 %}
8039 
8040 // Arithmetic Shift Right by 8-bit immediate
8041 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8042 %{
8043   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8044   effect(KILL cr);
8045 
8046   format %{ "sarl    $dst, $shift" %}
8047   opcode(0xC1, 0x7); /* C1 /7 ib */
8048   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8049   ins_pipe(ialu_mem_imm);
8050 %}
8051 
8052 // Arithmetic Shift Right by variable
8053 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8054 %{
8055   match(Set dst (RShiftI dst shift));
8056   effect(KILL cr);
8057 
8058   format %{ "sarl    $dst, $shift" %}
8059   opcode(0xD3, 0x7); /* D3 /7 */
8060   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8061   ins_pipe(ialu_reg_reg);
8062 %}
8063 
8064 // Arithmetic Shift Right by variable
8065 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8066 %{
8067   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8068   effect(KILL cr);
8069 
8070   format %{ "sarl    $dst, $shift" %}
8071   opcode(0xD3, 0x7); /* D3 /7 */
8072   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8073   ins_pipe(ialu_mem_reg);
8074 %}
8075 
8076 // Logical shift right by one
8077 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8078 %{
8079   match(Set dst (URShiftI dst shift));
8080   effect(KILL cr);
8081 
8082   format %{ "shrl    $dst, $shift" %}
8083   opcode(0xD1, 0x5); /* D1 /5 */
8084   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8085   ins_pipe(ialu_reg);
8086 %}
8087 
8088 // Logical shift right by one
8089 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8090 %{
8091   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8092   effect(KILL cr);
8093 
8094   format %{ "shrl    $dst, $shift" %}
8095   opcode(0xD1, 0x5); /* D1 /5 */
8096   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8097   ins_pipe(ialu_mem_imm);
8098 %}
8099 
8100 // Logical Shift Right by 8-bit immediate
8101 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8102 %{
8103   match(Set dst (URShiftI dst shift));
8104   effect(KILL cr);
8105 
8106   format %{ "shrl    $dst, $shift" %}
8107   opcode(0xC1, 0x5); /* C1 /5 ib */
8108   ins_encode(reg_opc_imm(dst, shift));
8109   ins_pipe(ialu_reg);
8110 %}
8111 
8112 // Logical Shift Right by 8-bit immediate
8113 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8114 %{
8115   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8116   effect(KILL cr);
8117 
8118   format %{ "shrl    $dst, $shift" %}
8119   opcode(0xC1, 0x5); /* C1 /5 ib */
8120   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8121   ins_pipe(ialu_mem_imm);
8122 %}
8123 
8124 // Logical Shift Right by variable
8125 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8126 %{
8127   match(Set dst (URShiftI dst shift));
8128   effect(KILL cr);
8129 
8130   format %{ "shrl    $dst, $shift" %}
8131   opcode(0xD3, 0x5); /* D3 /5 */
8132   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8133   ins_pipe(ialu_reg_reg);
8134 %}
8135 
8136 // Logical Shift Right by variable
8137 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8138 %{
8139   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8140   effect(KILL cr);
8141 
8142   format %{ "shrl    $dst, $shift" %}
8143   opcode(0xD3, 0x5); /* D3 /5 */
8144   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8145   ins_pipe(ialu_mem_reg);
8146 %}
8147 
8148 // Long Shift Instructions
8149 // Shift Left by one
8150 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8151 %{
8152   match(Set dst (LShiftL dst shift));
8153   effect(KILL cr);
8154 
8155   format %{ "salq    $dst, $shift" %}
8156   opcode(0xD1, 0x4); /* D1 /4 */
8157   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8158   ins_pipe(ialu_reg);
8159 %}
8160 
8161 // Shift Left by one
8162 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8163 %{
8164   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8165   effect(KILL cr);
8166 
8167   format %{ "salq    $dst, $shift" %}
8168   opcode(0xD1, 0x4); /* D1 /4 */
8169   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8170   ins_pipe(ialu_mem_imm);
8171 %}
8172 
8173 // Shift Left by 8-bit immediate
8174 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8175 %{
8176   match(Set dst (LShiftL dst shift));
8177   effect(KILL cr);
8178 
8179   format %{ "salq    $dst, $shift" %}
8180   opcode(0xC1, 0x4); /* C1 /4 ib */
8181   ins_encode(reg_opc_imm_wide(dst, shift));
8182   ins_pipe(ialu_reg);
8183 %}
8184 
8185 // Shift Left by 8-bit immediate
8186 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8187 %{
8188   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8189   effect(KILL cr);
8190 
8191   format %{ "salq    $dst, $shift" %}
8192   opcode(0xC1, 0x4); /* C1 /4 ib */
8193   ins_encode(REX_mem_wide(dst), OpcP,
8194              RM_opc_mem(secondary, dst), Con8or32(shift));
8195   ins_pipe(ialu_mem_imm);
8196 %}
8197 
8198 // Shift Left by variable
8199 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8200 %{
8201   match(Set dst (LShiftL dst shift));
8202   effect(KILL cr);
8203 
8204   format %{ "salq    $dst, $shift" %}
8205   opcode(0xD3, 0x4); /* D3 /4 */
8206   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8207   ins_pipe(ialu_reg_reg);
8208 %}
8209 
8210 // Shift Left by variable
8211 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8212 %{
8213   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8214   effect(KILL cr);
8215 
8216   format %{ "salq    $dst, $shift" %}
8217   opcode(0xD3, 0x4); /* D3 /4 */
8218   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8219   ins_pipe(ialu_mem_reg);
8220 %}
8221 
8222 // Arithmetic shift right by one
8223 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8224 %{
8225   match(Set dst (RShiftL dst shift));
8226   effect(KILL cr);
8227 
8228   format %{ "sarq    $dst, $shift" %}
8229   opcode(0xD1, 0x7); /* D1 /7 */
8230   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8231   ins_pipe(ialu_reg);
8232 %}
8233 
8234 // Arithmetic shift right by one
8235 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8236 %{
8237   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8238   effect(KILL cr);
8239 
8240   format %{ "sarq    $dst, $shift" %}
8241   opcode(0xD1, 0x7); /* D1 /7 */
8242   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8243   ins_pipe(ialu_mem_imm);
8244 %}
8245 
8246 // Arithmetic Shift Right by 8-bit immediate
8247 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8248 %{
8249   match(Set dst (RShiftL dst shift));
8250   effect(KILL cr);
8251 
8252   format %{ "sarq    $dst, $shift" %}
8253   opcode(0xC1, 0x7); /* C1 /7 ib */
8254   ins_encode(reg_opc_imm_wide(dst, shift));
8255   ins_pipe(ialu_mem_imm);
8256 %}
8257 
8258 // Arithmetic Shift Right by 8-bit immediate
8259 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8260 %{
8261   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8262   effect(KILL cr);
8263 
8264   format %{ "sarq    $dst, $shift" %}
8265   opcode(0xC1, 0x7); /* C1 /7 ib */
8266   ins_encode(REX_mem_wide(dst), OpcP,
8267              RM_opc_mem(secondary, dst), Con8or32(shift));
8268   ins_pipe(ialu_mem_imm);
8269 %}
8270 
8271 // Arithmetic Shift Right by variable
8272 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8273 %{
8274   match(Set dst (RShiftL dst shift));
8275   effect(KILL cr);
8276 
8277   format %{ "sarq    $dst, $shift" %}
8278   opcode(0xD3, 0x7); /* D3 /7 */
8279   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8280   ins_pipe(ialu_reg_reg);
8281 %}
8282 
8283 // Arithmetic Shift Right by variable
8284 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8285 %{
8286   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8287   effect(KILL cr);
8288 
8289   format %{ "sarq    $dst, $shift" %}
8290   opcode(0xD3, 0x7); /* D3 /7 */
8291   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8292   ins_pipe(ialu_mem_reg);
8293 %}
8294 
8295 // Logical shift right by one
8296 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8297 %{
8298   match(Set dst (URShiftL dst shift));
8299   effect(KILL cr);
8300 
8301   format %{ "shrq    $dst, $shift" %}
8302   opcode(0xD1, 0x5); /* D1 /5 */
8303   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8304   ins_pipe(ialu_reg);
8305 %}
8306 
8307 // Logical shift right by one
8308 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8309 %{
8310   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8311   effect(KILL cr);
8312 
8313   format %{ "shrq    $dst, $shift" %}
8314   opcode(0xD1, 0x5); /* D1 /5 */
8315   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8316   ins_pipe(ialu_mem_imm);
8317 %}
8318 
8319 // Logical Shift Right by 8-bit immediate
8320 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8321 %{
8322   match(Set dst (URShiftL dst shift));
8323   effect(KILL cr);
8324 
8325   format %{ "shrq    $dst, $shift" %}
8326   opcode(0xC1, 0x5); /* C1 /5 ib */
8327   ins_encode(reg_opc_imm_wide(dst, shift));
8328   ins_pipe(ialu_reg);
8329 %}
8330 
8331 
8332 // Logical Shift Right by 8-bit immediate
8333 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8334 %{
8335   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8336   effect(KILL cr);
8337 
8338   format %{ "shrq    $dst, $shift" %}
8339   opcode(0xC1, 0x5); /* C1 /5 ib */
8340   ins_encode(REX_mem_wide(dst), OpcP,
8341              RM_opc_mem(secondary, dst), Con8or32(shift));
8342   ins_pipe(ialu_mem_imm);
8343 %}
8344 
8345 // Logical Shift Right by variable
8346 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8347 %{
8348   match(Set dst (URShiftL dst shift));
8349   effect(KILL cr);
8350 
8351   format %{ "shrq    $dst, $shift" %}
8352   opcode(0xD3, 0x5); /* D3 /5 */
8353   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8354   ins_pipe(ialu_reg_reg);
8355 %}
8356 
8357 // Logical Shift Right by variable
8358 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8359 %{
8360   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8361   effect(KILL cr);
8362 
8363   format %{ "shrq    $dst, $shift" %}
8364   opcode(0xD3, 0x5); /* D3 /5 */
8365   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8366   ins_pipe(ialu_mem_reg);
8367 %}
8368 
8369 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8370 // This idiom is used by the compiler for the i2b bytecode.
8371 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8372 %{
8373   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8374 
8375   format %{ "movsbl  $dst, $src\t# i2b" %}
8376   opcode(0x0F, 0xBE);
8377   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8378   ins_pipe(ialu_reg_reg);
8379 %}
8380 
8381 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8382 // This idiom is used by the compiler the i2s bytecode.
8383 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8384 %{
8385   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8386 
8387   format %{ "movswl  $dst, $src\t# i2s" %}
8388   opcode(0x0F, 0xBF);
8389   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8390   ins_pipe(ialu_reg_reg);
8391 %}
8392 
8393 // ROL/ROR instructions
8394 
8395 // ROL expand
8396 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8397   effect(KILL cr, USE_DEF dst);
8398 
8399   format %{ "roll    $dst" %}
8400   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8401   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8402   ins_pipe(ialu_reg);
8403 %}
8404 
8405 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8406   effect(USE_DEF dst, USE shift, KILL cr);
8407 
8408   format %{ "roll    $dst, $shift" %}
8409   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8410   ins_encode( reg_opc_imm(dst, shift) );
8411   ins_pipe(ialu_reg);
8412 %}
8413 
8414 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8415 %{
8416   effect(USE_DEF dst, USE shift, KILL cr);
8417 
8418   format %{ "roll    $dst, $shift" %}
8419   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8420   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8421   ins_pipe(ialu_reg_reg);
8422 %}
8423 // end of ROL expand
8424 
8425 // Rotate Left by one
8426 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8427 %{
8428   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8429 
8430   expand %{
8431     rolI_rReg_imm1(dst, cr);
8432   %}
8433 %}
8434 
8435 // Rotate Left by 8-bit immediate
8436 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8437 %{
8438   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8439   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8440 
8441   expand %{
8442     rolI_rReg_imm8(dst, lshift, cr);
8443   %}
8444 %}
8445 
8446 // Rotate Left by variable
8447 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8448 %{
8449   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8450 
8451   expand %{
8452     rolI_rReg_CL(dst, shift, cr);
8453   %}
8454 %}
8455 
8456 // Rotate Left by variable
8457 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8458 %{
8459   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8460 
8461   expand %{
8462     rolI_rReg_CL(dst, shift, cr);
8463   %}
8464 %}
8465 
8466 // ROR expand
8467 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8468 %{
8469   effect(USE_DEF dst, KILL cr);
8470 
8471   format %{ "rorl    $dst" %}
8472   opcode(0xD1, 0x1); /* D1 /1 */
8473   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8474   ins_pipe(ialu_reg);
8475 %}
8476 
8477 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8478 %{
8479   effect(USE_DEF dst, USE shift, KILL cr);
8480 
8481   format %{ "rorl    $dst, $shift" %}
8482   opcode(0xC1, 0x1); /* C1 /1 ib */
8483   ins_encode(reg_opc_imm(dst, shift));
8484   ins_pipe(ialu_reg);
8485 %}
8486 
8487 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8488 %{
8489   effect(USE_DEF dst, USE shift, KILL cr);
8490 
8491   format %{ "rorl    $dst, $shift" %}
8492   opcode(0xD3, 0x1); /* D3 /1 */
8493   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8494   ins_pipe(ialu_reg_reg);
8495 %}
8496 // end of ROR expand
8497 
8498 // Rotate Right by one
8499 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8500 %{
8501   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8502 
8503   expand %{
8504     rorI_rReg_imm1(dst, cr);
8505   %}
8506 %}
8507 
8508 // Rotate Right by 8-bit immediate
8509 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8510 %{
8511   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8512   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8513 
8514   expand %{
8515     rorI_rReg_imm8(dst, rshift, cr);
8516   %}
8517 %}
8518 
8519 // Rotate Right by variable
8520 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8521 %{
8522   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8523 
8524   expand %{
8525     rorI_rReg_CL(dst, shift, cr);
8526   %}
8527 %}
8528 
8529 // Rotate Right by variable
8530 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8531 %{
8532   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8533 
8534   expand %{
8535     rorI_rReg_CL(dst, shift, cr);
8536   %}
8537 %}
8538 
8539 // for long rotate
8540 // ROL expand
8541 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8542   effect(USE_DEF dst, KILL cr);
8543 
8544   format %{ "rolq    $dst" %}
8545   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8546   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8547   ins_pipe(ialu_reg);
8548 %}
8549 
8550 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8551   effect(USE_DEF dst, USE shift, KILL cr);
8552 
8553   format %{ "rolq    $dst, $shift" %}
8554   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8555   ins_encode( reg_opc_imm_wide(dst, shift) );
8556   ins_pipe(ialu_reg);
8557 %}
8558 
8559 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8560 %{
8561   effect(USE_DEF dst, USE shift, KILL cr);
8562 
8563   format %{ "rolq    $dst, $shift" %}
8564   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8565   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8566   ins_pipe(ialu_reg_reg);
8567 %}
8568 // end of ROL expand
8569 
8570 // Rotate Left by one
8571 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8572 %{
8573   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8574 
8575   expand %{
8576     rolL_rReg_imm1(dst, cr);
8577   %}
8578 %}
8579 
8580 // Rotate Left by 8-bit immediate
8581 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8582 %{
8583   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8584   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8585 
8586   expand %{
8587     rolL_rReg_imm8(dst, lshift, cr);
8588   %}
8589 %}
8590 
8591 // Rotate Left by variable
8592 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8593 %{
8594   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8595 
8596   expand %{
8597     rolL_rReg_CL(dst, shift, cr);
8598   %}
8599 %}
8600 
8601 // Rotate Left by variable
8602 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8603 %{
8604   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8605 
8606   expand %{
8607     rolL_rReg_CL(dst, shift, cr);
8608   %}
8609 %}
8610 
8611 // ROR expand
8612 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8613 %{
8614   effect(USE_DEF dst, KILL cr);
8615 
8616   format %{ "rorq    $dst" %}
8617   opcode(0xD1, 0x1); /* D1 /1 */
8618   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8619   ins_pipe(ialu_reg);
8620 %}
8621 
8622 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8623 %{
8624   effect(USE_DEF dst, USE shift, KILL cr);
8625 
8626   format %{ "rorq    $dst, $shift" %}
8627   opcode(0xC1, 0x1); /* C1 /1 ib */
8628   ins_encode(reg_opc_imm_wide(dst, shift));
8629   ins_pipe(ialu_reg);
8630 %}
8631 
8632 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8633 %{
8634   effect(USE_DEF dst, USE shift, KILL cr);
8635 
8636   format %{ "rorq    $dst, $shift" %}
8637   opcode(0xD3, 0x1); /* D3 /1 */
8638   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8639   ins_pipe(ialu_reg_reg);
8640 %}
8641 // end of ROR expand
8642 
8643 // Rotate Right by one
8644 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8645 %{
8646   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8647 
8648   expand %{
8649     rorL_rReg_imm1(dst, cr);
8650   %}
8651 %}
8652 
8653 // Rotate Right by 8-bit immediate
8654 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8655 %{
8656   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8657   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8658 
8659   expand %{
8660     rorL_rReg_imm8(dst, rshift, cr);
8661   %}
8662 %}
8663 
8664 // Rotate Right by variable
8665 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8666 %{
8667   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8668 
8669   expand %{
8670     rorL_rReg_CL(dst, shift, cr);
8671   %}
8672 %}
8673 
8674 // Rotate Right by variable
8675 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8676 %{
8677   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8678 
8679   expand %{
8680     rorL_rReg_CL(dst, shift, cr);
8681   %}
8682 %}
8683 
8684 // Logical Instructions
8685 
8686 // Integer Logical Instructions
8687 
8688 // And Instructions
8689 // And Register with Register
8690 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8691 %{
8692   match(Set dst (AndI dst src));
8693   effect(KILL cr);
8694 
8695   format %{ "andl    $dst, $src\t# int" %}
8696   opcode(0x23);
8697   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8698   ins_pipe(ialu_reg_reg);
8699 %}
8700 
8701 // And Register with Immediate 255
8702 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8703 %{
8704   match(Set dst (AndI dst src));
8705 
8706   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8707   opcode(0x0F, 0xB6);
8708   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8709   ins_pipe(ialu_reg);
8710 %}
8711 
8712 // And Register with Immediate 255 and promote to long
8713 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8714 %{
8715   match(Set dst (ConvI2L (AndI src mask)));
8716 
8717   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8718   opcode(0x0F, 0xB6);
8719   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8720   ins_pipe(ialu_reg);
8721 %}
8722 
8723 // And Register with Immediate 65535
8724 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8725 %{
8726   match(Set dst (AndI dst src));
8727 
8728   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8729   opcode(0x0F, 0xB7);
8730   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8731   ins_pipe(ialu_reg);
8732 %}
8733 
8734 // And Register with Immediate 65535 and promote to long
8735 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8736 %{
8737   match(Set dst (ConvI2L (AndI src mask)));
8738 
8739   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8740   opcode(0x0F, 0xB7);
8741   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8742   ins_pipe(ialu_reg);
8743 %}
8744 
8745 // And Register with Immediate
8746 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8747 %{
8748   match(Set dst (AndI dst src));
8749   effect(KILL cr);
8750 
8751   format %{ "andl    $dst, $src\t# int" %}
8752   opcode(0x81, 0x04); /* Opcode 81 /4 */
8753   ins_encode(OpcSErm(dst, src), Con8or32(src));
8754   ins_pipe(ialu_reg);
8755 %}
8756 
8757 // And Register with Memory
8758 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8759 %{
8760   match(Set dst (AndI dst (LoadI src)));
8761   effect(KILL cr);
8762 
8763   ins_cost(125);
8764   format %{ "andl    $dst, $src\t# int" %}
8765   opcode(0x23);
8766   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8767   ins_pipe(ialu_reg_mem);
8768 %}
8769 
8770 // And Memory with Register
8771 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8772 %{
8773   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8774   effect(KILL cr);
8775 
8776   ins_cost(150);
8777   format %{ "andl    $dst, $src\t# int" %}
8778   opcode(0x21); /* Opcode 21 /r */
8779   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8780   ins_pipe(ialu_mem_reg);
8781 %}
8782 
8783 // And Memory with Immediate
8784 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8785 %{
8786   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8787   effect(KILL cr);
8788 
8789   ins_cost(125);
8790   format %{ "andl    $dst, $src\t# int" %}
8791   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8792   ins_encode(REX_mem(dst), OpcSE(src),
8793              RM_opc_mem(secondary, dst), Con8or32(src));
8794   ins_pipe(ialu_mem_imm);
8795 %}
8796 
8797 // BMI1 instructions
8798 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8799   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8800   predicate(UseBMI1Instructions);
8801   effect(KILL cr);
8802 
8803   ins_cost(125);
8804   format %{ "andnl  $dst, $src1, $src2" %}
8805 
8806   ins_encode %{
8807     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8808   %}
8809   ins_pipe(ialu_reg_mem);
8810 %}
8811 
8812 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8813   match(Set dst (AndI (XorI src1 minus_1) src2));
8814   predicate(UseBMI1Instructions);
8815   effect(KILL cr);
8816 
8817   format %{ "andnl  $dst, $src1, $src2" %}
8818 
8819   ins_encode %{
8820     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8821   %}
8822   ins_pipe(ialu_reg);
8823 %}
8824 
8825 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8826   match(Set dst (AndI (SubI imm_zero src) src));
8827   predicate(UseBMI1Instructions);
8828   effect(KILL cr);
8829 
8830   format %{ "blsil  $dst, $src" %}
8831 
8832   ins_encode %{
8833     __ blsil($dst$$Register, $src$$Register);
8834   %}
8835   ins_pipe(ialu_reg);
8836 %}
8837 
8838 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8839   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8840   predicate(UseBMI1Instructions);
8841   effect(KILL cr);
8842 
8843   ins_cost(125);
8844   format %{ "blsil  $dst, $src" %}
8845 
8846   ins_encode %{
8847     __ blsil($dst$$Register, $src$$Address);
8848   %}
8849   ins_pipe(ialu_reg_mem);
8850 %}
8851 
8852 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8853 %{
8854   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8855   predicate(UseBMI1Instructions);
8856   effect(KILL cr);
8857 
8858   ins_cost(125);
8859   format %{ "blsmskl $dst, $src" %}
8860 
8861   ins_encode %{
8862     __ blsmskl($dst$$Register, $src$$Address);
8863   %}
8864   ins_pipe(ialu_reg_mem);
8865 %}
8866 
8867 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8868 %{
8869   match(Set dst (XorI (AddI src minus_1) src));
8870   predicate(UseBMI1Instructions);
8871   effect(KILL cr);
8872 
8873   format %{ "blsmskl $dst, $src" %}
8874 
8875   ins_encode %{
8876     __ blsmskl($dst$$Register, $src$$Register);
8877   %}
8878 
8879   ins_pipe(ialu_reg);
8880 %}
8881 
8882 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8883 %{
8884   match(Set dst (AndI (AddI src minus_1) src) );
8885   predicate(UseBMI1Instructions);
8886   effect(KILL cr);
8887 
8888   format %{ "blsrl  $dst, $src" %}
8889 
8890   ins_encode %{
8891     __ blsrl($dst$$Register, $src$$Register);
8892   %}
8893 
8894   ins_pipe(ialu_reg_mem);
8895 %}
8896 
8897 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8898 %{
8899   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8900   predicate(UseBMI1Instructions);
8901   effect(KILL cr);
8902 
8903   ins_cost(125);
8904   format %{ "blsrl  $dst, $src" %}
8905 
8906   ins_encode %{
8907     __ blsrl($dst$$Register, $src$$Address);
8908   %}
8909 
8910   ins_pipe(ialu_reg);
8911 %}
8912 
8913 // Or Instructions
8914 // Or Register with Register
8915 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8916 %{
8917   match(Set dst (OrI dst src));
8918   effect(KILL cr);
8919 
8920   format %{ "orl     $dst, $src\t# int" %}
8921   opcode(0x0B);
8922   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8923   ins_pipe(ialu_reg_reg);
8924 %}
8925 
8926 // Or Register with Immediate
8927 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8928 %{
8929   match(Set dst (OrI dst src));
8930   effect(KILL cr);
8931 
8932   format %{ "orl     $dst, $src\t# int" %}
8933   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8934   ins_encode(OpcSErm(dst, src), Con8or32(src));
8935   ins_pipe(ialu_reg);
8936 %}
8937 
8938 // Or Register with Memory
8939 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8940 %{
8941   match(Set dst (OrI dst (LoadI src)));
8942   effect(KILL cr);
8943 
8944   ins_cost(125);
8945   format %{ "orl     $dst, $src\t# int" %}
8946   opcode(0x0B);
8947   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8948   ins_pipe(ialu_reg_mem);
8949 %}
8950 
8951 // Or Memory with Register
8952 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8953 %{
8954   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8955   effect(KILL cr);
8956 
8957   ins_cost(150);
8958   format %{ "orl     $dst, $src\t# int" %}
8959   opcode(0x09); /* Opcode 09 /r */
8960   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8961   ins_pipe(ialu_mem_reg);
8962 %}
8963 
8964 // Or Memory with Immediate
8965 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8966 %{
8967   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8968   effect(KILL cr);
8969 
8970   ins_cost(125);
8971   format %{ "orl     $dst, $src\t# int" %}
8972   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8973   ins_encode(REX_mem(dst), OpcSE(src),
8974              RM_opc_mem(secondary, dst), Con8or32(src));
8975   ins_pipe(ialu_mem_imm);
8976 %}
8977 
8978 // Xor Instructions
8979 // Xor Register with Register
8980 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8981 %{
8982   match(Set dst (XorI dst src));
8983   effect(KILL cr);
8984 
8985   format %{ "xorl    $dst, $src\t# int" %}
8986   opcode(0x33);
8987   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8988   ins_pipe(ialu_reg_reg);
8989 %}
8990 
8991 // Xor Register with Immediate -1
8992 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8993   match(Set dst (XorI dst imm));
8994 
8995   format %{ "not    $dst" %}
8996   ins_encode %{
8997      __ notl($dst$$Register);
8998   %}
8999   ins_pipe(ialu_reg);
9000 %}
9001 
9002 // Xor Register with Immediate
9003 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9004 %{
9005   match(Set dst (XorI dst src));
9006   effect(KILL cr);
9007 
9008   format %{ "xorl    $dst, $src\t# int" %}
9009   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9010   ins_encode(OpcSErm(dst, src), Con8or32(src));
9011   ins_pipe(ialu_reg);
9012 %}
9013 
9014 // Xor Register with Memory
9015 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9016 %{
9017   match(Set dst (XorI dst (LoadI src)));
9018   effect(KILL cr);
9019 
9020   ins_cost(125);
9021   format %{ "xorl    $dst, $src\t# int" %}
9022   opcode(0x33);
9023   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9024   ins_pipe(ialu_reg_mem);
9025 %}
9026 
9027 // Xor Memory with Register
9028 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9029 %{
9030   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9031   effect(KILL cr);
9032 
9033   ins_cost(150);
9034   format %{ "xorl    $dst, $src\t# int" %}
9035   opcode(0x31); /* Opcode 31 /r */
9036   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9037   ins_pipe(ialu_mem_reg);
9038 %}
9039 
9040 // Xor Memory with Immediate
9041 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9042 %{
9043   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9044   effect(KILL cr);
9045 
9046   ins_cost(125);
9047   format %{ "xorl    $dst, $src\t# int" %}
9048   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9049   ins_encode(REX_mem(dst), OpcSE(src),
9050              RM_opc_mem(secondary, dst), Con8or32(src));
9051   ins_pipe(ialu_mem_imm);
9052 %}
9053 
9054 
9055 // Long Logical Instructions
9056 
9057 // And Instructions
9058 // And Register with Register
9059 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9060 %{
9061   match(Set dst (AndL dst src));
9062   effect(KILL cr);
9063 
9064   format %{ "andq    $dst, $src\t# long" %}
9065   opcode(0x23);
9066   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9067   ins_pipe(ialu_reg_reg);
9068 %}
9069 
9070 // And Register with Immediate 255
9071 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9072 %{
9073   match(Set dst (AndL dst src));
9074 
9075   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9076   opcode(0x0F, 0xB6);
9077   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9078   ins_pipe(ialu_reg);
9079 %}
9080 
9081 // And Register with Immediate 65535
9082 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9083 %{
9084   match(Set dst (AndL dst src));
9085 
9086   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9087   opcode(0x0F, 0xB7);
9088   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9089   ins_pipe(ialu_reg);
9090 %}
9091 
9092 // And Register with Immediate
9093 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9094 %{
9095   match(Set dst (AndL dst src));
9096   effect(KILL cr);
9097 
9098   format %{ "andq    $dst, $src\t# long" %}
9099   opcode(0x81, 0x04); /* Opcode 81 /4 */
9100   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9101   ins_pipe(ialu_reg);
9102 %}
9103 
9104 // And Register with Memory
9105 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9106 %{
9107   match(Set dst (AndL dst (LoadL src)));
9108   effect(KILL cr);
9109 
9110   ins_cost(125);
9111   format %{ "andq    $dst, $src\t# long" %}
9112   opcode(0x23);
9113   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9114   ins_pipe(ialu_reg_mem);
9115 %}
9116 
9117 // And Memory with Register
9118 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9119 %{
9120   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9121   effect(KILL cr);
9122 
9123   ins_cost(150);
9124   format %{ "andq    $dst, $src\t# long" %}
9125   opcode(0x21); /* Opcode 21 /r */
9126   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9127   ins_pipe(ialu_mem_reg);
9128 %}
9129 
9130 // And Memory with Immediate
9131 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9132 %{
9133   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9134   effect(KILL cr);
9135 
9136   ins_cost(125);
9137   format %{ "andq    $dst, $src\t# long" %}
9138   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9139   ins_encode(REX_mem_wide(dst), OpcSE(src),
9140              RM_opc_mem(secondary, dst), Con8or32(src));
9141   ins_pipe(ialu_mem_imm);
9142 %}
9143 
9144 // BMI1 instructions
9145 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9146   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9147   predicate(UseBMI1Instructions);
9148   effect(KILL cr);
9149 
9150   ins_cost(125);
9151   format %{ "andnq  $dst, $src1, $src2" %}
9152 
9153   ins_encode %{
9154     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9155   %}
9156   ins_pipe(ialu_reg_mem);
9157 %}
9158 
9159 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9160   match(Set dst (AndL (XorL src1 minus_1) src2));
9161   predicate(UseBMI1Instructions);
9162   effect(KILL cr);
9163 
9164   format %{ "andnq  $dst, $src1, $src2" %}
9165 
9166   ins_encode %{
9167   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9168   %}
9169   ins_pipe(ialu_reg_mem);
9170 %}
9171 
9172 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9173   match(Set dst (AndL (SubL imm_zero src) src));
9174   predicate(UseBMI1Instructions);
9175   effect(KILL cr);
9176 
9177   format %{ "blsiq  $dst, $src" %}
9178 
9179   ins_encode %{
9180     __ blsiq($dst$$Register, $src$$Register);
9181   %}
9182   ins_pipe(ialu_reg);
9183 %}
9184 
9185 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9186   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9187   predicate(UseBMI1Instructions);
9188   effect(KILL cr);
9189 
9190   ins_cost(125);
9191   format %{ "blsiq  $dst, $src" %}
9192 
9193   ins_encode %{
9194     __ blsiq($dst$$Register, $src$$Address);
9195   %}
9196   ins_pipe(ialu_reg_mem);
9197 %}
9198 
9199 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9200 %{
9201   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9202   predicate(UseBMI1Instructions);
9203   effect(KILL cr);
9204 
9205   ins_cost(125);
9206   format %{ "blsmskq $dst, $src" %}
9207 
9208   ins_encode %{
9209     __ blsmskq($dst$$Register, $src$$Address);
9210   %}
9211   ins_pipe(ialu_reg_mem);
9212 %}
9213 
9214 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9215 %{
9216   match(Set dst (XorL (AddL src minus_1) src));
9217   predicate(UseBMI1Instructions);
9218   effect(KILL cr);
9219 
9220   format %{ "blsmskq $dst, $src" %}
9221 
9222   ins_encode %{
9223     __ blsmskq($dst$$Register, $src$$Register);
9224   %}
9225 
9226   ins_pipe(ialu_reg);
9227 %}
9228 
9229 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9230 %{
9231   match(Set dst (AndL (AddL src minus_1) src) );
9232   predicate(UseBMI1Instructions);
9233   effect(KILL cr);
9234 
9235   format %{ "blsrq  $dst, $src" %}
9236 
9237   ins_encode %{
9238     __ blsrq($dst$$Register, $src$$Register);
9239   %}
9240 
9241   ins_pipe(ialu_reg);
9242 %}
9243 
9244 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9245 %{
9246   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9247   predicate(UseBMI1Instructions);
9248   effect(KILL cr);
9249 
9250   ins_cost(125);
9251   format %{ "blsrq  $dst, $src" %}
9252 
9253   ins_encode %{
9254     __ blsrq($dst$$Register, $src$$Address);
9255   %}
9256 
9257   ins_pipe(ialu_reg);
9258 %}
9259 
9260 // Or Instructions
9261 // Or Register with Register
9262 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9263 %{
9264   match(Set dst (OrL dst src));
9265   effect(KILL cr);
9266 
9267   format %{ "orq     $dst, $src\t# long" %}
9268   opcode(0x0B);
9269   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9270   ins_pipe(ialu_reg_reg);
9271 %}
9272 
9273 // Use any_RegP to match R15 (TLS register) without spilling.
9274 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9275   match(Set dst (OrL dst (CastP2X src)));
9276   effect(KILL cr);
9277 
9278   format %{ "orq     $dst, $src\t# long" %}
9279   opcode(0x0B);
9280   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9281   ins_pipe(ialu_reg_reg);
9282 %}
9283 
9284 
9285 // Or Register with Immediate
9286 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9287 %{
9288   match(Set dst (OrL dst src));
9289   effect(KILL cr);
9290 
9291   format %{ "orq     $dst, $src\t# long" %}
9292   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9293   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9294   ins_pipe(ialu_reg);
9295 %}
9296 
9297 // Or Register with Memory
9298 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9299 %{
9300   match(Set dst (OrL dst (LoadL src)));
9301   effect(KILL cr);
9302 
9303   ins_cost(125);
9304   format %{ "orq     $dst, $src\t# long" %}
9305   opcode(0x0B);
9306   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9307   ins_pipe(ialu_reg_mem);
9308 %}
9309 
9310 // Or Memory with Register
9311 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9312 %{
9313   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9314   effect(KILL cr);
9315 
9316   ins_cost(150);
9317   format %{ "orq     $dst, $src\t# long" %}
9318   opcode(0x09); /* Opcode 09 /r */
9319   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9320   ins_pipe(ialu_mem_reg);
9321 %}
9322 
9323 // Or Memory with Immediate
9324 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9325 %{
9326   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9327   effect(KILL cr);
9328 
9329   ins_cost(125);
9330   format %{ "orq     $dst, $src\t# long" %}
9331   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9332   ins_encode(REX_mem_wide(dst), OpcSE(src),
9333              RM_opc_mem(secondary, dst), Con8or32(src));
9334   ins_pipe(ialu_mem_imm);
9335 %}
9336 
9337 // Xor Instructions
9338 // Xor Register with Register
9339 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9340 %{
9341   match(Set dst (XorL dst src));
9342   effect(KILL cr);
9343 
9344   format %{ "xorq    $dst, $src\t# long" %}
9345   opcode(0x33);
9346   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9347   ins_pipe(ialu_reg_reg);
9348 %}
9349 
9350 // Xor Register with Immediate -1
9351 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9352   match(Set dst (XorL dst imm));
9353 
9354   format %{ "notq   $dst" %}
9355   ins_encode %{
9356      __ notq($dst$$Register);
9357   %}
9358   ins_pipe(ialu_reg);
9359 %}
9360 
9361 // Xor Register with Immediate
9362 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9363 %{
9364   match(Set dst (XorL dst src));
9365   effect(KILL cr);
9366 
9367   format %{ "xorq    $dst, $src\t# long" %}
9368   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9369   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9370   ins_pipe(ialu_reg);
9371 %}
9372 
9373 // Xor Register with Memory
9374 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9375 %{
9376   match(Set dst (XorL dst (LoadL src)));
9377   effect(KILL cr);
9378 
9379   ins_cost(125);
9380   format %{ "xorq    $dst, $src\t# long" %}
9381   opcode(0x33);
9382   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9383   ins_pipe(ialu_reg_mem);
9384 %}
9385 
9386 // Xor Memory with Register
9387 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9388 %{
9389   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9390   effect(KILL cr);
9391 
9392   ins_cost(150);
9393   format %{ "xorq    $dst, $src\t# long" %}
9394   opcode(0x31); /* Opcode 31 /r */
9395   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9396   ins_pipe(ialu_mem_reg);
9397 %}
9398 
9399 // Xor Memory with Immediate
9400 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9401 %{
9402   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9403   effect(KILL cr);
9404 
9405   ins_cost(125);
9406   format %{ "xorq    $dst, $src\t# long" %}
9407   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9408   ins_encode(REX_mem_wide(dst), OpcSE(src),
9409              RM_opc_mem(secondary, dst), Con8or32(src));
9410   ins_pipe(ialu_mem_imm);
9411 %}
9412 
9413 // Convert Int to Boolean
9414 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9415 %{
9416   match(Set dst (Conv2B src));
9417   effect(KILL cr);
9418 
9419   format %{ "testl   $src, $src\t# ci2b\n\t"
9420             "setnz   $dst\n\t"
9421             "movzbl  $dst, $dst" %}
9422   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9423              setNZ_reg(dst),
9424              REX_reg_breg(dst, dst), // movzbl
9425              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9426   ins_pipe(pipe_slow); // XXX
9427 %}
9428 
9429 // Convert Pointer to Boolean
9430 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9431 %{
9432   match(Set dst (Conv2B src));
9433   effect(KILL cr);
9434 
9435   format %{ "testq   $src, $src\t# cp2b\n\t"
9436             "setnz   $dst\n\t"
9437             "movzbl  $dst, $dst" %}
9438   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9439              setNZ_reg(dst),
9440              REX_reg_breg(dst, dst), // movzbl
9441              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9442   ins_pipe(pipe_slow); // XXX
9443 %}
9444 
9445 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9446 %{
9447   match(Set dst (CmpLTMask p q));
9448   effect(KILL cr);
9449 
9450   ins_cost(400);
9451   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9452             "setlt   $dst\n\t"
9453             "movzbl  $dst, $dst\n\t"
9454             "negl    $dst" %}
9455   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9456              setLT_reg(dst),
9457              REX_reg_breg(dst, dst), // movzbl
9458              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9459              neg_reg(dst));
9460   ins_pipe(pipe_slow);
9461 %}
9462 
9463 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9464 %{
9465   match(Set dst (CmpLTMask dst zero));
9466   effect(KILL cr);
9467 
9468   ins_cost(100);
9469   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9470   ins_encode %{
9471   __ sarl($dst$$Register, 31);
9472   %}
9473   ins_pipe(ialu_reg);
9474 %}
9475 
9476 /* Better to save a register than avoid a branch */
9477 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9478 %{
9479   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9480   effect(KILL cr);
9481   ins_cost(300);
9482   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9483             "jge    done\n\t"
9484             "addl   $p,$y\n"
9485             "done:  " %}
9486   ins_encode %{
9487     Register Rp = $p$$Register;
9488     Register Rq = $q$$Register;
9489     Register Ry = $y$$Register;
9490     Label done;
9491     __ subl(Rp, Rq);
9492     __ jccb(Assembler::greaterEqual, done);
9493     __ addl(Rp, Ry);
9494     __ bind(done);
9495   %}
9496   ins_pipe(pipe_cmplt);
9497 %}
9498 
9499 /* Better to save a register than avoid a branch */
9500 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9501 %{
9502   match(Set y (AndI (CmpLTMask p q) y));
9503   effect(KILL cr);
9504 
9505   ins_cost(300);
9506 
9507   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9508             "jlt      done\n\t"
9509             "xorl     $y, $y\n"
9510             "done:  " %}
9511   ins_encode %{
9512     Register Rp = $p$$Register;
9513     Register Rq = $q$$Register;
9514     Register Ry = $y$$Register;
9515     Label done;
9516     __ cmpl(Rp, Rq);
9517     __ jccb(Assembler::less, done);
9518     __ xorl(Ry, Ry);
9519     __ bind(done);
9520   %}
9521   ins_pipe(pipe_cmplt);
9522 %}
9523 
9524 
9525 //---------- FP Instructions------------------------------------------------
9526 
9527 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9528 %{
9529   match(Set cr (CmpF src1 src2));
9530 
9531   ins_cost(145);
9532   format %{ "ucomiss $src1, $src2\n\t"
9533             "jnp,s   exit\n\t"
9534             "pushfq\t# saw NaN, set CF\n\t"
9535             "andq    [rsp], #0xffffff2b\n\t"
9536             "popfq\n"
9537     "exit:" %}
9538   ins_encode %{
9539     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9540     emit_cmpfp_fixup(_masm);
9541   %}
9542   ins_pipe(pipe_slow);
9543 %}
9544 
9545 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9546   match(Set cr (CmpF src1 src2));
9547 
9548   ins_cost(100);
9549   format %{ "ucomiss $src1, $src2" %}
9550   ins_encode %{
9551     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9552   %}
9553   ins_pipe(pipe_slow);
9554 %}
9555 
9556 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9557 %{
9558   match(Set cr (CmpF src1 (LoadF src2)));
9559 
9560   ins_cost(145);
9561   format %{ "ucomiss $src1, $src2\n\t"
9562             "jnp,s   exit\n\t"
9563             "pushfq\t# saw NaN, set CF\n\t"
9564             "andq    [rsp], #0xffffff2b\n\t"
9565             "popfq\n"
9566     "exit:" %}
9567   ins_encode %{
9568     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9569     emit_cmpfp_fixup(_masm);
9570   %}
9571   ins_pipe(pipe_slow);
9572 %}
9573 
9574 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9575   match(Set cr (CmpF src1 (LoadF src2)));
9576 
9577   ins_cost(100);
9578   format %{ "ucomiss $src1, $src2" %}
9579   ins_encode %{
9580     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9581   %}
9582   ins_pipe(pipe_slow);
9583 %}
9584 
9585 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9586   match(Set cr (CmpF src con));
9587 
9588   ins_cost(145);
9589   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9590             "jnp,s   exit\n\t"
9591             "pushfq\t# saw NaN, set CF\n\t"
9592             "andq    [rsp], #0xffffff2b\n\t"
9593             "popfq\n"
9594     "exit:" %}
9595   ins_encode %{
9596     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9597     emit_cmpfp_fixup(_masm);
9598   %}
9599   ins_pipe(pipe_slow);
9600 %}
9601 
9602 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9603   match(Set cr (CmpF src con));
9604   ins_cost(100);
9605   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9606   ins_encode %{
9607     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9608   %}
9609   ins_pipe(pipe_slow);
9610 %}
9611 
9612 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9613 %{
9614   match(Set cr (CmpD src1 src2));
9615 
9616   ins_cost(145);
9617   format %{ "ucomisd $src1, $src2\n\t"
9618             "jnp,s   exit\n\t"
9619             "pushfq\t# saw NaN, set CF\n\t"
9620             "andq    [rsp], #0xffffff2b\n\t"
9621             "popfq\n"
9622     "exit:" %}
9623   ins_encode %{
9624     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9625     emit_cmpfp_fixup(_masm);
9626   %}
9627   ins_pipe(pipe_slow);
9628 %}
9629 
9630 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9631   match(Set cr (CmpD src1 src2));
9632 
9633   ins_cost(100);
9634   format %{ "ucomisd $src1, $src2 test" %}
9635   ins_encode %{
9636     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9637   %}
9638   ins_pipe(pipe_slow);
9639 %}
9640 
9641 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9642 %{
9643   match(Set cr (CmpD src1 (LoadD src2)));
9644 
9645   ins_cost(145);
9646   format %{ "ucomisd $src1, $src2\n\t"
9647             "jnp,s   exit\n\t"
9648             "pushfq\t# saw NaN, set CF\n\t"
9649             "andq    [rsp], #0xffffff2b\n\t"
9650             "popfq\n"
9651     "exit:" %}
9652   ins_encode %{
9653     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9654     emit_cmpfp_fixup(_masm);
9655   %}
9656   ins_pipe(pipe_slow);
9657 %}
9658 
9659 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9660   match(Set cr (CmpD src1 (LoadD src2)));
9661 
9662   ins_cost(100);
9663   format %{ "ucomisd $src1, $src2" %}
9664   ins_encode %{
9665     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9666   %}
9667   ins_pipe(pipe_slow);
9668 %}
9669 
9670 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9671   match(Set cr (CmpD src con));
9672 
9673   ins_cost(145);
9674   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9675             "jnp,s   exit\n\t"
9676             "pushfq\t# saw NaN, set CF\n\t"
9677             "andq    [rsp], #0xffffff2b\n\t"
9678             "popfq\n"
9679     "exit:" %}
9680   ins_encode %{
9681     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9682     emit_cmpfp_fixup(_masm);
9683   %}
9684   ins_pipe(pipe_slow);
9685 %}
9686 
9687 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9688   match(Set cr (CmpD src con));
9689   ins_cost(100);
9690   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9691   ins_encode %{
9692     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9693   %}
9694   ins_pipe(pipe_slow);
9695 %}
9696 
9697 // Compare into -1,0,1
9698 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9699 %{
9700   match(Set dst (CmpF3 src1 src2));
9701   effect(KILL cr);
9702 
9703   ins_cost(275);
9704   format %{ "ucomiss $src1, $src2\n\t"
9705             "movl    $dst, #-1\n\t"
9706             "jp,s    done\n\t"
9707             "jb,s    done\n\t"
9708             "setne   $dst\n\t"
9709             "movzbl  $dst, $dst\n"
9710     "done:" %}
9711   ins_encode %{
9712     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9713     emit_cmpfp3(_masm, $dst$$Register);
9714   %}
9715   ins_pipe(pipe_slow);
9716 %}
9717 
9718 // Compare into -1,0,1
9719 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9720 %{
9721   match(Set dst (CmpF3 src1 (LoadF src2)));
9722   effect(KILL cr);
9723 
9724   ins_cost(275);
9725   format %{ "ucomiss $src1, $src2\n\t"
9726             "movl    $dst, #-1\n\t"
9727             "jp,s    done\n\t"
9728             "jb,s    done\n\t"
9729             "setne   $dst\n\t"
9730             "movzbl  $dst, $dst\n"
9731     "done:" %}
9732   ins_encode %{
9733     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9734     emit_cmpfp3(_masm, $dst$$Register);
9735   %}
9736   ins_pipe(pipe_slow);
9737 %}
9738 
9739 // Compare into -1,0,1
9740 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9741   match(Set dst (CmpF3 src con));
9742   effect(KILL cr);
9743 
9744   ins_cost(275);
9745   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9746             "movl    $dst, #-1\n\t"
9747             "jp,s    done\n\t"
9748             "jb,s    done\n\t"
9749             "setne   $dst\n\t"
9750             "movzbl  $dst, $dst\n"
9751     "done:" %}
9752   ins_encode %{
9753     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9754     emit_cmpfp3(_masm, $dst$$Register);
9755   %}
9756   ins_pipe(pipe_slow);
9757 %}
9758 
9759 // Compare into -1,0,1
9760 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9761 %{
9762   match(Set dst (CmpD3 src1 src2));
9763   effect(KILL cr);
9764 
9765   ins_cost(275);
9766   format %{ "ucomisd $src1, $src2\n\t"
9767             "movl    $dst, #-1\n\t"
9768             "jp,s    done\n\t"
9769             "jb,s    done\n\t"
9770             "setne   $dst\n\t"
9771             "movzbl  $dst, $dst\n"
9772     "done:" %}
9773   ins_encode %{
9774     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9775     emit_cmpfp3(_masm, $dst$$Register);
9776   %}
9777   ins_pipe(pipe_slow);
9778 %}
9779 
9780 // Compare into -1,0,1
9781 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9782 %{
9783   match(Set dst (CmpD3 src1 (LoadD src2)));
9784   effect(KILL cr);
9785 
9786   ins_cost(275);
9787   format %{ "ucomisd $src1, $src2\n\t"
9788             "movl    $dst, #-1\n\t"
9789             "jp,s    done\n\t"
9790             "jb,s    done\n\t"
9791             "setne   $dst\n\t"
9792             "movzbl  $dst, $dst\n"
9793     "done:" %}
9794   ins_encode %{
9795     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9796     emit_cmpfp3(_masm, $dst$$Register);
9797   %}
9798   ins_pipe(pipe_slow);
9799 %}
9800 
9801 // Compare into -1,0,1
9802 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9803   match(Set dst (CmpD3 src con));
9804   effect(KILL cr);
9805 
9806   ins_cost(275);
9807   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9808             "movl    $dst, #-1\n\t"
9809             "jp,s    done\n\t"
9810             "jb,s    done\n\t"
9811             "setne   $dst\n\t"
9812             "movzbl  $dst, $dst\n"
9813     "done:" %}
9814   ins_encode %{
9815     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9816     emit_cmpfp3(_masm, $dst$$Register);
9817   %}
9818   ins_pipe(pipe_slow);
9819 %}
9820 
9821 // -----------Trig and Trancendental Instructions------------------------------
9822 instruct cosD_reg(regD dst) %{
9823   match(Set dst (CosD dst));
9824 
9825   format %{ "dcos   $dst\n\t" %}
9826   opcode(0xD9, 0xFF);
9827   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9828   ins_pipe( pipe_slow );
9829 %}
9830 
9831 instruct sinD_reg(regD dst) %{
9832   match(Set dst (SinD dst));
9833 
9834   format %{ "dsin   $dst\n\t" %}
9835   opcode(0xD9, 0xFE);
9836   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9837   ins_pipe( pipe_slow );
9838 %}
9839 
9840 instruct tanD_reg(regD dst) %{
9841   match(Set dst (TanD dst));
9842 
9843   format %{ "dtan   $dst\n\t" %}
9844   ins_encode( Push_SrcXD(dst),
9845               Opcode(0xD9), Opcode(0xF2),   //fptan
9846               Opcode(0xDD), Opcode(0xD8),   //fstp st
9847               Push_ResultXD(dst) );
9848   ins_pipe( pipe_slow );
9849 %}
9850 
9851 instruct log10D_reg(regD dst) %{
9852   // The source and result Double operands in XMM registers
9853   match(Set dst (Log10D dst));
9854   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9855   // fyl2x        ; compute log_10(2) * log_2(x)
9856   format %{ "fldlg2\t\t\t#Log10\n\t"
9857             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9858          %}
9859    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9860               Push_SrcXD(dst),
9861               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9862               Push_ResultXD(dst));
9863 
9864   ins_pipe( pipe_slow );
9865 %}
9866 
9867 //----------Arithmetic Conversion Instructions---------------------------------
9868 
9869 instruct roundFloat_nop(regF dst)
9870 %{
9871   match(Set dst (RoundFloat dst));
9872 
9873   ins_cost(0);
9874   ins_encode();
9875   ins_pipe(empty);
9876 %}
9877 
9878 instruct roundDouble_nop(regD dst)
9879 %{
9880   match(Set dst (RoundDouble dst));
9881 
9882   ins_cost(0);
9883   ins_encode();
9884   ins_pipe(empty);
9885 %}
9886 
9887 instruct convF2D_reg_reg(regD dst, regF src)
9888 %{
9889   match(Set dst (ConvF2D src));
9890 
9891   format %{ "cvtss2sd $dst, $src" %}
9892   ins_encode %{
9893     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9894   %}
9895   ins_pipe(pipe_slow); // XXX
9896 %}
9897 
9898 instruct convF2D_reg_mem(regD dst, memory src)
9899 %{
9900   match(Set dst (ConvF2D (LoadF src)));
9901 
9902   format %{ "cvtss2sd $dst, $src" %}
9903   ins_encode %{
9904     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9905   %}
9906   ins_pipe(pipe_slow); // XXX
9907 %}
9908 
9909 instruct convD2F_reg_reg(regF dst, regD src)
9910 %{
9911   match(Set dst (ConvD2F src));
9912 
9913   format %{ "cvtsd2ss $dst, $src" %}
9914   ins_encode %{
9915     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9916   %}
9917   ins_pipe(pipe_slow); // XXX
9918 %}
9919 
9920 instruct convD2F_reg_mem(regF dst, memory src)
9921 %{
9922   match(Set dst (ConvD2F (LoadD src)));
9923 
9924   format %{ "cvtsd2ss $dst, $src" %}
9925   ins_encode %{
9926     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9927   %}
9928   ins_pipe(pipe_slow); // XXX
9929 %}
9930 
9931 // XXX do mem variants
9932 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9933 %{
9934   match(Set dst (ConvF2I src));
9935   effect(KILL cr);
9936 
9937   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9938             "cmpl    $dst, #0x80000000\n\t"
9939             "jne,s   done\n\t"
9940             "subq    rsp, #8\n\t"
9941             "movss   [rsp], $src\n\t"
9942             "call    f2i_fixup\n\t"
9943             "popq    $dst\n"
9944     "done:   "%}
9945   ins_encode %{
9946     Label done;
9947     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9948     __ cmpl($dst$$Register, 0x80000000);
9949     __ jccb(Assembler::notEqual, done);
9950     __ subptr(rsp, 8);
9951     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9952     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9953     __ pop($dst$$Register);
9954     __ bind(done);
9955   %}
9956   ins_pipe(pipe_slow);
9957 %}
9958 
9959 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9960 %{
9961   match(Set dst (ConvF2L src));
9962   effect(KILL cr);
9963 
9964   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
9965             "cmpq    $dst, [0x8000000000000000]\n\t"
9966             "jne,s   done\n\t"
9967             "subq    rsp, #8\n\t"
9968             "movss   [rsp], $src\n\t"
9969             "call    f2l_fixup\n\t"
9970             "popq    $dst\n"
9971     "done:   "%}
9972   ins_encode %{
9973     Label done;
9974     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
9975     __ cmp64($dst$$Register,
9976              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
9977     __ jccb(Assembler::notEqual, done);
9978     __ subptr(rsp, 8);
9979     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9980     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
9981     __ pop($dst$$Register);
9982     __ bind(done);
9983   %}
9984   ins_pipe(pipe_slow);
9985 %}
9986 
9987 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
9988 %{
9989   match(Set dst (ConvD2I src));
9990   effect(KILL cr);
9991 
9992   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
9993             "cmpl    $dst, #0x80000000\n\t"
9994             "jne,s   done\n\t"
9995             "subq    rsp, #8\n\t"
9996             "movsd   [rsp], $src\n\t"
9997             "call    d2i_fixup\n\t"
9998             "popq    $dst\n"
9999     "done:   "%}
10000   ins_encode %{
10001     Label done;
10002     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10003     __ cmpl($dst$$Register, 0x80000000);
10004     __ jccb(Assembler::notEqual, done);
10005     __ subptr(rsp, 8);
10006     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10007     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10008     __ pop($dst$$Register);
10009     __ bind(done);
10010   %}
10011   ins_pipe(pipe_slow);
10012 %}
10013 
10014 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10015 %{
10016   match(Set dst (ConvD2L src));
10017   effect(KILL cr);
10018 
10019   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10020             "cmpq    $dst, [0x8000000000000000]\n\t"
10021             "jne,s   done\n\t"
10022             "subq    rsp, #8\n\t"
10023             "movsd   [rsp], $src\n\t"
10024             "call    d2l_fixup\n\t"
10025             "popq    $dst\n"
10026     "done:   "%}
10027   ins_encode %{
10028     Label done;
10029     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10030     __ cmp64($dst$$Register,
10031              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10032     __ jccb(Assembler::notEqual, done);
10033     __ subptr(rsp, 8);
10034     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10035     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10036     __ pop($dst$$Register);
10037     __ bind(done);
10038   %}
10039   ins_pipe(pipe_slow);
10040 %}
10041 
10042 instruct convI2F_reg_reg(regF dst, rRegI src)
10043 %{
10044   predicate(!UseXmmI2F);
10045   match(Set dst (ConvI2F src));
10046 
10047   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10048   ins_encode %{
10049     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10050   %}
10051   ins_pipe(pipe_slow); // XXX
10052 %}
10053 
10054 instruct convI2F_reg_mem(regF dst, memory src)
10055 %{
10056   match(Set dst (ConvI2F (LoadI src)));
10057 
10058   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10059   ins_encode %{
10060     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10061   %}
10062   ins_pipe(pipe_slow); // XXX
10063 %}
10064 
10065 instruct convI2D_reg_reg(regD dst, rRegI src)
10066 %{
10067   predicate(!UseXmmI2D);
10068   match(Set dst (ConvI2D src));
10069 
10070   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10071   ins_encode %{
10072     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10073   %}
10074   ins_pipe(pipe_slow); // XXX
10075 %}
10076 
10077 instruct convI2D_reg_mem(regD dst, memory src)
10078 %{
10079   match(Set dst (ConvI2D (LoadI src)));
10080 
10081   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10082   ins_encode %{
10083     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10084   %}
10085   ins_pipe(pipe_slow); // XXX
10086 %}
10087 
10088 instruct convXI2F_reg(regF dst, rRegI src)
10089 %{
10090   predicate(UseXmmI2F);
10091   match(Set dst (ConvI2F src));
10092 
10093   format %{ "movdl $dst, $src\n\t"
10094             "cvtdq2psl $dst, $dst\t# i2f" %}
10095   ins_encode %{
10096     __ movdl($dst$$XMMRegister, $src$$Register);
10097     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10098   %}
10099   ins_pipe(pipe_slow); // XXX
10100 %}
10101 
10102 instruct convXI2D_reg(regD dst, rRegI src)
10103 %{
10104   predicate(UseXmmI2D);
10105   match(Set dst (ConvI2D src));
10106 
10107   format %{ "movdl $dst, $src\n\t"
10108             "cvtdq2pdl $dst, $dst\t# i2d" %}
10109   ins_encode %{
10110     __ movdl($dst$$XMMRegister, $src$$Register);
10111     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10112   %}
10113   ins_pipe(pipe_slow); // XXX
10114 %}
10115 
10116 instruct convL2F_reg_reg(regF dst, rRegL src)
10117 %{
10118   match(Set dst (ConvL2F src));
10119 
10120   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10121   ins_encode %{
10122     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10123   %}
10124   ins_pipe(pipe_slow); // XXX
10125 %}
10126 
10127 instruct convL2F_reg_mem(regF dst, memory src)
10128 %{
10129   match(Set dst (ConvL2F (LoadL src)));
10130 
10131   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10132   ins_encode %{
10133     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10134   %}
10135   ins_pipe(pipe_slow); // XXX
10136 %}
10137 
10138 instruct convL2D_reg_reg(regD dst, rRegL src)
10139 %{
10140   match(Set dst (ConvL2D src));
10141 
10142   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10143   ins_encode %{
10144     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10145   %}
10146   ins_pipe(pipe_slow); // XXX
10147 %}
10148 
10149 instruct convL2D_reg_mem(regD dst, memory src)
10150 %{
10151   match(Set dst (ConvL2D (LoadL src)));
10152 
10153   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10154   ins_encode %{
10155     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10156   %}
10157   ins_pipe(pipe_slow); // XXX
10158 %}
10159 
10160 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10161 %{
10162   match(Set dst (ConvI2L src));
10163 
10164   ins_cost(125);
10165   format %{ "movslq  $dst, $src\t# i2l" %}
10166   ins_encode %{
10167     __ movslq($dst$$Register, $src$$Register);
10168   %}
10169   ins_pipe(ialu_reg_reg);
10170 %}
10171 
10172 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10173 // %{
10174 //   match(Set dst (ConvI2L src));
10175 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10176 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10177 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10178 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10179 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10180 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10181 
10182 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10183 //   ins_encode(enc_copy(dst, src));
10184 // //   opcode(0x63); // needs REX.W
10185 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10186 //   ins_pipe(ialu_reg_reg);
10187 // %}
10188 
10189 // Zero-extend convert int to long
10190 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10191 %{
10192   match(Set dst (AndL (ConvI2L src) mask));
10193 
10194   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10195   ins_encode %{
10196     if ($dst$$reg != $src$$reg) {
10197       __ movl($dst$$Register, $src$$Register);
10198     }
10199   %}
10200   ins_pipe(ialu_reg_reg);
10201 %}
10202 
10203 // Zero-extend convert int to long
10204 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10205 %{
10206   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10207 
10208   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10209   ins_encode %{
10210     __ movl($dst$$Register, $src$$Address);
10211   %}
10212   ins_pipe(ialu_reg_mem);
10213 %}
10214 
10215 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10216 %{
10217   match(Set dst (AndL src mask));
10218 
10219   format %{ "movl    $dst, $src\t# zero-extend long" %}
10220   ins_encode %{
10221     __ movl($dst$$Register, $src$$Register);
10222   %}
10223   ins_pipe(ialu_reg_reg);
10224 %}
10225 
10226 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10227 %{
10228   match(Set dst (ConvL2I src));
10229 
10230   format %{ "movl    $dst, $src\t# l2i" %}
10231   ins_encode %{
10232     __ movl($dst$$Register, $src$$Register);
10233   %}
10234   ins_pipe(ialu_reg_reg);
10235 %}
10236 
10237 
10238 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10239   match(Set dst (MoveF2I src));
10240   effect(DEF dst, USE src);
10241 
10242   ins_cost(125);
10243   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10244   ins_encode %{
10245     __ movl($dst$$Register, Address(rsp, $src$$disp));
10246   %}
10247   ins_pipe(ialu_reg_mem);
10248 %}
10249 
10250 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10251   match(Set dst (MoveI2F src));
10252   effect(DEF dst, USE src);
10253 
10254   ins_cost(125);
10255   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10256   ins_encode %{
10257     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10258   %}
10259   ins_pipe(pipe_slow);
10260 %}
10261 
10262 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10263   match(Set dst (MoveD2L src));
10264   effect(DEF dst, USE src);
10265 
10266   ins_cost(125);
10267   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10268   ins_encode %{
10269     __ movq($dst$$Register, Address(rsp, $src$$disp));
10270   %}
10271   ins_pipe(ialu_reg_mem);
10272 %}
10273 
10274 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10275   predicate(!UseXmmLoadAndClearUpper);
10276   match(Set dst (MoveL2D src));
10277   effect(DEF dst, USE src);
10278 
10279   ins_cost(125);
10280   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10281   ins_encode %{
10282     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10283   %}
10284   ins_pipe(pipe_slow);
10285 %}
10286 
10287 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10288   predicate(UseXmmLoadAndClearUpper);
10289   match(Set dst (MoveL2D src));
10290   effect(DEF dst, USE src);
10291 
10292   ins_cost(125);
10293   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10294   ins_encode %{
10295     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10296   %}
10297   ins_pipe(pipe_slow);
10298 %}
10299 
10300 
10301 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10302   match(Set dst (MoveF2I src));
10303   effect(DEF dst, USE src);
10304 
10305   ins_cost(95); // XXX
10306   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10307   ins_encode %{
10308     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10309   %}
10310   ins_pipe(pipe_slow);
10311 %}
10312 
10313 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10314   match(Set dst (MoveI2F src));
10315   effect(DEF dst, USE src);
10316 
10317   ins_cost(100);
10318   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10319   ins_encode %{
10320     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10321   %}
10322   ins_pipe( ialu_mem_reg );
10323 %}
10324 
10325 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10326   match(Set dst (MoveD2L src));
10327   effect(DEF dst, USE src);
10328 
10329   ins_cost(95); // XXX
10330   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10331   ins_encode %{
10332     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10333   %}
10334   ins_pipe(pipe_slow);
10335 %}
10336 
10337 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10338   match(Set dst (MoveL2D src));
10339   effect(DEF dst, USE src);
10340 
10341   ins_cost(100);
10342   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10343   ins_encode %{
10344     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10345   %}
10346   ins_pipe(ialu_mem_reg);
10347 %}
10348 
10349 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10350   match(Set dst (MoveF2I src));
10351   effect(DEF dst, USE src);
10352   ins_cost(85);
10353   format %{ "movd    $dst,$src\t# MoveF2I" %}
10354   ins_encode %{
10355     __ movdl($dst$$Register, $src$$XMMRegister);
10356   %}
10357   ins_pipe( pipe_slow );
10358 %}
10359 
10360 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10361   match(Set dst (MoveD2L src));
10362   effect(DEF dst, USE src);
10363   ins_cost(85);
10364   format %{ "movd    $dst,$src\t# MoveD2L" %}
10365   ins_encode %{
10366     __ movdq($dst$$Register, $src$$XMMRegister);
10367   %}
10368   ins_pipe( pipe_slow );
10369 %}
10370 
10371 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10372   match(Set dst (MoveI2F src));
10373   effect(DEF dst, USE src);
10374   ins_cost(100);
10375   format %{ "movd    $dst,$src\t# MoveI2F" %}
10376   ins_encode %{
10377     __ movdl($dst$$XMMRegister, $src$$Register);
10378   %}
10379   ins_pipe( pipe_slow );
10380 %}
10381 
10382 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10383   match(Set dst (MoveL2D src));
10384   effect(DEF dst, USE src);
10385   ins_cost(100);
10386   format %{ "movd    $dst,$src\t# MoveL2D" %}
10387   ins_encode %{
10388      __ movdq($dst$$XMMRegister, $src$$Register);
10389   %}
10390   ins_pipe( pipe_slow );
10391 %}
10392 
10393 
10394 // =======================================================================
10395 // fast clearing of an array
10396 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10397                   rFlagsReg cr)
10398 %{
10399   predicate(!UseFastStosb);
10400   match(Set dummy (ClearArray cnt base));
10401   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10402 
10403   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10404             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10405   ins_encode %{
10406     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10407   %}
10408   ins_pipe(pipe_slow);
10409 %}
10410 
10411 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10412                         rFlagsReg cr)
10413 %{
10414   predicate(UseFastStosb);
10415   match(Set dummy (ClearArray cnt base));
10416   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10417   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10418             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10419             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10420   ins_encode %{
10421     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10422   %}
10423   ins_pipe( pipe_slow );
10424 %}
10425 
10426 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10427                          rax_RegI result, regD tmp1, rFlagsReg cr)
10428 %{
10429   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10430   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10431   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10432 
10433   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10434   ins_encode %{
10435     __ string_compare($str1$$Register, $str2$$Register,
10436                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10437                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10438   %}
10439   ins_pipe( pipe_slow );
10440 %}
10441 
10442 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10443                          rax_RegI result, regD tmp1, rFlagsReg cr)
10444 %{
10445   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10446   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10447   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10448 
10449   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10450   ins_encode %{
10451     __ string_compare($str1$$Register, $str2$$Register,
10452                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10453                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10454   %}
10455   ins_pipe( pipe_slow );
10456 %}
10457 
10458 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10459                           rax_RegI result, regD tmp1, rFlagsReg cr)
10460 %{
10461   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10462   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10463   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10464 
10465   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10466   ins_encode %{
10467     __ string_compare($str1$$Register, $str2$$Register,
10468                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10469                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10470   %}
10471   ins_pipe( pipe_slow );
10472 %}
10473 
10474 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10475                           rax_RegI result, regD tmp1, rFlagsReg cr)
10476 %{
10477   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10478   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10479   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10480 
10481   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10482   ins_encode %{
10483     __ string_compare($str2$$Register, $str1$$Register,
10484                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10485                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10486   %}
10487   ins_pipe( pipe_slow );
10488 %}
10489 
10490 // fast search of substring with known size.
10491 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10492                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10493 %{
10494   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10495   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10496   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10497 
10498   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10499   ins_encode %{
10500     int icnt2 = (int)$int_cnt2$$constant;
10501     if (icnt2 >= 16) {
10502       // IndexOf for constant substrings with size >= 16 elements
10503       // which don't need to be loaded through stack.
10504       __ string_indexofC8($str1$$Register, $str2$$Register,
10505                           $cnt1$$Register, $cnt2$$Register,
10506                           icnt2, $result$$Register,
10507                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10508     } else {
10509       // Small strings are loaded through stack if they cross page boundary.
10510       __ string_indexof($str1$$Register, $str2$$Register,
10511                         $cnt1$$Register, $cnt2$$Register,
10512                         icnt2, $result$$Register,
10513                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10514     }
10515   %}
10516   ins_pipe( pipe_slow );
10517 %}
10518 
10519 // fast search of substring with known size.
10520 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10521                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10522 %{
10523   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10524   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10525   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10526 
10527   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10528   ins_encode %{
10529     int icnt2 = (int)$int_cnt2$$constant;
10530     if (icnt2 >= 8) {
10531       // IndexOf for constant substrings with size >= 8 elements
10532       // which don't need to be loaded through stack.
10533       __ string_indexofC8($str1$$Register, $str2$$Register,
10534                           $cnt1$$Register, $cnt2$$Register,
10535                           icnt2, $result$$Register,
10536                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10537     } else {
10538       // Small strings are loaded through stack if they cross page boundary.
10539       __ string_indexof($str1$$Register, $str2$$Register,
10540                         $cnt1$$Register, $cnt2$$Register,
10541                         icnt2, $result$$Register,
10542                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10543     }
10544   %}
10545   ins_pipe( pipe_slow );
10546 %}
10547 
10548 // fast search of substring with known size.
10549 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10550                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10551 %{
10552   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10553   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10554   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10555 
10556   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10557   ins_encode %{
10558     int icnt2 = (int)$int_cnt2$$constant;
10559     if (icnt2 >= 8) {
10560       // IndexOf for constant substrings with size >= 8 elements
10561       // which don't need to be loaded through stack.
10562       __ string_indexofC8($str1$$Register, $str2$$Register,
10563                           $cnt1$$Register, $cnt2$$Register,
10564                           icnt2, $result$$Register,
10565                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10566     } else {
10567       // Small strings are loaded through stack if they cross page boundary.
10568       __ string_indexof($str1$$Register, $str2$$Register,
10569                         $cnt1$$Register, $cnt2$$Register,
10570                         icnt2, $result$$Register,
10571                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10572     }
10573   %}
10574   ins_pipe( pipe_slow );
10575 %}
10576 
10577 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10578                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10579 %{
10580   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10581   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10582   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10583 
10584   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10585   ins_encode %{
10586     __ string_indexof($str1$$Register, $str2$$Register,
10587                       $cnt1$$Register, $cnt2$$Register,
10588                       (-1), $result$$Register,
10589                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10590   %}
10591   ins_pipe( pipe_slow );
10592 %}
10593 
10594 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10595                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10596 %{
10597   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10598   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10599   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10600 
10601   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10602   ins_encode %{
10603     __ string_indexof($str1$$Register, $str2$$Register,
10604                       $cnt1$$Register, $cnt2$$Register,
10605                       (-1), $result$$Register,
10606                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10607   %}
10608   ins_pipe( pipe_slow );
10609 %}
10610 
10611 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10612                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10613 %{
10614   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10615   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10616   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10617 
10618   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10619   ins_encode %{
10620     __ string_indexof($str1$$Register, $str2$$Register,
10621                       $cnt1$$Register, $cnt2$$Register,
10622                       (-1), $result$$Register,
10623                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10624   %}
10625   ins_pipe( pipe_slow );
10626 %}
10627 
10628 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10629                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10630 %{
10631   predicate(UseSSE42Intrinsics);
10632   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10633   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10634   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10635   ins_encode %{
10636     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10637                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10638   %}
10639   ins_pipe( pipe_slow );
10640 %}
10641 
10642 // fast string equals
10643 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10644                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10645 %{
10646   match(Set result (StrEquals (Binary str1 str2) cnt));
10647   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10648 
10649   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10650   ins_encode %{
10651     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10652                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10653                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10654   %}
10655   ins_pipe( pipe_slow );
10656 %}
10657 
10658 // fast array equals
10659 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10660                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10661 %{
10662   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10663   match(Set result (AryEq ary1 ary2));
10664   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10665 
10666   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10667   ins_encode %{
10668     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10669                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10670                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10671   %}
10672   ins_pipe( pipe_slow );
10673 %}
10674 
10675 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10676                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10677 %{
10678   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10679   match(Set result (AryEq ary1 ary2));
10680   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10681 
10682   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10683   ins_encode %{
10684     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10685                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10686                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10687   %}
10688   ins_pipe( pipe_slow );
10689 %}
10690 
10691 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10692                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10693 %{
10694   match(Set result (HasNegatives ary1 len));
10695   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10696 
10697   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10698   ins_encode %{
10699     __ has_negatives($ary1$$Register, $len$$Register,
10700                      $result$$Register, $tmp3$$Register,
10701                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10702   %}
10703   ins_pipe( pipe_slow );
10704 %}
10705 
10706 // fast char[] to byte[] compression
10707 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10708                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10709   match(Set result (StrCompressedCopy src (Binary dst len)));
10710   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10711 
10712   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10713   ins_encode %{
10714     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10715                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10716                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10717   %}
10718   ins_pipe( pipe_slow );
10719 %}
10720 
10721 // fast byte[] to char[] inflation
10722 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10723                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10724   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10725   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10726 
10727   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10728   ins_encode %{
10729     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10730                           $tmp1$$XMMRegister, $tmp2$$Register);
10731   %}
10732   ins_pipe( pipe_slow );
10733 %}
10734 
10735 // encode char[] to byte[] in ISO_8859_1
10736 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10737                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10738                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10739   match(Set result (EncodeISOArray src (Binary dst len)));
10740   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10741 
10742   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10743   ins_encode %{
10744     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10745                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10746                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10747   %}
10748   ins_pipe( pipe_slow );
10749 %}
10750 
10751 //----------Overflow Math Instructions-----------------------------------------
10752 
10753 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10754 %{
10755   match(Set cr (OverflowAddI op1 op2));
10756   effect(DEF cr, USE_KILL op1, USE op2);
10757 
10758   format %{ "addl    $op1, $op2\t# overflow check int" %}
10759 
10760   ins_encode %{
10761     __ addl($op1$$Register, $op2$$Register);
10762   %}
10763   ins_pipe(ialu_reg_reg);
10764 %}
10765 
10766 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10767 %{
10768   match(Set cr (OverflowAddI op1 op2));
10769   effect(DEF cr, USE_KILL op1, USE op2);
10770 
10771   format %{ "addl    $op1, $op2\t# overflow check int" %}
10772 
10773   ins_encode %{
10774     __ addl($op1$$Register, $op2$$constant);
10775   %}
10776   ins_pipe(ialu_reg_reg);
10777 %}
10778 
10779 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10780 %{
10781   match(Set cr (OverflowAddL op1 op2));
10782   effect(DEF cr, USE_KILL op1, USE op2);
10783 
10784   format %{ "addq    $op1, $op2\t# overflow check long" %}
10785   ins_encode %{
10786     __ addq($op1$$Register, $op2$$Register);
10787   %}
10788   ins_pipe(ialu_reg_reg);
10789 %}
10790 
10791 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10792 %{
10793   match(Set cr (OverflowAddL op1 op2));
10794   effect(DEF cr, USE_KILL op1, USE op2);
10795 
10796   format %{ "addq    $op1, $op2\t# overflow check long" %}
10797   ins_encode %{
10798     __ addq($op1$$Register, $op2$$constant);
10799   %}
10800   ins_pipe(ialu_reg_reg);
10801 %}
10802 
10803 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10804 %{
10805   match(Set cr (OverflowSubI op1 op2));
10806 
10807   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10808   ins_encode %{
10809     __ cmpl($op1$$Register, $op2$$Register);
10810   %}
10811   ins_pipe(ialu_reg_reg);
10812 %}
10813 
10814 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10815 %{
10816   match(Set cr (OverflowSubI op1 op2));
10817 
10818   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10819   ins_encode %{
10820     __ cmpl($op1$$Register, $op2$$constant);
10821   %}
10822   ins_pipe(ialu_reg_reg);
10823 %}
10824 
10825 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10826 %{
10827   match(Set cr (OverflowSubL op1 op2));
10828 
10829   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10830   ins_encode %{
10831     __ cmpq($op1$$Register, $op2$$Register);
10832   %}
10833   ins_pipe(ialu_reg_reg);
10834 %}
10835 
10836 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10837 %{
10838   match(Set cr (OverflowSubL op1 op2));
10839 
10840   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10841   ins_encode %{
10842     __ cmpq($op1$$Register, $op2$$constant);
10843   %}
10844   ins_pipe(ialu_reg_reg);
10845 %}
10846 
10847 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10848 %{
10849   match(Set cr (OverflowSubI zero op2));
10850   effect(DEF cr, USE_KILL op2);
10851 
10852   format %{ "negl    $op2\t# overflow check int" %}
10853   ins_encode %{
10854     __ negl($op2$$Register);
10855   %}
10856   ins_pipe(ialu_reg_reg);
10857 %}
10858 
10859 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10860 %{
10861   match(Set cr (OverflowSubL zero op2));
10862   effect(DEF cr, USE_KILL op2);
10863 
10864   format %{ "negq    $op2\t# overflow check long" %}
10865   ins_encode %{
10866     __ negq($op2$$Register);
10867   %}
10868   ins_pipe(ialu_reg_reg);
10869 %}
10870 
10871 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10872 %{
10873   match(Set cr (OverflowMulI op1 op2));
10874   effect(DEF cr, USE_KILL op1, USE op2);
10875 
10876   format %{ "imull    $op1, $op2\t# overflow check int" %}
10877   ins_encode %{
10878     __ imull($op1$$Register, $op2$$Register);
10879   %}
10880   ins_pipe(ialu_reg_reg_alu0);
10881 %}
10882 
10883 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10884 %{
10885   match(Set cr (OverflowMulI op1 op2));
10886   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10887 
10888   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10889   ins_encode %{
10890     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10891   %}
10892   ins_pipe(ialu_reg_reg_alu0);
10893 %}
10894 
10895 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10896 %{
10897   match(Set cr (OverflowMulL op1 op2));
10898   effect(DEF cr, USE_KILL op1, USE op2);
10899 
10900   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10901   ins_encode %{
10902     __ imulq($op1$$Register, $op2$$Register);
10903   %}
10904   ins_pipe(ialu_reg_reg_alu0);
10905 %}
10906 
10907 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10908 %{
10909   match(Set cr (OverflowMulL op1 op2));
10910   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10911 
10912   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10913   ins_encode %{
10914     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10915   %}
10916   ins_pipe(ialu_reg_reg_alu0);
10917 %}
10918 
10919 
10920 //----------Control Flow Instructions------------------------------------------
10921 // Signed compare Instructions
10922 
10923 // XXX more variants!!
10924 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10925 %{
10926   match(Set cr (CmpI op1 op2));
10927   effect(DEF cr, USE op1, USE op2);
10928 
10929   format %{ "cmpl    $op1, $op2" %}
10930   opcode(0x3B);  /* Opcode 3B /r */
10931   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10932   ins_pipe(ialu_cr_reg_reg);
10933 %}
10934 
10935 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10936 %{
10937   match(Set cr (CmpI op1 op2));
10938 
10939   format %{ "cmpl    $op1, $op2" %}
10940   opcode(0x81, 0x07); /* Opcode 81 /7 */
10941   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10942   ins_pipe(ialu_cr_reg_imm);
10943 %}
10944 
10945 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10946 %{
10947   match(Set cr (CmpI op1 (LoadI op2)));
10948 
10949   ins_cost(500); // XXX
10950   format %{ "cmpl    $op1, $op2" %}
10951   opcode(0x3B); /* Opcode 3B /r */
10952   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10953   ins_pipe(ialu_cr_reg_mem);
10954 %}
10955 
10956 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10957 %{
10958   match(Set cr (CmpI src zero));
10959 
10960   format %{ "testl   $src, $src" %}
10961   opcode(0x85);
10962   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10963   ins_pipe(ialu_cr_reg_imm);
10964 %}
10965 
10966 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10967 %{
10968   match(Set cr (CmpI (AndI src con) zero));
10969 
10970   format %{ "testl   $src, $con" %}
10971   opcode(0xF7, 0x00);
10972   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10973   ins_pipe(ialu_cr_reg_imm);
10974 %}
10975 
10976 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10977 %{
10978   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10979 
10980   format %{ "testl   $src, $mem" %}
10981   opcode(0x85);
10982   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10983   ins_pipe(ialu_cr_reg_mem);
10984 %}
10985 
10986 // Unsigned compare Instructions; really, same as signed except they
10987 // produce an rFlagsRegU instead of rFlagsReg.
10988 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10989 %{
10990   match(Set cr (CmpU op1 op2));
10991 
10992   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10993   opcode(0x3B); /* Opcode 3B /r */
10994   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10995   ins_pipe(ialu_cr_reg_reg);
10996 %}
10997 
10998 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10999 %{
11000   match(Set cr (CmpU op1 op2));
11001 
11002   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11003   opcode(0x81,0x07); /* Opcode 81 /7 */
11004   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11005   ins_pipe(ialu_cr_reg_imm);
11006 %}
11007 
11008 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11009 %{
11010   match(Set cr (CmpU op1 (LoadI op2)));
11011 
11012   ins_cost(500); // XXX
11013   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11014   opcode(0x3B); /* Opcode 3B /r */
11015   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11016   ins_pipe(ialu_cr_reg_mem);
11017 %}
11018 
11019 // // // Cisc-spilled version of cmpU_rReg
11020 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11021 // //%{
11022 // //  match(Set cr (CmpU (LoadI op1) op2));
11023 // //
11024 // //  format %{ "CMPu   $op1,$op2" %}
11025 // //  ins_cost(500);
11026 // //  opcode(0x39);  /* Opcode 39 /r */
11027 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11028 // //%}
11029 
11030 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11031 %{
11032   match(Set cr (CmpU src zero));
11033 
11034   format %{ "testl  $src, $src\t# unsigned" %}
11035   opcode(0x85);
11036   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11037   ins_pipe(ialu_cr_reg_imm);
11038 %}
11039 
11040 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11041 %{
11042   match(Set cr (CmpP op1 op2));
11043 
11044   format %{ "cmpq    $op1, $op2\t# ptr" %}
11045   opcode(0x3B); /* Opcode 3B /r */
11046   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11047   ins_pipe(ialu_cr_reg_reg);
11048 %}
11049 
11050 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11051 %{
11052   match(Set cr (CmpP op1 (LoadP op2)));
11053 
11054   ins_cost(500); // XXX
11055   format %{ "cmpq    $op1, $op2\t# ptr" %}
11056   opcode(0x3B); /* Opcode 3B /r */
11057   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11058   ins_pipe(ialu_cr_reg_mem);
11059 %}
11060 
11061 // // // Cisc-spilled version of cmpP_rReg
11062 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11063 // //%{
11064 // //  match(Set cr (CmpP (LoadP op1) op2));
11065 // //
11066 // //  format %{ "CMPu   $op1,$op2" %}
11067 // //  ins_cost(500);
11068 // //  opcode(0x39);  /* Opcode 39 /r */
11069 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11070 // //%}
11071 
11072 // XXX this is generalized by compP_rReg_mem???
11073 // Compare raw pointer (used in out-of-heap check).
11074 // Only works because non-oop pointers must be raw pointers
11075 // and raw pointers have no anti-dependencies.
11076 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11077 %{
11078   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11079   match(Set cr (CmpP op1 (LoadP op2)));
11080 
11081   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11082   opcode(0x3B); /* Opcode 3B /r */
11083   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11084   ins_pipe(ialu_cr_reg_mem);
11085 %}
11086 
11087 // This will generate a signed flags result. This should be OK since
11088 // any compare to a zero should be eq/neq.
11089 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11090 %{
11091   match(Set cr (CmpP src zero));
11092 
11093   format %{ "testq   $src, $src\t# ptr" %}
11094   opcode(0x85);
11095   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11096   ins_pipe(ialu_cr_reg_imm);
11097 %}
11098 
11099 // This will generate a signed flags result. This should be OK since
11100 // any compare to a zero should be eq/neq.
11101 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11102 %{
11103   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11104   match(Set cr (CmpP (LoadP op) zero));
11105 
11106   ins_cost(500); // XXX
11107   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11108   opcode(0xF7); /* Opcode F7 /0 */
11109   ins_encode(REX_mem_wide(op),
11110              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11111   ins_pipe(ialu_cr_reg_imm);
11112 %}
11113 
11114 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11115 %{
11116   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11117   match(Set cr (CmpP (LoadP mem) zero));
11118 
11119   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11120   ins_encode %{
11121     __ cmpq(r12, $mem$$Address);
11122   %}
11123   ins_pipe(ialu_cr_reg_mem);
11124 %}
11125 
11126 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11127 %{
11128   match(Set cr (CmpN op1 op2));
11129 
11130   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11131   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11132   ins_pipe(ialu_cr_reg_reg);
11133 %}
11134 
11135 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11136 %{
11137   match(Set cr (CmpN src (LoadN mem)));
11138 
11139   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11140   ins_encode %{
11141     __ cmpl($src$$Register, $mem$$Address);
11142   %}
11143   ins_pipe(ialu_cr_reg_mem);
11144 %}
11145 
11146 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11147   match(Set cr (CmpN op1 op2));
11148 
11149   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11150   ins_encode %{
11151     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11152   %}
11153   ins_pipe(ialu_cr_reg_imm);
11154 %}
11155 
11156 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11157 %{
11158   match(Set cr (CmpN src (LoadN mem)));
11159 
11160   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11161   ins_encode %{
11162     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11163   %}
11164   ins_pipe(ialu_cr_reg_mem);
11165 %}
11166 
11167 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11168   match(Set cr (CmpN op1 op2));
11169 
11170   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11171   ins_encode %{
11172     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11173   %}
11174   ins_pipe(ialu_cr_reg_imm);
11175 %}
11176 
11177 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11178 %{
11179   match(Set cr (CmpN src (LoadNKlass mem)));
11180 
11181   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11182   ins_encode %{
11183     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11184   %}
11185   ins_pipe(ialu_cr_reg_mem);
11186 %}
11187 
11188 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11189   match(Set cr (CmpN src zero));
11190 
11191   format %{ "testl   $src, $src\t# compressed ptr" %}
11192   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11193   ins_pipe(ialu_cr_reg_imm);
11194 %}
11195 
11196 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11197 %{
11198   predicate(Universe::narrow_oop_base() != NULL);
11199   match(Set cr (CmpN (LoadN mem) zero));
11200 
11201   ins_cost(500); // XXX
11202   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11203   ins_encode %{
11204     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11205   %}
11206   ins_pipe(ialu_cr_reg_mem);
11207 %}
11208 
11209 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11210 %{
11211   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11212   match(Set cr (CmpN (LoadN mem) zero));
11213 
11214   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11215   ins_encode %{
11216     __ cmpl(r12, $mem$$Address);
11217   %}
11218   ins_pipe(ialu_cr_reg_mem);
11219 %}
11220 
11221 // Yanked all unsigned pointer compare operations.
11222 // Pointer compares are done with CmpP which is already unsigned.
11223 
11224 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11225 %{
11226   match(Set cr (CmpL op1 op2));
11227 
11228   format %{ "cmpq    $op1, $op2" %}
11229   opcode(0x3B);  /* Opcode 3B /r */
11230   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11231   ins_pipe(ialu_cr_reg_reg);
11232 %}
11233 
11234 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11235 %{
11236   match(Set cr (CmpL op1 op2));
11237 
11238   format %{ "cmpq    $op1, $op2" %}
11239   opcode(0x81, 0x07); /* Opcode 81 /7 */
11240   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11241   ins_pipe(ialu_cr_reg_imm);
11242 %}
11243 
11244 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11245 %{
11246   match(Set cr (CmpL op1 (LoadL op2)));
11247 
11248   format %{ "cmpq    $op1, $op2" %}
11249   opcode(0x3B); /* Opcode 3B /r */
11250   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11251   ins_pipe(ialu_cr_reg_mem);
11252 %}
11253 
11254 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11255 %{
11256   match(Set cr (CmpL src zero));
11257 
11258   format %{ "testq   $src, $src" %}
11259   opcode(0x85);
11260   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11261   ins_pipe(ialu_cr_reg_imm);
11262 %}
11263 
11264 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11265 %{
11266   match(Set cr (CmpL (AndL src con) zero));
11267 
11268   format %{ "testq   $src, $con\t# long" %}
11269   opcode(0xF7, 0x00);
11270   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11271   ins_pipe(ialu_cr_reg_imm);
11272 %}
11273 
11274 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11275 %{
11276   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11277 
11278   format %{ "testq   $src, $mem" %}
11279   opcode(0x85);
11280   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11281   ins_pipe(ialu_cr_reg_mem);
11282 %}
11283 
11284 // Manifest a CmpL result in an integer register.  Very painful.
11285 // This is the test to avoid.
11286 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11287 %{
11288   match(Set dst (CmpL3 src1 src2));
11289   effect(KILL flags);
11290 
11291   ins_cost(275); // XXX
11292   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11293             "movl    $dst, -1\n\t"
11294             "jl,s    done\n\t"
11295             "setne   $dst\n\t"
11296             "movzbl  $dst, $dst\n\t"
11297     "done:" %}
11298   ins_encode(cmpl3_flag(src1, src2, dst));
11299   ins_pipe(pipe_slow);
11300 %}
11301 
11302 //----------Max and Min--------------------------------------------------------
11303 // Min Instructions
11304 
11305 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11306 %{
11307   effect(USE_DEF dst, USE src, USE cr);
11308 
11309   format %{ "cmovlgt $dst, $src\t# min" %}
11310   opcode(0x0F, 0x4F);
11311   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11312   ins_pipe(pipe_cmov_reg);
11313 %}
11314 
11315 
11316 instruct minI_rReg(rRegI dst, rRegI src)
11317 %{
11318   match(Set dst (MinI dst src));
11319 
11320   ins_cost(200);
11321   expand %{
11322     rFlagsReg cr;
11323     compI_rReg(cr, dst, src);
11324     cmovI_reg_g(dst, src, cr);
11325   %}
11326 %}
11327 
11328 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11329 %{
11330   effect(USE_DEF dst, USE src, USE cr);
11331 
11332   format %{ "cmovllt $dst, $src\t# max" %}
11333   opcode(0x0F, 0x4C);
11334   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11335   ins_pipe(pipe_cmov_reg);
11336 %}
11337 
11338 
11339 instruct maxI_rReg(rRegI dst, rRegI src)
11340 %{
11341   match(Set dst (MaxI dst src));
11342 
11343   ins_cost(200);
11344   expand %{
11345     rFlagsReg cr;
11346     compI_rReg(cr, dst, src);
11347     cmovI_reg_l(dst, src, cr);
11348   %}
11349 %}
11350 
11351 // ============================================================================
11352 // Branch Instructions
11353 
11354 // Jump Direct - Label defines a relative address from JMP+1
11355 instruct jmpDir(label labl)
11356 %{
11357   match(Goto);
11358   effect(USE labl);
11359 
11360   ins_cost(300);
11361   format %{ "jmp     $labl" %}
11362   size(5);
11363   ins_encode %{
11364     Label* L = $labl$$label;
11365     __ jmp(*L, false); // Always long jump
11366   %}
11367   ins_pipe(pipe_jmp);
11368 %}
11369 
11370 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11371 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11372 %{
11373   match(If cop cr);
11374   effect(USE labl);
11375 
11376   ins_cost(300);
11377   format %{ "j$cop     $labl" %}
11378   size(6);
11379   ins_encode %{
11380     Label* L = $labl$$label;
11381     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11382   %}
11383   ins_pipe(pipe_jcc);
11384 %}
11385 
11386 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11387 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11388 %{
11389   match(CountedLoopEnd cop cr);
11390   effect(USE labl);
11391 
11392   ins_cost(300);
11393   format %{ "j$cop     $labl\t# loop end" %}
11394   size(6);
11395   ins_encode %{
11396     Label* L = $labl$$label;
11397     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11398   %}
11399   ins_pipe(pipe_jcc);
11400 %}
11401 
11402 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11403 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11404   match(CountedLoopEnd cop cmp);
11405   effect(USE labl);
11406 
11407   ins_cost(300);
11408   format %{ "j$cop,u   $labl\t# loop end" %}
11409   size(6);
11410   ins_encode %{
11411     Label* L = $labl$$label;
11412     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11413   %}
11414   ins_pipe(pipe_jcc);
11415 %}
11416 
11417 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11418   match(CountedLoopEnd cop cmp);
11419   effect(USE labl);
11420 
11421   ins_cost(200);
11422   format %{ "j$cop,u   $labl\t# loop end" %}
11423   size(6);
11424   ins_encode %{
11425     Label* L = $labl$$label;
11426     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11427   %}
11428   ins_pipe(pipe_jcc);
11429 %}
11430 
11431 // Jump Direct Conditional - using unsigned comparison
11432 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11433   match(If cop cmp);
11434   effect(USE labl);
11435 
11436   ins_cost(300);
11437   format %{ "j$cop,u  $labl" %}
11438   size(6);
11439   ins_encode %{
11440     Label* L = $labl$$label;
11441     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11442   %}
11443   ins_pipe(pipe_jcc);
11444 %}
11445 
11446 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11447   match(If cop cmp);
11448   effect(USE labl);
11449 
11450   ins_cost(200);
11451   format %{ "j$cop,u  $labl" %}
11452   size(6);
11453   ins_encode %{
11454     Label* L = $labl$$label;
11455     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11456   %}
11457   ins_pipe(pipe_jcc);
11458 %}
11459 
11460 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11461   match(If cop cmp);
11462   effect(USE labl);
11463 
11464   ins_cost(200);
11465   format %{ $$template
11466     if ($cop$$cmpcode == Assembler::notEqual) {
11467       $$emit$$"jp,u   $labl\n\t"
11468       $$emit$$"j$cop,u   $labl"
11469     } else {
11470       $$emit$$"jp,u   done\n\t"
11471       $$emit$$"j$cop,u   $labl\n\t"
11472       $$emit$$"done:"
11473     }
11474   %}
11475   ins_encode %{
11476     Label* l = $labl$$label;
11477     if ($cop$$cmpcode == Assembler::notEqual) {
11478       __ jcc(Assembler::parity, *l, false);
11479       __ jcc(Assembler::notEqual, *l, false);
11480     } else if ($cop$$cmpcode == Assembler::equal) {
11481       Label done;
11482       __ jccb(Assembler::parity, done);
11483       __ jcc(Assembler::equal, *l, false);
11484       __ bind(done);
11485     } else {
11486        ShouldNotReachHere();
11487     }
11488   %}
11489   ins_pipe(pipe_jcc);
11490 %}
11491 
11492 // ============================================================================
11493 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11494 // superklass array for an instance of the superklass.  Set a hidden
11495 // internal cache on a hit (cache is checked with exposed code in
11496 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11497 // encoding ALSO sets flags.
11498 
11499 instruct partialSubtypeCheck(rdi_RegP result,
11500                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11501                              rFlagsReg cr)
11502 %{
11503   match(Set result (PartialSubtypeCheck sub super));
11504   effect(KILL rcx, KILL cr);
11505 
11506   ins_cost(1100);  // slightly larger than the next version
11507   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11508             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11509             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11510             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11511             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11512             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11513             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11514     "miss:\t" %}
11515 
11516   opcode(0x1); // Force a XOR of RDI
11517   ins_encode(enc_PartialSubtypeCheck());
11518   ins_pipe(pipe_slow);
11519 %}
11520 
11521 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11522                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11523                                      immP0 zero,
11524                                      rdi_RegP result)
11525 %{
11526   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11527   effect(KILL rcx, KILL result);
11528 
11529   ins_cost(1000);
11530   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11531             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11532             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11533             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11534             "jne,s   miss\t\t# Missed: flags nz\n\t"
11535             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11536     "miss:\t" %}
11537 
11538   opcode(0x0); // No need to XOR RDI
11539   ins_encode(enc_PartialSubtypeCheck());
11540   ins_pipe(pipe_slow);
11541 %}
11542 
11543 // ============================================================================
11544 // Branch Instructions -- short offset versions
11545 //
11546 // These instructions are used to replace jumps of a long offset (the default
11547 // match) with jumps of a shorter offset.  These instructions are all tagged
11548 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11549 // match rules in general matching.  Instead, the ADLC generates a conversion
11550 // method in the MachNode which can be used to do in-place replacement of the
11551 // long variant with the shorter variant.  The compiler will determine if a
11552 // branch can be taken by the is_short_branch_offset() predicate in the machine
11553 // specific code section of the file.
11554 
11555 // Jump Direct - Label defines a relative address from JMP+1
11556 instruct jmpDir_short(label labl) %{
11557   match(Goto);
11558   effect(USE labl);
11559 
11560   ins_cost(300);
11561   format %{ "jmp,s   $labl" %}
11562   size(2);
11563   ins_encode %{
11564     Label* L = $labl$$label;
11565     __ jmpb(*L);
11566   %}
11567   ins_pipe(pipe_jmp);
11568   ins_short_branch(1);
11569 %}
11570 
11571 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11572 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11573   match(If cop cr);
11574   effect(USE labl);
11575 
11576   ins_cost(300);
11577   format %{ "j$cop,s   $labl" %}
11578   size(2);
11579   ins_encode %{
11580     Label* L = $labl$$label;
11581     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11582   %}
11583   ins_pipe(pipe_jcc);
11584   ins_short_branch(1);
11585 %}
11586 
11587 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11588 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11589   match(CountedLoopEnd cop cr);
11590   effect(USE labl);
11591 
11592   ins_cost(300);
11593   format %{ "j$cop,s   $labl\t# loop end" %}
11594   size(2);
11595   ins_encode %{
11596     Label* L = $labl$$label;
11597     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11598   %}
11599   ins_pipe(pipe_jcc);
11600   ins_short_branch(1);
11601 %}
11602 
11603 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11604 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11605   match(CountedLoopEnd cop cmp);
11606   effect(USE labl);
11607 
11608   ins_cost(300);
11609   format %{ "j$cop,us  $labl\t# loop end" %}
11610   size(2);
11611   ins_encode %{
11612     Label* L = $labl$$label;
11613     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11614   %}
11615   ins_pipe(pipe_jcc);
11616   ins_short_branch(1);
11617 %}
11618 
11619 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11620   match(CountedLoopEnd cop cmp);
11621   effect(USE labl);
11622 
11623   ins_cost(300);
11624   format %{ "j$cop,us  $labl\t# loop end" %}
11625   size(2);
11626   ins_encode %{
11627     Label* L = $labl$$label;
11628     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11629   %}
11630   ins_pipe(pipe_jcc);
11631   ins_short_branch(1);
11632 %}
11633 
11634 // Jump Direct Conditional - using unsigned comparison
11635 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11636   match(If cop cmp);
11637   effect(USE labl);
11638 
11639   ins_cost(300);
11640   format %{ "j$cop,us  $labl" %}
11641   size(2);
11642   ins_encode %{
11643     Label* L = $labl$$label;
11644     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11645   %}
11646   ins_pipe(pipe_jcc);
11647   ins_short_branch(1);
11648 %}
11649 
11650 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11651   match(If cop cmp);
11652   effect(USE labl);
11653 
11654   ins_cost(300);
11655   format %{ "j$cop,us  $labl" %}
11656   size(2);
11657   ins_encode %{
11658     Label* L = $labl$$label;
11659     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11660   %}
11661   ins_pipe(pipe_jcc);
11662   ins_short_branch(1);
11663 %}
11664 
11665 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11666   match(If cop cmp);
11667   effect(USE labl);
11668 
11669   ins_cost(300);
11670   format %{ $$template
11671     if ($cop$$cmpcode == Assembler::notEqual) {
11672       $$emit$$"jp,u,s   $labl\n\t"
11673       $$emit$$"j$cop,u,s   $labl"
11674     } else {
11675       $$emit$$"jp,u,s   done\n\t"
11676       $$emit$$"j$cop,u,s  $labl\n\t"
11677       $$emit$$"done:"
11678     }
11679   %}
11680   size(4);
11681   ins_encode %{
11682     Label* l = $labl$$label;
11683     if ($cop$$cmpcode == Assembler::notEqual) {
11684       __ jccb(Assembler::parity, *l);
11685       __ jccb(Assembler::notEqual, *l);
11686     } else if ($cop$$cmpcode == Assembler::equal) {
11687       Label done;
11688       __ jccb(Assembler::parity, done);
11689       __ jccb(Assembler::equal, *l);
11690       __ bind(done);
11691     } else {
11692        ShouldNotReachHere();
11693     }
11694   %}
11695   ins_pipe(pipe_jcc);
11696   ins_short_branch(1);
11697 %}
11698 
11699 // ============================================================================
11700 // inlined locking and unlocking
11701 
11702 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11703   predicate(Compile::current()->use_rtm());
11704   match(Set cr (FastLock object box));
11705   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11706   ins_cost(300);
11707   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11708   ins_encode %{
11709     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11710                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11711                  _counters, _rtm_counters, _stack_rtm_counters,
11712                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11713                  true, ra_->C->profile_rtm());
11714   %}
11715   ins_pipe(pipe_slow);
11716 %}
11717 
11718 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11719   predicate(!Compile::current()->use_rtm());
11720   match(Set cr (FastLock object box));
11721   effect(TEMP tmp, TEMP scr, USE_KILL box);
11722   ins_cost(300);
11723   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11724   ins_encode %{
11725     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11726                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11727   %}
11728   ins_pipe(pipe_slow);
11729 %}
11730 
11731 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11732   match(Set cr (FastUnlock object box));
11733   effect(TEMP tmp, USE_KILL box);
11734   ins_cost(300);
11735   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11736   ins_encode %{
11737     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11738   %}
11739   ins_pipe(pipe_slow);
11740 %}
11741 
11742 
11743 // ============================================================================
11744 // Safepoint Instructions
11745 instruct safePoint_poll(rFlagsReg cr)
11746 %{
11747   predicate(!Assembler::is_polling_page_far());
11748   match(SafePoint);
11749   effect(KILL cr);
11750 
11751   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11752             "# Safepoint: poll for GC" %}
11753   ins_cost(125);
11754   ins_encode %{
11755     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11756     __ testl(rax, addr);
11757   %}
11758   ins_pipe(ialu_reg_mem);
11759 %}
11760 
11761 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11762 %{
11763   predicate(Assembler::is_polling_page_far());
11764   match(SafePoint poll);
11765   effect(KILL cr, USE poll);
11766 
11767   format %{ "testl  rax, [$poll]\t"
11768             "# Safepoint: poll for GC" %}
11769   ins_cost(125);
11770   ins_encode %{
11771     __ relocate(relocInfo::poll_type);
11772     __ testl(rax, Address($poll$$Register, 0));
11773   %}
11774   ins_pipe(ialu_reg_mem);
11775 %}
11776 
11777 // ============================================================================
11778 // Procedure Call/Return Instructions
11779 // Call Java Static Instruction
11780 // Note: If this code changes, the corresponding ret_addr_offset() and
11781 //       compute_padding() functions will have to be adjusted.
11782 instruct CallStaticJavaDirect(method meth) %{
11783   match(CallStaticJava);
11784   effect(USE meth);
11785 
11786   ins_cost(300);
11787   format %{ "call,static " %}
11788   opcode(0xE8); /* E8 cd */
11789   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11790   ins_pipe(pipe_slow);
11791   ins_alignment(4);
11792 %}
11793 
11794 // Call Java Dynamic Instruction
11795 // Note: If this code changes, the corresponding ret_addr_offset() and
11796 //       compute_padding() functions will have to be adjusted.
11797 instruct CallDynamicJavaDirect(method meth)
11798 %{
11799   match(CallDynamicJava);
11800   effect(USE meth);
11801 
11802   ins_cost(300);
11803   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11804             "call,dynamic " %}
11805   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11806   ins_pipe(pipe_slow);
11807   ins_alignment(4);
11808 %}
11809 
11810 // Call Runtime Instruction
11811 instruct CallRuntimeDirect(method meth)
11812 %{
11813   match(CallRuntime);
11814   effect(USE meth);
11815 
11816   ins_cost(300);
11817   format %{ "call,runtime " %}
11818   ins_encode(clear_avx, Java_To_Runtime(meth));
11819   ins_pipe(pipe_slow);
11820 %}
11821 
11822 // Call runtime without safepoint
11823 instruct CallLeafDirect(method meth)
11824 %{
11825   match(CallLeaf);
11826   effect(USE meth);
11827 
11828   ins_cost(300);
11829   format %{ "call_leaf,runtime " %}
11830   ins_encode(clear_avx, Java_To_Runtime(meth));
11831   ins_pipe(pipe_slow);
11832 %}
11833 
11834 // Call runtime without safepoint
11835 instruct CallLeafNoFPDirect(method meth)
11836 %{
11837   match(CallLeafNoFP);
11838   effect(USE meth);
11839 
11840   ins_cost(300);
11841   format %{ "call_leaf_nofp,runtime " %}
11842   ins_encode(Java_To_Runtime(meth));
11843   ins_pipe(pipe_slow);
11844 %}
11845 
11846 // Return Instruction
11847 // Remove the return address & jump to it.
11848 // Notice: We always emit a nop after a ret to make sure there is room
11849 // for safepoint patching
11850 instruct Ret()
11851 %{
11852   match(Return);
11853 
11854   format %{ "ret" %}
11855   opcode(0xC3);
11856   ins_encode(OpcP);
11857   ins_pipe(pipe_jmp);
11858 %}
11859 
11860 // Tail Call; Jump from runtime stub to Java code.
11861 // Also known as an 'interprocedural jump'.
11862 // Target of jump will eventually return to caller.
11863 // TailJump below removes the return address.
11864 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11865 %{
11866   match(TailCall jump_target method_oop);
11867 
11868   ins_cost(300);
11869   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11870   opcode(0xFF, 0x4); /* Opcode FF /4 */
11871   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11872   ins_pipe(pipe_jmp);
11873 %}
11874 
11875 // Tail Jump; remove the return address; jump to target.
11876 // TailCall above leaves the return address around.
11877 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11878 %{
11879   match(TailJump jump_target ex_oop);
11880 
11881   ins_cost(300);
11882   format %{ "popq    rdx\t# pop return address\n\t"
11883             "jmp     $jump_target" %}
11884   opcode(0xFF, 0x4); /* Opcode FF /4 */
11885   ins_encode(Opcode(0x5a), // popq rdx
11886              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11887   ins_pipe(pipe_jmp);
11888 %}
11889 
11890 // Create exception oop: created by stack-crawling runtime code.
11891 // Created exception is now available to this handler, and is setup
11892 // just prior to jumping to this handler.  No code emitted.
11893 instruct CreateException(rax_RegP ex_oop)
11894 %{
11895   match(Set ex_oop (CreateEx));
11896 
11897   size(0);
11898   // use the following format syntax
11899   format %{ "# exception oop is in rax; no code emitted" %}
11900   ins_encode();
11901   ins_pipe(empty);
11902 %}
11903 
11904 // Rethrow exception:
11905 // The exception oop will come in the first argument position.
11906 // Then JUMP (not call) to the rethrow stub code.
11907 instruct RethrowException()
11908 %{
11909   match(Rethrow);
11910 
11911   // use the following format syntax
11912   format %{ "jmp     rethrow_stub" %}
11913   ins_encode(enc_rethrow);
11914   ins_pipe(pipe_jmp);
11915 %}
11916 
11917 
11918 // ============================================================================
11919 // This name is KNOWN by the ADLC and cannot be changed.
11920 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11921 // for this guy.
11922 instruct tlsLoadP(r15_RegP dst) %{
11923   match(Set dst (ThreadLocal));
11924   effect(DEF dst);
11925 
11926   size(0);
11927   format %{ "# TLS is in R15" %}
11928   ins_encode( /*empty encoding*/ );
11929   ins_pipe(ialu_reg_reg);
11930 %}
11931 
11932 
11933 //----------PEEPHOLE RULES-----------------------------------------------------
11934 // These must follow all instruction definitions as they use the names
11935 // defined in the instructions definitions.
11936 //
11937 // peepmatch ( root_instr_name [preceding_instruction]* );
11938 //
11939 // peepconstraint %{
11940 // (instruction_number.operand_name relational_op instruction_number.operand_name
11941 //  [, ...] );
11942 // // instruction numbers are zero-based using left to right order in peepmatch
11943 //
11944 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11945 // // provide an instruction_number.operand_name for each operand that appears
11946 // // in the replacement instruction's match rule
11947 //
11948 // ---------VM FLAGS---------------------------------------------------------
11949 //
11950 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11951 //
11952 // Each peephole rule is given an identifying number starting with zero and
11953 // increasing by one in the order seen by the parser.  An individual peephole
11954 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11955 // on the command-line.
11956 //
11957 // ---------CURRENT LIMITATIONS----------------------------------------------
11958 //
11959 // Only match adjacent instructions in same basic block
11960 // Only equality constraints
11961 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11962 // Only one replacement instruction
11963 //
11964 // ---------EXAMPLE----------------------------------------------------------
11965 //
11966 // // pertinent parts of existing instructions in architecture description
11967 // instruct movI(rRegI dst, rRegI src)
11968 // %{
11969 //   match(Set dst (CopyI src));
11970 // %}
11971 //
11972 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11973 // %{
11974 //   match(Set dst (AddI dst src));
11975 //   effect(KILL cr);
11976 // %}
11977 //
11978 // // Change (inc mov) to lea
11979 // peephole %{
11980 //   // increment preceeded by register-register move
11981 //   peepmatch ( incI_rReg movI );
11982 //   // require that the destination register of the increment
11983 //   // match the destination register of the move
11984 //   peepconstraint ( 0.dst == 1.dst );
11985 //   // construct a replacement instruction that sets
11986 //   // the destination to ( move's source register + one )
11987 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11988 // %}
11989 //
11990 
11991 // Implementation no longer uses movX instructions since
11992 // machine-independent system no longer uses CopyX nodes.
11993 //
11994 // peephole
11995 // %{
11996 //   peepmatch (incI_rReg movI);
11997 //   peepconstraint (0.dst == 1.dst);
11998 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11999 // %}
12000 
12001 // peephole
12002 // %{
12003 //   peepmatch (decI_rReg movI);
12004 //   peepconstraint (0.dst == 1.dst);
12005 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12006 // %}
12007 
12008 // peephole
12009 // %{
12010 //   peepmatch (addI_rReg_imm movI);
12011 //   peepconstraint (0.dst == 1.dst);
12012 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12013 // %}
12014 
12015 // peephole
12016 // %{
12017 //   peepmatch (incL_rReg movL);
12018 //   peepconstraint (0.dst == 1.dst);
12019 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12020 // %}
12021 
12022 // peephole
12023 // %{
12024 //   peepmatch (decL_rReg movL);
12025 //   peepconstraint (0.dst == 1.dst);
12026 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12027 // %}
12028 
12029 // peephole
12030 // %{
12031 //   peepmatch (addL_rReg_imm movL);
12032 //   peepconstraint (0.dst == 1.dst);
12033 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12034 // %}
12035 
12036 // peephole
12037 // %{
12038 //   peepmatch (addP_rReg_imm movP);
12039 //   peepconstraint (0.dst == 1.dst);
12040 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12041 // %}
12042 
12043 // // Change load of spilled value to only a spill
12044 // instruct storeI(memory mem, rRegI src)
12045 // %{
12046 //   match(Set mem (StoreI mem src));
12047 // %}
12048 //
12049 // instruct loadI(rRegI dst, memory mem)
12050 // %{
12051 //   match(Set dst (LoadI mem));
12052 // %}
12053 //
12054 
12055 peephole
12056 %{
12057   peepmatch (loadI storeI);
12058   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12059   peepreplace (storeI(1.mem 1.mem 1.src));
12060 %}
12061 
12062 peephole
12063 %{
12064   peepmatch (loadL storeL);
12065   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12066   peepreplace (storeL(1.mem 1.mem 1.src));
12067 %}
12068 
12069 //----------SMARTSPILL RULES---------------------------------------------------
12070 // These must follow all instruction definitions as they use the names
12071 // defined in the instructions definitions.