1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   if(UseAVX > 2) {
 540     return 0; // vzeroupper is ignored
 541   } else {
 542     return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 543   }
 544 }
 545 
 546 // !!!!! Special hack to get all types of calls to specify the byte offset
 547 //       from the start of the call to the point where the return address
 548 //       will point.
 549 int MachCallStaticJavaNode::ret_addr_offset()
 550 {
 551   int offset = 5; // 5 bytes from start of call to where return address points
 552   offset += clear_avx_size();
 553   return offset;
 554 }
 555 
 556 int MachCallDynamicJavaNode::ret_addr_offset()
 557 {
 558   int offset = 15; // 15 bytes from start of call to where return address points
 559   offset += clear_avx_size();
 560   return offset;
 561 }
 562 
 563 int MachCallRuntimeNode::ret_addr_offset() {
 564   int offset = 13; // movq r10,#addr; callq (r10)
 565   offset += clear_avx_size();
 566   return offset;
 567 }
 568 
 569 // Indicate if the safepoint node needs the polling page as an input,
 570 // it does if the polling page is more than disp32 away.
 571 bool SafePointNode::needs_polling_address_input()
 572 {
 573   return Assembler::is_polling_page_far();
 574 }
 575 
 576 //
 577 // Compute padding required for nodes which need alignment
 578 //
 579 
 580 // The address of the call instruction needs to be 4-byte aligned to
 581 // ensure that it does not span a cache line so that it can be patched.
 582 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 583 {
 584   current_offset += clear_avx_size(); // skip vzeroupper
 585   current_offset += 1; // skip call opcode byte
 586   return round_to(current_offset, alignment_required()) - current_offset;
 587 }
 588 
 589 // The address of the call instruction needs to be 4-byte aligned to
 590 // ensure that it does not span a cache line so that it can be patched.
 591 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 592 {
 593   current_offset += clear_avx_size(); // skip vzeroupper
 594   current_offset += 11; // skip movq instruction + call opcode byte
 595   return round_to(current_offset, alignment_required()) - current_offset;
 596 }
 597 
 598 // EMIT_RM()
 599 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 600   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 601   cbuf.insts()->emit_int8(c);
 602 }
 603 
 604 // EMIT_CC()
 605 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 606   unsigned char c = (unsigned char) (f1 | f2);
 607   cbuf.insts()->emit_int8(c);
 608 }
 609 
 610 // EMIT_OPCODE()
 611 void emit_opcode(CodeBuffer &cbuf, int code) {
 612   cbuf.insts()->emit_int8((unsigned char) code);
 613 }
 614 
 615 // EMIT_OPCODE() w/ relocation information
 616 void emit_opcode(CodeBuffer &cbuf,
 617                  int code, relocInfo::relocType reloc, int offset, int format)
 618 {
 619   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 620   emit_opcode(cbuf, code);
 621 }
 622 
 623 // EMIT_D8()
 624 void emit_d8(CodeBuffer &cbuf, int d8) {
 625   cbuf.insts()->emit_int8((unsigned char) d8);
 626 }
 627 
 628 // EMIT_D16()
 629 void emit_d16(CodeBuffer &cbuf, int d16) {
 630   cbuf.insts()->emit_int16(d16);
 631 }
 632 
 633 // EMIT_D32()
 634 void emit_d32(CodeBuffer &cbuf, int d32) {
 635   cbuf.insts()->emit_int32(d32);
 636 }
 637 
 638 // EMIT_D64()
 639 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 640   cbuf.insts()->emit_int64(d64);
 641 }
 642 
 643 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 644 void emit_d32_reloc(CodeBuffer& cbuf,
 645                     int d32,
 646                     relocInfo::relocType reloc,
 647                     int format)
 648 {
 649   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 650   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 651   cbuf.insts()->emit_int32(d32);
 652 }
 653 
 654 // emit 32 bit value and construct relocation entry from RelocationHolder
 655 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 656 #ifdef ASSERT
 657   if (rspec.reloc()->type() == relocInfo::oop_type &&
 658       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 659     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 660     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 661   }
 662 #endif
 663   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 664   cbuf.insts()->emit_int32(d32);
 665 }
 666 
 667 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 668   address next_ip = cbuf.insts_end() + 4;
 669   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 670                  external_word_Relocation::spec(addr),
 671                  RELOC_DISP32);
 672 }
 673 
 674 
 675 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 676 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 677   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 678   cbuf.insts()->emit_int64(d64);
 679 }
 680 
 681 // emit 64 bit value and construct relocation entry from RelocationHolder
 682 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 683 #ifdef ASSERT
 684   if (rspec.reloc()->type() == relocInfo::oop_type &&
 685       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 686     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 687     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 688            "cannot embed scavengable oops in code");
 689   }
 690 #endif
 691   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 692   cbuf.insts()->emit_int64(d64);
 693 }
 694 
 695 // Access stack slot for load or store
 696 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 697 {
 698   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 699   if (-0x80 <= disp && disp < 0x80) {
 700     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 703   } else {
 704     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 705     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 706     emit_d32(cbuf, disp);     // Displacement // R/M byte
 707   }
 708 }
 709 
 710    // rRegI ereg, memory mem) %{    // emit_reg_mem
 711 void encode_RegMem(CodeBuffer &cbuf,
 712                    int reg,
 713                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 714 {
 715   assert(disp_reloc == relocInfo::none, "cannot have disp");
 716   int regenc = reg & 7;
 717   int baseenc = base & 7;
 718   int indexenc = index & 7;
 719 
 720   // There is no index & no scale, use form without SIB byte
 721   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 722     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 723     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 724       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 725     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 726       // If 8-bit displacement, mode 0x1
 727       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 728       emit_d8(cbuf, disp);
 729     } else {
 730       // If 32-bit displacement
 731       if (base == -1) { // Special flag for absolute address
 732         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 733         if (disp_reloc != relocInfo::none) {
 734           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 735         } else {
 736           emit_d32(cbuf, disp);
 737         }
 738       } else {
 739         // Normal base + offset
 740         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 741         if (disp_reloc != relocInfo::none) {
 742           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 743         } else {
 744           emit_d32(cbuf, disp);
 745         }
 746       }
 747     }
 748   } else {
 749     // Else, encode with the SIB byte
 750     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 751     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 752       // If no displacement
 753       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 754       emit_rm(cbuf, scale, indexenc, baseenc);
 755     } else {
 756       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 757         // If 8-bit displacement, mode 0x1
 758         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 759         emit_rm(cbuf, scale, indexenc, baseenc);
 760         emit_d8(cbuf, disp);
 761       } else {
 762         // If 32-bit displacement
 763         if (base == 0x04 ) {
 764           emit_rm(cbuf, 0x2, regenc, 0x4);
 765           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 766         } else {
 767           emit_rm(cbuf, 0x2, regenc, 0x4);
 768           emit_rm(cbuf, scale, indexenc, baseenc); // *
 769         }
 770         if (disp_reloc != relocInfo::none) {
 771           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 772         } else {
 773           emit_d32(cbuf, disp);
 774         }
 775       }
 776     }
 777   }
 778 }
 779 
 780 // This could be in MacroAssembler but it's fairly C2 specific
 781 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 782   Label exit;
 783   __ jccb(Assembler::noParity, exit);
 784   __ pushf();
 785   //
 786   // comiss/ucomiss instructions set ZF,PF,CF flags and
 787   // zero OF,AF,SF for NaN values.
 788   // Fixup flags by zeroing ZF,PF so that compare of NaN
 789   // values returns 'less than' result (CF is set).
 790   // Leave the rest of flags unchanged.
 791   //
 792   //    7 6 5 4 3 2 1 0
 793   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 794   //    0 0 1 0 1 0 1 1   (0x2B)
 795   //
 796   __ andq(Address(rsp, 0), 0xffffff2b);
 797   __ popf();
 798   __ bind(exit);
 799 }
 800 
 801 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 802   Label done;
 803   __ movl(dst, -1);
 804   __ jcc(Assembler::parity, done);
 805   __ jcc(Assembler::below, done);
 806   __ setb(Assembler::notEqual, dst);
 807   __ movzbl(dst, dst);
 808   __ bind(done);
 809 }
 810 
 811 
 812 //=============================================================================
 813 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 814 
 815 int Compile::ConstantTable::calculate_table_base_offset() const {
 816   return 0;  // absolute addressing, no offset
 817 }
 818 
 819 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 820 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 821   ShouldNotReachHere();
 822 }
 823 
 824 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 825   // Empty encoding
 826 }
 827 
 828 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 829   return 0;
 830 }
 831 
 832 #ifndef PRODUCT
 833 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 834   st->print("# MachConstantBaseNode (empty encoding)");
 835 }
 836 #endif
 837 
 838 
 839 //=============================================================================
 840 #ifndef PRODUCT
 841 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 842   Compile* C = ra_->C;
 843 
 844   int framesize = C->frame_size_in_bytes();
 845   int bangsize = C->bang_size_in_bytes();
 846   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 847   // Remove wordSize for return addr which is already pushed.
 848   framesize -= wordSize;
 849 
 850   if (C->need_stack_bang(bangsize)) {
 851     framesize -= wordSize;
 852     st->print("# stack bang (%d bytes)", bangsize);
 853     st->print("\n\t");
 854     st->print("pushq   rbp\t# Save rbp");
 855     if (PreserveFramePointer) {
 856         st->print("\n\t");
 857         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 858     }
 859     if (framesize) {
 860       st->print("\n\t");
 861       st->print("subq    rsp, #%d\t# Create frame",framesize);
 862     }
 863   } else {
 864     st->print("subq    rsp, #%d\t# Create frame",framesize);
 865     st->print("\n\t");
 866     framesize -= wordSize;
 867     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 868     if (PreserveFramePointer) {
 869       st->print("\n\t");
 870       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 871       if (framesize > 0) {
 872         st->print("\n\t");
 873         st->print("addq    rbp, #%d", framesize);
 874       }      
 875     }
 876   }
 877 
 878   if (VerifyStackAtCalls) {
 879     st->print("\n\t");
 880     framesize -= wordSize;
 881     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 882 #ifdef ASSERT
 883     st->print("\n\t");
 884     st->print("# stack alignment check");
 885 #endif
 886   }
 887   st->cr();
 888 }
 889 #endif
 890 
 891 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 892   Compile* C = ra_->C;
 893   MacroAssembler _masm(&cbuf);
 894 
 895   int framesize = C->frame_size_in_bytes();
 896   int bangsize = C->bang_size_in_bytes();
 897 
 898   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 899 
 900   C->set_frame_complete(cbuf.insts_size());
 901 
 902   if (C->has_mach_constant_base_node()) {
 903     // NOTE: We set the table base offset here because users might be
 904     // emitted before MachConstantBaseNode.
 905     Compile::ConstantTable& constant_table = C->constant_table();
 906     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 907   }
 908 }
 909 
 910 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 911 {
 912   return MachNode::size(ra_); // too many variables; just compute it
 913                               // the hard way
 914 }
 915 
 916 int MachPrologNode::reloc() const
 917 {
 918   return 0; // a large enough number
 919 }
 920 
 921 //=============================================================================
 922 #ifndef PRODUCT
 923 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 924 {
 925   Compile* C = ra_->C;
 926   if (C->max_vector_size() > 16) {
 927     st->print("vzeroupper");
 928     st->cr(); st->print("\t");
 929   }
 930 
 931   int framesize = C->frame_size_in_bytes();
 932   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 933   // Remove word for return adr already pushed
 934   // and RBP
 935   framesize -= 2*wordSize;
 936 
 937   if (framesize) {
 938     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 939     st->print("\t");
 940   }
 941 
 942   st->print_cr("popq   rbp");
 943   if (do_polling() && C->is_method_compilation()) {
 944     st->print("\t");
 945     if (Assembler::is_polling_page_far()) {
 946       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 947                    "testl  rax, [rscratch1]\t"
 948                    "# Safepoint: poll for GC");
 949     } else {
 950       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 951                    "# Safepoint: poll for GC");
 952     }
 953   }
 954 }
 955 #endif
 956 
 957 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 958 {
 959   Compile* C = ra_->C;
 960   if (C->max_vector_size() > 16) {
 961     // Clear upper bits of YMM registers when current compiled code uses
 962     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 963     MacroAssembler _masm(&cbuf);
 964     __ vzeroupper();
 965   }
 966 
 967   int framesize = C->frame_size_in_bytes();
 968   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 969   // Remove word for return adr already pushed
 970   // and RBP
 971   framesize -= 2*wordSize;
 972 
 973   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 974 
 975   if (framesize) {
 976     emit_opcode(cbuf, Assembler::REX_W);
 977     if (framesize < 0x80) {
 978       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d8(cbuf, framesize);
 981     } else {
 982       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 983       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 984       emit_d32(cbuf, framesize);
 985     }
 986   }
 987 
 988   // popq rbp
 989   emit_opcode(cbuf, 0x58 | RBP_enc);
 990 
 991   if (do_polling() && C->is_method_compilation()) {
 992     MacroAssembler _masm(&cbuf);
 993     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 994     if (Assembler::is_polling_page_far()) {
 995       __ lea(rscratch1, polling_page);
 996       __ relocate(relocInfo::poll_return_type);
 997       __ testl(rax, Address(rscratch1, 0));
 998     } else {
 999       __ testl(rax, polling_page);
1000     }
1001   }
1002 }
1003 
1004 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1005 {
1006   return MachNode::size(ra_); // too many variables; just compute it
1007                               // the hard way
1008 }
1009 
1010 int MachEpilogNode::reloc() const
1011 {
1012   return 2; // a large enough number
1013 }
1014 
1015 const Pipeline* MachEpilogNode::pipeline() const
1016 {
1017   return MachNode::pipeline_class();
1018 }
1019 
1020 int MachEpilogNode::safepoint_offset() const
1021 {
1022   return 0;
1023 }
1024 
1025 //=============================================================================
1026 
1027 enum RC {
1028   rc_bad,
1029   rc_int,
1030   rc_float,
1031   rc_stack
1032 };
1033 
1034 static enum RC rc_class(OptoReg::Name reg)
1035 {
1036   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1037 
1038   if (OptoReg::is_stack(reg)) return rc_stack;
1039 
1040   VMReg r = OptoReg::as_VMReg(reg);
1041 
1042   if (r->is_Register()) return rc_int;
1043 
1044   assert(r->is_XMMRegister(), "must be");
1045   return rc_float;
1046 }
1047 
1048 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1049 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1050                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1051 
1052 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1053                             int stack_offset, int reg, uint ireg, outputStream* st);
1054 
1055 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1056                                       int dst_offset, uint ireg, outputStream* st) {
1057   if (cbuf) {
1058     MacroAssembler _masm(cbuf);
1059     switch (ireg) {
1060     case Op_VecS:
1061       __ movq(Address(rsp, -8), rax);
1062       __ movl(rax, Address(rsp, src_offset));
1063       __ movl(Address(rsp, dst_offset), rax);
1064       __ movq(rax, Address(rsp, -8));
1065       break;
1066     case Op_VecD:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       break;
1070     case Op_VecX:
1071       __ pushq(Address(rsp, src_offset));
1072       __ popq (Address(rsp, dst_offset));
1073       __ pushq(Address(rsp, src_offset+8));
1074       __ popq (Address(rsp, dst_offset+8));
1075       break;
1076     case Op_VecY:
1077       __ vmovdqu(Address(rsp, -32), xmm0);
1078       __ vmovdqu(xmm0, Address(rsp, src_offset));
1079       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1080       __ vmovdqu(xmm0, Address(rsp, -32));
1081     case Op_VecZ:
1082       __ evmovdqul(Address(rsp, -64), xmm0, 2);
1083       __ evmovdqul(xmm0, Address(rsp, src_offset), 2);
1084       __ evmovdqul(Address(rsp, dst_offset), xmm0, 2);
1085       __ evmovdqul(xmm0, Address(rsp, -64), 2);
1086       break;
1087     default:
1088       ShouldNotReachHere();
1089     }
1090 #ifndef PRODUCT
1091   } else {
1092     switch (ireg) {
1093     case Op_VecS:
1094       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1095                 "movl    rax, [rsp + #%d]\n\t"
1096                 "movl    [rsp + #%d], rax\n\t"
1097                 "movq    rax, [rsp - #8]",
1098                 src_offset, dst_offset);
1099       break;
1100     case Op_VecD:
1101       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1102                 "popq    [rsp + #%d]",
1103                 src_offset, dst_offset);
1104       break;
1105      case Op_VecX:
1106       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1107                 "popq    [rsp + #%d]\n\t"
1108                 "pushq   [rsp + #%d]\n\t"
1109                 "popq    [rsp + #%d]",
1110                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1111       break;
1112     case Op_VecY:
1113       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1114                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1115                 "vmovdqu [rsp + #%d], xmm0\n\t"
1116                 "vmovdqu xmm0, [rsp - #32]",
1117                 src_offset, dst_offset);
1118       break;
1119     case Op_VecZ:
1120       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1121                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1122                 "vmovdqu [rsp + #%d], xmm0\n\t"
1123                 "vmovdqu xmm0, [rsp - #64]",
1124                 src_offset, dst_offset);
1125       break;
1126     default:
1127       ShouldNotReachHere();
1128     }
1129 #endif
1130   }
1131 }
1132 
1133 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1134                                        PhaseRegAlloc* ra_,
1135                                        bool do_size,
1136                                        outputStream* st) const {
1137   assert(cbuf != NULL || st  != NULL, "sanity");
1138   // Get registers to move
1139   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1140   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1141   OptoReg::Name dst_second = ra_->get_reg_second(this);
1142   OptoReg::Name dst_first = ra_->get_reg_first(this);
1143 
1144   enum RC src_second_rc = rc_class(src_second);
1145   enum RC src_first_rc = rc_class(src_first);
1146   enum RC dst_second_rc = rc_class(dst_second);
1147   enum RC dst_first_rc = rc_class(dst_first);
1148 
1149   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1150          "must move at least 1 register" );
1151 
1152   if (src_first == dst_first && src_second == dst_second) {
1153     // Self copy, no move
1154     return 0;
1155   }
1156   if (bottom_type()->isa_vect() != NULL) {
1157     uint ireg = ideal_reg();
1158     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1159     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1160     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1161       // mem -> mem
1162       int src_offset = ra_->reg2offset(src_first);
1163       int dst_offset = ra_->reg2offset(dst_first);
1164       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1165     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1166       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1167     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1168       int stack_offset = ra_->reg2offset(dst_first);
1169       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1170     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1171       int stack_offset = ra_->reg2offset(src_first);
1172       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1173     } else {
1174       ShouldNotReachHere();
1175     }
1176     return 0;
1177   }
1178   if (src_first_rc == rc_stack) {
1179     // mem ->
1180     if (dst_first_rc == rc_stack) {
1181       // mem -> mem
1182       assert(src_second != dst_first, "overlap");
1183       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1184           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1185         // 64-bit
1186         int src_offset = ra_->reg2offset(src_first);
1187         int dst_offset = ra_->reg2offset(dst_first);
1188         if (cbuf) {
1189           MacroAssembler _masm(cbuf);
1190           __ pushq(Address(rsp, src_offset));
1191           __ popq (Address(rsp, dst_offset));
1192 #ifndef PRODUCT
1193         } else {
1194           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1195                     "popq    [rsp + #%d]",
1196                      src_offset, dst_offset);
1197 #endif
1198         }
1199       } else {
1200         // 32-bit
1201         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1202         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1203         // No pushl/popl, so:
1204         int src_offset = ra_->reg2offset(src_first);
1205         int dst_offset = ra_->reg2offset(dst_first);
1206         if (cbuf) {
1207           MacroAssembler _masm(cbuf);
1208           __ movq(Address(rsp, -8), rax);
1209           __ movl(rax, Address(rsp, src_offset));
1210           __ movl(Address(rsp, dst_offset), rax);
1211           __ movq(rax, Address(rsp, -8));
1212 #ifndef PRODUCT
1213         } else {
1214           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1215                     "movl    rax, [rsp + #%d]\n\t"
1216                     "movl    [rsp + #%d], rax\n\t"
1217                     "movq    rax, [rsp - #8]",
1218                      src_offset, dst_offset);
1219 #endif
1220         }
1221       }
1222       return 0;
1223     } else if (dst_first_rc == rc_int) {
1224       // mem -> gpr
1225       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1226           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1227         // 64-bit
1228         int offset = ra_->reg2offset(src_first);
1229         if (cbuf) {
1230           MacroAssembler _masm(cbuf);
1231           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1232 #ifndef PRODUCT
1233         } else {
1234           st->print("movq    %s, [rsp + #%d]\t# spill",
1235                      Matcher::regName[dst_first],
1236                      offset);
1237 #endif
1238         }
1239       } else {
1240         // 32-bit
1241         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1242         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1243         int offset = ra_->reg2offset(src_first);
1244         if (cbuf) {
1245           MacroAssembler _masm(cbuf);
1246           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1247 #ifndef PRODUCT
1248         } else {
1249           st->print("movl    %s, [rsp + #%d]\t# spill",
1250                      Matcher::regName[dst_first],
1251                      offset);
1252 #endif
1253         }
1254       }
1255       return 0;
1256     } else if (dst_first_rc == rc_float) {
1257       // mem-> xmm
1258       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1259           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1260         // 64-bit
1261         int offset = ra_->reg2offset(src_first);
1262         if (cbuf) {
1263           MacroAssembler _masm(cbuf);
1264           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1265 #ifndef PRODUCT
1266         } else {
1267           st->print("%s  %s, [rsp + #%d]\t# spill",
1268                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1269                      Matcher::regName[dst_first],
1270                      offset);
1271 #endif
1272         }
1273       } else {
1274         // 32-bit
1275         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1276         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1277         int offset = ra_->reg2offset(src_first);
1278         if (cbuf) {
1279           MacroAssembler _masm(cbuf);
1280           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1281 #ifndef PRODUCT
1282         } else {
1283           st->print("movss   %s, [rsp + #%d]\t# spill",
1284                      Matcher::regName[dst_first],
1285                      offset);
1286 #endif
1287         }
1288       }
1289       return 0;
1290     }
1291   } else if (src_first_rc == rc_int) {
1292     // gpr ->
1293     if (dst_first_rc == rc_stack) {
1294       // gpr -> mem
1295       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1296           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1297         // 64-bit
1298         int offset = ra_->reg2offset(dst_first);
1299         if (cbuf) {
1300           MacroAssembler _masm(cbuf);
1301           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1302 #ifndef PRODUCT
1303         } else {
1304           st->print("movq    [rsp + #%d], %s\t# spill",
1305                      offset,
1306                      Matcher::regName[src_first]);
1307 #endif
1308         }
1309       } else {
1310         // 32-bit
1311         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1312         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1313         int offset = ra_->reg2offset(dst_first);
1314         if (cbuf) {
1315           MacroAssembler _masm(cbuf);
1316           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1317 #ifndef PRODUCT
1318         } else {
1319           st->print("movl    [rsp + #%d], %s\t# spill",
1320                      offset,
1321                      Matcher::regName[src_first]);
1322 #endif
1323         }
1324       }
1325       return 0;
1326     } else if (dst_first_rc == rc_int) {
1327       // gpr -> gpr
1328       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1329           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1330         // 64-bit
1331         if (cbuf) {
1332           MacroAssembler _masm(cbuf);
1333           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1334                   as_Register(Matcher::_regEncode[src_first]));
1335 #ifndef PRODUCT
1336         } else {
1337           st->print("movq    %s, %s\t# spill",
1338                      Matcher::regName[dst_first],
1339                      Matcher::regName[src_first]);
1340 #endif
1341         }
1342         return 0;
1343       } else {
1344         // 32-bit
1345         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1346         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1347         if (cbuf) {
1348           MacroAssembler _masm(cbuf);
1349           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1350                   as_Register(Matcher::_regEncode[src_first]));
1351 #ifndef PRODUCT
1352         } else {
1353           st->print("movl    %s, %s\t# spill",
1354                      Matcher::regName[dst_first],
1355                      Matcher::regName[src_first]);
1356 #endif
1357         }
1358         return 0;
1359       }
1360     } else if (dst_first_rc == rc_float) {
1361       // gpr -> xmm
1362       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1363           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1364         // 64-bit
1365         if (cbuf) {
1366           MacroAssembler _masm(cbuf);
1367           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1368 #ifndef PRODUCT
1369         } else {
1370           st->print("movdq   %s, %s\t# spill",
1371                      Matcher::regName[dst_first],
1372                      Matcher::regName[src_first]);
1373 #endif
1374         }
1375       } else {
1376         // 32-bit
1377         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1378         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1379         if (cbuf) {
1380           MacroAssembler _masm(cbuf);
1381           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1382 #ifndef PRODUCT
1383         } else {
1384           st->print("movdl   %s, %s\t# spill",
1385                      Matcher::regName[dst_first],
1386                      Matcher::regName[src_first]);
1387 #endif
1388         }
1389       }
1390       return 0;
1391     }
1392   } else if (src_first_rc == rc_float) {
1393     // xmm ->
1394     if (dst_first_rc == rc_stack) {
1395       // xmm -> mem
1396       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1397           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1398         // 64-bit
1399         int offset = ra_->reg2offset(dst_first);
1400         if (cbuf) {
1401           MacroAssembler _masm(cbuf);
1402           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1403 #ifndef PRODUCT
1404         } else {
1405           st->print("movsd   [rsp + #%d], %s\t# spill",
1406                      offset,
1407                      Matcher::regName[src_first]);
1408 #endif
1409         }
1410       } else {
1411         // 32-bit
1412         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1413         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1414         int offset = ra_->reg2offset(dst_first);
1415         if (cbuf) {
1416           MacroAssembler _masm(cbuf);
1417           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1418 #ifndef PRODUCT
1419         } else {
1420           st->print("movss   [rsp + #%d], %s\t# spill",
1421                      offset,
1422                      Matcher::regName[src_first]);
1423 #endif
1424         }
1425       }
1426       return 0;
1427     } else if (dst_first_rc == rc_int) {
1428       // xmm -> gpr
1429       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1430           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1431         // 64-bit
1432         if (cbuf) {
1433           MacroAssembler _masm(cbuf);
1434           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1435 #ifndef PRODUCT
1436         } else {
1437           st->print("movdq   %s, %s\t# spill",
1438                      Matcher::regName[dst_first],
1439                      Matcher::regName[src_first]);
1440 #endif
1441         }
1442       } else {
1443         // 32-bit
1444         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1445         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1446         if (cbuf) {
1447           MacroAssembler _masm(cbuf);
1448           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1449 #ifndef PRODUCT
1450         } else {
1451           st->print("movdl   %s, %s\t# spill",
1452                      Matcher::regName[dst_first],
1453                      Matcher::regName[src_first]);
1454 #endif
1455         }
1456       }
1457       return 0;
1458     } else if (dst_first_rc == rc_float) {
1459       // xmm -> xmm
1460       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1461           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1462         // 64-bit
1463         if (cbuf) {
1464           MacroAssembler _masm(cbuf);
1465           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1466 #ifndef PRODUCT
1467         } else {
1468           st->print("%s  %s, %s\t# spill",
1469                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1470                      Matcher::regName[dst_first],
1471                      Matcher::regName[src_first]);
1472 #endif
1473         }
1474       } else {
1475         // 32-bit
1476         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1477         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1478         if (cbuf) {
1479           MacroAssembler _masm(cbuf);
1480           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1481 #ifndef PRODUCT
1482         } else {
1483           st->print("%s  %s, %s\t# spill",
1484                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1485                      Matcher::regName[dst_first],
1486                      Matcher::regName[src_first]);
1487 #endif
1488         }
1489       }
1490       return 0;
1491     }
1492   }
1493 
1494   assert(0," foo ");
1495   Unimplemented();
1496   return 0;
1497 }
1498 
1499 #ifndef PRODUCT
1500 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1501   implementation(NULL, ra_, false, st);
1502 }
1503 #endif
1504 
1505 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1506   implementation(&cbuf, ra_, false, NULL);
1507 }
1508 
1509 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1510   return MachNode::size(ra_);
1511 }
1512 
1513 //=============================================================================
1514 #ifndef PRODUCT
1515 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1516 {
1517   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1518   int reg = ra_->get_reg_first(this);
1519   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1520             Matcher::regName[reg], offset);
1521 }
1522 #endif
1523 
1524 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1525 {
1526   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1527   int reg = ra_->get_encode(this);
1528   if (offset >= 0x80) {
1529     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1530     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1531     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1532     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1533     emit_d32(cbuf, offset);
1534   } else {
1535     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1536     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1537     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1538     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1539     emit_d8(cbuf, offset);
1540   }
1541 }
1542 
1543 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1544 {
1545   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1546   return (offset < 0x80) ? 5 : 8; // REX
1547 }
1548 
1549 //=============================================================================
1550 #ifndef PRODUCT
1551 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1552 {
1553   if (UseCompressedClassPointers) {
1554     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1555     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1556     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1557   } else {
1558     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1559                  "# Inline cache check");
1560   }
1561   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1562   st->print_cr("\tnop\t# nops to align entry point");
1563 }
1564 #endif
1565 
1566 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1567 {
1568   MacroAssembler masm(&cbuf);
1569   uint insts_size = cbuf.insts_size();
1570   if (UseCompressedClassPointers) {
1571     masm.load_klass(rscratch1, j_rarg0);
1572     masm.cmpptr(rax, rscratch1);
1573   } else {
1574     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1575   }
1576 
1577   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1578 
1579   /* WARNING these NOPs are critical so that verified entry point is properly
1580      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1581   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1582   if (OptoBreakpoint) {
1583     // Leave space for int3
1584     nops_cnt -= 1;
1585   }
1586   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1587   if (nops_cnt > 0)
1588     masm.nop(nops_cnt);
1589 }
1590 
1591 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1592 {
1593   return MachNode::size(ra_); // too many variables; just compute it
1594                               // the hard way
1595 }
1596 
1597 
1598 //=============================================================================
1599 
1600 int Matcher::regnum_to_fpu_offset(int regnum)
1601 {
1602   return regnum - 32; // The FP registers are in the second chunk
1603 }
1604 
1605 // This is UltraSparc specific, true just means we have fast l2f conversion
1606 const bool Matcher::convL2FSupported(void) {
1607   return true;
1608 }
1609 
1610 // Is this branch offset short enough that a short branch can be used?
1611 //
1612 // NOTE: If the platform does not provide any short branch variants, then
1613 //       this method should return false for offset 0.
1614 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1615   // The passed offset is relative to address of the branch.
1616   // On 86 a branch displacement is calculated relative to address
1617   // of a next instruction.
1618   offset -= br_size;
1619 
1620   // the short version of jmpConUCF2 contains multiple branches,
1621   // making the reach slightly less
1622   if (rule == jmpConUCF2_rule)
1623     return (-126 <= offset && offset <= 125);
1624   return (-128 <= offset && offset <= 127);
1625 }
1626 
1627 const bool Matcher::isSimpleConstant64(jlong value) {
1628   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1629   //return value == (int) value;  // Cf. storeImmL and immL32.
1630 
1631   // Probably always true, even if a temp register is required.
1632   return true;
1633 }
1634 
1635 // The ecx parameter to rep stosq for the ClearArray node is in words.
1636 const bool Matcher::init_array_count_is_in_bytes = false;
1637 
1638 // Threshold size for cleararray.
1639 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1640 
1641 // No additional cost for CMOVL.
1642 const int Matcher::long_cmove_cost() { return 0; }
1643 
1644 // No CMOVF/CMOVD with SSE2
1645 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1646 
1647 // Does the CPU require late expand (see block.cpp for description of late expand)?
1648 const bool Matcher::require_postalloc_expand = false;
1649 
1650 // Should the Matcher clone shifts on addressing modes, expecting them
1651 // to be subsumed into complex addressing expressions or compute them
1652 // into registers?  True for Intel but false for most RISCs
1653 const bool Matcher::clone_shift_expressions = true;
1654 
1655 // Do we need to mask the count passed to shift instructions or does
1656 // the cpu only look at the lower 5/6 bits anyway?
1657 const bool Matcher::need_masked_shift_count = false;
1658 
1659 bool Matcher::narrow_oop_use_complex_address() {
1660   assert(UseCompressedOops, "only for compressed oops code");
1661   return (LogMinObjAlignmentInBytes <= 3);
1662 }
1663 
1664 bool Matcher::narrow_klass_use_complex_address() {
1665   assert(UseCompressedClassPointers, "only for compressed klass code");
1666   return (LogKlassAlignmentInBytes <= 3);
1667 }
1668 
1669 // Is it better to copy float constants, or load them directly from
1670 // memory?  Intel can load a float constant from a direct address,
1671 // requiring no extra registers.  Most RISCs will have to materialize
1672 // an address into a register first, so they would do better to copy
1673 // the constant from stack.
1674 const bool Matcher::rematerialize_float_constants = true; // XXX
1675 
1676 // If CPU can load and store mis-aligned doubles directly then no
1677 // fixup is needed.  Else we split the double into 2 integer pieces
1678 // and move it piece-by-piece.  Only happens when passing doubles into
1679 // C code as the Java calling convention forces doubles to be aligned.
1680 const bool Matcher::misaligned_doubles_ok = true;
1681 
1682 // No-op on amd64
1683 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1684 
1685 // Advertise here if the CPU requires explicit rounding operations to
1686 // implement the UseStrictFP mode.
1687 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1688 
1689 // Are floats conerted to double when stored to stack during deoptimization?
1690 // On x64 it is stored without convertion so we can use normal access.
1691 bool Matcher::float_in_double() { return false; }
1692 
1693 // Do ints take an entire long register or just half?
1694 const bool Matcher::int_in_long = true;
1695 
1696 // Return whether or not this register is ever used as an argument.
1697 // This function is used on startup to build the trampoline stubs in
1698 // generateOptoStub.  Registers not mentioned will be killed by the VM
1699 // call in the trampoline, and arguments in those registers not be
1700 // available to the callee.
1701 bool Matcher::can_be_java_arg(int reg)
1702 {
1703   return
1704     reg ==  RDI_num || reg == RDI_H_num ||
1705     reg ==  RSI_num || reg == RSI_H_num ||
1706     reg ==  RDX_num || reg == RDX_H_num ||
1707     reg ==  RCX_num || reg == RCX_H_num ||
1708     reg ==   R8_num || reg ==  R8_H_num ||
1709     reg ==   R9_num || reg ==  R9_H_num ||
1710     reg ==  R12_num || reg == R12_H_num ||
1711     reg == XMM0_num || reg == XMM0b_num ||
1712     reg == XMM1_num || reg == XMM1b_num ||
1713     reg == XMM2_num || reg == XMM2b_num ||
1714     reg == XMM3_num || reg == XMM3b_num ||
1715     reg == XMM4_num || reg == XMM4b_num ||
1716     reg == XMM5_num || reg == XMM5b_num ||
1717     reg == XMM6_num || reg == XMM6b_num ||
1718     reg == XMM7_num || reg == XMM7b_num;
1719 }
1720 
1721 bool Matcher::is_spillable_arg(int reg)
1722 {
1723   return can_be_java_arg(reg);
1724 }
1725 
1726 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1727   // In 64 bit mode a code which use multiply when
1728   // devisor is constant is faster than hardware
1729   // DIV instruction (it uses MulHiL).
1730   return false;
1731 }
1732 
1733 // Register for DIVI projection of divmodI
1734 RegMask Matcher::divI_proj_mask() {
1735   return INT_RAX_REG_mask();
1736 }
1737 
1738 // Register for MODI projection of divmodI
1739 RegMask Matcher::modI_proj_mask() {
1740   return INT_RDX_REG_mask();
1741 }
1742 
1743 // Register for DIVL projection of divmodL
1744 RegMask Matcher::divL_proj_mask() {
1745   return LONG_RAX_REG_mask();
1746 }
1747 
1748 // Register for MODL projection of divmodL
1749 RegMask Matcher::modL_proj_mask() {
1750   return LONG_RDX_REG_mask();
1751 }
1752 
1753 // Register for saving SP into on method handle invokes. Not used on x86_64.
1754 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1755     return NO_REG_mask();
1756 }
1757 
1758 %}
1759 
1760 //----------ENCODING BLOCK-----------------------------------------------------
1761 // This block specifies the encoding classes used by the compiler to
1762 // output byte streams.  Encoding classes are parameterized macros
1763 // used by Machine Instruction Nodes in order to generate the bit
1764 // encoding of the instruction.  Operands specify their base encoding
1765 // interface with the interface keyword.  There are currently
1766 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1767 // COND_INTER.  REG_INTER causes an operand to generate a function
1768 // which returns its register number when queried.  CONST_INTER causes
1769 // an operand to generate a function which returns the value of the
1770 // constant when queried.  MEMORY_INTER causes an operand to generate
1771 // four functions which return the Base Register, the Index Register,
1772 // the Scale Value, and the Offset Value of the operand when queried.
1773 // COND_INTER causes an operand to generate six functions which return
1774 // the encoding code (ie - encoding bits for the instruction)
1775 // associated with each basic boolean condition for a conditional
1776 // instruction.
1777 //
1778 // Instructions specify two basic values for encoding.  Again, a
1779 // function is available to check if the constant displacement is an
1780 // oop. They use the ins_encode keyword to specify their encoding
1781 // classes (which must be a sequence of enc_class names, and their
1782 // parameters, specified in the encoding block), and they use the
1783 // opcode keyword to specify, in order, their primary, secondary, and
1784 // tertiary opcode.  Only the opcode sections which a particular
1785 // instruction needs for encoding need to be specified.
1786 encode %{
1787   // Build emit functions for each basic byte or larger field in the
1788   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1789   // from C++ code in the enc_class source block.  Emit functions will
1790   // live in the main source block for now.  In future, we can
1791   // generalize this by adding a syntax that specifies the sizes of
1792   // fields in an order, so that the adlc can build the emit functions
1793   // automagically
1794 
1795   // Emit primary opcode
1796   enc_class OpcP
1797   %{
1798     emit_opcode(cbuf, $primary);
1799   %}
1800 
1801   // Emit secondary opcode
1802   enc_class OpcS
1803   %{
1804     emit_opcode(cbuf, $secondary);
1805   %}
1806 
1807   // Emit tertiary opcode
1808   enc_class OpcT
1809   %{
1810     emit_opcode(cbuf, $tertiary);
1811   %}
1812 
1813   // Emit opcode directly
1814   enc_class Opcode(immI d8)
1815   %{
1816     emit_opcode(cbuf, $d8$$constant);
1817   %}
1818 
1819   // Emit size prefix
1820   enc_class SizePrefix
1821   %{
1822     emit_opcode(cbuf, 0x66);
1823   %}
1824 
1825   enc_class reg(rRegI reg)
1826   %{
1827     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1828   %}
1829 
1830   enc_class reg_reg(rRegI dst, rRegI src)
1831   %{
1832     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1833   %}
1834 
1835   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1836   %{
1837     emit_opcode(cbuf, $opcode$$constant);
1838     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1839   %}
1840 
1841   enc_class cdql_enc(no_rax_rdx_RegI div)
1842   %{
1843     // Full implementation of Java idiv and irem; checks for
1844     // special case as described in JVM spec., p.243 & p.271.
1845     //
1846     //         normal case                           special case
1847     //
1848     // input : rax: dividend                         min_int
1849     //         reg: divisor                          -1
1850     //
1851     // output: rax: quotient  (= rax idiv reg)       min_int
1852     //         rdx: remainder (= rax irem reg)       0
1853     //
1854     //  Code sequnce:
1855     //
1856     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1857     //    5:   75 07/08                jne    e <normal>
1858     //    7:   33 d2                   xor    %edx,%edx
1859     //  [div >= 8 -> offset + 1]
1860     //  [REX_B]
1861     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1862     //    c:   74 03/04                je     11 <done>
1863     // 000000000000000e <normal>:
1864     //    e:   99                      cltd
1865     //  [div >= 8 -> offset + 1]
1866     //  [REX_B]
1867     //    f:   f7 f9                   idiv   $div
1868     // 0000000000000011 <done>:
1869 
1870     // cmp    $0x80000000,%eax
1871     emit_opcode(cbuf, 0x3d);
1872     emit_d8(cbuf, 0x00);
1873     emit_d8(cbuf, 0x00);
1874     emit_d8(cbuf, 0x00);
1875     emit_d8(cbuf, 0x80);
1876 
1877     // jne    e <normal>
1878     emit_opcode(cbuf, 0x75);
1879     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1880 
1881     // xor    %edx,%edx
1882     emit_opcode(cbuf, 0x33);
1883     emit_d8(cbuf, 0xD2);
1884 
1885     // cmp    $0xffffffffffffffff,%ecx
1886     if ($div$$reg >= 8) {
1887       emit_opcode(cbuf, Assembler::REX_B);
1888     }
1889     emit_opcode(cbuf, 0x83);
1890     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1891     emit_d8(cbuf, 0xFF);
1892 
1893     // je     11 <done>
1894     emit_opcode(cbuf, 0x74);
1895     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1896 
1897     // <normal>
1898     // cltd
1899     emit_opcode(cbuf, 0x99);
1900 
1901     // idivl (note: must be emitted by the user of this rule)
1902     // <done>
1903   %}
1904 
1905   enc_class cdqq_enc(no_rax_rdx_RegL div)
1906   %{
1907     // Full implementation of Java ldiv and lrem; checks for
1908     // special case as described in JVM spec., p.243 & p.271.
1909     //
1910     //         normal case                           special case
1911     //
1912     // input : rax: dividend                         min_long
1913     //         reg: divisor                          -1
1914     //
1915     // output: rax: quotient  (= rax idiv reg)       min_long
1916     //         rdx: remainder (= rax irem reg)       0
1917     //
1918     //  Code sequnce:
1919     //
1920     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1921     //    7:   00 00 80
1922     //    a:   48 39 d0                cmp    %rdx,%rax
1923     //    d:   75 08                   jne    17 <normal>
1924     //    f:   33 d2                   xor    %edx,%edx
1925     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1926     //   15:   74 05                   je     1c <done>
1927     // 0000000000000017 <normal>:
1928     //   17:   48 99                   cqto
1929     //   19:   48 f7 f9                idiv   $div
1930     // 000000000000001c <done>:
1931 
1932     // mov    $0x8000000000000000,%rdx
1933     emit_opcode(cbuf, Assembler::REX_W);
1934     emit_opcode(cbuf, 0xBA);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x00);
1939     emit_d8(cbuf, 0x00);
1940     emit_d8(cbuf, 0x00);
1941     emit_d8(cbuf, 0x00);
1942     emit_d8(cbuf, 0x80);
1943 
1944     // cmp    %rdx,%rax
1945     emit_opcode(cbuf, Assembler::REX_W);
1946     emit_opcode(cbuf, 0x39);
1947     emit_d8(cbuf, 0xD0);
1948 
1949     // jne    17 <normal>
1950     emit_opcode(cbuf, 0x75);
1951     emit_d8(cbuf, 0x08);
1952 
1953     // xor    %edx,%edx
1954     emit_opcode(cbuf, 0x33);
1955     emit_d8(cbuf, 0xD2);
1956 
1957     // cmp    $0xffffffffffffffff,$div
1958     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1959     emit_opcode(cbuf, 0x83);
1960     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1961     emit_d8(cbuf, 0xFF);
1962 
1963     // je     1e <done>
1964     emit_opcode(cbuf, 0x74);
1965     emit_d8(cbuf, 0x05);
1966 
1967     // <normal>
1968     // cqto
1969     emit_opcode(cbuf, Assembler::REX_W);
1970     emit_opcode(cbuf, 0x99);
1971 
1972     // idivq (note: must be emitted by the user of this rule)
1973     // <done>
1974   %}
1975 
1976   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1977   enc_class OpcSE(immI imm)
1978   %{
1979     // Emit primary opcode and set sign-extend bit
1980     // Check for 8-bit immediate, and set sign extend bit in opcode
1981     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1982       emit_opcode(cbuf, $primary | 0x02);
1983     } else {
1984       // 32-bit immediate
1985       emit_opcode(cbuf, $primary);
1986     }
1987   %}
1988 
1989   enc_class OpcSErm(rRegI dst, immI imm)
1990   %{
1991     // OpcSEr/m
1992     int dstenc = $dst$$reg;
1993     if (dstenc >= 8) {
1994       emit_opcode(cbuf, Assembler::REX_B);
1995       dstenc -= 8;
1996     }
1997     // Emit primary opcode and set sign-extend bit
1998     // Check for 8-bit immediate, and set sign extend bit in opcode
1999     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2000       emit_opcode(cbuf, $primary | 0x02);
2001     } else {
2002       // 32-bit immediate
2003       emit_opcode(cbuf, $primary);
2004     }
2005     // Emit r/m byte with secondary opcode, after primary opcode.
2006     emit_rm(cbuf, 0x3, $secondary, dstenc);
2007   %}
2008 
2009   enc_class OpcSErm_wide(rRegL dst, immI imm)
2010   %{
2011     // OpcSEr/m
2012     int dstenc = $dst$$reg;
2013     if (dstenc < 8) {
2014       emit_opcode(cbuf, Assembler::REX_W);
2015     } else {
2016       emit_opcode(cbuf, Assembler::REX_WB);
2017       dstenc -= 8;
2018     }
2019     // Emit primary opcode and set sign-extend bit
2020     // Check for 8-bit immediate, and set sign extend bit in opcode
2021     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2022       emit_opcode(cbuf, $primary | 0x02);
2023     } else {
2024       // 32-bit immediate
2025       emit_opcode(cbuf, $primary);
2026     }
2027     // Emit r/m byte with secondary opcode, after primary opcode.
2028     emit_rm(cbuf, 0x3, $secondary, dstenc);
2029   %}
2030 
2031   enc_class Con8or32(immI imm)
2032   %{
2033     // Check for 8-bit immediate, and set sign extend bit in opcode
2034     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2035       $$$emit8$imm$$constant;
2036     } else {
2037       // 32-bit immediate
2038       $$$emit32$imm$$constant;
2039     }
2040   %}
2041 
2042   enc_class opc2_reg(rRegI dst)
2043   %{
2044     // BSWAP
2045     emit_cc(cbuf, $secondary, $dst$$reg);
2046   %}
2047 
2048   enc_class opc3_reg(rRegI dst)
2049   %{
2050     // BSWAP
2051     emit_cc(cbuf, $tertiary, $dst$$reg);
2052   %}
2053 
2054   enc_class reg_opc(rRegI div)
2055   %{
2056     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2057     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2058   %}
2059 
2060   enc_class enc_cmov(cmpOp cop)
2061   %{
2062     // CMOV
2063     $$$emit8$primary;
2064     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2065   %}
2066 
2067   enc_class enc_PartialSubtypeCheck()
2068   %{
2069     Register Rrdi = as_Register(RDI_enc); // result register
2070     Register Rrax = as_Register(RAX_enc); // super class
2071     Register Rrcx = as_Register(RCX_enc); // killed
2072     Register Rrsi = as_Register(RSI_enc); // sub class
2073     Label miss;
2074     const bool set_cond_codes = true;
2075 
2076     MacroAssembler _masm(&cbuf);
2077     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2078                                      NULL, &miss,
2079                                      /*set_cond_codes:*/ true);
2080     if ($primary) {
2081       __ xorptr(Rrdi, Rrdi);
2082     }
2083     __ bind(miss);
2084   %}
2085 
2086   enc_class clear_avx %{
2087     debug_only(int off0 = cbuf.insts_size());
2088     if (ra_->C->max_vector_size() > 16) {
2089       // Clear upper bits of YMM registers when current compiled code uses
2090       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2091       MacroAssembler _masm(&cbuf);
2092       __ vzeroupper();
2093     }
2094     debug_only(int off1 = cbuf.insts_size());
2095     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2096   %}
2097 
2098   enc_class Java_To_Runtime(method meth) %{
2099     // No relocation needed
2100     MacroAssembler _masm(&cbuf);
2101     __ mov64(r10, (int64_t) $meth$$method);
2102     __ call(r10);
2103   %}
2104 
2105   enc_class Java_To_Interpreter(method meth)
2106   %{
2107     // CALL Java_To_Interpreter
2108     // This is the instruction starting address for relocation info.
2109     cbuf.set_insts_mark();
2110     $$$emit8$primary;
2111     // CALL directly to the runtime
2112     emit_d32_reloc(cbuf,
2113                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2114                    runtime_call_Relocation::spec(),
2115                    RELOC_DISP32);
2116   %}
2117 
2118   enc_class Java_Static_Call(method meth)
2119   %{
2120     // JAVA STATIC CALL
2121     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2122     // determine who we intended to call.
2123     cbuf.set_insts_mark();
2124     $$$emit8$primary;
2125 
2126     if (!_method) {
2127       emit_d32_reloc(cbuf,
2128                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2129                      runtime_call_Relocation::spec(),
2130                      RELOC_DISP32);
2131     } else if (_optimized_virtual) {
2132       emit_d32_reloc(cbuf,
2133                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2134                      opt_virtual_call_Relocation::spec(),
2135                      RELOC_DISP32);
2136     } else {
2137       emit_d32_reloc(cbuf,
2138                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2139                      static_call_Relocation::spec(),
2140                      RELOC_DISP32);
2141     }
2142     if (_method) {
2143       // Emit stubs for static call.
2144       address mark = cbuf.insts_mark();
2145       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2146       if (stub == NULL) {
2147         ciEnv::current()->record_failure("CodeCache is full");
2148         return;
2149       }
2150     }
2151   %}
2152 
2153   enc_class Java_Dynamic_Call(method meth) %{
2154     MacroAssembler _masm(&cbuf);
2155     __ ic_call((address)$meth$$method);
2156   %}
2157 
2158   enc_class Java_Compiled_Call(method meth)
2159   %{
2160     // JAVA COMPILED CALL
2161     int disp = in_bytes(Method:: from_compiled_offset());
2162 
2163     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2164     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2165 
2166     // callq *disp(%rax)
2167     cbuf.set_insts_mark();
2168     $$$emit8$primary;
2169     if (disp < 0x80) {
2170       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2171       emit_d8(cbuf, disp); // Displacement
2172     } else {
2173       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2174       emit_d32(cbuf, disp); // Displacement
2175     }
2176   %}
2177 
2178   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2179   %{
2180     // SAL, SAR, SHR
2181     int dstenc = $dst$$reg;
2182     if (dstenc >= 8) {
2183       emit_opcode(cbuf, Assembler::REX_B);
2184       dstenc -= 8;
2185     }
2186     $$$emit8$primary;
2187     emit_rm(cbuf, 0x3, $secondary, dstenc);
2188     $$$emit8$shift$$constant;
2189   %}
2190 
2191   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2192   %{
2193     // SAL, SAR, SHR
2194     int dstenc = $dst$$reg;
2195     if (dstenc < 8) {
2196       emit_opcode(cbuf, Assembler::REX_W);
2197     } else {
2198       emit_opcode(cbuf, Assembler::REX_WB);
2199       dstenc -= 8;
2200     }
2201     $$$emit8$primary;
2202     emit_rm(cbuf, 0x3, $secondary, dstenc);
2203     $$$emit8$shift$$constant;
2204   %}
2205 
2206   enc_class load_immI(rRegI dst, immI src)
2207   %{
2208     int dstenc = $dst$$reg;
2209     if (dstenc >= 8) {
2210       emit_opcode(cbuf, Assembler::REX_B);
2211       dstenc -= 8;
2212     }
2213     emit_opcode(cbuf, 0xB8 | dstenc);
2214     $$$emit32$src$$constant;
2215   %}
2216 
2217   enc_class load_immL(rRegL dst, immL src)
2218   %{
2219     int dstenc = $dst$$reg;
2220     if (dstenc < 8) {
2221       emit_opcode(cbuf, Assembler::REX_W);
2222     } else {
2223       emit_opcode(cbuf, Assembler::REX_WB);
2224       dstenc -= 8;
2225     }
2226     emit_opcode(cbuf, 0xB8 | dstenc);
2227     emit_d64(cbuf, $src$$constant);
2228   %}
2229 
2230   enc_class load_immUL32(rRegL dst, immUL32 src)
2231   %{
2232     // same as load_immI, but this time we care about zeroes in the high word
2233     int dstenc = $dst$$reg;
2234     if (dstenc >= 8) {
2235       emit_opcode(cbuf, Assembler::REX_B);
2236       dstenc -= 8;
2237     }
2238     emit_opcode(cbuf, 0xB8 | dstenc);
2239     $$$emit32$src$$constant;
2240   %}
2241 
2242   enc_class load_immL32(rRegL dst, immL32 src)
2243   %{
2244     int dstenc = $dst$$reg;
2245     if (dstenc < 8) {
2246       emit_opcode(cbuf, Assembler::REX_W);
2247     } else {
2248       emit_opcode(cbuf, Assembler::REX_WB);
2249       dstenc -= 8;
2250     }
2251     emit_opcode(cbuf, 0xC7);
2252     emit_rm(cbuf, 0x03, 0x00, dstenc);
2253     $$$emit32$src$$constant;
2254   %}
2255 
2256   enc_class load_immP31(rRegP dst, immP32 src)
2257   %{
2258     // same as load_immI, but this time we care about zeroes in the high word
2259     int dstenc = $dst$$reg;
2260     if (dstenc >= 8) {
2261       emit_opcode(cbuf, Assembler::REX_B);
2262       dstenc -= 8;
2263     }
2264     emit_opcode(cbuf, 0xB8 | dstenc);
2265     $$$emit32$src$$constant;
2266   %}
2267 
2268   enc_class load_immP(rRegP dst, immP src)
2269   %{
2270     int dstenc = $dst$$reg;
2271     if (dstenc < 8) {
2272       emit_opcode(cbuf, Assembler::REX_W);
2273     } else {
2274       emit_opcode(cbuf, Assembler::REX_WB);
2275       dstenc -= 8;
2276     }
2277     emit_opcode(cbuf, 0xB8 | dstenc);
2278     // This next line should be generated from ADLC
2279     if ($src->constant_reloc() != relocInfo::none) {
2280       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2281     } else {
2282       emit_d64(cbuf, $src$$constant);
2283     }
2284   %}
2285 
2286   enc_class Con32(immI src)
2287   %{
2288     // Output immediate
2289     $$$emit32$src$$constant;
2290   %}
2291 
2292   enc_class Con32F_as_bits(immF src)
2293   %{
2294     // Output Float immediate bits
2295     jfloat jf = $src$$constant;
2296     jint jf_as_bits = jint_cast(jf);
2297     emit_d32(cbuf, jf_as_bits);
2298   %}
2299 
2300   enc_class Con16(immI src)
2301   %{
2302     // Output immediate
2303     $$$emit16$src$$constant;
2304   %}
2305 
2306   // How is this different from Con32??? XXX
2307   enc_class Con_d32(immI src)
2308   %{
2309     emit_d32(cbuf,$src$$constant);
2310   %}
2311 
2312   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2313     // Output immediate memory reference
2314     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2315     emit_d32(cbuf, 0x00);
2316   %}
2317 
2318   enc_class lock_prefix()
2319   %{
2320     if (os::is_MP()) {
2321       emit_opcode(cbuf, 0xF0); // lock
2322     }
2323   %}
2324 
2325   enc_class REX_mem(memory mem)
2326   %{
2327     if ($mem$$base >= 8) {
2328       if ($mem$$index < 8) {
2329         emit_opcode(cbuf, Assembler::REX_B);
2330       } else {
2331         emit_opcode(cbuf, Assembler::REX_XB);
2332       }
2333     } else {
2334       if ($mem$$index >= 8) {
2335         emit_opcode(cbuf, Assembler::REX_X);
2336       }
2337     }
2338   %}
2339 
2340   enc_class REX_mem_wide(memory mem)
2341   %{
2342     if ($mem$$base >= 8) {
2343       if ($mem$$index < 8) {
2344         emit_opcode(cbuf, Assembler::REX_WB);
2345       } else {
2346         emit_opcode(cbuf, Assembler::REX_WXB);
2347       }
2348     } else {
2349       if ($mem$$index < 8) {
2350         emit_opcode(cbuf, Assembler::REX_W);
2351       } else {
2352         emit_opcode(cbuf, Assembler::REX_WX);
2353       }
2354     }
2355   %}
2356 
2357   // for byte regs
2358   enc_class REX_breg(rRegI reg)
2359   %{
2360     if ($reg$$reg >= 4) {
2361       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2362     }
2363   %}
2364 
2365   // for byte regs
2366   enc_class REX_reg_breg(rRegI dst, rRegI src)
2367   %{
2368     if ($dst$$reg < 8) {
2369       if ($src$$reg >= 4) {
2370         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2371       }
2372     } else {
2373       if ($src$$reg < 8) {
2374         emit_opcode(cbuf, Assembler::REX_R);
2375       } else {
2376         emit_opcode(cbuf, Assembler::REX_RB);
2377       }
2378     }
2379   %}
2380 
2381   // for byte regs
2382   enc_class REX_breg_mem(rRegI reg, memory mem)
2383   %{
2384     if ($reg$$reg < 8) {
2385       if ($mem$$base < 8) {
2386         if ($mem$$index >= 8) {
2387           emit_opcode(cbuf, Assembler::REX_X);
2388         } else if ($reg$$reg >= 4) {
2389           emit_opcode(cbuf, Assembler::REX);
2390         }
2391       } else {
2392         if ($mem$$index < 8) {
2393           emit_opcode(cbuf, Assembler::REX_B);
2394         } else {
2395           emit_opcode(cbuf, Assembler::REX_XB);
2396         }
2397       }
2398     } else {
2399       if ($mem$$base < 8) {
2400         if ($mem$$index < 8) {
2401           emit_opcode(cbuf, Assembler::REX_R);
2402         } else {
2403           emit_opcode(cbuf, Assembler::REX_RX);
2404         }
2405       } else {
2406         if ($mem$$index < 8) {
2407           emit_opcode(cbuf, Assembler::REX_RB);
2408         } else {
2409           emit_opcode(cbuf, Assembler::REX_RXB);
2410         }
2411       }
2412     }
2413   %}
2414 
2415   enc_class REX_reg(rRegI reg)
2416   %{
2417     if ($reg$$reg >= 8) {
2418       emit_opcode(cbuf, Assembler::REX_B);
2419     }
2420   %}
2421 
2422   enc_class REX_reg_wide(rRegI reg)
2423   %{
2424     if ($reg$$reg < 8) {
2425       emit_opcode(cbuf, Assembler::REX_W);
2426     } else {
2427       emit_opcode(cbuf, Assembler::REX_WB);
2428     }
2429   %}
2430 
2431   enc_class REX_reg_reg(rRegI dst, rRegI src)
2432   %{
2433     if ($dst$$reg < 8) {
2434       if ($src$$reg >= 8) {
2435         emit_opcode(cbuf, Assembler::REX_B);
2436       }
2437     } else {
2438       if ($src$$reg < 8) {
2439         emit_opcode(cbuf, Assembler::REX_R);
2440       } else {
2441         emit_opcode(cbuf, Assembler::REX_RB);
2442       }
2443     }
2444   %}
2445 
2446   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2447   %{
2448     if ($dst$$reg < 8) {
2449       if ($src$$reg < 8) {
2450         emit_opcode(cbuf, Assembler::REX_W);
2451       } else {
2452         emit_opcode(cbuf, Assembler::REX_WB);
2453       }
2454     } else {
2455       if ($src$$reg < 8) {
2456         emit_opcode(cbuf, Assembler::REX_WR);
2457       } else {
2458         emit_opcode(cbuf, Assembler::REX_WRB);
2459       }
2460     }
2461   %}
2462 
2463   enc_class REX_reg_mem(rRegI reg, memory mem)
2464   %{
2465     if ($reg$$reg < 8) {
2466       if ($mem$$base < 8) {
2467         if ($mem$$index >= 8) {
2468           emit_opcode(cbuf, Assembler::REX_X);
2469         }
2470       } else {
2471         if ($mem$$index < 8) {
2472           emit_opcode(cbuf, Assembler::REX_B);
2473         } else {
2474           emit_opcode(cbuf, Assembler::REX_XB);
2475         }
2476       }
2477     } else {
2478       if ($mem$$base < 8) {
2479         if ($mem$$index < 8) {
2480           emit_opcode(cbuf, Assembler::REX_R);
2481         } else {
2482           emit_opcode(cbuf, Assembler::REX_RX);
2483         }
2484       } else {
2485         if ($mem$$index < 8) {
2486           emit_opcode(cbuf, Assembler::REX_RB);
2487         } else {
2488           emit_opcode(cbuf, Assembler::REX_RXB);
2489         }
2490       }
2491     }
2492   %}
2493 
2494   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2495   %{
2496     if ($reg$$reg < 8) {
2497       if ($mem$$base < 8) {
2498         if ($mem$$index < 8) {
2499           emit_opcode(cbuf, Assembler::REX_W);
2500         } else {
2501           emit_opcode(cbuf, Assembler::REX_WX);
2502         }
2503       } else {
2504         if ($mem$$index < 8) {
2505           emit_opcode(cbuf, Assembler::REX_WB);
2506         } else {
2507           emit_opcode(cbuf, Assembler::REX_WXB);
2508         }
2509       }
2510     } else {
2511       if ($mem$$base < 8) {
2512         if ($mem$$index < 8) {
2513           emit_opcode(cbuf, Assembler::REX_WR);
2514         } else {
2515           emit_opcode(cbuf, Assembler::REX_WRX);
2516         }
2517       } else {
2518         if ($mem$$index < 8) {
2519           emit_opcode(cbuf, Assembler::REX_WRB);
2520         } else {
2521           emit_opcode(cbuf, Assembler::REX_WRXB);
2522         }
2523       }
2524     }
2525   %}
2526 
2527   enc_class reg_mem(rRegI ereg, memory mem)
2528   %{
2529     // High registers handle in encode_RegMem
2530     int reg = $ereg$$reg;
2531     int base = $mem$$base;
2532     int index = $mem$$index;
2533     int scale = $mem$$scale;
2534     int disp = $mem$$disp;
2535     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2536 
2537     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2538   %}
2539 
2540   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2541   %{
2542     int rm_byte_opcode = $rm_opcode$$constant;
2543 
2544     // High registers handle in encode_RegMem
2545     int base = $mem$$base;
2546     int index = $mem$$index;
2547     int scale = $mem$$scale;
2548     int displace = $mem$$disp;
2549 
2550     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2551                                             // working with static
2552                                             // globals
2553     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2554                   disp_reloc);
2555   %}
2556 
2557   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2558   %{
2559     int reg_encoding = $dst$$reg;
2560     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2561     int index        = 0x04;            // 0x04 indicates no index
2562     int scale        = 0x00;            // 0x00 indicates no scale
2563     int displace     = $src1$$constant; // 0x00 indicates no displacement
2564     relocInfo::relocType disp_reloc = relocInfo::none;
2565     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2566                   disp_reloc);
2567   %}
2568 
2569   enc_class neg_reg(rRegI dst)
2570   %{
2571     int dstenc = $dst$$reg;
2572     if (dstenc >= 8) {
2573       emit_opcode(cbuf, Assembler::REX_B);
2574       dstenc -= 8;
2575     }
2576     // NEG $dst
2577     emit_opcode(cbuf, 0xF7);
2578     emit_rm(cbuf, 0x3, 0x03, dstenc);
2579   %}
2580 
2581   enc_class neg_reg_wide(rRegI dst)
2582   %{
2583     int dstenc = $dst$$reg;
2584     if (dstenc < 8) {
2585       emit_opcode(cbuf, Assembler::REX_W);
2586     } else {
2587       emit_opcode(cbuf, Assembler::REX_WB);
2588       dstenc -= 8;
2589     }
2590     // NEG $dst
2591     emit_opcode(cbuf, 0xF7);
2592     emit_rm(cbuf, 0x3, 0x03, dstenc);
2593   %}
2594 
2595   enc_class setLT_reg(rRegI dst)
2596   %{
2597     int dstenc = $dst$$reg;
2598     if (dstenc >= 8) {
2599       emit_opcode(cbuf, Assembler::REX_B);
2600       dstenc -= 8;
2601     } else if (dstenc >= 4) {
2602       emit_opcode(cbuf, Assembler::REX);
2603     }
2604     // SETLT $dst
2605     emit_opcode(cbuf, 0x0F);
2606     emit_opcode(cbuf, 0x9C);
2607     emit_rm(cbuf, 0x3, 0x0, dstenc);
2608   %}
2609 
2610   enc_class setNZ_reg(rRegI dst)
2611   %{
2612     int dstenc = $dst$$reg;
2613     if (dstenc >= 8) {
2614       emit_opcode(cbuf, Assembler::REX_B);
2615       dstenc -= 8;
2616     } else if (dstenc >= 4) {
2617       emit_opcode(cbuf, Assembler::REX);
2618     }
2619     // SETNZ $dst
2620     emit_opcode(cbuf, 0x0F);
2621     emit_opcode(cbuf, 0x95);
2622     emit_rm(cbuf, 0x3, 0x0, dstenc);
2623   %}
2624 
2625 
2626   // Compare the lonogs and set -1, 0, or 1 into dst
2627   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2628   %{
2629     int src1enc = $src1$$reg;
2630     int src2enc = $src2$$reg;
2631     int dstenc = $dst$$reg;
2632 
2633     // cmpq $src1, $src2
2634     if (src1enc < 8) {
2635       if (src2enc < 8) {
2636         emit_opcode(cbuf, Assembler::REX_W);
2637       } else {
2638         emit_opcode(cbuf, Assembler::REX_WB);
2639       }
2640     } else {
2641       if (src2enc < 8) {
2642         emit_opcode(cbuf, Assembler::REX_WR);
2643       } else {
2644         emit_opcode(cbuf, Assembler::REX_WRB);
2645       }
2646     }
2647     emit_opcode(cbuf, 0x3B);
2648     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2649 
2650     // movl $dst, -1
2651     if (dstenc >= 8) {
2652       emit_opcode(cbuf, Assembler::REX_B);
2653     }
2654     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2655     emit_d32(cbuf, -1);
2656 
2657     // jl,s done
2658     emit_opcode(cbuf, 0x7C);
2659     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2660 
2661     // setne $dst
2662     if (dstenc >= 4) {
2663       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2664     }
2665     emit_opcode(cbuf, 0x0F);
2666     emit_opcode(cbuf, 0x95);
2667     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2668 
2669     // movzbl $dst, $dst
2670     if (dstenc >= 4) {
2671       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2672     }
2673     emit_opcode(cbuf, 0x0F);
2674     emit_opcode(cbuf, 0xB6);
2675     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2676   %}
2677 
2678   enc_class Push_ResultXD(regD dst) %{
2679     MacroAssembler _masm(&cbuf);
2680     __ fstp_d(Address(rsp, 0));
2681     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2682     __ addptr(rsp, 8);
2683   %}
2684 
2685   enc_class Push_SrcXD(regD src) %{
2686     MacroAssembler _masm(&cbuf);
2687     __ subptr(rsp, 8);
2688     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2689     __ fld_d(Address(rsp, 0));
2690   %}
2691 
2692 
2693   enc_class enc_rethrow()
2694   %{
2695     cbuf.set_insts_mark();
2696     emit_opcode(cbuf, 0xE9); // jmp entry
2697     emit_d32_reloc(cbuf,
2698                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2699                    runtime_call_Relocation::spec(),
2700                    RELOC_DISP32);
2701   %}
2702 
2703 %}
2704 
2705 
2706 
2707 //----------FRAME--------------------------------------------------------------
2708 // Definition of frame structure and management information.
2709 //
2710 //  S T A C K   L A Y O U T    Allocators stack-slot number
2711 //                             |   (to get allocators register number
2712 //  G  Owned by    |        |  v    add OptoReg::stack0())
2713 //  r   CALLER     |        |
2714 //  o     |        +--------+      pad to even-align allocators stack-slot
2715 //  w     V        |  pad0  |        numbers; owned by CALLER
2716 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2717 //  h     ^        |   in   |  5
2718 //        |        |  args  |  4   Holes in incoming args owned by SELF
2719 //  |     |        |        |  3
2720 //  |     |        +--------+
2721 //  V     |        | old out|      Empty on Intel, window on Sparc
2722 //        |    old |preserve|      Must be even aligned.
2723 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2724 //        |        |   in   |  3   area for Intel ret address
2725 //     Owned by    |preserve|      Empty on Sparc.
2726 //       SELF      +--------+
2727 //        |        |  pad2  |  2   pad to align old SP
2728 //        |        +--------+  1
2729 //        |        | locks  |  0
2730 //        |        +--------+----> OptoReg::stack0(), even aligned
2731 //        |        |  pad1  | 11   pad to align new SP
2732 //        |        +--------+
2733 //        |        |        | 10
2734 //        |        | spills |  9   spills
2735 //        V        |        |  8   (pad0 slot for callee)
2736 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2737 //        ^        |  out   |  7
2738 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2739 //     Owned by    +--------+
2740 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2741 //        |    new |preserve|      Must be even-aligned.
2742 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2743 //        |        |        |
2744 //
2745 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2746 //         known from SELF's arguments and the Java calling convention.
2747 //         Region 6-7 is determined per call site.
2748 // Note 2: If the calling convention leaves holes in the incoming argument
2749 //         area, those holes are owned by SELF.  Holes in the outgoing area
2750 //         are owned by the CALLEE.  Holes should not be nessecary in the
2751 //         incoming area, as the Java calling convention is completely under
2752 //         the control of the AD file.  Doubles can be sorted and packed to
2753 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2754 //         varargs C calling conventions.
2755 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2756 //         even aligned with pad0 as needed.
2757 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2758 //         region 6-11 is even aligned; it may be padded out more so that
2759 //         the region from SP to FP meets the minimum stack alignment.
2760 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2761 //         alignment.  Region 11, pad1, may be dynamically extended so that
2762 //         SP meets the minimum alignment.
2763 
2764 frame
2765 %{
2766   // What direction does stack grow in (assumed to be same for C & Java)
2767   stack_direction(TOWARDS_LOW);
2768 
2769   // These three registers define part of the calling convention
2770   // between compiled code and the interpreter.
2771   inline_cache_reg(RAX);                // Inline Cache Register
2772   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2773                                         // calling interpreter
2774 
2775   // Optional: name the operand used by cisc-spilling to access
2776   // [stack_pointer + offset]
2777   cisc_spilling_operand_name(indOffset32);
2778 
2779   // Number of stack slots consumed by locking an object
2780   sync_stack_slots(2);
2781 
2782   // Compiled code's Frame Pointer
2783   frame_pointer(RSP);
2784 
2785   // Interpreter stores its frame pointer in a register which is
2786   // stored to the stack by I2CAdaptors.
2787   // I2CAdaptors convert from interpreted java to compiled java.
2788   interpreter_frame_pointer(RBP);
2789 
2790   // Stack alignment requirement
2791   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2792 
2793   // Number of stack slots between incoming argument block and the start of
2794   // a new frame.  The PROLOG must add this many slots to the stack.  The
2795   // EPILOG must remove this many slots.  amd64 needs two slots for
2796   // return address.
2797   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2798 
2799   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2800   // for calls to C.  Supports the var-args backing area for register parms.
2801   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2802 
2803   // The after-PROLOG location of the return address.  Location of
2804   // return address specifies a type (REG or STACK) and a number
2805   // representing the register number (i.e. - use a register name) or
2806   // stack slot.
2807   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2808   // Otherwise, it is above the locks and verification slot and alignment word
2809   return_addr(STACK - 2 +
2810               round_to((Compile::current()->in_preserve_stack_slots() +
2811                         Compile::current()->fixed_slots()),
2812                        stack_alignment_in_slots()));
2813 
2814   // Body of function which returns an integer array locating
2815   // arguments either in registers or in stack slots.  Passed an array
2816   // of ideal registers called "sig" and a "length" count.  Stack-slot
2817   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2818   // arguments for a CALLEE.  Incoming stack arguments are
2819   // automatically biased by the preserve_stack_slots field above.
2820 
2821   calling_convention
2822   %{
2823     // No difference between ingoing/outgoing just pass false
2824     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2825   %}
2826 
2827   c_calling_convention
2828   %{
2829     // This is obviously always outgoing
2830     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2831   %}
2832 
2833   // Location of compiled Java return values.  Same as C for now.
2834   return_value
2835   %{
2836     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2837            "only return normal values");
2838 
2839     static const int lo[Op_RegL + 1] = {
2840       0,
2841       0,
2842       RAX_num,  // Op_RegN
2843       RAX_num,  // Op_RegI
2844       RAX_num,  // Op_RegP
2845       XMM0_num, // Op_RegF
2846       XMM0_num, // Op_RegD
2847       RAX_num   // Op_RegL
2848     };
2849     static const int hi[Op_RegL + 1] = {
2850       0,
2851       0,
2852       OptoReg::Bad, // Op_RegN
2853       OptoReg::Bad, // Op_RegI
2854       RAX_H_num,    // Op_RegP
2855       OptoReg::Bad, // Op_RegF
2856       XMM0b_num,    // Op_RegD
2857       RAX_H_num     // Op_RegL
2858     };
2859     // Excluded flags and vector registers.
2860     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2861     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2862   %}
2863 %}
2864 
2865 //----------ATTRIBUTES---------------------------------------------------------
2866 //----------Operand Attributes-------------------------------------------------
2867 op_attrib op_cost(0);        // Required cost attribute
2868 
2869 //----------Instruction Attributes---------------------------------------------
2870 ins_attrib ins_cost(100);       // Required cost attribute
2871 ins_attrib ins_size(8);         // Required size attribute (in bits)
2872 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2873                                 // a non-matching short branch variant
2874                                 // of some long branch?
2875 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2876                                 // be a power of 2) specifies the
2877                                 // alignment that some part of the
2878                                 // instruction (not necessarily the
2879                                 // start) requires.  If > 1, a
2880                                 // compute_padding() function must be
2881                                 // provided for the instruction
2882 
2883 //----------OPERANDS-----------------------------------------------------------
2884 // Operand definitions must precede instruction definitions for correct parsing
2885 // in the ADLC because operands constitute user defined types which are used in
2886 // instruction definitions.
2887 
2888 //----------Simple Operands----------------------------------------------------
2889 // Immediate Operands
2890 // Integer Immediate
2891 operand immI()
2892 %{
2893   match(ConI);
2894 
2895   op_cost(10);
2896   format %{ %}
2897   interface(CONST_INTER);
2898 %}
2899 
2900 // Constant for test vs zero
2901 operand immI0()
2902 %{
2903   predicate(n->get_int() == 0);
2904   match(ConI);
2905 
2906   op_cost(0);
2907   format %{ %}
2908   interface(CONST_INTER);
2909 %}
2910 
2911 // Constant for increment
2912 operand immI1()
2913 %{
2914   predicate(n->get_int() == 1);
2915   match(ConI);
2916 
2917   op_cost(0);
2918   format %{ %}
2919   interface(CONST_INTER);
2920 %}
2921 
2922 // Constant for decrement
2923 operand immI_M1()
2924 %{
2925   predicate(n->get_int() == -1);
2926   match(ConI);
2927 
2928   op_cost(0);
2929   format %{ %}
2930   interface(CONST_INTER);
2931 %}
2932 
2933 // Valid scale values for addressing modes
2934 operand immI2()
2935 %{
2936   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2937   match(ConI);
2938 
2939   format %{ %}
2940   interface(CONST_INTER);
2941 %}
2942 
2943 operand immI8()
2944 %{
2945   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2946   match(ConI);
2947 
2948   op_cost(5);
2949   format %{ %}
2950   interface(CONST_INTER);
2951 %}
2952 
2953 operand immI16()
2954 %{
2955   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2956   match(ConI);
2957 
2958   op_cost(10);
2959   format %{ %}
2960   interface(CONST_INTER);
2961 %}
2962 
2963 // Int Immediate non-negative
2964 operand immU31()
2965 %{
2966   predicate(n->get_int() >= 0);
2967   match(ConI);
2968 
2969   op_cost(0);
2970   format %{ %}
2971   interface(CONST_INTER);
2972 %}
2973 
2974 // Constant for long shifts
2975 operand immI_32()
2976 %{
2977   predicate( n->get_int() == 32 );
2978   match(ConI);
2979 
2980   op_cost(0);
2981   format %{ %}
2982   interface(CONST_INTER);
2983 %}
2984 
2985 // Constant for long shifts
2986 operand immI_64()
2987 %{
2988   predicate( n->get_int() == 64 );
2989   match(ConI);
2990 
2991   op_cost(0);
2992   format %{ %}
2993   interface(CONST_INTER);
2994 %}
2995 
2996 // Pointer Immediate
2997 operand immP()
2998 %{
2999   match(ConP);
3000 
3001   op_cost(10);
3002   format %{ %}
3003   interface(CONST_INTER);
3004 %}
3005 
3006 // NULL Pointer Immediate
3007 operand immP0()
3008 %{
3009   predicate(n->get_ptr() == 0);
3010   match(ConP);
3011 
3012   op_cost(5);
3013   format %{ %}
3014   interface(CONST_INTER);
3015 %}
3016 
3017 // Pointer Immediate
3018 operand immN() %{
3019   match(ConN);
3020 
3021   op_cost(10);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 operand immNKlass() %{
3027   match(ConNKlass);
3028 
3029   op_cost(10);
3030   format %{ %}
3031   interface(CONST_INTER);
3032 %}
3033 
3034 // NULL Pointer Immediate
3035 operand immN0() %{
3036   predicate(n->get_narrowcon() == 0);
3037   match(ConN);
3038 
3039   op_cost(5);
3040   format %{ %}
3041   interface(CONST_INTER);
3042 %}
3043 
3044 operand immP31()
3045 %{
3046   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3047             && (n->get_ptr() >> 31) == 0);
3048   match(ConP);
3049 
3050   op_cost(5);
3051   format %{ %}
3052   interface(CONST_INTER);
3053 %}
3054 
3055 
3056 // Long Immediate
3057 operand immL()
3058 %{
3059   match(ConL);
3060 
3061   op_cost(20);
3062   format %{ %}
3063   interface(CONST_INTER);
3064 %}
3065 
3066 // Long Immediate 8-bit
3067 operand immL8()
3068 %{
3069   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3070   match(ConL);
3071 
3072   op_cost(5);
3073   format %{ %}
3074   interface(CONST_INTER);
3075 %}
3076 
3077 // Long Immediate 32-bit unsigned
3078 operand immUL32()
3079 %{
3080   predicate(n->get_long() == (unsigned int) (n->get_long()));
3081   match(ConL);
3082 
3083   op_cost(10);
3084   format %{ %}
3085   interface(CONST_INTER);
3086 %}
3087 
3088 // Long Immediate 32-bit signed
3089 operand immL32()
3090 %{
3091   predicate(n->get_long() == (int) (n->get_long()));
3092   match(ConL);
3093 
3094   op_cost(15);
3095   format %{ %}
3096   interface(CONST_INTER);
3097 %}
3098 
3099 // Long Immediate zero
3100 operand immL0()
3101 %{
3102   predicate(n->get_long() == 0L);
3103   match(ConL);
3104 
3105   op_cost(10);
3106   format %{ %}
3107   interface(CONST_INTER);
3108 %}
3109 
3110 // Constant for increment
3111 operand immL1()
3112 %{
3113   predicate(n->get_long() == 1);
3114   match(ConL);
3115 
3116   format %{ %}
3117   interface(CONST_INTER);
3118 %}
3119 
3120 // Constant for decrement
3121 operand immL_M1()
3122 %{
3123   predicate(n->get_long() == -1);
3124   match(ConL);
3125 
3126   format %{ %}
3127   interface(CONST_INTER);
3128 %}
3129 
3130 // Long Immediate: the value 10
3131 operand immL10()
3132 %{
3133   predicate(n->get_long() == 10);
3134   match(ConL);
3135 
3136   format %{ %}
3137   interface(CONST_INTER);
3138 %}
3139 
3140 // Long immediate from 0 to 127.
3141 // Used for a shorter form of long mul by 10.
3142 operand immL_127()
3143 %{
3144   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3145   match(ConL);
3146 
3147   op_cost(10);
3148   format %{ %}
3149   interface(CONST_INTER);
3150 %}
3151 
3152 // Long Immediate: low 32-bit mask
3153 operand immL_32bits()
3154 %{
3155   predicate(n->get_long() == 0xFFFFFFFFL);
3156   match(ConL);
3157   op_cost(20);
3158 
3159   format %{ %}
3160   interface(CONST_INTER);
3161 %}
3162 
3163 // Float Immediate zero
3164 operand immF0()
3165 %{
3166   predicate(jint_cast(n->getf()) == 0);
3167   match(ConF);
3168 
3169   op_cost(5);
3170   format %{ %}
3171   interface(CONST_INTER);
3172 %}
3173 
3174 // Float Immediate
3175 operand immF()
3176 %{
3177   match(ConF);
3178 
3179   op_cost(15);
3180   format %{ %}
3181   interface(CONST_INTER);
3182 %}
3183 
3184 // Double Immediate zero
3185 operand immD0()
3186 %{
3187   predicate(jlong_cast(n->getd()) == 0);
3188   match(ConD);
3189 
3190   op_cost(5);
3191   format %{ %}
3192   interface(CONST_INTER);
3193 %}
3194 
3195 // Double Immediate
3196 operand immD()
3197 %{
3198   match(ConD);
3199 
3200   op_cost(15);
3201   format %{ %}
3202   interface(CONST_INTER);
3203 %}
3204 
3205 // Immediates for special shifts (sign extend)
3206 
3207 // Constants for increment
3208 operand immI_16()
3209 %{
3210   predicate(n->get_int() == 16);
3211   match(ConI);
3212 
3213   format %{ %}
3214   interface(CONST_INTER);
3215 %}
3216 
3217 operand immI_24()
3218 %{
3219   predicate(n->get_int() == 24);
3220   match(ConI);
3221 
3222   format %{ %}
3223   interface(CONST_INTER);
3224 %}
3225 
3226 // Constant for byte-wide masking
3227 operand immI_255()
3228 %{
3229   predicate(n->get_int() == 255);
3230   match(ConI);
3231 
3232   format %{ %}
3233   interface(CONST_INTER);
3234 %}
3235 
3236 // Constant for short-wide masking
3237 operand immI_65535()
3238 %{
3239   predicate(n->get_int() == 65535);
3240   match(ConI);
3241 
3242   format %{ %}
3243   interface(CONST_INTER);
3244 %}
3245 
3246 // Constant for byte-wide masking
3247 operand immL_255()
3248 %{
3249   predicate(n->get_long() == 255);
3250   match(ConL);
3251 
3252   format %{ %}
3253   interface(CONST_INTER);
3254 %}
3255 
3256 // Constant for short-wide masking
3257 operand immL_65535()
3258 %{
3259   predicate(n->get_long() == 65535);
3260   match(ConL);
3261 
3262   format %{ %}
3263   interface(CONST_INTER);
3264 %}
3265 
3266 // Register Operands
3267 // Integer Register
3268 operand rRegI()
3269 %{
3270   constraint(ALLOC_IN_RC(int_reg));
3271   match(RegI);
3272 
3273   match(rax_RegI);
3274   match(rbx_RegI);
3275   match(rcx_RegI);
3276   match(rdx_RegI);
3277   match(rdi_RegI);
3278 
3279   format %{ %}
3280   interface(REG_INTER);
3281 %}
3282 
3283 // Special Registers
3284 operand rax_RegI()
3285 %{
3286   constraint(ALLOC_IN_RC(int_rax_reg));
3287   match(RegI);
3288   match(rRegI);
3289 
3290   format %{ "RAX" %}
3291   interface(REG_INTER);
3292 %}
3293 
3294 // Special Registers
3295 operand rbx_RegI()
3296 %{
3297   constraint(ALLOC_IN_RC(int_rbx_reg));
3298   match(RegI);
3299   match(rRegI);
3300 
3301   format %{ "RBX" %}
3302   interface(REG_INTER);
3303 %}
3304 
3305 operand rcx_RegI()
3306 %{
3307   constraint(ALLOC_IN_RC(int_rcx_reg));
3308   match(RegI);
3309   match(rRegI);
3310 
3311   format %{ "RCX" %}
3312   interface(REG_INTER);
3313 %}
3314 
3315 operand rdx_RegI()
3316 %{
3317   constraint(ALLOC_IN_RC(int_rdx_reg));
3318   match(RegI);
3319   match(rRegI);
3320 
3321   format %{ "RDX" %}
3322   interface(REG_INTER);
3323 %}
3324 
3325 operand rdi_RegI()
3326 %{
3327   constraint(ALLOC_IN_RC(int_rdi_reg));
3328   match(RegI);
3329   match(rRegI);
3330 
3331   format %{ "RDI" %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 operand no_rcx_RegI()
3336 %{
3337   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3338   match(RegI);
3339   match(rax_RegI);
3340   match(rbx_RegI);
3341   match(rdx_RegI);
3342   match(rdi_RegI);
3343 
3344   format %{ %}
3345   interface(REG_INTER);
3346 %}
3347 
3348 operand no_rax_rdx_RegI()
3349 %{
3350   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3351   match(RegI);
3352   match(rbx_RegI);
3353   match(rcx_RegI);
3354   match(rdi_RegI);
3355 
3356   format %{ %}
3357   interface(REG_INTER);
3358 %}
3359 
3360 // Pointer Register
3361 operand any_RegP()
3362 %{
3363   constraint(ALLOC_IN_RC(any_reg));
3364   match(RegP);
3365   match(rax_RegP);
3366   match(rbx_RegP);
3367   match(rdi_RegP);
3368   match(rsi_RegP);
3369   match(rbp_RegP);
3370   match(r15_RegP);
3371   match(rRegP);
3372 
3373   format %{ %}
3374   interface(REG_INTER);
3375 %}
3376 
3377 operand rRegP()
3378 %{
3379   constraint(ALLOC_IN_RC(ptr_reg));
3380   match(RegP);
3381   match(rax_RegP);
3382   match(rbx_RegP);
3383   match(rdi_RegP);
3384   match(rsi_RegP);
3385   match(rbp_RegP);  // See Q&A below about
3386   match(r15_RegP);  // r15_RegP and rbp_RegP.
3387 
3388   format %{ %}
3389   interface(REG_INTER);
3390 %}
3391 
3392 operand rRegN() %{
3393   constraint(ALLOC_IN_RC(int_reg));
3394   match(RegN);
3395 
3396   format %{ %}
3397   interface(REG_INTER);
3398 %}
3399 
3400 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3401 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3402 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3403 // The output of an instruction is controlled by the allocator, which respects
3404 // register class masks, not match rules.  Unless an instruction mentions
3405 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3406 // by the allocator as an input.
3407 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3408 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3409 // result, RBP is not included in the output of the instruction either.
3410 
3411 operand no_rax_RegP()
3412 %{
3413   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3414   match(RegP);
3415   match(rbx_RegP);
3416   match(rsi_RegP);
3417   match(rdi_RegP);
3418 
3419   format %{ %}
3420   interface(REG_INTER);
3421 %}
3422 
3423 // This operand is not allowed to use RBP even if
3424 // RBP is not used to hold the frame pointer.
3425 operand no_rbp_RegP()
3426 %{
3427   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3428   match(RegP);
3429   match(rbx_RegP);
3430   match(rsi_RegP);
3431   match(rdi_RegP);
3432 
3433   format %{ %}
3434   interface(REG_INTER);
3435 %}
3436 
3437 operand no_rax_rbx_RegP()
3438 %{
3439   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3440   match(RegP);
3441   match(rsi_RegP);
3442   match(rdi_RegP);
3443 
3444   format %{ %}
3445   interface(REG_INTER);
3446 %}
3447 
3448 // Special Registers
3449 // Return a pointer value
3450 operand rax_RegP()
3451 %{
3452   constraint(ALLOC_IN_RC(ptr_rax_reg));
3453   match(RegP);
3454   match(rRegP);
3455 
3456   format %{ %}
3457   interface(REG_INTER);
3458 %}
3459 
3460 // Special Registers
3461 // Return a compressed pointer value
3462 operand rax_RegN()
3463 %{
3464   constraint(ALLOC_IN_RC(int_rax_reg));
3465   match(RegN);
3466   match(rRegN);
3467 
3468   format %{ %}
3469   interface(REG_INTER);
3470 %}
3471 
3472 // Used in AtomicAdd
3473 operand rbx_RegP()
3474 %{
3475   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3476   match(RegP);
3477   match(rRegP);
3478 
3479   format %{ %}
3480   interface(REG_INTER);
3481 %}
3482 
3483 operand rsi_RegP()
3484 %{
3485   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3486   match(RegP);
3487   match(rRegP);
3488 
3489   format %{ %}
3490   interface(REG_INTER);
3491 %}
3492 
3493 // Used in rep stosq
3494 operand rdi_RegP()
3495 %{
3496   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3497   match(RegP);
3498   match(rRegP);
3499 
3500   format %{ %}
3501   interface(REG_INTER);
3502 %}
3503 
3504 operand r15_RegP()
3505 %{
3506   constraint(ALLOC_IN_RC(ptr_r15_reg));
3507   match(RegP);
3508   match(rRegP);
3509 
3510   format %{ %}
3511   interface(REG_INTER);
3512 %}
3513 
3514 operand rRegL()
3515 %{
3516   constraint(ALLOC_IN_RC(long_reg));
3517   match(RegL);
3518   match(rax_RegL);
3519   match(rdx_RegL);
3520 
3521   format %{ %}
3522   interface(REG_INTER);
3523 %}
3524 
3525 // Special Registers
3526 operand no_rax_rdx_RegL()
3527 %{
3528   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3529   match(RegL);
3530   match(rRegL);
3531 
3532   format %{ %}
3533   interface(REG_INTER);
3534 %}
3535 
3536 operand no_rax_RegL()
3537 %{
3538   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3539   match(RegL);
3540   match(rRegL);
3541   match(rdx_RegL);
3542 
3543   format %{ %}
3544   interface(REG_INTER);
3545 %}
3546 
3547 operand no_rcx_RegL()
3548 %{
3549   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3550   match(RegL);
3551   match(rRegL);
3552 
3553   format %{ %}
3554   interface(REG_INTER);
3555 %}
3556 
3557 operand rax_RegL()
3558 %{
3559   constraint(ALLOC_IN_RC(long_rax_reg));
3560   match(RegL);
3561   match(rRegL);
3562 
3563   format %{ "RAX" %}
3564   interface(REG_INTER);
3565 %}
3566 
3567 operand rcx_RegL()
3568 %{
3569   constraint(ALLOC_IN_RC(long_rcx_reg));
3570   match(RegL);
3571   match(rRegL);
3572 
3573   format %{ %}
3574   interface(REG_INTER);
3575 %}
3576 
3577 operand rdx_RegL()
3578 %{
3579   constraint(ALLOC_IN_RC(long_rdx_reg));
3580   match(RegL);
3581   match(rRegL);
3582 
3583   format %{ %}
3584   interface(REG_INTER);
3585 %}
3586 
3587 // Flags register, used as output of compare instructions
3588 operand rFlagsReg()
3589 %{
3590   constraint(ALLOC_IN_RC(int_flags));
3591   match(RegFlags);
3592 
3593   format %{ "RFLAGS" %}
3594   interface(REG_INTER);
3595 %}
3596 
3597 // Flags register, used as output of FLOATING POINT compare instructions
3598 operand rFlagsRegU()
3599 %{
3600   constraint(ALLOC_IN_RC(int_flags));
3601   match(RegFlags);
3602 
3603   format %{ "RFLAGS_U" %}
3604   interface(REG_INTER);
3605 %}
3606 
3607 operand rFlagsRegUCF() %{
3608   constraint(ALLOC_IN_RC(int_flags));
3609   match(RegFlags);
3610   predicate(false);
3611 
3612   format %{ "RFLAGS_U_CF" %}
3613   interface(REG_INTER);
3614 %}
3615 
3616 // Float register operands
3617 operand regF() %{
3618    constraint(ALLOC_IN_RC(float_reg));
3619    match(RegF);
3620 
3621    format %{ %}
3622    interface(REG_INTER);
3623 %}
3624 
3625 // Double register operands
3626 operand regD() %{
3627    constraint(ALLOC_IN_RC(double_reg));
3628    match(RegD);
3629 
3630    format %{ %}
3631    interface(REG_INTER);
3632 %}
3633 
3634 // Vectors
3635 operand vecS() %{
3636   constraint(ALLOC_IN_RC(vectors_reg));
3637   match(VecS);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 operand vecD() %{
3644   constraint(ALLOC_IN_RC(vectord_reg));
3645   match(VecD);
3646 
3647   format %{ %}
3648   interface(REG_INTER);
3649 %}
3650 
3651 operand vecX() %{
3652   constraint(ALLOC_IN_RC(vectorx_reg));
3653   match(VecX);
3654 
3655   format %{ %}
3656   interface(REG_INTER);
3657 %}
3658 
3659 operand vecY() %{
3660   constraint(ALLOC_IN_RC(vectory_reg));
3661   match(VecY);
3662 
3663   format %{ %}
3664   interface(REG_INTER);
3665 %}
3666 
3667 //----------Memory Operands----------------------------------------------------
3668 // Direct Memory Operand
3669 // operand direct(immP addr)
3670 // %{
3671 //   match(addr);
3672 
3673 //   format %{ "[$addr]" %}
3674 //   interface(MEMORY_INTER) %{
3675 //     base(0xFFFFFFFF);
3676 //     index(0x4);
3677 //     scale(0x0);
3678 //     disp($addr);
3679 //   %}
3680 // %}
3681 
3682 // Indirect Memory Operand
3683 operand indirect(any_RegP reg)
3684 %{
3685   constraint(ALLOC_IN_RC(ptr_reg));
3686   match(reg);
3687 
3688   format %{ "[$reg]" %}
3689   interface(MEMORY_INTER) %{
3690     base($reg);
3691     index(0x4);
3692     scale(0x0);
3693     disp(0x0);
3694   %}
3695 %}
3696 
3697 // Indirect Memory Plus Short Offset Operand
3698 operand indOffset8(any_RegP reg, immL8 off)
3699 %{
3700   constraint(ALLOC_IN_RC(ptr_reg));
3701   match(AddP reg off);
3702 
3703   format %{ "[$reg + $off (8-bit)]" %}
3704   interface(MEMORY_INTER) %{
3705     base($reg);
3706     index(0x4);
3707     scale(0x0);
3708     disp($off);
3709   %}
3710 %}
3711 
3712 // Indirect Memory Plus Long Offset Operand
3713 operand indOffset32(any_RegP reg, immL32 off)
3714 %{
3715   constraint(ALLOC_IN_RC(ptr_reg));
3716   match(AddP reg off);
3717 
3718   format %{ "[$reg + $off (32-bit)]" %}
3719   interface(MEMORY_INTER) %{
3720     base($reg);
3721     index(0x4);
3722     scale(0x0);
3723     disp($off);
3724   %}
3725 %}
3726 
3727 // Indirect Memory Plus Index Register Plus Offset Operand
3728 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3729 %{
3730   constraint(ALLOC_IN_RC(ptr_reg));
3731   match(AddP (AddP reg lreg) off);
3732 
3733   op_cost(10);
3734   format %{"[$reg + $off + $lreg]" %}
3735   interface(MEMORY_INTER) %{
3736     base($reg);
3737     index($lreg);
3738     scale(0x0);
3739     disp($off);
3740   %}
3741 %}
3742 
3743 // Indirect Memory Plus Index Register Plus Offset Operand
3744 operand indIndex(any_RegP reg, rRegL lreg)
3745 %{
3746   constraint(ALLOC_IN_RC(ptr_reg));
3747   match(AddP reg lreg);
3748 
3749   op_cost(10);
3750   format %{"[$reg + $lreg]" %}
3751   interface(MEMORY_INTER) %{
3752     base($reg);
3753     index($lreg);
3754     scale(0x0);
3755     disp(0x0);
3756   %}
3757 %}
3758 
3759 // Indirect Memory Times Scale Plus Index Register
3760 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3761 %{
3762   constraint(ALLOC_IN_RC(ptr_reg));
3763   match(AddP reg (LShiftL lreg scale));
3764 
3765   op_cost(10);
3766   format %{"[$reg + $lreg << $scale]" %}
3767   interface(MEMORY_INTER) %{
3768     base($reg);
3769     index($lreg);
3770     scale($scale);
3771     disp(0x0);
3772   %}
3773 %}
3774 
3775 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3776 %{
3777   constraint(ALLOC_IN_RC(ptr_reg));
3778   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3779   match(AddP reg (LShiftL (ConvI2L idx) scale));
3780 
3781   op_cost(10);
3782   format %{"[$reg + pos $idx << $scale]" %}
3783   interface(MEMORY_INTER) %{
3784     base($reg);
3785     index($idx);
3786     scale($scale);
3787     disp(0x0);
3788   %}
3789 %}
3790 
3791 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3792 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3793 %{
3794   constraint(ALLOC_IN_RC(ptr_reg));
3795   match(AddP (AddP reg (LShiftL lreg scale)) off);
3796 
3797   op_cost(10);
3798   format %{"[$reg + $off + $lreg << $scale]" %}
3799   interface(MEMORY_INTER) %{
3800     base($reg);
3801     index($lreg);
3802     scale($scale);
3803     disp($off);
3804   %}
3805 %}
3806 
3807 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3808 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3809 %{
3810   constraint(ALLOC_IN_RC(ptr_reg));
3811   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3812   match(AddP (AddP reg (ConvI2L idx)) off);
3813 
3814   op_cost(10);
3815   format %{"[$reg + $off + $idx]" %}
3816   interface(MEMORY_INTER) %{
3817     base($reg);
3818     index($idx);
3819     scale(0x0);
3820     disp($off);
3821   %}
3822 %}
3823 
3824 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3825 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3826 %{
3827   constraint(ALLOC_IN_RC(ptr_reg));
3828   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3829   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3830 
3831   op_cost(10);
3832   format %{"[$reg + $off + $idx << $scale]" %}
3833   interface(MEMORY_INTER) %{
3834     base($reg);
3835     index($idx);
3836     scale($scale);
3837     disp($off);
3838   %}
3839 %}
3840 
3841 // Indirect Narrow Oop Plus Offset Operand
3842 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3843 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3844 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3845   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3846   constraint(ALLOC_IN_RC(ptr_reg));
3847   match(AddP (DecodeN reg) off);
3848 
3849   op_cost(10);
3850   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3851   interface(MEMORY_INTER) %{
3852     base(0xc); // R12
3853     index($reg);
3854     scale(0x3);
3855     disp($off);
3856   %}
3857 %}
3858 
3859 // Indirect Memory Operand
3860 operand indirectNarrow(rRegN reg)
3861 %{
3862   predicate(Universe::narrow_oop_shift() == 0);
3863   constraint(ALLOC_IN_RC(ptr_reg));
3864   match(DecodeN reg);
3865 
3866   format %{ "[$reg]" %}
3867   interface(MEMORY_INTER) %{
3868     base($reg);
3869     index(0x4);
3870     scale(0x0);
3871     disp(0x0);
3872   %}
3873 %}
3874 
3875 // Indirect Memory Plus Short Offset Operand
3876 operand indOffset8Narrow(rRegN reg, immL8 off)
3877 %{
3878   predicate(Universe::narrow_oop_shift() == 0);
3879   constraint(ALLOC_IN_RC(ptr_reg));
3880   match(AddP (DecodeN reg) off);
3881 
3882   format %{ "[$reg + $off (8-bit)]" %}
3883   interface(MEMORY_INTER) %{
3884     base($reg);
3885     index(0x4);
3886     scale(0x0);
3887     disp($off);
3888   %}
3889 %}
3890 
3891 // Indirect Memory Plus Long Offset Operand
3892 operand indOffset32Narrow(rRegN reg, immL32 off)
3893 %{
3894   predicate(Universe::narrow_oop_shift() == 0);
3895   constraint(ALLOC_IN_RC(ptr_reg));
3896   match(AddP (DecodeN reg) off);
3897 
3898   format %{ "[$reg + $off (32-bit)]" %}
3899   interface(MEMORY_INTER) %{
3900     base($reg);
3901     index(0x4);
3902     scale(0x0);
3903     disp($off);
3904   %}
3905 %}
3906 
3907 // Indirect Memory Plus Index Register Plus Offset Operand
3908 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3909 %{
3910   predicate(Universe::narrow_oop_shift() == 0);
3911   constraint(ALLOC_IN_RC(ptr_reg));
3912   match(AddP (AddP (DecodeN reg) lreg) off);
3913 
3914   op_cost(10);
3915   format %{"[$reg + $off + $lreg]" %}
3916   interface(MEMORY_INTER) %{
3917     base($reg);
3918     index($lreg);
3919     scale(0x0);
3920     disp($off);
3921   %}
3922 %}
3923 
3924 // Indirect Memory Plus Index Register Plus Offset Operand
3925 operand indIndexNarrow(rRegN reg, rRegL lreg)
3926 %{
3927   predicate(Universe::narrow_oop_shift() == 0);
3928   constraint(ALLOC_IN_RC(ptr_reg));
3929   match(AddP (DecodeN reg) lreg);
3930 
3931   op_cost(10);
3932   format %{"[$reg + $lreg]" %}
3933   interface(MEMORY_INTER) %{
3934     base($reg);
3935     index($lreg);
3936     scale(0x0);
3937     disp(0x0);
3938   %}
3939 %}
3940 
3941 // Indirect Memory Times Scale Plus Index Register
3942 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3943 %{
3944   predicate(Universe::narrow_oop_shift() == 0);
3945   constraint(ALLOC_IN_RC(ptr_reg));
3946   match(AddP (DecodeN reg) (LShiftL lreg scale));
3947 
3948   op_cost(10);
3949   format %{"[$reg + $lreg << $scale]" %}
3950   interface(MEMORY_INTER) %{
3951     base($reg);
3952     index($lreg);
3953     scale($scale);
3954     disp(0x0);
3955   %}
3956 %}
3957 
3958 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3959 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3960 %{
3961   predicate(Universe::narrow_oop_shift() == 0);
3962   constraint(ALLOC_IN_RC(ptr_reg));
3963   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3964 
3965   op_cost(10);
3966   format %{"[$reg + $off + $lreg << $scale]" %}
3967   interface(MEMORY_INTER) %{
3968     base($reg);
3969     index($lreg);
3970     scale($scale);
3971     disp($off);
3972   %}
3973 %}
3974 
3975 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3976 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3977 %{
3978   constraint(ALLOC_IN_RC(ptr_reg));
3979   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3980   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3981 
3982   op_cost(10);
3983   format %{"[$reg + $off + $idx]" %}
3984   interface(MEMORY_INTER) %{
3985     base($reg);
3986     index($idx);
3987     scale(0x0);
3988     disp($off);
3989   %}
3990 %}
3991 
3992 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3993 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3994 %{
3995   constraint(ALLOC_IN_RC(ptr_reg));
3996   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3997   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3998 
3999   op_cost(10);
4000   format %{"[$reg + $off + $idx << $scale]" %}
4001   interface(MEMORY_INTER) %{
4002     base($reg);
4003     index($idx);
4004     scale($scale);
4005     disp($off);
4006   %}
4007 %}
4008 
4009 //----------Special Memory Operands--------------------------------------------
4010 // Stack Slot Operand - This operand is used for loading and storing temporary
4011 //                      values on the stack where a match requires a value to
4012 //                      flow through memory.
4013 operand stackSlotP(sRegP reg)
4014 %{
4015   constraint(ALLOC_IN_RC(stack_slots));
4016   // No match rule because this operand is only generated in matching
4017 
4018   format %{ "[$reg]" %}
4019   interface(MEMORY_INTER) %{
4020     base(0x4);   // RSP
4021     index(0x4);  // No Index
4022     scale(0x0);  // No Scale
4023     disp($reg);  // Stack Offset
4024   %}
4025 %}
4026 
4027 operand stackSlotI(sRegI reg)
4028 %{
4029   constraint(ALLOC_IN_RC(stack_slots));
4030   // No match rule because this operand is only generated in matching
4031 
4032   format %{ "[$reg]" %}
4033   interface(MEMORY_INTER) %{
4034     base(0x4);   // RSP
4035     index(0x4);  // No Index
4036     scale(0x0);  // No Scale
4037     disp($reg);  // Stack Offset
4038   %}
4039 %}
4040 
4041 operand stackSlotF(sRegF reg)
4042 %{
4043   constraint(ALLOC_IN_RC(stack_slots));
4044   // No match rule because this operand is only generated in matching
4045 
4046   format %{ "[$reg]" %}
4047   interface(MEMORY_INTER) %{
4048     base(0x4);   // RSP
4049     index(0x4);  // No Index
4050     scale(0x0);  // No Scale
4051     disp($reg);  // Stack Offset
4052   %}
4053 %}
4054 
4055 operand stackSlotD(sRegD reg)
4056 %{
4057   constraint(ALLOC_IN_RC(stack_slots));
4058   // No match rule because this operand is only generated in matching
4059 
4060   format %{ "[$reg]" %}
4061   interface(MEMORY_INTER) %{
4062     base(0x4);   // RSP
4063     index(0x4);  // No Index
4064     scale(0x0);  // No Scale
4065     disp($reg);  // Stack Offset
4066   %}
4067 %}
4068 operand stackSlotL(sRegL reg)
4069 %{
4070   constraint(ALLOC_IN_RC(stack_slots));
4071   // No match rule because this operand is only generated in matching
4072 
4073   format %{ "[$reg]" %}
4074   interface(MEMORY_INTER) %{
4075     base(0x4);   // RSP
4076     index(0x4);  // No Index
4077     scale(0x0);  // No Scale
4078     disp($reg);  // Stack Offset
4079   %}
4080 %}
4081 
4082 //----------Conditional Branch Operands----------------------------------------
4083 // Comparison Op  - This is the operation of the comparison, and is limited to
4084 //                  the following set of codes:
4085 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4086 //
4087 // Other attributes of the comparison, such as unsignedness, are specified
4088 // by the comparison instruction that sets a condition code flags register.
4089 // That result is represented by a flags operand whose subtype is appropriate
4090 // to the unsignedness (etc.) of the comparison.
4091 //
4092 // Later, the instruction which matches both the Comparison Op (a Bool) and
4093 // the flags (produced by the Cmp) specifies the coding of the comparison op
4094 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4095 
4096 // Comparision Code
4097 operand cmpOp()
4098 %{
4099   match(Bool);
4100 
4101   format %{ "" %}
4102   interface(COND_INTER) %{
4103     equal(0x4, "e");
4104     not_equal(0x5, "ne");
4105     less(0xC, "l");
4106     greater_equal(0xD, "ge");
4107     less_equal(0xE, "le");
4108     greater(0xF, "g");
4109     overflow(0x0, "o");
4110     no_overflow(0x1, "no");
4111   %}
4112 %}
4113 
4114 // Comparison Code, unsigned compare.  Used by FP also, with
4115 // C2 (unordered) turned into GT or LT already.  The other bits
4116 // C0 and C3 are turned into Carry & Zero flags.
4117 operand cmpOpU()
4118 %{
4119   match(Bool);
4120 
4121   format %{ "" %}
4122   interface(COND_INTER) %{
4123     equal(0x4, "e");
4124     not_equal(0x5, "ne");
4125     less(0x2, "b");
4126     greater_equal(0x3, "nb");
4127     less_equal(0x6, "be");
4128     greater(0x7, "nbe");
4129     overflow(0x0, "o");
4130     no_overflow(0x1, "no");
4131   %}
4132 %}
4133 
4134 
4135 // Floating comparisons that don't require any fixup for the unordered case
4136 operand cmpOpUCF() %{
4137   match(Bool);
4138   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4139             n->as_Bool()->_test._test == BoolTest::ge ||
4140             n->as_Bool()->_test._test == BoolTest::le ||
4141             n->as_Bool()->_test._test == BoolTest::gt);
4142   format %{ "" %}
4143   interface(COND_INTER) %{
4144     equal(0x4, "e");
4145     not_equal(0x5, "ne");
4146     less(0x2, "b");
4147     greater_equal(0x3, "nb");
4148     less_equal(0x6, "be");
4149     greater(0x7, "nbe");
4150     overflow(0x0, "o");
4151     no_overflow(0x1, "no");
4152   %}
4153 %}
4154 
4155 
4156 // Floating comparisons that can be fixed up with extra conditional jumps
4157 operand cmpOpUCF2() %{
4158   match(Bool);
4159   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4160             n->as_Bool()->_test._test == BoolTest::eq);
4161   format %{ "" %}
4162   interface(COND_INTER) %{
4163     equal(0x4, "e");
4164     not_equal(0x5, "ne");
4165     less(0x2, "b");
4166     greater_equal(0x3, "nb");
4167     less_equal(0x6, "be");
4168     greater(0x7, "nbe");
4169     overflow(0x0, "o");
4170     no_overflow(0x1, "no");
4171   %}
4172 %}
4173 
4174 
4175 //----------OPERAND CLASSES----------------------------------------------------
4176 // Operand Classes are groups of operands that are used as to simplify
4177 // instruction definitions by not requiring the AD writer to specify separate
4178 // instructions for every form of operand when the instruction accepts
4179 // multiple operand types with the same basic encoding and format.  The classic
4180 // case of this is memory operands.
4181 
4182 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4183                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4184                indCompressedOopOffset,
4185                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4186                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4187                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4188 
4189 //----------PIPELINE-----------------------------------------------------------
4190 // Rules which define the behavior of the target architectures pipeline.
4191 pipeline %{
4192 
4193 //----------ATTRIBUTES---------------------------------------------------------
4194 attributes %{
4195   variable_size_instructions;        // Fixed size instructions
4196   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4197   instruction_unit_size = 1;         // An instruction is 1 bytes long
4198   instruction_fetch_unit_size = 16;  // The processor fetches one line
4199   instruction_fetch_units = 1;       // of 16 bytes
4200 
4201   // List of nop instructions
4202   nops( MachNop );
4203 %}
4204 
4205 //----------RESOURCES----------------------------------------------------------
4206 // Resources are the functional units available to the machine
4207 
4208 // Generic P2/P3 pipeline
4209 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4210 // 3 instructions decoded per cycle.
4211 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4212 // 3 ALU op, only ALU0 handles mul instructions.
4213 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4214            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4215            BR, FPU,
4216            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4217 
4218 //----------PIPELINE DESCRIPTION-----------------------------------------------
4219 // Pipeline Description specifies the stages in the machine's pipeline
4220 
4221 // Generic P2/P3 pipeline
4222 pipe_desc(S0, S1, S2, S3, S4, S5);
4223 
4224 //----------PIPELINE CLASSES---------------------------------------------------
4225 // Pipeline Classes describe the stages in which input and output are
4226 // referenced by the hardware pipeline.
4227 
4228 // Naming convention: ialu or fpu
4229 // Then: _reg
4230 // Then: _reg if there is a 2nd register
4231 // Then: _long if it's a pair of instructions implementing a long
4232 // Then: _fat if it requires the big decoder
4233 //   Or: _mem if it requires the big decoder and a memory unit.
4234 
4235 // Integer ALU reg operation
4236 pipe_class ialu_reg(rRegI dst)
4237 %{
4238     single_instruction;
4239     dst    : S4(write);
4240     dst    : S3(read);
4241     DECODE : S0;        // any decoder
4242     ALU    : S3;        // any alu
4243 %}
4244 
4245 // Long ALU reg operation
4246 pipe_class ialu_reg_long(rRegL dst)
4247 %{
4248     instruction_count(2);
4249     dst    : S4(write);
4250     dst    : S3(read);
4251     DECODE : S0(2);     // any 2 decoders
4252     ALU    : S3(2);     // both alus
4253 %}
4254 
4255 // Integer ALU reg operation using big decoder
4256 pipe_class ialu_reg_fat(rRegI dst)
4257 %{
4258     single_instruction;
4259     dst    : S4(write);
4260     dst    : S3(read);
4261     D0     : S0;        // big decoder only
4262     ALU    : S3;        // any alu
4263 %}
4264 
4265 // Long ALU reg operation using big decoder
4266 pipe_class ialu_reg_long_fat(rRegL dst)
4267 %{
4268     instruction_count(2);
4269     dst    : S4(write);
4270     dst    : S3(read);
4271     D0     : S0(2);     // big decoder only; twice
4272     ALU    : S3(2);     // any 2 alus
4273 %}
4274 
4275 // Integer ALU reg-reg operation
4276 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4277 %{
4278     single_instruction;
4279     dst    : S4(write);
4280     src    : S3(read);
4281     DECODE : S0;        // any decoder
4282     ALU    : S3;        // any alu
4283 %}
4284 
4285 // Long ALU reg-reg operation
4286 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4287 %{
4288     instruction_count(2);
4289     dst    : S4(write);
4290     src    : S3(read);
4291     DECODE : S0(2);     // any 2 decoders
4292     ALU    : S3(2);     // both alus
4293 %}
4294 
4295 // Integer ALU reg-reg operation
4296 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4297 %{
4298     single_instruction;
4299     dst    : S4(write);
4300     src    : S3(read);
4301     D0     : S0;        // big decoder only
4302     ALU    : S3;        // any alu
4303 %}
4304 
4305 // Long ALU reg-reg operation
4306 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4307 %{
4308     instruction_count(2);
4309     dst    : S4(write);
4310     src    : S3(read);
4311     D0     : S0(2);     // big decoder only; twice
4312     ALU    : S3(2);     // both alus
4313 %}
4314 
4315 // Integer ALU reg-mem operation
4316 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4317 %{
4318     single_instruction;
4319     dst    : S5(write);
4320     mem    : S3(read);
4321     D0     : S0;        // big decoder only
4322     ALU    : S4;        // any alu
4323     MEM    : S3;        // any mem
4324 %}
4325 
4326 // Integer mem operation (prefetch)
4327 pipe_class ialu_mem(memory mem)
4328 %{
4329     single_instruction;
4330     mem    : S3(read);
4331     D0     : S0;        // big decoder only
4332     MEM    : S3;        // any mem
4333 %}
4334 
4335 // Integer Store to Memory
4336 pipe_class ialu_mem_reg(memory mem, rRegI src)
4337 %{
4338     single_instruction;
4339     mem    : S3(read);
4340     src    : S5(read);
4341     D0     : S0;        // big decoder only
4342     ALU    : S4;        // any alu
4343     MEM    : S3;
4344 %}
4345 
4346 // // Long Store to Memory
4347 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4348 // %{
4349 //     instruction_count(2);
4350 //     mem    : S3(read);
4351 //     src    : S5(read);
4352 //     D0     : S0(2);          // big decoder only; twice
4353 //     ALU    : S4(2);     // any 2 alus
4354 //     MEM    : S3(2);  // Both mems
4355 // %}
4356 
4357 // Integer Store to Memory
4358 pipe_class ialu_mem_imm(memory mem)
4359 %{
4360     single_instruction;
4361     mem    : S3(read);
4362     D0     : S0;        // big decoder only
4363     ALU    : S4;        // any alu
4364     MEM    : S3;
4365 %}
4366 
4367 // Integer ALU0 reg-reg operation
4368 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4369 %{
4370     single_instruction;
4371     dst    : S4(write);
4372     src    : S3(read);
4373     D0     : S0;        // Big decoder only
4374     ALU0   : S3;        // only alu0
4375 %}
4376 
4377 // Integer ALU0 reg-mem operation
4378 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4379 %{
4380     single_instruction;
4381     dst    : S5(write);
4382     mem    : S3(read);
4383     D0     : S0;        // big decoder only
4384     ALU0   : S4;        // ALU0 only
4385     MEM    : S3;        // any mem
4386 %}
4387 
4388 // Integer ALU reg-reg operation
4389 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4390 %{
4391     single_instruction;
4392     cr     : S4(write);
4393     src1   : S3(read);
4394     src2   : S3(read);
4395     DECODE : S0;        // any decoder
4396     ALU    : S3;        // any alu
4397 %}
4398 
4399 // Integer ALU reg-imm operation
4400 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4401 %{
4402     single_instruction;
4403     cr     : S4(write);
4404     src1   : S3(read);
4405     DECODE : S0;        // any decoder
4406     ALU    : S3;        // any alu
4407 %}
4408 
4409 // Integer ALU reg-mem operation
4410 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4411 %{
4412     single_instruction;
4413     cr     : S4(write);
4414     src1   : S3(read);
4415     src2   : S3(read);
4416     D0     : S0;        // big decoder only
4417     ALU    : S4;        // any alu
4418     MEM    : S3;
4419 %}
4420 
4421 // Conditional move reg-reg
4422 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4423 %{
4424     instruction_count(4);
4425     y      : S4(read);
4426     q      : S3(read);
4427     p      : S3(read);
4428     DECODE : S0(4);     // any decoder
4429 %}
4430 
4431 // Conditional move reg-reg
4432 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4433 %{
4434     single_instruction;
4435     dst    : S4(write);
4436     src    : S3(read);
4437     cr     : S3(read);
4438     DECODE : S0;        // any decoder
4439 %}
4440 
4441 // Conditional move reg-mem
4442 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4443 %{
4444     single_instruction;
4445     dst    : S4(write);
4446     src    : S3(read);
4447     cr     : S3(read);
4448     DECODE : S0;        // any decoder
4449     MEM    : S3;
4450 %}
4451 
4452 // Conditional move reg-reg long
4453 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4454 %{
4455     single_instruction;
4456     dst    : S4(write);
4457     src    : S3(read);
4458     cr     : S3(read);
4459     DECODE : S0(2);     // any 2 decoders
4460 %}
4461 
4462 // XXX
4463 // // Conditional move double reg-reg
4464 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4465 // %{
4466 //     single_instruction;
4467 //     dst    : S4(write);
4468 //     src    : S3(read);
4469 //     cr     : S3(read);
4470 //     DECODE : S0;     // any decoder
4471 // %}
4472 
4473 // Float reg-reg operation
4474 pipe_class fpu_reg(regD dst)
4475 %{
4476     instruction_count(2);
4477     dst    : S3(read);
4478     DECODE : S0(2);     // any 2 decoders
4479     FPU    : S3;
4480 %}
4481 
4482 // Float reg-reg operation
4483 pipe_class fpu_reg_reg(regD dst, regD src)
4484 %{
4485     instruction_count(2);
4486     dst    : S4(write);
4487     src    : S3(read);
4488     DECODE : S0(2);     // any 2 decoders
4489     FPU    : S3;
4490 %}
4491 
4492 // Float reg-reg operation
4493 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4494 %{
4495     instruction_count(3);
4496     dst    : S4(write);
4497     src1   : S3(read);
4498     src2   : S3(read);
4499     DECODE : S0(3);     // any 3 decoders
4500     FPU    : S3(2);
4501 %}
4502 
4503 // Float reg-reg operation
4504 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4505 %{
4506     instruction_count(4);
4507     dst    : S4(write);
4508     src1   : S3(read);
4509     src2   : S3(read);
4510     src3   : S3(read);
4511     DECODE : S0(4);     // any 3 decoders
4512     FPU    : S3(2);
4513 %}
4514 
4515 // Float reg-reg operation
4516 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4517 %{
4518     instruction_count(4);
4519     dst    : S4(write);
4520     src1   : S3(read);
4521     src2   : S3(read);
4522     src3   : S3(read);
4523     DECODE : S1(3);     // any 3 decoders
4524     D0     : S0;        // Big decoder only
4525     FPU    : S3(2);
4526     MEM    : S3;
4527 %}
4528 
4529 // Float reg-mem operation
4530 pipe_class fpu_reg_mem(regD dst, memory mem)
4531 %{
4532     instruction_count(2);
4533     dst    : S5(write);
4534     mem    : S3(read);
4535     D0     : S0;        // big decoder only
4536     DECODE : S1;        // any decoder for FPU POP
4537     FPU    : S4;
4538     MEM    : S3;        // any mem
4539 %}
4540 
4541 // Float reg-mem operation
4542 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4543 %{
4544     instruction_count(3);
4545     dst    : S5(write);
4546     src1   : S3(read);
4547     mem    : S3(read);
4548     D0     : S0;        // big decoder only
4549     DECODE : S1(2);     // any decoder for FPU POP
4550     FPU    : S4;
4551     MEM    : S3;        // any mem
4552 %}
4553 
4554 // Float mem-reg operation
4555 pipe_class fpu_mem_reg(memory mem, regD src)
4556 %{
4557     instruction_count(2);
4558     src    : S5(read);
4559     mem    : S3(read);
4560     DECODE : S0;        // any decoder for FPU PUSH
4561     D0     : S1;        // big decoder only
4562     FPU    : S4;
4563     MEM    : S3;        // any mem
4564 %}
4565 
4566 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4567 %{
4568     instruction_count(3);
4569     src1   : S3(read);
4570     src2   : S3(read);
4571     mem    : S3(read);
4572     DECODE : S0(2);     // any decoder for FPU PUSH
4573     D0     : S1;        // big decoder only
4574     FPU    : S4;
4575     MEM    : S3;        // any mem
4576 %}
4577 
4578 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4579 %{
4580     instruction_count(3);
4581     src1   : S3(read);
4582     src2   : S3(read);
4583     mem    : S4(read);
4584     DECODE : S0;        // any decoder for FPU PUSH
4585     D0     : S0(2);     // big decoder only
4586     FPU    : S4;
4587     MEM    : S3(2);     // any mem
4588 %}
4589 
4590 pipe_class fpu_mem_mem(memory dst, memory src1)
4591 %{
4592     instruction_count(2);
4593     src1   : S3(read);
4594     dst    : S4(read);
4595     D0     : S0(2);     // big decoder only
4596     MEM    : S3(2);     // any mem
4597 %}
4598 
4599 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4600 %{
4601     instruction_count(3);
4602     src1   : S3(read);
4603     src2   : S3(read);
4604     dst    : S4(read);
4605     D0     : S0(3);     // big decoder only
4606     FPU    : S4;
4607     MEM    : S3(3);     // any mem
4608 %}
4609 
4610 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4611 %{
4612     instruction_count(3);
4613     src1   : S4(read);
4614     mem    : S4(read);
4615     DECODE : S0;        // any decoder for FPU PUSH
4616     D0     : S0(2);     // big decoder only
4617     FPU    : S4;
4618     MEM    : S3(2);     // any mem
4619 %}
4620 
4621 // Float load constant
4622 pipe_class fpu_reg_con(regD dst)
4623 %{
4624     instruction_count(2);
4625     dst    : S5(write);
4626     D0     : S0;        // big decoder only for the load
4627     DECODE : S1;        // any decoder for FPU POP
4628     FPU    : S4;
4629     MEM    : S3;        // any mem
4630 %}
4631 
4632 // Float load constant
4633 pipe_class fpu_reg_reg_con(regD dst, regD src)
4634 %{
4635     instruction_count(3);
4636     dst    : S5(write);
4637     src    : S3(read);
4638     D0     : S0;        // big decoder only for the load
4639     DECODE : S1(2);     // any decoder for FPU POP
4640     FPU    : S4;
4641     MEM    : S3;        // any mem
4642 %}
4643 
4644 // UnConditional branch
4645 pipe_class pipe_jmp(label labl)
4646 %{
4647     single_instruction;
4648     BR   : S3;
4649 %}
4650 
4651 // Conditional branch
4652 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4653 %{
4654     single_instruction;
4655     cr    : S1(read);
4656     BR    : S3;
4657 %}
4658 
4659 // Allocation idiom
4660 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4661 %{
4662     instruction_count(1); force_serialization;
4663     fixed_latency(6);
4664     heap_ptr : S3(read);
4665     DECODE   : S0(3);
4666     D0       : S2;
4667     MEM      : S3;
4668     ALU      : S3(2);
4669     dst      : S5(write);
4670     BR       : S5;
4671 %}
4672 
4673 // Generic big/slow expanded idiom
4674 pipe_class pipe_slow()
4675 %{
4676     instruction_count(10); multiple_bundles; force_serialization;
4677     fixed_latency(100);
4678     D0  : S0(2);
4679     MEM : S3(2);
4680 %}
4681 
4682 // The real do-nothing guy
4683 pipe_class empty()
4684 %{
4685     instruction_count(0);
4686 %}
4687 
4688 // Define the class for the Nop node
4689 define
4690 %{
4691    MachNop = empty;
4692 %}
4693 
4694 %}
4695 
4696 //----------INSTRUCTIONS-------------------------------------------------------
4697 //
4698 // match      -- States which machine-independent subtree may be replaced
4699 //               by this instruction.
4700 // ins_cost   -- The estimated cost of this instruction is used by instruction
4701 //               selection to identify a minimum cost tree of machine
4702 //               instructions that matches a tree of machine-independent
4703 //               instructions.
4704 // format     -- A string providing the disassembly for this instruction.
4705 //               The value of an instruction's operand may be inserted
4706 //               by referring to it with a '$' prefix.
4707 // opcode     -- Three instruction opcodes may be provided.  These are referred
4708 //               to within an encode class as $primary, $secondary, and $tertiary
4709 //               rrspectively.  The primary opcode is commonly used to
4710 //               indicate the type of machine instruction, while secondary
4711 //               and tertiary are often used for prefix options or addressing
4712 //               modes.
4713 // ins_encode -- A list of encode classes with parameters. The encode class
4714 //               name must have been defined in an 'enc_class' specification
4715 //               in the encode section of the architecture description.
4716 
4717 
4718 //----------Load/Store/Move Instructions---------------------------------------
4719 //----------Load Instructions--------------------------------------------------
4720 
4721 // Load Byte (8 bit signed)
4722 instruct loadB(rRegI dst, memory mem)
4723 %{
4724   match(Set dst (LoadB mem));
4725 
4726   ins_cost(125);
4727   format %{ "movsbl  $dst, $mem\t# byte" %}
4728 
4729   ins_encode %{
4730     __ movsbl($dst$$Register, $mem$$Address);
4731   %}
4732 
4733   ins_pipe(ialu_reg_mem);
4734 %}
4735 
4736 // Load Byte (8 bit signed) into Long Register
4737 instruct loadB2L(rRegL dst, memory mem)
4738 %{
4739   match(Set dst (ConvI2L (LoadB mem)));
4740 
4741   ins_cost(125);
4742   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4743 
4744   ins_encode %{
4745     __ movsbq($dst$$Register, $mem$$Address);
4746   %}
4747 
4748   ins_pipe(ialu_reg_mem);
4749 %}
4750 
4751 // Load Unsigned Byte (8 bit UNsigned)
4752 instruct loadUB(rRegI dst, memory mem)
4753 %{
4754   match(Set dst (LoadUB mem));
4755 
4756   ins_cost(125);
4757   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4758 
4759   ins_encode %{
4760     __ movzbl($dst$$Register, $mem$$Address);
4761   %}
4762 
4763   ins_pipe(ialu_reg_mem);
4764 %}
4765 
4766 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4767 instruct loadUB2L(rRegL dst, memory mem)
4768 %{
4769   match(Set dst (ConvI2L (LoadUB mem)));
4770 
4771   ins_cost(125);
4772   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4773 
4774   ins_encode %{
4775     __ movzbq($dst$$Register, $mem$$Address);
4776   %}
4777 
4778   ins_pipe(ialu_reg_mem);
4779 %}
4780 
4781 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4782 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4783   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4784   effect(KILL cr);
4785 
4786   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4787             "andl    $dst, right_n_bits($mask, 8)" %}
4788   ins_encode %{
4789     Register Rdst = $dst$$Register;
4790     __ movzbq(Rdst, $mem$$Address);
4791     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4792   %}
4793   ins_pipe(ialu_reg_mem);
4794 %}
4795 
4796 // Load Short (16 bit signed)
4797 instruct loadS(rRegI dst, memory mem)
4798 %{
4799   match(Set dst (LoadS mem));
4800 
4801   ins_cost(125);
4802   format %{ "movswl $dst, $mem\t# short" %}
4803 
4804   ins_encode %{
4805     __ movswl($dst$$Register, $mem$$Address);
4806   %}
4807 
4808   ins_pipe(ialu_reg_mem);
4809 %}
4810 
4811 // Load Short (16 bit signed) to Byte (8 bit signed)
4812 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4813   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4814 
4815   ins_cost(125);
4816   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4817   ins_encode %{
4818     __ movsbl($dst$$Register, $mem$$Address);
4819   %}
4820   ins_pipe(ialu_reg_mem);
4821 %}
4822 
4823 // Load Short (16 bit signed) into Long Register
4824 instruct loadS2L(rRegL dst, memory mem)
4825 %{
4826   match(Set dst (ConvI2L (LoadS mem)));
4827 
4828   ins_cost(125);
4829   format %{ "movswq $dst, $mem\t# short -> long" %}
4830 
4831   ins_encode %{
4832     __ movswq($dst$$Register, $mem$$Address);
4833   %}
4834 
4835   ins_pipe(ialu_reg_mem);
4836 %}
4837 
4838 // Load Unsigned Short/Char (16 bit UNsigned)
4839 instruct loadUS(rRegI dst, memory mem)
4840 %{
4841   match(Set dst (LoadUS mem));
4842 
4843   ins_cost(125);
4844   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4845 
4846   ins_encode %{
4847     __ movzwl($dst$$Register, $mem$$Address);
4848   %}
4849 
4850   ins_pipe(ialu_reg_mem);
4851 %}
4852 
4853 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4854 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4855   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4856 
4857   ins_cost(125);
4858   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4859   ins_encode %{
4860     __ movsbl($dst$$Register, $mem$$Address);
4861   %}
4862   ins_pipe(ialu_reg_mem);
4863 %}
4864 
4865 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4866 instruct loadUS2L(rRegL dst, memory mem)
4867 %{
4868   match(Set dst (ConvI2L (LoadUS mem)));
4869 
4870   ins_cost(125);
4871   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4872 
4873   ins_encode %{
4874     __ movzwq($dst$$Register, $mem$$Address);
4875   %}
4876 
4877   ins_pipe(ialu_reg_mem);
4878 %}
4879 
4880 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4881 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4882   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4883 
4884   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4885   ins_encode %{
4886     __ movzbq($dst$$Register, $mem$$Address);
4887   %}
4888   ins_pipe(ialu_reg_mem);
4889 %}
4890 
4891 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4892 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4893   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4894   effect(KILL cr);
4895 
4896   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4897             "andl    $dst, right_n_bits($mask, 16)" %}
4898   ins_encode %{
4899     Register Rdst = $dst$$Register;
4900     __ movzwq(Rdst, $mem$$Address);
4901     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4902   %}
4903   ins_pipe(ialu_reg_mem);
4904 %}
4905 
4906 // Load Integer
4907 instruct loadI(rRegI dst, memory mem)
4908 %{
4909   match(Set dst (LoadI mem));
4910 
4911   ins_cost(125);
4912   format %{ "movl    $dst, $mem\t# int" %}
4913 
4914   ins_encode %{
4915     __ movl($dst$$Register, $mem$$Address);
4916   %}
4917 
4918   ins_pipe(ialu_reg_mem);
4919 %}
4920 
4921 // Load Integer (32 bit signed) to Byte (8 bit signed)
4922 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4923   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4924 
4925   ins_cost(125);
4926   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4927   ins_encode %{
4928     __ movsbl($dst$$Register, $mem$$Address);
4929   %}
4930   ins_pipe(ialu_reg_mem);
4931 %}
4932 
4933 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4934 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4935   match(Set dst (AndI (LoadI mem) mask));
4936 
4937   ins_cost(125);
4938   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4939   ins_encode %{
4940     __ movzbl($dst$$Register, $mem$$Address);
4941   %}
4942   ins_pipe(ialu_reg_mem);
4943 %}
4944 
4945 // Load Integer (32 bit signed) to Short (16 bit signed)
4946 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4947   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4948 
4949   ins_cost(125);
4950   format %{ "movswl  $dst, $mem\t# int -> short" %}
4951   ins_encode %{
4952     __ movswl($dst$$Register, $mem$$Address);
4953   %}
4954   ins_pipe(ialu_reg_mem);
4955 %}
4956 
4957 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4958 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4959   match(Set dst (AndI (LoadI mem) mask));
4960 
4961   ins_cost(125);
4962   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4963   ins_encode %{
4964     __ movzwl($dst$$Register, $mem$$Address);
4965   %}
4966   ins_pipe(ialu_reg_mem);
4967 %}
4968 
4969 // Load Integer into Long Register
4970 instruct loadI2L(rRegL dst, memory mem)
4971 %{
4972   match(Set dst (ConvI2L (LoadI mem)));
4973 
4974   ins_cost(125);
4975   format %{ "movslq  $dst, $mem\t# int -> long" %}
4976 
4977   ins_encode %{
4978     __ movslq($dst$$Register, $mem$$Address);
4979   %}
4980 
4981   ins_pipe(ialu_reg_mem);
4982 %}
4983 
4984 // Load Integer with mask 0xFF into Long Register
4985 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4986   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4987 
4988   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4989   ins_encode %{
4990     __ movzbq($dst$$Register, $mem$$Address);
4991   %}
4992   ins_pipe(ialu_reg_mem);
4993 %}
4994 
4995 // Load Integer with mask 0xFFFF into Long Register
4996 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4997   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4998 
4999   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
5000   ins_encode %{
5001     __ movzwq($dst$$Register, $mem$$Address);
5002   %}
5003   ins_pipe(ialu_reg_mem);
5004 %}
5005 
5006 // Load Integer with a 31-bit mask into Long Register
5007 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5008   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5009   effect(KILL cr);
5010 
5011   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5012             "andl    $dst, $mask" %}
5013   ins_encode %{
5014     Register Rdst = $dst$$Register;
5015     __ movl(Rdst, $mem$$Address);
5016     __ andl(Rdst, $mask$$constant);
5017   %}
5018   ins_pipe(ialu_reg_mem);
5019 %}
5020 
5021 // Load Unsigned Integer into Long Register
5022 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5023 %{
5024   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5025 
5026   ins_cost(125);
5027   format %{ "movl    $dst, $mem\t# uint -> long" %}
5028 
5029   ins_encode %{
5030     __ movl($dst$$Register, $mem$$Address);
5031   %}
5032 
5033   ins_pipe(ialu_reg_mem);
5034 %}
5035 
5036 // Load Long
5037 instruct loadL(rRegL dst, memory mem)
5038 %{
5039   match(Set dst (LoadL mem));
5040 
5041   ins_cost(125);
5042   format %{ "movq    $dst, $mem\t# long" %}
5043 
5044   ins_encode %{
5045     __ movq($dst$$Register, $mem$$Address);
5046   %}
5047 
5048   ins_pipe(ialu_reg_mem); // XXX
5049 %}
5050 
5051 // Load Range
5052 instruct loadRange(rRegI dst, memory mem)
5053 %{
5054   match(Set dst (LoadRange mem));
5055 
5056   ins_cost(125); // XXX
5057   format %{ "movl    $dst, $mem\t# range" %}
5058   opcode(0x8B);
5059   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5060   ins_pipe(ialu_reg_mem);
5061 %}
5062 
5063 // Load Pointer
5064 instruct loadP(rRegP dst, memory mem)
5065 %{
5066   match(Set dst (LoadP mem));
5067 
5068   ins_cost(125); // XXX
5069   format %{ "movq    $dst, $mem\t# ptr" %}
5070   opcode(0x8B);
5071   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5072   ins_pipe(ialu_reg_mem); // XXX
5073 %}
5074 
5075 // Load Compressed Pointer
5076 instruct loadN(rRegN dst, memory mem)
5077 %{
5078    match(Set dst (LoadN mem));
5079 
5080    ins_cost(125); // XXX
5081    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5082    ins_encode %{
5083      __ movl($dst$$Register, $mem$$Address);
5084    %}
5085    ins_pipe(ialu_reg_mem); // XXX
5086 %}
5087 
5088 
5089 // Load Klass Pointer
5090 instruct loadKlass(rRegP dst, memory mem)
5091 %{
5092   match(Set dst (LoadKlass mem));
5093 
5094   ins_cost(125); // XXX
5095   format %{ "movq    $dst, $mem\t# class" %}
5096   opcode(0x8B);
5097   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5098   ins_pipe(ialu_reg_mem); // XXX
5099 %}
5100 
5101 // Load narrow Klass Pointer
5102 instruct loadNKlass(rRegN dst, memory mem)
5103 %{
5104   match(Set dst (LoadNKlass mem));
5105 
5106   ins_cost(125); // XXX
5107   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5108   ins_encode %{
5109     __ movl($dst$$Register, $mem$$Address);
5110   %}
5111   ins_pipe(ialu_reg_mem); // XXX
5112 %}
5113 
5114 // Load Float
5115 instruct loadF(regF dst, memory mem)
5116 %{
5117   match(Set dst (LoadF mem));
5118 
5119   ins_cost(145); // XXX
5120   format %{ "movss   $dst, $mem\t# float" %}
5121   ins_encode %{
5122     __ movflt($dst$$XMMRegister, $mem$$Address);
5123   %}
5124   ins_pipe(pipe_slow); // XXX
5125 %}
5126 
5127 // Load Double
5128 instruct loadD_partial(regD dst, memory mem)
5129 %{
5130   predicate(!UseXmmLoadAndClearUpper);
5131   match(Set dst (LoadD mem));
5132 
5133   ins_cost(145); // XXX
5134   format %{ "movlpd  $dst, $mem\t# double" %}
5135   ins_encode %{
5136     __ movdbl($dst$$XMMRegister, $mem$$Address);
5137   %}
5138   ins_pipe(pipe_slow); // XXX
5139 %}
5140 
5141 instruct loadD(regD dst, memory mem)
5142 %{
5143   predicate(UseXmmLoadAndClearUpper);
5144   match(Set dst (LoadD mem));
5145 
5146   ins_cost(145); // XXX
5147   format %{ "movsd   $dst, $mem\t# double" %}
5148   ins_encode %{
5149     __ movdbl($dst$$XMMRegister, $mem$$Address);
5150   %}
5151   ins_pipe(pipe_slow); // XXX
5152 %}
5153 
5154 // Load Effective Address
5155 instruct leaP8(rRegP dst, indOffset8 mem)
5156 %{
5157   match(Set dst mem);
5158 
5159   ins_cost(110); // XXX
5160   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5161   opcode(0x8D);
5162   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5163   ins_pipe(ialu_reg_reg_fat);
5164 %}
5165 
5166 instruct leaP32(rRegP dst, indOffset32 mem)
5167 %{
5168   match(Set dst mem);
5169 
5170   ins_cost(110);
5171   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5172   opcode(0x8D);
5173   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5174   ins_pipe(ialu_reg_reg_fat);
5175 %}
5176 
5177 // instruct leaPIdx(rRegP dst, indIndex mem)
5178 // %{
5179 //   match(Set dst mem);
5180 
5181 //   ins_cost(110);
5182 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5183 //   opcode(0x8D);
5184 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5185 //   ins_pipe(ialu_reg_reg_fat);
5186 // %}
5187 
5188 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5189 %{
5190   match(Set dst mem);
5191 
5192   ins_cost(110);
5193   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5194   opcode(0x8D);
5195   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5196   ins_pipe(ialu_reg_reg_fat);
5197 %}
5198 
5199 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5200 %{
5201   match(Set dst mem);
5202 
5203   ins_cost(110);
5204   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5205   opcode(0x8D);
5206   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5207   ins_pipe(ialu_reg_reg_fat);
5208 %}
5209 
5210 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5211 %{
5212   match(Set dst mem);
5213 
5214   ins_cost(110);
5215   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5216   opcode(0x8D);
5217   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5218   ins_pipe(ialu_reg_reg_fat);
5219 %}
5220 
5221 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5222 %{
5223   match(Set dst mem);
5224 
5225   ins_cost(110);
5226   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5227   opcode(0x8D);
5228   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5229   ins_pipe(ialu_reg_reg_fat);
5230 %}
5231 
5232 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5233 %{
5234   match(Set dst mem);
5235 
5236   ins_cost(110);
5237   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5238   opcode(0x8D);
5239   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5240   ins_pipe(ialu_reg_reg_fat);
5241 %}
5242 
5243 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5244 %{
5245   match(Set dst mem);
5246 
5247   ins_cost(110);
5248   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5249   opcode(0x8D);
5250   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5251   ins_pipe(ialu_reg_reg_fat);
5252 %}
5253 
5254 // Load Effective Address which uses Narrow (32-bits) oop
5255 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5256 %{
5257   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5258   match(Set dst mem);
5259 
5260   ins_cost(110);
5261   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5262   opcode(0x8D);
5263   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5264   ins_pipe(ialu_reg_reg_fat);
5265 %}
5266 
5267 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5268 %{
5269   predicate(Universe::narrow_oop_shift() == 0);
5270   match(Set dst mem);
5271 
5272   ins_cost(110); // XXX
5273   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5274   opcode(0x8D);
5275   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5276   ins_pipe(ialu_reg_reg_fat);
5277 %}
5278 
5279 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5280 %{
5281   predicate(Universe::narrow_oop_shift() == 0);
5282   match(Set dst mem);
5283 
5284   ins_cost(110);
5285   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5286   opcode(0x8D);
5287   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5288   ins_pipe(ialu_reg_reg_fat);
5289 %}
5290 
5291 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5292 %{
5293   predicate(Universe::narrow_oop_shift() == 0);
5294   match(Set dst mem);
5295 
5296   ins_cost(110);
5297   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5298   opcode(0x8D);
5299   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5300   ins_pipe(ialu_reg_reg_fat);
5301 %}
5302 
5303 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5304 %{
5305   predicate(Universe::narrow_oop_shift() == 0);
5306   match(Set dst mem);
5307 
5308   ins_cost(110);
5309   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5310   opcode(0x8D);
5311   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5312   ins_pipe(ialu_reg_reg_fat);
5313 %}
5314 
5315 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5316 %{
5317   predicate(Universe::narrow_oop_shift() == 0);
5318   match(Set dst mem);
5319 
5320   ins_cost(110);
5321   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5322   opcode(0x8D);
5323   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5324   ins_pipe(ialu_reg_reg_fat);
5325 %}
5326 
5327 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5328 %{
5329   predicate(Universe::narrow_oop_shift() == 0);
5330   match(Set dst mem);
5331 
5332   ins_cost(110);
5333   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5334   opcode(0x8D);
5335   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5336   ins_pipe(ialu_reg_reg_fat);
5337 %}
5338 
5339 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5340 %{
5341   predicate(Universe::narrow_oop_shift() == 0);
5342   match(Set dst mem);
5343 
5344   ins_cost(110);
5345   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5346   opcode(0x8D);
5347   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5348   ins_pipe(ialu_reg_reg_fat);
5349 %}
5350 
5351 instruct loadConI(rRegI dst, immI src)
5352 %{
5353   match(Set dst src);
5354 
5355   format %{ "movl    $dst, $src\t# int" %}
5356   ins_encode(load_immI(dst, src));
5357   ins_pipe(ialu_reg_fat); // XXX
5358 %}
5359 
5360 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5361 %{
5362   match(Set dst src);
5363   effect(KILL cr);
5364 
5365   ins_cost(50);
5366   format %{ "xorl    $dst, $dst\t# int" %}
5367   opcode(0x33); /* + rd */
5368   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5369   ins_pipe(ialu_reg);
5370 %}
5371 
5372 instruct loadConL(rRegL dst, immL src)
5373 %{
5374   match(Set dst src);
5375 
5376   ins_cost(150);
5377   format %{ "movq    $dst, $src\t# long" %}
5378   ins_encode(load_immL(dst, src));
5379   ins_pipe(ialu_reg);
5380 %}
5381 
5382 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5383 %{
5384   match(Set dst src);
5385   effect(KILL cr);
5386 
5387   ins_cost(50);
5388   format %{ "xorl    $dst, $dst\t# long" %}
5389   opcode(0x33); /* + rd */
5390   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5391   ins_pipe(ialu_reg); // XXX
5392 %}
5393 
5394 instruct loadConUL32(rRegL dst, immUL32 src)
5395 %{
5396   match(Set dst src);
5397 
5398   ins_cost(60);
5399   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5400   ins_encode(load_immUL32(dst, src));
5401   ins_pipe(ialu_reg);
5402 %}
5403 
5404 instruct loadConL32(rRegL dst, immL32 src)
5405 %{
5406   match(Set dst src);
5407 
5408   ins_cost(70);
5409   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5410   ins_encode(load_immL32(dst, src));
5411   ins_pipe(ialu_reg);
5412 %}
5413 
5414 instruct loadConP(rRegP dst, immP con) %{
5415   match(Set dst con);
5416 
5417   format %{ "movq    $dst, $con\t# ptr" %}
5418   ins_encode(load_immP(dst, con));
5419   ins_pipe(ialu_reg_fat); // XXX
5420 %}
5421 
5422 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5423 %{
5424   match(Set dst src);
5425   effect(KILL cr);
5426 
5427   ins_cost(50);
5428   format %{ "xorl    $dst, $dst\t# ptr" %}
5429   opcode(0x33); /* + rd */
5430   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5431   ins_pipe(ialu_reg);
5432 %}
5433 
5434 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5435 %{
5436   match(Set dst src);
5437   effect(KILL cr);
5438 
5439   ins_cost(60);
5440   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5441   ins_encode(load_immP31(dst, src));
5442   ins_pipe(ialu_reg);
5443 %}
5444 
5445 instruct loadConF(regF dst, immF con) %{
5446   match(Set dst con);
5447   ins_cost(125);
5448   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5449   ins_encode %{
5450     __ movflt($dst$$XMMRegister, $constantaddress($con));
5451   %}
5452   ins_pipe(pipe_slow);
5453 %}
5454 
5455 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5456   match(Set dst src);
5457   effect(KILL cr);
5458   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5459   ins_encode %{
5460     __ xorq($dst$$Register, $dst$$Register);
5461   %}
5462   ins_pipe(ialu_reg);
5463 %}
5464 
5465 instruct loadConN(rRegN dst, immN src) %{
5466   match(Set dst src);
5467 
5468   ins_cost(125);
5469   format %{ "movl    $dst, $src\t# compressed ptr" %}
5470   ins_encode %{
5471     address con = (address)$src$$constant;
5472     if (con == NULL) {
5473       ShouldNotReachHere();
5474     } else {
5475       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5476     }
5477   %}
5478   ins_pipe(ialu_reg_fat); // XXX
5479 %}
5480 
5481 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5482   match(Set dst src);
5483 
5484   ins_cost(125);
5485   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5486   ins_encode %{
5487     address con = (address)$src$$constant;
5488     if (con == NULL) {
5489       ShouldNotReachHere();
5490     } else {
5491       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5492     }
5493   %}
5494   ins_pipe(ialu_reg_fat); // XXX
5495 %}
5496 
5497 instruct loadConF0(regF dst, immF0 src)
5498 %{
5499   match(Set dst src);
5500   ins_cost(100);
5501 
5502   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5503   ins_encode %{
5504     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5505   %}
5506   ins_pipe(pipe_slow);
5507 %}
5508 
5509 // Use the same format since predicate() can not be used here.
5510 instruct loadConD(regD dst, immD con) %{
5511   match(Set dst con);
5512   ins_cost(125);
5513   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5514   ins_encode %{
5515     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5516   %}
5517   ins_pipe(pipe_slow);
5518 %}
5519 
5520 instruct loadConD0(regD dst, immD0 src)
5521 %{
5522   match(Set dst src);
5523   ins_cost(100);
5524 
5525   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5526   ins_encode %{
5527     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5528   %}
5529   ins_pipe(pipe_slow);
5530 %}
5531 
5532 instruct loadSSI(rRegI dst, stackSlotI src)
5533 %{
5534   match(Set dst src);
5535 
5536   ins_cost(125);
5537   format %{ "movl    $dst, $src\t# int stk" %}
5538   opcode(0x8B);
5539   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5540   ins_pipe(ialu_reg_mem);
5541 %}
5542 
5543 instruct loadSSL(rRegL dst, stackSlotL src)
5544 %{
5545   match(Set dst src);
5546 
5547   ins_cost(125);
5548   format %{ "movq    $dst, $src\t# long stk" %}
5549   opcode(0x8B);
5550   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5551   ins_pipe(ialu_reg_mem);
5552 %}
5553 
5554 instruct loadSSP(rRegP dst, stackSlotP src)
5555 %{
5556   match(Set dst src);
5557 
5558   ins_cost(125);
5559   format %{ "movq    $dst, $src\t# ptr stk" %}
5560   opcode(0x8B);
5561   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5562   ins_pipe(ialu_reg_mem);
5563 %}
5564 
5565 instruct loadSSF(regF dst, stackSlotF src)
5566 %{
5567   match(Set dst src);
5568 
5569   ins_cost(125);
5570   format %{ "movss   $dst, $src\t# float stk" %}
5571   ins_encode %{
5572     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5573   %}
5574   ins_pipe(pipe_slow); // XXX
5575 %}
5576 
5577 // Use the same format since predicate() can not be used here.
5578 instruct loadSSD(regD dst, stackSlotD src)
5579 %{
5580   match(Set dst src);
5581 
5582   ins_cost(125);
5583   format %{ "movsd   $dst, $src\t# double stk" %}
5584   ins_encode  %{
5585     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5586   %}
5587   ins_pipe(pipe_slow); // XXX
5588 %}
5589 
5590 // Prefetch instructions for allocation.
5591 // Must be safe to execute with invalid address (cannot fault).
5592 
5593 instruct prefetchAlloc( memory mem ) %{
5594   predicate(AllocatePrefetchInstr==3);
5595   match(PrefetchAllocation mem);
5596   ins_cost(125);
5597 
5598   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5599   ins_encode %{
5600     __ prefetchw($mem$$Address);
5601   %}
5602   ins_pipe(ialu_mem);
5603 %}
5604 
5605 instruct prefetchAllocNTA( memory mem ) %{
5606   predicate(AllocatePrefetchInstr==0);
5607   match(PrefetchAllocation mem);
5608   ins_cost(125);
5609 
5610   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5611   ins_encode %{
5612     __ prefetchnta($mem$$Address);
5613   %}
5614   ins_pipe(ialu_mem);
5615 %}
5616 
5617 instruct prefetchAllocT0( memory mem ) %{
5618   predicate(AllocatePrefetchInstr==1);
5619   match(PrefetchAllocation mem);
5620   ins_cost(125);
5621 
5622   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5623   ins_encode %{
5624     __ prefetcht0($mem$$Address);
5625   %}
5626   ins_pipe(ialu_mem);
5627 %}
5628 
5629 instruct prefetchAllocT2( memory mem ) %{
5630   predicate(AllocatePrefetchInstr==2);
5631   match(PrefetchAllocation mem);
5632   ins_cost(125);
5633 
5634   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5635   ins_encode %{
5636     __ prefetcht2($mem$$Address);
5637   %}
5638   ins_pipe(ialu_mem);
5639 %}
5640 
5641 //----------Store Instructions-------------------------------------------------
5642 
5643 // Store Byte
5644 instruct storeB(memory mem, rRegI src)
5645 %{
5646   match(Set mem (StoreB mem src));
5647 
5648   ins_cost(125); // XXX
5649   format %{ "movb    $mem, $src\t# byte" %}
5650   opcode(0x88);
5651   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5652   ins_pipe(ialu_mem_reg);
5653 %}
5654 
5655 // Store Char/Short
5656 instruct storeC(memory mem, rRegI src)
5657 %{
5658   match(Set mem (StoreC mem src));
5659 
5660   ins_cost(125); // XXX
5661   format %{ "movw    $mem, $src\t# char/short" %}
5662   opcode(0x89);
5663   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5664   ins_pipe(ialu_mem_reg);
5665 %}
5666 
5667 // Store Integer
5668 instruct storeI(memory mem, rRegI src)
5669 %{
5670   match(Set mem (StoreI mem src));
5671 
5672   ins_cost(125); // XXX
5673   format %{ "movl    $mem, $src\t# int" %}
5674   opcode(0x89);
5675   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5676   ins_pipe(ialu_mem_reg);
5677 %}
5678 
5679 // Store Long
5680 instruct storeL(memory mem, rRegL src)
5681 %{
5682   match(Set mem (StoreL mem src));
5683 
5684   ins_cost(125); // XXX
5685   format %{ "movq    $mem, $src\t# long" %}
5686   opcode(0x89);
5687   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5688   ins_pipe(ialu_mem_reg); // XXX
5689 %}
5690 
5691 // Store Pointer
5692 instruct storeP(memory mem, any_RegP src)
5693 %{
5694   match(Set mem (StoreP mem src));
5695 
5696   ins_cost(125); // XXX
5697   format %{ "movq    $mem, $src\t# ptr" %}
5698   opcode(0x89);
5699   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5700   ins_pipe(ialu_mem_reg);
5701 %}
5702 
5703 instruct storeImmP0(memory mem, immP0 zero)
5704 %{
5705   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5706   match(Set mem (StoreP mem zero));
5707 
5708   ins_cost(125); // XXX
5709   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5710   ins_encode %{
5711     __ movq($mem$$Address, r12);
5712   %}
5713   ins_pipe(ialu_mem_reg);
5714 %}
5715 
5716 // Store NULL Pointer, mark word, or other simple pointer constant.
5717 instruct storeImmP(memory mem, immP31 src)
5718 %{
5719   match(Set mem (StoreP mem src));
5720 
5721   ins_cost(150); // XXX
5722   format %{ "movq    $mem, $src\t# ptr" %}
5723   opcode(0xC7); /* C7 /0 */
5724   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5725   ins_pipe(ialu_mem_imm);
5726 %}
5727 
5728 // Store Compressed Pointer
5729 instruct storeN(memory mem, rRegN src)
5730 %{
5731   match(Set mem (StoreN mem src));
5732 
5733   ins_cost(125); // XXX
5734   format %{ "movl    $mem, $src\t# compressed ptr" %}
5735   ins_encode %{
5736     __ movl($mem$$Address, $src$$Register);
5737   %}
5738   ins_pipe(ialu_mem_reg);
5739 %}
5740 
5741 instruct storeNKlass(memory mem, rRegN src)
5742 %{
5743   match(Set mem (StoreNKlass mem src));
5744 
5745   ins_cost(125); // XXX
5746   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5747   ins_encode %{
5748     __ movl($mem$$Address, $src$$Register);
5749   %}
5750   ins_pipe(ialu_mem_reg);
5751 %}
5752 
5753 instruct storeImmN0(memory mem, immN0 zero)
5754 %{
5755   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5756   match(Set mem (StoreN mem zero));
5757 
5758   ins_cost(125); // XXX
5759   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5760   ins_encode %{
5761     __ movl($mem$$Address, r12);
5762   %}
5763   ins_pipe(ialu_mem_reg);
5764 %}
5765 
5766 instruct storeImmN(memory mem, immN src)
5767 %{
5768   match(Set mem (StoreN mem src));
5769 
5770   ins_cost(150); // XXX
5771   format %{ "movl    $mem, $src\t# compressed ptr" %}
5772   ins_encode %{
5773     address con = (address)$src$$constant;
5774     if (con == NULL) {
5775       __ movl($mem$$Address, (int32_t)0);
5776     } else {
5777       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5778     }
5779   %}
5780   ins_pipe(ialu_mem_imm);
5781 %}
5782 
5783 instruct storeImmNKlass(memory mem, immNKlass src)
5784 %{
5785   match(Set mem (StoreNKlass mem src));
5786 
5787   ins_cost(150); // XXX
5788   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5789   ins_encode %{
5790     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5791   %}
5792   ins_pipe(ialu_mem_imm);
5793 %}
5794 
5795 // Store Integer Immediate
5796 instruct storeImmI0(memory mem, immI0 zero)
5797 %{
5798   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5799   match(Set mem (StoreI mem zero));
5800 
5801   ins_cost(125); // XXX
5802   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5803   ins_encode %{
5804     __ movl($mem$$Address, r12);
5805   %}
5806   ins_pipe(ialu_mem_reg);
5807 %}
5808 
5809 instruct storeImmI(memory mem, immI src)
5810 %{
5811   match(Set mem (StoreI mem src));
5812 
5813   ins_cost(150);
5814   format %{ "movl    $mem, $src\t# int" %}
5815   opcode(0xC7); /* C7 /0 */
5816   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5817   ins_pipe(ialu_mem_imm);
5818 %}
5819 
5820 // Store Long Immediate
5821 instruct storeImmL0(memory mem, immL0 zero)
5822 %{
5823   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5824   match(Set mem (StoreL mem zero));
5825 
5826   ins_cost(125); // XXX
5827   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5828   ins_encode %{
5829     __ movq($mem$$Address, r12);
5830   %}
5831   ins_pipe(ialu_mem_reg);
5832 %}
5833 
5834 instruct storeImmL(memory mem, immL32 src)
5835 %{
5836   match(Set mem (StoreL mem src));
5837 
5838   ins_cost(150);
5839   format %{ "movq    $mem, $src\t# long" %}
5840   opcode(0xC7); /* C7 /0 */
5841   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5842   ins_pipe(ialu_mem_imm);
5843 %}
5844 
5845 // Store Short/Char Immediate
5846 instruct storeImmC0(memory mem, immI0 zero)
5847 %{
5848   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5849   match(Set mem (StoreC mem zero));
5850 
5851   ins_cost(125); // XXX
5852   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5853   ins_encode %{
5854     __ movw($mem$$Address, r12);
5855   %}
5856   ins_pipe(ialu_mem_reg);
5857 %}
5858 
5859 instruct storeImmI16(memory mem, immI16 src)
5860 %{
5861   predicate(UseStoreImmI16);
5862   match(Set mem (StoreC mem src));
5863 
5864   ins_cost(150);
5865   format %{ "movw    $mem, $src\t# short/char" %}
5866   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5867   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5868   ins_pipe(ialu_mem_imm);
5869 %}
5870 
5871 // Store Byte Immediate
5872 instruct storeImmB0(memory mem, immI0 zero)
5873 %{
5874   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5875   match(Set mem (StoreB mem zero));
5876 
5877   ins_cost(125); // XXX
5878   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5879   ins_encode %{
5880     __ movb($mem$$Address, r12);
5881   %}
5882   ins_pipe(ialu_mem_reg);
5883 %}
5884 
5885 instruct storeImmB(memory mem, immI8 src)
5886 %{
5887   match(Set mem (StoreB mem src));
5888 
5889   ins_cost(150); // XXX
5890   format %{ "movb    $mem, $src\t# byte" %}
5891   opcode(0xC6); /* C6 /0 */
5892   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5893   ins_pipe(ialu_mem_imm);
5894 %}
5895 
5896 // Store CMS card-mark Immediate
5897 instruct storeImmCM0_reg(memory mem, immI0 zero)
5898 %{
5899   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5900   match(Set mem (StoreCM mem zero));
5901 
5902   ins_cost(125); // XXX
5903   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5904   ins_encode %{
5905     __ movb($mem$$Address, r12);
5906   %}
5907   ins_pipe(ialu_mem_reg);
5908 %}
5909 
5910 instruct storeImmCM0(memory mem, immI0 src)
5911 %{
5912   match(Set mem (StoreCM mem src));
5913 
5914   ins_cost(150); // XXX
5915   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5916   opcode(0xC6); /* C6 /0 */
5917   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5918   ins_pipe(ialu_mem_imm);
5919 %}
5920 
5921 // Store Float
5922 instruct storeF(memory mem, regF src)
5923 %{
5924   match(Set mem (StoreF mem src));
5925 
5926   ins_cost(95); // XXX
5927   format %{ "movss   $mem, $src\t# float" %}
5928   ins_encode %{
5929     __ movflt($mem$$Address, $src$$XMMRegister);
5930   %}
5931   ins_pipe(pipe_slow); // XXX
5932 %}
5933 
5934 // Store immediate Float value (it is faster than store from XMM register)
5935 instruct storeF0(memory mem, immF0 zero)
5936 %{
5937   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5938   match(Set mem (StoreF mem zero));
5939 
5940   ins_cost(25); // XXX
5941   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5942   ins_encode %{
5943     __ movl($mem$$Address, r12);
5944   %}
5945   ins_pipe(ialu_mem_reg);
5946 %}
5947 
5948 instruct storeF_imm(memory mem, immF src)
5949 %{
5950   match(Set mem (StoreF mem src));
5951 
5952   ins_cost(50);
5953   format %{ "movl    $mem, $src\t# float" %}
5954   opcode(0xC7); /* C7 /0 */
5955   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5956   ins_pipe(ialu_mem_imm);
5957 %}
5958 
5959 // Store Double
5960 instruct storeD(memory mem, regD src)
5961 %{
5962   match(Set mem (StoreD mem src));
5963 
5964   ins_cost(95); // XXX
5965   format %{ "movsd   $mem, $src\t# double" %}
5966   ins_encode %{
5967     __ movdbl($mem$$Address, $src$$XMMRegister);
5968   %}
5969   ins_pipe(pipe_slow); // XXX
5970 %}
5971 
5972 // Store immediate double 0.0 (it is faster than store from XMM register)
5973 instruct storeD0_imm(memory mem, immD0 src)
5974 %{
5975   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5976   match(Set mem (StoreD mem src));
5977 
5978   ins_cost(50);
5979   format %{ "movq    $mem, $src\t# double 0." %}
5980   opcode(0xC7); /* C7 /0 */
5981   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5982   ins_pipe(ialu_mem_imm);
5983 %}
5984 
5985 instruct storeD0(memory mem, immD0 zero)
5986 %{
5987   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5988   match(Set mem (StoreD mem zero));
5989 
5990   ins_cost(25); // XXX
5991   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5992   ins_encode %{
5993     __ movq($mem$$Address, r12);
5994   %}
5995   ins_pipe(ialu_mem_reg);
5996 %}
5997 
5998 instruct storeSSI(stackSlotI dst, rRegI src)
5999 %{
6000   match(Set dst src);
6001 
6002   ins_cost(100);
6003   format %{ "movl    $dst, $src\t# int stk" %}
6004   opcode(0x89);
6005   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6006   ins_pipe( ialu_mem_reg );
6007 %}
6008 
6009 instruct storeSSL(stackSlotL dst, rRegL src)
6010 %{
6011   match(Set dst src);
6012 
6013   ins_cost(100);
6014   format %{ "movq    $dst, $src\t# long stk" %}
6015   opcode(0x89);
6016   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6017   ins_pipe(ialu_mem_reg);
6018 %}
6019 
6020 instruct storeSSP(stackSlotP dst, rRegP src)
6021 %{
6022   match(Set dst src);
6023 
6024   ins_cost(100);
6025   format %{ "movq    $dst, $src\t# ptr stk" %}
6026   opcode(0x89);
6027   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6028   ins_pipe(ialu_mem_reg);
6029 %}
6030 
6031 instruct storeSSF(stackSlotF dst, regF src)
6032 %{
6033   match(Set dst src);
6034 
6035   ins_cost(95); // XXX
6036   format %{ "movss   $dst, $src\t# float stk" %}
6037   ins_encode %{
6038     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6039   %}
6040   ins_pipe(pipe_slow); // XXX
6041 %}
6042 
6043 instruct storeSSD(stackSlotD dst, regD src)
6044 %{
6045   match(Set dst src);
6046 
6047   ins_cost(95); // XXX
6048   format %{ "movsd   $dst, $src\t# double stk" %}
6049   ins_encode %{
6050     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6051   %}
6052   ins_pipe(pipe_slow); // XXX
6053 %}
6054 
6055 //----------BSWAP Instructions-------------------------------------------------
6056 instruct bytes_reverse_int(rRegI dst) %{
6057   match(Set dst (ReverseBytesI dst));
6058 
6059   format %{ "bswapl  $dst" %}
6060   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6061   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6062   ins_pipe( ialu_reg );
6063 %}
6064 
6065 instruct bytes_reverse_long(rRegL dst) %{
6066   match(Set dst (ReverseBytesL dst));
6067 
6068   format %{ "bswapq  $dst" %}
6069   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6070   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6071   ins_pipe( ialu_reg);
6072 %}
6073 
6074 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6075   match(Set dst (ReverseBytesUS dst));
6076   effect(KILL cr);
6077 
6078   format %{ "bswapl  $dst\n\t"
6079             "shrl    $dst,16\n\t" %}
6080   ins_encode %{
6081     __ bswapl($dst$$Register);
6082     __ shrl($dst$$Register, 16);
6083   %}
6084   ins_pipe( ialu_reg );
6085 %}
6086 
6087 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6088   match(Set dst (ReverseBytesS dst));
6089   effect(KILL cr);
6090 
6091   format %{ "bswapl  $dst\n\t"
6092             "sar     $dst,16\n\t" %}
6093   ins_encode %{
6094     __ bswapl($dst$$Register);
6095     __ sarl($dst$$Register, 16);
6096   %}
6097   ins_pipe( ialu_reg );
6098 %}
6099 
6100 //---------- Zeros Count Instructions ------------------------------------------
6101 
6102 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6103   predicate(UseCountLeadingZerosInstruction);
6104   match(Set dst (CountLeadingZerosI src));
6105   effect(KILL cr);
6106 
6107   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6108   ins_encode %{
6109     __ lzcntl($dst$$Register, $src$$Register);
6110   %}
6111   ins_pipe(ialu_reg);
6112 %}
6113 
6114 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6115   predicate(!UseCountLeadingZerosInstruction);
6116   match(Set dst (CountLeadingZerosI src));
6117   effect(KILL cr);
6118 
6119   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6120             "jnz     skip\n\t"
6121             "movl    $dst, -1\n"
6122       "skip:\n\t"
6123             "negl    $dst\n\t"
6124             "addl    $dst, 31" %}
6125   ins_encode %{
6126     Register Rdst = $dst$$Register;
6127     Register Rsrc = $src$$Register;
6128     Label skip;
6129     __ bsrl(Rdst, Rsrc);
6130     __ jccb(Assembler::notZero, skip);
6131     __ movl(Rdst, -1);
6132     __ bind(skip);
6133     __ negl(Rdst);
6134     __ addl(Rdst, BitsPerInt - 1);
6135   %}
6136   ins_pipe(ialu_reg);
6137 %}
6138 
6139 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6140   predicate(UseCountLeadingZerosInstruction);
6141   match(Set dst (CountLeadingZerosL src));
6142   effect(KILL cr);
6143 
6144   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6145   ins_encode %{
6146     __ lzcntq($dst$$Register, $src$$Register);
6147   %}
6148   ins_pipe(ialu_reg);
6149 %}
6150 
6151 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6152   predicate(!UseCountLeadingZerosInstruction);
6153   match(Set dst (CountLeadingZerosL src));
6154   effect(KILL cr);
6155 
6156   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6157             "jnz     skip\n\t"
6158             "movl    $dst, -1\n"
6159       "skip:\n\t"
6160             "negl    $dst\n\t"
6161             "addl    $dst, 63" %}
6162   ins_encode %{
6163     Register Rdst = $dst$$Register;
6164     Register Rsrc = $src$$Register;
6165     Label skip;
6166     __ bsrq(Rdst, Rsrc);
6167     __ jccb(Assembler::notZero, skip);
6168     __ movl(Rdst, -1);
6169     __ bind(skip);
6170     __ negl(Rdst);
6171     __ addl(Rdst, BitsPerLong - 1);
6172   %}
6173   ins_pipe(ialu_reg);
6174 %}
6175 
6176 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6177   predicate(UseCountTrailingZerosInstruction);
6178   match(Set dst (CountTrailingZerosI src));
6179   effect(KILL cr);
6180 
6181   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6182   ins_encode %{
6183     __ tzcntl($dst$$Register, $src$$Register);
6184   %}
6185   ins_pipe(ialu_reg);
6186 %}
6187 
6188 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6189   predicate(!UseCountTrailingZerosInstruction);
6190   match(Set dst (CountTrailingZerosI src));
6191   effect(KILL cr);
6192 
6193   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6194             "jnz     done\n\t"
6195             "movl    $dst, 32\n"
6196       "done:" %}
6197   ins_encode %{
6198     Register Rdst = $dst$$Register;
6199     Label done;
6200     __ bsfl(Rdst, $src$$Register);
6201     __ jccb(Assembler::notZero, done);
6202     __ movl(Rdst, BitsPerInt);
6203     __ bind(done);
6204   %}
6205   ins_pipe(ialu_reg);
6206 %}
6207 
6208 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6209   predicate(UseCountTrailingZerosInstruction);
6210   match(Set dst (CountTrailingZerosL src));
6211   effect(KILL cr);
6212 
6213   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6214   ins_encode %{
6215     __ tzcntq($dst$$Register, $src$$Register);
6216   %}
6217   ins_pipe(ialu_reg);
6218 %}
6219 
6220 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6221   predicate(!UseCountTrailingZerosInstruction);
6222   match(Set dst (CountTrailingZerosL src));
6223   effect(KILL cr);
6224 
6225   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6226             "jnz     done\n\t"
6227             "movl    $dst, 64\n"
6228       "done:" %}
6229   ins_encode %{
6230     Register Rdst = $dst$$Register;
6231     Label done;
6232     __ bsfq(Rdst, $src$$Register);
6233     __ jccb(Assembler::notZero, done);
6234     __ movl(Rdst, BitsPerLong);
6235     __ bind(done);
6236   %}
6237   ins_pipe(ialu_reg);
6238 %}
6239 
6240 
6241 //---------- Population Count Instructions -------------------------------------
6242 
6243 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6244   predicate(UsePopCountInstruction);
6245   match(Set dst (PopCountI src));
6246   effect(KILL cr);
6247 
6248   format %{ "popcnt  $dst, $src" %}
6249   ins_encode %{
6250     __ popcntl($dst$$Register, $src$$Register);
6251   %}
6252   ins_pipe(ialu_reg);
6253 %}
6254 
6255 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6256   predicate(UsePopCountInstruction);
6257   match(Set dst (PopCountI (LoadI mem)));
6258   effect(KILL cr);
6259 
6260   format %{ "popcnt  $dst, $mem" %}
6261   ins_encode %{
6262     __ popcntl($dst$$Register, $mem$$Address);
6263   %}
6264   ins_pipe(ialu_reg);
6265 %}
6266 
6267 // Note: Long.bitCount(long) returns an int.
6268 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6269   predicate(UsePopCountInstruction);
6270   match(Set dst (PopCountL src));
6271   effect(KILL cr);
6272 
6273   format %{ "popcnt  $dst, $src" %}
6274   ins_encode %{
6275     __ popcntq($dst$$Register, $src$$Register);
6276   %}
6277   ins_pipe(ialu_reg);
6278 %}
6279 
6280 // Note: Long.bitCount(long) returns an int.
6281 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6282   predicate(UsePopCountInstruction);
6283   match(Set dst (PopCountL (LoadL mem)));
6284   effect(KILL cr);
6285 
6286   format %{ "popcnt  $dst, $mem" %}
6287   ins_encode %{
6288     __ popcntq($dst$$Register, $mem$$Address);
6289   %}
6290   ins_pipe(ialu_reg);
6291 %}
6292 
6293 
6294 //----------MemBar Instructions-----------------------------------------------
6295 // Memory barrier flavors
6296 
6297 instruct membar_acquire()
6298 %{
6299   match(MemBarAcquire);
6300   match(LoadFence);
6301   ins_cost(0);
6302 
6303   size(0);
6304   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6305   ins_encode();
6306   ins_pipe(empty);
6307 %}
6308 
6309 instruct membar_acquire_lock()
6310 %{
6311   match(MemBarAcquireLock);
6312   ins_cost(0);
6313 
6314   size(0);
6315   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6316   ins_encode();
6317   ins_pipe(empty);
6318 %}
6319 
6320 instruct membar_release()
6321 %{
6322   match(MemBarRelease);
6323   match(StoreFence);
6324   ins_cost(0);
6325 
6326   size(0);
6327   format %{ "MEMBAR-release ! (empty encoding)" %}
6328   ins_encode();
6329   ins_pipe(empty);
6330 %}
6331 
6332 instruct membar_release_lock()
6333 %{
6334   match(MemBarReleaseLock);
6335   ins_cost(0);
6336 
6337   size(0);
6338   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6339   ins_encode();
6340   ins_pipe(empty);
6341 %}
6342 
6343 instruct membar_volatile(rFlagsReg cr) %{
6344   match(MemBarVolatile);
6345   effect(KILL cr);
6346   ins_cost(400);
6347 
6348   format %{
6349     $$template
6350     if (os::is_MP()) {
6351       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6352     } else {
6353       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6354     }
6355   %}
6356   ins_encode %{
6357     __ membar(Assembler::StoreLoad);
6358   %}
6359   ins_pipe(pipe_slow);
6360 %}
6361 
6362 instruct unnecessary_membar_volatile()
6363 %{
6364   match(MemBarVolatile);
6365   predicate(Matcher::post_store_load_barrier(n));
6366   ins_cost(0);
6367 
6368   size(0);
6369   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6370   ins_encode();
6371   ins_pipe(empty);
6372 %}
6373 
6374 instruct membar_storestore() %{
6375   match(MemBarStoreStore);
6376   ins_cost(0);
6377 
6378   size(0);
6379   format %{ "MEMBAR-storestore (empty encoding)" %}
6380   ins_encode( );
6381   ins_pipe(empty);
6382 %}
6383 
6384 //----------Move Instructions--------------------------------------------------
6385 
6386 instruct castX2P(rRegP dst, rRegL src)
6387 %{
6388   match(Set dst (CastX2P src));
6389 
6390   format %{ "movq    $dst, $src\t# long->ptr" %}
6391   ins_encode %{
6392     if ($dst$$reg != $src$$reg) {
6393       __ movptr($dst$$Register, $src$$Register);
6394     }
6395   %}
6396   ins_pipe(ialu_reg_reg); // XXX
6397 %}
6398 
6399 instruct castP2X(rRegL dst, rRegP src)
6400 %{
6401   match(Set dst (CastP2X src));
6402 
6403   format %{ "movq    $dst, $src\t# ptr -> long" %}
6404   ins_encode %{
6405     if ($dst$$reg != $src$$reg) {
6406       __ movptr($dst$$Register, $src$$Register);
6407     }
6408   %}
6409   ins_pipe(ialu_reg_reg); // XXX
6410 %}
6411 
6412 // Convert oop into int for vectors alignment masking
6413 instruct convP2I(rRegI dst, rRegP src)
6414 %{
6415   match(Set dst (ConvL2I (CastP2X src)));
6416 
6417   format %{ "movl    $dst, $src\t# ptr -> int" %}
6418   ins_encode %{
6419     __ movl($dst$$Register, $src$$Register);
6420   %}
6421   ins_pipe(ialu_reg_reg); // XXX
6422 %}
6423 
6424 // Convert compressed oop into int for vectors alignment masking
6425 // in case of 32bit oops (heap < 4Gb).
6426 instruct convN2I(rRegI dst, rRegN src)
6427 %{
6428   predicate(Universe::narrow_oop_shift() == 0);
6429   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6430 
6431   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6432   ins_encode %{
6433     __ movl($dst$$Register, $src$$Register);
6434   %}
6435   ins_pipe(ialu_reg_reg); // XXX
6436 %}
6437 
6438 // Convert oop pointer into compressed form
6439 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6440   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6441   match(Set dst (EncodeP src));
6442   effect(KILL cr);
6443   format %{ "encode_heap_oop $dst,$src" %}
6444   ins_encode %{
6445     Register s = $src$$Register;
6446     Register d = $dst$$Register;
6447     if (s != d) {
6448       __ movq(d, s);
6449     }
6450     __ encode_heap_oop(d);
6451   %}
6452   ins_pipe(ialu_reg_long);
6453 %}
6454 
6455 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6456   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6457   match(Set dst (EncodeP src));
6458   effect(KILL cr);
6459   format %{ "encode_heap_oop_not_null $dst,$src" %}
6460   ins_encode %{
6461     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6462   %}
6463   ins_pipe(ialu_reg_long);
6464 %}
6465 
6466 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6467   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6468             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6469   match(Set dst (DecodeN src));
6470   effect(KILL cr);
6471   format %{ "decode_heap_oop $dst,$src" %}
6472   ins_encode %{
6473     Register s = $src$$Register;
6474     Register d = $dst$$Register;
6475     if (s != d) {
6476       __ movq(d, s);
6477     }
6478     __ decode_heap_oop(d);
6479   %}
6480   ins_pipe(ialu_reg_long);
6481 %}
6482 
6483 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6484   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6485             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6486   match(Set dst (DecodeN src));
6487   effect(KILL cr);
6488   format %{ "decode_heap_oop_not_null $dst,$src" %}
6489   ins_encode %{
6490     Register s = $src$$Register;
6491     Register d = $dst$$Register;
6492     if (s != d) {
6493       __ decode_heap_oop_not_null(d, s);
6494     } else {
6495       __ decode_heap_oop_not_null(d);
6496     }
6497   %}
6498   ins_pipe(ialu_reg_long);
6499 %}
6500 
6501 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6502   match(Set dst (EncodePKlass src));
6503   effect(KILL cr);
6504   format %{ "encode_klass_not_null $dst,$src" %}
6505   ins_encode %{
6506     __ encode_klass_not_null($dst$$Register, $src$$Register);
6507   %}
6508   ins_pipe(ialu_reg_long);
6509 %}
6510 
6511 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6512   match(Set dst (DecodeNKlass src));
6513   effect(KILL cr);
6514   format %{ "decode_klass_not_null $dst,$src" %}
6515   ins_encode %{
6516     Register s = $src$$Register;
6517     Register d = $dst$$Register;
6518     if (s != d) {
6519       __ decode_klass_not_null(d, s);
6520     } else {
6521       __ decode_klass_not_null(d);
6522     }
6523   %}
6524   ins_pipe(ialu_reg_long);
6525 %}
6526 
6527 
6528 //----------Conditional Move---------------------------------------------------
6529 // Jump
6530 // dummy instruction for generating temp registers
6531 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6532   match(Jump (LShiftL switch_val shift));
6533   ins_cost(350);
6534   predicate(false);
6535   effect(TEMP dest);
6536 
6537   format %{ "leaq    $dest, [$constantaddress]\n\t"
6538             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6539   ins_encode %{
6540     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6541     // to do that and the compiler is using that register as one it can allocate.
6542     // So we build it all by hand.
6543     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6544     // ArrayAddress dispatch(table, index);
6545     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6546     __ lea($dest$$Register, $constantaddress);
6547     __ jmp(dispatch);
6548   %}
6549   ins_pipe(pipe_jmp);
6550 %}
6551 
6552 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6553   match(Jump (AddL (LShiftL switch_val shift) offset));
6554   ins_cost(350);
6555   effect(TEMP dest);
6556 
6557   format %{ "leaq    $dest, [$constantaddress]\n\t"
6558             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6559   ins_encode %{
6560     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6561     // to do that and the compiler is using that register as one it can allocate.
6562     // So we build it all by hand.
6563     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6564     // ArrayAddress dispatch(table, index);
6565     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6566     __ lea($dest$$Register, $constantaddress);
6567     __ jmp(dispatch);
6568   %}
6569   ins_pipe(pipe_jmp);
6570 %}
6571 
6572 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6573   match(Jump switch_val);
6574   ins_cost(350);
6575   effect(TEMP dest);
6576 
6577   format %{ "leaq    $dest, [$constantaddress]\n\t"
6578             "jmp     [$dest + $switch_val]\n\t" %}
6579   ins_encode %{
6580     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6581     // to do that and the compiler is using that register as one it can allocate.
6582     // So we build it all by hand.
6583     // Address index(noreg, switch_reg, Address::times_1);
6584     // ArrayAddress dispatch(table, index);
6585     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6586     __ lea($dest$$Register, $constantaddress);
6587     __ jmp(dispatch);
6588   %}
6589   ins_pipe(pipe_jmp);
6590 %}
6591 
6592 // Conditional move
6593 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6594 %{
6595   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6596 
6597   ins_cost(200); // XXX
6598   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6599   opcode(0x0F, 0x40);
6600   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6601   ins_pipe(pipe_cmov_reg);
6602 %}
6603 
6604 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6605   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6606 
6607   ins_cost(200); // XXX
6608   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6609   opcode(0x0F, 0x40);
6610   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6611   ins_pipe(pipe_cmov_reg);
6612 %}
6613 
6614 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6615   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6616   ins_cost(200);
6617   expand %{
6618     cmovI_regU(cop, cr, dst, src);
6619   %}
6620 %}
6621 
6622 // Conditional move
6623 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6624   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6625 
6626   ins_cost(250); // XXX
6627   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6628   opcode(0x0F, 0x40);
6629   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6630   ins_pipe(pipe_cmov_mem);
6631 %}
6632 
6633 // Conditional move
6634 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6635 %{
6636   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6637 
6638   ins_cost(250); // XXX
6639   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6640   opcode(0x0F, 0x40);
6641   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6642   ins_pipe(pipe_cmov_mem);
6643 %}
6644 
6645 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6646   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6647   ins_cost(250);
6648   expand %{
6649     cmovI_memU(cop, cr, dst, src);
6650   %}
6651 %}
6652 
6653 // Conditional move
6654 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6655 %{
6656   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6657 
6658   ins_cost(200); // XXX
6659   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6660   opcode(0x0F, 0x40);
6661   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6662   ins_pipe(pipe_cmov_reg);
6663 %}
6664 
6665 // Conditional move
6666 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6667 %{
6668   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6669 
6670   ins_cost(200); // XXX
6671   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6672   opcode(0x0F, 0x40);
6673   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6674   ins_pipe(pipe_cmov_reg);
6675 %}
6676 
6677 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6678   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6679   ins_cost(200);
6680   expand %{
6681     cmovN_regU(cop, cr, dst, src);
6682   %}
6683 %}
6684 
6685 // Conditional move
6686 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6687 %{
6688   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6689 
6690   ins_cost(200); // XXX
6691   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6692   opcode(0x0F, 0x40);
6693   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6694   ins_pipe(pipe_cmov_reg);  // XXX
6695 %}
6696 
6697 // Conditional move
6698 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6699 %{
6700   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6701 
6702   ins_cost(200); // XXX
6703   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6704   opcode(0x0F, 0x40);
6705   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6706   ins_pipe(pipe_cmov_reg); // XXX
6707 %}
6708 
6709 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6710   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6711   ins_cost(200);
6712   expand %{
6713     cmovP_regU(cop, cr, dst, src);
6714   %}
6715 %}
6716 
6717 // DISABLED: Requires the ADLC to emit a bottom_type call that
6718 // correctly meets the two pointer arguments; one is an incoming
6719 // register but the other is a memory operand.  ALSO appears to
6720 // be buggy with implicit null checks.
6721 //
6722 //// Conditional move
6723 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6724 //%{
6725 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6726 //  ins_cost(250);
6727 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6728 //  opcode(0x0F,0x40);
6729 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6730 //  ins_pipe( pipe_cmov_mem );
6731 //%}
6732 //
6733 //// Conditional move
6734 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6735 //%{
6736 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6737 //  ins_cost(250);
6738 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6739 //  opcode(0x0F,0x40);
6740 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6741 //  ins_pipe( pipe_cmov_mem );
6742 //%}
6743 
6744 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6745 %{
6746   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6747 
6748   ins_cost(200); // XXX
6749   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6750   opcode(0x0F, 0x40);
6751   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6752   ins_pipe(pipe_cmov_reg);  // XXX
6753 %}
6754 
6755 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6756 %{
6757   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6758 
6759   ins_cost(200); // XXX
6760   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6761   opcode(0x0F, 0x40);
6762   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6763   ins_pipe(pipe_cmov_mem);  // XXX
6764 %}
6765 
6766 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6767 %{
6768   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6769 
6770   ins_cost(200); // XXX
6771   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6772   opcode(0x0F, 0x40);
6773   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6774   ins_pipe(pipe_cmov_reg); // XXX
6775 %}
6776 
6777 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6778   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6779   ins_cost(200);
6780   expand %{
6781     cmovL_regU(cop, cr, dst, src);
6782   %}
6783 %}
6784 
6785 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6786 %{
6787   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6788 
6789   ins_cost(200); // XXX
6790   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6791   opcode(0x0F, 0x40);
6792   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6793   ins_pipe(pipe_cmov_mem); // XXX
6794 %}
6795 
6796 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6797   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6798   ins_cost(200);
6799   expand %{
6800     cmovL_memU(cop, cr, dst, src);
6801   %}
6802 %}
6803 
6804 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6805 %{
6806   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6807 
6808   ins_cost(200); // XXX
6809   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6810             "movss     $dst, $src\n"
6811     "skip:" %}
6812   ins_encode %{
6813     Label Lskip;
6814     // Invert sense of branch from sense of CMOV
6815     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6816     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6817     __ bind(Lskip);
6818   %}
6819   ins_pipe(pipe_slow);
6820 %}
6821 
6822 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6823 // %{
6824 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6825 
6826 //   ins_cost(200); // XXX
6827 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6828 //             "movss     $dst, $src\n"
6829 //     "skip:" %}
6830 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6831 //   ins_pipe(pipe_slow);
6832 // %}
6833 
6834 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6835 %{
6836   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6837 
6838   ins_cost(200); // XXX
6839   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6840             "movss     $dst, $src\n"
6841     "skip:" %}
6842   ins_encode %{
6843     Label Lskip;
6844     // Invert sense of branch from sense of CMOV
6845     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6846     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6847     __ bind(Lskip);
6848   %}
6849   ins_pipe(pipe_slow);
6850 %}
6851 
6852 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6853   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6854   ins_cost(200);
6855   expand %{
6856     cmovF_regU(cop, cr, dst, src);
6857   %}
6858 %}
6859 
6860 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6861 %{
6862   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6863 
6864   ins_cost(200); // XXX
6865   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6866             "movsd     $dst, $src\n"
6867     "skip:" %}
6868   ins_encode %{
6869     Label Lskip;
6870     // Invert sense of branch from sense of CMOV
6871     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6872     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6873     __ bind(Lskip);
6874   %}
6875   ins_pipe(pipe_slow);
6876 %}
6877 
6878 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6879 %{
6880   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6881 
6882   ins_cost(200); // XXX
6883   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6884             "movsd     $dst, $src\n"
6885     "skip:" %}
6886   ins_encode %{
6887     Label Lskip;
6888     // Invert sense of branch from sense of CMOV
6889     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6890     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6891     __ bind(Lskip);
6892   %}
6893   ins_pipe(pipe_slow);
6894 %}
6895 
6896 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6897   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6898   ins_cost(200);
6899   expand %{
6900     cmovD_regU(cop, cr, dst, src);
6901   %}
6902 %}
6903 
6904 //----------Arithmetic Instructions--------------------------------------------
6905 //----------Addition Instructions----------------------------------------------
6906 
6907 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6908 %{
6909   match(Set dst (AddI dst src));
6910   effect(KILL cr);
6911 
6912   format %{ "addl    $dst, $src\t# int" %}
6913   opcode(0x03);
6914   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6915   ins_pipe(ialu_reg_reg);
6916 %}
6917 
6918 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6919 %{
6920   match(Set dst (AddI dst src));
6921   effect(KILL cr);
6922 
6923   format %{ "addl    $dst, $src\t# int" %}
6924   opcode(0x81, 0x00); /* /0 id */
6925   ins_encode(OpcSErm(dst, src), Con8or32(src));
6926   ins_pipe( ialu_reg );
6927 %}
6928 
6929 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6930 %{
6931   match(Set dst (AddI dst (LoadI src)));
6932   effect(KILL cr);
6933 
6934   ins_cost(125); // XXX
6935   format %{ "addl    $dst, $src\t# int" %}
6936   opcode(0x03);
6937   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6938   ins_pipe(ialu_reg_mem);
6939 %}
6940 
6941 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6942 %{
6943   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6944   effect(KILL cr);
6945 
6946   ins_cost(150); // XXX
6947   format %{ "addl    $dst, $src\t# int" %}
6948   opcode(0x01); /* Opcode 01 /r */
6949   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6950   ins_pipe(ialu_mem_reg);
6951 %}
6952 
6953 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6954 %{
6955   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6956   effect(KILL cr);
6957 
6958   ins_cost(125); // XXX
6959   format %{ "addl    $dst, $src\t# int" %}
6960   opcode(0x81); /* Opcode 81 /0 id */
6961   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6962   ins_pipe(ialu_mem_imm);
6963 %}
6964 
6965 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6966 %{
6967   predicate(UseIncDec);
6968   match(Set dst (AddI dst src));
6969   effect(KILL cr);
6970 
6971   format %{ "incl    $dst\t# int" %}
6972   opcode(0xFF, 0x00); // FF /0
6973   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6974   ins_pipe(ialu_reg);
6975 %}
6976 
6977 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6978 %{
6979   predicate(UseIncDec);
6980   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6981   effect(KILL cr);
6982 
6983   ins_cost(125); // XXX
6984   format %{ "incl    $dst\t# int" %}
6985   opcode(0xFF); /* Opcode FF /0 */
6986   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6987   ins_pipe(ialu_mem_imm);
6988 %}
6989 
6990 // XXX why does that use AddI
6991 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6992 %{
6993   predicate(UseIncDec);
6994   match(Set dst (AddI dst src));
6995   effect(KILL cr);
6996 
6997   format %{ "decl    $dst\t# int" %}
6998   opcode(0xFF, 0x01); // FF /1
6999   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7000   ins_pipe(ialu_reg);
7001 %}
7002 
7003 // XXX why does that use AddI
7004 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7005 %{
7006   predicate(UseIncDec);
7007   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7008   effect(KILL cr);
7009 
7010   ins_cost(125); // XXX
7011   format %{ "decl    $dst\t# int" %}
7012   opcode(0xFF); /* Opcode FF /1 */
7013   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7014   ins_pipe(ialu_mem_imm);
7015 %}
7016 
7017 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7018 %{
7019   match(Set dst (AddI src0 src1));
7020 
7021   ins_cost(110);
7022   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7023   opcode(0x8D); /* 0x8D /r */
7024   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7025   ins_pipe(ialu_reg_reg);
7026 %}
7027 
7028 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7029 %{
7030   match(Set dst (AddL dst src));
7031   effect(KILL cr);
7032 
7033   format %{ "addq    $dst, $src\t# long" %}
7034   opcode(0x03);
7035   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7036   ins_pipe(ialu_reg_reg);
7037 %}
7038 
7039 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7040 %{
7041   match(Set dst (AddL dst src));
7042   effect(KILL cr);
7043 
7044   format %{ "addq    $dst, $src\t# long" %}
7045   opcode(0x81, 0x00); /* /0 id */
7046   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7047   ins_pipe( ialu_reg );
7048 %}
7049 
7050 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7051 %{
7052   match(Set dst (AddL dst (LoadL src)));
7053   effect(KILL cr);
7054 
7055   ins_cost(125); // XXX
7056   format %{ "addq    $dst, $src\t# long" %}
7057   opcode(0x03);
7058   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7059   ins_pipe(ialu_reg_mem);
7060 %}
7061 
7062 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7063 %{
7064   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7065   effect(KILL cr);
7066 
7067   ins_cost(150); // XXX
7068   format %{ "addq    $dst, $src\t# long" %}
7069   opcode(0x01); /* Opcode 01 /r */
7070   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7071   ins_pipe(ialu_mem_reg);
7072 %}
7073 
7074 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7075 %{
7076   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7077   effect(KILL cr);
7078 
7079   ins_cost(125); // XXX
7080   format %{ "addq    $dst, $src\t# long" %}
7081   opcode(0x81); /* Opcode 81 /0 id */
7082   ins_encode(REX_mem_wide(dst),
7083              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7084   ins_pipe(ialu_mem_imm);
7085 %}
7086 
7087 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7088 %{
7089   predicate(UseIncDec);
7090   match(Set dst (AddL dst src));
7091   effect(KILL cr);
7092 
7093   format %{ "incq    $dst\t# long" %}
7094   opcode(0xFF, 0x00); // FF /0
7095   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7096   ins_pipe(ialu_reg);
7097 %}
7098 
7099 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7100 %{
7101   predicate(UseIncDec);
7102   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7103   effect(KILL cr);
7104 
7105   ins_cost(125); // XXX
7106   format %{ "incq    $dst\t# long" %}
7107   opcode(0xFF); /* Opcode FF /0 */
7108   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7109   ins_pipe(ialu_mem_imm);
7110 %}
7111 
7112 // XXX why does that use AddL
7113 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7114 %{
7115   predicate(UseIncDec);
7116   match(Set dst (AddL dst src));
7117   effect(KILL cr);
7118 
7119   format %{ "decq    $dst\t# long" %}
7120   opcode(0xFF, 0x01); // FF /1
7121   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7122   ins_pipe(ialu_reg);
7123 %}
7124 
7125 // XXX why does that use AddL
7126 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7127 %{
7128   predicate(UseIncDec);
7129   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7130   effect(KILL cr);
7131 
7132   ins_cost(125); // XXX
7133   format %{ "decq    $dst\t# long" %}
7134   opcode(0xFF); /* Opcode FF /1 */
7135   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7136   ins_pipe(ialu_mem_imm);
7137 %}
7138 
7139 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7140 %{
7141   match(Set dst (AddL src0 src1));
7142 
7143   ins_cost(110);
7144   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7145   opcode(0x8D); /* 0x8D /r */
7146   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7147   ins_pipe(ialu_reg_reg);
7148 %}
7149 
7150 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7151 %{
7152   match(Set dst (AddP dst src));
7153   effect(KILL cr);
7154 
7155   format %{ "addq    $dst, $src\t# ptr" %}
7156   opcode(0x03);
7157   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7158   ins_pipe(ialu_reg_reg);
7159 %}
7160 
7161 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7162 %{
7163   match(Set dst (AddP dst src));
7164   effect(KILL cr);
7165 
7166   format %{ "addq    $dst, $src\t# ptr" %}
7167   opcode(0x81, 0x00); /* /0 id */
7168   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7169   ins_pipe( ialu_reg );
7170 %}
7171 
7172 // XXX addP mem ops ????
7173 
7174 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7175 %{
7176   match(Set dst (AddP src0 src1));
7177 
7178   ins_cost(110);
7179   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7180   opcode(0x8D); /* 0x8D /r */
7181   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7182   ins_pipe(ialu_reg_reg);
7183 %}
7184 
7185 instruct checkCastPP(rRegP dst)
7186 %{
7187   match(Set dst (CheckCastPP dst));
7188 
7189   size(0);
7190   format %{ "# checkcastPP of $dst" %}
7191   ins_encode(/* empty encoding */);
7192   ins_pipe(empty);
7193 %}
7194 
7195 instruct castPP(rRegP dst)
7196 %{
7197   match(Set dst (CastPP dst));
7198 
7199   size(0);
7200   format %{ "# castPP of $dst" %}
7201   ins_encode(/* empty encoding */);
7202   ins_pipe(empty);
7203 %}
7204 
7205 instruct castII(rRegI dst)
7206 %{
7207   match(Set dst (CastII dst));
7208 
7209   size(0);
7210   format %{ "# castII of $dst" %}
7211   ins_encode(/* empty encoding */);
7212   ins_cost(0);
7213   ins_pipe(empty);
7214 %}
7215 
7216 // LoadP-locked same as a regular LoadP when used with compare-swap
7217 instruct loadPLocked(rRegP dst, memory mem)
7218 %{
7219   match(Set dst (LoadPLocked mem));
7220 
7221   ins_cost(125); // XXX
7222   format %{ "movq    $dst, $mem\t# ptr locked" %}
7223   opcode(0x8B);
7224   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7225   ins_pipe(ialu_reg_mem); // XXX
7226 %}
7227 
7228 // Conditional-store of the updated heap-top.
7229 // Used during allocation of the shared heap.
7230 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7231 
7232 instruct storePConditional(memory heap_top_ptr,
7233                            rax_RegP oldval, rRegP newval,
7234                            rFlagsReg cr)
7235 %{
7236   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7237 
7238   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7239             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7240   opcode(0x0F, 0xB1);
7241   ins_encode(lock_prefix,
7242              REX_reg_mem_wide(newval, heap_top_ptr),
7243              OpcP, OpcS,
7244              reg_mem(newval, heap_top_ptr));
7245   ins_pipe(pipe_cmpxchg);
7246 %}
7247 
7248 // Conditional-store of an int value.
7249 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7250 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7251 %{
7252   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7253   effect(KILL oldval);
7254 
7255   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7256   opcode(0x0F, 0xB1);
7257   ins_encode(lock_prefix,
7258              REX_reg_mem(newval, mem),
7259              OpcP, OpcS,
7260              reg_mem(newval, mem));
7261   ins_pipe(pipe_cmpxchg);
7262 %}
7263 
7264 // Conditional-store of a long value.
7265 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7266 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7267 %{
7268   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7269   effect(KILL oldval);
7270 
7271   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7272   opcode(0x0F, 0xB1);
7273   ins_encode(lock_prefix,
7274              REX_reg_mem_wide(newval, mem),
7275              OpcP, OpcS,
7276              reg_mem(newval, mem));
7277   ins_pipe(pipe_cmpxchg);
7278 %}
7279 
7280 
7281 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7282 instruct compareAndSwapP(rRegI res,
7283                          memory mem_ptr,
7284                          rax_RegP oldval, rRegP newval,
7285                          rFlagsReg cr)
7286 %{
7287   predicate(VM_Version::supports_cx8());
7288   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7289   effect(KILL cr, KILL oldval);
7290 
7291   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7292             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7293             "sete    $res\n\t"
7294             "movzbl  $res, $res" %}
7295   opcode(0x0F, 0xB1);
7296   ins_encode(lock_prefix,
7297              REX_reg_mem_wide(newval, mem_ptr),
7298              OpcP, OpcS,
7299              reg_mem(newval, mem_ptr),
7300              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7301              REX_reg_breg(res, res), // movzbl
7302              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7303   ins_pipe( pipe_cmpxchg );
7304 %}
7305 
7306 instruct compareAndSwapL(rRegI res,
7307                          memory mem_ptr,
7308                          rax_RegL oldval, rRegL newval,
7309                          rFlagsReg cr)
7310 %{
7311   predicate(VM_Version::supports_cx8());
7312   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7313   effect(KILL cr, KILL oldval);
7314 
7315   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7316             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7317             "sete    $res\n\t"
7318             "movzbl  $res, $res" %}
7319   opcode(0x0F, 0xB1);
7320   ins_encode(lock_prefix,
7321              REX_reg_mem_wide(newval, mem_ptr),
7322              OpcP, OpcS,
7323              reg_mem(newval, mem_ptr),
7324              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7325              REX_reg_breg(res, res), // movzbl
7326              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7327   ins_pipe( pipe_cmpxchg );
7328 %}
7329 
7330 instruct compareAndSwapI(rRegI res,
7331                          memory mem_ptr,
7332                          rax_RegI oldval, rRegI newval,
7333                          rFlagsReg cr)
7334 %{
7335   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7336   effect(KILL cr, KILL oldval);
7337 
7338   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7339             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7340             "sete    $res\n\t"
7341             "movzbl  $res, $res" %}
7342   opcode(0x0F, 0xB1);
7343   ins_encode(lock_prefix,
7344              REX_reg_mem(newval, mem_ptr),
7345              OpcP, OpcS,
7346              reg_mem(newval, mem_ptr),
7347              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7348              REX_reg_breg(res, res), // movzbl
7349              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7350   ins_pipe( pipe_cmpxchg );
7351 %}
7352 
7353 
7354 instruct compareAndSwapN(rRegI res,
7355                           memory mem_ptr,
7356                           rax_RegN oldval, rRegN newval,
7357                           rFlagsReg cr) %{
7358   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7359   effect(KILL cr, KILL oldval);
7360 
7361   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7362             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7363             "sete    $res\n\t"
7364             "movzbl  $res, $res" %}
7365   opcode(0x0F, 0xB1);
7366   ins_encode(lock_prefix,
7367              REX_reg_mem(newval, mem_ptr),
7368              OpcP, OpcS,
7369              reg_mem(newval, mem_ptr),
7370              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7371              REX_reg_breg(res, res), // movzbl
7372              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7373   ins_pipe( pipe_cmpxchg );
7374 %}
7375 
7376 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7377   predicate(n->as_LoadStore()->result_not_used());
7378   match(Set dummy (GetAndAddI mem add));
7379   effect(KILL cr);
7380   format %{ "ADDL  [$mem],$add" %}
7381   ins_encode %{
7382     if (os::is_MP()) { __ lock(); }
7383     __ addl($mem$$Address, $add$$constant);
7384   %}
7385   ins_pipe( pipe_cmpxchg );
7386 %}
7387 
7388 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7389   match(Set newval (GetAndAddI mem newval));
7390   effect(KILL cr);
7391   format %{ "XADDL  [$mem],$newval" %}
7392   ins_encode %{
7393     if (os::is_MP()) { __ lock(); }
7394     __ xaddl($mem$$Address, $newval$$Register);
7395   %}
7396   ins_pipe( pipe_cmpxchg );
7397 %}
7398 
7399 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7400   predicate(n->as_LoadStore()->result_not_used());
7401   match(Set dummy (GetAndAddL mem add));
7402   effect(KILL cr);
7403   format %{ "ADDQ  [$mem],$add" %}
7404   ins_encode %{
7405     if (os::is_MP()) { __ lock(); }
7406     __ addq($mem$$Address, $add$$constant);
7407   %}
7408   ins_pipe( pipe_cmpxchg );
7409 %}
7410 
7411 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7412   match(Set newval (GetAndAddL mem newval));
7413   effect(KILL cr);
7414   format %{ "XADDQ  [$mem],$newval" %}
7415   ins_encode %{
7416     if (os::is_MP()) { __ lock(); }
7417     __ xaddq($mem$$Address, $newval$$Register);
7418   %}
7419   ins_pipe( pipe_cmpxchg );
7420 %}
7421 
7422 instruct xchgI( memory mem, rRegI newval) %{
7423   match(Set newval (GetAndSetI mem newval));
7424   format %{ "XCHGL  $newval,[$mem]" %}
7425   ins_encode %{
7426     __ xchgl($newval$$Register, $mem$$Address);
7427   %}
7428   ins_pipe( pipe_cmpxchg );
7429 %}
7430 
7431 instruct xchgL( memory mem, rRegL newval) %{
7432   match(Set newval (GetAndSetL mem newval));
7433   format %{ "XCHGL  $newval,[$mem]" %}
7434   ins_encode %{
7435     __ xchgq($newval$$Register, $mem$$Address);
7436   %}
7437   ins_pipe( pipe_cmpxchg );
7438 %}
7439 
7440 instruct xchgP( memory mem, rRegP newval) %{
7441   match(Set newval (GetAndSetP mem newval));
7442   format %{ "XCHGQ  $newval,[$mem]" %}
7443   ins_encode %{
7444     __ xchgq($newval$$Register, $mem$$Address);
7445   %}
7446   ins_pipe( pipe_cmpxchg );
7447 %}
7448 
7449 instruct xchgN( memory mem, rRegN newval) %{
7450   match(Set newval (GetAndSetN mem newval));
7451   format %{ "XCHGL  $newval,$mem]" %}
7452   ins_encode %{
7453     __ xchgl($newval$$Register, $mem$$Address);
7454   %}
7455   ins_pipe( pipe_cmpxchg );
7456 %}
7457 
7458 //----------Subtraction Instructions-------------------------------------------
7459 
7460 // Integer Subtraction Instructions
7461 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7462 %{
7463   match(Set dst (SubI dst src));
7464   effect(KILL cr);
7465 
7466   format %{ "subl    $dst, $src\t# int" %}
7467   opcode(0x2B);
7468   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7469   ins_pipe(ialu_reg_reg);
7470 %}
7471 
7472 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7473 %{
7474   match(Set dst (SubI dst src));
7475   effect(KILL cr);
7476 
7477   format %{ "subl    $dst, $src\t# int" %}
7478   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7479   ins_encode(OpcSErm(dst, src), Con8or32(src));
7480   ins_pipe(ialu_reg);
7481 %}
7482 
7483 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7484 %{
7485   match(Set dst (SubI dst (LoadI src)));
7486   effect(KILL cr);
7487 
7488   ins_cost(125);
7489   format %{ "subl    $dst, $src\t# int" %}
7490   opcode(0x2B);
7491   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7492   ins_pipe(ialu_reg_mem);
7493 %}
7494 
7495 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7496 %{
7497   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7498   effect(KILL cr);
7499 
7500   ins_cost(150);
7501   format %{ "subl    $dst, $src\t# int" %}
7502   opcode(0x29); /* Opcode 29 /r */
7503   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7504   ins_pipe(ialu_mem_reg);
7505 %}
7506 
7507 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7508 %{
7509   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7510   effect(KILL cr);
7511 
7512   ins_cost(125); // XXX
7513   format %{ "subl    $dst, $src\t# int" %}
7514   opcode(0x81); /* Opcode 81 /5 id */
7515   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7516   ins_pipe(ialu_mem_imm);
7517 %}
7518 
7519 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7520 %{
7521   match(Set dst (SubL dst src));
7522   effect(KILL cr);
7523 
7524   format %{ "subq    $dst, $src\t# long" %}
7525   opcode(0x2B);
7526   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7527   ins_pipe(ialu_reg_reg);
7528 %}
7529 
7530 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7531 %{
7532   match(Set dst (SubL dst src));
7533   effect(KILL cr);
7534 
7535   format %{ "subq    $dst, $src\t# long" %}
7536   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7537   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7538   ins_pipe(ialu_reg);
7539 %}
7540 
7541 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7542 %{
7543   match(Set dst (SubL dst (LoadL src)));
7544   effect(KILL cr);
7545 
7546   ins_cost(125);
7547   format %{ "subq    $dst, $src\t# long" %}
7548   opcode(0x2B);
7549   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7550   ins_pipe(ialu_reg_mem);
7551 %}
7552 
7553 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7554 %{
7555   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7556   effect(KILL cr);
7557 
7558   ins_cost(150);
7559   format %{ "subq    $dst, $src\t# long" %}
7560   opcode(0x29); /* Opcode 29 /r */
7561   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7562   ins_pipe(ialu_mem_reg);
7563 %}
7564 
7565 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7566 %{
7567   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7568   effect(KILL cr);
7569 
7570   ins_cost(125); // XXX
7571   format %{ "subq    $dst, $src\t# long" %}
7572   opcode(0x81); /* Opcode 81 /5 id */
7573   ins_encode(REX_mem_wide(dst),
7574              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7575   ins_pipe(ialu_mem_imm);
7576 %}
7577 
7578 // Subtract from a pointer
7579 // XXX hmpf???
7580 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7581 %{
7582   match(Set dst (AddP dst (SubI zero src)));
7583   effect(KILL cr);
7584 
7585   format %{ "subq    $dst, $src\t# ptr - int" %}
7586   opcode(0x2B);
7587   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7588   ins_pipe(ialu_reg_reg);
7589 %}
7590 
7591 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7592 %{
7593   match(Set dst (SubI zero dst));
7594   effect(KILL cr);
7595 
7596   format %{ "negl    $dst\t# int" %}
7597   opcode(0xF7, 0x03);  // Opcode F7 /3
7598   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7599   ins_pipe(ialu_reg);
7600 %}
7601 
7602 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7603 %{
7604   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7605   effect(KILL cr);
7606 
7607   format %{ "negl    $dst\t# int" %}
7608   opcode(0xF7, 0x03);  // Opcode F7 /3
7609   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7610   ins_pipe(ialu_reg);
7611 %}
7612 
7613 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7614 %{
7615   match(Set dst (SubL zero dst));
7616   effect(KILL cr);
7617 
7618   format %{ "negq    $dst\t# long" %}
7619   opcode(0xF7, 0x03);  // Opcode F7 /3
7620   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7621   ins_pipe(ialu_reg);
7622 %}
7623 
7624 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7625 %{
7626   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7627   effect(KILL cr);
7628 
7629   format %{ "negq    $dst\t# long" %}
7630   opcode(0xF7, 0x03);  // Opcode F7 /3
7631   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7632   ins_pipe(ialu_reg);
7633 %}
7634 
7635 //----------Multiplication/Division Instructions-------------------------------
7636 // Integer Multiplication Instructions
7637 // Multiply Register
7638 
7639 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7640 %{
7641   match(Set dst (MulI dst src));
7642   effect(KILL cr);
7643 
7644   ins_cost(300);
7645   format %{ "imull   $dst, $src\t# int" %}
7646   opcode(0x0F, 0xAF);
7647   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7648   ins_pipe(ialu_reg_reg_alu0);
7649 %}
7650 
7651 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7652 %{
7653   match(Set dst (MulI src imm));
7654   effect(KILL cr);
7655 
7656   ins_cost(300);
7657   format %{ "imull   $dst, $src, $imm\t# int" %}
7658   opcode(0x69); /* 69 /r id */
7659   ins_encode(REX_reg_reg(dst, src),
7660              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7661   ins_pipe(ialu_reg_reg_alu0);
7662 %}
7663 
7664 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7665 %{
7666   match(Set dst (MulI dst (LoadI src)));
7667   effect(KILL cr);
7668 
7669   ins_cost(350);
7670   format %{ "imull   $dst, $src\t# int" %}
7671   opcode(0x0F, 0xAF);
7672   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7673   ins_pipe(ialu_reg_mem_alu0);
7674 %}
7675 
7676 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7677 %{
7678   match(Set dst (MulI (LoadI src) imm));
7679   effect(KILL cr);
7680 
7681   ins_cost(300);
7682   format %{ "imull   $dst, $src, $imm\t# int" %}
7683   opcode(0x69); /* 69 /r id */
7684   ins_encode(REX_reg_mem(dst, src),
7685              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7686   ins_pipe(ialu_reg_mem_alu0);
7687 %}
7688 
7689 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7690 %{
7691   match(Set dst (MulL dst src));
7692   effect(KILL cr);
7693 
7694   ins_cost(300);
7695   format %{ "imulq   $dst, $src\t# long" %}
7696   opcode(0x0F, 0xAF);
7697   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7698   ins_pipe(ialu_reg_reg_alu0);
7699 %}
7700 
7701 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7702 %{
7703   match(Set dst (MulL src imm));
7704   effect(KILL cr);
7705 
7706   ins_cost(300);
7707   format %{ "imulq   $dst, $src, $imm\t# long" %}
7708   opcode(0x69); /* 69 /r id */
7709   ins_encode(REX_reg_reg_wide(dst, src),
7710              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7711   ins_pipe(ialu_reg_reg_alu0);
7712 %}
7713 
7714 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7715 %{
7716   match(Set dst (MulL dst (LoadL src)));
7717   effect(KILL cr);
7718 
7719   ins_cost(350);
7720   format %{ "imulq   $dst, $src\t# long" %}
7721   opcode(0x0F, 0xAF);
7722   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7723   ins_pipe(ialu_reg_mem_alu0);
7724 %}
7725 
7726 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7727 %{
7728   match(Set dst (MulL (LoadL src) imm));
7729   effect(KILL cr);
7730 
7731   ins_cost(300);
7732   format %{ "imulq   $dst, $src, $imm\t# long" %}
7733   opcode(0x69); /* 69 /r id */
7734   ins_encode(REX_reg_mem_wide(dst, src),
7735              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7736   ins_pipe(ialu_reg_mem_alu0);
7737 %}
7738 
7739 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7740 %{
7741   match(Set dst (MulHiL src rax));
7742   effect(USE_KILL rax, KILL cr);
7743 
7744   ins_cost(300);
7745   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7746   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7747   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7748   ins_pipe(ialu_reg_reg_alu0);
7749 %}
7750 
7751 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7752                    rFlagsReg cr)
7753 %{
7754   match(Set rax (DivI rax div));
7755   effect(KILL rdx, KILL cr);
7756 
7757   ins_cost(30*100+10*100); // XXX
7758   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7759             "jne,s   normal\n\t"
7760             "xorl    rdx, rdx\n\t"
7761             "cmpl    $div, -1\n\t"
7762             "je,s    done\n"
7763     "normal: cdql\n\t"
7764             "idivl   $div\n"
7765     "done:"        %}
7766   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7767   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7768   ins_pipe(ialu_reg_reg_alu0);
7769 %}
7770 
7771 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7772                    rFlagsReg cr)
7773 %{
7774   match(Set rax (DivL rax div));
7775   effect(KILL rdx, KILL cr);
7776 
7777   ins_cost(30*100+10*100); // XXX
7778   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7779             "cmpq    rax, rdx\n\t"
7780             "jne,s   normal\n\t"
7781             "xorl    rdx, rdx\n\t"
7782             "cmpq    $div, -1\n\t"
7783             "je,s    done\n"
7784     "normal: cdqq\n\t"
7785             "idivq   $div\n"
7786     "done:"        %}
7787   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7788   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7789   ins_pipe(ialu_reg_reg_alu0);
7790 %}
7791 
7792 // Integer DIVMOD with Register, both quotient and mod results
7793 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7794                              rFlagsReg cr)
7795 %{
7796   match(DivModI rax div);
7797   effect(KILL cr);
7798 
7799   ins_cost(30*100+10*100); // XXX
7800   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7801             "jne,s   normal\n\t"
7802             "xorl    rdx, rdx\n\t"
7803             "cmpl    $div, -1\n\t"
7804             "je,s    done\n"
7805     "normal: cdql\n\t"
7806             "idivl   $div\n"
7807     "done:"        %}
7808   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7809   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7810   ins_pipe(pipe_slow);
7811 %}
7812 
7813 // Long DIVMOD with Register, both quotient and mod results
7814 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7815                              rFlagsReg cr)
7816 %{
7817   match(DivModL rax div);
7818   effect(KILL cr);
7819 
7820   ins_cost(30*100+10*100); // XXX
7821   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7822             "cmpq    rax, rdx\n\t"
7823             "jne,s   normal\n\t"
7824             "xorl    rdx, rdx\n\t"
7825             "cmpq    $div, -1\n\t"
7826             "je,s    done\n"
7827     "normal: cdqq\n\t"
7828             "idivq   $div\n"
7829     "done:"        %}
7830   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7831   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7832   ins_pipe(pipe_slow);
7833 %}
7834 
7835 //----------- DivL-By-Constant-Expansions--------------------------------------
7836 // DivI cases are handled by the compiler
7837 
7838 // Magic constant, reciprocal of 10
7839 instruct loadConL_0x6666666666666667(rRegL dst)
7840 %{
7841   effect(DEF dst);
7842 
7843   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7844   ins_encode(load_immL(dst, 0x6666666666666667));
7845   ins_pipe(ialu_reg);
7846 %}
7847 
7848 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7849 %{
7850   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7851 
7852   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7853   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7854   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7855   ins_pipe(ialu_reg_reg_alu0);
7856 %}
7857 
7858 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7859 %{
7860   effect(USE_DEF dst, KILL cr);
7861 
7862   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7863   opcode(0xC1, 0x7); /* C1 /7 ib */
7864   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7865   ins_pipe(ialu_reg);
7866 %}
7867 
7868 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7869 %{
7870   effect(USE_DEF dst, KILL cr);
7871 
7872   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7873   opcode(0xC1, 0x7); /* C1 /7 ib */
7874   ins_encode(reg_opc_imm_wide(dst, 0x2));
7875   ins_pipe(ialu_reg);
7876 %}
7877 
7878 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7879 %{
7880   match(Set dst (DivL src div));
7881 
7882   ins_cost((5+8)*100);
7883   expand %{
7884     rax_RegL rax;                     // Killed temp
7885     rFlagsReg cr;                     // Killed
7886     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7887     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7888     sarL_rReg_63(src, cr);            // sarq  src, 63
7889     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7890     subL_rReg(dst, src, cr);          // subl  rdx, src
7891   %}
7892 %}
7893 
7894 //-----------------------------------------------------------------------------
7895 
7896 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7897                    rFlagsReg cr)
7898 %{
7899   match(Set rdx (ModI rax div));
7900   effect(KILL rax, KILL cr);
7901 
7902   ins_cost(300); // XXX
7903   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7904             "jne,s   normal\n\t"
7905             "xorl    rdx, rdx\n\t"
7906             "cmpl    $div, -1\n\t"
7907             "je,s    done\n"
7908     "normal: cdql\n\t"
7909             "idivl   $div\n"
7910     "done:"        %}
7911   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7912   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7913   ins_pipe(ialu_reg_reg_alu0);
7914 %}
7915 
7916 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7917                    rFlagsReg cr)
7918 %{
7919   match(Set rdx (ModL rax div));
7920   effect(KILL rax, KILL cr);
7921 
7922   ins_cost(300); // XXX
7923   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7924             "cmpq    rax, rdx\n\t"
7925             "jne,s   normal\n\t"
7926             "xorl    rdx, rdx\n\t"
7927             "cmpq    $div, -1\n\t"
7928             "je,s    done\n"
7929     "normal: cdqq\n\t"
7930             "idivq   $div\n"
7931     "done:"        %}
7932   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7933   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7934   ins_pipe(ialu_reg_reg_alu0);
7935 %}
7936 
7937 // Integer Shift Instructions
7938 // Shift Left by one
7939 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7940 %{
7941   match(Set dst (LShiftI dst shift));
7942   effect(KILL cr);
7943 
7944   format %{ "sall    $dst, $shift" %}
7945   opcode(0xD1, 0x4); /* D1 /4 */
7946   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7947   ins_pipe(ialu_reg);
7948 %}
7949 
7950 // Shift Left by one
7951 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7952 %{
7953   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7954   effect(KILL cr);
7955 
7956   format %{ "sall    $dst, $shift\t" %}
7957   opcode(0xD1, 0x4); /* D1 /4 */
7958   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7959   ins_pipe(ialu_mem_imm);
7960 %}
7961 
7962 // Shift Left by 8-bit immediate
7963 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7964 %{
7965   match(Set dst (LShiftI dst shift));
7966   effect(KILL cr);
7967 
7968   format %{ "sall    $dst, $shift" %}
7969   opcode(0xC1, 0x4); /* C1 /4 ib */
7970   ins_encode(reg_opc_imm(dst, shift));
7971   ins_pipe(ialu_reg);
7972 %}
7973 
7974 // Shift Left by 8-bit immediate
7975 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7976 %{
7977   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7978   effect(KILL cr);
7979 
7980   format %{ "sall    $dst, $shift" %}
7981   opcode(0xC1, 0x4); /* C1 /4 ib */
7982   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7983   ins_pipe(ialu_mem_imm);
7984 %}
7985 
7986 // Shift Left by variable
7987 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7988 %{
7989   match(Set dst (LShiftI dst shift));
7990   effect(KILL cr);
7991 
7992   format %{ "sall    $dst, $shift" %}
7993   opcode(0xD3, 0x4); /* D3 /4 */
7994   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7995   ins_pipe(ialu_reg_reg);
7996 %}
7997 
7998 // Shift Left by variable
7999 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8000 %{
8001   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8002   effect(KILL cr);
8003 
8004   format %{ "sall    $dst, $shift" %}
8005   opcode(0xD3, 0x4); /* D3 /4 */
8006   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8007   ins_pipe(ialu_mem_reg);
8008 %}
8009 
8010 // Arithmetic shift right by one
8011 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8012 %{
8013   match(Set dst (RShiftI dst shift));
8014   effect(KILL cr);
8015 
8016   format %{ "sarl    $dst, $shift" %}
8017   opcode(0xD1, 0x7); /* D1 /7 */
8018   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8019   ins_pipe(ialu_reg);
8020 %}
8021 
8022 // Arithmetic shift right by one
8023 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8024 %{
8025   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8026   effect(KILL cr);
8027 
8028   format %{ "sarl    $dst, $shift" %}
8029   opcode(0xD1, 0x7); /* D1 /7 */
8030   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8031   ins_pipe(ialu_mem_imm);
8032 %}
8033 
8034 // Arithmetic Shift Right by 8-bit immediate
8035 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8036 %{
8037   match(Set dst (RShiftI dst shift));
8038   effect(KILL cr);
8039 
8040   format %{ "sarl    $dst, $shift" %}
8041   opcode(0xC1, 0x7); /* C1 /7 ib */
8042   ins_encode(reg_opc_imm(dst, shift));
8043   ins_pipe(ialu_mem_imm);
8044 %}
8045 
8046 // Arithmetic Shift Right by 8-bit immediate
8047 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8048 %{
8049   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8050   effect(KILL cr);
8051 
8052   format %{ "sarl    $dst, $shift" %}
8053   opcode(0xC1, 0x7); /* C1 /7 ib */
8054   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8055   ins_pipe(ialu_mem_imm);
8056 %}
8057 
8058 // Arithmetic Shift Right by variable
8059 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8060 %{
8061   match(Set dst (RShiftI dst shift));
8062   effect(KILL cr);
8063 
8064   format %{ "sarl    $dst, $shift" %}
8065   opcode(0xD3, 0x7); /* D3 /7 */
8066   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8067   ins_pipe(ialu_reg_reg);
8068 %}
8069 
8070 // Arithmetic Shift Right by variable
8071 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8072 %{
8073   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8074   effect(KILL cr);
8075 
8076   format %{ "sarl    $dst, $shift" %}
8077   opcode(0xD3, 0x7); /* D3 /7 */
8078   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8079   ins_pipe(ialu_mem_reg);
8080 %}
8081 
8082 // Logical shift right by one
8083 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8084 %{
8085   match(Set dst (URShiftI dst shift));
8086   effect(KILL cr);
8087 
8088   format %{ "shrl    $dst, $shift" %}
8089   opcode(0xD1, 0x5); /* D1 /5 */
8090   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8091   ins_pipe(ialu_reg);
8092 %}
8093 
8094 // Logical shift right by one
8095 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8096 %{
8097   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8098   effect(KILL cr);
8099 
8100   format %{ "shrl    $dst, $shift" %}
8101   opcode(0xD1, 0x5); /* D1 /5 */
8102   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8103   ins_pipe(ialu_mem_imm);
8104 %}
8105 
8106 // Logical Shift Right by 8-bit immediate
8107 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8108 %{
8109   match(Set dst (URShiftI dst shift));
8110   effect(KILL cr);
8111 
8112   format %{ "shrl    $dst, $shift" %}
8113   opcode(0xC1, 0x5); /* C1 /5 ib */
8114   ins_encode(reg_opc_imm(dst, shift));
8115   ins_pipe(ialu_reg);
8116 %}
8117 
8118 // Logical Shift Right by 8-bit immediate
8119 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8120 %{
8121   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8122   effect(KILL cr);
8123 
8124   format %{ "shrl    $dst, $shift" %}
8125   opcode(0xC1, 0x5); /* C1 /5 ib */
8126   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8127   ins_pipe(ialu_mem_imm);
8128 %}
8129 
8130 // Logical Shift Right by variable
8131 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8132 %{
8133   match(Set dst (URShiftI dst shift));
8134   effect(KILL cr);
8135 
8136   format %{ "shrl    $dst, $shift" %}
8137   opcode(0xD3, 0x5); /* D3 /5 */
8138   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8139   ins_pipe(ialu_reg_reg);
8140 %}
8141 
8142 // Logical Shift Right by variable
8143 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8144 %{
8145   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8146   effect(KILL cr);
8147 
8148   format %{ "shrl    $dst, $shift" %}
8149   opcode(0xD3, 0x5); /* D3 /5 */
8150   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8151   ins_pipe(ialu_mem_reg);
8152 %}
8153 
8154 // Long Shift Instructions
8155 // Shift Left by one
8156 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8157 %{
8158   match(Set dst (LShiftL dst shift));
8159   effect(KILL cr);
8160 
8161   format %{ "salq    $dst, $shift" %}
8162   opcode(0xD1, 0x4); /* D1 /4 */
8163   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8164   ins_pipe(ialu_reg);
8165 %}
8166 
8167 // Shift Left by one
8168 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8169 %{
8170   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8171   effect(KILL cr);
8172 
8173   format %{ "salq    $dst, $shift" %}
8174   opcode(0xD1, 0x4); /* D1 /4 */
8175   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8176   ins_pipe(ialu_mem_imm);
8177 %}
8178 
8179 // Shift Left by 8-bit immediate
8180 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8181 %{
8182   match(Set dst (LShiftL dst shift));
8183   effect(KILL cr);
8184 
8185   format %{ "salq    $dst, $shift" %}
8186   opcode(0xC1, 0x4); /* C1 /4 ib */
8187   ins_encode(reg_opc_imm_wide(dst, shift));
8188   ins_pipe(ialu_reg);
8189 %}
8190 
8191 // Shift Left by 8-bit immediate
8192 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8193 %{
8194   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8195   effect(KILL cr);
8196 
8197   format %{ "salq    $dst, $shift" %}
8198   opcode(0xC1, 0x4); /* C1 /4 ib */
8199   ins_encode(REX_mem_wide(dst), OpcP,
8200              RM_opc_mem(secondary, dst), Con8or32(shift));
8201   ins_pipe(ialu_mem_imm);
8202 %}
8203 
8204 // Shift Left by variable
8205 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8206 %{
8207   match(Set dst (LShiftL dst shift));
8208   effect(KILL cr);
8209 
8210   format %{ "salq    $dst, $shift" %}
8211   opcode(0xD3, 0x4); /* D3 /4 */
8212   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8213   ins_pipe(ialu_reg_reg);
8214 %}
8215 
8216 // Shift Left by variable
8217 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8218 %{
8219   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8220   effect(KILL cr);
8221 
8222   format %{ "salq    $dst, $shift" %}
8223   opcode(0xD3, 0x4); /* D3 /4 */
8224   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8225   ins_pipe(ialu_mem_reg);
8226 %}
8227 
8228 // Arithmetic shift right by one
8229 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8230 %{
8231   match(Set dst (RShiftL dst shift));
8232   effect(KILL cr);
8233 
8234   format %{ "sarq    $dst, $shift" %}
8235   opcode(0xD1, 0x7); /* D1 /7 */
8236   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8237   ins_pipe(ialu_reg);
8238 %}
8239 
8240 // Arithmetic shift right by one
8241 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8242 %{
8243   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8244   effect(KILL cr);
8245 
8246   format %{ "sarq    $dst, $shift" %}
8247   opcode(0xD1, 0x7); /* D1 /7 */
8248   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8249   ins_pipe(ialu_mem_imm);
8250 %}
8251 
8252 // Arithmetic Shift Right by 8-bit immediate
8253 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8254 %{
8255   match(Set dst (RShiftL dst shift));
8256   effect(KILL cr);
8257 
8258   format %{ "sarq    $dst, $shift" %}
8259   opcode(0xC1, 0x7); /* C1 /7 ib */
8260   ins_encode(reg_opc_imm_wide(dst, shift));
8261   ins_pipe(ialu_mem_imm);
8262 %}
8263 
8264 // Arithmetic Shift Right by 8-bit immediate
8265 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8266 %{
8267   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8268   effect(KILL cr);
8269 
8270   format %{ "sarq    $dst, $shift" %}
8271   opcode(0xC1, 0x7); /* C1 /7 ib */
8272   ins_encode(REX_mem_wide(dst), OpcP,
8273              RM_opc_mem(secondary, dst), Con8or32(shift));
8274   ins_pipe(ialu_mem_imm);
8275 %}
8276 
8277 // Arithmetic Shift Right by variable
8278 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8279 %{
8280   match(Set dst (RShiftL dst shift));
8281   effect(KILL cr);
8282 
8283   format %{ "sarq    $dst, $shift" %}
8284   opcode(0xD3, 0x7); /* D3 /7 */
8285   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8286   ins_pipe(ialu_reg_reg);
8287 %}
8288 
8289 // Arithmetic Shift Right by variable
8290 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8291 %{
8292   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8293   effect(KILL cr);
8294 
8295   format %{ "sarq    $dst, $shift" %}
8296   opcode(0xD3, 0x7); /* D3 /7 */
8297   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8298   ins_pipe(ialu_mem_reg);
8299 %}
8300 
8301 // Logical shift right by one
8302 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8303 %{
8304   match(Set dst (URShiftL dst shift));
8305   effect(KILL cr);
8306 
8307   format %{ "shrq    $dst, $shift" %}
8308   opcode(0xD1, 0x5); /* D1 /5 */
8309   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8310   ins_pipe(ialu_reg);
8311 %}
8312 
8313 // Logical shift right by one
8314 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8315 %{
8316   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8317   effect(KILL cr);
8318 
8319   format %{ "shrq    $dst, $shift" %}
8320   opcode(0xD1, 0x5); /* D1 /5 */
8321   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8322   ins_pipe(ialu_mem_imm);
8323 %}
8324 
8325 // Logical Shift Right by 8-bit immediate
8326 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8327 %{
8328   match(Set dst (URShiftL dst shift));
8329   effect(KILL cr);
8330 
8331   format %{ "shrq    $dst, $shift" %}
8332   opcode(0xC1, 0x5); /* C1 /5 ib */
8333   ins_encode(reg_opc_imm_wide(dst, shift));
8334   ins_pipe(ialu_reg);
8335 %}
8336 
8337 
8338 // Logical Shift Right by 8-bit immediate
8339 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8340 %{
8341   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8342   effect(KILL cr);
8343 
8344   format %{ "shrq    $dst, $shift" %}
8345   opcode(0xC1, 0x5); /* C1 /5 ib */
8346   ins_encode(REX_mem_wide(dst), OpcP,
8347              RM_opc_mem(secondary, dst), Con8or32(shift));
8348   ins_pipe(ialu_mem_imm);
8349 %}
8350 
8351 // Logical Shift Right by variable
8352 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8353 %{
8354   match(Set dst (URShiftL dst shift));
8355   effect(KILL cr);
8356 
8357   format %{ "shrq    $dst, $shift" %}
8358   opcode(0xD3, 0x5); /* D3 /5 */
8359   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8360   ins_pipe(ialu_reg_reg);
8361 %}
8362 
8363 // Logical Shift Right by variable
8364 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8365 %{
8366   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8367   effect(KILL cr);
8368 
8369   format %{ "shrq    $dst, $shift" %}
8370   opcode(0xD3, 0x5); /* D3 /5 */
8371   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8372   ins_pipe(ialu_mem_reg);
8373 %}
8374 
8375 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8376 // This idiom is used by the compiler for the i2b bytecode.
8377 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8378 %{
8379   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8380 
8381   format %{ "movsbl  $dst, $src\t# i2b" %}
8382   opcode(0x0F, 0xBE);
8383   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8384   ins_pipe(ialu_reg_reg);
8385 %}
8386 
8387 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8388 // This idiom is used by the compiler the i2s bytecode.
8389 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8390 %{
8391   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8392 
8393   format %{ "movswl  $dst, $src\t# i2s" %}
8394   opcode(0x0F, 0xBF);
8395   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8396   ins_pipe(ialu_reg_reg);
8397 %}
8398 
8399 // ROL/ROR instructions
8400 
8401 // ROL expand
8402 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8403   effect(KILL cr, USE_DEF dst);
8404 
8405   format %{ "roll    $dst" %}
8406   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8407   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8408   ins_pipe(ialu_reg);
8409 %}
8410 
8411 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8412   effect(USE_DEF dst, USE shift, KILL cr);
8413 
8414   format %{ "roll    $dst, $shift" %}
8415   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8416   ins_encode( reg_opc_imm(dst, shift) );
8417   ins_pipe(ialu_reg);
8418 %}
8419 
8420 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8421 %{
8422   effect(USE_DEF dst, USE shift, KILL cr);
8423 
8424   format %{ "roll    $dst, $shift" %}
8425   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8426   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8427   ins_pipe(ialu_reg_reg);
8428 %}
8429 // end of ROL expand
8430 
8431 // Rotate Left by one
8432 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8433 %{
8434   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8435 
8436   expand %{
8437     rolI_rReg_imm1(dst, cr);
8438   %}
8439 %}
8440 
8441 // Rotate Left by 8-bit immediate
8442 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8443 %{
8444   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8445   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8446 
8447   expand %{
8448     rolI_rReg_imm8(dst, lshift, cr);
8449   %}
8450 %}
8451 
8452 // Rotate Left by variable
8453 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8454 %{
8455   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8456 
8457   expand %{
8458     rolI_rReg_CL(dst, shift, cr);
8459   %}
8460 %}
8461 
8462 // Rotate Left by variable
8463 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8464 %{
8465   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8466 
8467   expand %{
8468     rolI_rReg_CL(dst, shift, cr);
8469   %}
8470 %}
8471 
8472 // ROR expand
8473 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8474 %{
8475   effect(USE_DEF dst, KILL cr);
8476 
8477   format %{ "rorl    $dst" %}
8478   opcode(0xD1, 0x1); /* D1 /1 */
8479   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8480   ins_pipe(ialu_reg);
8481 %}
8482 
8483 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8484 %{
8485   effect(USE_DEF dst, USE shift, KILL cr);
8486 
8487   format %{ "rorl    $dst, $shift" %}
8488   opcode(0xC1, 0x1); /* C1 /1 ib */
8489   ins_encode(reg_opc_imm(dst, shift));
8490   ins_pipe(ialu_reg);
8491 %}
8492 
8493 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8494 %{
8495   effect(USE_DEF dst, USE shift, KILL cr);
8496 
8497   format %{ "rorl    $dst, $shift" %}
8498   opcode(0xD3, 0x1); /* D3 /1 */
8499   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8500   ins_pipe(ialu_reg_reg);
8501 %}
8502 // end of ROR expand
8503 
8504 // Rotate Right by one
8505 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8506 %{
8507   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8508 
8509   expand %{
8510     rorI_rReg_imm1(dst, cr);
8511   %}
8512 %}
8513 
8514 // Rotate Right by 8-bit immediate
8515 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8516 %{
8517   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8518   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8519 
8520   expand %{
8521     rorI_rReg_imm8(dst, rshift, cr);
8522   %}
8523 %}
8524 
8525 // Rotate Right by variable
8526 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8527 %{
8528   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8529 
8530   expand %{
8531     rorI_rReg_CL(dst, shift, cr);
8532   %}
8533 %}
8534 
8535 // Rotate Right by variable
8536 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8537 %{
8538   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8539 
8540   expand %{
8541     rorI_rReg_CL(dst, shift, cr);
8542   %}
8543 %}
8544 
8545 // for long rotate
8546 // ROL expand
8547 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8548   effect(USE_DEF dst, KILL cr);
8549 
8550   format %{ "rolq    $dst" %}
8551   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8552   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8557   effect(USE_DEF dst, USE shift, KILL cr);
8558 
8559   format %{ "rolq    $dst, $shift" %}
8560   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8561   ins_encode( reg_opc_imm_wide(dst, shift) );
8562   ins_pipe(ialu_reg);
8563 %}
8564 
8565 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8566 %{
8567   effect(USE_DEF dst, USE shift, KILL cr);
8568 
8569   format %{ "rolq    $dst, $shift" %}
8570   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8571   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8572   ins_pipe(ialu_reg_reg);
8573 %}
8574 // end of ROL expand
8575 
8576 // Rotate Left by one
8577 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8578 %{
8579   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8580 
8581   expand %{
8582     rolL_rReg_imm1(dst, cr);
8583   %}
8584 %}
8585 
8586 // Rotate Left by 8-bit immediate
8587 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8588 %{
8589   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8590   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8591 
8592   expand %{
8593     rolL_rReg_imm8(dst, lshift, cr);
8594   %}
8595 %}
8596 
8597 // Rotate Left by variable
8598 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8599 %{
8600   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8601 
8602   expand %{
8603     rolL_rReg_CL(dst, shift, cr);
8604   %}
8605 %}
8606 
8607 // Rotate Left by variable
8608 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8609 %{
8610   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8611 
8612   expand %{
8613     rolL_rReg_CL(dst, shift, cr);
8614   %}
8615 %}
8616 
8617 // ROR expand
8618 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8619 %{
8620   effect(USE_DEF dst, KILL cr);
8621 
8622   format %{ "rorq    $dst" %}
8623   opcode(0xD1, 0x1); /* D1 /1 */
8624   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8625   ins_pipe(ialu_reg);
8626 %}
8627 
8628 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8629 %{
8630   effect(USE_DEF dst, USE shift, KILL cr);
8631 
8632   format %{ "rorq    $dst, $shift" %}
8633   opcode(0xC1, 0x1); /* C1 /1 ib */
8634   ins_encode(reg_opc_imm_wide(dst, shift));
8635   ins_pipe(ialu_reg);
8636 %}
8637 
8638 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8639 %{
8640   effect(USE_DEF dst, USE shift, KILL cr);
8641 
8642   format %{ "rorq    $dst, $shift" %}
8643   opcode(0xD3, 0x1); /* D3 /1 */
8644   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8645   ins_pipe(ialu_reg_reg);
8646 %}
8647 // end of ROR expand
8648 
8649 // Rotate Right by one
8650 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8651 %{
8652   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8653 
8654   expand %{
8655     rorL_rReg_imm1(dst, cr);
8656   %}
8657 %}
8658 
8659 // Rotate Right by 8-bit immediate
8660 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8661 %{
8662   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8663   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8664 
8665   expand %{
8666     rorL_rReg_imm8(dst, rshift, cr);
8667   %}
8668 %}
8669 
8670 // Rotate Right by variable
8671 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8672 %{
8673   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8674 
8675   expand %{
8676     rorL_rReg_CL(dst, shift, cr);
8677   %}
8678 %}
8679 
8680 // Rotate Right by variable
8681 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8682 %{
8683   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8684 
8685   expand %{
8686     rorL_rReg_CL(dst, shift, cr);
8687   %}
8688 %}
8689 
8690 // Logical Instructions
8691 
8692 // Integer Logical Instructions
8693 
8694 // And Instructions
8695 // And Register with Register
8696 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8697 %{
8698   match(Set dst (AndI dst src));
8699   effect(KILL cr);
8700 
8701   format %{ "andl    $dst, $src\t# int" %}
8702   opcode(0x23);
8703   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8704   ins_pipe(ialu_reg_reg);
8705 %}
8706 
8707 // And Register with Immediate 255
8708 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8709 %{
8710   match(Set dst (AndI dst src));
8711 
8712   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8713   opcode(0x0F, 0xB6);
8714   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8715   ins_pipe(ialu_reg);
8716 %}
8717 
8718 // And Register with Immediate 255 and promote to long
8719 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8720 %{
8721   match(Set dst (ConvI2L (AndI src mask)));
8722 
8723   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8724   opcode(0x0F, 0xB6);
8725   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8726   ins_pipe(ialu_reg);
8727 %}
8728 
8729 // And Register with Immediate 65535
8730 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8731 %{
8732   match(Set dst (AndI dst src));
8733 
8734   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8735   opcode(0x0F, 0xB7);
8736   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8737   ins_pipe(ialu_reg);
8738 %}
8739 
8740 // And Register with Immediate 65535 and promote to long
8741 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8742 %{
8743   match(Set dst (ConvI2L (AndI src mask)));
8744 
8745   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8746   opcode(0x0F, 0xB7);
8747   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8748   ins_pipe(ialu_reg);
8749 %}
8750 
8751 // And Register with Immediate
8752 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8753 %{
8754   match(Set dst (AndI dst src));
8755   effect(KILL cr);
8756 
8757   format %{ "andl    $dst, $src\t# int" %}
8758   opcode(0x81, 0x04); /* Opcode 81 /4 */
8759   ins_encode(OpcSErm(dst, src), Con8or32(src));
8760   ins_pipe(ialu_reg);
8761 %}
8762 
8763 // And Register with Memory
8764 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8765 %{
8766   match(Set dst (AndI dst (LoadI src)));
8767   effect(KILL cr);
8768 
8769   ins_cost(125);
8770   format %{ "andl    $dst, $src\t# int" %}
8771   opcode(0x23);
8772   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8773   ins_pipe(ialu_reg_mem);
8774 %}
8775 
8776 // And Memory with Register
8777 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8778 %{
8779   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8780   effect(KILL cr);
8781 
8782   ins_cost(150);
8783   format %{ "andl    $dst, $src\t# int" %}
8784   opcode(0x21); /* Opcode 21 /r */
8785   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8786   ins_pipe(ialu_mem_reg);
8787 %}
8788 
8789 // And Memory with Immediate
8790 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8791 %{
8792   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8793   effect(KILL cr);
8794 
8795   ins_cost(125);
8796   format %{ "andl    $dst, $src\t# int" %}
8797   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8798   ins_encode(REX_mem(dst), OpcSE(src),
8799              RM_opc_mem(secondary, dst), Con8or32(src));
8800   ins_pipe(ialu_mem_imm);
8801 %}
8802 
8803 // BMI1 instructions
8804 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8805   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8806   predicate(UseBMI1Instructions);
8807   effect(KILL cr);
8808 
8809   ins_cost(125);
8810   format %{ "andnl  $dst, $src1, $src2" %}
8811 
8812   ins_encode %{
8813     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8814   %}
8815   ins_pipe(ialu_reg_mem);
8816 %}
8817 
8818 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8819   match(Set dst (AndI (XorI src1 minus_1) src2));
8820   predicate(UseBMI1Instructions);
8821   effect(KILL cr);
8822 
8823   format %{ "andnl  $dst, $src1, $src2" %}
8824 
8825   ins_encode %{
8826     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8827   %}
8828   ins_pipe(ialu_reg);
8829 %}
8830 
8831 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8832   match(Set dst (AndI (SubI imm_zero src) src));
8833   predicate(UseBMI1Instructions);
8834   effect(KILL cr);
8835 
8836   format %{ "blsil  $dst, $src" %}
8837 
8838   ins_encode %{
8839     __ blsil($dst$$Register, $src$$Register);
8840   %}
8841   ins_pipe(ialu_reg);
8842 %}
8843 
8844 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8845   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8846   predicate(UseBMI1Instructions);
8847   effect(KILL cr);
8848 
8849   ins_cost(125);
8850   format %{ "blsil  $dst, $src" %}
8851 
8852   ins_encode %{
8853     __ blsil($dst$$Register, $src$$Address);
8854   %}
8855   ins_pipe(ialu_reg_mem);
8856 %}
8857 
8858 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8859 %{
8860   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8861   predicate(UseBMI1Instructions);
8862   effect(KILL cr);
8863 
8864   ins_cost(125);
8865   format %{ "blsmskl $dst, $src" %}
8866 
8867   ins_encode %{
8868     __ blsmskl($dst$$Register, $src$$Address);
8869   %}
8870   ins_pipe(ialu_reg_mem);
8871 %}
8872 
8873 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8874 %{
8875   match(Set dst (XorI (AddI src minus_1) src));
8876   predicate(UseBMI1Instructions);
8877   effect(KILL cr);
8878 
8879   format %{ "blsmskl $dst, $src" %}
8880 
8881   ins_encode %{
8882     __ blsmskl($dst$$Register, $src$$Register);
8883   %}
8884 
8885   ins_pipe(ialu_reg);
8886 %}
8887 
8888 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8889 %{
8890   match(Set dst (AndI (AddI src minus_1) src) );
8891   predicate(UseBMI1Instructions);
8892   effect(KILL cr);
8893 
8894   format %{ "blsrl  $dst, $src" %}
8895 
8896   ins_encode %{
8897     __ blsrl($dst$$Register, $src$$Register);
8898   %}
8899 
8900   ins_pipe(ialu_reg_mem);
8901 %}
8902 
8903 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8904 %{
8905   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8906   predicate(UseBMI1Instructions);
8907   effect(KILL cr);
8908 
8909   ins_cost(125);
8910   format %{ "blsrl  $dst, $src" %}
8911 
8912   ins_encode %{
8913     __ blsrl($dst$$Register, $src$$Address);
8914   %}
8915 
8916   ins_pipe(ialu_reg);
8917 %}
8918 
8919 // Or Instructions
8920 // Or Register with Register
8921 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8922 %{
8923   match(Set dst (OrI dst src));
8924   effect(KILL cr);
8925 
8926   format %{ "orl     $dst, $src\t# int" %}
8927   opcode(0x0B);
8928   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8929   ins_pipe(ialu_reg_reg);
8930 %}
8931 
8932 // Or Register with Immediate
8933 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8934 %{
8935   match(Set dst (OrI dst src));
8936   effect(KILL cr);
8937 
8938   format %{ "orl     $dst, $src\t# int" %}
8939   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8940   ins_encode(OpcSErm(dst, src), Con8or32(src));
8941   ins_pipe(ialu_reg);
8942 %}
8943 
8944 // Or Register with Memory
8945 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8946 %{
8947   match(Set dst (OrI dst (LoadI src)));
8948   effect(KILL cr);
8949 
8950   ins_cost(125);
8951   format %{ "orl     $dst, $src\t# int" %}
8952   opcode(0x0B);
8953   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8954   ins_pipe(ialu_reg_mem);
8955 %}
8956 
8957 // Or Memory with Register
8958 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8959 %{
8960   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8961   effect(KILL cr);
8962 
8963   ins_cost(150);
8964   format %{ "orl     $dst, $src\t# int" %}
8965   opcode(0x09); /* Opcode 09 /r */
8966   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8967   ins_pipe(ialu_mem_reg);
8968 %}
8969 
8970 // Or Memory with Immediate
8971 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8972 %{
8973   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8974   effect(KILL cr);
8975 
8976   ins_cost(125);
8977   format %{ "orl     $dst, $src\t# int" %}
8978   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8979   ins_encode(REX_mem(dst), OpcSE(src),
8980              RM_opc_mem(secondary, dst), Con8or32(src));
8981   ins_pipe(ialu_mem_imm);
8982 %}
8983 
8984 // Xor Instructions
8985 // Xor Register with Register
8986 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8987 %{
8988   match(Set dst (XorI dst src));
8989   effect(KILL cr);
8990 
8991   format %{ "xorl    $dst, $src\t# int" %}
8992   opcode(0x33);
8993   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8994   ins_pipe(ialu_reg_reg);
8995 %}
8996 
8997 // Xor Register with Immediate -1
8998 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8999   match(Set dst (XorI dst imm));
9000 
9001   format %{ "not    $dst" %}
9002   ins_encode %{
9003      __ notl($dst$$Register);
9004   %}
9005   ins_pipe(ialu_reg);
9006 %}
9007 
9008 // Xor Register with Immediate
9009 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9010 %{
9011   match(Set dst (XorI dst src));
9012   effect(KILL cr);
9013 
9014   format %{ "xorl    $dst, $src\t# int" %}
9015   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9016   ins_encode(OpcSErm(dst, src), Con8or32(src));
9017   ins_pipe(ialu_reg);
9018 %}
9019 
9020 // Xor Register with Memory
9021 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9022 %{
9023   match(Set dst (XorI dst (LoadI src)));
9024   effect(KILL cr);
9025 
9026   ins_cost(125);
9027   format %{ "xorl    $dst, $src\t# int" %}
9028   opcode(0x33);
9029   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9030   ins_pipe(ialu_reg_mem);
9031 %}
9032 
9033 // Xor Memory with Register
9034 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9035 %{
9036   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9037   effect(KILL cr);
9038 
9039   ins_cost(150);
9040   format %{ "xorl    $dst, $src\t# int" %}
9041   opcode(0x31); /* Opcode 31 /r */
9042   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9043   ins_pipe(ialu_mem_reg);
9044 %}
9045 
9046 // Xor Memory with Immediate
9047 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9048 %{
9049   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9050   effect(KILL cr);
9051 
9052   ins_cost(125);
9053   format %{ "xorl    $dst, $src\t# int" %}
9054   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9055   ins_encode(REX_mem(dst), OpcSE(src),
9056              RM_opc_mem(secondary, dst), Con8or32(src));
9057   ins_pipe(ialu_mem_imm);
9058 %}
9059 
9060 
9061 // Long Logical Instructions
9062 
9063 // And Instructions
9064 // And Register with Register
9065 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9066 %{
9067   match(Set dst (AndL dst src));
9068   effect(KILL cr);
9069 
9070   format %{ "andq    $dst, $src\t# long" %}
9071   opcode(0x23);
9072   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9073   ins_pipe(ialu_reg_reg);
9074 %}
9075 
9076 // And Register with Immediate 255
9077 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9078 %{
9079   match(Set dst (AndL dst src));
9080 
9081   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9082   opcode(0x0F, 0xB6);
9083   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9084   ins_pipe(ialu_reg);
9085 %}
9086 
9087 // And Register with Immediate 65535
9088 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9089 %{
9090   match(Set dst (AndL dst src));
9091 
9092   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9093   opcode(0x0F, 0xB7);
9094   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9095   ins_pipe(ialu_reg);
9096 %}
9097 
9098 // And Register with Immediate
9099 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9100 %{
9101   match(Set dst (AndL dst src));
9102   effect(KILL cr);
9103 
9104   format %{ "andq    $dst, $src\t# long" %}
9105   opcode(0x81, 0x04); /* Opcode 81 /4 */
9106   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9107   ins_pipe(ialu_reg);
9108 %}
9109 
9110 // And Register with Memory
9111 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9112 %{
9113   match(Set dst (AndL dst (LoadL src)));
9114   effect(KILL cr);
9115 
9116   ins_cost(125);
9117   format %{ "andq    $dst, $src\t# long" %}
9118   opcode(0x23);
9119   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9120   ins_pipe(ialu_reg_mem);
9121 %}
9122 
9123 // And Memory with Register
9124 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9125 %{
9126   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9127   effect(KILL cr);
9128 
9129   ins_cost(150);
9130   format %{ "andq    $dst, $src\t# long" %}
9131   opcode(0x21); /* Opcode 21 /r */
9132   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9133   ins_pipe(ialu_mem_reg);
9134 %}
9135 
9136 // And Memory with Immediate
9137 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9138 %{
9139   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9140   effect(KILL cr);
9141 
9142   ins_cost(125);
9143   format %{ "andq    $dst, $src\t# long" %}
9144   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9145   ins_encode(REX_mem_wide(dst), OpcSE(src),
9146              RM_opc_mem(secondary, dst), Con8or32(src));
9147   ins_pipe(ialu_mem_imm);
9148 %}
9149 
9150 // BMI1 instructions
9151 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9152   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9153   predicate(UseBMI1Instructions);
9154   effect(KILL cr);
9155 
9156   ins_cost(125);
9157   format %{ "andnq  $dst, $src1, $src2" %}
9158 
9159   ins_encode %{
9160     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9161   %}
9162   ins_pipe(ialu_reg_mem);
9163 %}
9164 
9165 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9166   match(Set dst (AndL (XorL src1 minus_1) src2));
9167   predicate(UseBMI1Instructions);
9168   effect(KILL cr);
9169 
9170   format %{ "andnq  $dst, $src1, $src2" %}
9171 
9172   ins_encode %{
9173   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9174   %}
9175   ins_pipe(ialu_reg_mem);
9176 %}
9177 
9178 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9179   match(Set dst (AndL (SubL imm_zero src) src));
9180   predicate(UseBMI1Instructions);
9181   effect(KILL cr);
9182 
9183   format %{ "blsiq  $dst, $src" %}
9184 
9185   ins_encode %{
9186     __ blsiq($dst$$Register, $src$$Register);
9187   %}
9188   ins_pipe(ialu_reg);
9189 %}
9190 
9191 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9192   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9193   predicate(UseBMI1Instructions);
9194   effect(KILL cr);
9195 
9196   ins_cost(125);
9197   format %{ "blsiq  $dst, $src" %}
9198 
9199   ins_encode %{
9200     __ blsiq($dst$$Register, $src$$Address);
9201   %}
9202   ins_pipe(ialu_reg_mem);
9203 %}
9204 
9205 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9206 %{
9207   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9208   predicate(UseBMI1Instructions);
9209   effect(KILL cr);
9210 
9211   ins_cost(125);
9212   format %{ "blsmskq $dst, $src" %}
9213 
9214   ins_encode %{
9215     __ blsmskq($dst$$Register, $src$$Address);
9216   %}
9217   ins_pipe(ialu_reg_mem);
9218 %}
9219 
9220 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9221 %{
9222   match(Set dst (XorL (AddL src minus_1) src));
9223   predicate(UseBMI1Instructions);
9224   effect(KILL cr);
9225 
9226   format %{ "blsmskq $dst, $src" %}
9227 
9228   ins_encode %{
9229     __ blsmskq($dst$$Register, $src$$Register);
9230   %}
9231 
9232   ins_pipe(ialu_reg);
9233 %}
9234 
9235 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9236 %{
9237   match(Set dst (AndL (AddL src minus_1) src) );
9238   predicate(UseBMI1Instructions);
9239   effect(KILL cr);
9240 
9241   format %{ "blsrq  $dst, $src" %}
9242 
9243   ins_encode %{
9244     __ blsrq($dst$$Register, $src$$Register);
9245   %}
9246 
9247   ins_pipe(ialu_reg);
9248 %}
9249 
9250 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9251 %{
9252   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9253   predicate(UseBMI1Instructions);
9254   effect(KILL cr);
9255 
9256   ins_cost(125);
9257   format %{ "blsrq  $dst, $src" %}
9258 
9259   ins_encode %{
9260     __ blsrq($dst$$Register, $src$$Address);
9261   %}
9262 
9263   ins_pipe(ialu_reg);
9264 %}
9265 
9266 // Or Instructions
9267 // Or Register with Register
9268 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9269 %{
9270   match(Set dst (OrL dst src));
9271   effect(KILL cr);
9272 
9273   format %{ "orq     $dst, $src\t# long" %}
9274   opcode(0x0B);
9275   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9276   ins_pipe(ialu_reg_reg);
9277 %}
9278 
9279 // Use any_RegP to match R15 (TLS register) without spilling.
9280 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9281   match(Set dst (OrL dst (CastP2X src)));
9282   effect(KILL cr);
9283 
9284   format %{ "orq     $dst, $src\t# long" %}
9285   opcode(0x0B);
9286   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9287   ins_pipe(ialu_reg_reg);
9288 %}
9289 
9290 
9291 // Or Register with Immediate
9292 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9293 %{
9294   match(Set dst (OrL dst src));
9295   effect(KILL cr);
9296 
9297   format %{ "orq     $dst, $src\t# long" %}
9298   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9299   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9300   ins_pipe(ialu_reg);
9301 %}
9302 
9303 // Or Register with Memory
9304 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9305 %{
9306   match(Set dst (OrL dst (LoadL src)));
9307   effect(KILL cr);
9308 
9309   ins_cost(125);
9310   format %{ "orq     $dst, $src\t# long" %}
9311   opcode(0x0B);
9312   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9313   ins_pipe(ialu_reg_mem);
9314 %}
9315 
9316 // Or Memory with Register
9317 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9318 %{
9319   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9320   effect(KILL cr);
9321 
9322   ins_cost(150);
9323   format %{ "orq     $dst, $src\t# long" %}
9324   opcode(0x09); /* Opcode 09 /r */
9325   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9326   ins_pipe(ialu_mem_reg);
9327 %}
9328 
9329 // Or Memory with Immediate
9330 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9331 %{
9332   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9333   effect(KILL cr);
9334 
9335   ins_cost(125);
9336   format %{ "orq     $dst, $src\t# long" %}
9337   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9338   ins_encode(REX_mem_wide(dst), OpcSE(src),
9339              RM_opc_mem(secondary, dst), Con8or32(src));
9340   ins_pipe(ialu_mem_imm);
9341 %}
9342 
9343 // Xor Instructions
9344 // Xor Register with Register
9345 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9346 %{
9347   match(Set dst (XorL dst src));
9348   effect(KILL cr);
9349 
9350   format %{ "xorq    $dst, $src\t# long" %}
9351   opcode(0x33);
9352   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9353   ins_pipe(ialu_reg_reg);
9354 %}
9355 
9356 // Xor Register with Immediate -1
9357 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9358   match(Set dst (XorL dst imm));
9359 
9360   format %{ "notq   $dst" %}
9361   ins_encode %{
9362      __ notq($dst$$Register);
9363   %}
9364   ins_pipe(ialu_reg);
9365 %}
9366 
9367 // Xor Register with Immediate
9368 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9369 %{
9370   match(Set dst (XorL dst src));
9371   effect(KILL cr);
9372 
9373   format %{ "xorq    $dst, $src\t# long" %}
9374   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9375   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9376   ins_pipe(ialu_reg);
9377 %}
9378 
9379 // Xor Register with Memory
9380 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9381 %{
9382   match(Set dst (XorL dst (LoadL src)));
9383   effect(KILL cr);
9384 
9385   ins_cost(125);
9386   format %{ "xorq    $dst, $src\t# long" %}
9387   opcode(0x33);
9388   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9389   ins_pipe(ialu_reg_mem);
9390 %}
9391 
9392 // Xor Memory with Register
9393 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9394 %{
9395   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9396   effect(KILL cr);
9397 
9398   ins_cost(150);
9399   format %{ "xorq    $dst, $src\t# long" %}
9400   opcode(0x31); /* Opcode 31 /r */
9401   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9402   ins_pipe(ialu_mem_reg);
9403 %}
9404 
9405 // Xor Memory with Immediate
9406 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9407 %{
9408   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9409   effect(KILL cr);
9410 
9411   ins_cost(125);
9412   format %{ "xorq    $dst, $src\t# long" %}
9413   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9414   ins_encode(REX_mem_wide(dst), OpcSE(src),
9415              RM_opc_mem(secondary, dst), Con8or32(src));
9416   ins_pipe(ialu_mem_imm);
9417 %}
9418 
9419 // Convert Int to Boolean
9420 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9421 %{
9422   match(Set dst (Conv2B src));
9423   effect(KILL cr);
9424 
9425   format %{ "testl   $src, $src\t# ci2b\n\t"
9426             "setnz   $dst\n\t"
9427             "movzbl  $dst, $dst" %}
9428   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9429              setNZ_reg(dst),
9430              REX_reg_breg(dst, dst), // movzbl
9431              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9432   ins_pipe(pipe_slow); // XXX
9433 %}
9434 
9435 // Convert Pointer to Boolean
9436 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9437 %{
9438   match(Set dst (Conv2B src));
9439   effect(KILL cr);
9440 
9441   format %{ "testq   $src, $src\t# cp2b\n\t"
9442             "setnz   $dst\n\t"
9443             "movzbl  $dst, $dst" %}
9444   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9445              setNZ_reg(dst),
9446              REX_reg_breg(dst, dst), // movzbl
9447              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9448   ins_pipe(pipe_slow); // XXX
9449 %}
9450 
9451 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9452 %{
9453   match(Set dst (CmpLTMask p q));
9454   effect(KILL cr);
9455 
9456   ins_cost(400);
9457   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9458             "setlt   $dst\n\t"
9459             "movzbl  $dst, $dst\n\t"
9460             "negl    $dst" %}
9461   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9462              setLT_reg(dst),
9463              REX_reg_breg(dst, dst), // movzbl
9464              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9465              neg_reg(dst));
9466   ins_pipe(pipe_slow);
9467 %}
9468 
9469 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9470 %{
9471   match(Set dst (CmpLTMask dst zero));
9472   effect(KILL cr);
9473 
9474   ins_cost(100);
9475   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9476   ins_encode %{
9477   __ sarl($dst$$Register, 31);
9478   %}
9479   ins_pipe(ialu_reg);
9480 %}
9481 
9482 /* Better to save a register than avoid a branch */
9483 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9484 %{
9485   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9486   effect(KILL cr);
9487   ins_cost(300);
9488   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9489             "jge    done\n\t"
9490             "addl   $p,$y\n"
9491             "done:  " %}
9492   ins_encode %{
9493     Register Rp = $p$$Register;
9494     Register Rq = $q$$Register;
9495     Register Ry = $y$$Register;
9496     Label done;
9497     __ subl(Rp, Rq);
9498     __ jccb(Assembler::greaterEqual, done);
9499     __ addl(Rp, Ry);
9500     __ bind(done);
9501   %}
9502   ins_pipe(pipe_cmplt);
9503 %}
9504 
9505 /* Better to save a register than avoid a branch */
9506 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9507 %{
9508   match(Set y (AndI (CmpLTMask p q) y));
9509   effect(KILL cr);
9510 
9511   ins_cost(300);
9512 
9513   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9514             "jlt      done\n\t"
9515             "xorl     $y, $y\n"
9516             "done:  " %}
9517   ins_encode %{
9518     Register Rp = $p$$Register;
9519     Register Rq = $q$$Register;
9520     Register Ry = $y$$Register;
9521     Label done;
9522     __ cmpl(Rp, Rq);
9523     __ jccb(Assembler::less, done);
9524     __ xorl(Ry, Ry);
9525     __ bind(done);
9526   %}
9527   ins_pipe(pipe_cmplt);
9528 %}
9529 
9530 
9531 //---------- FP Instructions------------------------------------------------
9532 
9533 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9534 %{
9535   match(Set cr (CmpF src1 src2));
9536 
9537   ins_cost(145);
9538   format %{ "ucomiss $src1, $src2\n\t"
9539             "jnp,s   exit\n\t"
9540             "pushfq\t# saw NaN, set CF\n\t"
9541             "andq    [rsp], #0xffffff2b\n\t"
9542             "popfq\n"
9543     "exit:" %}
9544   ins_encode %{
9545     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9546     emit_cmpfp_fixup(_masm);
9547   %}
9548   ins_pipe(pipe_slow);
9549 %}
9550 
9551 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9552   match(Set cr (CmpF src1 src2));
9553 
9554   ins_cost(100);
9555   format %{ "ucomiss $src1, $src2" %}
9556   ins_encode %{
9557     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9558   %}
9559   ins_pipe(pipe_slow);
9560 %}
9561 
9562 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9563 %{
9564   match(Set cr (CmpF src1 (LoadF src2)));
9565 
9566   ins_cost(145);
9567   format %{ "ucomiss $src1, $src2\n\t"
9568             "jnp,s   exit\n\t"
9569             "pushfq\t# saw NaN, set CF\n\t"
9570             "andq    [rsp], #0xffffff2b\n\t"
9571             "popfq\n"
9572     "exit:" %}
9573   ins_encode %{
9574     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9575     emit_cmpfp_fixup(_masm);
9576   %}
9577   ins_pipe(pipe_slow);
9578 %}
9579 
9580 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9581   match(Set cr (CmpF src1 (LoadF src2)));
9582 
9583   ins_cost(100);
9584   format %{ "ucomiss $src1, $src2" %}
9585   ins_encode %{
9586     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9587   %}
9588   ins_pipe(pipe_slow);
9589 %}
9590 
9591 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9592   match(Set cr (CmpF src con));
9593 
9594   ins_cost(145);
9595   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9596             "jnp,s   exit\n\t"
9597             "pushfq\t# saw NaN, set CF\n\t"
9598             "andq    [rsp], #0xffffff2b\n\t"
9599             "popfq\n"
9600     "exit:" %}
9601   ins_encode %{
9602     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9603     emit_cmpfp_fixup(_masm);
9604   %}
9605   ins_pipe(pipe_slow);
9606 %}
9607 
9608 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9609   match(Set cr (CmpF src con));
9610   ins_cost(100);
9611   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9612   ins_encode %{
9613     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9614   %}
9615   ins_pipe(pipe_slow);
9616 %}
9617 
9618 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9619 %{
9620   match(Set cr (CmpD src1 src2));
9621 
9622   ins_cost(145);
9623   format %{ "ucomisd $src1, $src2\n\t"
9624             "jnp,s   exit\n\t"
9625             "pushfq\t# saw NaN, set CF\n\t"
9626             "andq    [rsp], #0xffffff2b\n\t"
9627             "popfq\n"
9628     "exit:" %}
9629   ins_encode %{
9630     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9631     emit_cmpfp_fixup(_masm);
9632   %}
9633   ins_pipe(pipe_slow);
9634 %}
9635 
9636 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9637   match(Set cr (CmpD src1 src2));
9638 
9639   ins_cost(100);
9640   format %{ "ucomisd $src1, $src2 test" %}
9641   ins_encode %{
9642     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9643   %}
9644   ins_pipe(pipe_slow);
9645 %}
9646 
9647 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9648 %{
9649   match(Set cr (CmpD src1 (LoadD src2)));
9650 
9651   ins_cost(145);
9652   format %{ "ucomisd $src1, $src2\n\t"
9653             "jnp,s   exit\n\t"
9654             "pushfq\t# saw NaN, set CF\n\t"
9655             "andq    [rsp], #0xffffff2b\n\t"
9656             "popfq\n"
9657     "exit:" %}
9658   ins_encode %{
9659     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9660     emit_cmpfp_fixup(_masm);
9661   %}
9662   ins_pipe(pipe_slow);
9663 %}
9664 
9665 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9666   match(Set cr (CmpD src1 (LoadD src2)));
9667 
9668   ins_cost(100);
9669   format %{ "ucomisd $src1, $src2" %}
9670   ins_encode %{
9671     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9672   %}
9673   ins_pipe(pipe_slow);
9674 %}
9675 
9676 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9677   match(Set cr (CmpD src con));
9678 
9679   ins_cost(145);
9680   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9681             "jnp,s   exit\n\t"
9682             "pushfq\t# saw NaN, set CF\n\t"
9683             "andq    [rsp], #0xffffff2b\n\t"
9684             "popfq\n"
9685     "exit:" %}
9686   ins_encode %{
9687     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9688     emit_cmpfp_fixup(_masm);
9689   %}
9690   ins_pipe(pipe_slow);
9691 %}
9692 
9693 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9694   match(Set cr (CmpD src con));
9695   ins_cost(100);
9696   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9697   ins_encode %{
9698     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9699   %}
9700   ins_pipe(pipe_slow);
9701 %}
9702 
9703 // Compare into -1,0,1
9704 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9705 %{
9706   match(Set dst (CmpF3 src1 src2));
9707   effect(KILL cr);
9708 
9709   ins_cost(275);
9710   format %{ "ucomiss $src1, $src2\n\t"
9711             "movl    $dst, #-1\n\t"
9712             "jp,s    done\n\t"
9713             "jb,s    done\n\t"
9714             "setne   $dst\n\t"
9715             "movzbl  $dst, $dst\n"
9716     "done:" %}
9717   ins_encode %{
9718     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9719     emit_cmpfp3(_masm, $dst$$Register);
9720   %}
9721   ins_pipe(pipe_slow);
9722 %}
9723 
9724 // Compare into -1,0,1
9725 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9726 %{
9727   match(Set dst (CmpF3 src1 (LoadF src2)));
9728   effect(KILL cr);
9729 
9730   ins_cost(275);
9731   format %{ "ucomiss $src1, $src2\n\t"
9732             "movl    $dst, #-1\n\t"
9733             "jp,s    done\n\t"
9734             "jb,s    done\n\t"
9735             "setne   $dst\n\t"
9736             "movzbl  $dst, $dst\n"
9737     "done:" %}
9738   ins_encode %{
9739     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9740     emit_cmpfp3(_masm, $dst$$Register);
9741   %}
9742   ins_pipe(pipe_slow);
9743 %}
9744 
9745 // Compare into -1,0,1
9746 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9747   match(Set dst (CmpF3 src con));
9748   effect(KILL cr);
9749 
9750   ins_cost(275);
9751   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9752             "movl    $dst, #-1\n\t"
9753             "jp,s    done\n\t"
9754             "jb,s    done\n\t"
9755             "setne   $dst\n\t"
9756             "movzbl  $dst, $dst\n"
9757     "done:" %}
9758   ins_encode %{
9759     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9760     emit_cmpfp3(_masm, $dst$$Register);
9761   %}
9762   ins_pipe(pipe_slow);
9763 %}
9764 
9765 // Compare into -1,0,1
9766 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9767 %{
9768   match(Set dst (CmpD3 src1 src2));
9769   effect(KILL cr);
9770 
9771   ins_cost(275);
9772   format %{ "ucomisd $src1, $src2\n\t"
9773             "movl    $dst, #-1\n\t"
9774             "jp,s    done\n\t"
9775             "jb,s    done\n\t"
9776             "setne   $dst\n\t"
9777             "movzbl  $dst, $dst\n"
9778     "done:" %}
9779   ins_encode %{
9780     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9781     emit_cmpfp3(_masm, $dst$$Register);
9782   %}
9783   ins_pipe(pipe_slow);
9784 %}
9785 
9786 // Compare into -1,0,1
9787 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9788 %{
9789   match(Set dst (CmpD3 src1 (LoadD src2)));
9790   effect(KILL cr);
9791 
9792   ins_cost(275);
9793   format %{ "ucomisd $src1, $src2\n\t"
9794             "movl    $dst, #-1\n\t"
9795             "jp,s    done\n\t"
9796             "jb,s    done\n\t"
9797             "setne   $dst\n\t"
9798             "movzbl  $dst, $dst\n"
9799     "done:" %}
9800   ins_encode %{
9801     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9802     emit_cmpfp3(_masm, $dst$$Register);
9803   %}
9804   ins_pipe(pipe_slow);
9805 %}
9806 
9807 // Compare into -1,0,1
9808 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9809   match(Set dst (CmpD3 src con));
9810   effect(KILL cr);
9811 
9812   ins_cost(275);
9813   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9814             "movl    $dst, #-1\n\t"
9815             "jp,s    done\n\t"
9816             "jb,s    done\n\t"
9817             "setne   $dst\n\t"
9818             "movzbl  $dst, $dst\n"
9819     "done:" %}
9820   ins_encode %{
9821     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9822     emit_cmpfp3(_masm, $dst$$Register);
9823   %}
9824   ins_pipe(pipe_slow);
9825 %}
9826 
9827 // -----------Trig and Trancendental Instructions------------------------------
9828 instruct cosD_reg(regD dst) %{
9829   match(Set dst (CosD dst));
9830 
9831   format %{ "dcos   $dst\n\t" %}
9832   opcode(0xD9, 0xFF);
9833   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9834   ins_pipe( pipe_slow );
9835 %}
9836 
9837 instruct sinD_reg(regD dst) %{
9838   match(Set dst (SinD dst));
9839 
9840   format %{ "dsin   $dst\n\t" %}
9841   opcode(0xD9, 0xFE);
9842   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9843   ins_pipe( pipe_slow );
9844 %}
9845 
9846 instruct tanD_reg(regD dst) %{
9847   match(Set dst (TanD dst));
9848 
9849   format %{ "dtan   $dst\n\t" %}
9850   ins_encode( Push_SrcXD(dst),
9851               Opcode(0xD9), Opcode(0xF2),   //fptan
9852               Opcode(0xDD), Opcode(0xD8),   //fstp st
9853               Push_ResultXD(dst) );
9854   ins_pipe( pipe_slow );
9855 %}
9856 
9857 instruct log10D_reg(regD dst) %{
9858   // The source and result Double operands in XMM registers
9859   match(Set dst (Log10D dst));
9860   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9861   // fyl2x        ; compute log_10(2) * log_2(x)
9862   format %{ "fldlg2\t\t\t#Log10\n\t"
9863             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9864          %}
9865    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9866               Push_SrcXD(dst),
9867               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9868               Push_ResultXD(dst));
9869 
9870   ins_pipe( pipe_slow );
9871 %}
9872 
9873 instruct logD_reg(regD dst) %{
9874   // The source and result Double operands in XMM registers
9875   match(Set dst (LogD dst));
9876   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9877   // fyl2x        ; compute log_e(2) * log_2(x)
9878   format %{ "fldln2\t\t\t#Log_e\n\t"
9879             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9880          %}
9881   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9882               Push_SrcXD(dst),
9883               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9884               Push_ResultXD(dst));
9885   ins_pipe( pipe_slow );
9886 %}
9887 
9888 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9889   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9890   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9891   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9892   ins_encode %{
9893     __ subptr(rsp, 8);
9894     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9895     __ fld_d(Address(rsp, 0));
9896     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9897     __ fld_d(Address(rsp, 0));
9898     __ fast_pow();
9899     __ fstp_d(Address(rsp, 0));
9900     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9901     __ addptr(rsp, 8);
9902   %}
9903   ins_pipe( pipe_slow );
9904 %}
9905 
9906 //----------Arithmetic Conversion Instructions---------------------------------
9907 
9908 instruct roundFloat_nop(regF dst)
9909 %{
9910   match(Set dst (RoundFloat dst));
9911 
9912   ins_cost(0);
9913   ins_encode();
9914   ins_pipe(empty);
9915 %}
9916 
9917 instruct roundDouble_nop(regD dst)
9918 %{
9919   match(Set dst (RoundDouble dst));
9920 
9921   ins_cost(0);
9922   ins_encode();
9923   ins_pipe(empty);
9924 %}
9925 
9926 instruct convF2D_reg_reg(regD dst, regF src)
9927 %{
9928   match(Set dst (ConvF2D src));
9929 
9930   format %{ "cvtss2sd $dst, $src" %}
9931   ins_encode %{
9932     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9933   %}
9934   ins_pipe(pipe_slow); // XXX
9935 %}
9936 
9937 instruct convF2D_reg_mem(regD dst, memory src)
9938 %{
9939   match(Set dst (ConvF2D (LoadF src)));
9940 
9941   format %{ "cvtss2sd $dst, $src" %}
9942   ins_encode %{
9943     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9944   %}
9945   ins_pipe(pipe_slow); // XXX
9946 %}
9947 
9948 instruct convD2F_reg_reg(regF dst, regD src)
9949 %{
9950   match(Set dst (ConvD2F src));
9951 
9952   format %{ "cvtsd2ss $dst, $src" %}
9953   ins_encode %{
9954     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9955   %}
9956   ins_pipe(pipe_slow); // XXX
9957 %}
9958 
9959 instruct convD2F_reg_mem(regF dst, memory src)
9960 %{
9961   match(Set dst (ConvD2F (LoadD src)));
9962 
9963   format %{ "cvtsd2ss $dst, $src" %}
9964   ins_encode %{
9965     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9966   %}
9967   ins_pipe(pipe_slow); // XXX
9968 %}
9969 
9970 // XXX do mem variants
9971 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9972 %{
9973   match(Set dst (ConvF2I src));
9974   effect(KILL cr);
9975 
9976   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9977             "cmpl    $dst, #0x80000000\n\t"
9978             "jne,s   done\n\t"
9979             "subq    rsp, #8\n\t"
9980             "movss   [rsp], $src\n\t"
9981             "call    f2i_fixup\n\t"
9982             "popq    $dst\n"
9983     "done:   "%}
9984   ins_encode %{
9985     Label done;
9986     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9987     __ cmpl($dst$$Register, 0x80000000);
9988     __ jccb(Assembler::notEqual, done);
9989     __ subptr(rsp, 8);
9990     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9991     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9992     __ pop($dst$$Register);
9993     __ bind(done);
9994   %}
9995   ins_pipe(pipe_slow);
9996 %}
9997 
9998 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9999 %{
10000   match(Set dst (ConvF2L src));
10001   effect(KILL cr);
10002 
10003   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10004             "cmpq    $dst, [0x8000000000000000]\n\t"
10005             "jne,s   done\n\t"
10006             "subq    rsp, #8\n\t"
10007             "movss   [rsp], $src\n\t"
10008             "call    f2l_fixup\n\t"
10009             "popq    $dst\n"
10010     "done:   "%}
10011   ins_encode %{
10012     Label done;
10013     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10014     __ cmp64($dst$$Register,
10015              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10016     __ jccb(Assembler::notEqual, done);
10017     __ subptr(rsp, 8);
10018     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10019     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10020     __ pop($dst$$Register);
10021     __ bind(done);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10027 %{
10028   match(Set dst (ConvD2I src));
10029   effect(KILL cr);
10030 
10031   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10032             "cmpl    $dst, #0x80000000\n\t"
10033             "jne,s   done\n\t"
10034             "subq    rsp, #8\n\t"
10035             "movsd   [rsp], $src\n\t"
10036             "call    d2i_fixup\n\t"
10037             "popq    $dst\n"
10038     "done:   "%}
10039   ins_encode %{
10040     Label done;
10041     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10042     __ cmpl($dst$$Register, 0x80000000);
10043     __ jccb(Assembler::notEqual, done);
10044     __ subptr(rsp, 8);
10045     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10046     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10047     __ pop($dst$$Register);
10048     __ bind(done);
10049   %}
10050   ins_pipe(pipe_slow);
10051 %}
10052 
10053 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10054 %{
10055   match(Set dst (ConvD2L src));
10056   effect(KILL cr);
10057 
10058   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10059             "cmpq    $dst, [0x8000000000000000]\n\t"
10060             "jne,s   done\n\t"
10061             "subq    rsp, #8\n\t"
10062             "movsd   [rsp], $src\n\t"
10063             "call    d2l_fixup\n\t"
10064             "popq    $dst\n"
10065     "done:   "%}
10066   ins_encode %{
10067     Label done;
10068     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10069     __ cmp64($dst$$Register,
10070              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10071     __ jccb(Assembler::notEqual, done);
10072     __ subptr(rsp, 8);
10073     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10074     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10075     __ pop($dst$$Register);
10076     __ bind(done);
10077   %}
10078   ins_pipe(pipe_slow);
10079 %}
10080 
10081 instruct convI2F_reg_reg(regF dst, rRegI src)
10082 %{
10083   predicate(!UseXmmI2F);
10084   match(Set dst (ConvI2F src));
10085 
10086   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10087   ins_encode %{
10088     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10089   %}
10090   ins_pipe(pipe_slow); // XXX
10091 %}
10092 
10093 instruct convI2F_reg_mem(regF dst, memory src)
10094 %{
10095   match(Set dst (ConvI2F (LoadI src)));
10096 
10097   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10098   ins_encode %{
10099     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10100   %}
10101   ins_pipe(pipe_slow); // XXX
10102 %}
10103 
10104 instruct convI2D_reg_reg(regD dst, rRegI src)
10105 %{
10106   predicate(!UseXmmI2D);
10107   match(Set dst (ConvI2D src));
10108 
10109   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10110   ins_encode %{
10111     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10112   %}
10113   ins_pipe(pipe_slow); // XXX
10114 %}
10115 
10116 instruct convI2D_reg_mem(regD dst, memory src)
10117 %{
10118   match(Set dst (ConvI2D (LoadI src)));
10119 
10120   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10121   ins_encode %{
10122     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10123   %}
10124   ins_pipe(pipe_slow); // XXX
10125 %}
10126 
10127 instruct convXI2F_reg(regF dst, rRegI src)
10128 %{
10129   predicate(UseXmmI2F);
10130   match(Set dst (ConvI2F src));
10131 
10132   format %{ "movdl $dst, $src\n\t"
10133             "cvtdq2psl $dst, $dst\t# i2f" %}
10134   ins_encode %{
10135     __ movdl($dst$$XMMRegister, $src$$Register);
10136     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10137   %}
10138   ins_pipe(pipe_slow); // XXX
10139 %}
10140 
10141 instruct convXI2D_reg(regD dst, rRegI src)
10142 %{
10143   predicate(UseXmmI2D);
10144   match(Set dst (ConvI2D src));
10145 
10146   format %{ "movdl $dst, $src\n\t"
10147             "cvtdq2pdl $dst, $dst\t# i2d" %}
10148   ins_encode %{
10149     __ movdl($dst$$XMMRegister, $src$$Register);
10150     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10151   %}
10152   ins_pipe(pipe_slow); // XXX
10153 %}
10154 
10155 instruct convL2F_reg_reg(regF dst, rRegL src)
10156 %{
10157   match(Set dst (ConvL2F src));
10158 
10159   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10160   ins_encode %{
10161     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10162   %}
10163   ins_pipe(pipe_slow); // XXX
10164 %}
10165 
10166 instruct convL2F_reg_mem(regF dst, memory src)
10167 %{
10168   match(Set dst (ConvL2F (LoadL src)));
10169 
10170   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10171   ins_encode %{
10172     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10173   %}
10174   ins_pipe(pipe_slow); // XXX
10175 %}
10176 
10177 instruct convL2D_reg_reg(regD dst, rRegL src)
10178 %{
10179   match(Set dst (ConvL2D src));
10180 
10181   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10182   ins_encode %{
10183     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10184   %}
10185   ins_pipe(pipe_slow); // XXX
10186 %}
10187 
10188 instruct convL2D_reg_mem(regD dst, memory src)
10189 %{
10190   match(Set dst (ConvL2D (LoadL src)));
10191 
10192   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10193   ins_encode %{
10194     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10195   %}
10196   ins_pipe(pipe_slow); // XXX
10197 %}
10198 
10199 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10200 %{
10201   match(Set dst (ConvI2L src));
10202 
10203   ins_cost(125);
10204   format %{ "movslq  $dst, $src\t# i2l" %}
10205   ins_encode %{
10206     __ movslq($dst$$Register, $src$$Register);
10207   %}
10208   ins_pipe(ialu_reg_reg);
10209 %}
10210 
10211 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10212 // %{
10213 //   match(Set dst (ConvI2L src));
10214 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10215 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10216 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10217 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10218 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10219 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10220 
10221 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10222 //   ins_encode(enc_copy(dst, src));
10223 // //   opcode(0x63); // needs REX.W
10224 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10225 //   ins_pipe(ialu_reg_reg);
10226 // %}
10227 
10228 // Zero-extend convert int to long
10229 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10230 %{
10231   match(Set dst (AndL (ConvI2L src) mask));
10232 
10233   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10234   ins_encode %{
10235     if ($dst$$reg != $src$$reg) {
10236       __ movl($dst$$Register, $src$$Register);
10237     }
10238   %}
10239   ins_pipe(ialu_reg_reg);
10240 %}
10241 
10242 // Zero-extend convert int to long
10243 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10244 %{
10245   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10246 
10247   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10248   ins_encode %{
10249     __ movl($dst$$Register, $src$$Address);
10250   %}
10251   ins_pipe(ialu_reg_mem);
10252 %}
10253 
10254 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10255 %{
10256   match(Set dst (AndL src mask));
10257 
10258   format %{ "movl    $dst, $src\t# zero-extend long" %}
10259   ins_encode %{
10260     __ movl($dst$$Register, $src$$Register);
10261   %}
10262   ins_pipe(ialu_reg_reg);
10263 %}
10264 
10265 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10266 %{
10267   match(Set dst (ConvL2I src));
10268 
10269   format %{ "movl    $dst, $src\t# l2i" %}
10270   ins_encode %{
10271     __ movl($dst$$Register, $src$$Register);
10272   %}
10273   ins_pipe(ialu_reg_reg);
10274 %}
10275 
10276 
10277 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10278   match(Set dst (MoveF2I src));
10279   effect(DEF dst, USE src);
10280 
10281   ins_cost(125);
10282   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10283   ins_encode %{
10284     __ movl($dst$$Register, Address(rsp, $src$$disp));
10285   %}
10286   ins_pipe(ialu_reg_mem);
10287 %}
10288 
10289 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10290   match(Set dst (MoveI2F src));
10291   effect(DEF dst, USE src);
10292 
10293   ins_cost(125);
10294   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10295   ins_encode %{
10296     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10297   %}
10298   ins_pipe(pipe_slow);
10299 %}
10300 
10301 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10302   match(Set dst (MoveD2L src));
10303   effect(DEF dst, USE src);
10304 
10305   ins_cost(125);
10306   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10307   ins_encode %{
10308     __ movq($dst$$Register, Address(rsp, $src$$disp));
10309   %}
10310   ins_pipe(ialu_reg_mem);
10311 %}
10312 
10313 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10314   predicate(!UseXmmLoadAndClearUpper);
10315   match(Set dst (MoveL2D src));
10316   effect(DEF dst, USE src);
10317 
10318   ins_cost(125);
10319   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10320   ins_encode %{
10321     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10322   %}
10323   ins_pipe(pipe_slow);
10324 %}
10325 
10326 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10327   predicate(UseXmmLoadAndClearUpper);
10328   match(Set dst (MoveL2D src));
10329   effect(DEF dst, USE src);
10330 
10331   ins_cost(125);
10332   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10333   ins_encode %{
10334     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10335   %}
10336   ins_pipe(pipe_slow);
10337 %}
10338 
10339 
10340 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10341   match(Set dst (MoveF2I src));
10342   effect(DEF dst, USE src);
10343 
10344   ins_cost(95); // XXX
10345   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10346   ins_encode %{
10347     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10348   %}
10349   ins_pipe(pipe_slow);
10350 %}
10351 
10352 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10353   match(Set dst (MoveI2F src));
10354   effect(DEF dst, USE src);
10355 
10356   ins_cost(100);
10357   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10358   ins_encode %{
10359     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10360   %}
10361   ins_pipe( ialu_mem_reg );
10362 %}
10363 
10364 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10365   match(Set dst (MoveD2L src));
10366   effect(DEF dst, USE src);
10367 
10368   ins_cost(95); // XXX
10369   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10370   ins_encode %{
10371     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10372   %}
10373   ins_pipe(pipe_slow);
10374 %}
10375 
10376 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10377   match(Set dst (MoveL2D src));
10378   effect(DEF dst, USE src);
10379 
10380   ins_cost(100);
10381   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10382   ins_encode %{
10383     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10384   %}
10385   ins_pipe(ialu_mem_reg);
10386 %}
10387 
10388 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10389   match(Set dst (MoveF2I src));
10390   effect(DEF dst, USE src);
10391   ins_cost(85);
10392   format %{ "movd    $dst,$src\t# MoveF2I" %}
10393   ins_encode %{
10394     __ movdl($dst$$Register, $src$$XMMRegister);
10395   %}
10396   ins_pipe( pipe_slow );
10397 %}
10398 
10399 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10400   match(Set dst (MoveD2L src));
10401   effect(DEF dst, USE src);
10402   ins_cost(85);
10403   format %{ "movd    $dst,$src\t# MoveD2L" %}
10404   ins_encode %{
10405     __ movdq($dst$$Register, $src$$XMMRegister);
10406   %}
10407   ins_pipe( pipe_slow );
10408 %}
10409 
10410 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10411   match(Set dst (MoveI2F src));
10412   effect(DEF dst, USE src);
10413   ins_cost(100);
10414   format %{ "movd    $dst,$src\t# MoveI2F" %}
10415   ins_encode %{
10416     __ movdl($dst$$XMMRegister, $src$$Register);
10417   %}
10418   ins_pipe( pipe_slow );
10419 %}
10420 
10421 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10422   match(Set dst (MoveL2D src));
10423   effect(DEF dst, USE src);
10424   ins_cost(100);
10425   format %{ "movd    $dst,$src\t# MoveL2D" %}
10426   ins_encode %{
10427      __ movdq($dst$$XMMRegister, $src$$Register);
10428   %}
10429   ins_pipe( pipe_slow );
10430 %}
10431 
10432 
10433 // =======================================================================
10434 // fast clearing of an array
10435 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10436                   rFlagsReg cr)
10437 %{
10438   predicate(!UseFastStosb);
10439   match(Set dummy (ClearArray cnt base));
10440   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10441 
10442   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10443             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10444   ins_encode %{
10445     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10446   %}
10447   ins_pipe(pipe_slow);
10448 %}
10449 
10450 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10451                         rFlagsReg cr)
10452 %{
10453   predicate(UseFastStosb);
10454   match(Set dummy (ClearArray cnt base));
10455   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10456   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10457             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10458             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10459   ins_encode %{
10460     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10461   %}
10462   ins_pipe( pipe_slow );
10463 %}
10464 
10465 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10466                          rax_RegI result, regD tmp1, rFlagsReg cr)
10467 %{
10468   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10469   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10470   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10471 
10472   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10473   ins_encode %{
10474     __ string_compare($str1$$Register, $str2$$Register,
10475                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10476                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10477   %}
10478   ins_pipe( pipe_slow );
10479 %}
10480 
10481 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10482                          rax_RegI result, regD tmp1, rFlagsReg cr)
10483 %{
10484   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10485   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10486   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10487 
10488   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10489   ins_encode %{
10490     __ string_compare($str1$$Register, $str2$$Register,
10491                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10492                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10493   %}
10494   ins_pipe( pipe_slow );
10495 %}
10496 
10497 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10498                           rax_RegI result, regD tmp1, rFlagsReg cr)
10499 %{
10500   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10501   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10502   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10503 
10504   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10505   ins_encode %{
10506     __ string_compare($str1$$Register, $str2$$Register,
10507                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10508                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10509   %}
10510   ins_pipe( pipe_slow );
10511 %}
10512 
10513 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10514                           rax_RegI result, regD tmp1, rFlagsReg cr)
10515 %{
10516   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10517   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10518   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10519 
10520   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10521   ins_encode %{
10522     __ string_compare($str2$$Register, $str1$$Register,
10523                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10524                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10525   %}
10526   ins_pipe( pipe_slow );
10527 %}
10528 
10529 // fast search of substring with known size.
10530 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10531                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10532 %{
10533   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10534   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10535   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10536 
10537   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10538   ins_encode %{
10539     int icnt2 = (int)$int_cnt2$$constant;
10540     if (icnt2 >= 16) {
10541       // IndexOf for constant substrings with size >= 16 elements
10542       // which don't need to be loaded through stack.
10543       __ string_indexofC8($str1$$Register, $str2$$Register,
10544                           $cnt1$$Register, $cnt2$$Register,
10545                           icnt2, $result$$Register,
10546                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10547     } else {
10548       // Small strings are loaded through stack if they cross page boundary.
10549       __ string_indexof($str1$$Register, $str2$$Register,
10550                         $cnt1$$Register, $cnt2$$Register,
10551                         icnt2, $result$$Register,
10552                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10553     }
10554   %}
10555   ins_pipe( pipe_slow );
10556 %}
10557 
10558 // fast search of substring with known size.
10559 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10560                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10561 %{
10562   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10563   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10564   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10565 
10566   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10567   ins_encode %{
10568     int icnt2 = (int)$int_cnt2$$constant;
10569     if (icnt2 >= 8) {
10570       // IndexOf for constant substrings with size >= 8 elements
10571       // which don't need to be loaded through stack.
10572       __ string_indexofC8($str1$$Register, $str2$$Register,
10573                           $cnt1$$Register, $cnt2$$Register,
10574                           icnt2, $result$$Register,
10575                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10576     } else {
10577       // Small strings are loaded through stack if they cross page boundary.
10578       __ string_indexof($str1$$Register, $str2$$Register,
10579                         $cnt1$$Register, $cnt2$$Register,
10580                         icnt2, $result$$Register,
10581                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10582     }
10583   %}
10584   ins_pipe( pipe_slow );
10585 %}
10586 
10587 // fast search of substring with known size.
10588 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10589                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10590 %{
10591   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10592   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10593   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10594 
10595   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10596   ins_encode %{
10597     int icnt2 = (int)$int_cnt2$$constant;
10598     if (icnt2 >= 8) {
10599       // IndexOf for constant substrings with size >= 8 elements
10600       // which don't need to be loaded through stack.
10601       __ string_indexofC8($str1$$Register, $str2$$Register,
10602                           $cnt1$$Register, $cnt2$$Register,
10603                           icnt2, $result$$Register,
10604                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10605     } else {
10606       // Small strings are loaded through stack if they cross page boundary.
10607       __ string_indexof($str1$$Register, $str2$$Register,
10608                         $cnt1$$Register, $cnt2$$Register,
10609                         icnt2, $result$$Register,
10610                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10611     }
10612   %}
10613   ins_pipe( pipe_slow );
10614 %}
10615 
10616 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10617                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10618 %{
10619   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10620   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10621   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10622 
10623   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10624   ins_encode %{
10625     __ string_indexof($str1$$Register, $str2$$Register,
10626                       $cnt1$$Register, $cnt2$$Register,
10627                       (-1), $result$$Register,
10628                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10629   %}
10630   ins_pipe( pipe_slow );
10631 %}
10632 
10633 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10634                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10635 %{
10636   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10637   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10638   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10639 
10640   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10641   ins_encode %{
10642     __ string_indexof($str1$$Register, $str2$$Register,
10643                       $cnt1$$Register, $cnt2$$Register,
10644                       (-1), $result$$Register,
10645                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10646   %}
10647   ins_pipe( pipe_slow );
10648 %}
10649 
10650 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10651                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10652 %{
10653   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10654   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10655   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10656 
10657   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10658   ins_encode %{
10659     __ string_indexof($str1$$Register, $str2$$Register,
10660                       $cnt1$$Register, $cnt2$$Register,
10661                       (-1), $result$$Register,
10662                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10663   %}
10664   ins_pipe( pipe_slow );
10665 %}
10666 
10667 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10668                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10669 %{
10670   predicate(UseSSE42Intrinsics);
10671   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10672   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10673   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10674   ins_encode %{
10675     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10676                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10677   %}
10678   ins_pipe( pipe_slow );
10679 %}
10680 
10681 // fast string equals
10682 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10683                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10684 %{
10685   match(Set result (StrEquals (Binary str1 str2) cnt));
10686   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10687 
10688   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10689   ins_encode %{
10690     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10691                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10692                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10693   %}
10694   ins_pipe( pipe_slow );
10695 %}
10696 
10697 // fast array equals
10698 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10699                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10700 %{
10701   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10702   match(Set result (AryEq ary1 ary2));
10703   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10704 
10705   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10706   ins_encode %{
10707     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10708                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10709                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10710   %}
10711   ins_pipe( pipe_slow );
10712 %}
10713 
10714 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10715                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10716 %{
10717   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10718   match(Set result (AryEq ary1 ary2));
10719   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10720 
10721   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10722   ins_encode %{
10723     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10724                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10725                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10726   %}
10727   ins_pipe( pipe_slow );
10728 %}
10729 
10730 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10731                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10732 %{
10733   match(Set result (HasNegatives ary1 len));
10734   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10735 
10736   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10737   ins_encode %{
10738     __ has_negatives($ary1$$Register, $len$$Register,
10739                      $result$$Register, $tmp3$$Register,
10740                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10741   %}
10742   ins_pipe( pipe_slow );
10743 %}
10744 
10745 // fast char[] to byte[] compression
10746 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10747                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10748   match(Set result (StrCompressedCopy src (Binary dst len)));
10749   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10750 
10751   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10752   ins_encode %{
10753     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10754                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10755                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10756   %}
10757   ins_pipe( pipe_slow );
10758 %}
10759 
10760 // fast byte[] to char[] inflation
10761 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10762                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10763   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10764   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10765 
10766   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10767   ins_encode %{
10768     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10769                           $tmp1$$XMMRegister, $tmp2$$Register);
10770   %}
10771   ins_pipe( pipe_slow );
10772 %}
10773 
10774 // encode char[] to byte[] in ISO_8859_1
10775 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10776                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10777                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10778   match(Set result (EncodeISOArray src (Binary dst len)));
10779   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10780 
10781   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10782   ins_encode %{
10783     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10784                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10785                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10786   %}
10787   ins_pipe( pipe_slow );
10788 %}
10789 
10790 //----------Overflow Math Instructions-----------------------------------------
10791 
10792 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10793 %{
10794   match(Set cr (OverflowAddI op1 op2));
10795   effect(DEF cr, USE_KILL op1, USE op2);
10796 
10797   format %{ "addl    $op1, $op2\t# overflow check int" %}
10798 
10799   ins_encode %{
10800     __ addl($op1$$Register, $op2$$Register);
10801   %}
10802   ins_pipe(ialu_reg_reg);
10803 %}
10804 
10805 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10806 %{
10807   match(Set cr (OverflowAddI op1 op2));
10808   effect(DEF cr, USE_KILL op1, USE op2);
10809 
10810   format %{ "addl    $op1, $op2\t# overflow check int" %}
10811 
10812   ins_encode %{
10813     __ addl($op1$$Register, $op2$$constant);
10814   %}
10815   ins_pipe(ialu_reg_reg);
10816 %}
10817 
10818 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10819 %{
10820   match(Set cr (OverflowAddL op1 op2));
10821   effect(DEF cr, USE_KILL op1, USE op2);
10822 
10823   format %{ "addq    $op1, $op2\t# overflow check long" %}
10824   ins_encode %{
10825     __ addq($op1$$Register, $op2$$Register);
10826   %}
10827   ins_pipe(ialu_reg_reg);
10828 %}
10829 
10830 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10831 %{
10832   match(Set cr (OverflowAddL op1 op2));
10833   effect(DEF cr, USE_KILL op1, USE op2);
10834 
10835   format %{ "addq    $op1, $op2\t# overflow check long" %}
10836   ins_encode %{
10837     __ addq($op1$$Register, $op2$$constant);
10838   %}
10839   ins_pipe(ialu_reg_reg);
10840 %}
10841 
10842 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10843 %{
10844   match(Set cr (OverflowSubI op1 op2));
10845 
10846   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10847   ins_encode %{
10848     __ cmpl($op1$$Register, $op2$$Register);
10849   %}
10850   ins_pipe(ialu_reg_reg);
10851 %}
10852 
10853 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10854 %{
10855   match(Set cr (OverflowSubI op1 op2));
10856 
10857   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10858   ins_encode %{
10859     __ cmpl($op1$$Register, $op2$$constant);
10860   %}
10861   ins_pipe(ialu_reg_reg);
10862 %}
10863 
10864 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10865 %{
10866   match(Set cr (OverflowSubL op1 op2));
10867 
10868   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10869   ins_encode %{
10870     __ cmpq($op1$$Register, $op2$$Register);
10871   %}
10872   ins_pipe(ialu_reg_reg);
10873 %}
10874 
10875 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10876 %{
10877   match(Set cr (OverflowSubL op1 op2));
10878 
10879   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10880   ins_encode %{
10881     __ cmpq($op1$$Register, $op2$$constant);
10882   %}
10883   ins_pipe(ialu_reg_reg);
10884 %}
10885 
10886 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10887 %{
10888   match(Set cr (OverflowSubI zero op2));
10889   effect(DEF cr, USE_KILL op2);
10890 
10891   format %{ "negl    $op2\t# overflow check int" %}
10892   ins_encode %{
10893     __ negl($op2$$Register);
10894   %}
10895   ins_pipe(ialu_reg_reg);
10896 %}
10897 
10898 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10899 %{
10900   match(Set cr (OverflowSubL zero op2));
10901   effect(DEF cr, USE_KILL op2);
10902 
10903   format %{ "negq    $op2\t# overflow check long" %}
10904   ins_encode %{
10905     __ negq($op2$$Register);
10906   %}
10907   ins_pipe(ialu_reg_reg);
10908 %}
10909 
10910 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10911 %{
10912   match(Set cr (OverflowMulI op1 op2));
10913   effect(DEF cr, USE_KILL op1, USE op2);
10914 
10915   format %{ "imull    $op1, $op2\t# overflow check int" %}
10916   ins_encode %{
10917     __ imull($op1$$Register, $op2$$Register);
10918   %}
10919   ins_pipe(ialu_reg_reg_alu0);
10920 %}
10921 
10922 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10923 %{
10924   match(Set cr (OverflowMulI op1 op2));
10925   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10926 
10927   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10928   ins_encode %{
10929     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10930   %}
10931   ins_pipe(ialu_reg_reg_alu0);
10932 %}
10933 
10934 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10935 %{
10936   match(Set cr (OverflowMulL op1 op2));
10937   effect(DEF cr, USE_KILL op1, USE op2);
10938 
10939   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10940   ins_encode %{
10941     __ imulq($op1$$Register, $op2$$Register);
10942   %}
10943   ins_pipe(ialu_reg_reg_alu0);
10944 %}
10945 
10946 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10947 %{
10948   match(Set cr (OverflowMulL op1 op2));
10949   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10950 
10951   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10952   ins_encode %{
10953     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10954   %}
10955   ins_pipe(ialu_reg_reg_alu0);
10956 %}
10957 
10958 
10959 //----------Control Flow Instructions------------------------------------------
10960 // Signed compare Instructions
10961 
10962 // XXX more variants!!
10963 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10964 %{
10965   match(Set cr (CmpI op1 op2));
10966   effect(DEF cr, USE op1, USE op2);
10967 
10968   format %{ "cmpl    $op1, $op2" %}
10969   opcode(0x3B);  /* Opcode 3B /r */
10970   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10971   ins_pipe(ialu_cr_reg_reg);
10972 %}
10973 
10974 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10975 %{
10976   match(Set cr (CmpI op1 op2));
10977 
10978   format %{ "cmpl    $op1, $op2" %}
10979   opcode(0x81, 0x07); /* Opcode 81 /7 */
10980   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10981   ins_pipe(ialu_cr_reg_imm);
10982 %}
10983 
10984 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10985 %{
10986   match(Set cr (CmpI op1 (LoadI op2)));
10987 
10988   ins_cost(500); // XXX
10989   format %{ "cmpl    $op1, $op2" %}
10990   opcode(0x3B); /* Opcode 3B /r */
10991   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10992   ins_pipe(ialu_cr_reg_mem);
10993 %}
10994 
10995 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10996 %{
10997   match(Set cr (CmpI src zero));
10998 
10999   format %{ "testl   $src, $src" %}
11000   opcode(0x85);
11001   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11002   ins_pipe(ialu_cr_reg_imm);
11003 %}
11004 
11005 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11006 %{
11007   match(Set cr (CmpI (AndI src con) zero));
11008 
11009   format %{ "testl   $src, $con" %}
11010   opcode(0xF7, 0x00);
11011   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11012   ins_pipe(ialu_cr_reg_imm);
11013 %}
11014 
11015 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11016 %{
11017   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11018 
11019   format %{ "testl   $src, $mem" %}
11020   opcode(0x85);
11021   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11022   ins_pipe(ialu_cr_reg_mem);
11023 %}
11024 
11025 // Unsigned compare Instructions; really, same as signed except they
11026 // produce an rFlagsRegU instead of rFlagsReg.
11027 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11028 %{
11029   match(Set cr (CmpU op1 op2));
11030 
11031   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11032   opcode(0x3B); /* Opcode 3B /r */
11033   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11034   ins_pipe(ialu_cr_reg_reg);
11035 %}
11036 
11037 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11038 %{
11039   match(Set cr (CmpU op1 op2));
11040 
11041   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11042   opcode(0x81,0x07); /* Opcode 81 /7 */
11043   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11044   ins_pipe(ialu_cr_reg_imm);
11045 %}
11046 
11047 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11048 %{
11049   match(Set cr (CmpU op1 (LoadI op2)));
11050 
11051   ins_cost(500); // XXX
11052   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11053   opcode(0x3B); /* Opcode 3B /r */
11054   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11055   ins_pipe(ialu_cr_reg_mem);
11056 %}
11057 
11058 // // // Cisc-spilled version of cmpU_rReg
11059 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11060 // //%{
11061 // //  match(Set cr (CmpU (LoadI op1) op2));
11062 // //
11063 // //  format %{ "CMPu   $op1,$op2" %}
11064 // //  ins_cost(500);
11065 // //  opcode(0x39);  /* Opcode 39 /r */
11066 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11067 // //%}
11068 
11069 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11070 %{
11071   match(Set cr (CmpU src zero));
11072 
11073   format %{ "testl  $src, $src\t# unsigned" %}
11074   opcode(0x85);
11075   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11076   ins_pipe(ialu_cr_reg_imm);
11077 %}
11078 
11079 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11080 %{
11081   match(Set cr (CmpP op1 op2));
11082 
11083   format %{ "cmpq    $op1, $op2\t# ptr" %}
11084   opcode(0x3B); /* Opcode 3B /r */
11085   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11086   ins_pipe(ialu_cr_reg_reg);
11087 %}
11088 
11089 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11090 %{
11091   match(Set cr (CmpP op1 (LoadP op2)));
11092 
11093   ins_cost(500); // XXX
11094   format %{ "cmpq    $op1, $op2\t# ptr" %}
11095   opcode(0x3B); /* Opcode 3B /r */
11096   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11097   ins_pipe(ialu_cr_reg_mem);
11098 %}
11099 
11100 // // // Cisc-spilled version of cmpP_rReg
11101 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11102 // //%{
11103 // //  match(Set cr (CmpP (LoadP op1) op2));
11104 // //
11105 // //  format %{ "CMPu   $op1,$op2" %}
11106 // //  ins_cost(500);
11107 // //  opcode(0x39);  /* Opcode 39 /r */
11108 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11109 // //%}
11110 
11111 // XXX this is generalized by compP_rReg_mem???
11112 // Compare raw pointer (used in out-of-heap check).
11113 // Only works because non-oop pointers must be raw pointers
11114 // and raw pointers have no anti-dependencies.
11115 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11116 %{
11117   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11118   match(Set cr (CmpP op1 (LoadP op2)));
11119 
11120   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11121   opcode(0x3B); /* Opcode 3B /r */
11122   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11123   ins_pipe(ialu_cr_reg_mem);
11124 %}
11125 
11126 // This will generate a signed flags result. This should be OK since
11127 // any compare to a zero should be eq/neq.
11128 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11129 %{
11130   match(Set cr (CmpP src zero));
11131 
11132   format %{ "testq   $src, $src\t# ptr" %}
11133   opcode(0x85);
11134   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11135   ins_pipe(ialu_cr_reg_imm);
11136 %}
11137 
11138 // This will generate a signed flags result. This should be OK since
11139 // any compare to a zero should be eq/neq.
11140 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11141 %{
11142   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11143   match(Set cr (CmpP (LoadP op) zero));
11144 
11145   ins_cost(500); // XXX
11146   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11147   opcode(0xF7); /* Opcode F7 /0 */
11148   ins_encode(REX_mem_wide(op),
11149              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11150   ins_pipe(ialu_cr_reg_imm);
11151 %}
11152 
11153 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11154 %{
11155   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11156   match(Set cr (CmpP (LoadP mem) zero));
11157 
11158   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11159   ins_encode %{
11160     __ cmpq(r12, $mem$$Address);
11161   %}
11162   ins_pipe(ialu_cr_reg_mem);
11163 %}
11164 
11165 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11166 %{
11167   match(Set cr (CmpN op1 op2));
11168 
11169   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11170   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11171   ins_pipe(ialu_cr_reg_reg);
11172 %}
11173 
11174 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11175 %{
11176   match(Set cr (CmpN src (LoadN mem)));
11177 
11178   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11179   ins_encode %{
11180     __ cmpl($src$$Register, $mem$$Address);
11181   %}
11182   ins_pipe(ialu_cr_reg_mem);
11183 %}
11184 
11185 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11186   match(Set cr (CmpN op1 op2));
11187 
11188   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11189   ins_encode %{
11190     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11191   %}
11192   ins_pipe(ialu_cr_reg_imm);
11193 %}
11194 
11195 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11196 %{
11197   match(Set cr (CmpN src (LoadN mem)));
11198 
11199   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11200   ins_encode %{
11201     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11202   %}
11203   ins_pipe(ialu_cr_reg_mem);
11204 %}
11205 
11206 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11207   match(Set cr (CmpN op1 op2));
11208 
11209   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11210   ins_encode %{
11211     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11212   %}
11213   ins_pipe(ialu_cr_reg_imm);
11214 %}
11215 
11216 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11217 %{
11218   match(Set cr (CmpN src (LoadNKlass mem)));
11219 
11220   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11221   ins_encode %{
11222     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11223   %}
11224   ins_pipe(ialu_cr_reg_mem);
11225 %}
11226 
11227 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11228   match(Set cr (CmpN src zero));
11229 
11230   format %{ "testl   $src, $src\t# compressed ptr" %}
11231   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11232   ins_pipe(ialu_cr_reg_imm);
11233 %}
11234 
11235 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11236 %{
11237   predicate(Universe::narrow_oop_base() != NULL);
11238   match(Set cr (CmpN (LoadN mem) zero));
11239 
11240   ins_cost(500); // XXX
11241   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11242   ins_encode %{
11243     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11244   %}
11245   ins_pipe(ialu_cr_reg_mem);
11246 %}
11247 
11248 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11249 %{
11250   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11251   match(Set cr (CmpN (LoadN mem) zero));
11252 
11253   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11254   ins_encode %{
11255     __ cmpl(r12, $mem$$Address);
11256   %}
11257   ins_pipe(ialu_cr_reg_mem);
11258 %}
11259 
11260 // Yanked all unsigned pointer compare operations.
11261 // Pointer compares are done with CmpP which is already unsigned.
11262 
11263 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11264 %{
11265   match(Set cr (CmpL op1 op2));
11266 
11267   format %{ "cmpq    $op1, $op2" %}
11268   opcode(0x3B);  /* Opcode 3B /r */
11269   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11270   ins_pipe(ialu_cr_reg_reg);
11271 %}
11272 
11273 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11274 %{
11275   match(Set cr (CmpL op1 op2));
11276 
11277   format %{ "cmpq    $op1, $op2" %}
11278   opcode(0x81, 0x07); /* Opcode 81 /7 */
11279   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11280   ins_pipe(ialu_cr_reg_imm);
11281 %}
11282 
11283 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11284 %{
11285   match(Set cr (CmpL op1 (LoadL op2)));
11286 
11287   format %{ "cmpq    $op1, $op2" %}
11288   opcode(0x3B); /* Opcode 3B /r */
11289   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11290   ins_pipe(ialu_cr_reg_mem);
11291 %}
11292 
11293 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11294 %{
11295   match(Set cr (CmpL src zero));
11296 
11297   format %{ "testq   $src, $src" %}
11298   opcode(0x85);
11299   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11300   ins_pipe(ialu_cr_reg_imm);
11301 %}
11302 
11303 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11304 %{
11305   match(Set cr (CmpL (AndL src con) zero));
11306 
11307   format %{ "testq   $src, $con\t# long" %}
11308   opcode(0xF7, 0x00);
11309   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11310   ins_pipe(ialu_cr_reg_imm);
11311 %}
11312 
11313 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11314 %{
11315   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11316 
11317   format %{ "testq   $src, $mem" %}
11318   opcode(0x85);
11319   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11320   ins_pipe(ialu_cr_reg_mem);
11321 %}
11322 
11323 // Manifest a CmpL result in an integer register.  Very painful.
11324 // This is the test to avoid.
11325 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11326 %{
11327   match(Set dst (CmpL3 src1 src2));
11328   effect(KILL flags);
11329 
11330   ins_cost(275); // XXX
11331   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11332             "movl    $dst, -1\n\t"
11333             "jl,s    done\n\t"
11334             "setne   $dst\n\t"
11335             "movzbl  $dst, $dst\n\t"
11336     "done:" %}
11337   ins_encode(cmpl3_flag(src1, src2, dst));
11338   ins_pipe(pipe_slow);
11339 %}
11340 
11341 //----------Max and Min--------------------------------------------------------
11342 // Min Instructions
11343 
11344 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11345 %{
11346   effect(USE_DEF dst, USE src, USE cr);
11347 
11348   format %{ "cmovlgt $dst, $src\t# min" %}
11349   opcode(0x0F, 0x4F);
11350   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11351   ins_pipe(pipe_cmov_reg);
11352 %}
11353 
11354 
11355 instruct minI_rReg(rRegI dst, rRegI src)
11356 %{
11357   match(Set dst (MinI dst src));
11358 
11359   ins_cost(200);
11360   expand %{
11361     rFlagsReg cr;
11362     compI_rReg(cr, dst, src);
11363     cmovI_reg_g(dst, src, cr);
11364   %}
11365 %}
11366 
11367 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11368 %{
11369   effect(USE_DEF dst, USE src, USE cr);
11370 
11371   format %{ "cmovllt $dst, $src\t# max" %}
11372   opcode(0x0F, 0x4C);
11373   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11374   ins_pipe(pipe_cmov_reg);
11375 %}
11376 
11377 
11378 instruct maxI_rReg(rRegI dst, rRegI src)
11379 %{
11380   match(Set dst (MaxI dst src));
11381 
11382   ins_cost(200);
11383   expand %{
11384     rFlagsReg cr;
11385     compI_rReg(cr, dst, src);
11386     cmovI_reg_l(dst, src, cr);
11387   %}
11388 %}
11389 
11390 // ============================================================================
11391 // Branch Instructions
11392 
11393 // Jump Direct - Label defines a relative address from JMP+1
11394 instruct jmpDir(label labl)
11395 %{
11396   match(Goto);
11397   effect(USE labl);
11398 
11399   ins_cost(300);
11400   format %{ "jmp     $labl" %}
11401   size(5);
11402   ins_encode %{
11403     Label* L = $labl$$label;
11404     __ jmp(*L, false); // Always long jump
11405   %}
11406   ins_pipe(pipe_jmp);
11407 %}
11408 
11409 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11410 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11411 %{
11412   match(If cop cr);
11413   effect(USE labl);
11414 
11415   ins_cost(300);
11416   format %{ "j$cop     $labl" %}
11417   size(6);
11418   ins_encode %{
11419     Label* L = $labl$$label;
11420     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11421   %}
11422   ins_pipe(pipe_jcc);
11423 %}
11424 
11425 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11426 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11427 %{
11428   match(CountedLoopEnd cop cr);
11429   effect(USE labl);
11430 
11431   ins_cost(300);
11432   format %{ "j$cop     $labl\t# loop end" %}
11433   size(6);
11434   ins_encode %{
11435     Label* L = $labl$$label;
11436     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11437   %}
11438   ins_pipe(pipe_jcc);
11439 %}
11440 
11441 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11442 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11443   match(CountedLoopEnd cop cmp);
11444   effect(USE labl);
11445 
11446   ins_cost(300);
11447   format %{ "j$cop,u   $labl\t# loop end" %}
11448   size(6);
11449   ins_encode %{
11450     Label* L = $labl$$label;
11451     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11452   %}
11453   ins_pipe(pipe_jcc);
11454 %}
11455 
11456 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11457   match(CountedLoopEnd cop cmp);
11458   effect(USE labl);
11459 
11460   ins_cost(200);
11461   format %{ "j$cop,u   $labl\t# loop end" %}
11462   size(6);
11463   ins_encode %{
11464     Label* L = $labl$$label;
11465     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11466   %}
11467   ins_pipe(pipe_jcc);
11468 %}
11469 
11470 // Jump Direct Conditional - using unsigned comparison
11471 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11472   match(If cop cmp);
11473   effect(USE labl);
11474 
11475   ins_cost(300);
11476   format %{ "j$cop,u  $labl" %}
11477   size(6);
11478   ins_encode %{
11479     Label* L = $labl$$label;
11480     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11481   %}
11482   ins_pipe(pipe_jcc);
11483 %}
11484 
11485 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11486   match(If cop cmp);
11487   effect(USE labl);
11488 
11489   ins_cost(200);
11490   format %{ "j$cop,u  $labl" %}
11491   size(6);
11492   ins_encode %{
11493     Label* L = $labl$$label;
11494     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11495   %}
11496   ins_pipe(pipe_jcc);
11497 %}
11498 
11499 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11500   match(If cop cmp);
11501   effect(USE labl);
11502 
11503   ins_cost(200);
11504   format %{ $$template
11505     if ($cop$$cmpcode == Assembler::notEqual) {
11506       $$emit$$"jp,u   $labl\n\t"
11507       $$emit$$"j$cop,u   $labl"
11508     } else {
11509       $$emit$$"jp,u   done\n\t"
11510       $$emit$$"j$cop,u   $labl\n\t"
11511       $$emit$$"done:"
11512     }
11513   %}
11514   ins_encode %{
11515     Label* l = $labl$$label;
11516     if ($cop$$cmpcode == Assembler::notEqual) {
11517       __ jcc(Assembler::parity, *l, false);
11518       __ jcc(Assembler::notEqual, *l, false);
11519     } else if ($cop$$cmpcode == Assembler::equal) {
11520       Label done;
11521       __ jccb(Assembler::parity, done);
11522       __ jcc(Assembler::equal, *l, false);
11523       __ bind(done);
11524     } else {
11525        ShouldNotReachHere();
11526     }
11527   %}
11528   ins_pipe(pipe_jcc);
11529 %}
11530 
11531 // ============================================================================
11532 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11533 // superklass array for an instance of the superklass.  Set a hidden
11534 // internal cache on a hit (cache is checked with exposed code in
11535 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11536 // encoding ALSO sets flags.
11537 
11538 instruct partialSubtypeCheck(rdi_RegP result,
11539                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11540                              rFlagsReg cr)
11541 %{
11542   match(Set result (PartialSubtypeCheck sub super));
11543   effect(KILL rcx, KILL cr);
11544 
11545   ins_cost(1100);  // slightly larger than the next version
11546   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11547             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11548             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11549             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11550             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11551             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11552             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11553     "miss:\t" %}
11554 
11555   opcode(0x1); // Force a XOR of RDI
11556   ins_encode(enc_PartialSubtypeCheck());
11557   ins_pipe(pipe_slow);
11558 %}
11559 
11560 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11561                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11562                                      immP0 zero,
11563                                      rdi_RegP result)
11564 %{
11565   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11566   effect(KILL rcx, KILL result);
11567 
11568   ins_cost(1000);
11569   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11570             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11571             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11572             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11573             "jne,s   miss\t\t# Missed: flags nz\n\t"
11574             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11575     "miss:\t" %}
11576 
11577   opcode(0x0); // No need to XOR RDI
11578   ins_encode(enc_PartialSubtypeCheck());
11579   ins_pipe(pipe_slow);
11580 %}
11581 
11582 // ============================================================================
11583 // Branch Instructions -- short offset versions
11584 //
11585 // These instructions are used to replace jumps of a long offset (the default
11586 // match) with jumps of a shorter offset.  These instructions are all tagged
11587 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11588 // match rules in general matching.  Instead, the ADLC generates a conversion
11589 // method in the MachNode which can be used to do in-place replacement of the
11590 // long variant with the shorter variant.  The compiler will determine if a
11591 // branch can be taken by the is_short_branch_offset() predicate in the machine
11592 // specific code section of the file.
11593 
11594 // Jump Direct - Label defines a relative address from JMP+1
11595 instruct jmpDir_short(label labl) %{
11596   match(Goto);
11597   effect(USE labl);
11598 
11599   ins_cost(300);
11600   format %{ "jmp,s   $labl" %}
11601   size(2);
11602   ins_encode %{
11603     Label* L = $labl$$label;
11604     __ jmpb(*L);
11605   %}
11606   ins_pipe(pipe_jmp);
11607   ins_short_branch(1);
11608 %}
11609 
11610 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11611 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11612   match(If cop cr);
11613   effect(USE labl);
11614 
11615   ins_cost(300);
11616   format %{ "j$cop,s   $labl" %}
11617   size(2);
11618   ins_encode %{
11619     Label* L = $labl$$label;
11620     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11621   %}
11622   ins_pipe(pipe_jcc);
11623   ins_short_branch(1);
11624 %}
11625 
11626 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11627 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11628   match(CountedLoopEnd cop cr);
11629   effect(USE labl);
11630 
11631   ins_cost(300);
11632   format %{ "j$cop,s   $labl\t# loop end" %}
11633   size(2);
11634   ins_encode %{
11635     Label* L = $labl$$label;
11636     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11637   %}
11638   ins_pipe(pipe_jcc);
11639   ins_short_branch(1);
11640 %}
11641 
11642 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11643 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11644   match(CountedLoopEnd cop cmp);
11645   effect(USE labl);
11646 
11647   ins_cost(300);
11648   format %{ "j$cop,us  $labl\t# loop end" %}
11649   size(2);
11650   ins_encode %{
11651     Label* L = $labl$$label;
11652     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11653   %}
11654   ins_pipe(pipe_jcc);
11655   ins_short_branch(1);
11656 %}
11657 
11658 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11659   match(CountedLoopEnd cop cmp);
11660   effect(USE labl);
11661 
11662   ins_cost(300);
11663   format %{ "j$cop,us  $labl\t# loop end" %}
11664   size(2);
11665   ins_encode %{
11666     Label* L = $labl$$label;
11667     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11668   %}
11669   ins_pipe(pipe_jcc);
11670   ins_short_branch(1);
11671 %}
11672 
11673 // Jump Direct Conditional - using unsigned comparison
11674 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11675   match(If cop cmp);
11676   effect(USE labl);
11677 
11678   ins_cost(300);
11679   format %{ "j$cop,us  $labl" %}
11680   size(2);
11681   ins_encode %{
11682     Label* L = $labl$$label;
11683     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11684   %}
11685   ins_pipe(pipe_jcc);
11686   ins_short_branch(1);
11687 %}
11688 
11689 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11690   match(If cop cmp);
11691   effect(USE labl);
11692 
11693   ins_cost(300);
11694   format %{ "j$cop,us  $labl" %}
11695   size(2);
11696   ins_encode %{
11697     Label* L = $labl$$label;
11698     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11699   %}
11700   ins_pipe(pipe_jcc);
11701   ins_short_branch(1);
11702 %}
11703 
11704 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11705   match(If cop cmp);
11706   effect(USE labl);
11707 
11708   ins_cost(300);
11709   format %{ $$template
11710     if ($cop$$cmpcode == Assembler::notEqual) {
11711       $$emit$$"jp,u,s   $labl\n\t"
11712       $$emit$$"j$cop,u,s   $labl"
11713     } else {
11714       $$emit$$"jp,u,s   done\n\t"
11715       $$emit$$"j$cop,u,s  $labl\n\t"
11716       $$emit$$"done:"
11717     }
11718   %}
11719   size(4);
11720   ins_encode %{
11721     Label* l = $labl$$label;
11722     if ($cop$$cmpcode == Assembler::notEqual) {
11723       __ jccb(Assembler::parity, *l);
11724       __ jccb(Assembler::notEqual, *l);
11725     } else if ($cop$$cmpcode == Assembler::equal) {
11726       Label done;
11727       __ jccb(Assembler::parity, done);
11728       __ jccb(Assembler::equal, *l);
11729       __ bind(done);
11730     } else {
11731        ShouldNotReachHere();
11732     }
11733   %}
11734   ins_pipe(pipe_jcc);
11735   ins_short_branch(1);
11736 %}
11737 
11738 // ============================================================================
11739 // inlined locking and unlocking
11740 
11741 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11742   predicate(Compile::current()->use_rtm());
11743   match(Set cr (FastLock object box));
11744   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11745   ins_cost(300);
11746   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11747   ins_encode %{
11748     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11749                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11750                  _counters, _rtm_counters, _stack_rtm_counters,
11751                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11752                  true, ra_->C->profile_rtm());
11753   %}
11754   ins_pipe(pipe_slow);
11755 %}
11756 
11757 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11758   predicate(!Compile::current()->use_rtm());
11759   match(Set cr (FastLock object box));
11760   effect(TEMP tmp, TEMP scr, USE_KILL box);
11761   ins_cost(300);
11762   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11763   ins_encode %{
11764     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11765                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11766   %}
11767   ins_pipe(pipe_slow);
11768 %}
11769 
11770 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11771   match(Set cr (FastUnlock object box));
11772   effect(TEMP tmp, USE_KILL box);
11773   ins_cost(300);
11774   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11775   ins_encode %{
11776     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11777   %}
11778   ins_pipe(pipe_slow);
11779 %}
11780 
11781 
11782 // ============================================================================
11783 // Safepoint Instructions
11784 instruct safePoint_poll(rFlagsReg cr)
11785 %{
11786   predicate(!Assembler::is_polling_page_far());
11787   match(SafePoint);
11788   effect(KILL cr);
11789 
11790   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11791             "# Safepoint: poll for GC" %}
11792   ins_cost(125);
11793   ins_encode %{
11794     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11795     __ testl(rax, addr);
11796   %}
11797   ins_pipe(ialu_reg_mem);
11798 %}
11799 
11800 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11801 %{
11802   predicate(Assembler::is_polling_page_far());
11803   match(SafePoint poll);
11804   effect(KILL cr, USE poll);
11805 
11806   format %{ "testl  rax, [$poll]\t"
11807             "# Safepoint: poll for GC" %}
11808   ins_cost(125);
11809   ins_encode %{
11810     __ relocate(relocInfo::poll_type);
11811     __ testl(rax, Address($poll$$Register, 0));
11812   %}
11813   ins_pipe(ialu_reg_mem);
11814 %}
11815 
11816 // ============================================================================
11817 // Procedure Call/Return Instructions
11818 // Call Java Static Instruction
11819 // Note: If this code changes, the corresponding ret_addr_offset() and
11820 //       compute_padding() functions will have to be adjusted.
11821 instruct CallStaticJavaDirect(method meth) %{
11822   match(CallStaticJava);
11823   effect(USE meth);
11824 
11825   ins_cost(300);
11826   format %{ "call,static " %}
11827   opcode(0xE8); /* E8 cd */
11828   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11829   ins_pipe(pipe_slow);
11830   ins_alignment(4);
11831 %}
11832 
11833 // Call Java Dynamic Instruction
11834 // Note: If this code changes, the corresponding ret_addr_offset() and
11835 //       compute_padding() functions will have to be adjusted.
11836 instruct CallDynamicJavaDirect(method meth)
11837 %{
11838   match(CallDynamicJava);
11839   effect(USE meth);
11840 
11841   ins_cost(300);
11842   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11843             "call,dynamic " %}
11844   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11845   ins_pipe(pipe_slow);
11846   ins_alignment(4);
11847 %}
11848 
11849 // Call Runtime Instruction
11850 instruct CallRuntimeDirect(method meth)
11851 %{
11852   match(CallRuntime);
11853   effect(USE meth);
11854 
11855   ins_cost(300);
11856   format %{ "call,runtime " %}
11857   ins_encode(clear_avx, Java_To_Runtime(meth));
11858   ins_pipe(pipe_slow);
11859 %}
11860 
11861 // Call runtime without safepoint
11862 instruct CallLeafDirect(method meth)
11863 %{
11864   match(CallLeaf);
11865   effect(USE meth);
11866 
11867   ins_cost(300);
11868   format %{ "call_leaf,runtime " %}
11869   ins_encode(clear_avx, Java_To_Runtime(meth));
11870   ins_pipe(pipe_slow);
11871 %}
11872 
11873 // Call runtime without safepoint
11874 instruct CallLeafNoFPDirect(method meth)
11875 %{
11876   match(CallLeafNoFP);
11877   effect(USE meth);
11878 
11879   ins_cost(300);
11880   format %{ "call_leaf_nofp,runtime " %}
11881   ins_encode(Java_To_Runtime(meth));
11882   ins_pipe(pipe_slow);
11883 %}
11884 
11885 // Return Instruction
11886 // Remove the return address & jump to it.
11887 // Notice: We always emit a nop after a ret to make sure there is room
11888 // for safepoint patching
11889 instruct Ret()
11890 %{
11891   match(Return);
11892 
11893   format %{ "ret" %}
11894   opcode(0xC3);
11895   ins_encode(OpcP);
11896   ins_pipe(pipe_jmp);
11897 %}
11898 
11899 // Tail Call; Jump from runtime stub to Java code.
11900 // Also known as an 'interprocedural jump'.
11901 // Target of jump will eventually return to caller.
11902 // TailJump below removes the return address.
11903 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11904 %{
11905   match(TailCall jump_target method_oop);
11906 
11907   ins_cost(300);
11908   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11909   opcode(0xFF, 0x4); /* Opcode FF /4 */
11910   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11911   ins_pipe(pipe_jmp);
11912 %}
11913 
11914 // Tail Jump; remove the return address; jump to target.
11915 // TailCall above leaves the return address around.
11916 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11917 %{
11918   match(TailJump jump_target ex_oop);
11919 
11920   ins_cost(300);
11921   format %{ "popq    rdx\t# pop return address\n\t"
11922             "jmp     $jump_target" %}
11923   opcode(0xFF, 0x4); /* Opcode FF /4 */
11924   ins_encode(Opcode(0x5a), // popq rdx
11925              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11926   ins_pipe(pipe_jmp);
11927 %}
11928 
11929 // Create exception oop: created by stack-crawling runtime code.
11930 // Created exception is now available to this handler, and is setup
11931 // just prior to jumping to this handler.  No code emitted.
11932 instruct CreateException(rax_RegP ex_oop)
11933 %{
11934   match(Set ex_oop (CreateEx));
11935 
11936   size(0);
11937   // use the following format syntax
11938   format %{ "# exception oop is in rax; no code emitted" %}
11939   ins_encode();
11940   ins_pipe(empty);
11941 %}
11942 
11943 // Rethrow exception:
11944 // The exception oop will come in the first argument position.
11945 // Then JUMP (not call) to the rethrow stub code.
11946 instruct RethrowException()
11947 %{
11948   match(Rethrow);
11949 
11950   // use the following format syntax
11951   format %{ "jmp     rethrow_stub" %}
11952   ins_encode(enc_rethrow);
11953   ins_pipe(pipe_jmp);
11954 %}
11955 
11956 
11957 // ============================================================================
11958 // This name is KNOWN by the ADLC and cannot be changed.
11959 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11960 // for this guy.
11961 instruct tlsLoadP(r15_RegP dst) %{
11962   match(Set dst (ThreadLocal));
11963   effect(DEF dst);
11964 
11965   size(0);
11966   format %{ "# TLS is in R15" %}
11967   ins_encode( /*empty encoding*/ );
11968   ins_pipe(ialu_reg_reg);
11969 %}
11970 
11971 
11972 //----------PEEPHOLE RULES-----------------------------------------------------
11973 // These must follow all instruction definitions as they use the names
11974 // defined in the instructions definitions.
11975 //
11976 // peepmatch ( root_instr_name [preceding_instruction]* );
11977 //
11978 // peepconstraint %{
11979 // (instruction_number.operand_name relational_op instruction_number.operand_name
11980 //  [, ...] );
11981 // // instruction numbers are zero-based using left to right order in peepmatch
11982 //
11983 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11984 // // provide an instruction_number.operand_name for each operand that appears
11985 // // in the replacement instruction's match rule
11986 //
11987 // ---------VM FLAGS---------------------------------------------------------
11988 //
11989 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11990 //
11991 // Each peephole rule is given an identifying number starting with zero and
11992 // increasing by one in the order seen by the parser.  An individual peephole
11993 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11994 // on the command-line.
11995 //
11996 // ---------CURRENT LIMITATIONS----------------------------------------------
11997 //
11998 // Only match adjacent instructions in same basic block
11999 // Only equality constraints
12000 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12001 // Only one replacement instruction
12002 //
12003 // ---------EXAMPLE----------------------------------------------------------
12004 //
12005 // // pertinent parts of existing instructions in architecture description
12006 // instruct movI(rRegI dst, rRegI src)
12007 // %{
12008 //   match(Set dst (CopyI src));
12009 // %}
12010 //
12011 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12012 // %{
12013 //   match(Set dst (AddI dst src));
12014 //   effect(KILL cr);
12015 // %}
12016 //
12017 // // Change (inc mov) to lea
12018 // peephole %{
12019 //   // increment preceeded by register-register move
12020 //   peepmatch ( incI_rReg movI );
12021 //   // require that the destination register of the increment
12022 //   // match the destination register of the move
12023 //   peepconstraint ( 0.dst == 1.dst );
12024 //   // construct a replacement instruction that sets
12025 //   // the destination to ( move's source register + one )
12026 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12027 // %}
12028 //
12029 
12030 // Implementation no longer uses movX instructions since
12031 // machine-independent system no longer uses CopyX nodes.
12032 //
12033 // peephole
12034 // %{
12035 //   peepmatch (incI_rReg movI);
12036 //   peepconstraint (0.dst == 1.dst);
12037 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12038 // %}
12039 
12040 // peephole
12041 // %{
12042 //   peepmatch (decI_rReg movI);
12043 //   peepconstraint (0.dst == 1.dst);
12044 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12045 // %}
12046 
12047 // peephole
12048 // %{
12049 //   peepmatch (addI_rReg_imm movI);
12050 //   peepconstraint (0.dst == 1.dst);
12051 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12052 // %}
12053 
12054 // peephole
12055 // %{
12056 //   peepmatch (incL_rReg movL);
12057 //   peepconstraint (0.dst == 1.dst);
12058 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12059 // %}
12060 
12061 // peephole
12062 // %{
12063 //   peepmatch (decL_rReg movL);
12064 //   peepconstraint (0.dst == 1.dst);
12065 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12066 // %}
12067 
12068 // peephole
12069 // %{
12070 //   peepmatch (addL_rReg_imm movL);
12071 //   peepconstraint (0.dst == 1.dst);
12072 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12073 // %}
12074 
12075 // peephole
12076 // %{
12077 //   peepmatch (addP_rReg_imm movP);
12078 //   peepconstraint (0.dst == 1.dst);
12079 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12080 // %}
12081 
12082 // // Change load of spilled value to only a spill
12083 // instruct storeI(memory mem, rRegI src)
12084 // %{
12085 //   match(Set mem (StoreI mem src));
12086 // %}
12087 //
12088 // instruct loadI(rRegI dst, memory mem)
12089 // %{
12090 //   match(Set dst (LoadI mem));
12091 // %}
12092 //
12093 
12094 peephole
12095 %{
12096   peepmatch (loadI storeI);
12097   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12098   peepreplace (storeI(1.mem 1.mem 1.src));
12099 %}
12100 
12101 peephole
12102 %{
12103   peepmatch (loadL storeL);
12104   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12105   peepreplace (storeL(1.mem 1.mem 1.src));
12106 %}
12107 
12108 //----------SMARTSPILL RULES---------------------------------------------------
12109 // These must follow all instruction definitions as they use the names
12110 // defined in the instructions definitions.