1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   MacroAssembler _masm(&cbuf);
 957 
 958   if (C->max_vector_size() > 16) {
 959     // Clear upper bits of YMM registers when current compiled code uses
 960     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 961     __ vzeroupper();
 962   }
 963 
 964   int framesize = C->frame_size_in_bytes();
 965   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 966   // Remove word for return adr already pushed
 967   // and RBP
 968   framesize -= 2*wordSize;
 969 
 970   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 971 
 972   if (framesize) {
 973     emit_opcode(cbuf, Assembler::REX_W);
 974     if (framesize < 0x80) {
 975       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 976       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 977       emit_d8(cbuf, framesize);
 978     } else {
 979       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 980       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 981       emit_d32(cbuf, framesize);
 982     }
 983   }
 984 
 985   // popq rbp
 986   emit_opcode(cbuf, 0x58 | RBP_enc);
 987 
 988   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 989     __ reserved_stack_check(); 
 990   }
 991 
 992   if (do_polling() && C->is_method_compilation()) {
 993     MacroAssembler _masm(&cbuf);
 994     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 995     if (Assembler::is_polling_page_far()) {
 996       __ lea(rscratch1, polling_page);
 997       __ relocate(relocInfo::poll_return_type);
 998       __ testl(rax, Address(rscratch1, 0));
 999     } else {
1000       __ testl(rax, polling_page);
1001     }
1002   }
1003 }
1004 
1005 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1006 {
1007   return MachNode::size(ra_); // too many variables; just compute it
1008                               // the hard way
1009 }
1010 
1011 int MachEpilogNode::reloc() const
1012 {
1013   return 2; // a large enough number
1014 }
1015 
1016 const Pipeline* MachEpilogNode::pipeline() const
1017 {
1018   return MachNode::pipeline_class();
1019 }
1020 
1021 int MachEpilogNode::safepoint_offset() const
1022 {
1023   return 0;
1024 }
1025 
1026 //=============================================================================
1027 
1028 enum RC {
1029   rc_bad,
1030   rc_int,
1031   rc_float,
1032   rc_stack
1033 };
1034 
1035 static enum RC rc_class(OptoReg::Name reg)
1036 {
1037   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1038 
1039   if (OptoReg::is_stack(reg)) return rc_stack;
1040 
1041   VMReg r = OptoReg::as_VMReg(reg);
1042 
1043   if (r->is_Register()) return rc_int;
1044 
1045   assert(r->is_XMMRegister(), "must be");
1046   return rc_float;
1047 }
1048 
1049 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1050 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1051                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1052 
1053 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1054                             int stack_offset, int reg, uint ireg, outputStream* st);
1055 
1056 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1057                                       int dst_offset, uint ireg, outputStream* st) {
1058   if (cbuf) {
1059     MacroAssembler _masm(cbuf);
1060     switch (ireg) {
1061     case Op_VecS:
1062       __ movq(Address(rsp, -8), rax);
1063       __ movl(rax, Address(rsp, src_offset));
1064       __ movl(Address(rsp, dst_offset), rax);
1065       __ movq(rax, Address(rsp, -8));
1066       break;
1067     case Op_VecD:
1068       __ pushq(Address(rsp, src_offset));
1069       __ popq (Address(rsp, dst_offset));
1070       break;
1071     case Op_VecX:
1072       __ pushq(Address(rsp, src_offset));
1073       __ popq (Address(rsp, dst_offset));
1074       __ pushq(Address(rsp, src_offset+8));
1075       __ popq (Address(rsp, dst_offset+8));
1076       break;
1077     case Op_VecY:
1078       __ vmovdqu(Address(rsp, -32), xmm0);
1079       __ vmovdqu(xmm0, Address(rsp, src_offset));
1080       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1081       __ vmovdqu(xmm0, Address(rsp, -32));
1082       break;
1083     case Op_VecZ:
1084       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1085       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1086       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1087       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1088       break;
1089     default:
1090       ShouldNotReachHere();
1091     }
1092 #ifndef PRODUCT
1093   } else {
1094     switch (ireg) {
1095     case Op_VecS:
1096       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1097                 "movl    rax, [rsp + #%d]\n\t"
1098                 "movl    [rsp + #%d], rax\n\t"
1099                 "movq    rax, [rsp - #8]",
1100                 src_offset, dst_offset);
1101       break;
1102     case Op_VecD:
1103       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1104                 "popq    [rsp + #%d]",
1105                 src_offset, dst_offset);
1106       break;
1107      case Op_VecX:
1108       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1109                 "popq    [rsp + #%d]\n\t"
1110                 "pushq   [rsp + #%d]\n\t"
1111                 "popq    [rsp + #%d]",
1112                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1113       break;
1114     case Op_VecY:
1115       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1116                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1117                 "vmovdqu [rsp + #%d], xmm0\n\t"
1118                 "vmovdqu xmm0, [rsp - #32]",
1119                 src_offset, dst_offset);
1120       break;
1121     case Op_VecZ:
1122       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1123                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1124                 "vmovdqu [rsp + #%d], xmm0\n\t"
1125                 "vmovdqu xmm0, [rsp - #64]",
1126                 src_offset, dst_offset);
1127       break;
1128     default:
1129       ShouldNotReachHere();
1130     }
1131 #endif
1132   }
1133 }
1134 
1135 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1136                                        PhaseRegAlloc* ra_,
1137                                        bool do_size,
1138                                        outputStream* st) const {
1139   assert(cbuf != NULL || st  != NULL, "sanity");
1140   // Get registers to move
1141   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1142   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1143   OptoReg::Name dst_second = ra_->get_reg_second(this);
1144   OptoReg::Name dst_first = ra_->get_reg_first(this);
1145 
1146   enum RC src_second_rc = rc_class(src_second);
1147   enum RC src_first_rc = rc_class(src_first);
1148   enum RC dst_second_rc = rc_class(dst_second);
1149   enum RC dst_first_rc = rc_class(dst_first);
1150 
1151   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1152          "must move at least 1 register" );
1153 
1154   if (src_first == dst_first && src_second == dst_second) {
1155     // Self copy, no move
1156     return 0;
1157   }
1158   if (bottom_type()->isa_vect() != NULL) {
1159     uint ireg = ideal_reg();
1160     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1161     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1162     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1163       // mem -> mem
1164       int src_offset = ra_->reg2offset(src_first);
1165       int dst_offset = ra_->reg2offset(dst_first);
1166       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1167     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1168       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1169     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1170       int stack_offset = ra_->reg2offset(dst_first);
1171       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1172     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1173       int stack_offset = ra_->reg2offset(src_first);
1174       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1175     } else {
1176       ShouldNotReachHere();
1177     }
1178     return 0;
1179   }
1180   if (src_first_rc == rc_stack) {
1181     // mem ->
1182     if (dst_first_rc == rc_stack) {
1183       // mem -> mem
1184       assert(src_second != dst_first, "overlap");
1185       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1186           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1187         // 64-bit
1188         int src_offset = ra_->reg2offset(src_first);
1189         int dst_offset = ra_->reg2offset(dst_first);
1190         if (cbuf) {
1191           MacroAssembler _masm(cbuf);
1192           __ pushq(Address(rsp, src_offset));
1193           __ popq (Address(rsp, dst_offset));
1194 #ifndef PRODUCT
1195         } else {
1196           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1197                     "popq    [rsp + #%d]",
1198                      src_offset, dst_offset);
1199 #endif
1200         }
1201       } else {
1202         // 32-bit
1203         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1204         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1205         // No pushl/popl, so:
1206         int src_offset = ra_->reg2offset(src_first);
1207         int dst_offset = ra_->reg2offset(dst_first);
1208         if (cbuf) {
1209           MacroAssembler _masm(cbuf);
1210           __ movq(Address(rsp, -8), rax);
1211           __ movl(rax, Address(rsp, src_offset));
1212           __ movl(Address(rsp, dst_offset), rax);
1213           __ movq(rax, Address(rsp, -8));
1214 #ifndef PRODUCT
1215         } else {
1216           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1217                     "movl    rax, [rsp + #%d]\n\t"
1218                     "movl    [rsp + #%d], rax\n\t"
1219                     "movq    rax, [rsp - #8]",
1220                      src_offset, dst_offset);
1221 #endif
1222         }
1223       }
1224       return 0;
1225     } else if (dst_first_rc == rc_int) {
1226       // mem -> gpr
1227       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1228           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1229         // 64-bit
1230         int offset = ra_->reg2offset(src_first);
1231         if (cbuf) {
1232           MacroAssembler _masm(cbuf);
1233           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1234 #ifndef PRODUCT
1235         } else {
1236           st->print("movq    %s, [rsp + #%d]\t# spill",
1237                      Matcher::regName[dst_first],
1238                      offset);
1239 #endif
1240         }
1241       } else {
1242         // 32-bit
1243         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1244         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1245         int offset = ra_->reg2offset(src_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("movl    %s, [rsp + #%d]\t# spill",
1252                      Matcher::regName[dst_first],
1253                      offset);
1254 #endif
1255         }
1256       }
1257       return 0;
1258     } else if (dst_first_rc == rc_float) {
1259       // mem-> xmm
1260       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1261           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1262         // 64-bit
1263         int offset = ra_->reg2offset(src_first);
1264         if (cbuf) {
1265           MacroAssembler _masm(cbuf);
1266           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1267 #ifndef PRODUCT
1268         } else {
1269           st->print("%s  %s, [rsp + #%d]\t# spill",
1270                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1271                      Matcher::regName[dst_first],
1272                      offset);
1273 #endif
1274         }
1275       } else {
1276         // 32-bit
1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1279         int offset = ra_->reg2offset(src_first);
1280         if (cbuf) {
1281           MacroAssembler _masm(cbuf);
1282           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1283 #ifndef PRODUCT
1284         } else {
1285           st->print("movss   %s, [rsp + #%d]\t# spill",
1286                      Matcher::regName[dst_first],
1287                      offset);
1288 #endif
1289         }
1290       }
1291       return 0;
1292     }
1293   } else if (src_first_rc == rc_int) {
1294     // gpr ->
1295     if (dst_first_rc == rc_stack) {
1296       // gpr -> mem
1297       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1298           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1299         // 64-bit
1300         int offset = ra_->reg2offset(dst_first);
1301         if (cbuf) {
1302           MacroAssembler _masm(cbuf);
1303           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1304 #ifndef PRODUCT
1305         } else {
1306           st->print("movq    [rsp + #%d], %s\t# spill",
1307                      offset,
1308                      Matcher::regName[src_first]);
1309 #endif
1310         }
1311       } else {
1312         // 32-bit
1313         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1314         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1315         int offset = ra_->reg2offset(dst_first);
1316         if (cbuf) {
1317           MacroAssembler _masm(cbuf);
1318           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1319 #ifndef PRODUCT
1320         } else {
1321           st->print("movl    [rsp + #%d], %s\t# spill",
1322                      offset,
1323                      Matcher::regName[src_first]);
1324 #endif
1325         }
1326       }
1327       return 0;
1328     } else if (dst_first_rc == rc_int) {
1329       // gpr -> gpr
1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1332         // 64-bit
1333         if (cbuf) {
1334           MacroAssembler _masm(cbuf);
1335           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1336                   as_Register(Matcher::_regEncode[src_first]));
1337 #ifndef PRODUCT
1338         } else {
1339           st->print("movq    %s, %s\t# spill",
1340                      Matcher::regName[dst_first],
1341                      Matcher::regName[src_first]);
1342 #endif
1343         }
1344         return 0;
1345       } else {
1346         // 32-bit
1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1349         if (cbuf) {
1350           MacroAssembler _masm(cbuf);
1351           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1352                   as_Register(Matcher::_regEncode[src_first]));
1353 #ifndef PRODUCT
1354         } else {
1355           st->print("movl    %s, %s\t# spill",
1356                      Matcher::regName[dst_first],
1357                      Matcher::regName[src_first]);
1358 #endif
1359         }
1360         return 0;
1361       }
1362     } else if (dst_first_rc == rc_float) {
1363       // gpr -> xmm
1364       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1365           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1366         // 64-bit
1367         if (cbuf) {
1368           MacroAssembler _masm(cbuf);
1369           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1370 #ifndef PRODUCT
1371         } else {
1372           st->print("movdq   %s, %s\t# spill",
1373                      Matcher::regName[dst_first],
1374                      Matcher::regName[src_first]);
1375 #endif
1376         }
1377       } else {
1378         // 32-bit
1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1381         if (cbuf) {
1382           MacroAssembler _masm(cbuf);
1383           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1384 #ifndef PRODUCT
1385         } else {
1386           st->print("movdl   %s, %s\t# spill",
1387                      Matcher::regName[dst_first],
1388                      Matcher::regName[src_first]);
1389 #endif
1390         }
1391       }
1392       return 0;
1393     }
1394   } else if (src_first_rc == rc_float) {
1395     // xmm ->
1396     if (dst_first_rc == rc_stack) {
1397       // xmm -> mem
1398       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1399           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1400         // 64-bit
1401         int offset = ra_->reg2offset(dst_first);
1402         if (cbuf) {
1403           MacroAssembler _masm(cbuf);
1404           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1405 #ifndef PRODUCT
1406         } else {
1407           st->print("movsd   [rsp + #%d], %s\t# spill",
1408                      offset,
1409                      Matcher::regName[src_first]);
1410 #endif
1411         }
1412       } else {
1413         // 32-bit
1414         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1415         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1416         int offset = ra_->reg2offset(dst_first);
1417         if (cbuf) {
1418           MacroAssembler _masm(cbuf);
1419           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1420 #ifndef PRODUCT
1421         } else {
1422           st->print("movss   [rsp + #%d], %s\t# spill",
1423                      offset,
1424                      Matcher::regName[src_first]);
1425 #endif
1426         }
1427       }
1428       return 0;
1429     } else if (dst_first_rc == rc_int) {
1430       // xmm -> gpr
1431       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1432           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1433         // 64-bit
1434         if (cbuf) {
1435           MacroAssembler _masm(cbuf);
1436           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1437 #ifndef PRODUCT
1438         } else {
1439           st->print("movdq   %s, %s\t# spill",
1440                      Matcher::regName[dst_first],
1441                      Matcher::regName[src_first]);
1442 #endif
1443         }
1444       } else {
1445         // 32-bit
1446         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1447         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1448         if (cbuf) {
1449           MacroAssembler _masm(cbuf);
1450           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1451 #ifndef PRODUCT
1452         } else {
1453           st->print("movdl   %s, %s\t# spill",
1454                      Matcher::regName[dst_first],
1455                      Matcher::regName[src_first]);
1456 #endif
1457         }
1458       }
1459       return 0;
1460     } else if (dst_first_rc == rc_float) {
1461       // xmm -> xmm
1462       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1463           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1464         // 64-bit
1465         if (cbuf) {
1466           MacroAssembler _masm(cbuf);
1467           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1468 #ifndef PRODUCT
1469         } else {
1470           st->print("%s  %s, %s\t# spill",
1471                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1472                      Matcher::regName[dst_first],
1473                      Matcher::regName[src_first]);
1474 #endif
1475         }
1476       } else {
1477         // 32-bit
1478         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1479         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1480         if (cbuf) {
1481           MacroAssembler _masm(cbuf);
1482           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1483 #ifndef PRODUCT
1484         } else {
1485           st->print("%s  %s, %s\t# spill",
1486                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1487                      Matcher::regName[dst_first],
1488                      Matcher::regName[src_first]);
1489 #endif
1490         }
1491       }
1492       return 0;
1493     }
1494   }
1495 
1496   assert(0," foo ");
1497   Unimplemented();
1498   return 0;
1499 }
1500 
1501 #ifndef PRODUCT
1502 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1503   implementation(NULL, ra_, false, st);
1504 }
1505 #endif
1506 
1507 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1508   implementation(&cbuf, ra_, false, NULL);
1509 }
1510 
1511 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1512   return MachNode::size(ra_);
1513 }
1514 
1515 //=============================================================================
1516 #ifndef PRODUCT
1517 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1518 {
1519   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1520   int reg = ra_->get_reg_first(this);
1521   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1522             Matcher::regName[reg], offset);
1523 }
1524 #endif
1525 
1526 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1527 {
1528   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1529   int reg = ra_->get_encode(this);
1530   if (offset >= 0x80) {
1531     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1532     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1533     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1534     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1535     emit_d32(cbuf, offset);
1536   } else {
1537     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1538     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1539     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1540     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1541     emit_d8(cbuf, offset);
1542   }
1543 }
1544 
1545 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1546 {
1547   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1548   return (offset < 0x80) ? 5 : 8; // REX
1549 }
1550 
1551 //=============================================================================
1552 #ifndef PRODUCT
1553 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1554 {
1555   if (UseCompressedClassPointers) {
1556     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1557     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1558     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1559   } else {
1560     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1561                  "# Inline cache check");
1562   }
1563   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1564   st->print_cr("\tnop\t# nops to align entry point");
1565 }
1566 #endif
1567 
1568 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1569 {
1570   MacroAssembler masm(&cbuf);
1571   uint insts_size = cbuf.insts_size();
1572   if (UseCompressedClassPointers) {
1573     masm.load_klass(rscratch1, j_rarg0);
1574     masm.cmpptr(rax, rscratch1);
1575   } else {
1576     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1577   }
1578 
1579   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1580 
1581   /* WARNING these NOPs are critical so that verified entry point is properly
1582      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1583   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1584   if (OptoBreakpoint) {
1585     // Leave space for int3
1586     nops_cnt -= 1;
1587   }
1588   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1589   if (nops_cnt > 0)
1590     masm.nop(nops_cnt);
1591 }
1592 
1593 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1594 {
1595   return MachNode::size(ra_); // too many variables; just compute it
1596                               // the hard way
1597 }
1598 
1599 
1600 //=============================================================================
1601 
1602 int Matcher::regnum_to_fpu_offset(int regnum)
1603 {
1604   return regnum - 32; // The FP registers are in the second chunk
1605 }
1606 
1607 // This is UltraSparc specific, true just means we have fast l2f conversion
1608 const bool Matcher::convL2FSupported(void) {
1609   return true;
1610 }
1611 
1612 // Is this branch offset short enough that a short branch can be used?
1613 //
1614 // NOTE: If the platform does not provide any short branch variants, then
1615 //       this method should return false for offset 0.
1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1617   // The passed offset is relative to address of the branch.
1618   // On 86 a branch displacement is calculated relative to address
1619   // of a next instruction.
1620   offset -= br_size;
1621 
1622   // the short version of jmpConUCF2 contains multiple branches,
1623   // making the reach slightly less
1624   if (rule == jmpConUCF2_rule)
1625     return (-126 <= offset && offset <= 125);
1626   return (-128 <= offset && offset <= 127);
1627 }
1628 
1629 const bool Matcher::isSimpleConstant64(jlong value) {
1630   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1631   //return value == (int) value;  // Cf. storeImmL and immL32.
1632 
1633   // Probably always true, even if a temp register is required.
1634   return true;
1635 }
1636 
1637 // The ecx parameter to rep stosq for the ClearArray node is in words.
1638 const bool Matcher::init_array_count_is_in_bytes = false;
1639 
1640 // No additional cost for CMOVL.
1641 const int Matcher::long_cmove_cost() { return 0; }
1642 
1643 // No CMOVF/CMOVD with SSE2
1644 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1645 
1646 // Does the CPU require late expand (see block.cpp for description of late expand)?
1647 const bool Matcher::require_postalloc_expand = false;
1648 
1649 // Do we need to mask the count passed to shift instructions or does
1650 // the cpu only look at the lower 5/6 bits anyway?
1651 const bool Matcher::need_masked_shift_count = false;
1652 
1653 bool Matcher::narrow_oop_use_complex_address() {
1654   assert(UseCompressedOops, "only for compressed oops code");
1655   return (LogMinObjAlignmentInBytes <= 3);
1656 }
1657 
1658 bool Matcher::narrow_klass_use_complex_address() {
1659   assert(UseCompressedClassPointers, "only for compressed klass code");
1660   return (LogKlassAlignmentInBytes <= 3);
1661 }
1662 
1663 // Is it better to copy float constants, or load them directly from
1664 // memory?  Intel can load a float constant from a direct address,
1665 // requiring no extra registers.  Most RISCs will have to materialize
1666 // an address into a register first, so they would do better to copy
1667 // the constant from stack.
1668 const bool Matcher::rematerialize_float_constants = true; // XXX
1669 
1670 // If CPU can load and store mis-aligned doubles directly then no
1671 // fixup is needed.  Else we split the double into 2 integer pieces
1672 // and move it piece-by-piece.  Only happens when passing doubles into
1673 // C code as the Java calling convention forces doubles to be aligned.
1674 const bool Matcher::misaligned_doubles_ok = true;
1675 
1676 // No-op on amd64
1677 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1678 
1679 // Advertise here if the CPU requires explicit rounding operations to
1680 // implement the UseStrictFP mode.
1681 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1682 
1683 // Are floats conerted to double when stored to stack during deoptimization?
1684 // On x64 it is stored without convertion so we can use normal access.
1685 bool Matcher::float_in_double() { return false; }
1686 
1687 // Do ints take an entire long register or just half?
1688 const bool Matcher::int_in_long = true;
1689 
1690 // Return whether or not this register is ever used as an argument.
1691 // This function is used on startup to build the trampoline stubs in
1692 // generateOptoStub.  Registers not mentioned will be killed by the VM
1693 // call in the trampoline, and arguments in those registers not be
1694 // available to the callee.
1695 bool Matcher::can_be_java_arg(int reg)
1696 {
1697   return
1698     reg ==  RDI_num || reg == RDI_H_num ||
1699     reg ==  RSI_num || reg == RSI_H_num ||
1700     reg ==  RDX_num || reg == RDX_H_num ||
1701     reg ==  RCX_num || reg == RCX_H_num ||
1702     reg ==   R8_num || reg ==  R8_H_num ||
1703     reg ==   R9_num || reg ==  R9_H_num ||
1704     reg ==  R12_num || reg == R12_H_num ||
1705     reg == XMM0_num || reg == XMM0b_num ||
1706     reg == XMM1_num || reg == XMM1b_num ||
1707     reg == XMM2_num || reg == XMM2b_num ||
1708     reg == XMM3_num || reg == XMM3b_num ||
1709     reg == XMM4_num || reg == XMM4b_num ||
1710     reg == XMM5_num || reg == XMM5b_num ||
1711     reg == XMM6_num || reg == XMM6b_num ||
1712     reg == XMM7_num || reg == XMM7b_num;
1713 }
1714 
1715 bool Matcher::is_spillable_arg(int reg)
1716 {
1717   return can_be_java_arg(reg);
1718 }
1719 
1720 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1721   // In 64 bit mode a code which use multiply when
1722   // devisor is constant is faster than hardware
1723   // DIV instruction (it uses MulHiL).
1724   return false;
1725 }
1726 
1727 // Register for DIVI projection of divmodI
1728 RegMask Matcher::divI_proj_mask() {
1729   return INT_RAX_REG_mask();
1730 }
1731 
1732 // Register for MODI projection of divmodI
1733 RegMask Matcher::modI_proj_mask() {
1734   return INT_RDX_REG_mask();
1735 }
1736 
1737 // Register for DIVL projection of divmodL
1738 RegMask Matcher::divL_proj_mask() {
1739   return LONG_RAX_REG_mask();
1740 }
1741 
1742 // Register for MODL projection of divmodL
1743 RegMask Matcher::modL_proj_mask() {
1744   return LONG_RDX_REG_mask();
1745 }
1746 
1747 // Register for saving SP into on method handle invokes. Not used on x86_64.
1748 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1749     return NO_REG_mask();
1750 }
1751 
1752 %}
1753 
1754 //----------ENCODING BLOCK-----------------------------------------------------
1755 // This block specifies the encoding classes used by the compiler to
1756 // output byte streams.  Encoding classes are parameterized macros
1757 // used by Machine Instruction Nodes in order to generate the bit
1758 // encoding of the instruction.  Operands specify their base encoding
1759 // interface with the interface keyword.  There are currently
1760 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1761 // COND_INTER.  REG_INTER causes an operand to generate a function
1762 // which returns its register number when queried.  CONST_INTER causes
1763 // an operand to generate a function which returns the value of the
1764 // constant when queried.  MEMORY_INTER causes an operand to generate
1765 // four functions which return the Base Register, the Index Register,
1766 // the Scale Value, and the Offset Value of the operand when queried.
1767 // COND_INTER causes an operand to generate six functions which return
1768 // the encoding code (ie - encoding bits for the instruction)
1769 // associated with each basic boolean condition for a conditional
1770 // instruction.
1771 //
1772 // Instructions specify two basic values for encoding.  Again, a
1773 // function is available to check if the constant displacement is an
1774 // oop. They use the ins_encode keyword to specify their encoding
1775 // classes (which must be a sequence of enc_class names, and their
1776 // parameters, specified in the encoding block), and they use the
1777 // opcode keyword to specify, in order, their primary, secondary, and
1778 // tertiary opcode.  Only the opcode sections which a particular
1779 // instruction needs for encoding need to be specified.
1780 encode %{
1781   // Build emit functions for each basic byte or larger field in the
1782   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1783   // from C++ code in the enc_class source block.  Emit functions will
1784   // live in the main source block for now.  In future, we can
1785   // generalize this by adding a syntax that specifies the sizes of
1786   // fields in an order, so that the adlc can build the emit functions
1787   // automagically
1788 
1789   // Emit primary opcode
1790   enc_class OpcP
1791   %{
1792     emit_opcode(cbuf, $primary);
1793   %}
1794 
1795   // Emit secondary opcode
1796   enc_class OpcS
1797   %{
1798     emit_opcode(cbuf, $secondary);
1799   %}
1800 
1801   // Emit tertiary opcode
1802   enc_class OpcT
1803   %{
1804     emit_opcode(cbuf, $tertiary);
1805   %}
1806 
1807   // Emit opcode directly
1808   enc_class Opcode(immI d8)
1809   %{
1810     emit_opcode(cbuf, $d8$$constant);
1811   %}
1812 
1813   // Emit size prefix
1814   enc_class SizePrefix
1815   %{
1816     emit_opcode(cbuf, 0x66);
1817   %}
1818 
1819   enc_class reg(rRegI reg)
1820   %{
1821     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1822   %}
1823 
1824   enc_class reg_reg(rRegI dst, rRegI src)
1825   %{
1826     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1827   %}
1828 
1829   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1830   %{
1831     emit_opcode(cbuf, $opcode$$constant);
1832     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1833   %}
1834 
1835   enc_class cdql_enc(no_rax_rdx_RegI div)
1836   %{
1837     // Full implementation of Java idiv and irem; checks for
1838     // special case as described in JVM spec., p.243 & p.271.
1839     //
1840     //         normal case                           special case
1841     //
1842     // input : rax: dividend                         min_int
1843     //         reg: divisor                          -1
1844     //
1845     // output: rax: quotient  (= rax idiv reg)       min_int
1846     //         rdx: remainder (= rax irem reg)       0
1847     //
1848     //  Code sequnce:
1849     //
1850     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1851     //    5:   75 07/08                jne    e <normal>
1852     //    7:   33 d2                   xor    %edx,%edx
1853     //  [div >= 8 -> offset + 1]
1854     //  [REX_B]
1855     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1856     //    c:   74 03/04                je     11 <done>
1857     // 000000000000000e <normal>:
1858     //    e:   99                      cltd
1859     //  [div >= 8 -> offset + 1]
1860     //  [REX_B]
1861     //    f:   f7 f9                   idiv   $div
1862     // 0000000000000011 <done>:
1863 
1864     // cmp    $0x80000000,%eax
1865     emit_opcode(cbuf, 0x3d);
1866     emit_d8(cbuf, 0x00);
1867     emit_d8(cbuf, 0x00);
1868     emit_d8(cbuf, 0x00);
1869     emit_d8(cbuf, 0x80);
1870 
1871     // jne    e <normal>
1872     emit_opcode(cbuf, 0x75);
1873     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1874 
1875     // xor    %edx,%edx
1876     emit_opcode(cbuf, 0x33);
1877     emit_d8(cbuf, 0xD2);
1878 
1879     // cmp    $0xffffffffffffffff,%ecx
1880     if ($div$$reg >= 8) {
1881       emit_opcode(cbuf, Assembler::REX_B);
1882     }
1883     emit_opcode(cbuf, 0x83);
1884     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1885     emit_d8(cbuf, 0xFF);
1886 
1887     // je     11 <done>
1888     emit_opcode(cbuf, 0x74);
1889     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1890 
1891     // <normal>
1892     // cltd
1893     emit_opcode(cbuf, 0x99);
1894 
1895     // idivl (note: must be emitted by the user of this rule)
1896     // <done>
1897   %}
1898 
1899   enc_class cdqq_enc(no_rax_rdx_RegL div)
1900   %{
1901     // Full implementation of Java ldiv and lrem; checks for
1902     // special case as described in JVM spec., p.243 & p.271.
1903     //
1904     //         normal case                           special case
1905     //
1906     // input : rax: dividend                         min_long
1907     //         reg: divisor                          -1
1908     //
1909     // output: rax: quotient  (= rax idiv reg)       min_long
1910     //         rdx: remainder (= rax irem reg)       0
1911     //
1912     //  Code sequnce:
1913     //
1914     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1915     //    7:   00 00 80
1916     //    a:   48 39 d0                cmp    %rdx,%rax
1917     //    d:   75 08                   jne    17 <normal>
1918     //    f:   33 d2                   xor    %edx,%edx
1919     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1920     //   15:   74 05                   je     1c <done>
1921     // 0000000000000017 <normal>:
1922     //   17:   48 99                   cqto
1923     //   19:   48 f7 f9                idiv   $div
1924     // 000000000000001c <done>:
1925 
1926     // mov    $0x8000000000000000,%rdx
1927     emit_opcode(cbuf, Assembler::REX_W);
1928     emit_opcode(cbuf, 0xBA);
1929     emit_d8(cbuf, 0x00);
1930     emit_d8(cbuf, 0x00);
1931     emit_d8(cbuf, 0x00);
1932     emit_d8(cbuf, 0x00);
1933     emit_d8(cbuf, 0x00);
1934     emit_d8(cbuf, 0x00);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x80);
1937 
1938     // cmp    %rdx,%rax
1939     emit_opcode(cbuf, Assembler::REX_W);
1940     emit_opcode(cbuf, 0x39);
1941     emit_d8(cbuf, 0xD0);
1942 
1943     // jne    17 <normal>
1944     emit_opcode(cbuf, 0x75);
1945     emit_d8(cbuf, 0x08);
1946 
1947     // xor    %edx,%edx
1948     emit_opcode(cbuf, 0x33);
1949     emit_d8(cbuf, 0xD2);
1950 
1951     // cmp    $0xffffffffffffffff,$div
1952     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1953     emit_opcode(cbuf, 0x83);
1954     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1955     emit_d8(cbuf, 0xFF);
1956 
1957     // je     1e <done>
1958     emit_opcode(cbuf, 0x74);
1959     emit_d8(cbuf, 0x05);
1960 
1961     // <normal>
1962     // cqto
1963     emit_opcode(cbuf, Assembler::REX_W);
1964     emit_opcode(cbuf, 0x99);
1965 
1966     // idivq (note: must be emitted by the user of this rule)
1967     // <done>
1968   %}
1969 
1970   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1971   enc_class OpcSE(immI imm)
1972   %{
1973     // Emit primary opcode and set sign-extend bit
1974     // Check for 8-bit immediate, and set sign extend bit in opcode
1975     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1976       emit_opcode(cbuf, $primary | 0x02);
1977     } else {
1978       // 32-bit immediate
1979       emit_opcode(cbuf, $primary);
1980     }
1981   %}
1982 
1983   enc_class OpcSErm(rRegI dst, immI imm)
1984   %{
1985     // OpcSEr/m
1986     int dstenc = $dst$$reg;
1987     if (dstenc >= 8) {
1988       emit_opcode(cbuf, Assembler::REX_B);
1989       dstenc -= 8;
1990     }
1991     // Emit primary opcode and set sign-extend bit
1992     // Check for 8-bit immediate, and set sign extend bit in opcode
1993     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1994       emit_opcode(cbuf, $primary | 0x02);
1995     } else {
1996       // 32-bit immediate
1997       emit_opcode(cbuf, $primary);
1998     }
1999     // Emit r/m byte with secondary opcode, after primary opcode.
2000     emit_rm(cbuf, 0x3, $secondary, dstenc);
2001   %}
2002 
2003   enc_class OpcSErm_wide(rRegL dst, immI imm)
2004   %{
2005     // OpcSEr/m
2006     int dstenc = $dst$$reg;
2007     if (dstenc < 8) {
2008       emit_opcode(cbuf, Assembler::REX_W);
2009     } else {
2010       emit_opcode(cbuf, Assembler::REX_WB);
2011       dstenc -= 8;
2012     }
2013     // Emit primary opcode and set sign-extend bit
2014     // Check for 8-bit immediate, and set sign extend bit in opcode
2015     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2016       emit_opcode(cbuf, $primary | 0x02);
2017     } else {
2018       // 32-bit immediate
2019       emit_opcode(cbuf, $primary);
2020     }
2021     // Emit r/m byte with secondary opcode, after primary opcode.
2022     emit_rm(cbuf, 0x3, $secondary, dstenc);
2023   %}
2024 
2025   enc_class Con8or32(immI imm)
2026   %{
2027     // Check for 8-bit immediate, and set sign extend bit in opcode
2028     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2029       $$$emit8$imm$$constant;
2030     } else {
2031       // 32-bit immediate
2032       $$$emit32$imm$$constant;
2033     }
2034   %}
2035 
2036   enc_class opc2_reg(rRegI dst)
2037   %{
2038     // BSWAP
2039     emit_cc(cbuf, $secondary, $dst$$reg);
2040   %}
2041 
2042   enc_class opc3_reg(rRegI dst)
2043   %{
2044     // BSWAP
2045     emit_cc(cbuf, $tertiary, $dst$$reg);
2046   %}
2047 
2048   enc_class reg_opc(rRegI div)
2049   %{
2050     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2051     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2052   %}
2053 
2054   enc_class enc_cmov(cmpOp cop)
2055   %{
2056     // CMOV
2057     $$$emit8$primary;
2058     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2059   %}
2060 
2061   enc_class enc_PartialSubtypeCheck()
2062   %{
2063     Register Rrdi = as_Register(RDI_enc); // result register
2064     Register Rrax = as_Register(RAX_enc); // super class
2065     Register Rrcx = as_Register(RCX_enc); // killed
2066     Register Rrsi = as_Register(RSI_enc); // sub class
2067     Label miss;
2068     const bool set_cond_codes = true;
2069 
2070     MacroAssembler _masm(&cbuf);
2071     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2072                                      NULL, &miss,
2073                                      /*set_cond_codes:*/ true);
2074     if ($primary) {
2075       __ xorptr(Rrdi, Rrdi);
2076     }
2077     __ bind(miss);
2078   %}
2079 
2080   enc_class clear_avx %{
2081     debug_only(int off0 = cbuf.insts_size());
2082     if (ra_->C->max_vector_size() > 16) {
2083       // Clear upper bits of YMM registers when current compiled code uses
2084       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2085       MacroAssembler _masm(&cbuf);
2086       __ vzeroupper();
2087     }
2088     debug_only(int off1 = cbuf.insts_size());
2089     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2090   %}
2091 
2092   enc_class Java_To_Runtime(method meth) %{
2093     // No relocation needed
2094     MacroAssembler _masm(&cbuf);
2095     __ mov64(r10, (int64_t) $meth$$method);
2096     __ call(r10);
2097   %}
2098 
2099   enc_class Java_To_Interpreter(method meth)
2100   %{
2101     // CALL Java_To_Interpreter
2102     // This is the instruction starting address for relocation info.
2103     cbuf.set_insts_mark();
2104     $$$emit8$primary;
2105     // CALL directly to the runtime
2106     emit_d32_reloc(cbuf,
2107                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2108                    runtime_call_Relocation::spec(),
2109                    RELOC_DISP32);
2110   %}
2111 
2112   enc_class Java_Static_Call(method meth)
2113   %{
2114     // JAVA STATIC CALL
2115     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2116     // determine who we intended to call.
2117     cbuf.set_insts_mark();
2118     $$$emit8$primary;
2119 
2120     if (!_method) {
2121       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2122                      runtime_call_Relocation::spec(),
2123                      RELOC_DISP32);
2124     } else {
2125       int method_index = resolved_method_index(cbuf);
2126       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2127                                                   : static_call_Relocation::spec(method_index);
2128       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2129                      rspec, RELOC_DISP32);
2130       // Emit stubs for static call.
2131       address mark = cbuf.insts_mark();
2132       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2133       if (stub == NULL) {
2134         ciEnv::current()->record_failure("CodeCache is full");
2135         return;
2136       }
2137     }
2138   %}
2139 
2140   enc_class Java_Dynamic_Call(method meth) %{
2141     MacroAssembler _masm(&cbuf);
2142     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2143   %}
2144 
2145   enc_class Java_Compiled_Call(method meth)
2146   %{
2147     // JAVA COMPILED CALL
2148     int disp = in_bytes(Method:: from_compiled_offset());
2149 
2150     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2151     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2152 
2153     // callq *disp(%rax)
2154     cbuf.set_insts_mark();
2155     $$$emit8$primary;
2156     if (disp < 0x80) {
2157       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2158       emit_d8(cbuf, disp); // Displacement
2159     } else {
2160       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2161       emit_d32(cbuf, disp); // Displacement
2162     }
2163   %}
2164 
2165   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2166   %{
2167     // SAL, SAR, SHR
2168     int dstenc = $dst$$reg;
2169     if (dstenc >= 8) {
2170       emit_opcode(cbuf, Assembler::REX_B);
2171       dstenc -= 8;
2172     }
2173     $$$emit8$primary;
2174     emit_rm(cbuf, 0x3, $secondary, dstenc);
2175     $$$emit8$shift$$constant;
2176   %}
2177 
2178   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2179   %{
2180     // SAL, SAR, SHR
2181     int dstenc = $dst$$reg;
2182     if (dstenc < 8) {
2183       emit_opcode(cbuf, Assembler::REX_W);
2184     } else {
2185       emit_opcode(cbuf, Assembler::REX_WB);
2186       dstenc -= 8;
2187     }
2188     $$$emit8$primary;
2189     emit_rm(cbuf, 0x3, $secondary, dstenc);
2190     $$$emit8$shift$$constant;
2191   %}
2192 
2193   enc_class load_immI(rRegI dst, immI src)
2194   %{
2195     int dstenc = $dst$$reg;
2196     if (dstenc >= 8) {
2197       emit_opcode(cbuf, Assembler::REX_B);
2198       dstenc -= 8;
2199     }
2200     emit_opcode(cbuf, 0xB8 | dstenc);
2201     $$$emit32$src$$constant;
2202   %}
2203 
2204   enc_class load_immL(rRegL dst, immL src)
2205   %{
2206     int dstenc = $dst$$reg;
2207     if (dstenc < 8) {
2208       emit_opcode(cbuf, Assembler::REX_W);
2209     } else {
2210       emit_opcode(cbuf, Assembler::REX_WB);
2211       dstenc -= 8;
2212     }
2213     emit_opcode(cbuf, 0xB8 | dstenc);
2214     emit_d64(cbuf, $src$$constant);
2215   %}
2216 
2217   enc_class load_immUL32(rRegL dst, immUL32 src)
2218   %{
2219     // same as load_immI, but this time we care about zeroes in the high word
2220     int dstenc = $dst$$reg;
2221     if (dstenc >= 8) {
2222       emit_opcode(cbuf, Assembler::REX_B);
2223       dstenc -= 8;
2224     }
2225     emit_opcode(cbuf, 0xB8 | dstenc);
2226     $$$emit32$src$$constant;
2227   %}
2228 
2229   enc_class load_immL32(rRegL dst, immL32 src)
2230   %{
2231     int dstenc = $dst$$reg;
2232     if (dstenc < 8) {
2233       emit_opcode(cbuf, Assembler::REX_W);
2234     } else {
2235       emit_opcode(cbuf, Assembler::REX_WB);
2236       dstenc -= 8;
2237     }
2238     emit_opcode(cbuf, 0xC7);
2239     emit_rm(cbuf, 0x03, 0x00, dstenc);
2240     $$$emit32$src$$constant;
2241   %}
2242 
2243   enc_class load_immP31(rRegP dst, immP32 src)
2244   %{
2245     // same as load_immI, but this time we care about zeroes in the high word
2246     int dstenc = $dst$$reg;
2247     if (dstenc >= 8) {
2248       emit_opcode(cbuf, Assembler::REX_B);
2249       dstenc -= 8;
2250     }
2251     emit_opcode(cbuf, 0xB8 | dstenc);
2252     $$$emit32$src$$constant;
2253   %}
2254 
2255   enc_class load_immP(rRegP dst, immP src)
2256   %{
2257     int dstenc = $dst$$reg;
2258     if (dstenc < 8) {
2259       emit_opcode(cbuf, Assembler::REX_W);
2260     } else {
2261       emit_opcode(cbuf, Assembler::REX_WB);
2262       dstenc -= 8;
2263     }
2264     emit_opcode(cbuf, 0xB8 | dstenc);
2265     // This next line should be generated from ADLC
2266     if ($src->constant_reloc() != relocInfo::none) {
2267       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2268     } else {
2269       emit_d64(cbuf, $src$$constant);
2270     }
2271   %}
2272 
2273   enc_class Con32(immI src)
2274   %{
2275     // Output immediate
2276     $$$emit32$src$$constant;
2277   %}
2278 
2279   enc_class Con32F_as_bits(immF src)
2280   %{
2281     // Output Float immediate bits
2282     jfloat jf = $src$$constant;
2283     jint jf_as_bits = jint_cast(jf);
2284     emit_d32(cbuf, jf_as_bits);
2285   %}
2286 
2287   enc_class Con16(immI src)
2288   %{
2289     // Output immediate
2290     $$$emit16$src$$constant;
2291   %}
2292 
2293   // How is this different from Con32??? XXX
2294   enc_class Con_d32(immI src)
2295   %{
2296     emit_d32(cbuf,$src$$constant);
2297   %}
2298 
2299   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2300     // Output immediate memory reference
2301     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2302     emit_d32(cbuf, 0x00);
2303   %}
2304 
2305   enc_class lock_prefix()
2306   %{
2307     if (os::is_MP()) {
2308       emit_opcode(cbuf, 0xF0); // lock
2309     }
2310   %}
2311 
2312   enc_class REX_mem(memory mem)
2313   %{
2314     if ($mem$$base >= 8) {
2315       if ($mem$$index < 8) {
2316         emit_opcode(cbuf, Assembler::REX_B);
2317       } else {
2318         emit_opcode(cbuf, Assembler::REX_XB);
2319       }
2320     } else {
2321       if ($mem$$index >= 8) {
2322         emit_opcode(cbuf, Assembler::REX_X);
2323       }
2324     }
2325   %}
2326 
2327   enc_class REX_mem_wide(memory mem)
2328   %{
2329     if ($mem$$base >= 8) {
2330       if ($mem$$index < 8) {
2331         emit_opcode(cbuf, Assembler::REX_WB);
2332       } else {
2333         emit_opcode(cbuf, Assembler::REX_WXB);
2334       }
2335     } else {
2336       if ($mem$$index < 8) {
2337         emit_opcode(cbuf, Assembler::REX_W);
2338       } else {
2339         emit_opcode(cbuf, Assembler::REX_WX);
2340       }
2341     }
2342   %}
2343 
2344   // for byte regs
2345   enc_class REX_breg(rRegI reg)
2346   %{
2347     if ($reg$$reg >= 4) {
2348       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2349     }
2350   %}
2351 
2352   // for byte regs
2353   enc_class REX_reg_breg(rRegI dst, rRegI src)
2354   %{
2355     if ($dst$$reg < 8) {
2356       if ($src$$reg >= 4) {
2357         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2358       }
2359     } else {
2360       if ($src$$reg < 8) {
2361         emit_opcode(cbuf, Assembler::REX_R);
2362       } else {
2363         emit_opcode(cbuf, Assembler::REX_RB);
2364       }
2365     }
2366   %}
2367 
2368   // for byte regs
2369   enc_class REX_breg_mem(rRegI reg, memory mem)
2370   %{
2371     if ($reg$$reg < 8) {
2372       if ($mem$$base < 8) {
2373         if ($mem$$index >= 8) {
2374           emit_opcode(cbuf, Assembler::REX_X);
2375         } else if ($reg$$reg >= 4) {
2376           emit_opcode(cbuf, Assembler::REX);
2377         }
2378       } else {
2379         if ($mem$$index < 8) {
2380           emit_opcode(cbuf, Assembler::REX_B);
2381         } else {
2382           emit_opcode(cbuf, Assembler::REX_XB);
2383         }
2384       }
2385     } else {
2386       if ($mem$$base < 8) {
2387         if ($mem$$index < 8) {
2388           emit_opcode(cbuf, Assembler::REX_R);
2389         } else {
2390           emit_opcode(cbuf, Assembler::REX_RX);
2391         }
2392       } else {
2393         if ($mem$$index < 8) {
2394           emit_opcode(cbuf, Assembler::REX_RB);
2395         } else {
2396           emit_opcode(cbuf, Assembler::REX_RXB);
2397         }
2398       }
2399     }
2400   %}
2401 
2402   enc_class REX_reg(rRegI reg)
2403   %{
2404     if ($reg$$reg >= 8) {
2405       emit_opcode(cbuf, Assembler::REX_B);
2406     }
2407   %}
2408 
2409   enc_class REX_reg_wide(rRegI reg)
2410   %{
2411     if ($reg$$reg < 8) {
2412       emit_opcode(cbuf, Assembler::REX_W);
2413     } else {
2414       emit_opcode(cbuf, Assembler::REX_WB);
2415     }
2416   %}
2417 
2418   enc_class REX_reg_reg(rRegI dst, rRegI src)
2419   %{
2420     if ($dst$$reg < 8) {
2421       if ($src$$reg >= 8) {
2422         emit_opcode(cbuf, Assembler::REX_B);
2423       }
2424     } else {
2425       if ($src$$reg < 8) {
2426         emit_opcode(cbuf, Assembler::REX_R);
2427       } else {
2428         emit_opcode(cbuf, Assembler::REX_RB);
2429       }
2430     }
2431   %}
2432 
2433   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2434   %{
2435     if ($dst$$reg < 8) {
2436       if ($src$$reg < 8) {
2437         emit_opcode(cbuf, Assembler::REX_W);
2438       } else {
2439         emit_opcode(cbuf, Assembler::REX_WB);
2440       }
2441     } else {
2442       if ($src$$reg < 8) {
2443         emit_opcode(cbuf, Assembler::REX_WR);
2444       } else {
2445         emit_opcode(cbuf, Assembler::REX_WRB);
2446       }
2447     }
2448   %}
2449 
2450   enc_class REX_reg_mem(rRegI reg, memory mem)
2451   %{
2452     if ($reg$$reg < 8) {
2453       if ($mem$$base < 8) {
2454         if ($mem$$index >= 8) {
2455           emit_opcode(cbuf, Assembler::REX_X);
2456         }
2457       } else {
2458         if ($mem$$index < 8) {
2459           emit_opcode(cbuf, Assembler::REX_B);
2460         } else {
2461           emit_opcode(cbuf, Assembler::REX_XB);
2462         }
2463       }
2464     } else {
2465       if ($mem$$base < 8) {
2466         if ($mem$$index < 8) {
2467           emit_opcode(cbuf, Assembler::REX_R);
2468         } else {
2469           emit_opcode(cbuf, Assembler::REX_RX);
2470         }
2471       } else {
2472         if ($mem$$index < 8) {
2473           emit_opcode(cbuf, Assembler::REX_RB);
2474         } else {
2475           emit_opcode(cbuf, Assembler::REX_RXB);
2476         }
2477       }
2478     }
2479   %}
2480 
2481   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2482   %{
2483     if ($reg$$reg < 8) {
2484       if ($mem$$base < 8) {
2485         if ($mem$$index < 8) {
2486           emit_opcode(cbuf, Assembler::REX_W);
2487         } else {
2488           emit_opcode(cbuf, Assembler::REX_WX);
2489         }
2490       } else {
2491         if ($mem$$index < 8) {
2492           emit_opcode(cbuf, Assembler::REX_WB);
2493         } else {
2494           emit_opcode(cbuf, Assembler::REX_WXB);
2495         }
2496       }
2497     } else {
2498       if ($mem$$base < 8) {
2499         if ($mem$$index < 8) {
2500           emit_opcode(cbuf, Assembler::REX_WR);
2501         } else {
2502           emit_opcode(cbuf, Assembler::REX_WRX);
2503         }
2504       } else {
2505         if ($mem$$index < 8) {
2506           emit_opcode(cbuf, Assembler::REX_WRB);
2507         } else {
2508           emit_opcode(cbuf, Assembler::REX_WRXB);
2509         }
2510       }
2511     }
2512   %}
2513 
2514   enc_class reg_mem(rRegI ereg, memory mem)
2515   %{
2516     // High registers handle in encode_RegMem
2517     int reg = $ereg$$reg;
2518     int base = $mem$$base;
2519     int index = $mem$$index;
2520     int scale = $mem$$scale;
2521     int disp = $mem$$disp;
2522     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2523 
2524     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2525   %}
2526 
2527   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2528   %{
2529     int rm_byte_opcode = $rm_opcode$$constant;
2530 
2531     // High registers handle in encode_RegMem
2532     int base = $mem$$base;
2533     int index = $mem$$index;
2534     int scale = $mem$$scale;
2535     int displace = $mem$$disp;
2536 
2537     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2538                                             // working with static
2539                                             // globals
2540     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2541                   disp_reloc);
2542   %}
2543 
2544   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2545   %{
2546     int reg_encoding = $dst$$reg;
2547     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2548     int index        = 0x04;            // 0x04 indicates no index
2549     int scale        = 0x00;            // 0x00 indicates no scale
2550     int displace     = $src1$$constant; // 0x00 indicates no displacement
2551     relocInfo::relocType disp_reloc = relocInfo::none;
2552     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2553                   disp_reloc);
2554   %}
2555 
2556   enc_class neg_reg(rRegI dst)
2557   %{
2558     int dstenc = $dst$$reg;
2559     if (dstenc >= 8) {
2560       emit_opcode(cbuf, Assembler::REX_B);
2561       dstenc -= 8;
2562     }
2563     // NEG $dst
2564     emit_opcode(cbuf, 0xF7);
2565     emit_rm(cbuf, 0x3, 0x03, dstenc);
2566   %}
2567 
2568   enc_class neg_reg_wide(rRegI dst)
2569   %{
2570     int dstenc = $dst$$reg;
2571     if (dstenc < 8) {
2572       emit_opcode(cbuf, Assembler::REX_W);
2573     } else {
2574       emit_opcode(cbuf, Assembler::REX_WB);
2575       dstenc -= 8;
2576     }
2577     // NEG $dst
2578     emit_opcode(cbuf, 0xF7);
2579     emit_rm(cbuf, 0x3, 0x03, dstenc);
2580   %}
2581 
2582   enc_class setLT_reg(rRegI dst)
2583   %{
2584     int dstenc = $dst$$reg;
2585     if (dstenc >= 8) {
2586       emit_opcode(cbuf, Assembler::REX_B);
2587       dstenc -= 8;
2588     } else if (dstenc >= 4) {
2589       emit_opcode(cbuf, Assembler::REX);
2590     }
2591     // SETLT $dst
2592     emit_opcode(cbuf, 0x0F);
2593     emit_opcode(cbuf, 0x9C);
2594     emit_rm(cbuf, 0x3, 0x0, dstenc);
2595   %}
2596 
2597   enc_class setNZ_reg(rRegI dst)
2598   %{
2599     int dstenc = $dst$$reg;
2600     if (dstenc >= 8) {
2601       emit_opcode(cbuf, Assembler::REX_B);
2602       dstenc -= 8;
2603     } else if (dstenc >= 4) {
2604       emit_opcode(cbuf, Assembler::REX);
2605     }
2606     // SETNZ $dst
2607     emit_opcode(cbuf, 0x0F);
2608     emit_opcode(cbuf, 0x95);
2609     emit_rm(cbuf, 0x3, 0x0, dstenc);
2610   %}
2611 
2612 
2613   // Compare the lonogs and set -1, 0, or 1 into dst
2614   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2615   %{
2616     int src1enc = $src1$$reg;
2617     int src2enc = $src2$$reg;
2618     int dstenc = $dst$$reg;
2619 
2620     // cmpq $src1, $src2
2621     if (src1enc < 8) {
2622       if (src2enc < 8) {
2623         emit_opcode(cbuf, Assembler::REX_W);
2624       } else {
2625         emit_opcode(cbuf, Assembler::REX_WB);
2626       }
2627     } else {
2628       if (src2enc < 8) {
2629         emit_opcode(cbuf, Assembler::REX_WR);
2630       } else {
2631         emit_opcode(cbuf, Assembler::REX_WRB);
2632       }
2633     }
2634     emit_opcode(cbuf, 0x3B);
2635     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2636 
2637     // movl $dst, -1
2638     if (dstenc >= 8) {
2639       emit_opcode(cbuf, Assembler::REX_B);
2640     }
2641     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2642     emit_d32(cbuf, -1);
2643 
2644     // jl,s done
2645     emit_opcode(cbuf, 0x7C);
2646     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2647 
2648     // setne $dst
2649     if (dstenc >= 4) {
2650       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2651     }
2652     emit_opcode(cbuf, 0x0F);
2653     emit_opcode(cbuf, 0x95);
2654     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2655 
2656     // movzbl $dst, $dst
2657     if (dstenc >= 4) {
2658       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2659     }
2660     emit_opcode(cbuf, 0x0F);
2661     emit_opcode(cbuf, 0xB6);
2662     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2663   %}
2664 
2665   enc_class Push_ResultXD(regD dst) %{
2666     MacroAssembler _masm(&cbuf);
2667     __ fstp_d(Address(rsp, 0));
2668     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2669     __ addptr(rsp, 8);
2670   %}
2671 
2672   enc_class Push_SrcXD(regD src) %{
2673     MacroAssembler _masm(&cbuf);
2674     __ subptr(rsp, 8);
2675     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2676     __ fld_d(Address(rsp, 0));
2677   %}
2678 
2679 
2680   enc_class enc_rethrow()
2681   %{
2682     cbuf.set_insts_mark();
2683     emit_opcode(cbuf, 0xE9); // jmp entry
2684     emit_d32_reloc(cbuf,
2685                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2686                    runtime_call_Relocation::spec(),
2687                    RELOC_DISP32);
2688   %}
2689 
2690 %}
2691 
2692 
2693 
2694 //----------FRAME--------------------------------------------------------------
2695 // Definition of frame structure and management information.
2696 //
2697 //  S T A C K   L A Y O U T    Allocators stack-slot number
2698 //                             |   (to get allocators register number
2699 //  G  Owned by    |        |  v    add OptoReg::stack0())
2700 //  r   CALLER     |        |
2701 //  o     |        +--------+      pad to even-align allocators stack-slot
2702 //  w     V        |  pad0  |        numbers; owned by CALLER
2703 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2704 //  h     ^        |   in   |  5
2705 //        |        |  args  |  4   Holes in incoming args owned by SELF
2706 //  |     |        |        |  3
2707 //  |     |        +--------+
2708 //  V     |        | old out|      Empty on Intel, window on Sparc
2709 //        |    old |preserve|      Must be even aligned.
2710 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2711 //        |        |   in   |  3   area for Intel ret address
2712 //     Owned by    |preserve|      Empty on Sparc.
2713 //       SELF      +--------+
2714 //        |        |  pad2  |  2   pad to align old SP
2715 //        |        +--------+  1
2716 //        |        | locks  |  0
2717 //        |        +--------+----> OptoReg::stack0(), even aligned
2718 //        |        |  pad1  | 11   pad to align new SP
2719 //        |        +--------+
2720 //        |        |        | 10
2721 //        |        | spills |  9   spills
2722 //        V        |        |  8   (pad0 slot for callee)
2723 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2724 //        ^        |  out   |  7
2725 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2726 //     Owned by    +--------+
2727 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2728 //        |    new |preserve|      Must be even-aligned.
2729 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2730 //        |        |        |
2731 //
2732 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2733 //         known from SELF's arguments and the Java calling convention.
2734 //         Region 6-7 is determined per call site.
2735 // Note 2: If the calling convention leaves holes in the incoming argument
2736 //         area, those holes are owned by SELF.  Holes in the outgoing area
2737 //         are owned by the CALLEE.  Holes should not be nessecary in the
2738 //         incoming area, as the Java calling convention is completely under
2739 //         the control of the AD file.  Doubles can be sorted and packed to
2740 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2741 //         varargs C calling conventions.
2742 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2743 //         even aligned with pad0 as needed.
2744 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2745 //         region 6-11 is even aligned; it may be padded out more so that
2746 //         the region from SP to FP meets the minimum stack alignment.
2747 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2748 //         alignment.  Region 11, pad1, may be dynamically extended so that
2749 //         SP meets the minimum alignment.
2750 
2751 frame
2752 %{
2753   // What direction does stack grow in (assumed to be same for C & Java)
2754   stack_direction(TOWARDS_LOW);
2755 
2756   // These three registers define part of the calling convention
2757   // between compiled code and the interpreter.
2758   inline_cache_reg(RAX);                // Inline Cache Register
2759   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2760                                         // calling interpreter
2761 
2762   // Optional: name the operand used by cisc-spilling to access
2763   // [stack_pointer + offset]
2764   cisc_spilling_operand_name(indOffset32);
2765 
2766   // Number of stack slots consumed by locking an object
2767   sync_stack_slots(2);
2768 
2769   // Compiled code's Frame Pointer
2770   frame_pointer(RSP);
2771 
2772   // Interpreter stores its frame pointer in a register which is
2773   // stored to the stack by I2CAdaptors.
2774   // I2CAdaptors convert from interpreted java to compiled java.
2775   interpreter_frame_pointer(RBP);
2776 
2777   // Stack alignment requirement
2778   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2779 
2780   // Number of stack slots between incoming argument block and the start of
2781   // a new frame.  The PROLOG must add this many slots to the stack.  The
2782   // EPILOG must remove this many slots.  amd64 needs two slots for
2783   // return address.
2784   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2785 
2786   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2787   // for calls to C.  Supports the var-args backing area for register parms.
2788   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2789 
2790   // The after-PROLOG location of the return address.  Location of
2791   // return address specifies a type (REG or STACK) and a number
2792   // representing the register number (i.e. - use a register name) or
2793   // stack slot.
2794   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2795   // Otherwise, it is above the locks and verification slot and alignment word
2796   return_addr(STACK - 2 +
2797               round_to((Compile::current()->in_preserve_stack_slots() +
2798                         Compile::current()->fixed_slots()),
2799                        stack_alignment_in_slots()));
2800 
2801   // Body of function which returns an integer array locating
2802   // arguments either in registers or in stack slots.  Passed an array
2803   // of ideal registers called "sig" and a "length" count.  Stack-slot
2804   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2805   // arguments for a CALLEE.  Incoming stack arguments are
2806   // automatically biased by the preserve_stack_slots field above.
2807 
2808   calling_convention
2809   %{
2810     // No difference between ingoing/outgoing just pass false
2811     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2812   %}
2813 
2814   c_calling_convention
2815   %{
2816     // This is obviously always outgoing
2817     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2818   %}
2819 
2820   // Location of compiled Java return values.  Same as C for now.
2821   return_value
2822   %{
2823     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2824            "only return normal values");
2825 
2826     static const int lo[Op_RegL + 1] = {
2827       0,
2828       0,
2829       RAX_num,  // Op_RegN
2830       RAX_num,  // Op_RegI
2831       RAX_num,  // Op_RegP
2832       XMM0_num, // Op_RegF
2833       XMM0_num, // Op_RegD
2834       RAX_num   // Op_RegL
2835     };
2836     static const int hi[Op_RegL + 1] = {
2837       0,
2838       0,
2839       OptoReg::Bad, // Op_RegN
2840       OptoReg::Bad, // Op_RegI
2841       RAX_H_num,    // Op_RegP
2842       OptoReg::Bad, // Op_RegF
2843       XMM0b_num,    // Op_RegD
2844       RAX_H_num     // Op_RegL
2845     };
2846     // Excluded flags and vector registers.
2847     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2848     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2849   %}
2850 %}
2851 
2852 //----------ATTRIBUTES---------------------------------------------------------
2853 //----------Operand Attributes-------------------------------------------------
2854 op_attrib op_cost(0);        // Required cost attribute
2855 
2856 //----------Instruction Attributes---------------------------------------------
2857 ins_attrib ins_cost(100);       // Required cost attribute
2858 ins_attrib ins_size(8);         // Required size attribute (in bits)
2859 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2860                                 // a non-matching short branch variant
2861                                 // of some long branch?
2862 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2863                                 // be a power of 2) specifies the
2864                                 // alignment that some part of the
2865                                 // instruction (not necessarily the
2866                                 // start) requires.  If > 1, a
2867                                 // compute_padding() function must be
2868                                 // provided for the instruction
2869 
2870 //----------OPERANDS-----------------------------------------------------------
2871 // Operand definitions must precede instruction definitions for correct parsing
2872 // in the ADLC because operands constitute user defined types which are used in
2873 // instruction definitions.
2874 
2875 //----------Simple Operands----------------------------------------------------
2876 // Immediate Operands
2877 // Integer Immediate
2878 operand immI()
2879 %{
2880   match(ConI);
2881 
2882   op_cost(10);
2883   format %{ %}
2884   interface(CONST_INTER);
2885 %}
2886 
2887 // Constant for test vs zero
2888 operand immI0()
2889 %{
2890   predicate(n->get_int() == 0);
2891   match(ConI);
2892 
2893   op_cost(0);
2894   format %{ %}
2895   interface(CONST_INTER);
2896 %}
2897 
2898 // Constant for increment
2899 operand immI1()
2900 %{
2901   predicate(n->get_int() == 1);
2902   match(ConI);
2903 
2904   op_cost(0);
2905   format %{ %}
2906   interface(CONST_INTER);
2907 %}
2908 
2909 // Constant for decrement
2910 operand immI_M1()
2911 %{
2912   predicate(n->get_int() == -1);
2913   match(ConI);
2914 
2915   op_cost(0);
2916   format %{ %}
2917   interface(CONST_INTER);
2918 %}
2919 
2920 // Valid scale values for addressing modes
2921 operand immI2()
2922 %{
2923   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2924   match(ConI);
2925 
2926   format %{ %}
2927   interface(CONST_INTER);
2928 %}
2929 
2930 operand immI8()
2931 %{
2932   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2933   match(ConI);
2934 
2935   op_cost(5);
2936   format %{ %}
2937   interface(CONST_INTER);
2938 %}
2939 
2940 operand immI16()
2941 %{
2942   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2943   match(ConI);
2944 
2945   op_cost(10);
2946   format %{ %}
2947   interface(CONST_INTER);
2948 %}
2949 
2950 // Int Immediate non-negative
2951 operand immU31()
2952 %{
2953   predicate(n->get_int() >= 0);
2954   match(ConI);
2955 
2956   op_cost(0);
2957   format %{ %}
2958   interface(CONST_INTER);
2959 %}
2960 
2961 // Constant for long shifts
2962 operand immI_32()
2963 %{
2964   predicate( n->get_int() == 32 );
2965   match(ConI);
2966 
2967   op_cost(0);
2968   format %{ %}
2969   interface(CONST_INTER);
2970 %}
2971 
2972 // Constant for long shifts
2973 operand immI_64()
2974 %{
2975   predicate( n->get_int() == 64 );
2976   match(ConI);
2977 
2978   op_cost(0);
2979   format %{ %}
2980   interface(CONST_INTER);
2981 %}
2982 
2983 // Pointer Immediate
2984 operand immP()
2985 %{
2986   match(ConP);
2987 
2988   op_cost(10);
2989   format %{ %}
2990   interface(CONST_INTER);
2991 %}
2992 
2993 // NULL Pointer Immediate
2994 operand immP0()
2995 %{
2996   predicate(n->get_ptr() == 0);
2997   match(ConP);
2998 
2999   op_cost(5);
3000   format %{ %}
3001   interface(CONST_INTER);
3002 %}
3003 
3004 // Pointer Immediate
3005 operand immN() %{
3006   match(ConN);
3007 
3008   op_cost(10);
3009   format %{ %}
3010   interface(CONST_INTER);
3011 %}
3012 
3013 operand immNKlass() %{
3014   match(ConNKlass);
3015 
3016   op_cost(10);
3017   format %{ %}
3018   interface(CONST_INTER);
3019 %}
3020 
3021 // NULL Pointer Immediate
3022 operand immN0() %{
3023   predicate(n->get_narrowcon() == 0);
3024   match(ConN);
3025 
3026   op_cost(5);
3027   format %{ %}
3028   interface(CONST_INTER);
3029 %}
3030 
3031 operand immP31()
3032 %{
3033   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3034             && (n->get_ptr() >> 31) == 0);
3035   match(ConP);
3036 
3037   op_cost(5);
3038   format %{ %}
3039   interface(CONST_INTER);
3040 %}
3041 
3042 
3043 // Long Immediate
3044 operand immL()
3045 %{
3046   match(ConL);
3047 
3048   op_cost(20);
3049   format %{ %}
3050   interface(CONST_INTER);
3051 %}
3052 
3053 // Long Immediate 8-bit
3054 operand immL8()
3055 %{
3056   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3057   match(ConL);
3058 
3059   op_cost(5);
3060   format %{ %}
3061   interface(CONST_INTER);
3062 %}
3063 
3064 // Long Immediate 32-bit unsigned
3065 operand immUL32()
3066 %{
3067   predicate(n->get_long() == (unsigned int) (n->get_long()));
3068   match(ConL);
3069 
3070   op_cost(10);
3071   format %{ %}
3072   interface(CONST_INTER);
3073 %}
3074 
3075 // Long Immediate 32-bit signed
3076 operand immL32()
3077 %{
3078   predicate(n->get_long() == (int) (n->get_long()));
3079   match(ConL);
3080 
3081   op_cost(15);
3082   format %{ %}
3083   interface(CONST_INTER);
3084 %}
3085 
3086 // Long Immediate zero
3087 operand immL0()
3088 %{
3089   predicate(n->get_long() == 0L);
3090   match(ConL);
3091 
3092   op_cost(10);
3093   format %{ %}
3094   interface(CONST_INTER);
3095 %}
3096 
3097 // Constant for increment
3098 operand immL1()
3099 %{
3100   predicate(n->get_long() == 1);
3101   match(ConL);
3102 
3103   format %{ %}
3104   interface(CONST_INTER);
3105 %}
3106 
3107 // Constant for decrement
3108 operand immL_M1()
3109 %{
3110   predicate(n->get_long() == -1);
3111   match(ConL);
3112 
3113   format %{ %}
3114   interface(CONST_INTER);
3115 %}
3116 
3117 // Long Immediate: the value 10
3118 operand immL10()
3119 %{
3120   predicate(n->get_long() == 10);
3121   match(ConL);
3122 
3123   format %{ %}
3124   interface(CONST_INTER);
3125 %}
3126 
3127 // Long immediate from 0 to 127.
3128 // Used for a shorter form of long mul by 10.
3129 operand immL_127()
3130 %{
3131   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3132   match(ConL);
3133 
3134   op_cost(10);
3135   format %{ %}
3136   interface(CONST_INTER);
3137 %}
3138 
3139 // Long Immediate: low 32-bit mask
3140 operand immL_32bits()
3141 %{
3142   predicate(n->get_long() == 0xFFFFFFFFL);
3143   match(ConL);
3144   op_cost(20);
3145 
3146   format %{ %}
3147   interface(CONST_INTER);
3148 %}
3149 
3150 // Float Immediate zero
3151 operand immF0()
3152 %{
3153   predicate(jint_cast(n->getf()) == 0);
3154   match(ConF);
3155 
3156   op_cost(5);
3157   format %{ %}
3158   interface(CONST_INTER);
3159 %}
3160 
3161 // Float Immediate
3162 operand immF()
3163 %{
3164   match(ConF);
3165 
3166   op_cost(15);
3167   format %{ %}
3168   interface(CONST_INTER);
3169 %}
3170 
3171 // Double Immediate zero
3172 operand immD0()
3173 %{
3174   predicate(jlong_cast(n->getd()) == 0);
3175   match(ConD);
3176 
3177   op_cost(5);
3178   format %{ %}
3179   interface(CONST_INTER);
3180 %}
3181 
3182 // Double Immediate
3183 operand immD()
3184 %{
3185   match(ConD);
3186 
3187   op_cost(15);
3188   format %{ %}
3189   interface(CONST_INTER);
3190 %}
3191 
3192 // Immediates for special shifts (sign extend)
3193 
3194 // Constants for increment
3195 operand immI_16()
3196 %{
3197   predicate(n->get_int() == 16);
3198   match(ConI);
3199 
3200   format %{ %}
3201   interface(CONST_INTER);
3202 %}
3203 
3204 operand immI_24()
3205 %{
3206   predicate(n->get_int() == 24);
3207   match(ConI);
3208 
3209   format %{ %}
3210   interface(CONST_INTER);
3211 %}
3212 
3213 // Constant for byte-wide masking
3214 operand immI_255()
3215 %{
3216   predicate(n->get_int() == 255);
3217   match(ConI);
3218 
3219   format %{ %}
3220   interface(CONST_INTER);
3221 %}
3222 
3223 // Constant for short-wide masking
3224 operand immI_65535()
3225 %{
3226   predicate(n->get_int() == 65535);
3227   match(ConI);
3228 
3229   format %{ %}
3230   interface(CONST_INTER);
3231 %}
3232 
3233 // Constant for byte-wide masking
3234 operand immL_255()
3235 %{
3236   predicate(n->get_long() == 255);
3237   match(ConL);
3238 
3239   format %{ %}
3240   interface(CONST_INTER);
3241 %}
3242 
3243 // Constant for short-wide masking
3244 operand immL_65535()
3245 %{
3246   predicate(n->get_long() == 65535);
3247   match(ConL);
3248 
3249   format %{ %}
3250   interface(CONST_INTER);
3251 %}
3252 
3253 // Register Operands
3254 // Integer Register
3255 operand rRegI()
3256 %{
3257   constraint(ALLOC_IN_RC(int_reg));
3258   match(RegI);
3259 
3260   match(rax_RegI);
3261   match(rbx_RegI);
3262   match(rcx_RegI);
3263   match(rdx_RegI);
3264   match(rdi_RegI);
3265 
3266   format %{ %}
3267   interface(REG_INTER);
3268 %}
3269 
3270 // Special Registers
3271 operand rax_RegI()
3272 %{
3273   constraint(ALLOC_IN_RC(int_rax_reg));
3274   match(RegI);
3275   match(rRegI);
3276 
3277   format %{ "RAX" %}
3278   interface(REG_INTER);
3279 %}
3280 
3281 // Special Registers
3282 operand rbx_RegI()
3283 %{
3284   constraint(ALLOC_IN_RC(int_rbx_reg));
3285   match(RegI);
3286   match(rRegI);
3287 
3288   format %{ "RBX" %}
3289   interface(REG_INTER);
3290 %}
3291 
3292 operand rcx_RegI()
3293 %{
3294   constraint(ALLOC_IN_RC(int_rcx_reg));
3295   match(RegI);
3296   match(rRegI);
3297 
3298   format %{ "RCX" %}
3299   interface(REG_INTER);
3300 %}
3301 
3302 operand rdx_RegI()
3303 %{
3304   constraint(ALLOC_IN_RC(int_rdx_reg));
3305   match(RegI);
3306   match(rRegI);
3307 
3308   format %{ "RDX" %}
3309   interface(REG_INTER);
3310 %}
3311 
3312 operand rdi_RegI()
3313 %{
3314   constraint(ALLOC_IN_RC(int_rdi_reg));
3315   match(RegI);
3316   match(rRegI);
3317 
3318   format %{ "RDI" %}
3319   interface(REG_INTER);
3320 %}
3321 
3322 operand no_rcx_RegI()
3323 %{
3324   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3325   match(RegI);
3326   match(rax_RegI);
3327   match(rbx_RegI);
3328   match(rdx_RegI);
3329   match(rdi_RegI);
3330 
3331   format %{ %}
3332   interface(REG_INTER);
3333 %}
3334 
3335 operand no_rax_rdx_RegI()
3336 %{
3337   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3338   match(RegI);
3339   match(rbx_RegI);
3340   match(rcx_RegI);
3341   match(rdi_RegI);
3342 
3343   format %{ %}
3344   interface(REG_INTER);
3345 %}
3346 
3347 // Pointer Register
3348 operand any_RegP()
3349 %{
3350   constraint(ALLOC_IN_RC(any_reg));
3351   match(RegP);
3352   match(rax_RegP);
3353   match(rbx_RegP);
3354   match(rdi_RegP);
3355   match(rsi_RegP);
3356   match(rbp_RegP);
3357   match(r15_RegP);
3358   match(rRegP);
3359 
3360   format %{ %}
3361   interface(REG_INTER);
3362 %}
3363 
3364 operand rRegP()
3365 %{
3366   constraint(ALLOC_IN_RC(ptr_reg));
3367   match(RegP);
3368   match(rax_RegP);
3369   match(rbx_RegP);
3370   match(rdi_RegP);
3371   match(rsi_RegP);
3372   match(rbp_RegP);  // See Q&A below about
3373   match(r15_RegP);  // r15_RegP and rbp_RegP.
3374 
3375   format %{ %}
3376   interface(REG_INTER);
3377 %}
3378 
3379 operand rRegN() %{
3380   constraint(ALLOC_IN_RC(int_reg));
3381   match(RegN);
3382 
3383   format %{ %}
3384   interface(REG_INTER);
3385 %}
3386 
3387 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3388 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3389 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3390 // The output of an instruction is controlled by the allocator, which respects
3391 // register class masks, not match rules.  Unless an instruction mentions
3392 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3393 // by the allocator as an input.
3394 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3395 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3396 // result, RBP is not included in the output of the instruction either.
3397 
3398 operand no_rax_RegP()
3399 %{
3400   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3401   match(RegP);
3402   match(rbx_RegP);
3403   match(rsi_RegP);
3404   match(rdi_RegP);
3405 
3406   format %{ %}
3407   interface(REG_INTER);
3408 %}
3409 
3410 // This operand is not allowed to use RBP even if
3411 // RBP is not used to hold the frame pointer.
3412 operand no_rbp_RegP()
3413 %{
3414   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3415   match(RegP);
3416   match(rbx_RegP);
3417   match(rsi_RegP);
3418   match(rdi_RegP);
3419 
3420   format %{ %}
3421   interface(REG_INTER);
3422 %}
3423 
3424 operand no_rax_rbx_RegP()
3425 %{
3426   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3427   match(RegP);
3428   match(rsi_RegP);
3429   match(rdi_RegP);
3430 
3431   format %{ %}
3432   interface(REG_INTER);
3433 %}
3434 
3435 // Special Registers
3436 // Return a pointer value
3437 operand rax_RegP()
3438 %{
3439   constraint(ALLOC_IN_RC(ptr_rax_reg));
3440   match(RegP);
3441   match(rRegP);
3442 
3443   format %{ %}
3444   interface(REG_INTER);
3445 %}
3446 
3447 // Special Registers
3448 // Return a compressed pointer value
3449 operand rax_RegN()
3450 %{
3451   constraint(ALLOC_IN_RC(int_rax_reg));
3452   match(RegN);
3453   match(rRegN);
3454 
3455   format %{ %}
3456   interface(REG_INTER);
3457 %}
3458 
3459 // Used in AtomicAdd
3460 operand rbx_RegP()
3461 %{
3462   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3463   match(RegP);
3464   match(rRegP);
3465 
3466   format %{ %}
3467   interface(REG_INTER);
3468 %}
3469 
3470 operand rsi_RegP()
3471 %{
3472   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3473   match(RegP);
3474   match(rRegP);
3475 
3476   format %{ %}
3477   interface(REG_INTER);
3478 %}
3479 
3480 // Used in rep stosq
3481 operand rdi_RegP()
3482 %{
3483   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3484   match(RegP);
3485   match(rRegP);
3486 
3487   format %{ %}
3488   interface(REG_INTER);
3489 %}
3490 
3491 operand r15_RegP()
3492 %{
3493   constraint(ALLOC_IN_RC(ptr_r15_reg));
3494   match(RegP);
3495   match(rRegP);
3496 
3497   format %{ %}
3498   interface(REG_INTER);
3499 %}
3500 
3501 operand rRegL()
3502 %{
3503   constraint(ALLOC_IN_RC(long_reg));
3504   match(RegL);
3505   match(rax_RegL);
3506   match(rdx_RegL);
3507 
3508   format %{ %}
3509   interface(REG_INTER);
3510 %}
3511 
3512 // Special Registers
3513 operand no_rax_rdx_RegL()
3514 %{
3515   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3516   match(RegL);
3517   match(rRegL);
3518 
3519   format %{ %}
3520   interface(REG_INTER);
3521 %}
3522 
3523 operand no_rax_RegL()
3524 %{
3525   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3526   match(RegL);
3527   match(rRegL);
3528   match(rdx_RegL);
3529 
3530   format %{ %}
3531   interface(REG_INTER);
3532 %}
3533 
3534 operand no_rcx_RegL()
3535 %{
3536   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3537   match(RegL);
3538   match(rRegL);
3539 
3540   format %{ %}
3541   interface(REG_INTER);
3542 %}
3543 
3544 operand rax_RegL()
3545 %{
3546   constraint(ALLOC_IN_RC(long_rax_reg));
3547   match(RegL);
3548   match(rRegL);
3549 
3550   format %{ "RAX" %}
3551   interface(REG_INTER);
3552 %}
3553 
3554 operand rcx_RegL()
3555 %{
3556   constraint(ALLOC_IN_RC(long_rcx_reg));
3557   match(RegL);
3558   match(rRegL);
3559 
3560   format %{ %}
3561   interface(REG_INTER);
3562 %}
3563 
3564 operand rdx_RegL()
3565 %{
3566   constraint(ALLOC_IN_RC(long_rdx_reg));
3567   match(RegL);
3568   match(rRegL);
3569 
3570   format %{ %}
3571   interface(REG_INTER);
3572 %}
3573 
3574 // Flags register, used as output of compare instructions
3575 operand rFlagsReg()
3576 %{
3577   constraint(ALLOC_IN_RC(int_flags));
3578   match(RegFlags);
3579 
3580   format %{ "RFLAGS" %}
3581   interface(REG_INTER);
3582 %}
3583 
3584 // Flags register, used as output of FLOATING POINT compare instructions
3585 operand rFlagsRegU()
3586 %{
3587   constraint(ALLOC_IN_RC(int_flags));
3588   match(RegFlags);
3589 
3590   format %{ "RFLAGS_U" %}
3591   interface(REG_INTER);
3592 %}
3593 
3594 operand rFlagsRegUCF() %{
3595   constraint(ALLOC_IN_RC(int_flags));
3596   match(RegFlags);
3597   predicate(false);
3598 
3599   format %{ "RFLAGS_U_CF" %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 // Float register operands
3604 operand regF() %{
3605    constraint(ALLOC_IN_RC(float_reg));
3606    match(RegF);
3607 
3608    format %{ %}
3609    interface(REG_INTER);
3610 %}
3611 
3612 // Double register operands
3613 operand regD() %{
3614    constraint(ALLOC_IN_RC(double_reg));
3615    match(RegD);
3616 
3617    format %{ %}
3618    interface(REG_INTER);
3619 %}
3620 
3621 // Vectors
3622 operand vecS() %{
3623   constraint(ALLOC_IN_RC(vectors_reg));
3624   match(VecS);
3625 
3626   format %{ %}
3627   interface(REG_INTER);
3628 %}
3629 
3630 operand vecD() %{
3631   constraint(ALLOC_IN_RC(vectord_reg));
3632   match(VecD);
3633 
3634   format %{ %}
3635   interface(REG_INTER);
3636 %}
3637 
3638 operand vecX() %{
3639   constraint(ALLOC_IN_RC(vectorx_reg));
3640   match(VecX);
3641 
3642   format %{ %}
3643   interface(REG_INTER);
3644 %}
3645 
3646 operand vecY() %{
3647   constraint(ALLOC_IN_RC(vectory_reg));
3648   match(VecY);
3649 
3650   format %{ %}
3651   interface(REG_INTER);
3652 %}
3653 
3654 //----------Memory Operands----------------------------------------------------
3655 // Direct Memory Operand
3656 // operand direct(immP addr)
3657 // %{
3658 //   match(addr);
3659 
3660 //   format %{ "[$addr]" %}
3661 //   interface(MEMORY_INTER) %{
3662 //     base(0xFFFFFFFF);
3663 //     index(0x4);
3664 //     scale(0x0);
3665 //     disp($addr);
3666 //   %}
3667 // %}
3668 
3669 // Indirect Memory Operand
3670 operand indirect(any_RegP reg)
3671 %{
3672   constraint(ALLOC_IN_RC(ptr_reg));
3673   match(reg);
3674 
3675   format %{ "[$reg]" %}
3676   interface(MEMORY_INTER) %{
3677     base($reg);
3678     index(0x4);
3679     scale(0x0);
3680     disp(0x0);
3681   %}
3682 %}
3683 
3684 // Indirect Memory Plus Short Offset Operand
3685 operand indOffset8(any_RegP reg, immL8 off)
3686 %{
3687   constraint(ALLOC_IN_RC(ptr_reg));
3688   match(AddP reg off);
3689 
3690   format %{ "[$reg + $off (8-bit)]" %}
3691   interface(MEMORY_INTER) %{
3692     base($reg);
3693     index(0x4);
3694     scale(0x0);
3695     disp($off);
3696   %}
3697 %}
3698 
3699 // Indirect Memory Plus Long Offset Operand
3700 operand indOffset32(any_RegP reg, immL32 off)
3701 %{
3702   constraint(ALLOC_IN_RC(ptr_reg));
3703   match(AddP reg off);
3704 
3705   format %{ "[$reg + $off (32-bit)]" %}
3706   interface(MEMORY_INTER) %{
3707     base($reg);
3708     index(0x4);
3709     scale(0x0);
3710     disp($off);
3711   %}
3712 %}
3713 
3714 // Indirect Memory Plus Index Register Plus Offset Operand
3715 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3716 %{
3717   constraint(ALLOC_IN_RC(ptr_reg));
3718   match(AddP (AddP reg lreg) off);
3719 
3720   op_cost(10);
3721   format %{"[$reg + $off + $lreg]" %}
3722   interface(MEMORY_INTER) %{
3723     base($reg);
3724     index($lreg);
3725     scale(0x0);
3726     disp($off);
3727   %}
3728 %}
3729 
3730 // Indirect Memory Plus Index Register Plus Offset Operand
3731 operand indIndex(any_RegP reg, rRegL lreg)
3732 %{
3733   constraint(ALLOC_IN_RC(ptr_reg));
3734   match(AddP reg lreg);
3735 
3736   op_cost(10);
3737   format %{"[$reg + $lreg]" %}
3738   interface(MEMORY_INTER) %{
3739     base($reg);
3740     index($lreg);
3741     scale(0x0);
3742     disp(0x0);
3743   %}
3744 %}
3745 
3746 // Indirect Memory Times Scale Plus Index Register
3747 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3748 %{
3749   constraint(ALLOC_IN_RC(ptr_reg));
3750   match(AddP reg (LShiftL lreg scale));
3751 
3752   op_cost(10);
3753   format %{"[$reg + $lreg << $scale]" %}
3754   interface(MEMORY_INTER) %{
3755     base($reg);
3756     index($lreg);
3757     scale($scale);
3758     disp(0x0);
3759   %}
3760 %}
3761 
3762 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3763 %{
3764   constraint(ALLOC_IN_RC(ptr_reg));
3765   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3766   match(AddP reg (LShiftL (ConvI2L idx) scale));
3767 
3768   op_cost(10);
3769   format %{"[$reg + pos $idx << $scale]" %}
3770   interface(MEMORY_INTER) %{
3771     base($reg);
3772     index($idx);
3773     scale($scale);
3774     disp(0x0);
3775   %}
3776 %}
3777 
3778 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3779 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3780 %{
3781   constraint(ALLOC_IN_RC(ptr_reg));
3782   match(AddP (AddP reg (LShiftL lreg scale)) off);
3783 
3784   op_cost(10);
3785   format %{"[$reg + $off + $lreg << $scale]" %}
3786   interface(MEMORY_INTER) %{
3787     base($reg);
3788     index($lreg);
3789     scale($scale);
3790     disp($off);
3791   %}
3792 %}
3793 
3794 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3795 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3796 %{
3797   constraint(ALLOC_IN_RC(ptr_reg));
3798   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3799   match(AddP (AddP reg (ConvI2L idx)) off);
3800 
3801   op_cost(10);
3802   format %{"[$reg + $off + $idx]" %}
3803   interface(MEMORY_INTER) %{
3804     base($reg);
3805     index($idx);
3806     scale(0x0);
3807     disp($off);
3808   %}
3809 %}
3810 
3811 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3812 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3813 %{
3814   constraint(ALLOC_IN_RC(ptr_reg));
3815   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3816   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3817 
3818   op_cost(10);
3819   format %{"[$reg + $off + $idx << $scale]" %}
3820   interface(MEMORY_INTER) %{
3821     base($reg);
3822     index($idx);
3823     scale($scale);
3824     disp($off);
3825   %}
3826 %}
3827 
3828 // Indirect Narrow Oop Plus Offset Operand
3829 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3830 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3831 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3832   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3833   constraint(ALLOC_IN_RC(ptr_reg));
3834   match(AddP (DecodeN reg) off);
3835 
3836   op_cost(10);
3837   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3838   interface(MEMORY_INTER) %{
3839     base(0xc); // R12
3840     index($reg);
3841     scale(0x3);
3842     disp($off);
3843   %}
3844 %}
3845 
3846 // Indirect Memory Operand
3847 operand indirectNarrow(rRegN reg)
3848 %{
3849   predicate(Universe::narrow_oop_shift() == 0);
3850   constraint(ALLOC_IN_RC(ptr_reg));
3851   match(DecodeN reg);
3852 
3853   format %{ "[$reg]" %}
3854   interface(MEMORY_INTER) %{
3855     base($reg);
3856     index(0x4);
3857     scale(0x0);
3858     disp(0x0);
3859   %}
3860 %}
3861 
3862 // Indirect Memory Plus Short Offset Operand
3863 operand indOffset8Narrow(rRegN reg, immL8 off)
3864 %{
3865   predicate(Universe::narrow_oop_shift() == 0);
3866   constraint(ALLOC_IN_RC(ptr_reg));
3867   match(AddP (DecodeN reg) off);
3868 
3869   format %{ "[$reg + $off (8-bit)]" %}
3870   interface(MEMORY_INTER) %{
3871     base($reg);
3872     index(0x4);
3873     scale(0x0);
3874     disp($off);
3875   %}
3876 %}
3877 
3878 // Indirect Memory Plus Long Offset Operand
3879 operand indOffset32Narrow(rRegN reg, immL32 off)
3880 %{
3881   predicate(Universe::narrow_oop_shift() == 0);
3882   constraint(ALLOC_IN_RC(ptr_reg));
3883   match(AddP (DecodeN reg) off);
3884 
3885   format %{ "[$reg + $off (32-bit)]" %}
3886   interface(MEMORY_INTER) %{
3887     base($reg);
3888     index(0x4);
3889     scale(0x0);
3890     disp($off);
3891   %}
3892 %}
3893 
3894 // Indirect Memory Plus Index Register Plus Offset Operand
3895 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3896 %{
3897   predicate(Universe::narrow_oop_shift() == 0);
3898   constraint(ALLOC_IN_RC(ptr_reg));
3899   match(AddP (AddP (DecodeN reg) lreg) off);
3900 
3901   op_cost(10);
3902   format %{"[$reg + $off + $lreg]" %}
3903   interface(MEMORY_INTER) %{
3904     base($reg);
3905     index($lreg);
3906     scale(0x0);
3907     disp($off);
3908   %}
3909 %}
3910 
3911 // Indirect Memory Plus Index Register Plus Offset Operand
3912 operand indIndexNarrow(rRegN reg, rRegL lreg)
3913 %{
3914   predicate(Universe::narrow_oop_shift() == 0);
3915   constraint(ALLOC_IN_RC(ptr_reg));
3916   match(AddP (DecodeN reg) lreg);
3917 
3918   op_cost(10);
3919   format %{"[$reg + $lreg]" %}
3920   interface(MEMORY_INTER) %{
3921     base($reg);
3922     index($lreg);
3923     scale(0x0);
3924     disp(0x0);
3925   %}
3926 %}
3927 
3928 // Indirect Memory Times Scale Plus Index Register
3929 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3930 %{
3931   predicate(Universe::narrow_oop_shift() == 0);
3932   constraint(ALLOC_IN_RC(ptr_reg));
3933   match(AddP (DecodeN reg) (LShiftL lreg scale));
3934 
3935   op_cost(10);
3936   format %{"[$reg + $lreg << $scale]" %}
3937   interface(MEMORY_INTER) %{
3938     base($reg);
3939     index($lreg);
3940     scale($scale);
3941     disp(0x0);
3942   %}
3943 %}
3944 
3945 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3946 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3947 %{
3948   predicate(Universe::narrow_oop_shift() == 0);
3949   constraint(ALLOC_IN_RC(ptr_reg));
3950   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3951 
3952   op_cost(10);
3953   format %{"[$reg + $off + $lreg << $scale]" %}
3954   interface(MEMORY_INTER) %{
3955     base($reg);
3956     index($lreg);
3957     scale($scale);
3958     disp($off);
3959   %}
3960 %}
3961 
3962 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3963 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3964 %{
3965   constraint(ALLOC_IN_RC(ptr_reg));
3966   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3967   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3968 
3969   op_cost(10);
3970   format %{"[$reg + $off + $idx]" %}
3971   interface(MEMORY_INTER) %{
3972     base($reg);
3973     index($idx);
3974     scale(0x0);
3975     disp($off);
3976   %}
3977 %}
3978 
3979 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3980 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3981 %{
3982   constraint(ALLOC_IN_RC(ptr_reg));
3983   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3984   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3985 
3986   op_cost(10);
3987   format %{"[$reg + $off + $idx << $scale]" %}
3988   interface(MEMORY_INTER) %{
3989     base($reg);
3990     index($idx);
3991     scale($scale);
3992     disp($off);
3993   %}
3994 %}
3995 
3996 //----------Special Memory Operands--------------------------------------------
3997 // Stack Slot Operand - This operand is used for loading and storing temporary
3998 //                      values on the stack where a match requires a value to
3999 //                      flow through memory.
4000 operand stackSlotP(sRegP reg)
4001 %{
4002   constraint(ALLOC_IN_RC(stack_slots));
4003   // No match rule because this operand is only generated in matching
4004 
4005   format %{ "[$reg]" %}
4006   interface(MEMORY_INTER) %{
4007     base(0x4);   // RSP
4008     index(0x4);  // No Index
4009     scale(0x0);  // No Scale
4010     disp($reg);  // Stack Offset
4011   %}
4012 %}
4013 
4014 operand stackSlotI(sRegI reg)
4015 %{
4016   constraint(ALLOC_IN_RC(stack_slots));
4017   // No match rule because this operand is only generated in matching
4018 
4019   format %{ "[$reg]" %}
4020   interface(MEMORY_INTER) %{
4021     base(0x4);   // RSP
4022     index(0x4);  // No Index
4023     scale(0x0);  // No Scale
4024     disp($reg);  // Stack Offset
4025   %}
4026 %}
4027 
4028 operand stackSlotF(sRegF reg)
4029 %{
4030   constraint(ALLOC_IN_RC(stack_slots));
4031   // No match rule because this operand is only generated in matching
4032 
4033   format %{ "[$reg]" %}
4034   interface(MEMORY_INTER) %{
4035     base(0x4);   // RSP
4036     index(0x4);  // No Index
4037     scale(0x0);  // No Scale
4038     disp($reg);  // Stack Offset
4039   %}
4040 %}
4041 
4042 operand stackSlotD(sRegD reg)
4043 %{
4044   constraint(ALLOC_IN_RC(stack_slots));
4045   // No match rule because this operand is only generated in matching
4046 
4047   format %{ "[$reg]" %}
4048   interface(MEMORY_INTER) %{
4049     base(0x4);   // RSP
4050     index(0x4);  // No Index
4051     scale(0x0);  // No Scale
4052     disp($reg);  // Stack Offset
4053   %}
4054 %}
4055 operand stackSlotL(sRegL reg)
4056 %{
4057   constraint(ALLOC_IN_RC(stack_slots));
4058   // No match rule because this operand is only generated in matching
4059 
4060   format %{ "[$reg]" %}
4061   interface(MEMORY_INTER) %{
4062     base(0x4);   // RSP
4063     index(0x4);  // No Index
4064     scale(0x0);  // No Scale
4065     disp($reg);  // Stack Offset
4066   %}
4067 %}
4068 
4069 //----------Conditional Branch Operands----------------------------------------
4070 // Comparison Op  - This is the operation of the comparison, and is limited to
4071 //                  the following set of codes:
4072 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4073 //
4074 // Other attributes of the comparison, such as unsignedness, are specified
4075 // by the comparison instruction that sets a condition code flags register.
4076 // That result is represented by a flags operand whose subtype is appropriate
4077 // to the unsignedness (etc.) of the comparison.
4078 //
4079 // Later, the instruction which matches both the Comparison Op (a Bool) and
4080 // the flags (produced by the Cmp) specifies the coding of the comparison op
4081 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4082 
4083 // Comparision Code
4084 operand cmpOp()
4085 %{
4086   match(Bool);
4087 
4088   format %{ "" %}
4089   interface(COND_INTER) %{
4090     equal(0x4, "e");
4091     not_equal(0x5, "ne");
4092     less(0xC, "l");
4093     greater_equal(0xD, "ge");
4094     less_equal(0xE, "le");
4095     greater(0xF, "g");
4096     overflow(0x0, "o");
4097     no_overflow(0x1, "no");
4098   %}
4099 %}
4100 
4101 // Comparison Code, unsigned compare.  Used by FP also, with
4102 // C2 (unordered) turned into GT or LT already.  The other bits
4103 // C0 and C3 are turned into Carry & Zero flags.
4104 operand cmpOpU()
4105 %{
4106   match(Bool);
4107 
4108   format %{ "" %}
4109   interface(COND_INTER) %{
4110     equal(0x4, "e");
4111     not_equal(0x5, "ne");
4112     less(0x2, "b");
4113     greater_equal(0x3, "nb");
4114     less_equal(0x6, "be");
4115     greater(0x7, "nbe");
4116     overflow(0x0, "o");
4117     no_overflow(0x1, "no");
4118   %}
4119 %}
4120 
4121 
4122 // Floating comparisons that don't require any fixup for the unordered case
4123 operand cmpOpUCF() %{
4124   match(Bool);
4125   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4126             n->as_Bool()->_test._test == BoolTest::ge ||
4127             n->as_Bool()->_test._test == BoolTest::le ||
4128             n->as_Bool()->_test._test == BoolTest::gt);
4129   format %{ "" %}
4130   interface(COND_INTER) %{
4131     equal(0x4, "e");
4132     not_equal(0x5, "ne");
4133     less(0x2, "b");
4134     greater_equal(0x3, "nb");
4135     less_equal(0x6, "be");
4136     greater(0x7, "nbe");
4137     overflow(0x0, "o");
4138     no_overflow(0x1, "no");
4139   %}
4140 %}
4141 
4142 
4143 // Floating comparisons that can be fixed up with extra conditional jumps
4144 operand cmpOpUCF2() %{
4145   match(Bool);
4146   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4147             n->as_Bool()->_test._test == BoolTest::eq);
4148   format %{ "" %}
4149   interface(COND_INTER) %{
4150     equal(0x4, "e");
4151     not_equal(0x5, "ne");
4152     less(0x2, "b");
4153     greater_equal(0x3, "nb");
4154     less_equal(0x6, "be");
4155     greater(0x7, "nbe");
4156     overflow(0x0, "o");
4157     no_overflow(0x1, "no");
4158   %}
4159 %}
4160 
4161 
4162 //----------OPERAND CLASSES----------------------------------------------------
4163 // Operand Classes are groups of operands that are used as to simplify
4164 // instruction definitions by not requiring the AD writer to specify separate
4165 // instructions for every form of operand when the instruction accepts
4166 // multiple operand types with the same basic encoding and format.  The classic
4167 // case of this is memory operands.
4168 
4169 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4170                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4171                indCompressedOopOffset,
4172                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4173                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4174                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4175 
4176 //----------PIPELINE-----------------------------------------------------------
4177 // Rules which define the behavior of the target architectures pipeline.
4178 pipeline %{
4179 
4180 //----------ATTRIBUTES---------------------------------------------------------
4181 attributes %{
4182   variable_size_instructions;        // Fixed size instructions
4183   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4184   instruction_unit_size = 1;         // An instruction is 1 bytes long
4185   instruction_fetch_unit_size = 16;  // The processor fetches one line
4186   instruction_fetch_units = 1;       // of 16 bytes
4187 
4188   // List of nop instructions
4189   nops( MachNop );
4190 %}
4191 
4192 //----------RESOURCES----------------------------------------------------------
4193 // Resources are the functional units available to the machine
4194 
4195 // Generic P2/P3 pipeline
4196 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4197 // 3 instructions decoded per cycle.
4198 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4199 // 3 ALU op, only ALU0 handles mul instructions.
4200 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4201            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4202            BR, FPU,
4203            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4204 
4205 //----------PIPELINE DESCRIPTION-----------------------------------------------
4206 // Pipeline Description specifies the stages in the machine's pipeline
4207 
4208 // Generic P2/P3 pipeline
4209 pipe_desc(S0, S1, S2, S3, S4, S5);
4210 
4211 //----------PIPELINE CLASSES---------------------------------------------------
4212 // Pipeline Classes describe the stages in which input and output are
4213 // referenced by the hardware pipeline.
4214 
4215 // Naming convention: ialu or fpu
4216 // Then: _reg
4217 // Then: _reg if there is a 2nd register
4218 // Then: _long if it's a pair of instructions implementing a long
4219 // Then: _fat if it requires the big decoder
4220 //   Or: _mem if it requires the big decoder and a memory unit.
4221 
4222 // Integer ALU reg operation
4223 pipe_class ialu_reg(rRegI dst)
4224 %{
4225     single_instruction;
4226     dst    : S4(write);
4227     dst    : S3(read);
4228     DECODE : S0;        // any decoder
4229     ALU    : S3;        // any alu
4230 %}
4231 
4232 // Long ALU reg operation
4233 pipe_class ialu_reg_long(rRegL dst)
4234 %{
4235     instruction_count(2);
4236     dst    : S4(write);
4237     dst    : S3(read);
4238     DECODE : S0(2);     // any 2 decoders
4239     ALU    : S3(2);     // both alus
4240 %}
4241 
4242 // Integer ALU reg operation using big decoder
4243 pipe_class ialu_reg_fat(rRegI dst)
4244 %{
4245     single_instruction;
4246     dst    : S4(write);
4247     dst    : S3(read);
4248     D0     : S0;        // big decoder only
4249     ALU    : S3;        // any alu
4250 %}
4251 
4252 // Long ALU reg operation using big decoder
4253 pipe_class ialu_reg_long_fat(rRegL dst)
4254 %{
4255     instruction_count(2);
4256     dst    : S4(write);
4257     dst    : S3(read);
4258     D0     : S0(2);     // big decoder only; twice
4259     ALU    : S3(2);     // any 2 alus
4260 %}
4261 
4262 // Integer ALU reg-reg operation
4263 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4264 %{
4265     single_instruction;
4266     dst    : S4(write);
4267     src    : S3(read);
4268     DECODE : S0;        // any decoder
4269     ALU    : S3;        // any alu
4270 %}
4271 
4272 // Long ALU reg-reg operation
4273 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4274 %{
4275     instruction_count(2);
4276     dst    : S4(write);
4277     src    : S3(read);
4278     DECODE : S0(2);     // any 2 decoders
4279     ALU    : S3(2);     // both alus
4280 %}
4281 
4282 // Integer ALU reg-reg operation
4283 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4284 %{
4285     single_instruction;
4286     dst    : S4(write);
4287     src    : S3(read);
4288     D0     : S0;        // big decoder only
4289     ALU    : S3;        // any alu
4290 %}
4291 
4292 // Long ALU reg-reg operation
4293 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4294 %{
4295     instruction_count(2);
4296     dst    : S4(write);
4297     src    : S3(read);
4298     D0     : S0(2);     // big decoder only; twice
4299     ALU    : S3(2);     // both alus
4300 %}
4301 
4302 // Integer ALU reg-mem operation
4303 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4304 %{
4305     single_instruction;
4306     dst    : S5(write);
4307     mem    : S3(read);
4308     D0     : S0;        // big decoder only
4309     ALU    : S4;        // any alu
4310     MEM    : S3;        // any mem
4311 %}
4312 
4313 // Integer mem operation (prefetch)
4314 pipe_class ialu_mem(memory mem)
4315 %{
4316     single_instruction;
4317     mem    : S3(read);
4318     D0     : S0;        // big decoder only
4319     MEM    : S3;        // any mem
4320 %}
4321 
4322 // Integer Store to Memory
4323 pipe_class ialu_mem_reg(memory mem, rRegI src)
4324 %{
4325     single_instruction;
4326     mem    : S3(read);
4327     src    : S5(read);
4328     D0     : S0;        // big decoder only
4329     ALU    : S4;        // any alu
4330     MEM    : S3;
4331 %}
4332 
4333 // // Long Store to Memory
4334 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4335 // %{
4336 //     instruction_count(2);
4337 //     mem    : S3(read);
4338 //     src    : S5(read);
4339 //     D0     : S0(2);          // big decoder only; twice
4340 //     ALU    : S4(2);     // any 2 alus
4341 //     MEM    : S3(2);  // Both mems
4342 // %}
4343 
4344 // Integer Store to Memory
4345 pipe_class ialu_mem_imm(memory mem)
4346 %{
4347     single_instruction;
4348     mem    : S3(read);
4349     D0     : S0;        // big decoder only
4350     ALU    : S4;        // any alu
4351     MEM    : S3;
4352 %}
4353 
4354 // Integer ALU0 reg-reg operation
4355 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4356 %{
4357     single_instruction;
4358     dst    : S4(write);
4359     src    : S3(read);
4360     D0     : S0;        // Big decoder only
4361     ALU0   : S3;        // only alu0
4362 %}
4363 
4364 // Integer ALU0 reg-mem operation
4365 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4366 %{
4367     single_instruction;
4368     dst    : S5(write);
4369     mem    : S3(read);
4370     D0     : S0;        // big decoder only
4371     ALU0   : S4;        // ALU0 only
4372     MEM    : S3;        // any mem
4373 %}
4374 
4375 // Integer ALU reg-reg operation
4376 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4377 %{
4378     single_instruction;
4379     cr     : S4(write);
4380     src1   : S3(read);
4381     src2   : S3(read);
4382     DECODE : S0;        // any decoder
4383     ALU    : S3;        // any alu
4384 %}
4385 
4386 // Integer ALU reg-imm operation
4387 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4388 %{
4389     single_instruction;
4390     cr     : S4(write);
4391     src1   : S3(read);
4392     DECODE : S0;        // any decoder
4393     ALU    : S3;        // any alu
4394 %}
4395 
4396 // Integer ALU reg-mem operation
4397 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4398 %{
4399     single_instruction;
4400     cr     : S4(write);
4401     src1   : S3(read);
4402     src2   : S3(read);
4403     D0     : S0;        // big decoder only
4404     ALU    : S4;        // any alu
4405     MEM    : S3;
4406 %}
4407 
4408 // Conditional move reg-reg
4409 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4410 %{
4411     instruction_count(4);
4412     y      : S4(read);
4413     q      : S3(read);
4414     p      : S3(read);
4415     DECODE : S0(4);     // any decoder
4416 %}
4417 
4418 // Conditional move reg-reg
4419 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4420 %{
4421     single_instruction;
4422     dst    : S4(write);
4423     src    : S3(read);
4424     cr     : S3(read);
4425     DECODE : S0;        // any decoder
4426 %}
4427 
4428 // Conditional move reg-mem
4429 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4430 %{
4431     single_instruction;
4432     dst    : S4(write);
4433     src    : S3(read);
4434     cr     : S3(read);
4435     DECODE : S0;        // any decoder
4436     MEM    : S3;
4437 %}
4438 
4439 // Conditional move reg-reg long
4440 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4441 %{
4442     single_instruction;
4443     dst    : S4(write);
4444     src    : S3(read);
4445     cr     : S3(read);
4446     DECODE : S0(2);     // any 2 decoders
4447 %}
4448 
4449 // XXX
4450 // // Conditional move double reg-reg
4451 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4452 // %{
4453 //     single_instruction;
4454 //     dst    : S4(write);
4455 //     src    : S3(read);
4456 //     cr     : S3(read);
4457 //     DECODE : S0;     // any decoder
4458 // %}
4459 
4460 // Float reg-reg operation
4461 pipe_class fpu_reg(regD dst)
4462 %{
4463     instruction_count(2);
4464     dst    : S3(read);
4465     DECODE : S0(2);     // any 2 decoders
4466     FPU    : S3;
4467 %}
4468 
4469 // Float reg-reg operation
4470 pipe_class fpu_reg_reg(regD dst, regD src)
4471 %{
4472     instruction_count(2);
4473     dst    : S4(write);
4474     src    : S3(read);
4475     DECODE : S0(2);     // any 2 decoders
4476     FPU    : S3;
4477 %}
4478 
4479 // Float reg-reg operation
4480 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4481 %{
4482     instruction_count(3);
4483     dst    : S4(write);
4484     src1   : S3(read);
4485     src2   : S3(read);
4486     DECODE : S0(3);     // any 3 decoders
4487     FPU    : S3(2);
4488 %}
4489 
4490 // Float reg-reg operation
4491 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4492 %{
4493     instruction_count(4);
4494     dst    : S4(write);
4495     src1   : S3(read);
4496     src2   : S3(read);
4497     src3   : S3(read);
4498     DECODE : S0(4);     // any 3 decoders
4499     FPU    : S3(2);
4500 %}
4501 
4502 // Float reg-reg operation
4503 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4504 %{
4505     instruction_count(4);
4506     dst    : S4(write);
4507     src1   : S3(read);
4508     src2   : S3(read);
4509     src3   : S3(read);
4510     DECODE : S1(3);     // any 3 decoders
4511     D0     : S0;        // Big decoder only
4512     FPU    : S3(2);
4513     MEM    : S3;
4514 %}
4515 
4516 // Float reg-mem operation
4517 pipe_class fpu_reg_mem(regD dst, memory mem)
4518 %{
4519     instruction_count(2);
4520     dst    : S5(write);
4521     mem    : S3(read);
4522     D0     : S0;        // big decoder only
4523     DECODE : S1;        // any decoder for FPU POP
4524     FPU    : S4;
4525     MEM    : S3;        // any mem
4526 %}
4527 
4528 // Float reg-mem operation
4529 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4530 %{
4531     instruction_count(3);
4532     dst    : S5(write);
4533     src1   : S3(read);
4534     mem    : S3(read);
4535     D0     : S0;        // big decoder only
4536     DECODE : S1(2);     // any decoder for FPU POP
4537     FPU    : S4;
4538     MEM    : S3;        // any mem
4539 %}
4540 
4541 // Float mem-reg operation
4542 pipe_class fpu_mem_reg(memory mem, regD src)
4543 %{
4544     instruction_count(2);
4545     src    : S5(read);
4546     mem    : S3(read);
4547     DECODE : S0;        // any decoder for FPU PUSH
4548     D0     : S1;        // big decoder only
4549     FPU    : S4;
4550     MEM    : S3;        // any mem
4551 %}
4552 
4553 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4554 %{
4555     instruction_count(3);
4556     src1   : S3(read);
4557     src2   : S3(read);
4558     mem    : S3(read);
4559     DECODE : S0(2);     // any decoder for FPU PUSH
4560     D0     : S1;        // big decoder only
4561     FPU    : S4;
4562     MEM    : S3;        // any mem
4563 %}
4564 
4565 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4566 %{
4567     instruction_count(3);
4568     src1   : S3(read);
4569     src2   : S3(read);
4570     mem    : S4(read);
4571     DECODE : S0;        // any decoder for FPU PUSH
4572     D0     : S0(2);     // big decoder only
4573     FPU    : S4;
4574     MEM    : S3(2);     // any mem
4575 %}
4576 
4577 pipe_class fpu_mem_mem(memory dst, memory src1)
4578 %{
4579     instruction_count(2);
4580     src1   : S3(read);
4581     dst    : S4(read);
4582     D0     : S0(2);     // big decoder only
4583     MEM    : S3(2);     // any mem
4584 %}
4585 
4586 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4587 %{
4588     instruction_count(3);
4589     src1   : S3(read);
4590     src2   : S3(read);
4591     dst    : S4(read);
4592     D0     : S0(3);     // big decoder only
4593     FPU    : S4;
4594     MEM    : S3(3);     // any mem
4595 %}
4596 
4597 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4598 %{
4599     instruction_count(3);
4600     src1   : S4(read);
4601     mem    : S4(read);
4602     DECODE : S0;        // any decoder for FPU PUSH
4603     D0     : S0(2);     // big decoder only
4604     FPU    : S4;
4605     MEM    : S3(2);     // any mem
4606 %}
4607 
4608 // Float load constant
4609 pipe_class fpu_reg_con(regD dst)
4610 %{
4611     instruction_count(2);
4612     dst    : S5(write);
4613     D0     : S0;        // big decoder only for the load
4614     DECODE : S1;        // any decoder for FPU POP
4615     FPU    : S4;
4616     MEM    : S3;        // any mem
4617 %}
4618 
4619 // Float load constant
4620 pipe_class fpu_reg_reg_con(regD dst, regD src)
4621 %{
4622     instruction_count(3);
4623     dst    : S5(write);
4624     src    : S3(read);
4625     D0     : S0;        // big decoder only for the load
4626     DECODE : S1(2);     // any decoder for FPU POP
4627     FPU    : S4;
4628     MEM    : S3;        // any mem
4629 %}
4630 
4631 // UnConditional branch
4632 pipe_class pipe_jmp(label labl)
4633 %{
4634     single_instruction;
4635     BR   : S3;
4636 %}
4637 
4638 // Conditional branch
4639 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4640 %{
4641     single_instruction;
4642     cr    : S1(read);
4643     BR    : S3;
4644 %}
4645 
4646 // Allocation idiom
4647 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4648 %{
4649     instruction_count(1); force_serialization;
4650     fixed_latency(6);
4651     heap_ptr : S3(read);
4652     DECODE   : S0(3);
4653     D0       : S2;
4654     MEM      : S3;
4655     ALU      : S3(2);
4656     dst      : S5(write);
4657     BR       : S5;
4658 %}
4659 
4660 // Generic big/slow expanded idiom
4661 pipe_class pipe_slow()
4662 %{
4663     instruction_count(10); multiple_bundles; force_serialization;
4664     fixed_latency(100);
4665     D0  : S0(2);
4666     MEM : S3(2);
4667 %}
4668 
4669 // The real do-nothing guy
4670 pipe_class empty()
4671 %{
4672     instruction_count(0);
4673 %}
4674 
4675 // Define the class for the Nop node
4676 define
4677 %{
4678    MachNop = empty;
4679 %}
4680 
4681 %}
4682 
4683 //----------INSTRUCTIONS-------------------------------------------------------
4684 //
4685 // match      -- States which machine-independent subtree may be replaced
4686 //               by this instruction.
4687 // ins_cost   -- The estimated cost of this instruction is used by instruction
4688 //               selection to identify a minimum cost tree of machine
4689 //               instructions that matches a tree of machine-independent
4690 //               instructions.
4691 // format     -- A string providing the disassembly for this instruction.
4692 //               The value of an instruction's operand may be inserted
4693 //               by referring to it with a '$' prefix.
4694 // opcode     -- Three instruction opcodes may be provided.  These are referred
4695 //               to within an encode class as $primary, $secondary, and $tertiary
4696 //               rrspectively.  The primary opcode is commonly used to
4697 //               indicate the type of machine instruction, while secondary
4698 //               and tertiary are often used for prefix options or addressing
4699 //               modes.
4700 // ins_encode -- A list of encode classes with parameters. The encode class
4701 //               name must have been defined in an 'enc_class' specification
4702 //               in the encode section of the architecture description.
4703 
4704 
4705 //----------Load/Store/Move Instructions---------------------------------------
4706 //----------Load Instructions--------------------------------------------------
4707 
4708 // Load Byte (8 bit signed)
4709 instruct loadB(rRegI dst, memory mem)
4710 %{
4711   match(Set dst (LoadB mem));
4712 
4713   ins_cost(125);
4714   format %{ "movsbl  $dst, $mem\t# byte" %}
4715 
4716   ins_encode %{
4717     __ movsbl($dst$$Register, $mem$$Address);
4718   %}
4719 
4720   ins_pipe(ialu_reg_mem);
4721 %}
4722 
4723 // Load Byte (8 bit signed) into Long Register
4724 instruct loadB2L(rRegL dst, memory mem)
4725 %{
4726   match(Set dst (ConvI2L (LoadB mem)));
4727 
4728   ins_cost(125);
4729   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4730 
4731   ins_encode %{
4732     __ movsbq($dst$$Register, $mem$$Address);
4733   %}
4734 
4735   ins_pipe(ialu_reg_mem);
4736 %}
4737 
4738 // Load Unsigned Byte (8 bit UNsigned)
4739 instruct loadUB(rRegI dst, memory mem)
4740 %{
4741   match(Set dst (LoadUB mem));
4742 
4743   ins_cost(125);
4744   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4745 
4746   ins_encode %{
4747     __ movzbl($dst$$Register, $mem$$Address);
4748   %}
4749 
4750   ins_pipe(ialu_reg_mem);
4751 %}
4752 
4753 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4754 instruct loadUB2L(rRegL dst, memory mem)
4755 %{
4756   match(Set dst (ConvI2L (LoadUB mem)));
4757 
4758   ins_cost(125);
4759   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4760 
4761   ins_encode %{
4762     __ movzbq($dst$$Register, $mem$$Address);
4763   %}
4764 
4765   ins_pipe(ialu_reg_mem);
4766 %}
4767 
4768 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4769 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4770   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4771   effect(KILL cr);
4772 
4773   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4774             "andl    $dst, right_n_bits($mask, 8)" %}
4775   ins_encode %{
4776     Register Rdst = $dst$$Register;
4777     __ movzbq(Rdst, $mem$$Address);
4778     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4779   %}
4780   ins_pipe(ialu_reg_mem);
4781 %}
4782 
4783 // Load Short (16 bit signed)
4784 instruct loadS(rRegI dst, memory mem)
4785 %{
4786   match(Set dst (LoadS mem));
4787 
4788   ins_cost(125);
4789   format %{ "movswl $dst, $mem\t# short" %}
4790 
4791   ins_encode %{
4792     __ movswl($dst$$Register, $mem$$Address);
4793   %}
4794 
4795   ins_pipe(ialu_reg_mem);
4796 %}
4797 
4798 // Load Short (16 bit signed) to Byte (8 bit signed)
4799 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4800   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4801 
4802   ins_cost(125);
4803   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4804   ins_encode %{
4805     __ movsbl($dst$$Register, $mem$$Address);
4806   %}
4807   ins_pipe(ialu_reg_mem);
4808 %}
4809 
4810 // Load Short (16 bit signed) into Long Register
4811 instruct loadS2L(rRegL dst, memory mem)
4812 %{
4813   match(Set dst (ConvI2L (LoadS mem)));
4814 
4815   ins_cost(125);
4816   format %{ "movswq $dst, $mem\t# short -> long" %}
4817 
4818   ins_encode %{
4819     __ movswq($dst$$Register, $mem$$Address);
4820   %}
4821 
4822   ins_pipe(ialu_reg_mem);
4823 %}
4824 
4825 // Load Unsigned Short/Char (16 bit UNsigned)
4826 instruct loadUS(rRegI dst, memory mem)
4827 %{
4828   match(Set dst (LoadUS mem));
4829 
4830   ins_cost(125);
4831   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4832 
4833   ins_encode %{
4834     __ movzwl($dst$$Register, $mem$$Address);
4835   %}
4836 
4837   ins_pipe(ialu_reg_mem);
4838 %}
4839 
4840 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4841 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4842   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4843 
4844   ins_cost(125);
4845   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4846   ins_encode %{
4847     __ movsbl($dst$$Register, $mem$$Address);
4848   %}
4849   ins_pipe(ialu_reg_mem);
4850 %}
4851 
4852 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4853 instruct loadUS2L(rRegL dst, memory mem)
4854 %{
4855   match(Set dst (ConvI2L (LoadUS mem)));
4856 
4857   ins_cost(125);
4858   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4859 
4860   ins_encode %{
4861     __ movzwq($dst$$Register, $mem$$Address);
4862   %}
4863 
4864   ins_pipe(ialu_reg_mem);
4865 %}
4866 
4867 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4868 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4869   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4870 
4871   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4872   ins_encode %{
4873     __ movzbq($dst$$Register, $mem$$Address);
4874   %}
4875   ins_pipe(ialu_reg_mem);
4876 %}
4877 
4878 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4879 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4880   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4881   effect(KILL cr);
4882 
4883   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4884             "andl    $dst, right_n_bits($mask, 16)" %}
4885   ins_encode %{
4886     Register Rdst = $dst$$Register;
4887     __ movzwq(Rdst, $mem$$Address);
4888     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4889   %}
4890   ins_pipe(ialu_reg_mem);
4891 %}
4892 
4893 // Load Integer
4894 instruct loadI(rRegI dst, memory mem)
4895 %{
4896   match(Set dst (LoadI mem));
4897 
4898   ins_cost(125);
4899   format %{ "movl    $dst, $mem\t# int" %}
4900 
4901   ins_encode %{
4902     __ movl($dst$$Register, $mem$$Address);
4903   %}
4904 
4905   ins_pipe(ialu_reg_mem);
4906 %}
4907 
4908 // Load Integer (32 bit signed) to Byte (8 bit signed)
4909 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4910   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4911 
4912   ins_cost(125);
4913   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4914   ins_encode %{
4915     __ movsbl($dst$$Register, $mem$$Address);
4916   %}
4917   ins_pipe(ialu_reg_mem);
4918 %}
4919 
4920 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4921 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4922   match(Set dst (AndI (LoadI mem) mask));
4923 
4924   ins_cost(125);
4925   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4926   ins_encode %{
4927     __ movzbl($dst$$Register, $mem$$Address);
4928   %}
4929   ins_pipe(ialu_reg_mem);
4930 %}
4931 
4932 // Load Integer (32 bit signed) to Short (16 bit signed)
4933 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4934   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4935 
4936   ins_cost(125);
4937   format %{ "movswl  $dst, $mem\t# int -> short" %}
4938   ins_encode %{
4939     __ movswl($dst$$Register, $mem$$Address);
4940   %}
4941   ins_pipe(ialu_reg_mem);
4942 %}
4943 
4944 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4945 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4946   match(Set dst (AndI (LoadI mem) mask));
4947 
4948   ins_cost(125);
4949   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4950   ins_encode %{
4951     __ movzwl($dst$$Register, $mem$$Address);
4952   %}
4953   ins_pipe(ialu_reg_mem);
4954 %}
4955 
4956 // Load Integer into Long Register
4957 instruct loadI2L(rRegL dst, memory mem)
4958 %{
4959   match(Set dst (ConvI2L (LoadI mem)));
4960 
4961   ins_cost(125);
4962   format %{ "movslq  $dst, $mem\t# int -> long" %}
4963 
4964   ins_encode %{
4965     __ movslq($dst$$Register, $mem$$Address);
4966   %}
4967 
4968   ins_pipe(ialu_reg_mem);
4969 %}
4970 
4971 // Load Integer with mask 0xFF into Long Register
4972 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4973   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4974 
4975   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4976   ins_encode %{
4977     __ movzbq($dst$$Register, $mem$$Address);
4978   %}
4979   ins_pipe(ialu_reg_mem);
4980 %}
4981 
4982 // Load Integer with mask 0xFFFF into Long Register
4983 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4984   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4985 
4986   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4987   ins_encode %{
4988     __ movzwq($dst$$Register, $mem$$Address);
4989   %}
4990   ins_pipe(ialu_reg_mem);
4991 %}
4992 
4993 // Load Integer with a 31-bit mask into Long Register
4994 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
4995   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4996   effect(KILL cr);
4997 
4998   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
4999             "andl    $dst, $mask" %}
5000   ins_encode %{
5001     Register Rdst = $dst$$Register;
5002     __ movl(Rdst, $mem$$Address);
5003     __ andl(Rdst, $mask$$constant);
5004   %}
5005   ins_pipe(ialu_reg_mem);
5006 %}
5007 
5008 // Load Unsigned Integer into Long Register
5009 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5010 %{
5011   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5012 
5013   ins_cost(125);
5014   format %{ "movl    $dst, $mem\t# uint -> long" %}
5015 
5016   ins_encode %{
5017     __ movl($dst$$Register, $mem$$Address);
5018   %}
5019 
5020   ins_pipe(ialu_reg_mem);
5021 %}
5022 
5023 // Load Long
5024 instruct loadL(rRegL dst, memory mem)
5025 %{
5026   match(Set dst (LoadL mem));
5027 
5028   ins_cost(125);
5029   format %{ "movq    $dst, $mem\t# long" %}
5030 
5031   ins_encode %{
5032     __ movq($dst$$Register, $mem$$Address);
5033   %}
5034 
5035   ins_pipe(ialu_reg_mem); // XXX
5036 %}
5037 
5038 // Load Range
5039 instruct loadRange(rRegI dst, memory mem)
5040 %{
5041   match(Set dst (LoadRange mem));
5042 
5043   ins_cost(125); // XXX
5044   format %{ "movl    $dst, $mem\t# range" %}
5045   opcode(0x8B);
5046   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5047   ins_pipe(ialu_reg_mem);
5048 %}
5049 
5050 // Load Pointer
5051 instruct loadP(rRegP dst, memory mem)
5052 %{
5053   match(Set dst (LoadP mem));
5054 
5055   ins_cost(125); // XXX
5056   format %{ "movq    $dst, $mem\t# ptr" %}
5057   opcode(0x8B);
5058   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5059   ins_pipe(ialu_reg_mem); // XXX
5060 %}
5061 
5062 // Load Compressed Pointer
5063 instruct loadN(rRegN dst, memory mem)
5064 %{
5065    match(Set dst (LoadN mem));
5066 
5067    ins_cost(125); // XXX
5068    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5069    ins_encode %{
5070      __ movl($dst$$Register, $mem$$Address);
5071    %}
5072    ins_pipe(ialu_reg_mem); // XXX
5073 %}
5074 
5075 
5076 // Load Klass Pointer
5077 instruct loadKlass(rRegP dst, memory mem)
5078 %{
5079   match(Set dst (LoadKlass mem));
5080 
5081   ins_cost(125); // XXX
5082   format %{ "movq    $dst, $mem\t# class" %}
5083   opcode(0x8B);
5084   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5085   ins_pipe(ialu_reg_mem); // XXX
5086 %}
5087 
5088 // Load narrow Klass Pointer
5089 instruct loadNKlass(rRegN dst, memory mem)
5090 %{
5091   match(Set dst (LoadNKlass mem));
5092 
5093   ins_cost(125); // XXX
5094   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5095   ins_encode %{
5096     __ movl($dst$$Register, $mem$$Address);
5097   %}
5098   ins_pipe(ialu_reg_mem); // XXX
5099 %}
5100 
5101 // Load Float
5102 instruct loadF(regF dst, memory mem)
5103 %{
5104   match(Set dst (LoadF mem));
5105 
5106   ins_cost(145); // XXX
5107   format %{ "movss   $dst, $mem\t# float" %}
5108   ins_encode %{
5109     __ movflt($dst$$XMMRegister, $mem$$Address);
5110   %}
5111   ins_pipe(pipe_slow); // XXX
5112 %}
5113 
5114 // Load Double
5115 instruct loadD_partial(regD dst, memory mem)
5116 %{
5117   predicate(!UseXmmLoadAndClearUpper);
5118   match(Set dst (LoadD mem));
5119 
5120   ins_cost(145); // XXX
5121   format %{ "movlpd  $dst, $mem\t# double" %}
5122   ins_encode %{
5123     __ movdbl($dst$$XMMRegister, $mem$$Address);
5124   %}
5125   ins_pipe(pipe_slow); // XXX
5126 %}
5127 
5128 instruct loadD(regD dst, memory mem)
5129 %{
5130   predicate(UseXmmLoadAndClearUpper);
5131   match(Set dst (LoadD mem));
5132 
5133   ins_cost(145); // XXX
5134   format %{ "movsd   $dst, $mem\t# double" %}
5135   ins_encode %{
5136     __ movdbl($dst$$XMMRegister, $mem$$Address);
5137   %}
5138   ins_pipe(pipe_slow); // XXX
5139 %}
5140 
5141 // Load Effective Address
5142 instruct leaP8(rRegP dst, indOffset8 mem)
5143 %{
5144   match(Set dst mem);
5145 
5146   ins_cost(110); // XXX
5147   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5148   opcode(0x8D);
5149   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5150   ins_pipe(ialu_reg_reg_fat);
5151 %}
5152 
5153 instruct leaP32(rRegP dst, indOffset32 mem)
5154 %{
5155   match(Set dst mem);
5156 
5157   ins_cost(110);
5158   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5159   opcode(0x8D);
5160   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5161   ins_pipe(ialu_reg_reg_fat);
5162 %}
5163 
5164 // instruct leaPIdx(rRegP dst, indIndex mem)
5165 // %{
5166 //   match(Set dst mem);
5167 
5168 //   ins_cost(110);
5169 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5170 //   opcode(0x8D);
5171 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5172 //   ins_pipe(ialu_reg_reg_fat);
5173 // %}
5174 
5175 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5176 %{
5177   match(Set dst mem);
5178 
5179   ins_cost(110);
5180   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5181   opcode(0x8D);
5182   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5183   ins_pipe(ialu_reg_reg_fat);
5184 %}
5185 
5186 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5187 %{
5188   match(Set dst mem);
5189 
5190   ins_cost(110);
5191   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5192   opcode(0x8D);
5193   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5194   ins_pipe(ialu_reg_reg_fat);
5195 %}
5196 
5197 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5198 %{
5199   match(Set dst mem);
5200 
5201   ins_cost(110);
5202   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5203   opcode(0x8D);
5204   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5205   ins_pipe(ialu_reg_reg_fat);
5206 %}
5207 
5208 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5209 %{
5210   match(Set dst mem);
5211 
5212   ins_cost(110);
5213   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5214   opcode(0x8D);
5215   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5216   ins_pipe(ialu_reg_reg_fat);
5217 %}
5218 
5219 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5220 %{
5221   match(Set dst mem);
5222 
5223   ins_cost(110);
5224   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5225   opcode(0x8D);
5226   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5227   ins_pipe(ialu_reg_reg_fat);
5228 %}
5229 
5230 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5231 %{
5232   match(Set dst mem);
5233 
5234   ins_cost(110);
5235   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5236   opcode(0x8D);
5237   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5238   ins_pipe(ialu_reg_reg_fat);
5239 %}
5240 
5241 // Load Effective Address which uses Narrow (32-bits) oop
5242 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5243 %{
5244   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5245   match(Set dst mem);
5246 
5247   ins_cost(110);
5248   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5249   opcode(0x8D);
5250   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5251   ins_pipe(ialu_reg_reg_fat);
5252 %}
5253 
5254 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5255 %{
5256   predicate(Universe::narrow_oop_shift() == 0);
5257   match(Set dst mem);
5258 
5259   ins_cost(110); // XXX
5260   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5261   opcode(0x8D);
5262   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5263   ins_pipe(ialu_reg_reg_fat);
5264 %}
5265 
5266 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5267 %{
5268   predicate(Universe::narrow_oop_shift() == 0);
5269   match(Set dst mem);
5270 
5271   ins_cost(110);
5272   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5273   opcode(0x8D);
5274   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5275   ins_pipe(ialu_reg_reg_fat);
5276 %}
5277 
5278 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5279 %{
5280   predicate(Universe::narrow_oop_shift() == 0);
5281   match(Set dst mem);
5282 
5283   ins_cost(110);
5284   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5285   opcode(0x8D);
5286   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5287   ins_pipe(ialu_reg_reg_fat);
5288 %}
5289 
5290 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5291 %{
5292   predicate(Universe::narrow_oop_shift() == 0);
5293   match(Set dst mem);
5294 
5295   ins_cost(110);
5296   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5297   opcode(0x8D);
5298   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5299   ins_pipe(ialu_reg_reg_fat);
5300 %}
5301 
5302 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5303 %{
5304   predicate(Universe::narrow_oop_shift() == 0);
5305   match(Set dst mem);
5306 
5307   ins_cost(110);
5308   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5309   opcode(0x8D);
5310   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5311   ins_pipe(ialu_reg_reg_fat);
5312 %}
5313 
5314 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5315 %{
5316   predicate(Universe::narrow_oop_shift() == 0);
5317   match(Set dst mem);
5318 
5319   ins_cost(110);
5320   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5321   opcode(0x8D);
5322   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5323   ins_pipe(ialu_reg_reg_fat);
5324 %}
5325 
5326 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5327 %{
5328   predicate(Universe::narrow_oop_shift() == 0);
5329   match(Set dst mem);
5330 
5331   ins_cost(110);
5332   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5333   opcode(0x8D);
5334   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5335   ins_pipe(ialu_reg_reg_fat);
5336 %}
5337 
5338 instruct loadConI(rRegI dst, immI src)
5339 %{
5340   match(Set dst src);
5341 
5342   format %{ "movl    $dst, $src\t# int" %}
5343   ins_encode(load_immI(dst, src));
5344   ins_pipe(ialu_reg_fat); // XXX
5345 %}
5346 
5347 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5348 %{
5349   match(Set dst src);
5350   effect(KILL cr);
5351 
5352   ins_cost(50);
5353   format %{ "xorl    $dst, $dst\t# int" %}
5354   opcode(0x33); /* + rd */
5355   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5356   ins_pipe(ialu_reg);
5357 %}
5358 
5359 instruct loadConL(rRegL dst, immL src)
5360 %{
5361   match(Set dst src);
5362 
5363   ins_cost(150);
5364   format %{ "movq    $dst, $src\t# long" %}
5365   ins_encode(load_immL(dst, src));
5366   ins_pipe(ialu_reg);
5367 %}
5368 
5369 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5370 %{
5371   match(Set dst src);
5372   effect(KILL cr);
5373 
5374   ins_cost(50);
5375   format %{ "xorl    $dst, $dst\t# long" %}
5376   opcode(0x33); /* + rd */
5377   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5378   ins_pipe(ialu_reg); // XXX
5379 %}
5380 
5381 instruct loadConUL32(rRegL dst, immUL32 src)
5382 %{
5383   match(Set dst src);
5384 
5385   ins_cost(60);
5386   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5387   ins_encode(load_immUL32(dst, src));
5388   ins_pipe(ialu_reg);
5389 %}
5390 
5391 instruct loadConL32(rRegL dst, immL32 src)
5392 %{
5393   match(Set dst src);
5394 
5395   ins_cost(70);
5396   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5397   ins_encode(load_immL32(dst, src));
5398   ins_pipe(ialu_reg);
5399 %}
5400 
5401 instruct loadConP(rRegP dst, immP con) %{
5402   match(Set dst con);
5403 
5404   format %{ "movq    $dst, $con\t# ptr" %}
5405   ins_encode(load_immP(dst, con));
5406   ins_pipe(ialu_reg_fat); // XXX
5407 %}
5408 
5409 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5410 %{
5411   match(Set dst src);
5412   effect(KILL cr);
5413 
5414   ins_cost(50);
5415   format %{ "xorl    $dst, $dst\t# ptr" %}
5416   opcode(0x33); /* + rd */
5417   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5418   ins_pipe(ialu_reg);
5419 %}
5420 
5421 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5422 %{
5423   match(Set dst src);
5424   effect(KILL cr);
5425 
5426   ins_cost(60);
5427   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5428   ins_encode(load_immP31(dst, src));
5429   ins_pipe(ialu_reg);
5430 %}
5431 
5432 instruct loadConF(regF dst, immF con) %{
5433   match(Set dst con);
5434   ins_cost(125);
5435   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5436   ins_encode %{
5437     __ movflt($dst$$XMMRegister, $constantaddress($con));
5438   %}
5439   ins_pipe(pipe_slow);
5440 %}
5441 
5442 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5443   match(Set dst src);
5444   effect(KILL cr);
5445   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5446   ins_encode %{
5447     __ xorq($dst$$Register, $dst$$Register);
5448   %}
5449   ins_pipe(ialu_reg);
5450 %}
5451 
5452 instruct loadConN(rRegN dst, immN src) %{
5453   match(Set dst src);
5454 
5455   ins_cost(125);
5456   format %{ "movl    $dst, $src\t# compressed ptr" %}
5457   ins_encode %{
5458     address con = (address)$src$$constant;
5459     if (con == NULL) {
5460       ShouldNotReachHere();
5461     } else {
5462       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5463     }
5464   %}
5465   ins_pipe(ialu_reg_fat); // XXX
5466 %}
5467 
5468 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5469   match(Set dst src);
5470 
5471   ins_cost(125);
5472   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5473   ins_encode %{
5474     address con = (address)$src$$constant;
5475     if (con == NULL) {
5476       ShouldNotReachHere();
5477     } else {
5478       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5479     }
5480   %}
5481   ins_pipe(ialu_reg_fat); // XXX
5482 %}
5483 
5484 instruct loadConF0(regF dst, immF0 src)
5485 %{
5486   match(Set dst src);
5487   ins_cost(100);
5488 
5489   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5490   ins_encode %{
5491     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5492   %}
5493   ins_pipe(pipe_slow);
5494 %}
5495 
5496 // Use the same format since predicate() can not be used here.
5497 instruct loadConD(regD dst, immD con) %{
5498   match(Set dst con);
5499   ins_cost(125);
5500   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5501   ins_encode %{
5502     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5503   %}
5504   ins_pipe(pipe_slow);
5505 %}
5506 
5507 instruct loadConD0(regD dst, immD0 src)
5508 %{
5509   match(Set dst src);
5510   ins_cost(100);
5511 
5512   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5513   ins_encode %{
5514     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5515   %}
5516   ins_pipe(pipe_slow);
5517 %}
5518 
5519 instruct loadSSI(rRegI dst, stackSlotI src)
5520 %{
5521   match(Set dst src);
5522 
5523   ins_cost(125);
5524   format %{ "movl    $dst, $src\t# int stk" %}
5525   opcode(0x8B);
5526   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5527   ins_pipe(ialu_reg_mem);
5528 %}
5529 
5530 instruct loadSSL(rRegL dst, stackSlotL src)
5531 %{
5532   match(Set dst src);
5533 
5534   ins_cost(125);
5535   format %{ "movq    $dst, $src\t# long stk" %}
5536   opcode(0x8B);
5537   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5538   ins_pipe(ialu_reg_mem);
5539 %}
5540 
5541 instruct loadSSP(rRegP dst, stackSlotP src)
5542 %{
5543   match(Set dst src);
5544 
5545   ins_cost(125);
5546   format %{ "movq    $dst, $src\t# ptr stk" %}
5547   opcode(0x8B);
5548   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5549   ins_pipe(ialu_reg_mem);
5550 %}
5551 
5552 instruct loadSSF(regF dst, stackSlotF src)
5553 %{
5554   match(Set dst src);
5555 
5556   ins_cost(125);
5557   format %{ "movss   $dst, $src\t# float stk" %}
5558   ins_encode %{
5559     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5560   %}
5561   ins_pipe(pipe_slow); // XXX
5562 %}
5563 
5564 // Use the same format since predicate() can not be used here.
5565 instruct loadSSD(regD dst, stackSlotD src)
5566 %{
5567   match(Set dst src);
5568 
5569   ins_cost(125);
5570   format %{ "movsd   $dst, $src\t# double stk" %}
5571   ins_encode  %{
5572     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5573   %}
5574   ins_pipe(pipe_slow); // XXX
5575 %}
5576 
5577 // Prefetch instructions for allocation.
5578 // Must be safe to execute with invalid address (cannot fault).
5579 
5580 instruct prefetchAlloc( memory mem ) %{
5581   predicate(AllocatePrefetchInstr==3);
5582   match(PrefetchAllocation mem);
5583   ins_cost(125);
5584 
5585   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5586   ins_encode %{
5587     __ prefetchw($mem$$Address);
5588   %}
5589   ins_pipe(ialu_mem);
5590 %}
5591 
5592 instruct prefetchAllocNTA( memory mem ) %{
5593   predicate(AllocatePrefetchInstr==0);
5594   match(PrefetchAllocation mem);
5595   ins_cost(125);
5596 
5597   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5598   ins_encode %{
5599     __ prefetchnta($mem$$Address);
5600   %}
5601   ins_pipe(ialu_mem);
5602 %}
5603 
5604 instruct prefetchAllocT0( memory mem ) %{
5605   predicate(AllocatePrefetchInstr==1);
5606   match(PrefetchAllocation mem);
5607   ins_cost(125);
5608 
5609   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5610   ins_encode %{
5611     __ prefetcht0($mem$$Address);
5612   %}
5613   ins_pipe(ialu_mem);
5614 %}
5615 
5616 instruct prefetchAllocT2( memory mem ) %{
5617   predicate(AllocatePrefetchInstr==2);
5618   match(PrefetchAllocation mem);
5619   ins_cost(125);
5620 
5621   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5622   ins_encode %{
5623     __ prefetcht2($mem$$Address);
5624   %}
5625   ins_pipe(ialu_mem);
5626 %}
5627 
5628 //----------Store Instructions-------------------------------------------------
5629 
5630 // Store Byte
5631 instruct storeB(memory mem, rRegI src)
5632 %{
5633   match(Set mem (StoreB mem src));
5634 
5635   ins_cost(125); // XXX
5636   format %{ "movb    $mem, $src\t# byte" %}
5637   opcode(0x88);
5638   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5639   ins_pipe(ialu_mem_reg);
5640 %}
5641 
5642 // Store Char/Short
5643 instruct storeC(memory mem, rRegI src)
5644 %{
5645   match(Set mem (StoreC mem src));
5646 
5647   ins_cost(125); // XXX
5648   format %{ "movw    $mem, $src\t# char/short" %}
5649   opcode(0x89);
5650   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5651   ins_pipe(ialu_mem_reg);
5652 %}
5653 
5654 // Store Integer
5655 instruct storeI(memory mem, rRegI src)
5656 %{
5657   match(Set mem (StoreI mem src));
5658 
5659   ins_cost(125); // XXX
5660   format %{ "movl    $mem, $src\t# int" %}
5661   opcode(0x89);
5662   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5663   ins_pipe(ialu_mem_reg);
5664 %}
5665 
5666 // Store Long
5667 instruct storeL(memory mem, rRegL src)
5668 %{
5669   match(Set mem (StoreL mem src));
5670 
5671   ins_cost(125); // XXX
5672   format %{ "movq    $mem, $src\t# long" %}
5673   opcode(0x89);
5674   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5675   ins_pipe(ialu_mem_reg); // XXX
5676 %}
5677 
5678 // Store Pointer
5679 instruct storeP(memory mem, any_RegP src)
5680 %{
5681   match(Set mem (StoreP mem src));
5682 
5683   ins_cost(125); // XXX
5684   format %{ "movq    $mem, $src\t# ptr" %}
5685   opcode(0x89);
5686   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5687   ins_pipe(ialu_mem_reg);
5688 %}
5689 
5690 instruct storeImmP0(memory mem, immP0 zero)
5691 %{
5692   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5693   match(Set mem (StoreP mem zero));
5694 
5695   ins_cost(125); // XXX
5696   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5697   ins_encode %{
5698     __ movq($mem$$Address, r12);
5699   %}
5700   ins_pipe(ialu_mem_reg);
5701 %}
5702 
5703 // Store NULL Pointer, mark word, or other simple pointer constant.
5704 instruct storeImmP(memory mem, immP31 src)
5705 %{
5706   match(Set mem (StoreP mem src));
5707 
5708   ins_cost(150); // XXX
5709   format %{ "movq    $mem, $src\t# ptr" %}
5710   opcode(0xC7); /* C7 /0 */
5711   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5712   ins_pipe(ialu_mem_imm);
5713 %}
5714 
5715 // Store Compressed Pointer
5716 instruct storeN(memory mem, rRegN src)
5717 %{
5718   match(Set mem (StoreN mem src));
5719 
5720   ins_cost(125); // XXX
5721   format %{ "movl    $mem, $src\t# compressed ptr" %}
5722   ins_encode %{
5723     __ movl($mem$$Address, $src$$Register);
5724   %}
5725   ins_pipe(ialu_mem_reg);
5726 %}
5727 
5728 instruct storeNKlass(memory mem, rRegN src)
5729 %{
5730   match(Set mem (StoreNKlass mem src));
5731 
5732   ins_cost(125); // XXX
5733   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5734   ins_encode %{
5735     __ movl($mem$$Address, $src$$Register);
5736   %}
5737   ins_pipe(ialu_mem_reg);
5738 %}
5739 
5740 instruct storeImmN0(memory mem, immN0 zero)
5741 %{
5742   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5743   match(Set mem (StoreN mem zero));
5744 
5745   ins_cost(125); // XXX
5746   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5747   ins_encode %{
5748     __ movl($mem$$Address, r12);
5749   %}
5750   ins_pipe(ialu_mem_reg);
5751 %}
5752 
5753 instruct storeImmN(memory mem, immN src)
5754 %{
5755   match(Set mem (StoreN mem src));
5756 
5757   ins_cost(150); // XXX
5758   format %{ "movl    $mem, $src\t# compressed ptr" %}
5759   ins_encode %{
5760     address con = (address)$src$$constant;
5761     if (con == NULL) {
5762       __ movl($mem$$Address, (int32_t)0);
5763     } else {
5764       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5765     }
5766   %}
5767   ins_pipe(ialu_mem_imm);
5768 %}
5769 
5770 instruct storeImmNKlass(memory mem, immNKlass src)
5771 %{
5772   match(Set mem (StoreNKlass mem src));
5773 
5774   ins_cost(150); // XXX
5775   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5776   ins_encode %{
5777     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5778   %}
5779   ins_pipe(ialu_mem_imm);
5780 %}
5781 
5782 // Store Integer Immediate
5783 instruct storeImmI0(memory mem, immI0 zero)
5784 %{
5785   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5786   match(Set mem (StoreI mem zero));
5787 
5788   ins_cost(125); // XXX
5789   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5790   ins_encode %{
5791     __ movl($mem$$Address, r12);
5792   %}
5793   ins_pipe(ialu_mem_reg);
5794 %}
5795 
5796 instruct storeImmI(memory mem, immI src)
5797 %{
5798   match(Set mem (StoreI mem src));
5799 
5800   ins_cost(150);
5801   format %{ "movl    $mem, $src\t# int" %}
5802   opcode(0xC7); /* C7 /0 */
5803   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5804   ins_pipe(ialu_mem_imm);
5805 %}
5806 
5807 // Store Long Immediate
5808 instruct storeImmL0(memory mem, immL0 zero)
5809 %{
5810   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5811   match(Set mem (StoreL mem zero));
5812 
5813   ins_cost(125); // XXX
5814   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5815   ins_encode %{
5816     __ movq($mem$$Address, r12);
5817   %}
5818   ins_pipe(ialu_mem_reg);
5819 %}
5820 
5821 instruct storeImmL(memory mem, immL32 src)
5822 %{
5823   match(Set mem (StoreL mem src));
5824 
5825   ins_cost(150);
5826   format %{ "movq    $mem, $src\t# long" %}
5827   opcode(0xC7); /* C7 /0 */
5828   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5829   ins_pipe(ialu_mem_imm);
5830 %}
5831 
5832 // Store Short/Char Immediate
5833 instruct storeImmC0(memory mem, immI0 zero)
5834 %{
5835   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5836   match(Set mem (StoreC mem zero));
5837 
5838   ins_cost(125); // XXX
5839   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5840   ins_encode %{
5841     __ movw($mem$$Address, r12);
5842   %}
5843   ins_pipe(ialu_mem_reg);
5844 %}
5845 
5846 instruct storeImmI16(memory mem, immI16 src)
5847 %{
5848   predicate(UseStoreImmI16);
5849   match(Set mem (StoreC mem src));
5850 
5851   ins_cost(150);
5852   format %{ "movw    $mem, $src\t# short/char" %}
5853   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5854   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5855   ins_pipe(ialu_mem_imm);
5856 %}
5857 
5858 // Store Byte Immediate
5859 instruct storeImmB0(memory mem, immI0 zero)
5860 %{
5861   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5862   match(Set mem (StoreB mem zero));
5863 
5864   ins_cost(125); // XXX
5865   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5866   ins_encode %{
5867     __ movb($mem$$Address, r12);
5868   %}
5869   ins_pipe(ialu_mem_reg);
5870 %}
5871 
5872 instruct storeImmB(memory mem, immI8 src)
5873 %{
5874   match(Set mem (StoreB mem src));
5875 
5876   ins_cost(150); // XXX
5877   format %{ "movb    $mem, $src\t# byte" %}
5878   opcode(0xC6); /* C6 /0 */
5879   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5880   ins_pipe(ialu_mem_imm);
5881 %}
5882 
5883 // Store CMS card-mark Immediate
5884 instruct storeImmCM0_reg(memory mem, immI0 zero)
5885 %{
5886   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5887   match(Set mem (StoreCM mem zero));
5888 
5889   ins_cost(125); // XXX
5890   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5891   ins_encode %{
5892     __ movb($mem$$Address, r12);
5893   %}
5894   ins_pipe(ialu_mem_reg);
5895 %}
5896 
5897 instruct storeImmCM0(memory mem, immI0 src)
5898 %{
5899   match(Set mem (StoreCM mem src));
5900 
5901   ins_cost(150); // XXX
5902   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5903   opcode(0xC6); /* C6 /0 */
5904   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5905   ins_pipe(ialu_mem_imm);
5906 %}
5907 
5908 // Store Float
5909 instruct storeF(memory mem, regF src)
5910 %{
5911   match(Set mem (StoreF mem src));
5912 
5913   ins_cost(95); // XXX
5914   format %{ "movss   $mem, $src\t# float" %}
5915   ins_encode %{
5916     __ movflt($mem$$Address, $src$$XMMRegister);
5917   %}
5918   ins_pipe(pipe_slow); // XXX
5919 %}
5920 
5921 // Store immediate Float value (it is faster than store from XMM register)
5922 instruct storeF0(memory mem, immF0 zero)
5923 %{
5924   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5925   match(Set mem (StoreF mem zero));
5926 
5927   ins_cost(25); // XXX
5928   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5929   ins_encode %{
5930     __ movl($mem$$Address, r12);
5931   %}
5932   ins_pipe(ialu_mem_reg);
5933 %}
5934 
5935 instruct storeF_imm(memory mem, immF src)
5936 %{
5937   match(Set mem (StoreF mem src));
5938 
5939   ins_cost(50);
5940   format %{ "movl    $mem, $src\t# float" %}
5941   opcode(0xC7); /* C7 /0 */
5942   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5943   ins_pipe(ialu_mem_imm);
5944 %}
5945 
5946 // Store Double
5947 instruct storeD(memory mem, regD src)
5948 %{
5949   match(Set mem (StoreD mem src));
5950 
5951   ins_cost(95); // XXX
5952   format %{ "movsd   $mem, $src\t# double" %}
5953   ins_encode %{
5954     __ movdbl($mem$$Address, $src$$XMMRegister);
5955   %}
5956   ins_pipe(pipe_slow); // XXX
5957 %}
5958 
5959 // Store immediate double 0.0 (it is faster than store from XMM register)
5960 instruct storeD0_imm(memory mem, immD0 src)
5961 %{
5962   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5963   match(Set mem (StoreD mem src));
5964 
5965   ins_cost(50);
5966   format %{ "movq    $mem, $src\t# double 0." %}
5967   opcode(0xC7); /* C7 /0 */
5968   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5969   ins_pipe(ialu_mem_imm);
5970 %}
5971 
5972 instruct storeD0(memory mem, immD0 zero)
5973 %{
5974   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5975   match(Set mem (StoreD mem zero));
5976 
5977   ins_cost(25); // XXX
5978   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5979   ins_encode %{
5980     __ movq($mem$$Address, r12);
5981   %}
5982   ins_pipe(ialu_mem_reg);
5983 %}
5984 
5985 instruct storeSSI(stackSlotI dst, rRegI src)
5986 %{
5987   match(Set dst src);
5988 
5989   ins_cost(100);
5990   format %{ "movl    $dst, $src\t# int stk" %}
5991   opcode(0x89);
5992   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5993   ins_pipe( ialu_mem_reg );
5994 %}
5995 
5996 instruct storeSSL(stackSlotL dst, rRegL src)
5997 %{
5998   match(Set dst src);
5999 
6000   ins_cost(100);
6001   format %{ "movq    $dst, $src\t# long stk" %}
6002   opcode(0x89);
6003   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6004   ins_pipe(ialu_mem_reg);
6005 %}
6006 
6007 instruct storeSSP(stackSlotP dst, rRegP src)
6008 %{
6009   match(Set dst src);
6010 
6011   ins_cost(100);
6012   format %{ "movq    $dst, $src\t# ptr stk" %}
6013   opcode(0x89);
6014   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6015   ins_pipe(ialu_mem_reg);
6016 %}
6017 
6018 instruct storeSSF(stackSlotF dst, regF src)
6019 %{
6020   match(Set dst src);
6021 
6022   ins_cost(95); // XXX
6023   format %{ "movss   $dst, $src\t# float stk" %}
6024   ins_encode %{
6025     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6026   %}
6027   ins_pipe(pipe_slow); // XXX
6028 %}
6029 
6030 instruct storeSSD(stackSlotD dst, regD src)
6031 %{
6032   match(Set dst src);
6033 
6034   ins_cost(95); // XXX
6035   format %{ "movsd   $dst, $src\t# double stk" %}
6036   ins_encode %{
6037     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6038   %}
6039   ins_pipe(pipe_slow); // XXX
6040 %}
6041 
6042 //----------BSWAP Instructions-------------------------------------------------
6043 instruct bytes_reverse_int(rRegI dst) %{
6044   match(Set dst (ReverseBytesI dst));
6045 
6046   format %{ "bswapl  $dst" %}
6047   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6048   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6049   ins_pipe( ialu_reg );
6050 %}
6051 
6052 instruct bytes_reverse_long(rRegL dst) %{
6053   match(Set dst (ReverseBytesL dst));
6054 
6055   format %{ "bswapq  $dst" %}
6056   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6057   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6058   ins_pipe( ialu_reg);
6059 %}
6060 
6061 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6062   match(Set dst (ReverseBytesUS dst));
6063   effect(KILL cr);
6064 
6065   format %{ "bswapl  $dst\n\t"
6066             "shrl    $dst,16\n\t" %}
6067   ins_encode %{
6068     __ bswapl($dst$$Register);
6069     __ shrl($dst$$Register, 16);
6070   %}
6071   ins_pipe( ialu_reg );
6072 %}
6073 
6074 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6075   match(Set dst (ReverseBytesS dst));
6076   effect(KILL cr);
6077 
6078   format %{ "bswapl  $dst\n\t"
6079             "sar     $dst,16\n\t" %}
6080   ins_encode %{
6081     __ bswapl($dst$$Register);
6082     __ sarl($dst$$Register, 16);
6083   %}
6084   ins_pipe( ialu_reg );
6085 %}
6086 
6087 //---------- Zeros Count Instructions ------------------------------------------
6088 
6089 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6090   predicate(UseCountLeadingZerosInstruction);
6091   match(Set dst (CountLeadingZerosI src));
6092   effect(KILL cr);
6093 
6094   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6095   ins_encode %{
6096     __ lzcntl($dst$$Register, $src$$Register);
6097   %}
6098   ins_pipe(ialu_reg);
6099 %}
6100 
6101 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6102   predicate(!UseCountLeadingZerosInstruction);
6103   match(Set dst (CountLeadingZerosI src));
6104   effect(KILL cr);
6105 
6106   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6107             "jnz     skip\n\t"
6108             "movl    $dst, -1\n"
6109       "skip:\n\t"
6110             "negl    $dst\n\t"
6111             "addl    $dst, 31" %}
6112   ins_encode %{
6113     Register Rdst = $dst$$Register;
6114     Register Rsrc = $src$$Register;
6115     Label skip;
6116     __ bsrl(Rdst, Rsrc);
6117     __ jccb(Assembler::notZero, skip);
6118     __ movl(Rdst, -1);
6119     __ bind(skip);
6120     __ negl(Rdst);
6121     __ addl(Rdst, BitsPerInt - 1);
6122   %}
6123   ins_pipe(ialu_reg);
6124 %}
6125 
6126 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6127   predicate(UseCountLeadingZerosInstruction);
6128   match(Set dst (CountLeadingZerosL src));
6129   effect(KILL cr);
6130 
6131   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6132   ins_encode %{
6133     __ lzcntq($dst$$Register, $src$$Register);
6134   %}
6135   ins_pipe(ialu_reg);
6136 %}
6137 
6138 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6139   predicate(!UseCountLeadingZerosInstruction);
6140   match(Set dst (CountLeadingZerosL src));
6141   effect(KILL cr);
6142 
6143   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6144             "jnz     skip\n\t"
6145             "movl    $dst, -1\n"
6146       "skip:\n\t"
6147             "negl    $dst\n\t"
6148             "addl    $dst, 63" %}
6149   ins_encode %{
6150     Register Rdst = $dst$$Register;
6151     Register Rsrc = $src$$Register;
6152     Label skip;
6153     __ bsrq(Rdst, Rsrc);
6154     __ jccb(Assembler::notZero, skip);
6155     __ movl(Rdst, -1);
6156     __ bind(skip);
6157     __ negl(Rdst);
6158     __ addl(Rdst, BitsPerLong - 1);
6159   %}
6160   ins_pipe(ialu_reg);
6161 %}
6162 
6163 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6164   predicate(UseCountTrailingZerosInstruction);
6165   match(Set dst (CountTrailingZerosI src));
6166   effect(KILL cr);
6167 
6168   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6169   ins_encode %{
6170     __ tzcntl($dst$$Register, $src$$Register);
6171   %}
6172   ins_pipe(ialu_reg);
6173 %}
6174 
6175 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6176   predicate(!UseCountTrailingZerosInstruction);
6177   match(Set dst (CountTrailingZerosI src));
6178   effect(KILL cr);
6179 
6180   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6181             "jnz     done\n\t"
6182             "movl    $dst, 32\n"
6183       "done:" %}
6184   ins_encode %{
6185     Register Rdst = $dst$$Register;
6186     Label done;
6187     __ bsfl(Rdst, $src$$Register);
6188     __ jccb(Assembler::notZero, done);
6189     __ movl(Rdst, BitsPerInt);
6190     __ bind(done);
6191   %}
6192   ins_pipe(ialu_reg);
6193 %}
6194 
6195 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6196   predicate(UseCountTrailingZerosInstruction);
6197   match(Set dst (CountTrailingZerosL src));
6198   effect(KILL cr);
6199 
6200   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6201   ins_encode %{
6202     __ tzcntq($dst$$Register, $src$$Register);
6203   %}
6204   ins_pipe(ialu_reg);
6205 %}
6206 
6207 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6208   predicate(!UseCountTrailingZerosInstruction);
6209   match(Set dst (CountTrailingZerosL src));
6210   effect(KILL cr);
6211 
6212   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6213             "jnz     done\n\t"
6214             "movl    $dst, 64\n"
6215       "done:" %}
6216   ins_encode %{
6217     Register Rdst = $dst$$Register;
6218     Label done;
6219     __ bsfq(Rdst, $src$$Register);
6220     __ jccb(Assembler::notZero, done);
6221     __ movl(Rdst, BitsPerLong);
6222     __ bind(done);
6223   %}
6224   ins_pipe(ialu_reg);
6225 %}
6226 
6227 
6228 //---------- Population Count Instructions -------------------------------------
6229 
6230 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6231   predicate(UsePopCountInstruction);
6232   match(Set dst (PopCountI src));
6233   effect(KILL cr);
6234 
6235   format %{ "popcnt  $dst, $src" %}
6236   ins_encode %{
6237     __ popcntl($dst$$Register, $src$$Register);
6238   %}
6239   ins_pipe(ialu_reg);
6240 %}
6241 
6242 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6243   predicate(UsePopCountInstruction);
6244   match(Set dst (PopCountI (LoadI mem)));
6245   effect(KILL cr);
6246 
6247   format %{ "popcnt  $dst, $mem" %}
6248   ins_encode %{
6249     __ popcntl($dst$$Register, $mem$$Address);
6250   %}
6251   ins_pipe(ialu_reg);
6252 %}
6253 
6254 // Note: Long.bitCount(long) returns an int.
6255 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6256   predicate(UsePopCountInstruction);
6257   match(Set dst (PopCountL src));
6258   effect(KILL cr);
6259 
6260   format %{ "popcnt  $dst, $src" %}
6261   ins_encode %{
6262     __ popcntq($dst$$Register, $src$$Register);
6263   %}
6264   ins_pipe(ialu_reg);
6265 %}
6266 
6267 // Note: Long.bitCount(long) returns an int.
6268 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6269   predicate(UsePopCountInstruction);
6270   match(Set dst (PopCountL (LoadL mem)));
6271   effect(KILL cr);
6272 
6273   format %{ "popcnt  $dst, $mem" %}
6274   ins_encode %{
6275     __ popcntq($dst$$Register, $mem$$Address);
6276   %}
6277   ins_pipe(ialu_reg);
6278 %}
6279 
6280 
6281 //----------MemBar Instructions-----------------------------------------------
6282 // Memory barrier flavors
6283 
6284 instruct membar_acquire()
6285 %{
6286   match(MemBarAcquire);
6287   match(LoadFence);
6288   ins_cost(0);
6289 
6290   size(0);
6291   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6292   ins_encode();
6293   ins_pipe(empty);
6294 %}
6295 
6296 instruct membar_acquire_lock()
6297 %{
6298   match(MemBarAcquireLock);
6299   ins_cost(0);
6300 
6301   size(0);
6302   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6303   ins_encode();
6304   ins_pipe(empty);
6305 %}
6306 
6307 instruct membar_release()
6308 %{
6309   match(MemBarRelease);
6310   match(StoreFence);
6311   ins_cost(0);
6312 
6313   size(0);
6314   format %{ "MEMBAR-release ! (empty encoding)" %}
6315   ins_encode();
6316   ins_pipe(empty);
6317 %}
6318 
6319 instruct membar_release_lock()
6320 %{
6321   match(MemBarReleaseLock);
6322   ins_cost(0);
6323 
6324   size(0);
6325   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6326   ins_encode();
6327   ins_pipe(empty);
6328 %}
6329 
6330 instruct membar_volatile(rFlagsReg cr) %{
6331   match(MemBarVolatile);
6332   effect(KILL cr);
6333   ins_cost(400);
6334 
6335   format %{
6336     $$template
6337     if (os::is_MP()) {
6338       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6339     } else {
6340       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6341     }
6342   %}
6343   ins_encode %{
6344     __ membar(Assembler::StoreLoad);
6345   %}
6346   ins_pipe(pipe_slow);
6347 %}
6348 
6349 instruct unnecessary_membar_volatile()
6350 %{
6351   match(MemBarVolatile);
6352   predicate(Matcher::post_store_load_barrier(n));
6353   ins_cost(0);
6354 
6355   size(0);
6356   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6357   ins_encode();
6358   ins_pipe(empty);
6359 %}
6360 
6361 instruct membar_storestore() %{
6362   match(MemBarStoreStore);
6363   ins_cost(0);
6364 
6365   size(0);
6366   format %{ "MEMBAR-storestore (empty encoding)" %}
6367   ins_encode( );
6368   ins_pipe(empty);
6369 %}
6370 
6371 //----------Move Instructions--------------------------------------------------
6372 
6373 instruct castX2P(rRegP dst, rRegL src)
6374 %{
6375   match(Set dst (CastX2P src));
6376 
6377   format %{ "movq    $dst, $src\t# long->ptr" %}
6378   ins_encode %{
6379     if ($dst$$reg != $src$$reg) {
6380       __ movptr($dst$$Register, $src$$Register);
6381     }
6382   %}
6383   ins_pipe(ialu_reg_reg); // XXX
6384 %}
6385 
6386 instruct castP2X(rRegL dst, rRegP src)
6387 %{
6388   match(Set dst (CastP2X src));
6389 
6390   format %{ "movq    $dst, $src\t# ptr -> long" %}
6391   ins_encode %{
6392     if ($dst$$reg != $src$$reg) {
6393       __ movptr($dst$$Register, $src$$Register);
6394     }
6395   %}
6396   ins_pipe(ialu_reg_reg); // XXX
6397 %}
6398 
6399 // Convert oop into int for vectors alignment masking
6400 instruct convP2I(rRegI dst, rRegP src)
6401 %{
6402   match(Set dst (ConvL2I (CastP2X src)));
6403 
6404   format %{ "movl    $dst, $src\t# ptr -> int" %}
6405   ins_encode %{
6406     __ movl($dst$$Register, $src$$Register);
6407   %}
6408   ins_pipe(ialu_reg_reg); // XXX
6409 %}
6410 
6411 // Convert compressed oop into int for vectors alignment masking
6412 // in case of 32bit oops (heap < 4Gb).
6413 instruct convN2I(rRegI dst, rRegN src)
6414 %{
6415   predicate(Universe::narrow_oop_shift() == 0);
6416   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6417 
6418   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6419   ins_encode %{
6420     __ movl($dst$$Register, $src$$Register);
6421   %}
6422   ins_pipe(ialu_reg_reg); // XXX
6423 %}
6424 
6425 // Convert oop pointer into compressed form
6426 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6427   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6428   match(Set dst (EncodeP src));
6429   effect(KILL cr);
6430   format %{ "encode_heap_oop $dst,$src" %}
6431   ins_encode %{
6432     Register s = $src$$Register;
6433     Register d = $dst$$Register;
6434     if (s != d) {
6435       __ movq(d, s);
6436     }
6437     __ encode_heap_oop(d);
6438   %}
6439   ins_pipe(ialu_reg_long);
6440 %}
6441 
6442 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6443   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6444   match(Set dst (EncodeP src));
6445   effect(KILL cr);
6446   format %{ "encode_heap_oop_not_null $dst,$src" %}
6447   ins_encode %{
6448     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6449   %}
6450   ins_pipe(ialu_reg_long);
6451 %}
6452 
6453 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6454   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6455             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6456   match(Set dst (DecodeN src));
6457   effect(KILL cr);
6458   format %{ "decode_heap_oop $dst,$src" %}
6459   ins_encode %{
6460     Register s = $src$$Register;
6461     Register d = $dst$$Register;
6462     if (s != d) {
6463       __ movq(d, s);
6464     }
6465     __ decode_heap_oop(d);
6466   %}
6467   ins_pipe(ialu_reg_long);
6468 %}
6469 
6470 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6471   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6472             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6473   match(Set dst (DecodeN src));
6474   effect(KILL cr);
6475   format %{ "decode_heap_oop_not_null $dst,$src" %}
6476   ins_encode %{
6477     Register s = $src$$Register;
6478     Register d = $dst$$Register;
6479     if (s != d) {
6480       __ decode_heap_oop_not_null(d, s);
6481     } else {
6482       __ decode_heap_oop_not_null(d);
6483     }
6484   %}
6485   ins_pipe(ialu_reg_long);
6486 %}
6487 
6488 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6489   match(Set dst (EncodePKlass src));
6490   effect(KILL cr);
6491   format %{ "encode_klass_not_null $dst,$src" %}
6492   ins_encode %{
6493     __ encode_klass_not_null($dst$$Register, $src$$Register);
6494   %}
6495   ins_pipe(ialu_reg_long);
6496 %}
6497 
6498 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6499   match(Set dst (DecodeNKlass src));
6500   effect(KILL cr);
6501   format %{ "decode_klass_not_null $dst,$src" %}
6502   ins_encode %{
6503     Register s = $src$$Register;
6504     Register d = $dst$$Register;
6505     if (s != d) {
6506       __ decode_klass_not_null(d, s);
6507     } else {
6508       __ decode_klass_not_null(d);
6509     }
6510   %}
6511   ins_pipe(ialu_reg_long);
6512 %}
6513 
6514 
6515 //----------Conditional Move---------------------------------------------------
6516 // Jump
6517 // dummy instruction for generating temp registers
6518 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6519   match(Jump (LShiftL switch_val shift));
6520   ins_cost(350);
6521   predicate(false);
6522   effect(TEMP dest);
6523 
6524   format %{ "leaq    $dest, [$constantaddress]\n\t"
6525             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6526   ins_encode %{
6527     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6528     // to do that and the compiler is using that register as one it can allocate.
6529     // So we build it all by hand.
6530     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6531     // ArrayAddress dispatch(table, index);
6532     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6533     __ lea($dest$$Register, $constantaddress);
6534     __ jmp(dispatch);
6535   %}
6536   ins_pipe(pipe_jmp);
6537 %}
6538 
6539 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6540   match(Jump (AddL (LShiftL switch_val shift) offset));
6541   ins_cost(350);
6542   effect(TEMP dest);
6543 
6544   format %{ "leaq    $dest, [$constantaddress]\n\t"
6545             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6546   ins_encode %{
6547     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6548     // to do that and the compiler is using that register as one it can allocate.
6549     // So we build it all by hand.
6550     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6551     // ArrayAddress dispatch(table, index);
6552     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6553     __ lea($dest$$Register, $constantaddress);
6554     __ jmp(dispatch);
6555   %}
6556   ins_pipe(pipe_jmp);
6557 %}
6558 
6559 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6560   match(Jump switch_val);
6561   ins_cost(350);
6562   effect(TEMP dest);
6563 
6564   format %{ "leaq    $dest, [$constantaddress]\n\t"
6565             "jmp     [$dest + $switch_val]\n\t" %}
6566   ins_encode %{
6567     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6568     // to do that and the compiler is using that register as one it can allocate.
6569     // So we build it all by hand.
6570     // Address index(noreg, switch_reg, Address::times_1);
6571     // ArrayAddress dispatch(table, index);
6572     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6573     __ lea($dest$$Register, $constantaddress);
6574     __ jmp(dispatch);
6575   %}
6576   ins_pipe(pipe_jmp);
6577 %}
6578 
6579 // Conditional move
6580 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6581 %{
6582   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6583 
6584   ins_cost(200); // XXX
6585   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6586   opcode(0x0F, 0x40);
6587   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6588   ins_pipe(pipe_cmov_reg);
6589 %}
6590 
6591 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6592   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6593 
6594   ins_cost(200); // XXX
6595   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6596   opcode(0x0F, 0x40);
6597   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6598   ins_pipe(pipe_cmov_reg);
6599 %}
6600 
6601 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6602   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6603   ins_cost(200);
6604   expand %{
6605     cmovI_regU(cop, cr, dst, src);
6606   %}
6607 %}
6608 
6609 // Conditional move
6610 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6611   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6612 
6613   ins_cost(250); // XXX
6614   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6615   opcode(0x0F, 0x40);
6616   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6617   ins_pipe(pipe_cmov_mem);
6618 %}
6619 
6620 // Conditional move
6621 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6622 %{
6623   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6624 
6625   ins_cost(250); // XXX
6626   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6627   opcode(0x0F, 0x40);
6628   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6629   ins_pipe(pipe_cmov_mem);
6630 %}
6631 
6632 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6633   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6634   ins_cost(250);
6635   expand %{
6636     cmovI_memU(cop, cr, dst, src);
6637   %}
6638 %}
6639 
6640 // Conditional move
6641 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6642 %{
6643   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6644 
6645   ins_cost(200); // XXX
6646   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6647   opcode(0x0F, 0x40);
6648   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6649   ins_pipe(pipe_cmov_reg);
6650 %}
6651 
6652 // Conditional move
6653 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6654 %{
6655   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6656 
6657   ins_cost(200); // XXX
6658   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6659   opcode(0x0F, 0x40);
6660   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6661   ins_pipe(pipe_cmov_reg);
6662 %}
6663 
6664 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6665   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6666   ins_cost(200);
6667   expand %{
6668     cmovN_regU(cop, cr, dst, src);
6669   %}
6670 %}
6671 
6672 // Conditional move
6673 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6674 %{
6675   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6676 
6677   ins_cost(200); // XXX
6678   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6679   opcode(0x0F, 0x40);
6680   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6681   ins_pipe(pipe_cmov_reg);  // XXX
6682 %}
6683 
6684 // Conditional move
6685 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6686 %{
6687   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6688 
6689   ins_cost(200); // XXX
6690   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6691   opcode(0x0F, 0x40);
6692   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6693   ins_pipe(pipe_cmov_reg); // XXX
6694 %}
6695 
6696 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6697   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6698   ins_cost(200);
6699   expand %{
6700     cmovP_regU(cop, cr, dst, src);
6701   %}
6702 %}
6703 
6704 // DISABLED: Requires the ADLC to emit a bottom_type call that
6705 // correctly meets the two pointer arguments; one is an incoming
6706 // register but the other is a memory operand.  ALSO appears to
6707 // be buggy with implicit null checks.
6708 //
6709 //// Conditional move
6710 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6711 //%{
6712 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6713 //  ins_cost(250);
6714 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6715 //  opcode(0x0F,0x40);
6716 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6717 //  ins_pipe( pipe_cmov_mem );
6718 //%}
6719 //
6720 //// Conditional move
6721 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6722 //%{
6723 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6724 //  ins_cost(250);
6725 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6726 //  opcode(0x0F,0x40);
6727 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6728 //  ins_pipe( pipe_cmov_mem );
6729 //%}
6730 
6731 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6732 %{
6733   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6734 
6735   ins_cost(200); // XXX
6736   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6737   opcode(0x0F, 0x40);
6738   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6739   ins_pipe(pipe_cmov_reg);  // XXX
6740 %}
6741 
6742 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6743 %{
6744   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6745 
6746   ins_cost(200); // XXX
6747   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6748   opcode(0x0F, 0x40);
6749   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6750   ins_pipe(pipe_cmov_mem);  // XXX
6751 %}
6752 
6753 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6754 %{
6755   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6756 
6757   ins_cost(200); // XXX
6758   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6759   opcode(0x0F, 0x40);
6760   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6761   ins_pipe(pipe_cmov_reg); // XXX
6762 %}
6763 
6764 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6765   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6766   ins_cost(200);
6767   expand %{
6768     cmovL_regU(cop, cr, dst, src);
6769   %}
6770 %}
6771 
6772 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6773 %{
6774   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6775 
6776   ins_cost(200); // XXX
6777   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6778   opcode(0x0F, 0x40);
6779   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6780   ins_pipe(pipe_cmov_mem); // XXX
6781 %}
6782 
6783 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6784   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6785   ins_cost(200);
6786   expand %{
6787     cmovL_memU(cop, cr, dst, src);
6788   %}
6789 %}
6790 
6791 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6792 %{
6793   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6794 
6795   ins_cost(200); // XXX
6796   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6797             "movss     $dst, $src\n"
6798     "skip:" %}
6799   ins_encode %{
6800     Label Lskip;
6801     // Invert sense of branch from sense of CMOV
6802     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6803     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6804     __ bind(Lskip);
6805   %}
6806   ins_pipe(pipe_slow);
6807 %}
6808 
6809 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6810 // %{
6811 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6812 
6813 //   ins_cost(200); // XXX
6814 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6815 //             "movss     $dst, $src\n"
6816 //     "skip:" %}
6817 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6818 //   ins_pipe(pipe_slow);
6819 // %}
6820 
6821 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6822 %{
6823   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6824 
6825   ins_cost(200); // XXX
6826   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6827             "movss     $dst, $src\n"
6828     "skip:" %}
6829   ins_encode %{
6830     Label Lskip;
6831     // Invert sense of branch from sense of CMOV
6832     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6833     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6834     __ bind(Lskip);
6835   %}
6836   ins_pipe(pipe_slow);
6837 %}
6838 
6839 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6840   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6841   ins_cost(200);
6842   expand %{
6843     cmovF_regU(cop, cr, dst, src);
6844   %}
6845 %}
6846 
6847 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6848 %{
6849   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6850 
6851   ins_cost(200); // XXX
6852   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6853             "movsd     $dst, $src\n"
6854     "skip:" %}
6855   ins_encode %{
6856     Label Lskip;
6857     // Invert sense of branch from sense of CMOV
6858     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6859     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6860     __ bind(Lskip);
6861   %}
6862   ins_pipe(pipe_slow);
6863 %}
6864 
6865 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6866 %{
6867   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6868 
6869   ins_cost(200); // XXX
6870   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6871             "movsd     $dst, $src\n"
6872     "skip:" %}
6873   ins_encode %{
6874     Label Lskip;
6875     // Invert sense of branch from sense of CMOV
6876     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6877     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6878     __ bind(Lskip);
6879   %}
6880   ins_pipe(pipe_slow);
6881 %}
6882 
6883 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6884   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6885   ins_cost(200);
6886   expand %{
6887     cmovD_regU(cop, cr, dst, src);
6888   %}
6889 %}
6890 
6891 //----------Arithmetic Instructions--------------------------------------------
6892 //----------Addition Instructions----------------------------------------------
6893 
6894 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6895 %{
6896   match(Set dst (AddI dst src));
6897   effect(KILL cr);
6898 
6899   format %{ "addl    $dst, $src\t# int" %}
6900   opcode(0x03);
6901   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6902   ins_pipe(ialu_reg_reg);
6903 %}
6904 
6905 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6906 %{
6907   match(Set dst (AddI dst src));
6908   effect(KILL cr);
6909 
6910   format %{ "addl    $dst, $src\t# int" %}
6911   opcode(0x81, 0x00); /* /0 id */
6912   ins_encode(OpcSErm(dst, src), Con8or32(src));
6913   ins_pipe( ialu_reg );
6914 %}
6915 
6916 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6917 %{
6918   match(Set dst (AddI dst (LoadI src)));
6919   effect(KILL cr);
6920 
6921   ins_cost(125); // XXX
6922   format %{ "addl    $dst, $src\t# int" %}
6923   opcode(0x03);
6924   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6925   ins_pipe(ialu_reg_mem);
6926 %}
6927 
6928 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6929 %{
6930   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6931   effect(KILL cr);
6932 
6933   ins_cost(150); // XXX
6934   format %{ "addl    $dst, $src\t# int" %}
6935   opcode(0x01); /* Opcode 01 /r */
6936   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6937   ins_pipe(ialu_mem_reg);
6938 %}
6939 
6940 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6941 %{
6942   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6943   effect(KILL cr);
6944 
6945   ins_cost(125); // XXX
6946   format %{ "addl    $dst, $src\t# int" %}
6947   opcode(0x81); /* Opcode 81 /0 id */
6948   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6949   ins_pipe(ialu_mem_imm);
6950 %}
6951 
6952 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6953 %{
6954   predicate(UseIncDec);
6955   match(Set dst (AddI dst src));
6956   effect(KILL cr);
6957 
6958   format %{ "incl    $dst\t# int" %}
6959   opcode(0xFF, 0x00); // FF /0
6960   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6961   ins_pipe(ialu_reg);
6962 %}
6963 
6964 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6965 %{
6966   predicate(UseIncDec);
6967   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6968   effect(KILL cr);
6969 
6970   ins_cost(125); // XXX
6971   format %{ "incl    $dst\t# int" %}
6972   opcode(0xFF); /* Opcode FF /0 */
6973   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6974   ins_pipe(ialu_mem_imm);
6975 %}
6976 
6977 // XXX why does that use AddI
6978 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6979 %{
6980   predicate(UseIncDec);
6981   match(Set dst (AddI dst src));
6982   effect(KILL cr);
6983 
6984   format %{ "decl    $dst\t# int" %}
6985   opcode(0xFF, 0x01); // FF /1
6986   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6987   ins_pipe(ialu_reg);
6988 %}
6989 
6990 // XXX why does that use AddI
6991 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6992 %{
6993   predicate(UseIncDec);
6994   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6995   effect(KILL cr);
6996 
6997   ins_cost(125); // XXX
6998   format %{ "decl    $dst\t# int" %}
6999   opcode(0xFF); /* Opcode FF /1 */
7000   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7001   ins_pipe(ialu_mem_imm);
7002 %}
7003 
7004 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7005 %{
7006   match(Set dst (AddI src0 src1));
7007 
7008   ins_cost(110);
7009   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7010   opcode(0x8D); /* 0x8D /r */
7011   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7012   ins_pipe(ialu_reg_reg);
7013 %}
7014 
7015 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7016 %{
7017   match(Set dst (AddL dst src));
7018   effect(KILL cr);
7019 
7020   format %{ "addq    $dst, $src\t# long" %}
7021   opcode(0x03);
7022   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7023   ins_pipe(ialu_reg_reg);
7024 %}
7025 
7026 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7027 %{
7028   match(Set dst (AddL dst src));
7029   effect(KILL cr);
7030 
7031   format %{ "addq    $dst, $src\t# long" %}
7032   opcode(0x81, 0x00); /* /0 id */
7033   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7034   ins_pipe( ialu_reg );
7035 %}
7036 
7037 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7038 %{
7039   match(Set dst (AddL dst (LoadL src)));
7040   effect(KILL cr);
7041 
7042   ins_cost(125); // XXX
7043   format %{ "addq    $dst, $src\t# long" %}
7044   opcode(0x03);
7045   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7046   ins_pipe(ialu_reg_mem);
7047 %}
7048 
7049 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7050 %{
7051   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7052   effect(KILL cr);
7053 
7054   ins_cost(150); // XXX
7055   format %{ "addq    $dst, $src\t# long" %}
7056   opcode(0x01); /* Opcode 01 /r */
7057   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7058   ins_pipe(ialu_mem_reg);
7059 %}
7060 
7061 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7062 %{
7063   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7064   effect(KILL cr);
7065 
7066   ins_cost(125); // XXX
7067   format %{ "addq    $dst, $src\t# long" %}
7068   opcode(0x81); /* Opcode 81 /0 id */
7069   ins_encode(REX_mem_wide(dst),
7070              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7071   ins_pipe(ialu_mem_imm);
7072 %}
7073 
7074 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7075 %{
7076   predicate(UseIncDec);
7077   match(Set dst (AddL dst src));
7078   effect(KILL cr);
7079 
7080   format %{ "incq    $dst\t# long" %}
7081   opcode(0xFF, 0x00); // FF /0
7082   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7083   ins_pipe(ialu_reg);
7084 %}
7085 
7086 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7087 %{
7088   predicate(UseIncDec);
7089   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7090   effect(KILL cr);
7091 
7092   ins_cost(125); // XXX
7093   format %{ "incq    $dst\t# long" %}
7094   opcode(0xFF); /* Opcode FF /0 */
7095   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7096   ins_pipe(ialu_mem_imm);
7097 %}
7098 
7099 // XXX why does that use AddL
7100 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7101 %{
7102   predicate(UseIncDec);
7103   match(Set dst (AddL dst src));
7104   effect(KILL cr);
7105 
7106   format %{ "decq    $dst\t# long" %}
7107   opcode(0xFF, 0x01); // FF /1
7108   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7109   ins_pipe(ialu_reg);
7110 %}
7111 
7112 // XXX why does that use AddL
7113 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7114 %{
7115   predicate(UseIncDec);
7116   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7117   effect(KILL cr);
7118 
7119   ins_cost(125); // XXX
7120   format %{ "decq    $dst\t# long" %}
7121   opcode(0xFF); /* Opcode FF /1 */
7122   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7123   ins_pipe(ialu_mem_imm);
7124 %}
7125 
7126 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7127 %{
7128   match(Set dst (AddL src0 src1));
7129 
7130   ins_cost(110);
7131   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7132   opcode(0x8D); /* 0x8D /r */
7133   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7134   ins_pipe(ialu_reg_reg);
7135 %}
7136 
7137 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7138 %{
7139   match(Set dst (AddP dst src));
7140   effect(KILL cr);
7141 
7142   format %{ "addq    $dst, $src\t# ptr" %}
7143   opcode(0x03);
7144   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7145   ins_pipe(ialu_reg_reg);
7146 %}
7147 
7148 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7149 %{
7150   match(Set dst (AddP dst src));
7151   effect(KILL cr);
7152 
7153   format %{ "addq    $dst, $src\t# ptr" %}
7154   opcode(0x81, 0x00); /* /0 id */
7155   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7156   ins_pipe( ialu_reg );
7157 %}
7158 
7159 // XXX addP mem ops ????
7160 
7161 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7162 %{
7163   match(Set dst (AddP src0 src1));
7164 
7165   ins_cost(110);
7166   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7167   opcode(0x8D); /* 0x8D /r */
7168   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7169   ins_pipe(ialu_reg_reg);
7170 %}
7171 
7172 instruct checkCastPP(rRegP dst)
7173 %{
7174   match(Set dst (CheckCastPP dst));
7175 
7176   size(0);
7177   format %{ "# checkcastPP of $dst" %}
7178   ins_encode(/* empty encoding */);
7179   ins_pipe(empty);
7180 %}
7181 
7182 instruct castPP(rRegP dst)
7183 %{
7184   match(Set dst (CastPP dst));
7185 
7186   size(0);
7187   format %{ "# castPP of $dst" %}
7188   ins_encode(/* empty encoding */);
7189   ins_pipe(empty);
7190 %}
7191 
7192 instruct castII(rRegI dst)
7193 %{
7194   match(Set dst (CastII dst));
7195 
7196   size(0);
7197   format %{ "# castII of $dst" %}
7198   ins_encode(/* empty encoding */);
7199   ins_cost(0);
7200   ins_pipe(empty);
7201 %}
7202 
7203 // LoadP-locked same as a regular LoadP when used with compare-swap
7204 instruct loadPLocked(rRegP dst, memory mem)
7205 %{
7206   match(Set dst (LoadPLocked mem));
7207 
7208   ins_cost(125); // XXX
7209   format %{ "movq    $dst, $mem\t# ptr locked" %}
7210   opcode(0x8B);
7211   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7212   ins_pipe(ialu_reg_mem); // XXX
7213 %}
7214 
7215 // Conditional-store of the updated heap-top.
7216 // Used during allocation of the shared heap.
7217 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7218 
7219 instruct storePConditional(memory heap_top_ptr,
7220                            rax_RegP oldval, rRegP newval,
7221                            rFlagsReg cr)
7222 %{
7223   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7224 
7225   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7226             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7227   opcode(0x0F, 0xB1);
7228   ins_encode(lock_prefix,
7229              REX_reg_mem_wide(newval, heap_top_ptr),
7230              OpcP, OpcS,
7231              reg_mem(newval, heap_top_ptr));
7232   ins_pipe(pipe_cmpxchg);
7233 %}
7234 
7235 // Conditional-store of an int value.
7236 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7237 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7238 %{
7239   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7240   effect(KILL oldval);
7241 
7242   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7243   opcode(0x0F, 0xB1);
7244   ins_encode(lock_prefix,
7245              REX_reg_mem(newval, mem),
7246              OpcP, OpcS,
7247              reg_mem(newval, mem));
7248   ins_pipe(pipe_cmpxchg);
7249 %}
7250 
7251 // Conditional-store of a long value.
7252 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7253 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7254 %{
7255   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7256   effect(KILL oldval);
7257 
7258   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7259   opcode(0x0F, 0xB1);
7260   ins_encode(lock_prefix,
7261              REX_reg_mem_wide(newval, mem),
7262              OpcP, OpcS,
7263              reg_mem(newval, mem));
7264   ins_pipe(pipe_cmpxchg);
7265 %}
7266 
7267 
7268 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7269 instruct compareAndSwapP(rRegI res,
7270                          memory mem_ptr,
7271                          rax_RegP oldval, rRegP newval,
7272                          rFlagsReg cr)
7273 %{
7274   predicate(VM_Version::supports_cx8());
7275   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7276   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7277   effect(KILL cr, KILL oldval);
7278 
7279   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7280             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7281             "sete    $res\n\t"
7282             "movzbl  $res, $res" %}
7283   opcode(0x0F, 0xB1);
7284   ins_encode(lock_prefix,
7285              REX_reg_mem_wide(newval, mem_ptr),
7286              OpcP, OpcS,
7287              reg_mem(newval, mem_ptr),
7288              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7289              REX_reg_breg(res, res), // movzbl
7290              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7291   ins_pipe( pipe_cmpxchg );
7292 %}
7293 
7294 instruct compareAndSwapL(rRegI res,
7295                          memory mem_ptr,
7296                          rax_RegL oldval, rRegL newval,
7297                          rFlagsReg cr)
7298 %{
7299   predicate(VM_Version::supports_cx8());
7300   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7301   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7302   effect(KILL cr, KILL oldval);
7303 
7304   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7305             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7306             "sete    $res\n\t"
7307             "movzbl  $res, $res" %}
7308   opcode(0x0F, 0xB1);
7309   ins_encode(lock_prefix,
7310              REX_reg_mem_wide(newval, mem_ptr),
7311              OpcP, OpcS,
7312              reg_mem(newval, mem_ptr),
7313              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7314              REX_reg_breg(res, res), // movzbl
7315              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7316   ins_pipe( pipe_cmpxchg );
7317 %}
7318 
7319 instruct compareAndSwapI(rRegI res,
7320                          memory mem_ptr,
7321                          rax_RegI oldval, rRegI newval,
7322                          rFlagsReg cr)
7323 %{
7324   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7325   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7326   effect(KILL cr, KILL oldval);
7327 
7328   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7329             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7330             "sete    $res\n\t"
7331             "movzbl  $res, $res" %}
7332   opcode(0x0F, 0xB1);
7333   ins_encode(lock_prefix,
7334              REX_reg_mem(newval, mem_ptr),
7335              OpcP, OpcS,
7336              reg_mem(newval, mem_ptr),
7337              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7338              REX_reg_breg(res, res), // movzbl
7339              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7340   ins_pipe( pipe_cmpxchg );
7341 %}
7342 
7343 instruct compareAndSwapB(rRegI res,
7344                          memory mem_ptr,
7345                          rax_RegI oldval, rRegI newval,
7346                          rFlagsReg cr)
7347 %{
7348   match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
7349   match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
7350   effect(KILL cr, KILL oldval);
7351 
7352   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7353             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7354             "sete    $res\n\t"
7355             "movzbl  $res, $res" %}
7356   opcode(0x0F, 0xB0);
7357   ins_encode(lock_prefix,
7358              REX_reg_mem(newval, mem_ptr),
7359              OpcP, OpcS,
7360              reg_mem(newval, mem_ptr),
7361              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7362              REX_reg_breg(res, res), // movzbl
7363              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7364   ins_pipe( pipe_cmpxchg );
7365 %}
7366 
7367 instruct compareAndSwapS(rRegI res,
7368                          memory mem_ptr,
7369                          rax_RegI oldval, rRegI newval,
7370                          rFlagsReg cr)
7371 %{
7372   match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
7373   match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
7374   effect(KILL cr, KILL oldval);
7375 
7376   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7377             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7378             "sete    $res\n\t"
7379             "movzbl  $res, $res" %}
7380   opcode(0x0F, 0xB1);
7381   ins_encode(lock_prefix,
7382              SizePrefix,
7383              REX_reg_mem(newval, mem_ptr),          
7384              OpcP, OpcS,
7385              reg_mem(newval, mem_ptr),
7386              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7387              REX_reg_breg(res, res), // movzbl
7388              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7389   ins_pipe( pipe_cmpxchg );
7390 %}
7391 
7392 instruct compareAndSwapN(rRegI res,
7393                           memory mem_ptr,
7394                           rax_RegN oldval, rRegN newval,
7395                           rFlagsReg cr) %{
7396   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7397   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7398   effect(KILL cr, KILL oldval);
7399 
7400   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7401             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7402             "sete    $res\n\t"
7403             "movzbl  $res, $res" %}
7404   opcode(0x0F, 0xB1);
7405   ins_encode(lock_prefix,
7406              REX_reg_mem(newval, mem_ptr),
7407              OpcP, OpcS,
7408              reg_mem(newval, mem_ptr),
7409              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7410              REX_reg_breg(res, res), // movzbl
7411              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7412   ins_pipe( pipe_cmpxchg );
7413 %}
7414 
7415 instruct compareAndExchangeB(
7416                          memory mem_ptr,
7417                          rax_RegI oldval, rRegI newval,
7418                          rFlagsReg cr)
7419 %{
7420   match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
7421   effect(KILL cr);
7422 
7423   format %{ "cmpxchgb $mem_ptr,$newval\t# "
7424             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7425   opcode(0x0F, 0xB0);
7426   ins_encode(lock_prefix,
7427              REX_reg_mem(newval, mem_ptr),
7428              OpcP, OpcS,
7429              reg_mem(newval, mem_ptr) // lock cmpxchg
7430              );
7431   ins_pipe( pipe_cmpxchg );
7432 %}
7433 
7434 instruct compareAndExchangeS(
7435                          memory mem_ptr,
7436                          rax_RegI oldval, rRegI newval,
7437                          rFlagsReg cr)
7438 %{
7439   match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
7440   effect(KILL cr);
7441 
7442   format %{ "cmpxchgw $mem_ptr,$newval\t# "
7443             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7444   opcode(0x0F, 0xB1);
7445   ins_encode(lock_prefix,
7446              SizePrefix,
7447              REX_reg_mem(newval, mem_ptr),
7448              OpcP, OpcS,
7449              reg_mem(newval, mem_ptr) // lock cmpxchg
7450              );
7451   ins_pipe( pipe_cmpxchg );
7452 %}
7453 
7454 instruct compareAndExchangeI(
7455                          memory mem_ptr,
7456                          rax_RegI oldval, rRegI newval,
7457                          rFlagsReg cr)
7458 %{
7459   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7460   effect(KILL cr);
7461 
7462   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7463             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7464   opcode(0x0F, 0xB1);
7465   ins_encode(lock_prefix,
7466              REX_reg_mem(newval, mem_ptr),
7467              OpcP, OpcS,
7468              reg_mem(newval, mem_ptr) // lock cmpxchg
7469              );
7470   ins_pipe( pipe_cmpxchg );
7471 %}
7472 
7473 instruct compareAndExchangeL(
7474                          memory mem_ptr,
7475                          rax_RegL oldval, rRegL newval,
7476                          rFlagsReg cr)
7477 %{
7478   predicate(VM_Version::supports_cx8());
7479   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7480   effect(KILL cr);
7481 
7482   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7483             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7484   opcode(0x0F, 0xB1);
7485   ins_encode(lock_prefix,
7486              REX_reg_mem_wide(newval, mem_ptr),
7487              OpcP, OpcS,
7488              reg_mem(newval, mem_ptr)  // lock cmpxchg
7489             );
7490   ins_pipe( pipe_cmpxchg );
7491 %}
7492 
7493 instruct compareAndExchangeN(
7494                           memory mem_ptr,
7495                           rax_RegN oldval, rRegN newval,
7496                           rFlagsReg cr) %{
7497   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7498   effect(KILL cr);
7499 
7500   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7501             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7502   opcode(0x0F, 0xB1);
7503   ins_encode(lock_prefix,
7504              REX_reg_mem(newval, mem_ptr),
7505              OpcP, OpcS,
7506              reg_mem(newval, mem_ptr)  // lock cmpxchg
7507           );
7508   ins_pipe( pipe_cmpxchg );
7509 %}
7510 
7511 instruct compareAndExchangeP(
7512                          memory mem_ptr,
7513                          rax_RegP oldval, rRegP newval,
7514                          rFlagsReg cr)
7515 %{
7516   predicate(VM_Version::supports_cx8());
7517   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7518   effect(KILL cr);
7519 
7520   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7521             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7522   opcode(0x0F, 0xB1);
7523   ins_encode(lock_prefix,
7524              REX_reg_mem_wide(newval, mem_ptr),
7525              OpcP, OpcS,
7526              reg_mem(newval, mem_ptr)  // lock cmpxchg
7527           );
7528   ins_pipe( pipe_cmpxchg );
7529 %}
7530 
7531 instruct xaddB_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7532   predicate(n->as_LoadStore()->result_not_used());
7533   match(Set dummy (GetAndAddB mem add));
7534   effect(KILL cr);
7535   format %{ "ADDB  [$mem],$add" %}
7536   ins_encode %{
7537     if (os::is_MP()) { __ lock(); }
7538     __ addb($mem$$Address, $add$$constant);
7539   %}
7540   ins_pipe( pipe_cmpxchg );
7541 %}
7542 
7543 instruct xaddB( memory mem, rRegI newval, rFlagsReg cr) %{
7544   match(Set newval (GetAndAddB mem newval));
7545   effect(KILL cr);
7546   format %{ "XADDB  [$mem],$newval" %}
7547   ins_encode %{
7548     if (os::is_MP()) { __ lock(); }
7549     __ xaddb($mem$$Address, $newval$$Register);
7550   %}
7551   ins_pipe( pipe_cmpxchg );
7552 %}
7553 
7554 instruct xaddS_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7555   predicate(n->as_LoadStore()->result_not_used());
7556   match(Set dummy (GetAndAddS mem add));
7557   effect(KILL cr);
7558   format %{ "ADDW  [$mem],$add" %}
7559   ins_encode %{
7560     if (os::is_MP()) { __ lock(); }
7561     __ addw($mem$$Address, $add$$constant);
7562   %}
7563   ins_pipe( pipe_cmpxchg );
7564 %}
7565 
7566 instruct xaddS( memory mem, rRegI newval, rFlagsReg cr) %{
7567   match(Set newval (GetAndAddS mem newval));
7568   effect(KILL cr);
7569   format %{ "XADDW  [$mem],$newval" %}
7570   ins_encode %{
7571     if (os::is_MP()) { __ lock(); }
7572     __ xaddw($mem$$Address, $newval$$Register);
7573   %}
7574   ins_pipe( pipe_cmpxchg );
7575 %}
7576 
7577 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7578   predicate(n->as_LoadStore()->result_not_used());
7579   match(Set dummy (GetAndAddI mem add));
7580   effect(KILL cr);
7581   format %{ "ADDL  [$mem],$add" %}
7582   ins_encode %{
7583     if (os::is_MP()) { __ lock(); }
7584     __ addl($mem$$Address, $add$$constant);
7585   %}
7586   ins_pipe( pipe_cmpxchg );
7587 %}
7588 
7589 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7590   match(Set newval (GetAndAddI mem newval));
7591   effect(KILL cr);
7592   format %{ "XADDL  [$mem],$newval" %}
7593   ins_encode %{
7594     if (os::is_MP()) { __ lock(); }
7595     __ xaddl($mem$$Address, $newval$$Register);
7596   %}
7597   ins_pipe( pipe_cmpxchg );
7598 %}
7599 
7600 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7601   predicate(n->as_LoadStore()->result_not_used());
7602   match(Set dummy (GetAndAddL mem add));
7603   effect(KILL cr);
7604   format %{ "ADDQ  [$mem],$add" %}
7605   ins_encode %{
7606     if (os::is_MP()) { __ lock(); }
7607     __ addq($mem$$Address, $add$$constant);
7608   %}
7609   ins_pipe( pipe_cmpxchg );
7610 %}
7611 
7612 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7613   match(Set newval (GetAndAddL mem newval));
7614   effect(KILL cr);
7615   format %{ "XADDQ  [$mem],$newval" %}
7616   ins_encode %{
7617     if (os::is_MP()) { __ lock(); }
7618     __ xaddq($mem$$Address, $newval$$Register);
7619   %}
7620   ins_pipe( pipe_cmpxchg );
7621 %}
7622 
7623 instruct xchgB( memory mem, rRegI newval) %{
7624   match(Set newval (GetAndSetB mem newval));
7625   format %{ "XCHGB  $newval,[$mem]" %}
7626   ins_encode %{
7627     __ xchgb($newval$$Register, $mem$$Address);
7628   %}
7629   ins_pipe( pipe_cmpxchg );
7630 %}
7631 
7632 instruct xchgS( memory mem, rRegI newval) %{
7633   match(Set newval (GetAndSetS mem newval));
7634   format %{ "XCHGW  $newval,[$mem]" %}
7635   ins_encode %{
7636     __ xchgw($newval$$Register, $mem$$Address);
7637   %}
7638   ins_pipe( pipe_cmpxchg );
7639 %}
7640 
7641 instruct xchgI( memory mem, rRegI newval) %{
7642   match(Set newval (GetAndSetI mem newval));
7643   format %{ "XCHGL  $newval,[$mem]" %}
7644   ins_encode %{
7645     __ xchgl($newval$$Register, $mem$$Address);
7646   %}
7647   ins_pipe( pipe_cmpxchg );
7648 %}
7649 
7650 instruct xchgL( memory mem, rRegL newval) %{
7651   match(Set newval (GetAndSetL mem newval));
7652   format %{ "XCHGL  $newval,[$mem]" %}
7653   ins_encode %{
7654     __ xchgq($newval$$Register, $mem$$Address);
7655   %}
7656   ins_pipe( pipe_cmpxchg );
7657 %}
7658 
7659 instruct xchgP( memory mem, rRegP newval) %{
7660   match(Set newval (GetAndSetP mem newval));
7661   format %{ "XCHGQ  $newval,[$mem]" %}
7662   ins_encode %{
7663     __ xchgq($newval$$Register, $mem$$Address);
7664   %}
7665   ins_pipe( pipe_cmpxchg );
7666 %}
7667 
7668 instruct xchgN( memory mem, rRegN newval) %{
7669   match(Set newval (GetAndSetN mem newval));
7670   format %{ "XCHGL  $newval,$mem]" %}
7671   ins_encode %{
7672     __ xchgl($newval$$Register, $mem$$Address);
7673   %}
7674   ins_pipe( pipe_cmpxchg );
7675 %}
7676 
7677 //----------Subtraction Instructions-------------------------------------------
7678 
7679 // Integer Subtraction Instructions
7680 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7681 %{
7682   match(Set dst (SubI dst src));
7683   effect(KILL cr);
7684 
7685   format %{ "subl    $dst, $src\t# int" %}
7686   opcode(0x2B);
7687   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7688   ins_pipe(ialu_reg_reg);
7689 %}
7690 
7691 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7692 %{
7693   match(Set dst (SubI dst src));
7694   effect(KILL cr);
7695 
7696   format %{ "subl    $dst, $src\t# int" %}
7697   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7698   ins_encode(OpcSErm(dst, src), Con8or32(src));
7699   ins_pipe(ialu_reg);
7700 %}
7701 
7702 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7703 %{
7704   match(Set dst (SubI dst (LoadI src)));
7705   effect(KILL cr);
7706 
7707   ins_cost(125);
7708   format %{ "subl    $dst, $src\t# int" %}
7709   opcode(0x2B);
7710   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7711   ins_pipe(ialu_reg_mem);
7712 %}
7713 
7714 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7715 %{
7716   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7717   effect(KILL cr);
7718 
7719   ins_cost(150);
7720   format %{ "subl    $dst, $src\t# int" %}
7721   opcode(0x29); /* Opcode 29 /r */
7722   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7723   ins_pipe(ialu_mem_reg);
7724 %}
7725 
7726 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7727 %{
7728   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7729   effect(KILL cr);
7730 
7731   ins_cost(125); // XXX
7732   format %{ "subl    $dst, $src\t# int" %}
7733   opcode(0x81); /* Opcode 81 /5 id */
7734   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7735   ins_pipe(ialu_mem_imm);
7736 %}
7737 
7738 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7739 %{
7740   match(Set dst (SubL dst src));
7741   effect(KILL cr);
7742 
7743   format %{ "subq    $dst, $src\t# long" %}
7744   opcode(0x2B);
7745   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7746   ins_pipe(ialu_reg_reg);
7747 %}
7748 
7749 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7750 %{
7751   match(Set dst (SubL dst src));
7752   effect(KILL cr);
7753 
7754   format %{ "subq    $dst, $src\t# long" %}
7755   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7756   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7757   ins_pipe(ialu_reg);
7758 %}
7759 
7760 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7761 %{
7762   match(Set dst (SubL dst (LoadL src)));
7763   effect(KILL cr);
7764 
7765   ins_cost(125);
7766   format %{ "subq    $dst, $src\t# long" %}
7767   opcode(0x2B);
7768   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7769   ins_pipe(ialu_reg_mem);
7770 %}
7771 
7772 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7773 %{
7774   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7775   effect(KILL cr);
7776 
7777   ins_cost(150);
7778   format %{ "subq    $dst, $src\t# long" %}
7779   opcode(0x29); /* Opcode 29 /r */
7780   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7781   ins_pipe(ialu_mem_reg);
7782 %}
7783 
7784 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7785 %{
7786   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7787   effect(KILL cr);
7788 
7789   ins_cost(125); // XXX
7790   format %{ "subq    $dst, $src\t# long" %}
7791   opcode(0x81); /* Opcode 81 /5 id */
7792   ins_encode(REX_mem_wide(dst),
7793              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7794   ins_pipe(ialu_mem_imm);
7795 %}
7796 
7797 // Subtract from a pointer
7798 // XXX hmpf???
7799 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7800 %{
7801   match(Set dst (AddP dst (SubI zero src)));
7802   effect(KILL cr);
7803 
7804   format %{ "subq    $dst, $src\t# ptr - int" %}
7805   opcode(0x2B);
7806   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7807   ins_pipe(ialu_reg_reg);
7808 %}
7809 
7810 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7811 %{
7812   match(Set dst (SubI zero dst));
7813   effect(KILL cr);
7814 
7815   format %{ "negl    $dst\t# int" %}
7816   opcode(0xF7, 0x03);  // Opcode F7 /3
7817   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7818   ins_pipe(ialu_reg);
7819 %}
7820 
7821 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7822 %{
7823   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7824   effect(KILL cr);
7825 
7826   format %{ "negl    $dst\t# int" %}
7827   opcode(0xF7, 0x03);  // Opcode F7 /3
7828   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7829   ins_pipe(ialu_reg);
7830 %}
7831 
7832 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7833 %{
7834   match(Set dst (SubL zero dst));
7835   effect(KILL cr);
7836 
7837   format %{ "negq    $dst\t# long" %}
7838   opcode(0xF7, 0x03);  // Opcode F7 /3
7839   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7840   ins_pipe(ialu_reg);
7841 %}
7842 
7843 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7844 %{
7845   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7846   effect(KILL cr);
7847 
7848   format %{ "negq    $dst\t# long" %}
7849   opcode(0xF7, 0x03);  // Opcode F7 /3
7850   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7851   ins_pipe(ialu_reg);
7852 %}
7853 
7854 //----------Multiplication/Division Instructions-------------------------------
7855 // Integer Multiplication Instructions
7856 // Multiply Register
7857 
7858 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7859 %{
7860   match(Set dst (MulI dst src));
7861   effect(KILL cr);
7862 
7863   ins_cost(300);
7864   format %{ "imull   $dst, $src\t# int" %}
7865   opcode(0x0F, 0xAF);
7866   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7867   ins_pipe(ialu_reg_reg_alu0);
7868 %}
7869 
7870 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7871 %{
7872   match(Set dst (MulI src imm));
7873   effect(KILL cr);
7874 
7875   ins_cost(300);
7876   format %{ "imull   $dst, $src, $imm\t# int" %}
7877   opcode(0x69); /* 69 /r id */
7878   ins_encode(REX_reg_reg(dst, src),
7879              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7880   ins_pipe(ialu_reg_reg_alu0);
7881 %}
7882 
7883 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7884 %{
7885   match(Set dst (MulI dst (LoadI src)));
7886   effect(KILL cr);
7887 
7888   ins_cost(350);
7889   format %{ "imull   $dst, $src\t# int" %}
7890   opcode(0x0F, 0xAF);
7891   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7892   ins_pipe(ialu_reg_mem_alu0);
7893 %}
7894 
7895 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7896 %{
7897   match(Set dst (MulI (LoadI src) imm));
7898   effect(KILL cr);
7899 
7900   ins_cost(300);
7901   format %{ "imull   $dst, $src, $imm\t# int" %}
7902   opcode(0x69); /* 69 /r id */
7903   ins_encode(REX_reg_mem(dst, src),
7904              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7905   ins_pipe(ialu_reg_mem_alu0);
7906 %}
7907 
7908 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7909 %{
7910   match(Set dst (MulL dst src));
7911   effect(KILL cr);
7912 
7913   ins_cost(300);
7914   format %{ "imulq   $dst, $src\t# long" %}
7915   opcode(0x0F, 0xAF);
7916   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7917   ins_pipe(ialu_reg_reg_alu0);
7918 %}
7919 
7920 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7921 %{
7922   match(Set dst (MulL src imm));
7923   effect(KILL cr);
7924 
7925   ins_cost(300);
7926   format %{ "imulq   $dst, $src, $imm\t# long" %}
7927   opcode(0x69); /* 69 /r id */
7928   ins_encode(REX_reg_reg_wide(dst, src),
7929              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7930   ins_pipe(ialu_reg_reg_alu0);
7931 %}
7932 
7933 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7934 %{
7935   match(Set dst (MulL dst (LoadL src)));
7936   effect(KILL cr);
7937 
7938   ins_cost(350);
7939   format %{ "imulq   $dst, $src\t# long" %}
7940   opcode(0x0F, 0xAF);
7941   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7942   ins_pipe(ialu_reg_mem_alu0);
7943 %}
7944 
7945 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7946 %{
7947   match(Set dst (MulL (LoadL src) imm));
7948   effect(KILL cr);
7949 
7950   ins_cost(300);
7951   format %{ "imulq   $dst, $src, $imm\t# long" %}
7952   opcode(0x69); /* 69 /r id */
7953   ins_encode(REX_reg_mem_wide(dst, src),
7954              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7955   ins_pipe(ialu_reg_mem_alu0);
7956 %}
7957 
7958 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7959 %{
7960   match(Set dst (MulHiL src rax));
7961   effect(USE_KILL rax, KILL cr);
7962 
7963   ins_cost(300);
7964   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7965   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7966   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7967   ins_pipe(ialu_reg_reg_alu0);
7968 %}
7969 
7970 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7971                    rFlagsReg cr)
7972 %{
7973   match(Set rax (DivI rax div));
7974   effect(KILL rdx, KILL cr);
7975 
7976   ins_cost(30*100+10*100); // XXX
7977   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7978             "jne,s   normal\n\t"
7979             "xorl    rdx, rdx\n\t"
7980             "cmpl    $div, -1\n\t"
7981             "je,s    done\n"
7982     "normal: cdql\n\t"
7983             "idivl   $div\n"
7984     "done:"        %}
7985   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7986   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7987   ins_pipe(ialu_reg_reg_alu0);
7988 %}
7989 
7990 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7991                    rFlagsReg cr)
7992 %{
7993   match(Set rax (DivL rax div));
7994   effect(KILL rdx, KILL cr);
7995 
7996   ins_cost(30*100+10*100); // XXX
7997   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7998             "cmpq    rax, rdx\n\t"
7999             "jne,s   normal\n\t"
8000             "xorl    rdx, rdx\n\t"
8001             "cmpq    $div, -1\n\t"
8002             "je,s    done\n"
8003     "normal: cdqq\n\t"
8004             "idivq   $div\n"
8005     "done:"        %}
8006   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8007   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8008   ins_pipe(ialu_reg_reg_alu0);
8009 %}
8010 
8011 // Integer DIVMOD with Register, both quotient and mod results
8012 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
8013                              rFlagsReg cr)
8014 %{
8015   match(DivModI rax div);
8016   effect(KILL cr);
8017 
8018   ins_cost(30*100+10*100); // XXX
8019   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
8020             "jne,s   normal\n\t"
8021             "xorl    rdx, rdx\n\t"
8022             "cmpl    $div, -1\n\t"
8023             "je,s    done\n"
8024     "normal: cdql\n\t"
8025             "idivl   $div\n"
8026     "done:"        %}
8027   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8028   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8029   ins_pipe(pipe_slow);
8030 %}
8031 
8032 // Long DIVMOD with Register, both quotient and mod results
8033 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
8034                              rFlagsReg cr)
8035 %{
8036   match(DivModL rax div);
8037   effect(KILL cr);
8038 
8039   ins_cost(30*100+10*100); // XXX
8040   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
8041             "cmpq    rax, rdx\n\t"
8042             "jne,s   normal\n\t"
8043             "xorl    rdx, rdx\n\t"
8044             "cmpq    $div, -1\n\t"
8045             "je,s    done\n"
8046     "normal: cdqq\n\t"
8047             "idivq   $div\n"
8048     "done:"        %}
8049   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8050   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8051   ins_pipe(pipe_slow);
8052 %}
8053 
8054 //----------- DivL-By-Constant-Expansions--------------------------------------
8055 // DivI cases are handled by the compiler
8056 
8057 // Magic constant, reciprocal of 10
8058 instruct loadConL_0x6666666666666667(rRegL dst)
8059 %{
8060   effect(DEF dst);
8061 
8062   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
8063   ins_encode(load_immL(dst, 0x6666666666666667));
8064   ins_pipe(ialu_reg);
8065 %}
8066 
8067 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
8068 %{
8069   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
8070 
8071   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
8072   opcode(0xF7, 0x5); /* Opcode F7 /5 */
8073   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
8074   ins_pipe(ialu_reg_reg_alu0);
8075 %}
8076 
8077 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
8078 %{
8079   effect(USE_DEF dst, KILL cr);
8080 
8081   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
8082   opcode(0xC1, 0x7); /* C1 /7 ib */
8083   ins_encode(reg_opc_imm_wide(dst, 0x3F));
8084   ins_pipe(ialu_reg);
8085 %}
8086 
8087 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
8088 %{
8089   effect(USE_DEF dst, KILL cr);
8090 
8091   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
8092   opcode(0xC1, 0x7); /* C1 /7 ib */
8093   ins_encode(reg_opc_imm_wide(dst, 0x2));
8094   ins_pipe(ialu_reg);
8095 %}
8096 
8097 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
8098 %{
8099   match(Set dst (DivL src div));
8100 
8101   ins_cost((5+8)*100);
8102   expand %{
8103     rax_RegL rax;                     // Killed temp
8104     rFlagsReg cr;                     // Killed
8105     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
8106     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
8107     sarL_rReg_63(src, cr);            // sarq  src, 63
8108     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
8109     subL_rReg(dst, src, cr);          // subl  rdx, src
8110   %}
8111 %}
8112 
8113 //-----------------------------------------------------------------------------
8114 
8115 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
8116                    rFlagsReg cr)
8117 %{
8118   match(Set rdx (ModI rax div));
8119   effect(KILL rax, KILL cr);
8120 
8121   ins_cost(300); // XXX
8122   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
8123             "jne,s   normal\n\t"
8124             "xorl    rdx, rdx\n\t"
8125             "cmpl    $div, -1\n\t"
8126             "je,s    done\n"
8127     "normal: cdql\n\t"
8128             "idivl   $div\n"
8129     "done:"        %}
8130   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8131   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
8132   ins_pipe(ialu_reg_reg_alu0);
8133 %}
8134 
8135 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
8136                    rFlagsReg cr)
8137 %{
8138   match(Set rdx (ModL rax div));
8139   effect(KILL rax, KILL cr);
8140 
8141   ins_cost(300); // XXX
8142   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
8143             "cmpq    rax, rdx\n\t"
8144             "jne,s   normal\n\t"
8145             "xorl    rdx, rdx\n\t"
8146             "cmpq    $div, -1\n\t"
8147             "je,s    done\n"
8148     "normal: cdqq\n\t"
8149             "idivq   $div\n"
8150     "done:"        %}
8151   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8152   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8153   ins_pipe(ialu_reg_reg_alu0);
8154 %}
8155 
8156 // Integer Shift Instructions
8157 // Shift Left by one
8158 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8159 %{
8160   match(Set dst (LShiftI dst shift));
8161   effect(KILL cr);
8162 
8163   format %{ "sall    $dst, $shift" %}
8164   opcode(0xD1, 0x4); /* D1 /4 */
8165   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8166   ins_pipe(ialu_reg);
8167 %}
8168 
8169 // Shift Left by one
8170 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8171 %{
8172   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8173   effect(KILL cr);
8174 
8175   format %{ "sall    $dst, $shift\t" %}
8176   opcode(0xD1, 0x4); /* D1 /4 */
8177   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8178   ins_pipe(ialu_mem_imm);
8179 %}
8180 
8181 // Shift Left by 8-bit immediate
8182 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8183 %{
8184   match(Set dst (LShiftI dst shift));
8185   effect(KILL cr);
8186 
8187   format %{ "sall    $dst, $shift" %}
8188   opcode(0xC1, 0x4); /* C1 /4 ib */
8189   ins_encode(reg_opc_imm(dst, shift));
8190   ins_pipe(ialu_reg);
8191 %}
8192 
8193 // Shift Left by 8-bit immediate
8194 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8195 %{
8196   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8197   effect(KILL cr);
8198 
8199   format %{ "sall    $dst, $shift" %}
8200   opcode(0xC1, 0x4); /* C1 /4 ib */
8201   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8202   ins_pipe(ialu_mem_imm);
8203 %}
8204 
8205 // Shift Left by variable
8206 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8207 %{
8208   match(Set dst (LShiftI dst shift));
8209   effect(KILL cr);
8210 
8211   format %{ "sall    $dst, $shift" %}
8212   opcode(0xD3, 0x4); /* D3 /4 */
8213   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8214   ins_pipe(ialu_reg_reg);
8215 %}
8216 
8217 // Shift Left by variable
8218 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8219 %{
8220   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8221   effect(KILL cr);
8222 
8223   format %{ "sall    $dst, $shift" %}
8224   opcode(0xD3, 0x4); /* D3 /4 */
8225   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8226   ins_pipe(ialu_mem_reg);
8227 %}
8228 
8229 // Arithmetic shift right by one
8230 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8231 %{
8232   match(Set dst (RShiftI dst shift));
8233   effect(KILL cr);
8234 
8235   format %{ "sarl    $dst, $shift" %}
8236   opcode(0xD1, 0x7); /* D1 /7 */
8237   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8238   ins_pipe(ialu_reg);
8239 %}
8240 
8241 // Arithmetic shift right by one
8242 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8243 %{
8244   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8245   effect(KILL cr);
8246 
8247   format %{ "sarl    $dst, $shift" %}
8248   opcode(0xD1, 0x7); /* D1 /7 */
8249   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8250   ins_pipe(ialu_mem_imm);
8251 %}
8252 
8253 // Arithmetic Shift Right by 8-bit immediate
8254 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8255 %{
8256   match(Set dst (RShiftI dst shift));
8257   effect(KILL cr);
8258 
8259   format %{ "sarl    $dst, $shift" %}
8260   opcode(0xC1, 0x7); /* C1 /7 ib */
8261   ins_encode(reg_opc_imm(dst, shift));
8262   ins_pipe(ialu_mem_imm);
8263 %}
8264 
8265 // Arithmetic Shift Right by 8-bit immediate
8266 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8267 %{
8268   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8269   effect(KILL cr);
8270 
8271   format %{ "sarl    $dst, $shift" %}
8272   opcode(0xC1, 0x7); /* C1 /7 ib */
8273   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8274   ins_pipe(ialu_mem_imm);
8275 %}
8276 
8277 // Arithmetic Shift Right by variable
8278 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8279 %{
8280   match(Set dst (RShiftI dst shift));
8281   effect(KILL cr);
8282 
8283   format %{ "sarl    $dst, $shift" %}
8284   opcode(0xD3, 0x7); /* D3 /7 */
8285   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8286   ins_pipe(ialu_reg_reg);
8287 %}
8288 
8289 // Arithmetic Shift Right by variable
8290 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8291 %{
8292   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8293   effect(KILL cr);
8294 
8295   format %{ "sarl    $dst, $shift" %}
8296   opcode(0xD3, 0x7); /* D3 /7 */
8297   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8298   ins_pipe(ialu_mem_reg);
8299 %}
8300 
8301 // Logical shift right by one
8302 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8303 %{
8304   match(Set dst (URShiftI dst shift));
8305   effect(KILL cr);
8306 
8307   format %{ "shrl    $dst, $shift" %}
8308   opcode(0xD1, 0x5); /* D1 /5 */
8309   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8310   ins_pipe(ialu_reg);
8311 %}
8312 
8313 // Logical shift right by one
8314 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8315 %{
8316   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8317   effect(KILL cr);
8318 
8319   format %{ "shrl    $dst, $shift" %}
8320   opcode(0xD1, 0x5); /* D1 /5 */
8321   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8322   ins_pipe(ialu_mem_imm);
8323 %}
8324 
8325 // Logical Shift Right by 8-bit immediate
8326 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8327 %{
8328   match(Set dst (URShiftI dst shift));
8329   effect(KILL cr);
8330 
8331   format %{ "shrl    $dst, $shift" %}
8332   opcode(0xC1, 0x5); /* C1 /5 ib */
8333   ins_encode(reg_opc_imm(dst, shift));
8334   ins_pipe(ialu_reg);
8335 %}
8336 
8337 // Logical Shift Right by 8-bit immediate
8338 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8339 %{
8340   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8341   effect(KILL cr);
8342 
8343   format %{ "shrl    $dst, $shift" %}
8344   opcode(0xC1, 0x5); /* C1 /5 ib */
8345   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8346   ins_pipe(ialu_mem_imm);
8347 %}
8348 
8349 // Logical Shift Right by variable
8350 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8351 %{
8352   match(Set dst (URShiftI dst shift));
8353   effect(KILL cr);
8354 
8355   format %{ "shrl    $dst, $shift" %}
8356   opcode(0xD3, 0x5); /* D3 /5 */
8357   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8358   ins_pipe(ialu_reg_reg);
8359 %}
8360 
8361 // Logical Shift Right by variable
8362 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8363 %{
8364   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8365   effect(KILL cr);
8366 
8367   format %{ "shrl    $dst, $shift" %}
8368   opcode(0xD3, 0x5); /* D3 /5 */
8369   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8370   ins_pipe(ialu_mem_reg);
8371 %}
8372 
8373 // Long Shift Instructions
8374 // Shift Left by one
8375 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8376 %{
8377   match(Set dst (LShiftL dst shift));
8378   effect(KILL cr);
8379 
8380   format %{ "salq    $dst, $shift" %}
8381   opcode(0xD1, 0x4); /* D1 /4 */
8382   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8383   ins_pipe(ialu_reg);
8384 %}
8385 
8386 // Shift Left by one
8387 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8388 %{
8389   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8390   effect(KILL cr);
8391 
8392   format %{ "salq    $dst, $shift" %}
8393   opcode(0xD1, 0x4); /* D1 /4 */
8394   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8395   ins_pipe(ialu_mem_imm);
8396 %}
8397 
8398 // Shift Left by 8-bit immediate
8399 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8400 %{
8401   match(Set dst (LShiftL dst shift));
8402   effect(KILL cr);
8403 
8404   format %{ "salq    $dst, $shift" %}
8405   opcode(0xC1, 0x4); /* C1 /4 ib */
8406   ins_encode(reg_opc_imm_wide(dst, shift));
8407   ins_pipe(ialu_reg);
8408 %}
8409 
8410 // Shift Left by 8-bit immediate
8411 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8412 %{
8413   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8414   effect(KILL cr);
8415 
8416   format %{ "salq    $dst, $shift" %}
8417   opcode(0xC1, 0x4); /* C1 /4 ib */
8418   ins_encode(REX_mem_wide(dst), OpcP,
8419              RM_opc_mem(secondary, dst), Con8or32(shift));
8420   ins_pipe(ialu_mem_imm);
8421 %}
8422 
8423 // Shift Left by variable
8424 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8425 %{
8426   match(Set dst (LShiftL dst shift));
8427   effect(KILL cr);
8428 
8429   format %{ "salq    $dst, $shift" %}
8430   opcode(0xD3, 0x4); /* D3 /4 */
8431   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8432   ins_pipe(ialu_reg_reg);
8433 %}
8434 
8435 // Shift Left by variable
8436 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8437 %{
8438   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8439   effect(KILL cr);
8440 
8441   format %{ "salq    $dst, $shift" %}
8442   opcode(0xD3, 0x4); /* D3 /4 */
8443   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8444   ins_pipe(ialu_mem_reg);
8445 %}
8446 
8447 // Arithmetic shift right by one
8448 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8449 %{
8450   match(Set dst (RShiftL dst shift));
8451   effect(KILL cr);
8452 
8453   format %{ "sarq    $dst, $shift" %}
8454   opcode(0xD1, 0x7); /* D1 /7 */
8455   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8456   ins_pipe(ialu_reg);
8457 %}
8458 
8459 // Arithmetic shift right by one
8460 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8461 %{
8462   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8463   effect(KILL cr);
8464 
8465   format %{ "sarq    $dst, $shift" %}
8466   opcode(0xD1, 0x7); /* D1 /7 */
8467   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8468   ins_pipe(ialu_mem_imm);
8469 %}
8470 
8471 // Arithmetic Shift Right by 8-bit immediate
8472 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8473 %{
8474   match(Set dst (RShiftL dst shift));
8475   effect(KILL cr);
8476 
8477   format %{ "sarq    $dst, $shift" %}
8478   opcode(0xC1, 0x7); /* C1 /7 ib */
8479   ins_encode(reg_opc_imm_wide(dst, shift));
8480   ins_pipe(ialu_mem_imm);
8481 %}
8482 
8483 // Arithmetic Shift Right by 8-bit immediate
8484 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8485 %{
8486   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8487   effect(KILL cr);
8488 
8489   format %{ "sarq    $dst, $shift" %}
8490   opcode(0xC1, 0x7); /* C1 /7 ib */
8491   ins_encode(REX_mem_wide(dst), OpcP,
8492              RM_opc_mem(secondary, dst), Con8or32(shift));
8493   ins_pipe(ialu_mem_imm);
8494 %}
8495 
8496 // Arithmetic Shift Right by variable
8497 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8498 %{
8499   match(Set dst (RShiftL dst shift));
8500   effect(KILL cr);
8501 
8502   format %{ "sarq    $dst, $shift" %}
8503   opcode(0xD3, 0x7); /* D3 /7 */
8504   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8505   ins_pipe(ialu_reg_reg);
8506 %}
8507 
8508 // Arithmetic Shift Right by variable
8509 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8510 %{
8511   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8512   effect(KILL cr);
8513 
8514   format %{ "sarq    $dst, $shift" %}
8515   opcode(0xD3, 0x7); /* D3 /7 */
8516   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8517   ins_pipe(ialu_mem_reg);
8518 %}
8519 
8520 // Logical shift right by one
8521 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8522 %{
8523   match(Set dst (URShiftL dst shift));
8524   effect(KILL cr);
8525 
8526   format %{ "shrq    $dst, $shift" %}
8527   opcode(0xD1, 0x5); /* D1 /5 */
8528   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8529   ins_pipe(ialu_reg);
8530 %}
8531 
8532 // Logical shift right by one
8533 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8534 %{
8535   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8536   effect(KILL cr);
8537 
8538   format %{ "shrq    $dst, $shift" %}
8539   opcode(0xD1, 0x5); /* D1 /5 */
8540   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8541   ins_pipe(ialu_mem_imm);
8542 %}
8543 
8544 // Logical Shift Right by 8-bit immediate
8545 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8546 %{
8547   match(Set dst (URShiftL dst shift));
8548   effect(KILL cr);
8549 
8550   format %{ "shrq    $dst, $shift" %}
8551   opcode(0xC1, 0x5); /* C1 /5 ib */
8552   ins_encode(reg_opc_imm_wide(dst, shift));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 
8557 // Logical Shift Right by 8-bit immediate
8558 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8559 %{
8560   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8561   effect(KILL cr);
8562 
8563   format %{ "shrq    $dst, $shift" %}
8564   opcode(0xC1, 0x5); /* C1 /5 ib */
8565   ins_encode(REX_mem_wide(dst), OpcP,
8566              RM_opc_mem(secondary, dst), Con8or32(shift));
8567   ins_pipe(ialu_mem_imm);
8568 %}
8569 
8570 // Logical Shift Right by variable
8571 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8572 %{
8573   match(Set dst (URShiftL dst shift));
8574   effect(KILL cr);
8575 
8576   format %{ "shrq    $dst, $shift" %}
8577   opcode(0xD3, 0x5); /* D3 /5 */
8578   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8579   ins_pipe(ialu_reg_reg);
8580 %}
8581 
8582 // Logical Shift Right by variable
8583 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8584 %{
8585   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8586   effect(KILL cr);
8587 
8588   format %{ "shrq    $dst, $shift" %}
8589   opcode(0xD3, 0x5); /* D3 /5 */
8590   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8591   ins_pipe(ialu_mem_reg);
8592 %}
8593 
8594 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8595 // This idiom is used by the compiler for the i2b bytecode.
8596 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8597 %{
8598   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8599 
8600   format %{ "movsbl  $dst, $src\t# i2b" %}
8601   opcode(0x0F, 0xBE);
8602   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8603   ins_pipe(ialu_reg_reg);
8604 %}
8605 
8606 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8607 // This idiom is used by the compiler the i2s bytecode.
8608 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8609 %{
8610   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8611 
8612   format %{ "movswl  $dst, $src\t# i2s" %}
8613   opcode(0x0F, 0xBF);
8614   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8615   ins_pipe(ialu_reg_reg);
8616 %}
8617 
8618 // ROL/ROR instructions
8619 
8620 // ROL expand
8621 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8622   effect(KILL cr, USE_DEF dst);
8623 
8624   format %{ "roll    $dst" %}
8625   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8626   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8627   ins_pipe(ialu_reg);
8628 %}
8629 
8630 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8631   effect(USE_DEF dst, USE shift, KILL cr);
8632 
8633   format %{ "roll    $dst, $shift" %}
8634   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8635   ins_encode( reg_opc_imm(dst, shift) );
8636   ins_pipe(ialu_reg);
8637 %}
8638 
8639 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8640 %{
8641   effect(USE_DEF dst, USE shift, KILL cr);
8642 
8643   format %{ "roll    $dst, $shift" %}
8644   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8645   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8646   ins_pipe(ialu_reg_reg);
8647 %}
8648 // end of ROL expand
8649 
8650 // Rotate Left by one
8651 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8652 %{
8653   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8654 
8655   expand %{
8656     rolI_rReg_imm1(dst, cr);
8657   %}
8658 %}
8659 
8660 // Rotate Left by 8-bit immediate
8661 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8662 %{
8663   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8664   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8665 
8666   expand %{
8667     rolI_rReg_imm8(dst, lshift, cr);
8668   %}
8669 %}
8670 
8671 // Rotate Left by variable
8672 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8673 %{
8674   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8675 
8676   expand %{
8677     rolI_rReg_CL(dst, shift, cr);
8678   %}
8679 %}
8680 
8681 // Rotate Left by variable
8682 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8683 %{
8684   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8685 
8686   expand %{
8687     rolI_rReg_CL(dst, shift, cr);
8688   %}
8689 %}
8690 
8691 // ROR expand
8692 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8693 %{
8694   effect(USE_DEF dst, KILL cr);
8695 
8696   format %{ "rorl    $dst" %}
8697   opcode(0xD1, 0x1); /* D1 /1 */
8698   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8699   ins_pipe(ialu_reg);
8700 %}
8701 
8702 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8703 %{
8704   effect(USE_DEF dst, USE shift, KILL cr);
8705 
8706   format %{ "rorl    $dst, $shift" %}
8707   opcode(0xC1, 0x1); /* C1 /1 ib */
8708   ins_encode(reg_opc_imm(dst, shift));
8709   ins_pipe(ialu_reg);
8710 %}
8711 
8712 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8713 %{
8714   effect(USE_DEF dst, USE shift, KILL cr);
8715 
8716   format %{ "rorl    $dst, $shift" %}
8717   opcode(0xD3, 0x1); /* D3 /1 */
8718   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8719   ins_pipe(ialu_reg_reg);
8720 %}
8721 // end of ROR expand
8722 
8723 // Rotate Right by one
8724 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8725 %{
8726   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8727 
8728   expand %{
8729     rorI_rReg_imm1(dst, cr);
8730   %}
8731 %}
8732 
8733 // Rotate Right by 8-bit immediate
8734 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8735 %{
8736   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8737   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8738 
8739   expand %{
8740     rorI_rReg_imm8(dst, rshift, cr);
8741   %}
8742 %}
8743 
8744 // Rotate Right by variable
8745 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8746 %{
8747   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8748 
8749   expand %{
8750     rorI_rReg_CL(dst, shift, cr);
8751   %}
8752 %}
8753 
8754 // Rotate Right by variable
8755 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8756 %{
8757   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8758 
8759   expand %{
8760     rorI_rReg_CL(dst, shift, cr);
8761   %}
8762 %}
8763 
8764 // for long rotate
8765 // ROL expand
8766 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8767   effect(USE_DEF dst, KILL cr);
8768 
8769   format %{ "rolq    $dst" %}
8770   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8771   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8772   ins_pipe(ialu_reg);
8773 %}
8774 
8775 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8776   effect(USE_DEF dst, USE shift, KILL cr);
8777 
8778   format %{ "rolq    $dst, $shift" %}
8779   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8780   ins_encode( reg_opc_imm_wide(dst, shift) );
8781   ins_pipe(ialu_reg);
8782 %}
8783 
8784 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8785 %{
8786   effect(USE_DEF dst, USE shift, KILL cr);
8787 
8788   format %{ "rolq    $dst, $shift" %}
8789   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8790   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8791   ins_pipe(ialu_reg_reg);
8792 %}
8793 // end of ROL expand
8794 
8795 // Rotate Left by one
8796 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8797 %{
8798   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8799 
8800   expand %{
8801     rolL_rReg_imm1(dst, cr);
8802   %}
8803 %}
8804 
8805 // Rotate Left by 8-bit immediate
8806 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8807 %{
8808   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8809   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8810 
8811   expand %{
8812     rolL_rReg_imm8(dst, lshift, cr);
8813   %}
8814 %}
8815 
8816 // Rotate Left by variable
8817 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8818 %{
8819   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8820 
8821   expand %{
8822     rolL_rReg_CL(dst, shift, cr);
8823   %}
8824 %}
8825 
8826 // Rotate Left by variable
8827 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8828 %{
8829   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8830 
8831   expand %{
8832     rolL_rReg_CL(dst, shift, cr);
8833   %}
8834 %}
8835 
8836 // ROR expand
8837 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8838 %{
8839   effect(USE_DEF dst, KILL cr);
8840 
8841   format %{ "rorq    $dst" %}
8842   opcode(0xD1, 0x1); /* D1 /1 */
8843   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8844   ins_pipe(ialu_reg);
8845 %}
8846 
8847 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8848 %{
8849   effect(USE_DEF dst, USE shift, KILL cr);
8850 
8851   format %{ "rorq    $dst, $shift" %}
8852   opcode(0xC1, 0x1); /* C1 /1 ib */
8853   ins_encode(reg_opc_imm_wide(dst, shift));
8854   ins_pipe(ialu_reg);
8855 %}
8856 
8857 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8858 %{
8859   effect(USE_DEF dst, USE shift, KILL cr);
8860 
8861   format %{ "rorq    $dst, $shift" %}
8862   opcode(0xD3, 0x1); /* D3 /1 */
8863   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8864   ins_pipe(ialu_reg_reg);
8865 %}
8866 // end of ROR expand
8867 
8868 // Rotate Right by one
8869 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8870 %{
8871   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8872 
8873   expand %{
8874     rorL_rReg_imm1(dst, cr);
8875   %}
8876 %}
8877 
8878 // Rotate Right by 8-bit immediate
8879 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8880 %{
8881   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8882   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8883 
8884   expand %{
8885     rorL_rReg_imm8(dst, rshift, cr);
8886   %}
8887 %}
8888 
8889 // Rotate Right by variable
8890 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8891 %{
8892   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8893 
8894   expand %{
8895     rorL_rReg_CL(dst, shift, cr);
8896   %}
8897 %}
8898 
8899 // Rotate Right by variable
8900 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8901 %{
8902   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8903 
8904   expand %{
8905     rorL_rReg_CL(dst, shift, cr);
8906   %}
8907 %}
8908 
8909 // Logical Instructions
8910 
8911 // Integer Logical Instructions
8912 
8913 // And Instructions
8914 // And Register with Register
8915 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8916 %{
8917   match(Set dst (AndI dst src));
8918   effect(KILL cr);
8919 
8920   format %{ "andl    $dst, $src\t# int" %}
8921   opcode(0x23);
8922   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8923   ins_pipe(ialu_reg_reg);
8924 %}
8925 
8926 // And Register with Immediate 255
8927 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8928 %{
8929   match(Set dst (AndI dst src));
8930 
8931   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8932   opcode(0x0F, 0xB6);
8933   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8934   ins_pipe(ialu_reg);
8935 %}
8936 
8937 // And Register with Immediate 255 and promote to long
8938 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8939 %{
8940   match(Set dst (ConvI2L (AndI src mask)));
8941 
8942   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8943   opcode(0x0F, 0xB6);
8944   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8945   ins_pipe(ialu_reg);
8946 %}
8947 
8948 // And Register with Immediate 65535
8949 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8950 %{
8951   match(Set dst (AndI dst src));
8952 
8953   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8954   opcode(0x0F, 0xB7);
8955   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8956   ins_pipe(ialu_reg);
8957 %}
8958 
8959 // And Register with Immediate 65535 and promote to long
8960 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8961 %{
8962   match(Set dst (ConvI2L (AndI src mask)));
8963 
8964   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8965   opcode(0x0F, 0xB7);
8966   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8967   ins_pipe(ialu_reg);
8968 %}
8969 
8970 // And Register with Immediate
8971 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8972 %{
8973   match(Set dst (AndI dst src));
8974   effect(KILL cr);
8975 
8976   format %{ "andl    $dst, $src\t# int" %}
8977   opcode(0x81, 0x04); /* Opcode 81 /4 */
8978   ins_encode(OpcSErm(dst, src), Con8or32(src));
8979   ins_pipe(ialu_reg);
8980 %}
8981 
8982 // And Register with Memory
8983 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8984 %{
8985   match(Set dst (AndI dst (LoadI src)));
8986   effect(KILL cr);
8987 
8988   ins_cost(125);
8989   format %{ "andl    $dst, $src\t# int" %}
8990   opcode(0x23);
8991   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8992   ins_pipe(ialu_reg_mem);
8993 %}
8994 
8995 // And Memory with Register
8996 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8997 %{
8998   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8999   effect(KILL cr);
9000 
9001   ins_cost(150);
9002   format %{ "andl    $dst, $src\t# int" %}
9003   opcode(0x21); /* Opcode 21 /r */
9004   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9005   ins_pipe(ialu_mem_reg);
9006 %}
9007 
9008 // And Memory with Immediate
9009 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
9010 %{
9011   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
9012   effect(KILL cr);
9013 
9014   ins_cost(125);
9015   format %{ "andl    $dst, $src\t# int" %}
9016   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9017   ins_encode(REX_mem(dst), OpcSE(src),
9018              RM_opc_mem(secondary, dst), Con8or32(src));
9019   ins_pipe(ialu_mem_imm);
9020 %}
9021 
9022 // BMI1 instructions
9023 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
9024   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
9025   predicate(UseBMI1Instructions);
9026   effect(KILL cr);
9027 
9028   ins_cost(125);
9029   format %{ "andnl  $dst, $src1, $src2" %}
9030 
9031   ins_encode %{
9032     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
9033   %}
9034   ins_pipe(ialu_reg_mem);
9035 %}
9036 
9037 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
9038   match(Set dst (AndI (XorI src1 minus_1) src2));
9039   predicate(UseBMI1Instructions);
9040   effect(KILL cr);
9041 
9042   format %{ "andnl  $dst, $src1, $src2" %}
9043 
9044   ins_encode %{
9045     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
9046   %}
9047   ins_pipe(ialu_reg);
9048 %}
9049 
9050 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
9051   match(Set dst (AndI (SubI imm_zero src) src));
9052   predicate(UseBMI1Instructions);
9053   effect(KILL cr);
9054 
9055   format %{ "blsil  $dst, $src" %}
9056 
9057   ins_encode %{
9058     __ blsil($dst$$Register, $src$$Register);
9059   %}
9060   ins_pipe(ialu_reg);
9061 %}
9062 
9063 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
9064   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
9065   predicate(UseBMI1Instructions);
9066   effect(KILL cr);
9067 
9068   ins_cost(125);
9069   format %{ "blsil  $dst, $src" %}
9070 
9071   ins_encode %{
9072     __ blsil($dst$$Register, $src$$Address);
9073   %}
9074   ins_pipe(ialu_reg_mem);
9075 %}
9076 
9077 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9078 %{
9079   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
9080   predicate(UseBMI1Instructions);
9081   effect(KILL cr);
9082 
9083   ins_cost(125);
9084   format %{ "blsmskl $dst, $src" %}
9085 
9086   ins_encode %{
9087     __ blsmskl($dst$$Register, $src$$Address);
9088   %}
9089   ins_pipe(ialu_reg_mem);
9090 %}
9091 
9092 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9093 %{
9094   match(Set dst (XorI (AddI src minus_1) src));
9095   predicate(UseBMI1Instructions);
9096   effect(KILL cr);
9097 
9098   format %{ "blsmskl $dst, $src" %}
9099 
9100   ins_encode %{
9101     __ blsmskl($dst$$Register, $src$$Register);
9102   %}
9103 
9104   ins_pipe(ialu_reg);
9105 %}
9106 
9107 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
9108 %{
9109   match(Set dst (AndI (AddI src minus_1) src) );
9110   predicate(UseBMI1Instructions);
9111   effect(KILL cr);
9112 
9113   format %{ "blsrl  $dst, $src" %}
9114 
9115   ins_encode %{
9116     __ blsrl($dst$$Register, $src$$Register);
9117   %}
9118 
9119   ins_pipe(ialu_reg_mem);
9120 %}
9121 
9122 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
9123 %{
9124   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
9125   predicate(UseBMI1Instructions);
9126   effect(KILL cr);
9127 
9128   ins_cost(125);
9129   format %{ "blsrl  $dst, $src" %}
9130 
9131   ins_encode %{
9132     __ blsrl($dst$$Register, $src$$Address);
9133   %}
9134 
9135   ins_pipe(ialu_reg);
9136 %}
9137 
9138 // Or Instructions
9139 // Or Register with Register
9140 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9141 %{
9142   match(Set dst (OrI dst src));
9143   effect(KILL cr);
9144 
9145   format %{ "orl     $dst, $src\t# int" %}
9146   opcode(0x0B);
9147   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9148   ins_pipe(ialu_reg_reg);
9149 %}
9150 
9151 // Or Register with Immediate
9152 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9153 %{
9154   match(Set dst (OrI dst src));
9155   effect(KILL cr);
9156 
9157   format %{ "orl     $dst, $src\t# int" %}
9158   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9159   ins_encode(OpcSErm(dst, src), Con8or32(src));
9160   ins_pipe(ialu_reg);
9161 %}
9162 
9163 // Or Register with Memory
9164 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9165 %{
9166   match(Set dst (OrI dst (LoadI src)));
9167   effect(KILL cr);
9168 
9169   ins_cost(125);
9170   format %{ "orl     $dst, $src\t# int" %}
9171   opcode(0x0B);
9172   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9173   ins_pipe(ialu_reg_mem);
9174 %}
9175 
9176 // Or Memory with Register
9177 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9178 %{
9179   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9180   effect(KILL cr);
9181 
9182   ins_cost(150);
9183   format %{ "orl     $dst, $src\t# int" %}
9184   opcode(0x09); /* Opcode 09 /r */
9185   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9186   ins_pipe(ialu_mem_reg);
9187 %}
9188 
9189 // Or Memory with Immediate
9190 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9191 %{
9192   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9193   effect(KILL cr);
9194 
9195   ins_cost(125);
9196   format %{ "orl     $dst, $src\t# int" %}
9197   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9198   ins_encode(REX_mem(dst), OpcSE(src),
9199              RM_opc_mem(secondary, dst), Con8or32(src));
9200   ins_pipe(ialu_mem_imm);
9201 %}
9202 
9203 // Xor Instructions
9204 // Xor Register with Register
9205 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9206 %{
9207   match(Set dst (XorI dst src));
9208   effect(KILL cr);
9209 
9210   format %{ "xorl    $dst, $src\t# int" %}
9211   opcode(0x33);
9212   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9213   ins_pipe(ialu_reg_reg);
9214 %}
9215 
9216 // Xor Register with Immediate -1
9217 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9218   match(Set dst (XorI dst imm));
9219 
9220   format %{ "not    $dst" %}
9221   ins_encode %{
9222      __ notl($dst$$Register);
9223   %}
9224   ins_pipe(ialu_reg);
9225 %}
9226 
9227 // Xor Register with Immediate
9228 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9229 %{
9230   match(Set dst (XorI dst src));
9231   effect(KILL cr);
9232 
9233   format %{ "xorl    $dst, $src\t# int" %}
9234   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9235   ins_encode(OpcSErm(dst, src), Con8or32(src));
9236   ins_pipe(ialu_reg);
9237 %}
9238 
9239 // Xor Register with Memory
9240 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9241 %{
9242   match(Set dst (XorI dst (LoadI src)));
9243   effect(KILL cr);
9244 
9245   ins_cost(125);
9246   format %{ "xorl    $dst, $src\t# int" %}
9247   opcode(0x33);
9248   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9249   ins_pipe(ialu_reg_mem);
9250 %}
9251 
9252 // Xor Memory with Register
9253 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9254 %{
9255   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9256   effect(KILL cr);
9257 
9258   ins_cost(150);
9259   format %{ "xorl    $dst, $src\t# int" %}
9260   opcode(0x31); /* Opcode 31 /r */
9261   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9262   ins_pipe(ialu_mem_reg);
9263 %}
9264 
9265 // Xor Memory with Immediate
9266 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9267 %{
9268   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9269   effect(KILL cr);
9270 
9271   ins_cost(125);
9272   format %{ "xorl    $dst, $src\t# int" %}
9273   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9274   ins_encode(REX_mem(dst), OpcSE(src),
9275              RM_opc_mem(secondary, dst), Con8or32(src));
9276   ins_pipe(ialu_mem_imm);
9277 %}
9278 
9279 
9280 // Long Logical Instructions
9281 
9282 // And Instructions
9283 // And Register with Register
9284 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9285 %{
9286   match(Set dst (AndL dst src));
9287   effect(KILL cr);
9288 
9289   format %{ "andq    $dst, $src\t# long" %}
9290   opcode(0x23);
9291   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9292   ins_pipe(ialu_reg_reg);
9293 %}
9294 
9295 // And Register with Immediate 255
9296 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9297 %{
9298   match(Set dst (AndL dst src));
9299 
9300   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9301   opcode(0x0F, 0xB6);
9302   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9303   ins_pipe(ialu_reg);
9304 %}
9305 
9306 // And Register with Immediate 65535
9307 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9308 %{
9309   match(Set dst (AndL dst src));
9310 
9311   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9312   opcode(0x0F, 0xB7);
9313   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9314   ins_pipe(ialu_reg);
9315 %}
9316 
9317 // And Register with Immediate
9318 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9319 %{
9320   match(Set dst (AndL dst src));
9321   effect(KILL cr);
9322 
9323   format %{ "andq    $dst, $src\t# long" %}
9324   opcode(0x81, 0x04); /* Opcode 81 /4 */
9325   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9326   ins_pipe(ialu_reg);
9327 %}
9328 
9329 // And Register with Memory
9330 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9331 %{
9332   match(Set dst (AndL dst (LoadL src)));
9333   effect(KILL cr);
9334 
9335   ins_cost(125);
9336   format %{ "andq    $dst, $src\t# long" %}
9337   opcode(0x23);
9338   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9339   ins_pipe(ialu_reg_mem);
9340 %}
9341 
9342 // And Memory with Register
9343 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9344 %{
9345   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9346   effect(KILL cr);
9347 
9348   ins_cost(150);
9349   format %{ "andq    $dst, $src\t# long" %}
9350   opcode(0x21); /* Opcode 21 /r */
9351   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9352   ins_pipe(ialu_mem_reg);
9353 %}
9354 
9355 // And Memory with Immediate
9356 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9357 %{
9358   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9359   effect(KILL cr);
9360 
9361   ins_cost(125);
9362   format %{ "andq    $dst, $src\t# long" %}
9363   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9364   ins_encode(REX_mem_wide(dst), OpcSE(src),
9365              RM_opc_mem(secondary, dst), Con8or32(src));
9366   ins_pipe(ialu_mem_imm);
9367 %}
9368 
9369 // BMI1 instructions
9370 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9371   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9372   predicate(UseBMI1Instructions);
9373   effect(KILL cr);
9374 
9375   ins_cost(125);
9376   format %{ "andnq  $dst, $src1, $src2" %}
9377 
9378   ins_encode %{
9379     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9380   %}
9381   ins_pipe(ialu_reg_mem);
9382 %}
9383 
9384 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9385   match(Set dst (AndL (XorL src1 minus_1) src2));
9386   predicate(UseBMI1Instructions);
9387   effect(KILL cr);
9388 
9389   format %{ "andnq  $dst, $src1, $src2" %}
9390 
9391   ins_encode %{
9392   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9393   %}
9394   ins_pipe(ialu_reg_mem);
9395 %}
9396 
9397 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9398   match(Set dst (AndL (SubL imm_zero src) src));
9399   predicate(UseBMI1Instructions);
9400   effect(KILL cr);
9401 
9402   format %{ "blsiq  $dst, $src" %}
9403 
9404   ins_encode %{
9405     __ blsiq($dst$$Register, $src$$Register);
9406   %}
9407   ins_pipe(ialu_reg);
9408 %}
9409 
9410 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9411   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9412   predicate(UseBMI1Instructions);
9413   effect(KILL cr);
9414 
9415   ins_cost(125);
9416   format %{ "blsiq  $dst, $src" %}
9417 
9418   ins_encode %{
9419     __ blsiq($dst$$Register, $src$$Address);
9420   %}
9421   ins_pipe(ialu_reg_mem);
9422 %}
9423 
9424 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9425 %{
9426   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9427   predicate(UseBMI1Instructions);
9428   effect(KILL cr);
9429 
9430   ins_cost(125);
9431   format %{ "blsmskq $dst, $src" %}
9432 
9433   ins_encode %{
9434     __ blsmskq($dst$$Register, $src$$Address);
9435   %}
9436   ins_pipe(ialu_reg_mem);
9437 %}
9438 
9439 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9440 %{
9441   match(Set dst (XorL (AddL src minus_1) src));
9442   predicate(UseBMI1Instructions);
9443   effect(KILL cr);
9444 
9445   format %{ "blsmskq $dst, $src" %}
9446 
9447   ins_encode %{
9448     __ blsmskq($dst$$Register, $src$$Register);
9449   %}
9450 
9451   ins_pipe(ialu_reg);
9452 %}
9453 
9454 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9455 %{
9456   match(Set dst (AndL (AddL src minus_1) src) );
9457   predicate(UseBMI1Instructions);
9458   effect(KILL cr);
9459 
9460   format %{ "blsrq  $dst, $src" %}
9461 
9462   ins_encode %{
9463     __ blsrq($dst$$Register, $src$$Register);
9464   %}
9465 
9466   ins_pipe(ialu_reg);
9467 %}
9468 
9469 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9470 %{
9471   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9472   predicate(UseBMI1Instructions);
9473   effect(KILL cr);
9474 
9475   ins_cost(125);
9476   format %{ "blsrq  $dst, $src" %}
9477 
9478   ins_encode %{
9479     __ blsrq($dst$$Register, $src$$Address);
9480   %}
9481 
9482   ins_pipe(ialu_reg);
9483 %}
9484 
9485 // Or Instructions
9486 // Or Register with Register
9487 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9488 %{
9489   match(Set dst (OrL dst src));
9490   effect(KILL cr);
9491 
9492   format %{ "orq     $dst, $src\t# long" %}
9493   opcode(0x0B);
9494   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9495   ins_pipe(ialu_reg_reg);
9496 %}
9497 
9498 // Use any_RegP to match R15 (TLS register) without spilling.
9499 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9500   match(Set dst (OrL dst (CastP2X src)));
9501   effect(KILL cr);
9502 
9503   format %{ "orq     $dst, $src\t# long" %}
9504   opcode(0x0B);
9505   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9506   ins_pipe(ialu_reg_reg);
9507 %}
9508 
9509 
9510 // Or Register with Immediate
9511 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9512 %{
9513   match(Set dst (OrL dst src));
9514   effect(KILL cr);
9515 
9516   format %{ "orq     $dst, $src\t# long" %}
9517   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9518   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9519   ins_pipe(ialu_reg);
9520 %}
9521 
9522 // Or Register with Memory
9523 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9524 %{
9525   match(Set dst (OrL dst (LoadL src)));
9526   effect(KILL cr);
9527 
9528   ins_cost(125);
9529   format %{ "orq     $dst, $src\t# long" %}
9530   opcode(0x0B);
9531   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9532   ins_pipe(ialu_reg_mem);
9533 %}
9534 
9535 // Or Memory with Register
9536 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9537 %{
9538   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9539   effect(KILL cr);
9540 
9541   ins_cost(150);
9542   format %{ "orq     $dst, $src\t# long" %}
9543   opcode(0x09); /* Opcode 09 /r */
9544   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9545   ins_pipe(ialu_mem_reg);
9546 %}
9547 
9548 // Or Memory with Immediate
9549 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9550 %{
9551   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9552   effect(KILL cr);
9553 
9554   ins_cost(125);
9555   format %{ "orq     $dst, $src\t# long" %}
9556   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9557   ins_encode(REX_mem_wide(dst), OpcSE(src),
9558              RM_opc_mem(secondary, dst), Con8or32(src));
9559   ins_pipe(ialu_mem_imm);
9560 %}
9561 
9562 // Xor Instructions
9563 // Xor Register with Register
9564 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9565 %{
9566   match(Set dst (XorL dst src));
9567   effect(KILL cr);
9568 
9569   format %{ "xorq    $dst, $src\t# long" %}
9570   opcode(0x33);
9571   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9572   ins_pipe(ialu_reg_reg);
9573 %}
9574 
9575 // Xor Register with Immediate -1
9576 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9577   match(Set dst (XorL dst imm));
9578 
9579   format %{ "notq   $dst" %}
9580   ins_encode %{
9581      __ notq($dst$$Register);
9582   %}
9583   ins_pipe(ialu_reg);
9584 %}
9585 
9586 // Xor Register with Immediate
9587 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9588 %{
9589   match(Set dst (XorL dst src));
9590   effect(KILL cr);
9591 
9592   format %{ "xorq    $dst, $src\t# long" %}
9593   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9594   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9595   ins_pipe(ialu_reg);
9596 %}
9597 
9598 // Xor Register with Memory
9599 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9600 %{
9601   match(Set dst (XorL dst (LoadL src)));
9602   effect(KILL cr);
9603 
9604   ins_cost(125);
9605   format %{ "xorq    $dst, $src\t# long" %}
9606   opcode(0x33);
9607   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9608   ins_pipe(ialu_reg_mem);
9609 %}
9610 
9611 // Xor Memory with Register
9612 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9613 %{
9614   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9615   effect(KILL cr);
9616 
9617   ins_cost(150);
9618   format %{ "xorq    $dst, $src\t# long" %}
9619   opcode(0x31); /* Opcode 31 /r */
9620   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9621   ins_pipe(ialu_mem_reg);
9622 %}
9623 
9624 // Xor Memory with Immediate
9625 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9626 %{
9627   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9628   effect(KILL cr);
9629 
9630   ins_cost(125);
9631   format %{ "xorq    $dst, $src\t# long" %}
9632   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9633   ins_encode(REX_mem_wide(dst), OpcSE(src),
9634              RM_opc_mem(secondary, dst), Con8or32(src));
9635   ins_pipe(ialu_mem_imm);
9636 %}
9637 
9638 // Convert Int to Boolean
9639 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9640 %{
9641   match(Set dst (Conv2B src));
9642   effect(KILL cr);
9643 
9644   format %{ "testl   $src, $src\t# ci2b\n\t"
9645             "setnz   $dst\n\t"
9646             "movzbl  $dst, $dst" %}
9647   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9648              setNZ_reg(dst),
9649              REX_reg_breg(dst, dst), // movzbl
9650              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9651   ins_pipe(pipe_slow); // XXX
9652 %}
9653 
9654 // Convert Pointer to Boolean
9655 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9656 %{
9657   match(Set dst (Conv2B src));
9658   effect(KILL cr);
9659 
9660   format %{ "testq   $src, $src\t# cp2b\n\t"
9661             "setnz   $dst\n\t"
9662             "movzbl  $dst, $dst" %}
9663   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9664              setNZ_reg(dst),
9665              REX_reg_breg(dst, dst), // movzbl
9666              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9667   ins_pipe(pipe_slow); // XXX
9668 %}
9669 
9670 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9671 %{
9672   match(Set dst (CmpLTMask p q));
9673   effect(KILL cr);
9674 
9675   ins_cost(400);
9676   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9677             "setlt   $dst\n\t"
9678             "movzbl  $dst, $dst\n\t"
9679             "negl    $dst" %}
9680   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9681              setLT_reg(dst),
9682              REX_reg_breg(dst, dst), // movzbl
9683              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9684              neg_reg(dst));
9685   ins_pipe(pipe_slow);
9686 %}
9687 
9688 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9689 %{
9690   match(Set dst (CmpLTMask dst zero));
9691   effect(KILL cr);
9692 
9693   ins_cost(100);
9694   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9695   ins_encode %{
9696   __ sarl($dst$$Register, 31);
9697   %}
9698   ins_pipe(ialu_reg);
9699 %}
9700 
9701 /* Better to save a register than avoid a branch */
9702 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9703 %{
9704   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9705   effect(KILL cr);
9706   ins_cost(300);
9707   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9708             "jge    done\n\t"
9709             "addl   $p,$y\n"
9710             "done:  " %}
9711   ins_encode %{
9712     Register Rp = $p$$Register;
9713     Register Rq = $q$$Register;
9714     Register Ry = $y$$Register;
9715     Label done;
9716     __ subl(Rp, Rq);
9717     __ jccb(Assembler::greaterEqual, done);
9718     __ addl(Rp, Ry);
9719     __ bind(done);
9720   %}
9721   ins_pipe(pipe_cmplt);
9722 %}
9723 
9724 /* Better to save a register than avoid a branch */
9725 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9726 %{
9727   match(Set y (AndI (CmpLTMask p q) y));
9728   effect(KILL cr);
9729 
9730   ins_cost(300);
9731 
9732   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9733             "jlt      done\n\t"
9734             "xorl     $y, $y\n"
9735             "done:  " %}
9736   ins_encode %{
9737     Register Rp = $p$$Register;
9738     Register Rq = $q$$Register;
9739     Register Ry = $y$$Register;
9740     Label done;
9741     __ cmpl(Rp, Rq);
9742     __ jccb(Assembler::less, done);
9743     __ xorl(Ry, Ry);
9744     __ bind(done);
9745   %}
9746   ins_pipe(pipe_cmplt);
9747 %}
9748 
9749 
9750 //---------- FP Instructions------------------------------------------------
9751 
9752 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9753 %{
9754   match(Set cr (CmpF src1 src2));
9755 
9756   ins_cost(145);
9757   format %{ "ucomiss $src1, $src2\n\t"
9758             "jnp,s   exit\n\t"
9759             "pushfq\t# saw NaN, set CF\n\t"
9760             "andq    [rsp], #0xffffff2b\n\t"
9761             "popfq\n"
9762     "exit:" %}
9763   ins_encode %{
9764     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9765     emit_cmpfp_fixup(_masm);
9766   %}
9767   ins_pipe(pipe_slow);
9768 %}
9769 
9770 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9771   match(Set cr (CmpF src1 src2));
9772 
9773   ins_cost(100);
9774   format %{ "ucomiss $src1, $src2" %}
9775   ins_encode %{
9776     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9777   %}
9778   ins_pipe(pipe_slow);
9779 %}
9780 
9781 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9782 %{
9783   match(Set cr (CmpF src1 (LoadF src2)));
9784 
9785   ins_cost(145);
9786   format %{ "ucomiss $src1, $src2\n\t"
9787             "jnp,s   exit\n\t"
9788             "pushfq\t# saw NaN, set CF\n\t"
9789             "andq    [rsp], #0xffffff2b\n\t"
9790             "popfq\n"
9791     "exit:" %}
9792   ins_encode %{
9793     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9794     emit_cmpfp_fixup(_masm);
9795   %}
9796   ins_pipe(pipe_slow);
9797 %}
9798 
9799 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9800   match(Set cr (CmpF src1 (LoadF src2)));
9801 
9802   ins_cost(100);
9803   format %{ "ucomiss $src1, $src2" %}
9804   ins_encode %{
9805     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9806   %}
9807   ins_pipe(pipe_slow);
9808 %}
9809 
9810 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9811   match(Set cr (CmpF src con));
9812 
9813   ins_cost(145);
9814   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9815             "jnp,s   exit\n\t"
9816             "pushfq\t# saw NaN, set CF\n\t"
9817             "andq    [rsp], #0xffffff2b\n\t"
9818             "popfq\n"
9819     "exit:" %}
9820   ins_encode %{
9821     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9822     emit_cmpfp_fixup(_masm);
9823   %}
9824   ins_pipe(pipe_slow);
9825 %}
9826 
9827 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9828   match(Set cr (CmpF src con));
9829   ins_cost(100);
9830   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9831   ins_encode %{
9832     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9833   %}
9834   ins_pipe(pipe_slow);
9835 %}
9836 
9837 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9838 %{
9839   match(Set cr (CmpD src1 src2));
9840 
9841   ins_cost(145);
9842   format %{ "ucomisd $src1, $src2\n\t"
9843             "jnp,s   exit\n\t"
9844             "pushfq\t# saw NaN, set CF\n\t"
9845             "andq    [rsp], #0xffffff2b\n\t"
9846             "popfq\n"
9847     "exit:" %}
9848   ins_encode %{
9849     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9850     emit_cmpfp_fixup(_masm);
9851   %}
9852   ins_pipe(pipe_slow);
9853 %}
9854 
9855 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9856   match(Set cr (CmpD src1 src2));
9857 
9858   ins_cost(100);
9859   format %{ "ucomisd $src1, $src2 test" %}
9860   ins_encode %{
9861     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9862   %}
9863   ins_pipe(pipe_slow);
9864 %}
9865 
9866 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9867 %{
9868   match(Set cr (CmpD src1 (LoadD src2)));
9869 
9870   ins_cost(145);
9871   format %{ "ucomisd $src1, $src2\n\t"
9872             "jnp,s   exit\n\t"
9873             "pushfq\t# saw NaN, set CF\n\t"
9874             "andq    [rsp], #0xffffff2b\n\t"
9875             "popfq\n"
9876     "exit:" %}
9877   ins_encode %{
9878     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9879     emit_cmpfp_fixup(_masm);
9880   %}
9881   ins_pipe(pipe_slow);
9882 %}
9883 
9884 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9885   match(Set cr (CmpD src1 (LoadD src2)));
9886 
9887   ins_cost(100);
9888   format %{ "ucomisd $src1, $src2" %}
9889   ins_encode %{
9890     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9891   %}
9892   ins_pipe(pipe_slow);
9893 %}
9894 
9895 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9896   match(Set cr (CmpD src con));
9897 
9898   ins_cost(145);
9899   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9900             "jnp,s   exit\n\t"
9901             "pushfq\t# saw NaN, set CF\n\t"
9902             "andq    [rsp], #0xffffff2b\n\t"
9903             "popfq\n"
9904     "exit:" %}
9905   ins_encode %{
9906     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9907     emit_cmpfp_fixup(_masm);
9908   %}
9909   ins_pipe(pipe_slow);
9910 %}
9911 
9912 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9913   match(Set cr (CmpD src con));
9914   ins_cost(100);
9915   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9916   ins_encode %{
9917     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9918   %}
9919   ins_pipe(pipe_slow);
9920 %}
9921 
9922 // Compare into -1,0,1
9923 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9924 %{
9925   match(Set dst (CmpF3 src1 src2));
9926   effect(KILL cr);
9927 
9928   ins_cost(275);
9929   format %{ "ucomiss $src1, $src2\n\t"
9930             "movl    $dst, #-1\n\t"
9931             "jp,s    done\n\t"
9932             "jb,s    done\n\t"
9933             "setne   $dst\n\t"
9934             "movzbl  $dst, $dst\n"
9935     "done:" %}
9936   ins_encode %{
9937     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9938     emit_cmpfp3(_masm, $dst$$Register);
9939   %}
9940   ins_pipe(pipe_slow);
9941 %}
9942 
9943 // Compare into -1,0,1
9944 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9945 %{
9946   match(Set dst (CmpF3 src1 (LoadF src2)));
9947   effect(KILL cr);
9948 
9949   ins_cost(275);
9950   format %{ "ucomiss $src1, $src2\n\t"
9951             "movl    $dst, #-1\n\t"
9952             "jp,s    done\n\t"
9953             "jb,s    done\n\t"
9954             "setne   $dst\n\t"
9955             "movzbl  $dst, $dst\n"
9956     "done:" %}
9957   ins_encode %{
9958     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9959     emit_cmpfp3(_masm, $dst$$Register);
9960   %}
9961   ins_pipe(pipe_slow);
9962 %}
9963 
9964 // Compare into -1,0,1
9965 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9966   match(Set dst (CmpF3 src con));
9967   effect(KILL cr);
9968 
9969   ins_cost(275);
9970   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9971             "movl    $dst, #-1\n\t"
9972             "jp,s    done\n\t"
9973             "jb,s    done\n\t"
9974             "setne   $dst\n\t"
9975             "movzbl  $dst, $dst\n"
9976     "done:" %}
9977   ins_encode %{
9978     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9979     emit_cmpfp3(_masm, $dst$$Register);
9980   %}
9981   ins_pipe(pipe_slow);
9982 %}
9983 
9984 // Compare into -1,0,1
9985 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9986 %{
9987   match(Set dst (CmpD3 src1 src2));
9988   effect(KILL cr);
9989 
9990   ins_cost(275);
9991   format %{ "ucomisd $src1, $src2\n\t"
9992             "movl    $dst, #-1\n\t"
9993             "jp,s    done\n\t"
9994             "jb,s    done\n\t"
9995             "setne   $dst\n\t"
9996             "movzbl  $dst, $dst\n"
9997     "done:" %}
9998   ins_encode %{
9999     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
10000     emit_cmpfp3(_masm, $dst$$Register);
10001   %}
10002   ins_pipe(pipe_slow);
10003 %}
10004 
10005 // Compare into -1,0,1
10006 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
10007 %{
10008   match(Set dst (CmpD3 src1 (LoadD src2)));
10009   effect(KILL cr);
10010 
10011   ins_cost(275);
10012   format %{ "ucomisd $src1, $src2\n\t"
10013             "movl    $dst, #-1\n\t"
10014             "jp,s    done\n\t"
10015             "jb,s    done\n\t"
10016             "setne   $dst\n\t"
10017             "movzbl  $dst, $dst\n"
10018     "done:" %}
10019   ins_encode %{
10020     __ ucomisd($src1$$XMMRegister, $src2$$Address);
10021     emit_cmpfp3(_masm, $dst$$Register);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 // Compare into -1,0,1
10027 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
10028   match(Set dst (CmpD3 src con));
10029   effect(KILL cr);
10030 
10031   ins_cost(275);
10032   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
10033             "movl    $dst, #-1\n\t"
10034             "jp,s    done\n\t"
10035             "jb,s    done\n\t"
10036             "setne   $dst\n\t"
10037             "movzbl  $dst, $dst\n"
10038     "done:" %}
10039   ins_encode %{
10040     __ ucomisd($src$$XMMRegister, $constantaddress($con));
10041     emit_cmpfp3(_masm, $dst$$Register);
10042   %}
10043   ins_pipe(pipe_slow);
10044 %}
10045 
10046 //----------Arithmetic Conversion Instructions---------------------------------
10047 
10048 instruct roundFloat_nop(regF dst)
10049 %{
10050   match(Set dst (RoundFloat dst));
10051 
10052   ins_cost(0);
10053   ins_encode();
10054   ins_pipe(empty);
10055 %}
10056 
10057 instruct roundDouble_nop(regD dst)
10058 %{
10059   match(Set dst (RoundDouble dst));
10060 
10061   ins_cost(0);
10062   ins_encode();
10063   ins_pipe(empty);
10064 %}
10065 
10066 instruct convF2D_reg_reg(regD dst, regF src)
10067 %{
10068   match(Set dst (ConvF2D src));
10069 
10070   format %{ "cvtss2sd $dst, $src" %}
10071   ins_encode %{
10072     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
10073   %}
10074   ins_pipe(pipe_slow); // XXX
10075 %}
10076 
10077 instruct convF2D_reg_mem(regD dst, memory src)
10078 %{
10079   match(Set dst (ConvF2D (LoadF src)));
10080 
10081   format %{ "cvtss2sd $dst, $src" %}
10082   ins_encode %{
10083     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
10084   %}
10085   ins_pipe(pipe_slow); // XXX
10086 %}
10087 
10088 instruct convD2F_reg_reg(regF dst, regD src)
10089 %{
10090   match(Set dst (ConvD2F src));
10091 
10092   format %{ "cvtsd2ss $dst, $src" %}
10093   ins_encode %{
10094     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
10095   %}
10096   ins_pipe(pipe_slow); // XXX
10097 %}
10098 
10099 instruct convD2F_reg_mem(regF dst, memory src)
10100 %{
10101   match(Set dst (ConvD2F (LoadD src)));
10102 
10103   format %{ "cvtsd2ss $dst, $src" %}
10104   ins_encode %{
10105     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
10106   %}
10107   ins_pipe(pipe_slow); // XXX
10108 %}
10109 
10110 // XXX do mem variants
10111 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
10112 %{
10113   match(Set dst (ConvF2I src));
10114   effect(KILL cr);
10115 
10116   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
10117             "cmpl    $dst, #0x80000000\n\t"
10118             "jne,s   done\n\t"
10119             "subq    rsp, #8\n\t"
10120             "movss   [rsp], $src\n\t"
10121             "call    f2i_fixup\n\t"
10122             "popq    $dst\n"
10123     "done:   "%}
10124   ins_encode %{
10125     Label done;
10126     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10127     __ cmpl($dst$$Register, 0x80000000);
10128     __ jccb(Assembler::notEqual, done);
10129     __ subptr(rsp, 8);
10130     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10131     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10132     __ pop($dst$$Register);
10133     __ bind(done);
10134   %}
10135   ins_pipe(pipe_slow);
10136 %}
10137 
10138 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10139 %{
10140   match(Set dst (ConvF2L src));
10141   effect(KILL cr);
10142 
10143   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10144             "cmpq    $dst, [0x8000000000000000]\n\t"
10145             "jne,s   done\n\t"
10146             "subq    rsp, #8\n\t"
10147             "movss   [rsp], $src\n\t"
10148             "call    f2l_fixup\n\t"
10149             "popq    $dst\n"
10150     "done:   "%}
10151   ins_encode %{
10152     Label done;
10153     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10154     __ cmp64($dst$$Register,
10155              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10156     __ jccb(Assembler::notEqual, done);
10157     __ subptr(rsp, 8);
10158     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10159     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10160     __ pop($dst$$Register);
10161     __ bind(done);
10162   %}
10163   ins_pipe(pipe_slow);
10164 %}
10165 
10166 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10167 %{
10168   match(Set dst (ConvD2I src));
10169   effect(KILL cr);
10170 
10171   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10172             "cmpl    $dst, #0x80000000\n\t"
10173             "jne,s   done\n\t"
10174             "subq    rsp, #8\n\t"
10175             "movsd   [rsp], $src\n\t"
10176             "call    d2i_fixup\n\t"
10177             "popq    $dst\n"
10178     "done:   "%}
10179   ins_encode %{
10180     Label done;
10181     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10182     __ cmpl($dst$$Register, 0x80000000);
10183     __ jccb(Assembler::notEqual, done);
10184     __ subptr(rsp, 8);
10185     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10186     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10187     __ pop($dst$$Register);
10188     __ bind(done);
10189   %}
10190   ins_pipe(pipe_slow);
10191 %}
10192 
10193 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10194 %{
10195   match(Set dst (ConvD2L src));
10196   effect(KILL cr);
10197 
10198   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10199             "cmpq    $dst, [0x8000000000000000]\n\t"
10200             "jne,s   done\n\t"
10201             "subq    rsp, #8\n\t"
10202             "movsd   [rsp], $src\n\t"
10203             "call    d2l_fixup\n\t"
10204             "popq    $dst\n"
10205     "done:   "%}
10206   ins_encode %{
10207     Label done;
10208     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10209     __ cmp64($dst$$Register,
10210              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10211     __ jccb(Assembler::notEqual, done);
10212     __ subptr(rsp, 8);
10213     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10214     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10215     __ pop($dst$$Register);
10216     __ bind(done);
10217   %}
10218   ins_pipe(pipe_slow);
10219 %}
10220 
10221 instruct convI2F_reg_reg(regF dst, rRegI src)
10222 %{
10223   predicate(!UseXmmI2F);
10224   match(Set dst (ConvI2F src));
10225 
10226   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10227   ins_encode %{
10228     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10229   %}
10230   ins_pipe(pipe_slow); // XXX
10231 %}
10232 
10233 instruct convI2F_reg_mem(regF dst, memory src)
10234 %{
10235   match(Set dst (ConvI2F (LoadI src)));
10236 
10237   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10238   ins_encode %{
10239     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10240   %}
10241   ins_pipe(pipe_slow); // XXX
10242 %}
10243 
10244 instruct convI2D_reg_reg(regD dst, rRegI src)
10245 %{
10246   predicate(!UseXmmI2D);
10247   match(Set dst (ConvI2D src));
10248 
10249   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10250   ins_encode %{
10251     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10252   %}
10253   ins_pipe(pipe_slow); // XXX
10254 %}
10255 
10256 instruct convI2D_reg_mem(regD dst, memory src)
10257 %{
10258   match(Set dst (ConvI2D (LoadI src)));
10259 
10260   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10261   ins_encode %{
10262     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10263   %}
10264   ins_pipe(pipe_slow); // XXX
10265 %}
10266 
10267 instruct convXI2F_reg(regF dst, rRegI src)
10268 %{
10269   predicate(UseXmmI2F);
10270   match(Set dst (ConvI2F src));
10271 
10272   format %{ "movdl $dst, $src\n\t"
10273             "cvtdq2psl $dst, $dst\t# i2f" %}
10274   ins_encode %{
10275     __ movdl($dst$$XMMRegister, $src$$Register);
10276     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10277   %}
10278   ins_pipe(pipe_slow); // XXX
10279 %}
10280 
10281 instruct convXI2D_reg(regD dst, rRegI src)
10282 %{
10283   predicate(UseXmmI2D);
10284   match(Set dst (ConvI2D src));
10285 
10286   format %{ "movdl $dst, $src\n\t"
10287             "cvtdq2pdl $dst, $dst\t# i2d" %}
10288   ins_encode %{
10289     __ movdl($dst$$XMMRegister, $src$$Register);
10290     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10291   %}
10292   ins_pipe(pipe_slow); // XXX
10293 %}
10294 
10295 instruct convL2F_reg_reg(regF dst, rRegL src)
10296 %{
10297   match(Set dst (ConvL2F src));
10298 
10299   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10300   ins_encode %{
10301     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10302   %}
10303   ins_pipe(pipe_slow); // XXX
10304 %}
10305 
10306 instruct convL2F_reg_mem(regF dst, memory src)
10307 %{
10308   match(Set dst (ConvL2F (LoadL src)));
10309 
10310   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10311   ins_encode %{
10312     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10313   %}
10314   ins_pipe(pipe_slow); // XXX
10315 %}
10316 
10317 instruct convL2D_reg_reg(regD dst, rRegL src)
10318 %{
10319   match(Set dst (ConvL2D src));
10320 
10321   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10322   ins_encode %{
10323     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10324   %}
10325   ins_pipe(pipe_slow); // XXX
10326 %}
10327 
10328 instruct convL2D_reg_mem(regD dst, memory src)
10329 %{
10330   match(Set dst (ConvL2D (LoadL src)));
10331 
10332   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10333   ins_encode %{
10334     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10335   %}
10336   ins_pipe(pipe_slow); // XXX
10337 %}
10338 
10339 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10340 %{
10341   match(Set dst (ConvI2L src));
10342 
10343   ins_cost(125);
10344   format %{ "movslq  $dst, $src\t# i2l" %}
10345   ins_encode %{
10346     __ movslq($dst$$Register, $src$$Register);
10347   %}
10348   ins_pipe(ialu_reg_reg);
10349 %}
10350 
10351 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10352 // %{
10353 //   match(Set dst (ConvI2L src));
10354 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10355 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10356 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10357 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10358 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10359 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10360 
10361 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10362 //   ins_encode(enc_copy(dst, src));
10363 // //   opcode(0x63); // needs REX.W
10364 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10365 //   ins_pipe(ialu_reg_reg);
10366 // %}
10367 
10368 // Zero-extend convert int to long
10369 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10370 %{
10371   match(Set dst (AndL (ConvI2L src) mask));
10372 
10373   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10374   ins_encode %{
10375     if ($dst$$reg != $src$$reg) {
10376       __ movl($dst$$Register, $src$$Register);
10377     }
10378   %}
10379   ins_pipe(ialu_reg_reg);
10380 %}
10381 
10382 // Zero-extend convert int to long
10383 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10384 %{
10385   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10386 
10387   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10388   ins_encode %{
10389     __ movl($dst$$Register, $src$$Address);
10390   %}
10391   ins_pipe(ialu_reg_mem);
10392 %}
10393 
10394 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10395 %{
10396   match(Set dst (AndL src mask));
10397 
10398   format %{ "movl    $dst, $src\t# zero-extend long" %}
10399   ins_encode %{
10400     __ movl($dst$$Register, $src$$Register);
10401   %}
10402   ins_pipe(ialu_reg_reg);
10403 %}
10404 
10405 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10406 %{
10407   match(Set dst (ConvL2I src));
10408 
10409   format %{ "movl    $dst, $src\t# l2i" %}
10410   ins_encode %{
10411     __ movl($dst$$Register, $src$$Register);
10412   %}
10413   ins_pipe(ialu_reg_reg);
10414 %}
10415 
10416 
10417 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10418   match(Set dst (MoveF2I src));
10419   effect(DEF dst, USE src);
10420 
10421   ins_cost(125);
10422   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10423   ins_encode %{
10424     __ movl($dst$$Register, Address(rsp, $src$$disp));
10425   %}
10426   ins_pipe(ialu_reg_mem);
10427 %}
10428 
10429 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10430   match(Set dst (MoveI2F src));
10431   effect(DEF dst, USE src);
10432 
10433   ins_cost(125);
10434   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10435   ins_encode %{
10436     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10437   %}
10438   ins_pipe(pipe_slow);
10439 %}
10440 
10441 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10442   match(Set dst (MoveD2L src));
10443   effect(DEF dst, USE src);
10444 
10445   ins_cost(125);
10446   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10447   ins_encode %{
10448     __ movq($dst$$Register, Address(rsp, $src$$disp));
10449   %}
10450   ins_pipe(ialu_reg_mem);
10451 %}
10452 
10453 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10454   predicate(!UseXmmLoadAndClearUpper);
10455   match(Set dst (MoveL2D src));
10456   effect(DEF dst, USE src);
10457 
10458   ins_cost(125);
10459   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10460   ins_encode %{
10461     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10462   %}
10463   ins_pipe(pipe_slow);
10464 %}
10465 
10466 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10467   predicate(UseXmmLoadAndClearUpper);
10468   match(Set dst (MoveL2D src));
10469   effect(DEF dst, USE src);
10470 
10471   ins_cost(125);
10472   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10473   ins_encode %{
10474     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10475   %}
10476   ins_pipe(pipe_slow);
10477 %}
10478 
10479 
10480 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10481   match(Set dst (MoveF2I src));
10482   effect(DEF dst, USE src);
10483 
10484   ins_cost(95); // XXX
10485   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10486   ins_encode %{
10487     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10488   %}
10489   ins_pipe(pipe_slow);
10490 %}
10491 
10492 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10493   match(Set dst (MoveI2F src));
10494   effect(DEF dst, USE src);
10495 
10496   ins_cost(100);
10497   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10498   ins_encode %{
10499     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10500   %}
10501   ins_pipe( ialu_mem_reg );
10502 %}
10503 
10504 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10505   match(Set dst (MoveD2L src));
10506   effect(DEF dst, USE src);
10507 
10508   ins_cost(95); // XXX
10509   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10510   ins_encode %{
10511     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10512   %}
10513   ins_pipe(pipe_slow);
10514 %}
10515 
10516 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10517   match(Set dst (MoveL2D src));
10518   effect(DEF dst, USE src);
10519 
10520   ins_cost(100);
10521   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10522   ins_encode %{
10523     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10524   %}
10525   ins_pipe(ialu_mem_reg);
10526 %}
10527 
10528 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10529   match(Set dst (MoveF2I src));
10530   effect(DEF dst, USE src);
10531   ins_cost(85);
10532   format %{ "movd    $dst,$src\t# MoveF2I" %}
10533   ins_encode %{
10534     __ movdl($dst$$Register, $src$$XMMRegister);
10535   %}
10536   ins_pipe( pipe_slow );
10537 %}
10538 
10539 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10540   match(Set dst (MoveD2L src));
10541   effect(DEF dst, USE src);
10542   ins_cost(85);
10543   format %{ "movd    $dst,$src\t# MoveD2L" %}
10544   ins_encode %{
10545     __ movdq($dst$$Register, $src$$XMMRegister);
10546   %}
10547   ins_pipe( pipe_slow );
10548 %}
10549 
10550 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10551   match(Set dst (MoveI2F src));
10552   effect(DEF dst, USE src);
10553   ins_cost(100);
10554   format %{ "movd    $dst,$src\t# MoveI2F" %}
10555   ins_encode %{
10556     __ movdl($dst$$XMMRegister, $src$$Register);
10557   %}
10558   ins_pipe( pipe_slow );
10559 %}
10560 
10561 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10562   match(Set dst (MoveL2D src));
10563   effect(DEF dst, USE src);
10564   ins_cost(100);
10565   format %{ "movd    $dst,$src\t# MoveL2D" %}
10566   ins_encode %{
10567      __ movdq($dst$$XMMRegister, $src$$Register);
10568   %}
10569   ins_pipe( pipe_slow );
10570 %}
10571 
10572 
10573 // =======================================================================
10574 // fast clearing of an array
10575 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10576                   rFlagsReg cr)
10577 %{
10578   predicate(!((ClearArrayNode*)n)->is_large());
10579   match(Set dummy (ClearArray cnt base));
10580   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10581 
10582   format %{ $$template
10583     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10584     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10585     $$emit$$"jg      LARGE\n\t"
10586     $$emit$$"dec     rcx\n\t"
10587     $$emit$$"js      DONE\t# Zero length\n\t"
10588     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10589     $$emit$$"dec     rcx\n\t"
10590     $$emit$$"jge     LOOP\n\t"
10591     $$emit$$"jmp     DONE\n\t"
10592     $$emit$$"# LARGE:\n\t"
10593     if (UseFastStosb) {
10594        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10595        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10596     } else {
10597        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10598     }
10599     $$emit$$"# DONE"
10600   %}
10601   ins_encode %{
10602     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, false);
10603   %}
10604   ins_pipe(pipe_slow);
10605 %}
10606 
10607 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10608                   rFlagsReg cr)
10609 %{
10610   predicate(((ClearArrayNode*)n)->is_large());
10611   match(Set dummy (ClearArray cnt base));
10612   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10613 
10614   format %{ $$template
10615     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10616     if (UseFastStosb) {
10617        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10618        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10619     } else {
10620        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10621     }
10622   %}
10623   ins_encode %{
10624     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, true);
10625   %}
10626   ins_pipe(pipe_slow);
10627 %}
10628 
10629 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10630                          rax_RegI result, regD tmp1, rFlagsReg cr)
10631 %{
10632   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10633   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10634   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10635 
10636   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10637   ins_encode %{
10638     __ string_compare($str1$$Register, $str2$$Register,
10639                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10640                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10641   %}
10642   ins_pipe( pipe_slow );
10643 %}
10644 
10645 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10646                          rax_RegI result, regD tmp1, rFlagsReg cr)
10647 %{
10648   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10649   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10650   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10651 
10652   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10653   ins_encode %{
10654     __ string_compare($str1$$Register, $str2$$Register,
10655                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10656                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10657   %}
10658   ins_pipe( pipe_slow );
10659 %}
10660 
10661 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10662                           rax_RegI result, regD tmp1, rFlagsReg cr)
10663 %{
10664   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10665   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10666   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10667 
10668   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10669   ins_encode %{
10670     __ string_compare($str1$$Register, $str2$$Register,
10671                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10672                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10673   %}
10674   ins_pipe( pipe_slow );
10675 %}
10676 
10677 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10678                           rax_RegI result, regD tmp1, rFlagsReg cr)
10679 %{
10680   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10681   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10682   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10683 
10684   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10685   ins_encode %{
10686     __ string_compare($str2$$Register, $str1$$Register,
10687                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10688                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10689   %}
10690   ins_pipe( pipe_slow );
10691 %}
10692 
10693 // fast search of substring with known size.
10694 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10695                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10696 %{
10697   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10698   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10699   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10700 
10701   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10702   ins_encode %{
10703     int icnt2 = (int)$int_cnt2$$constant;
10704     if (icnt2 >= 16) {
10705       // IndexOf for constant substrings with size >= 16 elements
10706       // which don't need to be loaded through stack.
10707       __ string_indexofC8($str1$$Register, $str2$$Register,
10708                           $cnt1$$Register, $cnt2$$Register,
10709                           icnt2, $result$$Register,
10710                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10711     } else {
10712       // Small strings are loaded through stack if they cross page boundary.
10713       __ string_indexof($str1$$Register, $str2$$Register,
10714                         $cnt1$$Register, $cnt2$$Register,
10715                         icnt2, $result$$Register,
10716                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10717     }
10718   %}
10719   ins_pipe( pipe_slow );
10720 %}
10721 
10722 // fast search of substring with known size.
10723 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10724                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10725 %{
10726   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10727   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10728   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10729 
10730   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10731   ins_encode %{
10732     int icnt2 = (int)$int_cnt2$$constant;
10733     if (icnt2 >= 8) {
10734       // IndexOf for constant substrings with size >= 8 elements
10735       // which don't need to be loaded through stack.
10736       __ string_indexofC8($str1$$Register, $str2$$Register,
10737                           $cnt1$$Register, $cnt2$$Register,
10738                           icnt2, $result$$Register,
10739                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10740     } else {
10741       // Small strings are loaded through stack if they cross page boundary.
10742       __ string_indexof($str1$$Register, $str2$$Register,
10743                         $cnt1$$Register, $cnt2$$Register,
10744                         icnt2, $result$$Register,
10745                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10746     }
10747   %}
10748   ins_pipe( pipe_slow );
10749 %}
10750 
10751 // fast search of substring with known size.
10752 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10753                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10754 %{
10755   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10756   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10757   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10758 
10759   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10760   ins_encode %{
10761     int icnt2 = (int)$int_cnt2$$constant;
10762     if (icnt2 >= 8) {
10763       // IndexOf for constant substrings with size >= 8 elements
10764       // which don't need to be loaded through stack.
10765       __ string_indexofC8($str1$$Register, $str2$$Register,
10766                           $cnt1$$Register, $cnt2$$Register,
10767                           icnt2, $result$$Register,
10768                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10769     } else {
10770       // Small strings are loaded through stack if they cross page boundary.
10771       __ string_indexof($str1$$Register, $str2$$Register,
10772                         $cnt1$$Register, $cnt2$$Register,
10773                         icnt2, $result$$Register,
10774                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10775     }
10776   %}
10777   ins_pipe( pipe_slow );
10778 %}
10779 
10780 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10781                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10782 %{
10783   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10784   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10785   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10786 
10787   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10788   ins_encode %{
10789     __ string_indexof($str1$$Register, $str2$$Register,
10790                       $cnt1$$Register, $cnt2$$Register,
10791                       (-1), $result$$Register,
10792                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10793   %}
10794   ins_pipe( pipe_slow );
10795 %}
10796 
10797 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10798                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10799 %{
10800   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10801   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10802   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10803 
10804   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10805   ins_encode %{
10806     __ string_indexof($str1$$Register, $str2$$Register,
10807                       $cnt1$$Register, $cnt2$$Register,
10808                       (-1), $result$$Register,
10809                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10810   %}
10811   ins_pipe( pipe_slow );
10812 %}
10813 
10814 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10815                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10816 %{
10817   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10818   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10819   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10820 
10821   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10822   ins_encode %{
10823     __ string_indexof($str1$$Register, $str2$$Register,
10824                       $cnt1$$Register, $cnt2$$Register,
10825                       (-1), $result$$Register,
10826                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10827   %}
10828   ins_pipe( pipe_slow );
10829 %}
10830 
10831 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10832                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10833 %{
10834   predicate(UseSSE42Intrinsics);
10835   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10836   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10837   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10838   ins_encode %{
10839     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10840                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10841   %}
10842   ins_pipe( pipe_slow );
10843 %}
10844 
10845 // fast string equals
10846 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10847                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10848 %{
10849   match(Set result (StrEquals (Binary str1 str2) cnt));
10850   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10851 
10852   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10853   ins_encode %{
10854     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10855                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10856                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10857   %}
10858   ins_pipe( pipe_slow );
10859 %}
10860 
10861 // fast array equals
10862 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10863                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10864 %{
10865   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10866   match(Set result (AryEq ary1 ary2));
10867   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10868 
10869   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10870   ins_encode %{
10871     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10872                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10873                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10874   %}
10875   ins_pipe( pipe_slow );
10876 %}
10877 
10878 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10879                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10880 %{
10881   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10882   match(Set result (AryEq ary1 ary2));
10883   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10884 
10885   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10886   ins_encode %{
10887     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10888                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10889                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10890   %}
10891   ins_pipe( pipe_slow );
10892 %}
10893 
10894 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10895                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10896 %{
10897   match(Set result (HasNegatives ary1 len));
10898   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10899 
10900   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10901   ins_encode %{
10902     __ has_negatives($ary1$$Register, $len$$Register,
10903                      $result$$Register, $tmp3$$Register,
10904                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10905   %}
10906   ins_pipe( pipe_slow );
10907 %}
10908 
10909 // fast char[] to byte[] compression
10910 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10911                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10912   match(Set result (StrCompressedCopy src (Binary dst len)));
10913   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10914 
10915   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10916   ins_encode %{
10917     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10918                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10919                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10920   %}
10921   ins_pipe( pipe_slow );
10922 %}
10923 
10924 // fast byte[] to char[] inflation
10925 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10926                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10927   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10928   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10929 
10930   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10931   ins_encode %{
10932     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10933                           $tmp1$$XMMRegister, $tmp2$$Register);
10934   %}
10935   ins_pipe( pipe_slow );
10936 %}
10937 
10938 // encode char[] to byte[] in ISO_8859_1
10939 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10940                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10941                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10942   match(Set result (EncodeISOArray src (Binary dst len)));
10943   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10944 
10945   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10946   ins_encode %{
10947     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10948                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10949                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10950   %}
10951   ins_pipe( pipe_slow );
10952 %}
10953 
10954 // fma - double - a * b + c
10955 instruct fmaD_reg(regD a, regD b, regD c) %{
10956   predicate(UseFMA);
10957   match(Set c (FmaD  c (Binary a b)));
10958   format %{ "fmasd $a,$b,$c -> $c" %}
10959   ins_cost(150);
10960   ins_encode %{
10961     __ vfmadd231sd($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister);
10962   %}
10963   ins_pipe( pipe_slow );
10964 %}
10965 
10966 // fma - float - a * b + c
10967 instruct fmaF_reg(regF a, regF b, regF c) %{
10968   predicate(UseFMA);
10969   match(Set c (FmaF  c (Binary a b)));
10970   format %{ "fmass $a,$b,$c -> $c" %}
10971   ins_cost(150);
10972   ins_encode %{
10973     __ vfmadd231ss($c$$XMMRegister, $a$$XMMRegister, $b$$XMMRegister);
10974   %}
10975   ins_pipe( pipe_slow );
10976 %}
10977 
10978 //----------Overflow Math Instructions-----------------------------------------
10979 
10980 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10981 %{
10982   match(Set cr (OverflowAddI op1 op2));
10983   effect(DEF cr, USE_KILL op1, USE op2);
10984 
10985   format %{ "addl    $op1, $op2\t# overflow check int" %}
10986 
10987   ins_encode %{
10988     __ addl($op1$$Register, $op2$$Register);
10989   %}
10990   ins_pipe(ialu_reg_reg);
10991 %}
10992 
10993 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10994 %{
10995   match(Set cr (OverflowAddI op1 op2));
10996   effect(DEF cr, USE_KILL op1, USE op2);
10997 
10998   format %{ "addl    $op1, $op2\t# overflow check int" %}
10999 
11000   ins_encode %{
11001     __ addl($op1$$Register, $op2$$constant);
11002   %}
11003   ins_pipe(ialu_reg_reg);
11004 %}
11005 
11006 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11007 %{
11008   match(Set cr (OverflowAddL op1 op2));
11009   effect(DEF cr, USE_KILL op1, USE op2);
11010 
11011   format %{ "addq    $op1, $op2\t# overflow check long" %}
11012   ins_encode %{
11013     __ addq($op1$$Register, $op2$$Register);
11014   %}
11015   ins_pipe(ialu_reg_reg);
11016 %}
11017 
11018 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
11019 %{
11020   match(Set cr (OverflowAddL op1 op2));
11021   effect(DEF cr, USE_KILL op1, USE op2);
11022 
11023   format %{ "addq    $op1, $op2\t# overflow check long" %}
11024   ins_encode %{
11025     __ addq($op1$$Register, $op2$$constant);
11026   %}
11027   ins_pipe(ialu_reg_reg);
11028 %}
11029 
11030 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11031 %{
11032   match(Set cr (OverflowSubI op1 op2));
11033 
11034   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11035   ins_encode %{
11036     __ cmpl($op1$$Register, $op2$$Register);
11037   %}
11038   ins_pipe(ialu_reg_reg);
11039 %}
11040 
11041 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11042 %{
11043   match(Set cr (OverflowSubI op1 op2));
11044 
11045   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
11046   ins_encode %{
11047     __ cmpl($op1$$Register, $op2$$constant);
11048   %}
11049   ins_pipe(ialu_reg_reg);
11050 %}
11051 
11052 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11053 %{
11054   match(Set cr (OverflowSubL op1 op2));
11055 
11056   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11057   ins_encode %{
11058     __ cmpq($op1$$Register, $op2$$Register);
11059   %}
11060   ins_pipe(ialu_reg_reg);
11061 %}
11062 
11063 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11064 %{
11065   match(Set cr (OverflowSubL op1 op2));
11066 
11067   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
11068   ins_encode %{
11069     __ cmpq($op1$$Register, $op2$$constant);
11070   %}
11071   ins_pipe(ialu_reg_reg);
11072 %}
11073 
11074 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
11075 %{
11076   match(Set cr (OverflowSubI zero op2));
11077   effect(DEF cr, USE_KILL op2);
11078 
11079   format %{ "negl    $op2\t# overflow check int" %}
11080   ins_encode %{
11081     __ negl($op2$$Register);
11082   %}
11083   ins_pipe(ialu_reg_reg);
11084 %}
11085 
11086 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
11087 %{
11088   match(Set cr (OverflowSubL zero op2));
11089   effect(DEF cr, USE_KILL op2);
11090 
11091   format %{ "negq    $op2\t# overflow check long" %}
11092   ins_encode %{
11093     __ negq($op2$$Register);
11094   %}
11095   ins_pipe(ialu_reg_reg);
11096 %}
11097 
11098 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
11099 %{
11100   match(Set cr (OverflowMulI op1 op2));
11101   effect(DEF cr, USE_KILL op1, USE op2);
11102 
11103   format %{ "imull    $op1, $op2\t# overflow check int" %}
11104   ins_encode %{
11105     __ imull($op1$$Register, $op2$$Register);
11106   %}
11107   ins_pipe(ialu_reg_reg_alu0);
11108 %}
11109 
11110 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
11111 %{
11112   match(Set cr (OverflowMulI op1 op2));
11113   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11114 
11115   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
11116   ins_encode %{
11117     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
11118   %}
11119   ins_pipe(ialu_reg_reg_alu0);
11120 %}
11121 
11122 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
11123 %{
11124   match(Set cr (OverflowMulL op1 op2));
11125   effect(DEF cr, USE_KILL op1, USE op2);
11126 
11127   format %{ "imulq    $op1, $op2\t# overflow check long" %}
11128   ins_encode %{
11129     __ imulq($op1$$Register, $op2$$Register);
11130   %}
11131   ins_pipe(ialu_reg_reg_alu0);
11132 %}
11133 
11134 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
11135 %{
11136   match(Set cr (OverflowMulL op1 op2));
11137   effect(DEF cr, TEMP tmp, USE op1, USE op2);
11138 
11139   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
11140   ins_encode %{
11141     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11142   %}
11143   ins_pipe(ialu_reg_reg_alu0);
11144 %}
11145 
11146 
11147 //----------Control Flow Instructions------------------------------------------
11148 // Signed compare Instructions
11149 
11150 // XXX more variants!!
11151 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11152 %{
11153   match(Set cr (CmpI op1 op2));
11154   effect(DEF cr, USE op1, USE op2);
11155 
11156   format %{ "cmpl    $op1, $op2" %}
11157   opcode(0x3B);  /* Opcode 3B /r */
11158   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11159   ins_pipe(ialu_cr_reg_reg);
11160 %}
11161 
11162 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11163 %{
11164   match(Set cr (CmpI op1 op2));
11165 
11166   format %{ "cmpl    $op1, $op2" %}
11167   opcode(0x81, 0x07); /* Opcode 81 /7 */
11168   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11169   ins_pipe(ialu_cr_reg_imm);
11170 %}
11171 
11172 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11173 %{
11174   match(Set cr (CmpI op1 (LoadI op2)));
11175 
11176   ins_cost(500); // XXX
11177   format %{ "cmpl    $op1, $op2" %}
11178   opcode(0x3B); /* Opcode 3B /r */
11179   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11180   ins_pipe(ialu_cr_reg_mem);
11181 %}
11182 
11183 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11184 %{
11185   match(Set cr (CmpI src zero));
11186 
11187   format %{ "testl   $src, $src" %}
11188   opcode(0x85);
11189   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11190   ins_pipe(ialu_cr_reg_imm);
11191 %}
11192 
11193 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11194 %{
11195   match(Set cr (CmpI (AndI src con) zero));
11196 
11197   format %{ "testl   $src, $con" %}
11198   opcode(0xF7, 0x00);
11199   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11200   ins_pipe(ialu_cr_reg_imm);
11201 %}
11202 
11203 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11204 %{
11205   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11206 
11207   format %{ "testl   $src, $mem" %}
11208   opcode(0x85);
11209   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11210   ins_pipe(ialu_cr_reg_mem);
11211 %}
11212 
11213 // Unsigned compare Instructions; really, same as signed except they
11214 // produce an rFlagsRegU instead of rFlagsReg.
11215 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11216 %{
11217   match(Set cr (CmpU op1 op2));
11218 
11219   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11220   opcode(0x3B); /* Opcode 3B /r */
11221   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11222   ins_pipe(ialu_cr_reg_reg);
11223 %}
11224 
11225 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11226 %{
11227   match(Set cr (CmpU op1 op2));
11228 
11229   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11230   opcode(0x81,0x07); /* Opcode 81 /7 */
11231   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11232   ins_pipe(ialu_cr_reg_imm);
11233 %}
11234 
11235 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11236 %{
11237   match(Set cr (CmpU op1 (LoadI op2)));
11238 
11239   ins_cost(500); // XXX
11240   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11241   opcode(0x3B); /* Opcode 3B /r */
11242   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11243   ins_pipe(ialu_cr_reg_mem);
11244 %}
11245 
11246 // // // Cisc-spilled version of cmpU_rReg
11247 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11248 // //%{
11249 // //  match(Set cr (CmpU (LoadI op1) op2));
11250 // //
11251 // //  format %{ "CMPu   $op1,$op2" %}
11252 // //  ins_cost(500);
11253 // //  opcode(0x39);  /* Opcode 39 /r */
11254 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11255 // //%}
11256 
11257 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11258 %{
11259   match(Set cr (CmpU src zero));
11260 
11261   format %{ "testl  $src, $src\t# unsigned" %}
11262   opcode(0x85);
11263   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11264   ins_pipe(ialu_cr_reg_imm);
11265 %}
11266 
11267 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11268 %{
11269   match(Set cr (CmpP op1 op2));
11270 
11271   format %{ "cmpq    $op1, $op2\t# ptr" %}
11272   opcode(0x3B); /* Opcode 3B /r */
11273   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11274   ins_pipe(ialu_cr_reg_reg);
11275 %}
11276 
11277 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11278 %{
11279   match(Set cr (CmpP op1 (LoadP op2)));
11280 
11281   ins_cost(500); // XXX
11282   format %{ "cmpq    $op1, $op2\t# ptr" %}
11283   opcode(0x3B); /* Opcode 3B /r */
11284   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11285   ins_pipe(ialu_cr_reg_mem);
11286 %}
11287 
11288 // // // Cisc-spilled version of cmpP_rReg
11289 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11290 // //%{
11291 // //  match(Set cr (CmpP (LoadP op1) op2));
11292 // //
11293 // //  format %{ "CMPu   $op1,$op2" %}
11294 // //  ins_cost(500);
11295 // //  opcode(0x39);  /* Opcode 39 /r */
11296 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11297 // //%}
11298 
11299 // XXX this is generalized by compP_rReg_mem???
11300 // Compare raw pointer (used in out-of-heap check).
11301 // Only works because non-oop pointers must be raw pointers
11302 // and raw pointers have no anti-dependencies.
11303 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11304 %{
11305   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11306   match(Set cr (CmpP op1 (LoadP op2)));
11307 
11308   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11309   opcode(0x3B); /* Opcode 3B /r */
11310   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11311   ins_pipe(ialu_cr_reg_mem);
11312 %}
11313 
11314 // This will generate a signed flags result. This should be OK since
11315 // any compare to a zero should be eq/neq.
11316 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11317 %{
11318   match(Set cr (CmpP src zero));
11319 
11320   format %{ "testq   $src, $src\t# ptr" %}
11321   opcode(0x85);
11322   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11323   ins_pipe(ialu_cr_reg_imm);
11324 %}
11325 
11326 // This will generate a signed flags result. This should be OK since
11327 // any compare to a zero should be eq/neq.
11328 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11329 %{
11330   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11331   match(Set cr (CmpP (LoadP op) zero));
11332 
11333   ins_cost(500); // XXX
11334   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11335   opcode(0xF7); /* Opcode F7 /0 */
11336   ins_encode(REX_mem_wide(op),
11337              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11338   ins_pipe(ialu_cr_reg_imm);
11339 %}
11340 
11341 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11342 %{
11343   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11344   match(Set cr (CmpP (LoadP mem) zero));
11345 
11346   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11347   ins_encode %{
11348     __ cmpq(r12, $mem$$Address);
11349   %}
11350   ins_pipe(ialu_cr_reg_mem);
11351 %}
11352 
11353 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11354 %{
11355   match(Set cr (CmpN op1 op2));
11356 
11357   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11358   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11359   ins_pipe(ialu_cr_reg_reg);
11360 %}
11361 
11362 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11363 %{
11364   match(Set cr (CmpN src (LoadN mem)));
11365 
11366   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11367   ins_encode %{
11368     __ cmpl($src$$Register, $mem$$Address);
11369   %}
11370   ins_pipe(ialu_cr_reg_mem);
11371 %}
11372 
11373 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11374   match(Set cr (CmpN op1 op2));
11375 
11376   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11377   ins_encode %{
11378     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11379   %}
11380   ins_pipe(ialu_cr_reg_imm);
11381 %}
11382 
11383 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11384 %{
11385   match(Set cr (CmpN src (LoadN mem)));
11386 
11387   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11388   ins_encode %{
11389     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11390   %}
11391   ins_pipe(ialu_cr_reg_mem);
11392 %}
11393 
11394 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11395   match(Set cr (CmpN op1 op2));
11396 
11397   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11398   ins_encode %{
11399     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11400   %}
11401   ins_pipe(ialu_cr_reg_imm);
11402 %}
11403 
11404 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11405 %{
11406   match(Set cr (CmpN src (LoadNKlass mem)));
11407 
11408   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11409   ins_encode %{
11410     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11411   %}
11412   ins_pipe(ialu_cr_reg_mem);
11413 %}
11414 
11415 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11416   match(Set cr (CmpN src zero));
11417 
11418   format %{ "testl   $src, $src\t# compressed ptr" %}
11419   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11420   ins_pipe(ialu_cr_reg_imm);
11421 %}
11422 
11423 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11424 %{
11425   predicate(Universe::narrow_oop_base() != NULL);
11426   match(Set cr (CmpN (LoadN mem) zero));
11427 
11428   ins_cost(500); // XXX
11429   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11430   ins_encode %{
11431     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11432   %}
11433   ins_pipe(ialu_cr_reg_mem);
11434 %}
11435 
11436 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11437 %{
11438   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11439   match(Set cr (CmpN (LoadN mem) zero));
11440 
11441   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11442   ins_encode %{
11443     __ cmpl(r12, $mem$$Address);
11444   %}
11445   ins_pipe(ialu_cr_reg_mem);
11446 %}
11447 
11448 // Yanked all unsigned pointer compare operations.
11449 // Pointer compares are done with CmpP which is already unsigned.
11450 
11451 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11452 %{
11453   match(Set cr (CmpL op1 op2));
11454 
11455   format %{ "cmpq    $op1, $op2" %}
11456   opcode(0x3B);  /* Opcode 3B /r */
11457   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11458   ins_pipe(ialu_cr_reg_reg);
11459 %}
11460 
11461 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11462 %{
11463   match(Set cr (CmpL op1 op2));
11464 
11465   format %{ "cmpq    $op1, $op2" %}
11466   opcode(0x81, 0x07); /* Opcode 81 /7 */
11467   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11468   ins_pipe(ialu_cr_reg_imm);
11469 %}
11470 
11471 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11472 %{
11473   match(Set cr (CmpL op1 (LoadL op2)));
11474 
11475   format %{ "cmpq    $op1, $op2" %}
11476   opcode(0x3B); /* Opcode 3B /r */
11477   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11478   ins_pipe(ialu_cr_reg_mem);
11479 %}
11480 
11481 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11482 %{
11483   match(Set cr (CmpL src zero));
11484 
11485   format %{ "testq   $src, $src" %}
11486   opcode(0x85);
11487   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11488   ins_pipe(ialu_cr_reg_imm);
11489 %}
11490 
11491 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11492 %{
11493   match(Set cr (CmpL (AndL src con) zero));
11494 
11495   format %{ "testq   $src, $con\t# long" %}
11496   opcode(0xF7, 0x00);
11497   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11498   ins_pipe(ialu_cr_reg_imm);
11499 %}
11500 
11501 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11502 %{
11503   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11504 
11505   format %{ "testq   $src, $mem" %}
11506   opcode(0x85);
11507   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11508   ins_pipe(ialu_cr_reg_mem);
11509 %}
11510 
11511 // Manifest a CmpL result in an integer register.  Very painful.
11512 // This is the test to avoid.
11513 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11514 %{
11515   match(Set dst (CmpL3 src1 src2));
11516   effect(KILL flags);
11517 
11518   ins_cost(275); // XXX
11519   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11520             "movl    $dst, -1\n\t"
11521             "jl,s    done\n\t"
11522             "setne   $dst\n\t"
11523             "movzbl  $dst, $dst\n\t"
11524     "done:" %}
11525   ins_encode(cmpl3_flag(src1, src2, dst));
11526   ins_pipe(pipe_slow);
11527 %}
11528 
11529 //----------Max and Min--------------------------------------------------------
11530 // Min Instructions
11531 
11532 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11533 %{
11534   effect(USE_DEF dst, USE src, USE cr);
11535 
11536   format %{ "cmovlgt $dst, $src\t# min" %}
11537   opcode(0x0F, 0x4F);
11538   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11539   ins_pipe(pipe_cmov_reg);
11540 %}
11541 
11542 
11543 instruct minI_rReg(rRegI dst, rRegI src)
11544 %{
11545   match(Set dst (MinI dst src));
11546 
11547   ins_cost(200);
11548   expand %{
11549     rFlagsReg cr;
11550     compI_rReg(cr, dst, src);
11551     cmovI_reg_g(dst, src, cr);
11552   %}
11553 %}
11554 
11555 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11556 %{
11557   effect(USE_DEF dst, USE src, USE cr);
11558 
11559   format %{ "cmovllt $dst, $src\t# max" %}
11560   opcode(0x0F, 0x4C);
11561   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11562   ins_pipe(pipe_cmov_reg);
11563 %}
11564 
11565 
11566 instruct maxI_rReg(rRegI dst, rRegI src)
11567 %{
11568   match(Set dst (MaxI dst src));
11569 
11570   ins_cost(200);
11571   expand %{
11572     rFlagsReg cr;
11573     compI_rReg(cr, dst, src);
11574     cmovI_reg_l(dst, src, cr);
11575   %}
11576 %}
11577 
11578 // ============================================================================
11579 // Branch Instructions
11580 
11581 // Jump Direct - Label defines a relative address from JMP+1
11582 instruct jmpDir(label labl)
11583 %{
11584   match(Goto);
11585   effect(USE labl);
11586 
11587   ins_cost(300);
11588   format %{ "jmp     $labl" %}
11589   size(5);
11590   ins_encode %{
11591     Label* L = $labl$$label;
11592     __ jmp(*L, false); // Always long jump
11593   %}
11594   ins_pipe(pipe_jmp);
11595 %}
11596 
11597 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11598 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11599 %{
11600   match(If cop cr);
11601   effect(USE labl);
11602 
11603   ins_cost(300);
11604   format %{ "j$cop     $labl" %}
11605   size(6);
11606   ins_encode %{
11607     Label* L = $labl$$label;
11608     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11609   %}
11610   ins_pipe(pipe_jcc);
11611 %}
11612 
11613 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11614 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11615 %{
11616   predicate(!n->has_vector_mask_set());
11617   match(CountedLoopEnd cop cr);
11618   effect(USE labl);
11619 
11620   ins_cost(300);
11621   format %{ "j$cop     $labl\t# loop end" %}
11622   size(6);
11623   ins_encode %{
11624     Label* L = $labl$$label;
11625     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11626   %}
11627   ins_pipe(pipe_jcc);
11628 %}
11629 
11630 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11631 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11632   predicate(!n->has_vector_mask_set());
11633   match(CountedLoopEnd cop cmp);
11634   effect(USE labl);
11635 
11636   ins_cost(300);
11637   format %{ "j$cop,u   $labl\t# loop end" %}
11638   size(6);
11639   ins_encode %{
11640     Label* L = $labl$$label;
11641     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11642   %}
11643   ins_pipe(pipe_jcc);
11644 %}
11645 
11646 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11647   predicate(!n->has_vector_mask_set());
11648   match(CountedLoopEnd cop cmp);
11649   effect(USE labl);
11650 
11651   ins_cost(200);
11652   format %{ "j$cop,u   $labl\t# loop end" %}
11653   size(6);
11654   ins_encode %{
11655     Label* L = $labl$$label;
11656     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11657   %}
11658   ins_pipe(pipe_jcc);
11659 %}
11660 
11661 // mask version
11662 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11663 instruct jmpLoopEnd_and_restoreMask(cmpOp cop, rFlagsReg cr, label labl)
11664 %{
11665   predicate(n->has_vector_mask_set());
11666   match(CountedLoopEnd cop cr);
11667   effect(USE labl);
11668 
11669   ins_cost(400);
11670   format %{ "j$cop     $labl\t# loop end\n\t"
11671             "restorevectmask \t# vector mask restore for loops" %}
11672   size(10);
11673   ins_encode %{
11674     Label* L = $labl$$label;
11675     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11676     __ restorevectmask();
11677   %}
11678   ins_pipe(pipe_jcc);
11679 %}
11680 
11681 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11682 instruct jmpLoopEndU_and_restoreMask(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11683   predicate(n->has_vector_mask_set());
11684   match(CountedLoopEnd cop cmp);
11685   effect(USE labl);
11686 
11687   ins_cost(400);
11688   format %{ "j$cop,u   $labl\t# loop end\n\t"
11689             "restorevectmask \t# vector mask restore for loops" %}
11690   size(10);
11691   ins_encode %{
11692     Label* L = $labl$$label;
11693     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11694     __ restorevectmask();
11695   %}
11696   ins_pipe(pipe_jcc);
11697 %}
11698 
11699 instruct jmpLoopEndUCF_and_restoreMask(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11700   predicate(n->has_vector_mask_set());
11701   match(CountedLoopEnd cop cmp);
11702   effect(USE labl);
11703 
11704   ins_cost(300);
11705   format %{ "j$cop,u   $labl\t# loop end\n\t"
11706             "restorevectmask \t# vector mask restore for loops" %}
11707   size(10);
11708   ins_encode %{
11709     Label* L = $labl$$label;
11710     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11711     __ restorevectmask();
11712   %}
11713   ins_pipe(pipe_jcc);
11714 %}
11715 
11716 // Jump Direct Conditional - using unsigned comparison
11717 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11718   match(If cop cmp);
11719   effect(USE labl);
11720 
11721   ins_cost(300);
11722   format %{ "j$cop,u  $labl" %}
11723   size(6);
11724   ins_encode %{
11725     Label* L = $labl$$label;
11726     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11727   %}
11728   ins_pipe(pipe_jcc);
11729 %}
11730 
11731 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11732   match(If cop cmp);
11733   effect(USE labl);
11734 
11735   ins_cost(200);
11736   format %{ "j$cop,u  $labl" %}
11737   size(6);
11738   ins_encode %{
11739     Label* L = $labl$$label;
11740     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11741   %}
11742   ins_pipe(pipe_jcc);
11743 %}
11744 
11745 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11746   match(If cop cmp);
11747   effect(USE labl);
11748 
11749   ins_cost(200);
11750   format %{ $$template
11751     if ($cop$$cmpcode == Assembler::notEqual) {
11752       $$emit$$"jp,u   $labl\n\t"
11753       $$emit$$"j$cop,u   $labl"
11754     } else {
11755       $$emit$$"jp,u   done\n\t"
11756       $$emit$$"j$cop,u   $labl\n\t"
11757       $$emit$$"done:"
11758     }
11759   %}
11760   ins_encode %{
11761     Label* l = $labl$$label;
11762     if ($cop$$cmpcode == Assembler::notEqual) {
11763       __ jcc(Assembler::parity, *l, false);
11764       __ jcc(Assembler::notEqual, *l, false);
11765     } else if ($cop$$cmpcode == Assembler::equal) {
11766       Label done;
11767       __ jccb(Assembler::parity, done);
11768       __ jcc(Assembler::equal, *l, false);
11769       __ bind(done);
11770     } else {
11771        ShouldNotReachHere();
11772     }
11773   %}
11774   ins_pipe(pipe_jcc);
11775 %}
11776 
11777 // ============================================================================
11778 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11779 // superklass array for an instance of the superklass.  Set a hidden
11780 // internal cache on a hit (cache is checked with exposed code in
11781 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11782 // encoding ALSO sets flags.
11783 
11784 instruct partialSubtypeCheck(rdi_RegP result,
11785                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11786                              rFlagsReg cr)
11787 %{
11788   match(Set result (PartialSubtypeCheck sub super));
11789   effect(KILL rcx, KILL cr);
11790 
11791   ins_cost(1100);  // slightly larger than the next version
11792   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11793             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11794             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11795             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11796             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11797             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11798             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11799     "miss:\t" %}
11800 
11801   opcode(0x1); // Force a XOR of RDI
11802   ins_encode(enc_PartialSubtypeCheck());
11803   ins_pipe(pipe_slow);
11804 %}
11805 
11806 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11807                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11808                                      immP0 zero,
11809                                      rdi_RegP result)
11810 %{
11811   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11812   effect(KILL rcx, KILL result);
11813 
11814   ins_cost(1000);
11815   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11816             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11817             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11818             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11819             "jne,s   miss\t\t# Missed: flags nz\n\t"
11820             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11821     "miss:\t" %}
11822 
11823   opcode(0x0); // No need to XOR RDI
11824   ins_encode(enc_PartialSubtypeCheck());
11825   ins_pipe(pipe_slow);
11826 %}
11827 
11828 // ============================================================================
11829 // Branch Instructions -- short offset versions
11830 //
11831 // These instructions are used to replace jumps of a long offset (the default
11832 // match) with jumps of a shorter offset.  These instructions are all tagged
11833 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11834 // match rules in general matching.  Instead, the ADLC generates a conversion
11835 // method in the MachNode which can be used to do in-place replacement of the
11836 // long variant with the shorter variant.  The compiler will determine if a
11837 // branch can be taken by the is_short_branch_offset() predicate in the machine
11838 // specific code section of the file.
11839 
11840 // Jump Direct - Label defines a relative address from JMP+1
11841 instruct jmpDir_short(label labl) %{
11842   match(Goto);
11843   effect(USE labl);
11844 
11845   ins_cost(300);
11846   format %{ "jmp,s   $labl" %}
11847   size(2);
11848   ins_encode %{
11849     Label* L = $labl$$label;
11850     __ jmpb(*L);
11851   %}
11852   ins_pipe(pipe_jmp);
11853   ins_short_branch(1);
11854 %}
11855 
11856 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11857 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11858   match(If cop cr);
11859   effect(USE labl);
11860 
11861   ins_cost(300);
11862   format %{ "j$cop,s   $labl" %}
11863   size(2);
11864   ins_encode %{
11865     Label* L = $labl$$label;
11866     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11867   %}
11868   ins_pipe(pipe_jcc);
11869   ins_short_branch(1);
11870 %}
11871 
11872 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11873 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11874   match(CountedLoopEnd cop cr);
11875   effect(USE labl);
11876 
11877   ins_cost(300);
11878   format %{ "j$cop,s   $labl\t# loop end" %}
11879   size(2);
11880   ins_encode %{
11881     Label* L = $labl$$label;
11882     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11883   %}
11884   ins_pipe(pipe_jcc);
11885   ins_short_branch(1);
11886 %}
11887 
11888 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11889 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11890   match(CountedLoopEnd cop cmp);
11891   effect(USE labl);
11892 
11893   ins_cost(300);
11894   format %{ "j$cop,us  $labl\t# loop end" %}
11895   size(2);
11896   ins_encode %{
11897     Label* L = $labl$$label;
11898     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11899   %}
11900   ins_pipe(pipe_jcc);
11901   ins_short_branch(1);
11902 %}
11903 
11904 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11905   match(CountedLoopEnd cop cmp);
11906   effect(USE labl);
11907 
11908   ins_cost(300);
11909   format %{ "j$cop,us  $labl\t# loop end" %}
11910   size(2);
11911   ins_encode %{
11912     Label* L = $labl$$label;
11913     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11914   %}
11915   ins_pipe(pipe_jcc);
11916   ins_short_branch(1);
11917 %}
11918 
11919 // Jump Direct Conditional - using unsigned comparison
11920 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11921   match(If cop cmp);
11922   effect(USE labl);
11923 
11924   ins_cost(300);
11925   format %{ "j$cop,us  $labl" %}
11926   size(2);
11927   ins_encode %{
11928     Label* L = $labl$$label;
11929     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11930   %}
11931   ins_pipe(pipe_jcc);
11932   ins_short_branch(1);
11933 %}
11934 
11935 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11936   match(If cop cmp);
11937   effect(USE labl);
11938 
11939   ins_cost(300);
11940   format %{ "j$cop,us  $labl" %}
11941   size(2);
11942   ins_encode %{
11943     Label* L = $labl$$label;
11944     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11945   %}
11946   ins_pipe(pipe_jcc);
11947   ins_short_branch(1);
11948 %}
11949 
11950 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11951   match(If cop cmp);
11952   effect(USE labl);
11953 
11954   ins_cost(300);
11955   format %{ $$template
11956     if ($cop$$cmpcode == Assembler::notEqual) {
11957       $$emit$$"jp,u,s   $labl\n\t"
11958       $$emit$$"j$cop,u,s   $labl"
11959     } else {
11960       $$emit$$"jp,u,s   done\n\t"
11961       $$emit$$"j$cop,u,s  $labl\n\t"
11962       $$emit$$"done:"
11963     }
11964   %}
11965   size(4);
11966   ins_encode %{
11967     Label* l = $labl$$label;
11968     if ($cop$$cmpcode == Assembler::notEqual) {
11969       __ jccb(Assembler::parity, *l);
11970       __ jccb(Assembler::notEqual, *l);
11971     } else if ($cop$$cmpcode == Assembler::equal) {
11972       Label done;
11973       __ jccb(Assembler::parity, done);
11974       __ jccb(Assembler::equal, *l);
11975       __ bind(done);
11976     } else {
11977        ShouldNotReachHere();
11978     }
11979   %}
11980   ins_pipe(pipe_jcc);
11981   ins_short_branch(1);
11982 %}
11983 
11984 // ============================================================================
11985 // inlined locking and unlocking
11986 
11987 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11988   predicate(Compile::current()->use_rtm());
11989   match(Set cr (FastLock object box));
11990   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11991   ins_cost(300);
11992   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11993   ins_encode %{
11994     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11995                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11996                  _counters, _rtm_counters, _stack_rtm_counters,
11997                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11998                  true, ra_->C->profile_rtm());
11999   %}
12000   ins_pipe(pipe_slow);
12001 %}
12002 
12003 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
12004   predicate(!Compile::current()->use_rtm());
12005   match(Set cr (FastLock object box));
12006   effect(TEMP tmp, TEMP scr, USE_KILL box);
12007   ins_cost(300);
12008   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
12009   ins_encode %{
12010     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
12011                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
12012   %}
12013   ins_pipe(pipe_slow);
12014 %}
12015 
12016 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
12017   match(Set cr (FastUnlock object box));
12018   effect(TEMP tmp, USE_KILL box);
12019   ins_cost(300);
12020   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
12021   ins_encode %{
12022     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
12023   %}
12024   ins_pipe(pipe_slow);
12025 %}
12026 
12027 
12028 // ============================================================================
12029 // Safepoint Instructions
12030 instruct safePoint_poll(rFlagsReg cr)
12031 %{
12032   predicate(!Assembler::is_polling_page_far());
12033   match(SafePoint);
12034   effect(KILL cr);
12035 
12036   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
12037             "# Safepoint: poll for GC" %}
12038   ins_cost(125);
12039   ins_encode %{
12040     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
12041     __ testl(rax, addr);
12042   %}
12043   ins_pipe(ialu_reg_mem);
12044 %}
12045 
12046 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
12047 %{
12048   predicate(Assembler::is_polling_page_far());
12049   match(SafePoint poll);
12050   effect(KILL cr, USE poll);
12051 
12052   format %{ "testl  rax, [$poll]\t"
12053             "# Safepoint: poll for GC" %}
12054   ins_cost(125);
12055   ins_encode %{
12056     __ relocate(relocInfo::poll_type);
12057     __ testl(rax, Address($poll$$Register, 0));
12058   %}
12059   ins_pipe(ialu_reg_mem);
12060 %}
12061 
12062 // ============================================================================
12063 // Procedure Call/Return Instructions
12064 // Call Java Static Instruction
12065 // Note: If this code changes, the corresponding ret_addr_offset() and
12066 //       compute_padding() functions will have to be adjusted.
12067 instruct CallStaticJavaDirect(method meth) %{
12068   match(CallStaticJava);
12069   effect(USE meth);
12070 
12071   ins_cost(300);
12072   format %{ "call,static " %}
12073   opcode(0xE8); /* E8 cd */
12074   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
12075   ins_pipe(pipe_slow);
12076   ins_alignment(4);
12077 %}
12078 
12079 // Call Java Dynamic Instruction
12080 // Note: If this code changes, the corresponding ret_addr_offset() and
12081 //       compute_padding() functions will have to be adjusted.
12082 instruct CallDynamicJavaDirect(method meth)
12083 %{
12084   match(CallDynamicJava);
12085   effect(USE meth);
12086 
12087   ins_cost(300);
12088   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
12089             "call,dynamic " %}
12090   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
12091   ins_pipe(pipe_slow);
12092   ins_alignment(4);
12093 %}
12094 
12095 // Call Runtime Instruction
12096 instruct CallRuntimeDirect(method meth)
12097 %{
12098   match(CallRuntime);
12099   effect(USE meth);
12100 
12101   ins_cost(300);
12102   format %{ "call,runtime " %}
12103   ins_encode(clear_avx, Java_To_Runtime(meth));
12104   ins_pipe(pipe_slow);
12105 %}
12106 
12107 // Call runtime without safepoint
12108 instruct CallLeafDirect(method meth)
12109 %{
12110   match(CallLeaf);
12111   effect(USE meth);
12112 
12113   ins_cost(300);
12114   format %{ "call_leaf,runtime " %}
12115   ins_encode(clear_avx, Java_To_Runtime(meth));
12116   ins_pipe(pipe_slow);
12117 %}
12118 
12119 // Call runtime without safepoint
12120 instruct CallLeafNoFPDirect(method meth)
12121 %{
12122   match(CallLeafNoFP);
12123   effect(USE meth);
12124 
12125   ins_cost(300);
12126   format %{ "call_leaf_nofp,runtime " %}
12127   ins_encode(Java_To_Runtime(meth));
12128   ins_pipe(pipe_slow);
12129 %}
12130 
12131 // Return Instruction
12132 // Remove the return address & jump to it.
12133 // Notice: We always emit a nop after a ret to make sure there is room
12134 // for safepoint patching
12135 instruct Ret()
12136 %{
12137   match(Return);
12138 
12139   format %{ "ret" %}
12140   opcode(0xC3);
12141   ins_encode(OpcP);
12142   ins_pipe(pipe_jmp);
12143 %}
12144 
12145 // Tail Call; Jump from runtime stub to Java code.
12146 // Also known as an 'interprocedural jump'.
12147 // Target of jump will eventually return to caller.
12148 // TailJump below removes the return address.
12149 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
12150 %{
12151   match(TailCall jump_target method_oop);
12152 
12153   ins_cost(300);
12154   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
12155   opcode(0xFF, 0x4); /* Opcode FF /4 */
12156   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
12157   ins_pipe(pipe_jmp);
12158 %}
12159 
12160 // Tail Jump; remove the return address; jump to target.
12161 // TailCall above leaves the return address around.
12162 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
12163 %{
12164   match(TailJump jump_target ex_oop);
12165 
12166   ins_cost(300);
12167   format %{ "popq    rdx\t# pop return address\n\t"
12168             "jmp     $jump_target" %}
12169   opcode(0xFF, 0x4); /* Opcode FF /4 */
12170   ins_encode(Opcode(0x5a), // popq rdx
12171              REX_reg(jump_target), OpcP, reg_opc(jump_target));
12172   ins_pipe(pipe_jmp);
12173 %}
12174 
12175 // Create exception oop: created by stack-crawling runtime code.
12176 // Created exception is now available to this handler, and is setup
12177 // just prior to jumping to this handler.  No code emitted.
12178 instruct CreateException(rax_RegP ex_oop)
12179 %{
12180   match(Set ex_oop (CreateEx));
12181 
12182   size(0);
12183   // use the following format syntax
12184   format %{ "# exception oop is in rax; no code emitted" %}
12185   ins_encode();
12186   ins_pipe(empty);
12187 %}
12188 
12189 // Rethrow exception:
12190 // The exception oop will come in the first argument position.
12191 // Then JUMP (not call) to the rethrow stub code.
12192 instruct RethrowException()
12193 %{
12194   match(Rethrow);
12195 
12196   // use the following format syntax
12197   format %{ "jmp     rethrow_stub" %}
12198   ins_encode(enc_rethrow);
12199   ins_pipe(pipe_jmp);
12200 %}
12201 
12202 
12203 // ============================================================================
12204 // This name is KNOWN by the ADLC and cannot be changed.
12205 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12206 // for this guy.
12207 instruct tlsLoadP(r15_RegP dst) %{
12208   match(Set dst (ThreadLocal));
12209   effect(DEF dst);
12210 
12211   size(0);
12212   format %{ "# TLS is in R15" %}
12213   ins_encode( /*empty encoding*/ );
12214   ins_pipe(ialu_reg_reg);
12215 %}
12216 
12217 
12218 //----------PEEPHOLE RULES-----------------------------------------------------
12219 // These must follow all instruction definitions as they use the names
12220 // defined in the instructions definitions.
12221 //
12222 // peepmatch ( root_instr_name [preceding_instruction]* );
12223 //
12224 // peepconstraint %{
12225 // (instruction_number.operand_name relational_op instruction_number.operand_name
12226 //  [, ...] );
12227 // // instruction numbers are zero-based using left to right order in peepmatch
12228 //
12229 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12230 // // provide an instruction_number.operand_name for each operand that appears
12231 // // in the replacement instruction's match rule
12232 //
12233 // ---------VM FLAGS---------------------------------------------------------
12234 //
12235 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12236 //
12237 // Each peephole rule is given an identifying number starting with zero and
12238 // increasing by one in the order seen by the parser.  An individual peephole
12239 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12240 // on the command-line.
12241 //
12242 // ---------CURRENT LIMITATIONS----------------------------------------------
12243 //
12244 // Only match adjacent instructions in same basic block
12245 // Only equality constraints
12246 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12247 // Only one replacement instruction
12248 //
12249 // ---------EXAMPLE----------------------------------------------------------
12250 //
12251 // // pertinent parts of existing instructions in architecture description
12252 // instruct movI(rRegI dst, rRegI src)
12253 // %{
12254 //   match(Set dst (CopyI src));
12255 // %}
12256 //
12257 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12258 // %{
12259 //   match(Set dst (AddI dst src));
12260 //   effect(KILL cr);
12261 // %}
12262 //
12263 // // Change (inc mov) to lea
12264 // peephole %{
12265 //   // increment preceeded by register-register move
12266 //   peepmatch ( incI_rReg movI );
12267 //   // require that the destination register of the increment
12268 //   // match the destination register of the move
12269 //   peepconstraint ( 0.dst == 1.dst );
12270 //   // construct a replacement instruction that sets
12271 //   // the destination to ( move's source register + one )
12272 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12273 // %}
12274 //
12275 
12276 // Implementation no longer uses movX instructions since
12277 // machine-independent system no longer uses CopyX nodes.
12278 //
12279 // peephole
12280 // %{
12281 //   peepmatch (incI_rReg movI);
12282 //   peepconstraint (0.dst == 1.dst);
12283 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12284 // %}
12285 
12286 // peephole
12287 // %{
12288 //   peepmatch (decI_rReg movI);
12289 //   peepconstraint (0.dst == 1.dst);
12290 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12291 // %}
12292 
12293 // peephole
12294 // %{
12295 //   peepmatch (addI_rReg_imm movI);
12296 //   peepconstraint (0.dst == 1.dst);
12297 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12298 // %}
12299 
12300 // peephole
12301 // %{
12302 //   peepmatch (incL_rReg movL);
12303 //   peepconstraint (0.dst == 1.dst);
12304 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12305 // %}
12306 
12307 // peephole
12308 // %{
12309 //   peepmatch (decL_rReg movL);
12310 //   peepconstraint (0.dst == 1.dst);
12311 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12312 // %}
12313 
12314 // peephole
12315 // %{
12316 //   peepmatch (addL_rReg_imm movL);
12317 //   peepconstraint (0.dst == 1.dst);
12318 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12319 // %}
12320 
12321 // peephole
12322 // %{
12323 //   peepmatch (addP_rReg_imm movP);
12324 //   peepconstraint (0.dst == 1.dst);
12325 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12326 // %}
12327 
12328 // // Change load of spilled value to only a spill
12329 // instruct storeI(memory mem, rRegI src)
12330 // %{
12331 //   match(Set mem (StoreI mem src));
12332 // %}
12333 //
12334 // instruct loadI(rRegI dst, memory mem)
12335 // %{
12336 //   match(Set dst (LoadI mem));
12337 // %}
12338 //
12339 
12340 peephole
12341 %{
12342   peepmatch (loadI storeI);
12343   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12344   peepreplace (storeI(1.mem 1.mem 1.src));
12345 %}
12346 
12347 peephole
12348 %{
12349   peepmatch (loadL storeL);
12350   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12351   peepreplace (storeL(1.mem 1.mem 1.src));
12352 %}
12353 
12354 //----------SMARTSPILL RULES---------------------------------------------------
12355 // These must follow all instruction definitions as they use the names
12356 // defined in the instructions definitions.