1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   MacroAssembler _masm(&cbuf);
 957 
 958   if (C->max_vector_size() > 16) {
 959     // Clear upper bits of YMM registers when current compiled code uses
 960     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 961     __ vzeroupper();
 962   }
 963 
 964   int framesize = C->frame_size_in_bytes();
 965   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 966   // Remove word for return adr already pushed
 967   // and RBP
 968   framesize -= 2*wordSize;
 969 
 970   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 971 
 972   if (framesize) {
 973     emit_opcode(cbuf, Assembler::REX_W);
 974     if (framesize < 0x80) {
 975       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 976       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 977       emit_d8(cbuf, framesize);
 978     } else {
 979       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 980       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 981       emit_d32(cbuf, framesize);
 982     }
 983   }
 984 
 985   // popq rbp
 986   emit_opcode(cbuf, 0x58 | RBP_enc);
 987 
 988   if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
 989     __ reserved_stack_check(); 
 990   }
 991 
 992   if (do_polling() && C->is_method_compilation()) {
 993     MacroAssembler _masm(&cbuf);
 994     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 995     if (Assembler::is_polling_page_far()) {
 996       __ lea(rscratch1, polling_page);
 997       __ relocate(relocInfo::poll_return_type);
 998       __ testl(rax, Address(rscratch1, 0));
 999     } else {
1000       __ testl(rax, polling_page);
1001     }
1002   }
1003 }
1004 
1005 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1006 {
1007   return MachNode::size(ra_); // too many variables; just compute it
1008                               // the hard way
1009 }
1010 
1011 int MachEpilogNode::reloc() const
1012 {
1013   return 2; // a large enough number
1014 }
1015 
1016 const Pipeline* MachEpilogNode::pipeline() const
1017 {
1018   return MachNode::pipeline_class();
1019 }
1020 
1021 int MachEpilogNode::safepoint_offset() const
1022 {
1023   return 0;
1024 }
1025 
1026 //=============================================================================
1027 
1028 enum RC {
1029   rc_bad,
1030   rc_int,
1031   rc_float,
1032   rc_stack
1033 };
1034 
1035 static enum RC rc_class(OptoReg::Name reg)
1036 {
1037   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1038 
1039   if (OptoReg::is_stack(reg)) return rc_stack;
1040 
1041   VMReg r = OptoReg::as_VMReg(reg);
1042 
1043   if (r->is_Register()) return rc_int;
1044 
1045   assert(r->is_XMMRegister(), "must be");
1046   return rc_float;
1047 }
1048 
1049 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1050 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1051                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1052 
1053 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1054                             int stack_offset, int reg, uint ireg, outputStream* st);
1055 
1056 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1057                                       int dst_offset, uint ireg, outputStream* st) {
1058   if (cbuf) {
1059     MacroAssembler _masm(cbuf);
1060     switch (ireg) {
1061     case Op_VecS:
1062       __ movq(Address(rsp, -8), rax);
1063       __ movl(rax, Address(rsp, src_offset));
1064       __ movl(Address(rsp, dst_offset), rax);
1065       __ movq(rax, Address(rsp, -8));
1066       break;
1067     case Op_VecD:
1068       __ pushq(Address(rsp, src_offset));
1069       __ popq (Address(rsp, dst_offset));
1070       break;
1071     case Op_VecX:
1072       __ pushq(Address(rsp, src_offset));
1073       __ popq (Address(rsp, dst_offset));
1074       __ pushq(Address(rsp, src_offset+8));
1075       __ popq (Address(rsp, dst_offset+8));
1076       break;
1077     case Op_VecY:
1078       __ vmovdqu(Address(rsp, -32), xmm0);
1079       __ vmovdqu(xmm0, Address(rsp, src_offset));
1080       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1081       __ vmovdqu(xmm0, Address(rsp, -32));
1082       break;
1083     case Op_VecZ:
1084       __ evmovdquq(Address(rsp, -64), xmm0, 2);
1085       __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
1086       __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
1087       __ evmovdquq(xmm0, Address(rsp, -64), 2);
1088       break;
1089     default:
1090       ShouldNotReachHere();
1091     }
1092 #ifndef PRODUCT
1093   } else {
1094     switch (ireg) {
1095     case Op_VecS:
1096       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1097                 "movl    rax, [rsp + #%d]\n\t"
1098                 "movl    [rsp + #%d], rax\n\t"
1099                 "movq    rax, [rsp - #8]",
1100                 src_offset, dst_offset);
1101       break;
1102     case Op_VecD:
1103       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1104                 "popq    [rsp + #%d]",
1105                 src_offset, dst_offset);
1106       break;
1107      case Op_VecX:
1108       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1109                 "popq    [rsp + #%d]\n\t"
1110                 "pushq   [rsp + #%d]\n\t"
1111                 "popq    [rsp + #%d]",
1112                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1113       break;
1114     case Op_VecY:
1115       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1116                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1117                 "vmovdqu [rsp + #%d], xmm0\n\t"
1118                 "vmovdqu xmm0, [rsp - #32]",
1119                 src_offset, dst_offset);
1120       break;
1121     case Op_VecZ:
1122       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1123                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1124                 "vmovdqu [rsp + #%d], xmm0\n\t"
1125                 "vmovdqu xmm0, [rsp - #64]",
1126                 src_offset, dst_offset);
1127       break;
1128     default:
1129       ShouldNotReachHere();
1130     }
1131 #endif
1132   }
1133 }
1134 
1135 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1136                                        PhaseRegAlloc* ra_,
1137                                        bool do_size,
1138                                        outputStream* st) const {
1139   assert(cbuf != NULL || st  != NULL, "sanity");
1140   // Get registers to move
1141   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1142   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1143   OptoReg::Name dst_second = ra_->get_reg_second(this);
1144   OptoReg::Name dst_first = ra_->get_reg_first(this);
1145 
1146   enum RC src_second_rc = rc_class(src_second);
1147   enum RC src_first_rc = rc_class(src_first);
1148   enum RC dst_second_rc = rc_class(dst_second);
1149   enum RC dst_first_rc = rc_class(dst_first);
1150 
1151   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1152          "must move at least 1 register" );
1153 
1154   if (src_first == dst_first && src_second == dst_second) {
1155     // Self copy, no move
1156     return 0;
1157   }
1158   if (bottom_type()->isa_vect() != NULL) {
1159     uint ireg = ideal_reg();
1160     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1161     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1162     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1163       // mem -> mem
1164       int src_offset = ra_->reg2offset(src_first);
1165       int dst_offset = ra_->reg2offset(dst_first);
1166       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1167     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1168       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1169     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1170       int stack_offset = ra_->reg2offset(dst_first);
1171       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1172     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1173       int stack_offset = ra_->reg2offset(src_first);
1174       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1175     } else {
1176       ShouldNotReachHere();
1177     }
1178     return 0;
1179   }
1180   if (src_first_rc == rc_stack) {
1181     // mem ->
1182     if (dst_first_rc == rc_stack) {
1183       // mem -> mem
1184       assert(src_second != dst_first, "overlap");
1185       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1186           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1187         // 64-bit
1188         int src_offset = ra_->reg2offset(src_first);
1189         int dst_offset = ra_->reg2offset(dst_first);
1190         if (cbuf) {
1191           MacroAssembler _masm(cbuf);
1192           __ pushq(Address(rsp, src_offset));
1193           __ popq (Address(rsp, dst_offset));
1194 #ifndef PRODUCT
1195         } else {
1196           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1197                     "popq    [rsp + #%d]",
1198                      src_offset, dst_offset);
1199 #endif
1200         }
1201       } else {
1202         // 32-bit
1203         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1204         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1205         // No pushl/popl, so:
1206         int src_offset = ra_->reg2offset(src_first);
1207         int dst_offset = ra_->reg2offset(dst_first);
1208         if (cbuf) {
1209           MacroAssembler _masm(cbuf);
1210           __ movq(Address(rsp, -8), rax);
1211           __ movl(rax, Address(rsp, src_offset));
1212           __ movl(Address(rsp, dst_offset), rax);
1213           __ movq(rax, Address(rsp, -8));
1214 #ifndef PRODUCT
1215         } else {
1216           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1217                     "movl    rax, [rsp + #%d]\n\t"
1218                     "movl    [rsp + #%d], rax\n\t"
1219                     "movq    rax, [rsp - #8]",
1220                      src_offset, dst_offset);
1221 #endif
1222         }
1223       }
1224       return 0;
1225     } else if (dst_first_rc == rc_int) {
1226       // mem -> gpr
1227       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1228           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1229         // 64-bit
1230         int offset = ra_->reg2offset(src_first);
1231         if (cbuf) {
1232           MacroAssembler _masm(cbuf);
1233           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1234 #ifndef PRODUCT
1235         } else {
1236           st->print("movq    %s, [rsp + #%d]\t# spill",
1237                      Matcher::regName[dst_first],
1238                      offset);
1239 #endif
1240         }
1241       } else {
1242         // 32-bit
1243         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1244         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1245         int offset = ra_->reg2offset(src_first);
1246         if (cbuf) {
1247           MacroAssembler _masm(cbuf);
1248           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1249 #ifndef PRODUCT
1250         } else {
1251           st->print("movl    %s, [rsp + #%d]\t# spill",
1252                      Matcher::regName[dst_first],
1253                      offset);
1254 #endif
1255         }
1256       }
1257       return 0;
1258     } else if (dst_first_rc == rc_float) {
1259       // mem-> xmm
1260       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1261           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1262         // 64-bit
1263         int offset = ra_->reg2offset(src_first);
1264         if (cbuf) {
1265           MacroAssembler _masm(cbuf);
1266           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1267 #ifndef PRODUCT
1268         } else {
1269           st->print("%s  %s, [rsp + #%d]\t# spill",
1270                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1271                      Matcher::regName[dst_first],
1272                      offset);
1273 #endif
1274         }
1275       } else {
1276         // 32-bit
1277         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1278         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1279         int offset = ra_->reg2offset(src_first);
1280         if (cbuf) {
1281           MacroAssembler _masm(cbuf);
1282           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1283 #ifndef PRODUCT
1284         } else {
1285           st->print("movss   %s, [rsp + #%d]\t# spill",
1286                      Matcher::regName[dst_first],
1287                      offset);
1288 #endif
1289         }
1290       }
1291       return 0;
1292     }
1293   } else if (src_first_rc == rc_int) {
1294     // gpr ->
1295     if (dst_first_rc == rc_stack) {
1296       // gpr -> mem
1297       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1298           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1299         // 64-bit
1300         int offset = ra_->reg2offset(dst_first);
1301         if (cbuf) {
1302           MacroAssembler _masm(cbuf);
1303           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1304 #ifndef PRODUCT
1305         } else {
1306           st->print("movq    [rsp + #%d], %s\t# spill",
1307                      offset,
1308                      Matcher::regName[src_first]);
1309 #endif
1310         }
1311       } else {
1312         // 32-bit
1313         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1314         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1315         int offset = ra_->reg2offset(dst_first);
1316         if (cbuf) {
1317           MacroAssembler _masm(cbuf);
1318           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1319 #ifndef PRODUCT
1320         } else {
1321           st->print("movl    [rsp + #%d], %s\t# spill",
1322                      offset,
1323                      Matcher::regName[src_first]);
1324 #endif
1325         }
1326       }
1327       return 0;
1328     } else if (dst_first_rc == rc_int) {
1329       // gpr -> gpr
1330       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1331           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1332         // 64-bit
1333         if (cbuf) {
1334           MacroAssembler _masm(cbuf);
1335           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1336                   as_Register(Matcher::_regEncode[src_first]));
1337 #ifndef PRODUCT
1338         } else {
1339           st->print("movq    %s, %s\t# spill",
1340                      Matcher::regName[dst_first],
1341                      Matcher::regName[src_first]);
1342 #endif
1343         }
1344         return 0;
1345       } else {
1346         // 32-bit
1347         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1348         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1349         if (cbuf) {
1350           MacroAssembler _masm(cbuf);
1351           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1352                   as_Register(Matcher::_regEncode[src_first]));
1353 #ifndef PRODUCT
1354         } else {
1355           st->print("movl    %s, %s\t# spill",
1356                      Matcher::regName[dst_first],
1357                      Matcher::regName[src_first]);
1358 #endif
1359         }
1360         return 0;
1361       }
1362     } else if (dst_first_rc == rc_float) {
1363       // gpr -> xmm
1364       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1365           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1366         // 64-bit
1367         if (cbuf) {
1368           MacroAssembler _masm(cbuf);
1369           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1370 #ifndef PRODUCT
1371         } else {
1372           st->print("movdq   %s, %s\t# spill",
1373                      Matcher::regName[dst_first],
1374                      Matcher::regName[src_first]);
1375 #endif
1376         }
1377       } else {
1378         // 32-bit
1379         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1380         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1381         if (cbuf) {
1382           MacroAssembler _masm(cbuf);
1383           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1384 #ifndef PRODUCT
1385         } else {
1386           st->print("movdl   %s, %s\t# spill",
1387                      Matcher::regName[dst_first],
1388                      Matcher::regName[src_first]);
1389 #endif
1390         }
1391       }
1392       return 0;
1393     }
1394   } else if (src_first_rc == rc_float) {
1395     // xmm ->
1396     if (dst_first_rc == rc_stack) {
1397       // xmm -> mem
1398       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1399           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1400         // 64-bit
1401         int offset = ra_->reg2offset(dst_first);
1402         if (cbuf) {
1403           MacroAssembler _masm(cbuf);
1404           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1405 #ifndef PRODUCT
1406         } else {
1407           st->print("movsd   [rsp + #%d], %s\t# spill",
1408                      offset,
1409                      Matcher::regName[src_first]);
1410 #endif
1411         }
1412       } else {
1413         // 32-bit
1414         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1415         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1416         int offset = ra_->reg2offset(dst_first);
1417         if (cbuf) {
1418           MacroAssembler _masm(cbuf);
1419           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1420 #ifndef PRODUCT
1421         } else {
1422           st->print("movss   [rsp + #%d], %s\t# spill",
1423                      offset,
1424                      Matcher::regName[src_first]);
1425 #endif
1426         }
1427       }
1428       return 0;
1429     } else if (dst_first_rc == rc_int) {
1430       // xmm -> gpr
1431       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1432           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1433         // 64-bit
1434         if (cbuf) {
1435           MacroAssembler _masm(cbuf);
1436           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1437 #ifndef PRODUCT
1438         } else {
1439           st->print("movdq   %s, %s\t# spill",
1440                      Matcher::regName[dst_first],
1441                      Matcher::regName[src_first]);
1442 #endif
1443         }
1444       } else {
1445         // 32-bit
1446         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1447         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1448         if (cbuf) {
1449           MacroAssembler _masm(cbuf);
1450           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1451 #ifndef PRODUCT
1452         } else {
1453           st->print("movdl   %s, %s\t# spill",
1454                      Matcher::regName[dst_first],
1455                      Matcher::regName[src_first]);
1456 #endif
1457         }
1458       }
1459       return 0;
1460     } else if (dst_first_rc == rc_float) {
1461       // xmm -> xmm
1462       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1463           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1464         // 64-bit
1465         if (cbuf) {
1466           MacroAssembler _masm(cbuf);
1467           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1468 #ifndef PRODUCT
1469         } else {
1470           st->print("%s  %s, %s\t# spill",
1471                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1472                      Matcher::regName[dst_first],
1473                      Matcher::regName[src_first]);
1474 #endif
1475         }
1476       } else {
1477         // 32-bit
1478         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1479         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1480         if (cbuf) {
1481           MacroAssembler _masm(cbuf);
1482           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1483 #ifndef PRODUCT
1484         } else {
1485           st->print("%s  %s, %s\t# spill",
1486                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1487                      Matcher::regName[dst_first],
1488                      Matcher::regName[src_first]);
1489 #endif
1490         }
1491       }
1492       return 0;
1493     }
1494   }
1495 
1496   assert(0," foo ");
1497   Unimplemented();
1498   return 0;
1499 }
1500 
1501 #ifndef PRODUCT
1502 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1503   implementation(NULL, ra_, false, st);
1504 }
1505 #endif
1506 
1507 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1508   implementation(&cbuf, ra_, false, NULL);
1509 }
1510 
1511 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1512   return MachNode::size(ra_);
1513 }
1514 
1515 //=============================================================================
1516 #ifndef PRODUCT
1517 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1518 {
1519   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1520   int reg = ra_->get_reg_first(this);
1521   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1522             Matcher::regName[reg], offset);
1523 }
1524 #endif
1525 
1526 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1527 {
1528   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1529   int reg = ra_->get_encode(this);
1530   if (offset >= 0x80) {
1531     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1532     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1533     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1534     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1535     emit_d32(cbuf, offset);
1536   } else {
1537     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1538     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1539     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1540     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1541     emit_d8(cbuf, offset);
1542   }
1543 }
1544 
1545 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1546 {
1547   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1548   return (offset < 0x80) ? 5 : 8; // REX
1549 }
1550 
1551 //=============================================================================
1552 #ifndef PRODUCT
1553 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1554 {
1555   if (UseCompressedClassPointers) {
1556     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1557     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1558     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1559   } else {
1560     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1561                  "# Inline cache check");
1562   }
1563   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1564   st->print_cr("\tnop\t# nops to align entry point");
1565 }
1566 #endif
1567 
1568 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1569 {
1570   MacroAssembler masm(&cbuf);
1571   uint insts_size = cbuf.insts_size();
1572   if (UseCompressedClassPointers) {
1573     masm.load_klass(rscratch1, j_rarg0);
1574     masm.cmpptr(rax, rscratch1);
1575   } else {
1576     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1577   }
1578 
1579   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1580 
1581   /* WARNING these NOPs are critical so that verified entry point is properly
1582      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1583   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1584   if (OptoBreakpoint) {
1585     // Leave space for int3
1586     nops_cnt -= 1;
1587   }
1588   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1589   if (nops_cnt > 0)
1590     masm.nop(nops_cnt);
1591 }
1592 
1593 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1594 {
1595   return MachNode::size(ra_); // too many variables; just compute it
1596                               // the hard way
1597 }
1598 
1599 
1600 //=============================================================================
1601 
1602 int Matcher::regnum_to_fpu_offset(int regnum)
1603 {
1604   return regnum - 32; // The FP registers are in the second chunk
1605 }
1606 
1607 // This is UltraSparc specific, true just means we have fast l2f conversion
1608 const bool Matcher::convL2FSupported(void) {
1609   return true;
1610 }
1611 
1612 // Is this branch offset short enough that a short branch can be used?
1613 //
1614 // NOTE: If the platform does not provide any short branch variants, then
1615 //       this method should return false for offset 0.
1616 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1617   // The passed offset is relative to address of the branch.
1618   // On 86 a branch displacement is calculated relative to address
1619   // of a next instruction.
1620   offset -= br_size;
1621 
1622   // the short version of jmpConUCF2 contains multiple branches,
1623   // making the reach slightly less
1624   if (rule == jmpConUCF2_rule)
1625     return (-126 <= offset && offset <= 125);
1626   return (-128 <= offset && offset <= 127);
1627 }
1628 
1629 const bool Matcher::isSimpleConstant64(jlong value) {
1630   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1631   //return value == (int) value;  // Cf. storeImmL and immL32.
1632 
1633   // Probably always true, even if a temp register is required.
1634   return true;
1635 }
1636 
1637 // The ecx parameter to rep stosq for the ClearArray node is in words.
1638 const bool Matcher::init_array_count_is_in_bytes = false;
1639 
1640 // No additional cost for CMOVL.
1641 const int Matcher::long_cmove_cost() { return 0; }
1642 
1643 // No CMOVF/CMOVD with SSE2
1644 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1645 
1646 // Does the CPU require late expand (see block.cpp for description of late expand)?
1647 const bool Matcher::require_postalloc_expand = false;
1648 
1649 // Should the Matcher clone shifts on addressing modes, expecting them
1650 // to be subsumed into complex addressing expressions or compute them
1651 // into registers?  True for Intel but false for most RISCs
1652 const bool Matcher::clone_shift_expressions = true;
1653 
1654 // Do we need to mask the count passed to shift instructions or does
1655 // the cpu only look at the lower 5/6 bits anyway?
1656 const bool Matcher::need_masked_shift_count = false;
1657 
1658 bool Matcher::narrow_oop_use_complex_address() {
1659   assert(UseCompressedOops, "only for compressed oops code");
1660   return (LogMinObjAlignmentInBytes <= 3);
1661 }
1662 
1663 bool Matcher::narrow_klass_use_complex_address() {
1664   assert(UseCompressedClassPointers, "only for compressed klass code");
1665   return (LogKlassAlignmentInBytes <= 3);
1666 }
1667 
1668 // Is it better to copy float constants, or load them directly from
1669 // memory?  Intel can load a float constant from a direct address,
1670 // requiring no extra registers.  Most RISCs will have to materialize
1671 // an address into a register first, so they would do better to copy
1672 // the constant from stack.
1673 const bool Matcher::rematerialize_float_constants = true; // XXX
1674 
1675 // If CPU can load and store mis-aligned doubles directly then no
1676 // fixup is needed.  Else we split the double into 2 integer pieces
1677 // and move it piece-by-piece.  Only happens when passing doubles into
1678 // C code as the Java calling convention forces doubles to be aligned.
1679 const bool Matcher::misaligned_doubles_ok = true;
1680 
1681 // No-op on amd64
1682 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1683 
1684 // Advertise here if the CPU requires explicit rounding operations to
1685 // implement the UseStrictFP mode.
1686 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1687 
1688 // Are floats conerted to double when stored to stack during deoptimization?
1689 // On x64 it is stored without convertion so we can use normal access.
1690 bool Matcher::float_in_double() { return false; }
1691 
1692 // Do ints take an entire long register or just half?
1693 const bool Matcher::int_in_long = true;
1694 
1695 // Return whether or not this register is ever used as an argument.
1696 // This function is used on startup to build the trampoline stubs in
1697 // generateOptoStub.  Registers not mentioned will be killed by the VM
1698 // call in the trampoline, and arguments in those registers not be
1699 // available to the callee.
1700 bool Matcher::can_be_java_arg(int reg)
1701 {
1702   return
1703     reg ==  RDI_num || reg == RDI_H_num ||
1704     reg ==  RSI_num || reg == RSI_H_num ||
1705     reg ==  RDX_num || reg == RDX_H_num ||
1706     reg ==  RCX_num || reg == RCX_H_num ||
1707     reg ==   R8_num || reg ==  R8_H_num ||
1708     reg ==   R9_num || reg ==  R9_H_num ||
1709     reg ==  R12_num || reg == R12_H_num ||
1710     reg == XMM0_num || reg == XMM0b_num ||
1711     reg == XMM1_num || reg == XMM1b_num ||
1712     reg == XMM2_num || reg == XMM2b_num ||
1713     reg == XMM3_num || reg == XMM3b_num ||
1714     reg == XMM4_num || reg == XMM4b_num ||
1715     reg == XMM5_num || reg == XMM5b_num ||
1716     reg == XMM6_num || reg == XMM6b_num ||
1717     reg == XMM7_num || reg == XMM7b_num;
1718 }
1719 
1720 bool Matcher::is_spillable_arg(int reg)
1721 {
1722   return can_be_java_arg(reg);
1723 }
1724 
1725 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1726   // In 64 bit mode a code which use multiply when
1727   // devisor is constant is faster than hardware
1728   // DIV instruction (it uses MulHiL).
1729   return false;
1730 }
1731 
1732 // Register for DIVI projection of divmodI
1733 RegMask Matcher::divI_proj_mask() {
1734   return INT_RAX_REG_mask();
1735 }
1736 
1737 // Register for MODI projection of divmodI
1738 RegMask Matcher::modI_proj_mask() {
1739   return INT_RDX_REG_mask();
1740 }
1741 
1742 // Register for DIVL projection of divmodL
1743 RegMask Matcher::divL_proj_mask() {
1744   return LONG_RAX_REG_mask();
1745 }
1746 
1747 // Register for MODL projection of divmodL
1748 RegMask Matcher::modL_proj_mask() {
1749   return LONG_RDX_REG_mask();
1750 }
1751 
1752 // Register for saving SP into on method handle invokes. Not used on x86_64.
1753 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1754     return NO_REG_mask();
1755 }
1756 
1757 %}
1758 
1759 //----------ENCODING BLOCK-----------------------------------------------------
1760 // This block specifies the encoding classes used by the compiler to
1761 // output byte streams.  Encoding classes are parameterized macros
1762 // used by Machine Instruction Nodes in order to generate the bit
1763 // encoding of the instruction.  Operands specify their base encoding
1764 // interface with the interface keyword.  There are currently
1765 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1766 // COND_INTER.  REG_INTER causes an operand to generate a function
1767 // which returns its register number when queried.  CONST_INTER causes
1768 // an operand to generate a function which returns the value of the
1769 // constant when queried.  MEMORY_INTER causes an operand to generate
1770 // four functions which return the Base Register, the Index Register,
1771 // the Scale Value, and the Offset Value of the operand when queried.
1772 // COND_INTER causes an operand to generate six functions which return
1773 // the encoding code (ie - encoding bits for the instruction)
1774 // associated with each basic boolean condition for a conditional
1775 // instruction.
1776 //
1777 // Instructions specify two basic values for encoding.  Again, a
1778 // function is available to check if the constant displacement is an
1779 // oop. They use the ins_encode keyword to specify their encoding
1780 // classes (which must be a sequence of enc_class names, and their
1781 // parameters, specified in the encoding block), and they use the
1782 // opcode keyword to specify, in order, their primary, secondary, and
1783 // tertiary opcode.  Only the opcode sections which a particular
1784 // instruction needs for encoding need to be specified.
1785 encode %{
1786   // Build emit functions for each basic byte or larger field in the
1787   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1788   // from C++ code in the enc_class source block.  Emit functions will
1789   // live in the main source block for now.  In future, we can
1790   // generalize this by adding a syntax that specifies the sizes of
1791   // fields in an order, so that the adlc can build the emit functions
1792   // automagically
1793 
1794   // Emit primary opcode
1795   enc_class OpcP
1796   %{
1797     emit_opcode(cbuf, $primary);
1798   %}
1799 
1800   // Emit secondary opcode
1801   enc_class OpcS
1802   %{
1803     emit_opcode(cbuf, $secondary);
1804   %}
1805 
1806   // Emit tertiary opcode
1807   enc_class OpcT
1808   %{
1809     emit_opcode(cbuf, $tertiary);
1810   %}
1811 
1812   // Emit opcode directly
1813   enc_class Opcode(immI d8)
1814   %{
1815     emit_opcode(cbuf, $d8$$constant);
1816   %}
1817 
1818   // Emit size prefix
1819   enc_class SizePrefix
1820   %{
1821     emit_opcode(cbuf, 0x66);
1822   %}
1823 
1824   enc_class reg(rRegI reg)
1825   %{
1826     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1827   %}
1828 
1829   enc_class reg_reg(rRegI dst, rRegI src)
1830   %{
1831     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1832   %}
1833 
1834   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1835   %{
1836     emit_opcode(cbuf, $opcode$$constant);
1837     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1838   %}
1839 
1840   enc_class cdql_enc(no_rax_rdx_RegI div)
1841   %{
1842     // Full implementation of Java idiv and irem; checks for
1843     // special case as described in JVM spec., p.243 & p.271.
1844     //
1845     //         normal case                           special case
1846     //
1847     // input : rax: dividend                         min_int
1848     //         reg: divisor                          -1
1849     //
1850     // output: rax: quotient  (= rax idiv reg)       min_int
1851     //         rdx: remainder (= rax irem reg)       0
1852     //
1853     //  Code sequnce:
1854     //
1855     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1856     //    5:   75 07/08                jne    e <normal>
1857     //    7:   33 d2                   xor    %edx,%edx
1858     //  [div >= 8 -> offset + 1]
1859     //  [REX_B]
1860     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1861     //    c:   74 03/04                je     11 <done>
1862     // 000000000000000e <normal>:
1863     //    e:   99                      cltd
1864     //  [div >= 8 -> offset + 1]
1865     //  [REX_B]
1866     //    f:   f7 f9                   idiv   $div
1867     // 0000000000000011 <done>:
1868 
1869     // cmp    $0x80000000,%eax
1870     emit_opcode(cbuf, 0x3d);
1871     emit_d8(cbuf, 0x00);
1872     emit_d8(cbuf, 0x00);
1873     emit_d8(cbuf, 0x00);
1874     emit_d8(cbuf, 0x80);
1875 
1876     // jne    e <normal>
1877     emit_opcode(cbuf, 0x75);
1878     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1879 
1880     // xor    %edx,%edx
1881     emit_opcode(cbuf, 0x33);
1882     emit_d8(cbuf, 0xD2);
1883 
1884     // cmp    $0xffffffffffffffff,%ecx
1885     if ($div$$reg >= 8) {
1886       emit_opcode(cbuf, Assembler::REX_B);
1887     }
1888     emit_opcode(cbuf, 0x83);
1889     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1890     emit_d8(cbuf, 0xFF);
1891 
1892     // je     11 <done>
1893     emit_opcode(cbuf, 0x74);
1894     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1895 
1896     // <normal>
1897     // cltd
1898     emit_opcode(cbuf, 0x99);
1899 
1900     // idivl (note: must be emitted by the user of this rule)
1901     // <done>
1902   %}
1903 
1904   enc_class cdqq_enc(no_rax_rdx_RegL div)
1905   %{
1906     // Full implementation of Java ldiv and lrem; checks for
1907     // special case as described in JVM spec., p.243 & p.271.
1908     //
1909     //         normal case                           special case
1910     //
1911     // input : rax: dividend                         min_long
1912     //         reg: divisor                          -1
1913     //
1914     // output: rax: quotient  (= rax idiv reg)       min_long
1915     //         rdx: remainder (= rax irem reg)       0
1916     //
1917     //  Code sequnce:
1918     //
1919     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1920     //    7:   00 00 80
1921     //    a:   48 39 d0                cmp    %rdx,%rax
1922     //    d:   75 08                   jne    17 <normal>
1923     //    f:   33 d2                   xor    %edx,%edx
1924     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1925     //   15:   74 05                   je     1c <done>
1926     // 0000000000000017 <normal>:
1927     //   17:   48 99                   cqto
1928     //   19:   48 f7 f9                idiv   $div
1929     // 000000000000001c <done>:
1930 
1931     // mov    $0x8000000000000000,%rdx
1932     emit_opcode(cbuf, Assembler::REX_W);
1933     emit_opcode(cbuf, 0xBA);
1934     emit_d8(cbuf, 0x00);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x00);
1939     emit_d8(cbuf, 0x00);
1940     emit_d8(cbuf, 0x00);
1941     emit_d8(cbuf, 0x80);
1942 
1943     // cmp    %rdx,%rax
1944     emit_opcode(cbuf, Assembler::REX_W);
1945     emit_opcode(cbuf, 0x39);
1946     emit_d8(cbuf, 0xD0);
1947 
1948     // jne    17 <normal>
1949     emit_opcode(cbuf, 0x75);
1950     emit_d8(cbuf, 0x08);
1951 
1952     // xor    %edx,%edx
1953     emit_opcode(cbuf, 0x33);
1954     emit_d8(cbuf, 0xD2);
1955 
1956     // cmp    $0xffffffffffffffff,$div
1957     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1958     emit_opcode(cbuf, 0x83);
1959     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1960     emit_d8(cbuf, 0xFF);
1961 
1962     // je     1e <done>
1963     emit_opcode(cbuf, 0x74);
1964     emit_d8(cbuf, 0x05);
1965 
1966     // <normal>
1967     // cqto
1968     emit_opcode(cbuf, Assembler::REX_W);
1969     emit_opcode(cbuf, 0x99);
1970 
1971     // idivq (note: must be emitted by the user of this rule)
1972     // <done>
1973   %}
1974 
1975   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1976   enc_class OpcSE(immI imm)
1977   %{
1978     // Emit primary opcode and set sign-extend bit
1979     // Check for 8-bit immediate, and set sign extend bit in opcode
1980     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1981       emit_opcode(cbuf, $primary | 0x02);
1982     } else {
1983       // 32-bit immediate
1984       emit_opcode(cbuf, $primary);
1985     }
1986   %}
1987 
1988   enc_class OpcSErm(rRegI dst, immI imm)
1989   %{
1990     // OpcSEr/m
1991     int dstenc = $dst$$reg;
1992     if (dstenc >= 8) {
1993       emit_opcode(cbuf, Assembler::REX_B);
1994       dstenc -= 8;
1995     }
1996     // Emit primary opcode and set sign-extend bit
1997     // Check for 8-bit immediate, and set sign extend bit in opcode
1998     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1999       emit_opcode(cbuf, $primary | 0x02);
2000     } else {
2001       // 32-bit immediate
2002       emit_opcode(cbuf, $primary);
2003     }
2004     // Emit r/m byte with secondary opcode, after primary opcode.
2005     emit_rm(cbuf, 0x3, $secondary, dstenc);
2006   %}
2007 
2008   enc_class OpcSErm_wide(rRegL dst, immI imm)
2009   %{
2010     // OpcSEr/m
2011     int dstenc = $dst$$reg;
2012     if (dstenc < 8) {
2013       emit_opcode(cbuf, Assembler::REX_W);
2014     } else {
2015       emit_opcode(cbuf, Assembler::REX_WB);
2016       dstenc -= 8;
2017     }
2018     // Emit primary opcode and set sign-extend bit
2019     // Check for 8-bit immediate, and set sign extend bit in opcode
2020     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2021       emit_opcode(cbuf, $primary | 0x02);
2022     } else {
2023       // 32-bit immediate
2024       emit_opcode(cbuf, $primary);
2025     }
2026     // Emit r/m byte with secondary opcode, after primary opcode.
2027     emit_rm(cbuf, 0x3, $secondary, dstenc);
2028   %}
2029 
2030   enc_class Con8or32(immI imm)
2031   %{
2032     // Check for 8-bit immediate, and set sign extend bit in opcode
2033     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2034       $$$emit8$imm$$constant;
2035     } else {
2036       // 32-bit immediate
2037       $$$emit32$imm$$constant;
2038     }
2039   %}
2040 
2041   enc_class opc2_reg(rRegI dst)
2042   %{
2043     // BSWAP
2044     emit_cc(cbuf, $secondary, $dst$$reg);
2045   %}
2046 
2047   enc_class opc3_reg(rRegI dst)
2048   %{
2049     // BSWAP
2050     emit_cc(cbuf, $tertiary, $dst$$reg);
2051   %}
2052 
2053   enc_class reg_opc(rRegI div)
2054   %{
2055     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2056     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2057   %}
2058 
2059   enc_class enc_cmov(cmpOp cop)
2060   %{
2061     // CMOV
2062     $$$emit8$primary;
2063     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2064   %}
2065 
2066   enc_class enc_PartialSubtypeCheck()
2067   %{
2068     Register Rrdi = as_Register(RDI_enc); // result register
2069     Register Rrax = as_Register(RAX_enc); // super class
2070     Register Rrcx = as_Register(RCX_enc); // killed
2071     Register Rrsi = as_Register(RSI_enc); // sub class
2072     Label miss;
2073     const bool set_cond_codes = true;
2074 
2075     MacroAssembler _masm(&cbuf);
2076     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2077                                      NULL, &miss,
2078                                      /*set_cond_codes:*/ true);
2079     if ($primary) {
2080       __ xorptr(Rrdi, Rrdi);
2081     }
2082     __ bind(miss);
2083   %}
2084 
2085   enc_class clear_avx %{
2086     debug_only(int off0 = cbuf.insts_size());
2087     if (ra_->C->max_vector_size() > 16) {
2088       // Clear upper bits of YMM registers when current compiled code uses
2089       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2090       MacroAssembler _masm(&cbuf);
2091       __ vzeroupper();
2092     }
2093     debug_only(int off1 = cbuf.insts_size());
2094     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2095   %}
2096 
2097   enc_class Java_To_Runtime(method meth) %{
2098     // No relocation needed
2099     MacroAssembler _masm(&cbuf);
2100     __ mov64(r10, (int64_t) $meth$$method);
2101     __ call(r10);
2102   %}
2103 
2104   enc_class Java_To_Interpreter(method meth)
2105   %{
2106     // CALL Java_To_Interpreter
2107     // This is the instruction starting address for relocation info.
2108     cbuf.set_insts_mark();
2109     $$$emit8$primary;
2110     // CALL directly to the runtime
2111     emit_d32_reloc(cbuf,
2112                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2113                    runtime_call_Relocation::spec(),
2114                    RELOC_DISP32);
2115   %}
2116 
2117   enc_class Java_Static_Call(method meth)
2118   %{
2119     // JAVA STATIC CALL
2120     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2121     // determine who we intended to call.
2122     cbuf.set_insts_mark();
2123     $$$emit8$primary;
2124 
2125     if (!_method) {
2126       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2127                      runtime_call_Relocation::spec(),
2128                      RELOC_DISP32);
2129     } else {
2130       int method_index = resolved_method_index(cbuf);
2131       RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
2132                                                   : static_call_Relocation::spec(method_index);
2133       emit_d32_reloc(cbuf, (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2134                      rspec, RELOC_DISP32);
2135       // Emit stubs for static call.
2136       address mark = cbuf.insts_mark();
2137       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2138       if (stub == NULL) {
2139         ciEnv::current()->record_failure("CodeCache is full");
2140         return;
2141       }
2142     }
2143   %}
2144 
2145   enc_class Java_Dynamic_Call(method meth) %{
2146     MacroAssembler _masm(&cbuf);
2147     __ ic_call((address)$meth$$method, resolved_method_index(cbuf));
2148   %}
2149 
2150   enc_class Java_Compiled_Call(method meth)
2151   %{
2152     // JAVA COMPILED CALL
2153     int disp = in_bytes(Method:: from_compiled_offset());
2154 
2155     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2156     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2157 
2158     // callq *disp(%rax)
2159     cbuf.set_insts_mark();
2160     $$$emit8$primary;
2161     if (disp < 0x80) {
2162       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2163       emit_d8(cbuf, disp); // Displacement
2164     } else {
2165       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2166       emit_d32(cbuf, disp); // Displacement
2167     }
2168   %}
2169 
2170   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2171   %{
2172     // SAL, SAR, SHR
2173     int dstenc = $dst$$reg;
2174     if (dstenc >= 8) {
2175       emit_opcode(cbuf, Assembler::REX_B);
2176       dstenc -= 8;
2177     }
2178     $$$emit8$primary;
2179     emit_rm(cbuf, 0x3, $secondary, dstenc);
2180     $$$emit8$shift$$constant;
2181   %}
2182 
2183   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2184   %{
2185     // SAL, SAR, SHR
2186     int dstenc = $dst$$reg;
2187     if (dstenc < 8) {
2188       emit_opcode(cbuf, Assembler::REX_W);
2189     } else {
2190       emit_opcode(cbuf, Assembler::REX_WB);
2191       dstenc -= 8;
2192     }
2193     $$$emit8$primary;
2194     emit_rm(cbuf, 0x3, $secondary, dstenc);
2195     $$$emit8$shift$$constant;
2196   %}
2197 
2198   enc_class load_immI(rRegI dst, immI src)
2199   %{
2200     int dstenc = $dst$$reg;
2201     if (dstenc >= 8) {
2202       emit_opcode(cbuf, Assembler::REX_B);
2203       dstenc -= 8;
2204     }
2205     emit_opcode(cbuf, 0xB8 | dstenc);
2206     $$$emit32$src$$constant;
2207   %}
2208 
2209   enc_class load_immL(rRegL dst, immL src)
2210   %{
2211     int dstenc = $dst$$reg;
2212     if (dstenc < 8) {
2213       emit_opcode(cbuf, Assembler::REX_W);
2214     } else {
2215       emit_opcode(cbuf, Assembler::REX_WB);
2216       dstenc -= 8;
2217     }
2218     emit_opcode(cbuf, 0xB8 | dstenc);
2219     emit_d64(cbuf, $src$$constant);
2220   %}
2221 
2222   enc_class load_immUL32(rRegL dst, immUL32 src)
2223   %{
2224     // same as load_immI, but this time we care about zeroes in the high word
2225     int dstenc = $dst$$reg;
2226     if (dstenc >= 8) {
2227       emit_opcode(cbuf, Assembler::REX_B);
2228       dstenc -= 8;
2229     }
2230     emit_opcode(cbuf, 0xB8 | dstenc);
2231     $$$emit32$src$$constant;
2232   %}
2233 
2234   enc_class load_immL32(rRegL dst, immL32 src)
2235   %{
2236     int dstenc = $dst$$reg;
2237     if (dstenc < 8) {
2238       emit_opcode(cbuf, Assembler::REX_W);
2239     } else {
2240       emit_opcode(cbuf, Assembler::REX_WB);
2241       dstenc -= 8;
2242     }
2243     emit_opcode(cbuf, 0xC7);
2244     emit_rm(cbuf, 0x03, 0x00, dstenc);
2245     $$$emit32$src$$constant;
2246   %}
2247 
2248   enc_class load_immP31(rRegP dst, immP32 src)
2249   %{
2250     // same as load_immI, but this time we care about zeroes in the high word
2251     int dstenc = $dst$$reg;
2252     if (dstenc >= 8) {
2253       emit_opcode(cbuf, Assembler::REX_B);
2254       dstenc -= 8;
2255     }
2256     emit_opcode(cbuf, 0xB8 | dstenc);
2257     $$$emit32$src$$constant;
2258   %}
2259 
2260   enc_class load_immP(rRegP dst, immP src)
2261   %{
2262     int dstenc = $dst$$reg;
2263     if (dstenc < 8) {
2264       emit_opcode(cbuf, Assembler::REX_W);
2265     } else {
2266       emit_opcode(cbuf, Assembler::REX_WB);
2267       dstenc -= 8;
2268     }
2269     emit_opcode(cbuf, 0xB8 | dstenc);
2270     // This next line should be generated from ADLC
2271     if ($src->constant_reloc() != relocInfo::none) {
2272       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2273     } else {
2274       emit_d64(cbuf, $src$$constant);
2275     }
2276   %}
2277 
2278   enc_class Con32(immI src)
2279   %{
2280     // Output immediate
2281     $$$emit32$src$$constant;
2282   %}
2283 
2284   enc_class Con32F_as_bits(immF src)
2285   %{
2286     // Output Float immediate bits
2287     jfloat jf = $src$$constant;
2288     jint jf_as_bits = jint_cast(jf);
2289     emit_d32(cbuf, jf_as_bits);
2290   %}
2291 
2292   enc_class Con16(immI src)
2293   %{
2294     // Output immediate
2295     $$$emit16$src$$constant;
2296   %}
2297 
2298   // How is this different from Con32??? XXX
2299   enc_class Con_d32(immI src)
2300   %{
2301     emit_d32(cbuf,$src$$constant);
2302   %}
2303 
2304   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2305     // Output immediate memory reference
2306     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2307     emit_d32(cbuf, 0x00);
2308   %}
2309 
2310   enc_class lock_prefix()
2311   %{
2312     if (os::is_MP()) {
2313       emit_opcode(cbuf, 0xF0); // lock
2314     }
2315   %}
2316 
2317   enc_class REX_mem(memory mem)
2318   %{
2319     if ($mem$$base >= 8) {
2320       if ($mem$$index < 8) {
2321         emit_opcode(cbuf, Assembler::REX_B);
2322       } else {
2323         emit_opcode(cbuf, Assembler::REX_XB);
2324       }
2325     } else {
2326       if ($mem$$index >= 8) {
2327         emit_opcode(cbuf, Assembler::REX_X);
2328       }
2329     }
2330   %}
2331 
2332   enc_class REX_mem_wide(memory mem)
2333   %{
2334     if ($mem$$base >= 8) {
2335       if ($mem$$index < 8) {
2336         emit_opcode(cbuf, Assembler::REX_WB);
2337       } else {
2338         emit_opcode(cbuf, Assembler::REX_WXB);
2339       }
2340     } else {
2341       if ($mem$$index < 8) {
2342         emit_opcode(cbuf, Assembler::REX_W);
2343       } else {
2344         emit_opcode(cbuf, Assembler::REX_WX);
2345       }
2346     }
2347   %}
2348 
2349   // for byte regs
2350   enc_class REX_breg(rRegI reg)
2351   %{
2352     if ($reg$$reg >= 4) {
2353       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2354     }
2355   %}
2356 
2357   // for byte regs
2358   enc_class REX_reg_breg(rRegI dst, rRegI src)
2359   %{
2360     if ($dst$$reg < 8) {
2361       if ($src$$reg >= 4) {
2362         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2363       }
2364     } else {
2365       if ($src$$reg < 8) {
2366         emit_opcode(cbuf, Assembler::REX_R);
2367       } else {
2368         emit_opcode(cbuf, Assembler::REX_RB);
2369       }
2370     }
2371   %}
2372 
2373   // for byte regs
2374   enc_class REX_breg_mem(rRegI reg, memory mem)
2375   %{
2376     if ($reg$$reg < 8) {
2377       if ($mem$$base < 8) {
2378         if ($mem$$index >= 8) {
2379           emit_opcode(cbuf, Assembler::REX_X);
2380         } else if ($reg$$reg >= 4) {
2381           emit_opcode(cbuf, Assembler::REX);
2382         }
2383       } else {
2384         if ($mem$$index < 8) {
2385           emit_opcode(cbuf, Assembler::REX_B);
2386         } else {
2387           emit_opcode(cbuf, Assembler::REX_XB);
2388         }
2389       }
2390     } else {
2391       if ($mem$$base < 8) {
2392         if ($mem$$index < 8) {
2393           emit_opcode(cbuf, Assembler::REX_R);
2394         } else {
2395           emit_opcode(cbuf, Assembler::REX_RX);
2396         }
2397       } else {
2398         if ($mem$$index < 8) {
2399           emit_opcode(cbuf, Assembler::REX_RB);
2400         } else {
2401           emit_opcode(cbuf, Assembler::REX_RXB);
2402         }
2403       }
2404     }
2405   %}
2406 
2407   enc_class REX_reg(rRegI reg)
2408   %{
2409     if ($reg$$reg >= 8) {
2410       emit_opcode(cbuf, Assembler::REX_B);
2411     }
2412   %}
2413 
2414   enc_class REX_reg_wide(rRegI reg)
2415   %{
2416     if ($reg$$reg < 8) {
2417       emit_opcode(cbuf, Assembler::REX_W);
2418     } else {
2419       emit_opcode(cbuf, Assembler::REX_WB);
2420     }
2421   %}
2422 
2423   enc_class REX_reg_reg(rRegI dst, rRegI src)
2424   %{
2425     if ($dst$$reg < 8) {
2426       if ($src$$reg >= 8) {
2427         emit_opcode(cbuf, Assembler::REX_B);
2428       }
2429     } else {
2430       if ($src$$reg < 8) {
2431         emit_opcode(cbuf, Assembler::REX_R);
2432       } else {
2433         emit_opcode(cbuf, Assembler::REX_RB);
2434       }
2435     }
2436   %}
2437 
2438   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2439   %{
2440     if ($dst$$reg < 8) {
2441       if ($src$$reg < 8) {
2442         emit_opcode(cbuf, Assembler::REX_W);
2443       } else {
2444         emit_opcode(cbuf, Assembler::REX_WB);
2445       }
2446     } else {
2447       if ($src$$reg < 8) {
2448         emit_opcode(cbuf, Assembler::REX_WR);
2449       } else {
2450         emit_opcode(cbuf, Assembler::REX_WRB);
2451       }
2452     }
2453   %}
2454 
2455   enc_class REX_reg_mem(rRegI reg, memory mem)
2456   %{
2457     if ($reg$$reg < 8) {
2458       if ($mem$$base < 8) {
2459         if ($mem$$index >= 8) {
2460           emit_opcode(cbuf, Assembler::REX_X);
2461         }
2462       } else {
2463         if ($mem$$index < 8) {
2464           emit_opcode(cbuf, Assembler::REX_B);
2465         } else {
2466           emit_opcode(cbuf, Assembler::REX_XB);
2467         }
2468       }
2469     } else {
2470       if ($mem$$base < 8) {
2471         if ($mem$$index < 8) {
2472           emit_opcode(cbuf, Assembler::REX_R);
2473         } else {
2474           emit_opcode(cbuf, Assembler::REX_RX);
2475         }
2476       } else {
2477         if ($mem$$index < 8) {
2478           emit_opcode(cbuf, Assembler::REX_RB);
2479         } else {
2480           emit_opcode(cbuf, Assembler::REX_RXB);
2481         }
2482       }
2483     }
2484   %}
2485 
2486   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2487   %{
2488     if ($reg$$reg < 8) {
2489       if ($mem$$base < 8) {
2490         if ($mem$$index < 8) {
2491           emit_opcode(cbuf, Assembler::REX_W);
2492         } else {
2493           emit_opcode(cbuf, Assembler::REX_WX);
2494         }
2495       } else {
2496         if ($mem$$index < 8) {
2497           emit_opcode(cbuf, Assembler::REX_WB);
2498         } else {
2499           emit_opcode(cbuf, Assembler::REX_WXB);
2500         }
2501       }
2502     } else {
2503       if ($mem$$base < 8) {
2504         if ($mem$$index < 8) {
2505           emit_opcode(cbuf, Assembler::REX_WR);
2506         } else {
2507           emit_opcode(cbuf, Assembler::REX_WRX);
2508         }
2509       } else {
2510         if ($mem$$index < 8) {
2511           emit_opcode(cbuf, Assembler::REX_WRB);
2512         } else {
2513           emit_opcode(cbuf, Assembler::REX_WRXB);
2514         }
2515       }
2516     }
2517   %}
2518 
2519   enc_class reg_mem(rRegI ereg, memory mem)
2520   %{
2521     // High registers handle in encode_RegMem
2522     int reg = $ereg$$reg;
2523     int base = $mem$$base;
2524     int index = $mem$$index;
2525     int scale = $mem$$scale;
2526     int disp = $mem$$disp;
2527     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2528 
2529     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2530   %}
2531 
2532   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2533   %{
2534     int rm_byte_opcode = $rm_opcode$$constant;
2535 
2536     // High registers handle in encode_RegMem
2537     int base = $mem$$base;
2538     int index = $mem$$index;
2539     int scale = $mem$$scale;
2540     int displace = $mem$$disp;
2541 
2542     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2543                                             // working with static
2544                                             // globals
2545     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2546                   disp_reloc);
2547   %}
2548 
2549   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2550   %{
2551     int reg_encoding = $dst$$reg;
2552     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2553     int index        = 0x04;            // 0x04 indicates no index
2554     int scale        = 0x00;            // 0x00 indicates no scale
2555     int displace     = $src1$$constant; // 0x00 indicates no displacement
2556     relocInfo::relocType disp_reloc = relocInfo::none;
2557     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2558                   disp_reloc);
2559   %}
2560 
2561   enc_class neg_reg(rRegI dst)
2562   %{
2563     int dstenc = $dst$$reg;
2564     if (dstenc >= 8) {
2565       emit_opcode(cbuf, Assembler::REX_B);
2566       dstenc -= 8;
2567     }
2568     // NEG $dst
2569     emit_opcode(cbuf, 0xF7);
2570     emit_rm(cbuf, 0x3, 0x03, dstenc);
2571   %}
2572 
2573   enc_class neg_reg_wide(rRegI dst)
2574   %{
2575     int dstenc = $dst$$reg;
2576     if (dstenc < 8) {
2577       emit_opcode(cbuf, Assembler::REX_W);
2578     } else {
2579       emit_opcode(cbuf, Assembler::REX_WB);
2580       dstenc -= 8;
2581     }
2582     // NEG $dst
2583     emit_opcode(cbuf, 0xF7);
2584     emit_rm(cbuf, 0x3, 0x03, dstenc);
2585   %}
2586 
2587   enc_class setLT_reg(rRegI dst)
2588   %{
2589     int dstenc = $dst$$reg;
2590     if (dstenc >= 8) {
2591       emit_opcode(cbuf, Assembler::REX_B);
2592       dstenc -= 8;
2593     } else if (dstenc >= 4) {
2594       emit_opcode(cbuf, Assembler::REX);
2595     }
2596     // SETLT $dst
2597     emit_opcode(cbuf, 0x0F);
2598     emit_opcode(cbuf, 0x9C);
2599     emit_rm(cbuf, 0x3, 0x0, dstenc);
2600   %}
2601 
2602   enc_class setNZ_reg(rRegI dst)
2603   %{
2604     int dstenc = $dst$$reg;
2605     if (dstenc >= 8) {
2606       emit_opcode(cbuf, Assembler::REX_B);
2607       dstenc -= 8;
2608     } else if (dstenc >= 4) {
2609       emit_opcode(cbuf, Assembler::REX);
2610     }
2611     // SETNZ $dst
2612     emit_opcode(cbuf, 0x0F);
2613     emit_opcode(cbuf, 0x95);
2614     emit_rm(cbuf, 0x3, 0x0, dstenc);
2615   %}
2616 
2617 
2618   // Compare the lonogs and set -1, 0, or 1 into dst
2619   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2620   %{
2621     int src1enc = $src1$$reg;
2622     int src2enc = $src2$$reg;
2623     int dstenc = $dst$$reg;
2624 
2625     // cmpq $src1, $src2
2626     if (src1enc < 8) {
2627       if (src2enc < 8) {
2628         emit_opcode(cbuf, Assembler::REX_W);
2629       } else {
2630         emit_opcode(cbuf, Assembler::REX_WB);
2631       }
2632     } else {
2633       if (src2enc < 8) {
2634         emit_opcode(cbuf, Assembler::REX_WR);
2635       } else {
2636         emit_opcode(cbuf, Assembler::REX_WRB);
2637       }
2638     }
2639     emit_opcode(cbuf, 0x3B);
2640     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2641 
2642     // movl $dst, -1
2643     if (dstenc >= 8) {
2644       emit_opcode(cbuf, Assembler::REX_B);
2645     }
2646     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2647     emit_d32(cbuf, -1);
2648 
2649     // jl,s done
2650     emit_opcode(cbuf, 0x7C);
2651     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2652 
2653     // setne $dst
2654     if (dstenc >= 4) {
2655       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2656     }
2657     emit_opcode(cbuf, 0x0F);
2658     emit_opcode(cbuf, 0x95);
2659     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2660 
2661     // movzbl $dst, $dst
2662     if (dstenc >= 4) {
2663       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2664     }
2665     emit_opcode(cbuf, 0x0F);
2666     emit_opcode(cbuf, 0xB6);
2667     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2668   %}
2669 
2670   enc_class Push_ResultXD(regD dst) %{
2671     MacroAssembler _masm(&cbuf);
2672     __ fstp_d(Address(rsp, 0));
2673     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2674     __ addptr(rsp, 8);
2675   %}
2676 
2677   enc_class Push_SrcXD(regD src) %{
2678     MacroAssembler _masm(&cbuf);
2679     __ subptr(rsp, 8);
2680     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2681     __ fld_d(Address(rsp, 0));
2682   %}
2683 
2684 
2685   enc_class enc_rethrow()
2686   %{
2687     cbuf.set_insts_mark();
2688     emit_opcode(cbuf, 0xE9); // jmp entry
2689     emit_d32_reloc(cbuf,
2690                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2691                    runtime_call_Relocation::spec(),
2692                    RELOC_DISP32);
2693   %}
2694 
2695 %}
2696 
2697 
2698 
2699 //----------FRAME--------------------------------------------------------------
2700 // Definition of frame structure and management information.
2701 //
2702 //  S T A C K   L A Y O U T    Allocators stack-slot number
2703 //                             |   (to get allocators register number
2704 //  G  Owned by    |        |  v    add OptoReg::stack0())
2705 //  r   CALLER     |        |
2706 //  o     |        +--------+      pad to even-align allocators stack-slot
2707 //  w     V        |  pad0  |        numbers; owned by CALLER
2708 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2709 //  h     ^        |   in   |  5
2710 //        |        |  args  |  4   Holes in incoming args owned by SELF
2711 //  |     |        |        |  3
2712 //  |     |        +--------+
2713 //  V     |        | old out|      Empty on Intel, window on Sparc
2714 //        |    old |preserve|      Must be even aligned.
2715 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2716 //        |        |   in   |  3   area for Intel ret address
2717 //     Owned by    |preserve|      Empty on Sparc.
2718 //       SELF      +--------+
2719 //        |        |  pad2  |  2   pad to align old SP
2720 //        |        +--------+  1
2721 //        |        | locks  |  0
2722 //        |        +--------+----> OptoReg::stack0(), even aligned
2723 //        |        |  pad1  | 11   pad to align new SP
2724 //        |        +--------+
2725 //        |        |        | 10
2726 //        |        | spills |  9   spills
2727 //        V        |        |  8   (pad0 slot for callee)
2728 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2729 //        ^        |  out   |  7
2730 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2731 //     Owned by    +--------+
2732 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2733 //        |    new |preserve|      Must be even-aligned.
2734 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2735 //        |        |        |
2736 //
2737 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2738 //         known from SELF's arguments and the Java calling convention.
2739 //         Region 6-7 is determined per call site.
2740 // Note 2: If the calling convention leaves holes in the incoming argument
2741 //         area, those holes are owned by SELF.  Holes in the outgoing area
2742 //         are owned by the CALLEE.  Holes should not be nessecary in the
2743 //         incoming area, as the Java calling convention is completely under
2744 //         the control of the AD file.  Doubles can be sorted and packed to
2745 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2746 //         varargs C calling conventions.
2747 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2748 //         even aligned with pad0 as needed.
2749 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2750 //         region 6-11 is even aligned; it may be padded out more so that
2751 //         the region from SP to FP meets the minimum stack alignment.
2752 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2753 //         alignment.  Region 11, pad1, may be dynamically extended so that
2754 //         SP meets the minimum alignment.
2755 
2756 frame
2757 %{
2758   // What direction does stack grow in (assumed to be same for C & Java)
2759   stack_direction(TOWARDS_LOW);
2760 
2761   // These three registers define part of the calling convention
2762   // between compiled code and the interpreter.
2763   inline_cache_reg(RAX);                // Inline Cache Register
2764   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2765                                         // calling interpreter
2766 
2767   // Optional: name the operand used by cisc-spilling to access
2768   // [stack_pointer + offset]
2769   cisc_spilling_operand_name(indOffset32);
2770 
2771   // Number of stack slots consumed by locking an object
2772   sync_stack_slots(2);
2773 
2774   // Compiled code's Frame Pointer
2775   frame_pointer(RSP);
2776 
2777   // Interpreter stores its frame pointer in a register which is
2778   // stored to the stack by I2CAdaptors.
2779   // I2CAdaptors convert from interpreted java to compiled java.
2780   interpreter_frame_pointer(RBP);
2781 
2782   // Stack alignment requirement
2783   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2784 
2785   // Number of stack slots between incoming argument block and the start of
2786   // a new frame.  The PROLOG must add this many slots to the stack.  The
2787   // EPILOG must remove this many slots.  amd64 needs two slots for
2788   // return address.
2789   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2790 
2791   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2792   // for calls to C.  Supports the var-args backing area for register parms.
2793   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2794 
2795   // The after-PROLOG location of the return address.  Location of
2796   // return address specifies a type (REG or STACK) and a number
2797   // representing the register number (i.e. - use a register name) or
2798   // stack slot.
2799   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2800   // Otherwise, it is above the locks and verification slot and alignment word
2801   return_addr(STACK - 2 +
2802               round_to((Compile::current()->in_preserve_stack_slots() +
2803                         Compile::current()->fixed_slots()),
2804                        stack_alignment_in_slots()));
2805 
2806   // Body of function which returns an integer array locating
2807   // arguments either in registers or in stack slots.  Passed an array
2808   // of ideal registers called "sig" and a "length" count.  Stack-slot
2809   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2810   // arguments for a CALLEE.  Incoming stack arguments are
2811   // automatically biased by the preserve_stack_slots field above.
2812 
2813   calling_convention
2814   %{
2815     // No difference between ingoing/outgoing just pass false
2816     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2817   %}
2818 
2819   c_calling_convention
2820   %{
2821     // This is obviously always outgoing
2822     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2823   %}
2824 
2825   // Location of compiled Java return values.  Same as C for now.
2826   return_value
2827   %{
2828     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2829            "only return normal values");
2830 
2831     static const int lo[Op_RegL + 1] = {
2832       0,
2833       0,
2834       RAX_num,  // Op_RegN
2835       RAX_num,  // Op_RegI
2836       RAX_num,  // Op_RegP
2837       XMM0_num, // Op_RegF
2838       XMM0_num, // Op_RegD
2839       RAX_num   // Op_RegL
2840     };
2841     static const int hi[Op_RegL + 1] = {
2842       0,
2843       0,
2844       OptoReg::Bad, // Op_RegN
2845       OptoReg::Bad, // Op_RegI
2846       RAX_H_num,    // Op_RegP
2847       OptoReg::Bad, // Op_RegF
2848       XMM0b_num,    // Op_RegD
2849       RAX_H_num     // Op_RegL
2850     };
2851     // Excluded flags and vector registers.
2852     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2853     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2854   %}
2855 %}
2856 
2857 //----------ATTRIBUTES---------------------------------------------------------
2858 //----------Operand Attributes-------------------------------------------------
2859 op_attrib op_cost(0);        // Required cost attribute
2860 
2861 //----------Instruction Attributes---------------------------------------------
2862 ins_attrib ins_cost(100);       // Required cost attribute
2863 ins_attrib ins_size(8);         // Required size attribute (in bits)
2864 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2865                                 // a non-matching short branch variant
2866                                 // of some long branch?
2867 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2868                                 // be a power of 2) specifies the
2869                                 // alignment that some part of the
2870                                 // instruction (not necessarily the
2871                                 // start) requires.  If > 1, a
2872                                 // compute_padding() function must be
2873                                 // provided for the instruction
2874 
2875 //----------OPERANDS-----------------------------------------------------------
2876 // Operand definitions must precede instruction definitions for correct parsing
2877 // in the ADLC because operands constitute user defined types which are used in
2878 // instruction definitions.
2879 
2880 //----------Simple Operands----------------------------------------------------
2881 // Immediate Operands
2882 // Integer Immediate
2883 operand immI()
2884 %{
2885   match(ConI);
2886 
2887   op_cost(10);
2888   format %{ %}
2889   interface(CONST_INTER);
2890 %}
2891 
2892 // Constant for test vs zero
2893 operand immI0()
2894 %{
2895   predicate(n->get_int() == 0);
2896   match(ConI);
2897 
2898   op_cost(0);
2899   format %{ %}
2900   interface(CONST_INTER);
2901 %}
2902 
2903 // Constant for increment
2904 operand immI1()
2905 %{
2906   predicate(n->get_int() == 1);
2907   match(ConI);
2908 
2909   op_cost(0);
2910   format %{ %}
2911   interface(CONST_INTER);
2912 %}
2913 
2914 // Constant for decrement
2915 operand immI_M1()
2916 %{
2917   predicate(n->get_int() == -1);
2918   match(ConI);
2919 
2920   op_cost(0);
2921   format %{ %}
2922   interface(CONST_INTER);
2923 %}
2924 
2925 // Valid scale values for addressing modes
2926 operand immI2()
2927 %{
2928   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2929   match(ConI);
2930 
2931   format %{ %}
2932   interface(CONST_INTER);
2933 %}
2934 
2935 operand immI8()
2936 %{
2937   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2938   match(ConI);
2939 
2940   op_cost(5);
2941   format %{ %}
2942   interface(CONST_INTER);
2943 %}
2944 
2945 operand immI16()
2946 %{
2947   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2948   match(ConI);
2949 
2950   op_cost(10);
2951   format %{ %}
2952   interface(CONST_INTER);
2953 %}
2954 
2955 // Int Immediate non-negative
2956 operand immU31()
2957 %{
2958   predicate(n->get_int() >= 0);
2959   match(ConI);
2960 
2961   op_cost(0);
2962   format %{ %}
2963   interface(CONST_INTER);
2964 %}
2965 
2966 // Constant for long shifts
2967 operand immI_32()
2968 %{
2969   predicate( n->get_int() == 32 );
2970   match(ConI);
2971 
2972   op_cost(0);
2973   format %{ %}
2974   interface(CONST_INTER);
2975 %}
2976 
2977 // Constant for long shifts
2978 operand immI_64()
2979 %{
2980   predicate( n->get_int() == 64 );
2981   match(ConI);
2982 
2983   op_cost(0);
2984   format %{ %}
2985   interface(CONST_INTER);
2986 %}
2987 
2988 // Pointer Immediate
2989 operand immP()
2990 %{
2991   match(ConP);
2992 
2993   op_cost(10);
2994   format %{ %}
2995   interface(CONST_INTER);
2996 %}
2997 
2998 // NULL Pointer Immediate
2999 operand immP0()
3000 %{
3001   predicate(n->get_ptr() == 0);
3002   match(ConP);
3003 
3004   op_cost(5);
3005   format %{ %}
3006   interface(CONST_INTER);
3007 %}
3008 
3009 // Pointer Immediate
3010 operand immN() %{
3011   match(ConN);
3012 
3013   op_cost(10);
3014   format %{ %}
3015   interface(CONST_INTER);
3016 %}
3017 
3018 operand immNKlass() %{
3019   match(ConNKlass);
3020 
3021   op_cost(10);
3022   format %{ %}
3023   interface(CONST_INTER);
3024 %}
3025 
3026 // NULL Pointer Immediate
3027 operand immN0() %{
3028   predicate(n->get_narrowcon() == 0);
3029   match(ConN);
3030 
3031   op_cost(5);
3032   format %{ %}
3033   interface(CONST_INTER);
3034 %}
3035 
3036 operand immP31()
3037 %{
3038   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3039             && (n->get_ptr() >> 31) == 0);
3040   match(ConP);
3041 
3042   op_cost(5);
3043   format %{ %}
3044   interface(CONST_INTER);
3045 %}
3046 
3047 
3048 // Long Immediate
3049 operand immL()
3050 %{
3051   match(ConL);
3052 
3053   op_cost(20);
3054   format %{ %}
3055   interface(CONST_INTER);
3056 %}
3057 
3058 // Long Immediate 8-bit
3059 operand immL8()
3060 %{
3061   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3062   match(ConL);
3063 
3064   op_cost(5);
3065   format %{ %}
3066   interface(CONST_INTER);
3067 %}
3068 
3069 // Long Immediate 32-bit unsigned
3070 operand immUL32()
3071 %{
3072   predicate(n->get_long() == (unsigned int) (n->get_long()));
3073   match(ConL);
3074 
3075   op_cost(10);
3076   format %{ %}
3077   interface(CONST_INTER);
3078 %}
3079 
3080 // Long Immediate 32-bit signed
3081 operand immL32()
3082 %{
3083   predicate(n->get_long() == (int) (n->get_long()));
3084   match(ConL);
3085 
3086   op_cost(15);
3087   format %{ %}
3088   interface(CONST_INTER);
3089 %}
3090 
3091 // Long Immediate zero
3092 operand immL0()
3093 %{
3094   predicate(n->get_long() == 0L);
3095   match(ConL);
3096 
3097   op_cost(10);
3098   format %{ %}
3099   interface(CONST_INTER);
3100 %}
3101 
3102 // Constant for increment
3103 operand immL1()
3104 %{
3105   predicate(n->get_long() == 1);
3106   match(ConL);
3107 
3108   format %{ %}
3109   interface(CONST_INTER);
3110 %}
3111 
3112 // Constant for decrement
3113 operand immL_M1()
3114 %{
3115   predicate(n->get_long() == -1);
3116   match(ConL);
3117 
3118   format %{ %}
3119   interface(CONST_INTER);
3120 %}
3121 
3122 // Long Immediate: the value 10
3123 operand immL10()
3124 %{
3125   predicate(n->get_long() == 10);
3126   match(ConL);
3127 
3128   format %{ %}
3129   interface(CONST_INTER);
3130 %}
3131 
3132 // Long immediate from 0 to 127.
3133 // Used for a shorter form of long mul by 10.
3134 operand immL_127()
3135 %{
3136   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3137   match(ConL);
3138 
3139   op_cost(10);
3140   format %{ %}
3141   interface(CONST_INTER);
3142 %}
3143 
3144 // Long Immediate: low 32-bit mask
3145 operand immL_32bits()
3146 %{
3147   predicate(n->get_long() == 0xFFFFFFFFL);
3148   match(ConL);
3149   op_cost(20);
3150 
3151   format %{ %}
3152   interface(CONST_INTER);
3153 %}
3154 
3155 // Float Immediate zero
3156 operand immF0()
3157 %{
3158   predicate(jint_cast(n->getf()) == 0);
3159   match(ConF);
3160 
3161   op_cost(5);
3162   format %{ %}
3163   interface(CONST_INTER);
3164 %}
3165 
3166 // Float Immediate
3167 operand immF()
3168 %{
3169   match(ConF);
3170 
3171   op_cost(15);
3172   format %{ %}
3173   interface(CONST_INTER);
3174 %}
3175 
3176 // Double Immediate zero
3177 operand immD0()
3178 %{
3179   predicate(jlong_cast(n->getd()) == 0);
3180   match(ConD);
3181 
3182   op_cost(5);
3183   format %{ %}
3184   interface(CONST_INTER);
3185 %}
3186 
3187 // Double Immediate
3188 operand immD()
3189 %{
3190   match(ConD);
3191 
3192   op_cost(15);
3193   format %{ %}
3194   interface(CONST_INTER);
3195 %}
3196 
3197 // Immediates for special shifts (sign extend)
3198 
3199 // Constants for increment
3200 operand immI_16()
3201 %{
3202   predicate(n->get_int() == 16);
3203   match(ConI);
3204 
3205   format %{ %}
3206   interface(CONST_INTER);
3207 %}
3208 
3209 operand immI_24()
3210 %{
3211   predicate(n->get_int() == 24);
3212   match(ConI);
3213 
3214   format %{ %}
3215   interface(CONST_INTER);
3216 %}
3217 
3218 // Constant for byte-wide masking
3219 operand immI_255()
3220 %{
3221   predicate(n->get_int() == 255);
3222   match(ConI);
3223 
3224   format %{ %}
3225   interface(CONST_INTER);
3226 %}
3227 
3228 // Constant for short-wide masking
3229 operand immI_65535()
3230 %{
3231   predicate(n->get_int() == 65535);
3232   match(ConI);
3233 
3234   format %{ %}
3235   interface(CONST_INTER);
3236 %}
3237 
3238 // Constant for byte-wide masking
3239 operand immL_255()
3240 %{
3241   predicate(n->get_long() == 255);
3242   match(ConL);
3243 
3244   format %{ %}
3245   interface(CONST_INTER);
3246 %}
3247 
3248 // Constant for short-wide masking
3249 operand immL_65535()
3250 %{
3251   predicate(n->get_long() == 65535);
3252   match(ConL);
3253 
3254   format %{ %}
3255   interface(CONST_INTER);
3256 %}
3257 
3258 // Register Operands
3259 // Integer Register
3260 operand rRegI()
3261 %{
3262   constraint(ALLOC_IN_RC(int_reg));
3263   match(RegI);
3264 
3265   match(rax_RegI);
3266   match(rbx_RegI);
3267   match(rcx_RegI);
3268   match(rdx_RegI);
3269   match(rdi_RegI);
3270 
3271   format %{ %}
3272   interface(REG_INTER);
3273 %}
3274 
3275 // Special Registers
3276 operand rax_RegI()
3277 %{
3278   constraint(ALLOC_IN_RC(int_rax_reg));
3279   match(RegI);
3280   match(rRegI);
3281 
3282   format %{ "RAX" %}
3283   interface(REG_INTER);
3284 %}
3285 
3286 // Special Registers
3287 operand rbx_RegI()
3288 %{
3289   constraint(ALLOC_IN_RC(int_rbx_reg));
3290   match(RegI);
3291   match(rRegI);
3292 
3293   format %{ "RBX" %}
3294   interface(REG_INTER);
3295 %}
3296 
3297 operand rcx_RegI()
3298 %{
3299   constraint(ALLOC_IN_RC(int_rcx_reg));
3300   match(RegI);
3301   match(rRegI);
3302 
3303   format %{ "RCX" %}
3304   interface(REG_INTER);
3305 %}
3306 
3307 operand rdx_RegI()
3308 %{
3309   constraint(ALLOC_IN_RC(int_rdx_reg));
3310   match(RegI);
3311   match(rRegI);
3312 
3313   format %{ "RDX" %}
3314   interface(REG_INTER);
3315 %}
3316 
3317 operand rdi_RegI()
3318 %{
3319   constraint(ALLOC_IN_RC(int_rdi_reg));
3320   match(RegI);
3321   match(rRegI);
3322 
3323   format %{ "RDI" %}
3324   interface(REG_INTER);
3325 %}
3326 
3327 operand no_rcx_RegI()
3328 %{
3329   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3330   match(RegI);
3331   match(rax_RegI);
3332   match(rbx_RegI);
3333   match(rdx_RegI);
3334   match(rdi_RegI);
3335 
3336   format %{ %}
3337   interface(REG_INTER);
3338 %}
3339 
3340 operand no_rax_rdx_RegI()
3341 %{
3342   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3343   match(RegI);
3344   match(rbx_RegI);
3345   match(rcx_RegI);
3346   match(rdi_RegI);
3347 
3348   format %{ %}
3349   interface(REG_INTER);
3350 %}
3351 
3352 // Pointer Register
3353 operand any_RegP()
3354 %{
3355   constraint(ALLOC_IN_RC(any_reg));
3356   match(RegP);
3357   match(rax_RegP);
3358   match(rbx_RegP);
3359   match(rdi_RegP);
3360   match(rsi_RegP);
3361   match(rbp_RegP);
3362   match(r15_RegP);
3363   match(rRegP);
3364 
3365   format %{ %}
3366   interface(REG_INTER);
3367 %}
3368 
3369 operand rRegP()
3370 %{
3371   constraint(ALLOC_IN_RC(ptr_reg));
3372   match(RegP);
3373   match(rax_RegP);
3374   match(rbx_RegP);
3375   match(rdi_RegP);
3376   match(rsi_RegP);
3377   match(rbp_RegP);  // See Q&A below about
3378   match(r15_RegP);  // r15_RegP and rbp_RegP.
3379 
3380   format %{ %}
3381   interface(REG_INTER);
3382 %}
3383 
3384 operand rRegN() %{
3385   constraint(ALLOC_IN_RC(int_reg));
3386   match(RegN);
3387 
3388   format %{ %}
3389   interface(REG_INTER);
3390 %}
3391 
3392 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3393 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3394 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3395 // The output of an instruction is controlled by the allocator, which respects
3396 // register class masks, not match rules.  Unless an instruction mentions
3397 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3398 // by the allocator as an input.
3399 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3400 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3401 // result, RBP is not included in the output of the instruction either.
3402 
3403 operand no_rax_RegP()
3404 %{
3405   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3406   match(RegP);
3407   match(rbx_RegP);
3408   match(rsi_RegP);
3409   match(rdi_RegP);
3410 
3411   format %{ %}
3412   interface(REG_INTER);
3413 %}
3414 
3415 // This operand is not allowed to use RBP even if
3416 // RBP is not used to hold the frame pointer.
3417 operand no_rbp_RegP()
3418 %{
3419   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3420   match(RegP);
3421   match(rbx_RegP);
3422   match(rsi_RegP);
3423   match(rdi_RegP);
3424 
3425   format %{ %}
3426   interface(REG_INTER);
3427 %}
3428 
3429 operand no_rax_rbx_RegP()
3430 %{
3431   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3432   match(RegP);
3433   match(rsi_RegP);
3434   match(rdi_RegP);
3435 
3436   format %{ %}
3437   interface(REG_INTER);
3438 %}
3439 
3440 // Special Registers
3441 // Return a pointer value
3442 operand rax_RegP()
3443 %{
3444   constraint(ALLOC_IN_RC(ptr_rax_reg));
3445   match(RegP);
3446   match(rRegP);
3447 
3448   format %{ %}
3449   interface(REG_INTER);
3450 %}
3451 
3452 // Special Registers
3453 // Return a compressed pointer value
3454 operand rax_RegN()
3455 %{
3456   constraint(ALLOC_IN_RC(int_rax_reg));
3457   match(RegN);
3458   match(rRegN);
3459 
3460   format %{ %}
3461   interface(REG_INTER);
3462 %}
3463 
3464 // Used in AtomicAdd
3465 operand rbx_RegP()
3466 %{
3467   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3468   match(RegP);
3469   match(rRegP);
3470 
3471   format %{ %}
3472   interface(REG_INTER);
3473 %}
3474 
3475 operand rsi_RegP()
3476 %{
3477   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3478   match(RegP);
3479   match(rRegP);
3480 
3481   format %{ %}
3482   interface(REG_INTER);
3483 %}
3484 
3485 // Used in rep stosq
3486 operand rdi_RegP()
3487 %{
3488   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3489   match(RegP);
3490   match(rRegP);
3491 
3492   format %{ %}
3493   interface(REG_INTER);
3494 %}
3495 
3496 operand r15_RegP()
3497 %{
3498   constraint(ALLOC_IN_RC(ptr_r15_reg));
3499   match(RegP);
3500   match(rRegP);
3501 
3502   format %{ %}
3503   interface(REG_INTER);
3504 %}
3505 
3506 operand rRegL()
3507 %{
3508   constraint(ALLOC_IN_RC(long_reg));
3509   match(RegL);
3510   match(rax_RegL);
3511   match(rdx_RegL);
3512 
3513   format %{ %}
3514   interface(REG_INTER);
3515 %}
3516 
3517 // Special Registers
3518 operand no_rax_rdx_RegL()
3519 %{
3520   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3521   match(RegL);
3522   match(rRegL);
3523 
3524   format %{ %}
3525   interface(REG_INTER);
3526 %}
3527 
3528 operand no_rax_RegL()
3529 %{
3530   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3531   match(RegL);
3532   match(rRegL);
3533   match(rdx_RegL);
3534 
3535   format %{ %}
3536   interface(REG_INTER);
3537 %}
3538 
3539 operand no_rcx_RegL()
3540 %{
3541   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3542   match(RegL);
3543   match(rRegL);
3544 
3545   format %{ %}
3546   interface(REG_INTER);
3547 %}
3548 
3549 operand rax_RegL()
3550 %{
3551   constraint(ALLOC_IN_RC(long_rax_reg));
3552   match(RegL);
3553   match(rRegL);
3554 
3555   format %{ "RAX" %}
3556   interface(REG_INTER);
3557 %}
3558 
3559 operand rcx_RegL()
3560 %{
3561   constraint(ALLOC_IN_RC(long_rcx_reg));
3562   match(RegL);
3563   match(rRegL);
3564 
3565   format %{ %}
3566   interface(REG_INTER);
3567 %}
3568 
3569 operand rdx_RegL()
3570 %{
3571   constraint(ALLOC_IN_RC(long_rdx_reg));
3572   match(RegL);
3573   match(rRegL);
3574 
3575   format %{ %}
3576   interface(REG_INTER);
3577 %}
3578 
3579 // Flags register, used as output of compare instructions
3580 operand rFlagsReg()
3581 %{
3582   constraint(ALLOC_IN_RC(int_flags));
3583   match(RegFlags);
3584 
3585   format %{ "RFLAGS" %}
3586   interface(REG_INTER);
3587 %}
3588 
3589 // Flags register, used as output of FLOATING POINT compare instructions
3590 operand rFlagsRegU()
3591 %{
3592   constraint(ALLOC_IN_RC(int_flags));
3593   match(RegFlags);
3594 
3595   format %{ "RFLAGS_U" %}
3596   interface(REG_INTER);
3597 %}
3598 
3599 operand rFlagsRegUCF() %{
3600   constraint(ALLOC_IN_RC(int_flags));
3601   match(RegFlags);
3602   predicate(false);
3603 
3604   format %{ "RFLAGS_U_CF" %}
3605   interface(REG_INTER);
3606 %}
3607 
3608 // Float register operands
3609 operand regF() %{
3610    constraint(ALLOC_IN_RC(float_reg));
3611    match(RegF);
3612 
3613    format %{ %}
3614    interface(REG_INTER);
3615 %}
3616 
3617 // Double register operands
3618 operand regD() %{
3619    constraint(ALLOC_IN_RC(double_reg));
3620    match(RegD);
3621 
3622    format %{ %}
3623    interface(REG_INTER);
3624 %}
3625 
3626 // Vectors
3627 operand vecS() %{
3628   constraint(ALLOC_IN_RC(vectors_reg));
3629   match(VecS);
3630 
3631   format %{ %}
3632   interface(REG_INTER);
3633 %}
3634 
3635 operand vecD() %{
3636   constraint(ALLOC_IN_RC(vectord_reg));
3637   match(VecD);
3638 
3639   format %{ %}
3640   interface(REG_INTER);
3641 %}
3642 
3643 operand vecX() %{
3644   constraint(ALLOC_IN_RC(vectorx_reg));
3645   match(VecX);
3646 
3647   format %{ %}
3648   interface(REG_INTER);
3649 %}
3650 
3651 operand vecY() %{
3652   constraint(ALLOC_IN_RC(vectory_reg));
3653   match(VecY);
3654 
3655   format %{ %}
3656   interface(REG_INTER);
3657 %}
3658 
3659 //----------Memory Operands----------------------------------------------------
3660 // Direct Memory Operand
3661 // operand direct(immP addr)
3662 // %{
3663 //   match(addr);
3664 
3665 //   format %{ "[$addr]" %}
3666 //   interface(MEMORY_INTER) %{
3667 //     base(0xFFFFFFFF);
3668 //     index(0x4);
3669 //     scale(0x0);
3670 //     disp($addr);
3671 //   %}
3672 // %}
3673 
3674 // Indirect Memory Operand
3675 operand indirect(any_RegP reg)
3676 %{
3677   constraint(ALLOC_IN_RC(ptr_reg));
3678   match(reg);
3679 
3680   format %{ "[$reg]" %}
3681   interface(MEMORY_INTER) %{
3682     base($reg);
3683     index(0x4);
3684     scale(0x0);
3685     disp(0x0);
3686   %}
3687 %}
3688 
3689 // Indirect Memory Plus Short Offset Operand
3690 operand indOffset8(any_RegP reg, immL8 off)
3691 %{
3692   constraint(ALLOC_IN_RC(ptr_reg));
3693   match(AddP reg off);
3694 
3695   format %{ "[$reg + $off (8-bit)]" %}
3696   interface(MEMORY_INTER) %{
3697     base($reg);
3698     index(0x4);
3699     scale(0x0);
3700     disp($off);
3701   %}
3702 %}
3703 
3704 // Indirect Memory Plus Long Offset Operand
3705 operand indOffset32(any_RegP reg, immL32 off)
3706 %{
3707   constraint(ALLOC_IN_RC(ptr_reg));
3708   match(AddP reg off);
3709 
3710   format %{ "[$reg + $off (32-bit)]" %}
3711   interface(MEMORY_INTER) %{
3712     base($reg);
3713     index(0x4);
3714     scale(0x0);
3715     disp($off);
3716   %}
3717 %}
3718 
3719 // Indirect Memory Plus Index Register Plus Offset Operand
3720 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3721 %{
3722   constraint(ALLOC_IN_RC(ptr_reg));
3723   match(AddP (AddP reg lreg) off);
3724 
3725   op_cost(10);
3726   format %{"[$reg + $off + $lreg]" %}
3727   interface(MEMORY_INTER) %{
3728     base($reg);
3729     index($lreg);
3730     scale(0x0);
3731     disp($off);
3732   %}
3733 %}
3734 
3735 // Indirect Memory Plus Index Register Plus Offset Operand
3736 operand indIndex(any_RegP reg, rRegL lreg)
3737 %{
3738   constraint(ALLOC_IN_RC(ptr_reg));
3739   match(AddP reg lreg);
3740 
3741   op_cost(10);
3742   format %{"[$reg + $lreg]" %}
3743   interface(MEMORY_INTER) %{
3744     base($reg);
3745     index($lreg);
3746     scale(0x0);
3747     disp(0x0);
3748   %}
3749 %}
3750 
3751 // Indirect Memory Times Scale Plus Index Register
3752 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3753 %{
3754   constraint(ALLOC_IN_RC(ptr_reg));
3755   match(AddP reg (LShiftL lreg scale));
3756 
3757   op_cost(10);
3758   format %{"[$reg + $lreg << $scale]" %}
3759   interface(MEMORY_INTER) %{
3760     base($reg);
3761     index($lreg);
3762     scale($scale);
3763     disp(0x0);
3764   %}
3765 %}
3766 
3767 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3768 %{
3769   constraint(ALLOC_IN_RC(ptr_reg));
3770   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3771   match(AddP reg (LShiftL (ConvI2L idx) scale));
3772 
3773   op_cost(10);
3774   format %{"[$reg + pos $idx << $scale]" %}
3775   interface(MEMORY_INTER) %{
3776     base($reg);
3777     index($idx);
3778     scale($scale);
3779     disp(0x0);
3780   %}
3781 %}
3782 
3783 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3784 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3785 %{
3786   constraint(ALLOC_IN_RC(ptr_reg));
3787   match(AddP (AddP reg (LShiftL lreg scale)) off);
3788 
3789   op_cost(10);
3790   format %{"[$reg + $off + $lreg << $scale]" %}
3791   interface(MEMORY_INTER) %{
3792     base($reg);
3793     index($lreg);
3794     scale($scale);
3795     disp($off);
3796   %}
3797 %}
3798 
3799 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3800 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3801 %{
3802   constraint(ALLOC_IN_RC(ptr_reg));
3803   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3804   match(AddP (AddP reg (ConvI2L idx)) off);
3805 
3806   op_cost(10);
3807   format %{"[$reg + $off + $idx]" %}
3808   interface(MEMORY_INTER) %{
3809     base($reg);
3810     index($idx);
3811     scale(0x0);
3812     disp($off);
3813   %}
3814 %}
3815 
3816 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3817 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3818 %{
3819   constraint(ALLOC_IN_RC(ptr_reg));
3820   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3821   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3822 
3823   op_cost(10);
3824   format %{"[$reg + $off + $idx << $scale]" %}
3825   interface(MEMORY_INTER) %{
3826     base($reg);
3827     index($idx);
3828     scale($scale);
3829     disp($off);
3830   %}
3831 %}
3832 
3833 // Indirect Narrow Oop Plus Offset Operand
3834 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3835 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3836 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3837   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3838   constraint(ALLOC_IN_RC(ptr_reg));
3839   match(AddP (DecodeN reg) off);
3840 
3841   op_cost(10);
3842   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3843   interface(MEMORY_INTER) %{
3844     base(0xc); // R12
3845     index($reg);
3846     scale(0x3);
3847     disp($off);
3848   %}
3849 %}
3850 
3851 // Indirect Memory Operand
3852 operand indirectNarrow(rRegN reg)
3853 %{
3854   predicate(Universe::narrow_oop_shift() == 0);
3855   constraint(ALLOC_IN_RC(ptr_reg));
3856   match(DecodeN reg);
3857 
3858   format %{ "[$reg]" %}
3859   interface(MEMORY_INTER) %{
3860     base($reg);
3861     index(0x4);
3862     scale(0x0);
3863     disp(0x0);
3864   %}
3865 %}
3866 
3867 // Indirect Memory Plus Short Offset Operand
3868 operand indOffset8Narrow(rRegN reg, immL8 off)
3869 %{
3870   predicate(Universe::narrow_oop_shift() == 0);
3871   constraint(ALLOC_IN_RC(ptr_reg));
3872   match(AddP (DecodeN reg) off);
3873 
3874   format %{ "[$reg + $off (8-bit)]" %}
3875   interface(MEMORY_INTER) %{
3876     base($reg);
3877     index(0x4);
3878     scale(0x0);
3879     disp($off);
3880   %}
3881 %}
3882 
3883 // Indirect Memory Plus Long Offset Operand
3884 operand indOffset32Narrow(rRegN reg, immL32 off)
3885 %{
3886   predicate(Universe::narrow_oop_shift() == 0);
3887   constraint(ALLOC_IN_RC(ptr_reg));
3888   match(AddP (DecodeN reg) off);
3889 
3890   format %{ "[$reg + $off (32-bit)]" %}
3891   interface(MEMORY_INTER) %{
3892     base($reg);
3893     index(0x4);
3894     scale(0x0);
3895     disp($off);
3896   %}
3897 %}
3898 
3899 // Indirect Memory Plus Index Register Plus Offset Operand
3900 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3901 %{
3902   predicate(Universe::narrow_oop_shift() == 0);
3903   constraint(ALLOC_IN_RC(ptr_reg));
3904   match(AddP (AddP (DecodeN reg) lreg) off);
3905 
3906   op_cost(10);
3907   format %{"[$reg + $off + $lreg]" %}
3908   interface(MEMORY_INTER) %{
3909     base($reg);
3910     index($lreg);
3911     scale(0x0);
3912     disp($off);
3913   %}
3914 %}
3915 
3916 // Indirect Memory Plus Index Register Plus Offset Operand
3917 operand indIndexNarrow(rRegN reg, rRegL lreg)
3918 %{
3919   predicate(Universe::narrow_oop_shift() == 0);
3920   constraint(ALLOC_IN_RC(ptr_reg));
3921   match(AddP (DecodeN reg) lreg);
3922 
3923   op_cost(10);
3924   format %{"[$reg + $lreg]" %}
3925   interface(MEMORY_INTER) %{
3926     base($reg);
3927     index($lreg);
3928     scale(0x0);
3929     disp(0x0);
3930   %}
3931 %}
3932 
3933 // Indirect Memory Times Scale Plus Index Register
3934 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3935 %{
3936   predicate(Universe::narrow_oop_shift() == 0);
3937   constraint(ALLOC_IN_RC(ptr_reg));
3938   match(AddP (DecodeN reg) (LShiftL lreg scale));
3939 
3940   op_cost(10);
3941   format %{"[$reg + $lreg << $scale]" %}
3942   interface(MEMORY_INTER) %{
3943     base($reg);
3944     index($lreg);
3945     scale($scale);
3946     disp(0x0);
3947   %}
3948 %}
3949 
3950 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3951 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3952 %{
3953   predicate(Universe::narrow_oop_shift() == 0);
3954   constraint(ALLOC_IN_RC(ptr_reg));
3955   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3956 
3957   op_cost(10);
3958   format %{"[$reg + $off + $lreg << $scale]" %}
3959   interface(MEMORY_INTER) %{
3960     base($reg);
3961     index($lreg);
3962     scale($scale);
3963     disp($off);
3964   %}
3965 %}
3966 
3967 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3968 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3969 %{
3970   constraint(ALLOC_IN_RC(ptr_reg));
3971   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3972   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3973 
3974   op_cost(10);
3975   format %{"[$reg + $off + $idx]" %}
3976   interface(MEMORY_INTER) %{
3977     base($reg);
3978     index($idx);
3979     scale(0x0);
3980     disp($off);
3981   %}
3982 %}
3983 
3984 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3985 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3986 %{
3987   constraint(ALLOC_IN_RC(ptr_reg));
3988   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3989   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3990 
3991   op_cost(10);
3992   format %{"[$reg + $off + $idx << $scale]" %}
3993   interface(MEMORY_INTER) %{
3994     base($reg);
3995     index($idx);
3996     scale($scale);
3997     disp($off);
3998   %}
3999 %}
4000 
4001 //----------Special Memory Operands--------------------------------------------
4002 // Stack Slot Operand - This operand is used for loading and storing temporary
4003 //                      values on the stack where a match requires a value to
4004 //                      flow through memory.
4005 operand stackSlotP(sRegP reg)
4006 %{
4007   constraint(ALLOC_IN_RC(stack_slots));
4008   // No match rule because this operand is only generated in matching
4009 
4010   format %{ "[$reg]" %}
4011   interface(MEMORY_INTER) %{
4012     base(0x4);   // RSP
4013     index(0x4);  // No Index
4014     scale(0x0);  // No Scale
4015     disp($reg);  // Stack Offset
4016   %}
4017 %}
4018 
4019 operand stackSlotI(sRegI reg)
4020 %{
4021   constraint(ALLOC_IN_RC(stack_slots));
4022   // No match rule because this operand is only generated in matching
4023 
4024   format %{ "[$reg]" %}
4025   interface(MEMORY_INTER) %{
4026     base(0x4);   // RSP
4027     index(0x4);  // No Index
4028     scale(0x0);  // No Scale
4029     disp($reg);  // Stack Offset
4030   %}
4031 %}
4032 
4033 operand stackSlotF(sRegF reg)
4034 %{
4035   constraint(ALLOC_IN_RC(stack_slots));
4036   // No match rule because this operand is only generated in matching
4037 
4038   format %{ "[$reg]" %}
4039   interface(MEMORY_INTER) %{
4040     base(0x4);   // RSP
4041     index(0x4);  // No Index
4042     scale(0x0);  // No Scale
4043     disp($reg);  // Stack Offset
4044   %}
4045 %}
4046 
4047 operand stackSlotD(sRegD reg)
4048 %{
4049   constraint(ALLOC_IN_RC(stack_slots));
4050   // No match rule because this operand is only generated in matching
4051 
4052   format %{ "[$reg]" %}
4053   interface(MEMORY_INTER) %{
4054     base(0x4);   // RSP
4055     index(0x4);  // No Index
4056     scale(0x0);  // No Scale
4057     disp($reg);  // Stack Offset
4058   %}
4059 %}
4060 operand stackSlotL(sRegL reg)
4061 %{
4062   constraint(ALLOC_IN_RC(stack_slots));
4063   // No match rule because this operand is only generated in matching
4064 
4065   format %{ "[$reg]" %}
4066   interface(MEMORY_INTER) %{
4067     base(0x4);   // RSP
4068     index(0x4);  // No Index
4069     scale(0x0);  // No Scale
4070     disp($reg);  // Stack Offset
4071   %}
4072 %}
4073 
4074 //----------Conditional Branch Operands----------------------------------------
4075 // Comparison Op  - This is the operation of the comparison, and is limited to
4076 //                  the following set of codes:
4077 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4078 //
4079 // Other attributes of the comparison, such as unsignedness, are specified
4080 // by the comparison instruction that sets a condition code flags register.
4081 // That result is represented by a flags operand whose subtype is appropriate
4082 // to the unsignedness (etc.) of the comparison.
4083 //
4084 // Later, the instruction which matches both the Comparison Op (a Bool) and
4085 // the flags (produced by the Cmp) specifies the coding of the comparison op
4086 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4087 
4088 // Comparision Code
4089 operand cmpOp()
4090 %{
4091   match(Bool);
4092 
4093   format %{ "" %}
4094   interface(COND_INTER) %{
4095     equal(0x4, "e");
4096     not_equal(0x5, "ne");
4097     less(0xC, "l");
4098     greater_equal(0xD, "ge");
4099     less_equal(0xE, "le");
4100     greater(0xF, "g");
4101     overflow(0x0, "o");
4102     no_overflow(0x1, "no");
4103   %}
4104 %}
4105 
4106 // Comparison Code, unsigned compare.  Used by FP also, with
4107 // C2 (unordered) turned into GT or LT already.  The other bits
4108 // C0 and C3 are turned into Carry & Zero flags.
4109 operand cmpOpU()
4110 %{
4111   match(Bool);
4112 
4113   format %{ "" %}
4114   interface(COND_INTER) %{
4115     equal(0x4, "e");
4116     not_equal(0x5, "ne");
4117     less(0x2, "b");
4118     greater_equal(0x3, "nb");
4119     less_equal(0x6, "be");
4120     greater(0x7, "nbe");
4121     overflow(0x0, "o");
4122     no_overflow(0x1, "no");
4123   %}
4124 %}
4125 
4126 
4127 // Floating comparisons that don't require any fixup for the unordered case
4128 operand cmpOpUCF() %{
4129   match(Bool);
4130   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4131             n->as_Bool()->_test._test == BoolTest::ge ||
4132             n->as_Bool()->_test._test == BoolTest::le ||
4133             n->as_Bool()->_test._test == BoolTest::gt);
4134   format %{ "" %}
4135   interface(COND_INTER) %{
4136     equal(0x4, "e");
4137     not_equal(0x5, "ne");
4138     less(0x2, "b");
4139     greater_equal(0x3, "nb");
4140     less_equal(0x6, "be");
4141     greater(0x7, "nbe");
4142     overflow(0x0, "o");
4143     no_overflow(0x1, "no");
4144   %}
4145 %}
4146 
4147 
4148 // Floating comparisons that can be fixed up with extra conditional jumps
4149 operand cmpOpUCF2() %{
4150   match(Bool);
4151   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4152             n->as_Bool()->_test._test == BoolTest::eq);
4153   format %{ "" %}
4154   interface(COND_INTER) %{
4155     equal(0x4, "e");
4156     not_equal(0x5, "ne");
4157     less(0x2, "b");
4158     greater_equal(0x3, "nb");
4159     less_equal(0x6, "be");
4160     greater(0x7, "nbe");
4161     overflow(0x0, "o");
4162     no_overflow(0x1, "no");
4163   %}
4164 %}
4165 
4166 
4167 //----------OPERAND CLASSES----------------------------------------------------
4168 // Operand Classes are groups of operands that are used as to simplify
4169 // instruction definitions by not requiring the AD writer to specify separate
4170 // instructions for every form of operand when the instruction accepts
4171 // multiple operand types with the same basic encoding and format.  The classic
4172 // case of this is memory operands.
4173 
4174 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4175                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4176                indCompressedOopOffset,
4177                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4178                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4179                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4180 
4181 //----------PIPELINE-----------------------------------------------------------
4182 // Rules which define the behavior of the target architectures pipeline.
4183 pipeline %{
4184 
4185 //----------ATTRIBUTES---------------------------------------------------------
4186 attributes %{
4187   variable_size_instructions;        // Fixed size instructions
4188   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4189   instruction_unit_size = 1;         // An instruction is 1 bytes long
4190   instruction_fetch_unit_size = 16;  // The processor fetches one line
4191   instruction_fetch_units = 1;       // of 16 bytes
4192 
4193   // List of nop instructions
4194   nops( MachNop );
4195 %}
4196 
4197 //----------RESOURCES----------------------------------------------------------
4198 // Resources are the functional units available to the machine
4199 
4200 // Generic P2/P3 pipeline
4201 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4202 // 3 instructions decoded per cycle.
4203 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4204 // 3 ALU op, only ALU0 handles mul instructions.
4205 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4206            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4207            BR, FPU,
4208            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4209 
4210 //----------PIPELINE DESCRIPTION-----------------------------------------------
4211 // Pipeline Description specifies the stages in the machine's pipeline
4212 
4213 // Generic P2/P3 pipeline
4214 pipe_desc(S0, S1, S2, S3, S4, S5);
4215 
4216 //----------PIPELINE CLASSES---------------------------------------------------
4217 // Pipeline Classes describe the stages in which input and output are
4218 // referenced by the hardware pipeline.
4219 
4220 // Naming convention: ialu or fpu
4221 // Then: _reg
4222 // Then: _reg if there is a 2nd register
4223 // Then: _long if it's a pair of instructions implementing a long
4224 // Then: _fat if it requires the big decoder
4225 //   Or: _mem if it requires the big decoder and a memory unit.
4226 
4227 // Integer ALU reg operation
4228 pipe_class ialu_reg(rRegI dst)
4229 %{
4230     single_instruction;
4231     dst    : S4(write);
4232     dst    : S3(read);
4233     DECODE : S0;        // any decoder
4234     ALU    : S3;        // any alu
4235 %}
4236 
4237 // Long ALU reg operation
4238 pipe_class ialu_reg_long(rRegL dst)
4239 %{
4240     instruction_count(2);
4241     dst    : S4(write);
4242     dst    : S3(read);
4243     DECODE : S0(2);     // any 2 decoders
4244     ALU    : S3(2);     // both alus
4245 %}
4246 
4247 // Integer ALU reg operation using big decoder
4248 pipe_class ialu_reg_fat(rRegI dst)
4249 %{
4250     single_instruction;
4251     dst    : S4(write);
4252     dst    : S3(read);
4253     D0     : S0;        // big decoder only
4254     ALU    : S3;        // any alu
4255 %}
4256 
4257 // Long ALU reg operation using big decoder
4258 pipe_class ialu_reg_long_fat(rRegL dst)
4259 %{
4260     instruction_count(2);
4261     dst    : S4(write);
4262     dst    : S3(read);
4263     D0     : S0(2);     // big decoder only; twice
4264     ALU    : S3(2);     // any 2 alus
4265 %}
4266 
4267 // Integer ALU reg-reg operation
4268 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4269 %{
4270     single_instruction;
4271     dst    : S4(write);
4272     src    : S3(read);
4273     DECODE : S0;        // any decoder
4274     ALU    : S3;        // any alu
4275 %}
4276 
4277 // Long ALU reg-reg operation
4278 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4279 %{
4280     instruction_count(2);
4281     dst    : S4(write);
4282     src    : S3(read);
4283     DECODE : S0(2);     // any 2 decoders
4284     ALU    : S3(2);     // both alus
4285 %}
4286 
4287 // Integer ALU reg-reg operation
4288 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4289 %{
4290     single_instruction;
4291     dst    : S4(write);
4292     src    : S3(read);
4293     D0     : S0;        // big decoder only
4294     ALU    : S3;        // any alu
4295 %}
4296 
4297 // Long ALU reg-reg operation
4298 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4299 %{
4300     instruction_count(2);
4301     dst    : S4(write);
4302     src    : S3(read);
4303     D0     : S0(2);     // big decoder only; twice
4304     ALU    : S3(2);     // both alus
4305 %}
4306 
4307 // Integer ALU reg-mem operation
4308 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4309 %{
4310     single_instruction;
4311     dst    : S5(write);
4312     mem    : S3(read);
4313     D0     : S0;        // big decoder only
4314     ALU    : S4;        // any alu
4315     MEM    : S3;        // any mem
4316 %}
4317 
4318 // Integer mem operation (prefetch)
4319 pipe_class ialu_mem(memory mem)
4320 %{
4321     single_instruction;
4322     mem    : S3(read);
4323     D0     : S0;        // big decoder only
4324     MEM    : S3;        // any mem
4325 %}
4326 
4327 // Integer Store to Memory
4328 pipe_class ialu_mem_reg(memory mem, rRegI src)
4329 %{
4330     single_instruction;
4331     mem    : S3(read);
4332     src    : S5(read);
4333     D0     : S0;        // big decoder only
4334     ALU    : S4;        // any alu
4335     MEM    : S3;
4336 %}
4337 
4338 // // Long Store to Memory
4339 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4340 // %{
4341 //     instruction_count(2);
4342 //     mem    : S3(read);
4343 //     src    : S5(read);
4344 //     D0     : S0(2);          // big decoder only; twice
4345 //     ALU    : S4(2);     // any 2 alus
4346 //     MEM    : S3(2);  // Both mems
4347 // %}
4348 
4349 // Integer Store to Memory
4350 pipe_class ialu_mem_imm(memory mem)
4351 %{
4352     single_instruction;
4353     mem    : S3(read);
4354     D0     : S0;        // big decoder only
4355     ALU    : S4;        // any alu
4356     MEM    : S3;
4357 %}
4358 
4359 // Integer ALU0 reg-reg operation
4360 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4361 %{
4362     single_instruction;
4363     dst    : S4(write);
4364     src    : S3(read);
4365     D0     : S0;        // Big decoder only
4366     ALU0   : S3;        // only alu0
4367 %}
4368 
4369 // Integer ALU0 reg-mem operation
4370 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4371 %{
4372     single_instruction;
4373     dst    : S5(write);
4374     mem    : S3(read);
4375     D0     : S0;        // big decoder only
4376     ALU0   : S4;        // ALU0 only
4377     MEM    : S3;        // any mem
4378 %}
4379 
4380 // Integer ALU reg-reg operation
4381 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4382 %{
4383     single_instruction;
4384     cr     : S4(write);
4385     src1   : S3(read);
4386     src2   : S3(read);
4387     DECODE : S0;        // any decoder
4388     ALU    : S3;        // any alu
4389 %}
4390 
4391 // Integer ALU reg-imm operation
4392 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4393 %{
4394     single_instruction;
4395     cr     : S4(write);
4396     src1   : S3(read);
4397     DECODE : S0;        // any decoder
4398     ALU    : S3;        // any alu
4399 %}
4400 
4401 // Integer ALU reg-mem operation
4402 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4403 %{
4404     single_instruction;
4405     cr     : S4(write);
4406     src1   : S3(read);
4407     src2   : S3(read);
4408     D0     : S0;        // big decoder only
4409     ALU    : S4;        // any alu
4410     MEM    : S3;
4411 %}
4412 
4413 // Conditional move reg-reg
4414 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4415 %{
4416     instruction_count(4);
4417     y      : S4(read);
4418     q      : S3(read);
4419     p      : S3(read);
4420     DECODE : S0(4);     // any decoder
4421 %}
4422 
4423 // Conditional move reg-reg
4424 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4425 %{
4426     single_instruction;
4427     dst    : S4(write);
4428     src    : S3(read);
4429     cr     : S3(read);
4430     DECODE : S0;        // any decoder
4431 %}
4432 
4433 // Conditional move reg-mem
4434 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4435 %{
4436     single_instruction;
4437     dst    : S4(write);
4438     src    : S3(read);
4439     cr     : S3(read);
4440     DECODE : S0;        // any decoder
4441     MEM    : S3;
4442 %}
4443 
4444 // Conditional move reg-reg long
4445 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4446 %{
4447     single_instruction;
4448     dst    : S4(write);
4449     src    : S3(read);
4450     cr     : S3(read);
4451     DECODE : S0(2);     // any 2 decoders
4452 %}
4453 
4454 // XXX
4455 // // Conditional move double reg-reg
4456 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4457 // %{
4458 //     single_instruction;
4459 //     dst    : S4(write);
4460 //     src    : S3(read);
4461 //     cr     : S3(read);
4462 //     DECODE : S0;     // any decoder
4463 // %}
4464 
4465 // Float reg-reg operation
4466 pipe_class fpu_reg(regD dst)
4467 %{
4468     instruction_count(2);
4469     dst    : S3(read);
4470     DECODE : S0(2);     // any 2 decoders
4471     FPU    : S3;
4472 %}
4473 
4474 // Float reg-reg operation
4475 pipe_class fpu_reg_reg(regD dst, regD src)
4476 %{
4477     instruction_count(2);
4478     dst    : S4(write);
4479     src    : S3(read);
4480     DECODE : S0(2);     // any 2 decoders
4481     FPU    : S3;
4482 %}
4483 
4484 // Float reg-reg operation
4485 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4486 %{
4487     instruction_count(3);
4488     dst    : S4(write);
4489     src1   : S3(read);
4490     src2   : S3(read);
4491     DECODE : S0(3);     // any 3 decoders
4492     FPU    : S3(2);
4493 %}
4494 
4495 // Float reg-reg operation
4496 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4497 %{
4498     instruction_count(4);
4499     dst    : S4(write);
4500     src1   : S3(read);
4501     src2   : S3(read);
4502     src3   : S3(read);
4503     DECODE : S0(4);     // any 3 decoders
4504     FPU    : S3(2);
4505 %}
4506 
4507 // Float reg-reg operation
4508 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4509 %{
4510     instruction_count(4);
4511     dst    : S4(write);
4512     src1   : S3(read);
4513     src2   : S3(read);
4514     src3   : S3(read);
4515     DECODE : S1(3);     // any 3 decoders
4516     D0     : S0;        // Big decoder only
4517     FPU    : S3(2);
4518     MEM    : S3;
4519 %}
4520 
4521 // Float reg-mem operation
4522 pipe_class fpu_reg_mem(regD dst, memory mem)
4523 %{
4524     instruction_count(2);
4525     dst    : S5(write);
4526     mem    : S3(read);
4527     D0     : S0;        // big decoder only
4528     DECODE : S1;        // any decoder for FPU POP
4529     FPU    : S4;
4530     MEM    : S3;        // any mem
4531 %}
4532 
4533 // Float reg-mem operation
4534 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4535 %{
4536     instruction_count(3);
4537     dst    : S5(write);
4538     src1   : S3(read);
4539     mem    : S3(read);
4540     D0     : S0;        // big decoder only
4541     DECODE : S1(2);     // any decoder for FPU POP
4542     FPU    : S4;
4543     MEM    : S3;        // any mem
4544 %}
4545 
4546 // Float mem-reg operation
4547 pipe_class fpu_mem_reg(memory mem, regD src)
4548 %{
4549     instruction_count(2);
4550     src    : S5(read);
4551     mem    : S3(read);
4552     DECODE : S0;        // any decoder for FPU PUSH
4553     D0     : S1;        // big decoder only
4554     FPU    : S4;
4555     MEM    : S3;        // any mem
4556 %}
4557 
4558 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4559 %{
4560     instruction_count(3);
4561     src1   : S3(read);
4562     src2   : S3(read);
4563     mem    : S3(read);
4564     DECODE : S0(2);     // any decoder for FPU PUSH
4565     D0     : S1;        // big decoder only
4566     FPU    : S4;
4567     MEM    : S3;        // any mem
4568 %}
4569 
4570 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4571 %{
4572     instruction_count(3);
4573     src1   : S3(read);
4574     src2   : S3(read);
4575     mem    : S4(read);
4576     DECODE : S0;        // any decoder for FPU PUSH
4577     D0     : S0(2);     // big decoder only
4578     FPU    : S4;
4579     MEM    : S3(2);     // any mem
4580 %}
4581 
4582 pipe_class fpu_mem_mem(memory dst, memory src1)
4583 %{
4584     instruction_count(2);
4585     src1   : S3(read);
4586     dst    : S4(read);
4587     D0     : S0(2);     // big decoder only
4588     MEM    : S3(2);     // any mem
4589 %}
4590 
4591 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4592 %{
4593     instruction_count(3);
4594     src1   : S3(read);
4595     src2   : S3(read);
4596     dst    : S4(read);
4597     D0     : S0(3);     // big decoder only
4598     FPU    : S4;
4599     MEM    : S3(3);     // any mem
4600 %}
4601 
4602 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4603 %{
4604     instruction_count(3);
4605     src1   : S4(read);
4606     mem    : S4(read);
4607     DECODE : S0;        // any decoder for FPU PUSH
4608     D0     : S0(2);     // big decoder only
4609     FPU    : S4;
4610     MEM    : S3(2);     // any mem
4611 %}
4612 
4613 // Float load constant
4614 pipe_class fpu_reg_con(regD dst)
4615 %{
4616     instruction_count(2);
4617     dst    : S5(write);
4618     D0     : S0;        // big decoder only for the load
4619     DECODE : S1;        // any decoder for FPU POP
4620     FPU    : S4;
4621     MEM    : S3;        // any mem
4622 %}
4623 
4624 // Float load constant
4625 pipe_class fpu_reg_reg_con(regD dst, regD src)
4626 %{
4627     instruction_count(3);
4628     dst    : S5(write);
4629     src    : S3(read);
4630     D0     : S0;        // big decoder only for the load
4631     DECODE : S1(2);     // any decoder for FPU POP
4632     FPU    : S4;
4633     MEM    : S3;        // any mem
4634 %}
4635 
4636 // UnConditional branch
4637 pipe_class pipe_jmp(label labl)
4638 %{
4639     single_instruction;
4640     BR   : S3;
4641 %}
4642 
4643 // Conditional branch
4644 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4645 %{
4646     single_instruction;
4647     cr    : S1(read);
4648     BR    : S3;
4649 %}
4650 
4651 // Allocation idiom
4652 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4653 %{
4654     instruction_count(1); force_serialization;
4655     fixed_latency(6);
4656     heap_ptr : S3(read);
4657     DECODE   : S0(3);
4658     D0       : S2;
4659     MEM      : S3;
4660     ALU      : S3(2);
4661     dst      : S5(write);
4662     BR       : S5;
4663 %}
4664 
4665 // Generic big/slow expanded idiom
4666 pipe_class pipe_slow()
4667 %{
4668     instruction_count(10); multiple_bundles; force_serialization;
4669     fixed_latency(100);
4670     D0  : S0(2);
4671     MEM : S3(2);
4672 %}
4673 
4674 // The real do-nothing guy
4675 pipe_class empty()
4676 %{
4677     instruction_count(0);
4678 %}
4679 
4680 // Define the class for the Nop node
4681 define
4682 %{
4683    MachNop = empty;
4684 %}
4685 
4686 %}
4687 
4688 //----------INSTRUCTIONS-------------------------------------------------------
4689 //
4690 // match      -- States which machine-independent subtree may be replaced
4691 //               by this instruction.
4692 // ins_cost   -- The estimated cost of this instruction is used by instruction
4693 //               selection to identify a minimum cost tree of machine
4694 //               instructions that matches a tree of machine-independent
4695 //               instructions.
4696 // format     -- A string providing the disassembly for this instruction.
4697 //               The value of an instruction's operand may be inserted
4698 //               by referring to it with a '$' prefix.
4699 // opcode     -- Three instruction opcodes may be provided.  These are referred
4700 //               to within an encode class as $primary, $secondary, and $tertiary
4701 //               rrspectively.  The primary opcode is commonly used to
4702 //               indicate the type of machine instruction, while secondary
4703 //               and tertiary are often used for prefix options or addressing
4704 //               modes.
4705 // ins_encode -- A list of encode classes with parameters. The encode class
4706 //               name must have been defined in an 'enc_class' specification
4707 //               in the encode section of the architecture description.
4708 
4709 
4710 //----------Load/Store/Move Instructions---------------------------------------
4711 //----------Load Instructions--------------------------------------------------
4712 
4713 // Load Byte (8 bit signed)
4714 instruct loadB(rRegI dst, memory mem)
4715 %{
4716   match(Set dst (LoadB mem));
4717 
4718   ins_cost(125);
4719   format %{ "movsbl  $dst, $mem\t# byte" %}
4720 
4721   ins_encode %{
4722     __ movsbl($dst$$Register, $mem$$Address);
4723   %}
4724 
4725   ins_pipe(ialu_reg_mem);
4726 %}
4727 
4728 // Load Byte (8 bit signed) into Long Register
4729 instruct loadB2L(rRegL dst, memory mem)
4730 %{
4731   match(Set dst (ConvI2L (LoadB mem)));
4732 
4733   ins_cost(125);
4734   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4735 
4736   ins_encode %{
4737     __ movsbq($dst$$Register, $mem$$Address);
4738   %}
4739 
4740   ins_pipe(ialu_reg_mem);
4741 %}
4742 
4743 // Load Unsigned Byte (8 bit UNsigned)
4744 instruct loadUB(rRegI dst, memory mem)
4745 %{
4746   match(Set dst (LoadUB mem));
4747 
4748   ins_cost(125);
4749   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4750 
4751   ins_encode %{
4752     __ movzbl($dst$$Register, $mem$$Address);
4753   %}
4754 
4755   ins_pipe(ialu_reg_mem);
4756 %}
4757 
4758 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4759 instruct loadUB2L(rRegL dst, memory mem)
4760 %{
4761   match(Set dst (ConvI2L (LoadUB mem)));
4762 
4763   ins_cost(125);
4764   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4765 
4766   ins_encode %{
4767     __ movzbq($dst$$Register, $mem$$Address);
4768   %}
4769 
4770   ins_pipe(ialu_reg_mem);
4771 %}
4772 
4773 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4774 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4775   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4776   effect(KILL cr);
4777 
4778   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4779             "andl    $dst, right_n_bits($mask, 8)" %}
4780   ins_encode %{
4781     Register Rdst = $dst$$Register;
4782     __ movzbq(Rdst, $mem$$Address);
4783     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4784   %}
4785   ins_pipe(ialu_reg_mem);
4786 %}
4787 
4788 // Load Short (16 bit signed)
4789 instruct loadS(rRegI dst, memory mem)
4790 %{
4791   match(Set dst (LoadS mem));
4792 
4793   ins_cost(125);
4794   format %{ "movswl $dst, $mem\t# short" %}
4795 
4796   ins_encode %{
4797     __ movswl($dst$$Register, $mem$$Address);
4798   %}
4799 
4800   ins_pipe(ialu_reg_mem);
4801 %}
4802 
4803 // Load Short (16 bit signed) to Byte (8 bit signed)
4804 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4805   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4806 
4807   ins_cost(125);
4808   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4809   ins_encode %{
4810     __ movsbl($dst$$Register, $mem$$Address);
4811   %}
4812   ins_pipe(ialu_reg_mem);
4813 %}
4814 
4815 // Load Short (16 bit signed) into Long Register
4816 instruct loadS2L(rRegL dst, memory mem)
4817 %{
4818   match(Set dst (ConvI2L (LoadS mem)));
4819 
4820   ins_cost(125);
4821   format %{ "movswq $dst, $mem\t# short -> long" %}
4822 
4823   ins_encode %{
4824     __ movswq($dst$$Register, $mem$$Address);
4825   %}
4826 
4827   ins_pipe(ialu_reg_mem);
4828 %}
4829 
4830 // Load Unsigned Short/Char (16 bit UNsigned)
4831 instruct loadUS(rRegI dst, memory mem)
4832 %{
4833   match(Set dst (LoadUS mem));
4834 
4835   ins_cost(125);
4836   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4837 
4838   ins_encode %{
4839     __ movzwl($dst$$Register, $mem$$Address);
4840   %}
4841 
4842   ins_pipe(ialu_reg_mem);
4843 %}
4844 
4845 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4846 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4847   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4848 
4849   ins_cost(125);
4850   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4851   ins_encode %{
4852     __ movsbl($dst$$Register, $mem$$Address);
4853   %}
4854   ins_pipe(ialu_reg_mem);
4855 %}
4856 
4857 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4858 instruct loadUS2L(rRegL dst, memory mem)
4859 %{
4860   match(Set dst (ConvI2L (LoadUS mem)));
4861 
4862   ins_cost(125);
4863   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4864 
4865   ins_encode %{
4866     __ movzwq($dst$$Register, $mem$$Address);
4867   %}
4868 
4869   ins_pipe(ialu_reg_mem);
4870 %}
4871 
4872 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4873 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4874   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4875 
4876   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4877   ins_encode %{
4878     __ movzbq($dst$$Register, $mem$$Address);
4879   %}
4880   ins_pipe(ialu_reg_mem);
4881 %}
4882 
4883 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4884 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4885   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4886   effect(KILL cr);
4887 
4888   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4889             "andl    $dst, right_n_bits($mask, 16)" %}
4890   ins_encode %{
4891     Register Rdst = $dst$$Register;
4892     __ movzwq(Rdst, $mem$$Address);
4893     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4894   %}
4895   ins_pipe(ialu_reg_mem);
4896 %}
4897 
4898 // Load Integer
4899 instruct loadI(rRegI dst, memory mem)
4900 %{
4901   match(Set dst (LoadI mem));
4902 
4903   ins_cost(125);
4904   format %{ "movl    $dst, $mem\t# int" %}
4905 
4906   ins_encode %{
4907     __ movl($dst$$Register, $mem$$Address);
4908   %}
4909 
4910   ins_pipe(ialu_reg_mem);
4911 %}
4912 
4913 // Load Integer (32 bit signed) to Byte (8 bit signed)
4914 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4915   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4916 
4917   ins_cost(125);
4918   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4919   ins_encode %{
4920     __ movsbl($dst$$Register, $mem$$Address);
4921   %}
4922   ins_pipe(ialu_reg_mem);
4923 %}
4924 
4925 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4926 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4927   match(Set dst (AndI (LoadI mem) mask));
4928 
4929   ins_cost(125);
4930   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4931   ins_encode %{
4932     __ movzbl($dst$$Register, $mem$$Address);
4933   %}
4934   ins_pipe(ialu_reg_mem);
4935 %}
4936 
4937 // Load Integer (32 bit signed) to Short (16 bit signed)
4938 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4939   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4940 
4941   ins_cost(125);
4942   format %{ "movswl  $dst, $mem\t# int -> short" %}
4943   ins_encode %{
4944     __ movswl($dst$$Register, $mem$$Address);
4945   %}
4946   ins_pipe(ialu_reg_mem);
4947 %}
4948 
4949 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4950 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4951   match(Set dst (AndI (LoadI mem) mask));
4952 
4953   ins_cost(125);
4954   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4955   ins_encode %{
4956     __ movzwl($dst$$Register, $mem$$Address);
4957   %}
4958   ins_pipe(ialu_reg_mem);
4959 %}
4960 
4961 // Load Integer into Long Register
4962 instruct loadI2L(rRegL dst, memory mem)
4963 %{
4964   match(Set dst (ConvI2L (LoadI mem)));
4965 
4966   ins_cost(125);
4967   format %{ "movslq  $dst, $mem\t# int -> long" %}
4968 
4969   ins_encode %{
4970     __ movslq($dst$$Register, $mem$$Address);
4971   %}
4972 
4973   ins_pipe(ialu_reg_mem);
4974 %}
4975 
4976 // Load Integer with mask 0xFF into Long Register
4977 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4978   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4979 
4980   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4981   ins_encode %{
4982     __ movzbq($dst$$Register, $mem$$Address);
4983   %}
4984   ins_pipe(ialu_reg_mem);
4985 %}
4986 
4987 // Load Integer with mask 0xFFFF into Long Register
4988 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4989   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4990 
4991   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4992   ins_encode %{
4993     __ movzwq($dst$$Register, $mem$$Address);
4994   %}
4995   ins_pipe(ialu_reg_mem);
4996 %}
4997 
4998 // Load Integer with a 31-bit mask into Long Register
4999 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5000   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5001   effect(KILL cr);
5002 
5003   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5004             "andl    $dst, $mask" %}
5005   ins_encode %{
5006     Register Rdst = $dst$$Register;
5007     __ movl(Rdst, $mem$$Address);
5008     __ andl(Rdst, $mask$$constant);
5009   %}
5010   ins_pipe(ialu_reg_mem);
5011 %}
5012 
5013 // Load Unsigned Integer into Long Register
5014 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5015 %{
5016   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5017 
5018   ins_cost(125);
5019   format %{ "movl    $dst, $mem\t# uint -> long" %}
5020 
5021   ins_encode %{
5022     __ movl($dst$$Register, $mem$$Address);
5023   %}
5024 
5025   ins_pipe(ialu_reg_mem);
5026 %}
5027 
5028 // Load Long
5029 instruct loadL(rRegL dst, memory mem)
5030 %{
5031   match(Set dst (LoadL mem));
5032 
5033   ins_cost(125);
5034   format %{ "movq    $dst, $mem\t# long" %}
5035 
5036   ins_encode %{
5037     __ movq($dst$$Register, $mem$$Address);
5038   %}
5039 
5040   ins_pipe(ialu_reg_mem); // XXX
5041 %}
5042 
5043 // Load Range
5044 instruct loadRange(rRegI dst, memory mem)
5045 %{
5046   match(Set dst (LoadRange mem));
5047 
5048   ins_cost(125); // XXX
5049   format %{ "movl    $dst, $mem\t# range" %}
5050   opcode(0x8B);
5051   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5052   ins_pipe(ialu_reg_mem);
5053 %}
5054 
5055 // Load Pointer
5056 instruct loadP(rRegP dst, memory mem)
5057 %{
5058   match(Set dst (LoadP mem));
5059 
5060   ins_cost(125); // XXX
5061   format %{ "movq    $dst, $mem\t# ptr" %}
5062   opcode(0x8B);
5063   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5064   ins_pipe(ialu_reg_mem); // XXX
5065 %}
5066 
5067 // Load Compressed Pointer
5068 instruct loadN(rRegN dst, memory mem)
5069 %{
5070    match(Set dst (LoadN mem));
5071 
5072    ins_cost(125); // XXX
5073    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5074    ins_encode %{
5075      __ movl($dst$$Register, $mem$$Address);
5076    %}
5077    ins_pipe(ialu_reg_mem); // XXX
5078 %}
5079 
5080 
5081 // Load Klass Pointer
5082 instruct loadKlass(rRegP dst, memory mem)
5083 %{
5084   match(Set dst (LoadKlass mem));
5085 
5086   ins_cost(125); // XXX
5087   format %{ "movq    $dst, $mem\t# class" %}
5088   opcode(0x8B);
5089   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5090   ins_pipe(ialu_reg_mem); // XXX
5091 %}
5092 
5093 // Load narrow Klass Pointer
5094 instruct loadNKlass(rRegN dst, memory mem)
5095 %{
5096   match(Set dst (LoadNKlass mem));
5097 
5098   ins_cost(125); // XXX
5099   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5100   ins_encode %{
5101     __ movl($dst$$Register, $mem$$Address);
5102   %}
5103   ins_pipe(ialu_reg_mem); // XXX
5104 %}
5105 
5106 // Load Float
5107 instruct loadF(regF dst, memory mem)
5108 %{
5109   match(Set dst (LoadF mem));
5110 
5111   ins_cost(145); // XXX
5112   format %{ "movss   $dst, $mem\t# float" %}
5113   ins_encode %{
5114     __ movflt($dst$$XMMRegister, $mem$$Address);
5115   %}
5116   ins_pipe(pipe_slow); // XXX
5117 %}
5118 
5119 // Load Double
5120 instruct loadD_partial(regD dst, memory mem)
5121 %{
5122   predicate(!UseXmmLoadAndClearUpper);
5123   match(Set dst (LoadD mem));
5124 
5125   ins_cost(145); // XXX
5126   format %{ "movlpd  $dst, $mem\t# double" %}
5127   ins_encode %{
5128     __ movdbl($dst$$XMMRegister, $mem$$Address);
5129   %}
5130   ins_pipe(pipe_slow); // XXX
5131 %}
5132 
5133 instruct loadD(regD dst, memory mem)
5134 %{
5135   predicate(UseXmmLoadAndClearUpper);
5136   match(Set dst (LoadD mem));
5137 
5138   ins_cost(145); // XXX
5139   format %{ "movsd   $dst, $mem\t# double" %}
5140   ins_encode %{
5141     __ movdbl($dst$$XMMRegister, $mem$$Address);
5142   %}
5143   ins_pipe(pipe_slow); // XXX
5144 %}
5145 
5146 // Load Effective Address
5147 instruct leaP8(rRegP dst, indOffset8 mem)
5148 %{
5149   match(Set dst mem);
5150 
5151   ins_cost(110); // XXX
5152   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5153   opcode(0x8D);
5154   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5155   ins_pipe(ialu_reg_reg_fat);
5156 %}
5157 
5158 instruct leaP32(rRegP dst, indOffset32 mem)
5159 %{
5160   match(Set dst mem);
5161 
5162   ins_cost(110);
5163   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5164   opcode(0x8D);
5165   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5166   ins_pipe(ialu_reg_reg_fat);
5167 %}
5168 
5169 // instruct leaPIdx(rRegP dst, indIndex mem)
5170 // %{
5171 //   match(Set dst mem);
5172 
5173 //   ins_cost(110);
5174 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5175 //   opcode(0x8D);
5176 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5177 //   ins_pipe(ialu_reg_reg_fat);
5178 // %}
5179 
5180 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5181 %{
5182   match(Set dst mem);
5183 
5184   ins_cost(110);
5185   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5186   opcode(0x8D);
5187   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5188   ins_pipe(ialu_reg_reg_fat);
5189 %}
5190 
5191 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5192 %{
5193   match(Set dst mem);
5194 
5195   ins_cost(110);
5196   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5197   opcode(0x8D);
5198   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5199   ins_pipe(ialu_reg_reg_fat);
5200 %}
5201 
5202 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5203 %{
5204   match(Set dst mem);
5205 
5206   ins_cost(110);
5207   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5208   opcode(0x8D);
5209   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5210   ins_pipe(ialu_reg_reg_fat);
5211 %}
5212 
5213 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5214 %{
5215   match(Set dst mem);
5216 
5217   ins_cost(110);
5218   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5219   opcode(0x8D);
5220   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5221   ins_pipe(ialu_reg_reg_fat);
5222 %}
5223 
5224 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5225 %{
5226   match(Set dst mem);
5227 
5228   ins_cost(110);
5229   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5230   opcode(0x8D);
5231   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5232   ins_pipe(ialu_reg_reg_fat);
5233 %}
5234 
5235 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5236 %{
5237   match(Set dst mem);
5238 
5239   ins_cost(110);
5240   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5241   opcode(0x8D);
5242   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5243   ins_pipe(ialu_reg_reg_fat);
5244 %}
5245 
5246 // Load Effective Address which uses Narrow (32-bits) oop
5247 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5248 %{
5249   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5250   match(Set dst mem);
5251 
5252   ins_cost(110);
5253   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5254   opcode(0x8D);
5255   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5256   ins_pipe(ialu_reg_reg_fat);
5257 %}
5258 
5259 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5260 %{
5261   predicate(Universe::narrow_oop_shift() == 0);
5262   match(Set dst mem);
5263 
5264   ins_cost(110); // XXX
5265   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5266   opcode(0x8D);
5267   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5268   ins_pipe(ialu_reg_reg_fat);
5269 %}
5270 
5271 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5272 %{
5273   predicate(Universe::narrow_oop_shift() == 0);
5274   match(Set dst mem);
5275 
5276   ins_cost(110);
5277   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5278   opcode(0x8D);
5279   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5280   ins_pipe(ialu_reg_reg_fat);
5281 %}
5282 
5283 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5284 %{
5285   predicate(Universe::narrow_oop_shift() == 0);
5286   match(Set dst mem);
5287 
5288   ins_cost(110);
5289   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5290   opcode(0x8D);
5291   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5292   ins_pipe(ialu_reg_reg_fat);
5293 %}
5294 
5295 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5296 %{
5297   predicate(Universe::narrow_oop_shift() == 0);
5298   match(Set dst mem);
5299 
5300   ins_cost(110);
5301   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5302   opcode(0x8D);
5303   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5304   ins_pipe(ialu_reg_reg_fat);
5305 %}
5306 
5307 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5308 %{
5309   predicate(Universe::narrow_oop_shift() == 0);
5310   match(Set dst mem);
5311 
5312   ins_cost(110);
5313   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5314   opcode(0x8D);
5315   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5316   ins_pipe(ialu_reg_reg_fat);
5317 %}
5318 
5319 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5320 %{
5321   predicate(Universe::narrow_oop_shift() == 0);
5322   match(Set dst mem);
5323 
5324   ins_cost(110);
5325   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5326   opcode(0x8D);
5327   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5328   ins_pipe(ialu_reg_reg_fat);
5329 %}
5330 
5331 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5332 %{
5333   predicate(Universe::narrow_oop_shift() == 0);
5334   match(Set dst mem);
5335 
5336   ins_cost(110);
5337   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5338   opcode(0x8D);
5339   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5340   ins_pipe(ialu_reg_reg_fat);
5341 %}
5342 
5343 instruct loadConI(rRegI dst, immI src)
5344 %{
5345   match(Set dst src);
5346 
5347   format %{ "movl    $dst, $src\t# int" %}
5348   ins_encode(load_immI(dst, src));
5349   ins_pipe(ialu_reg_fat); // XXX
5350 %}
5351 
5352 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5353 %{
5354   match(Set dst src);
5355   effect(KILL cr);
5356 
5357   ins_cost(50);
5358   format %{ "xorl    $dst, $dst\t# int" %}
5359   opcode(0x33); /* + rd */
5360   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5361   ins_pipe(ialu_reg);
5362 %}
5363 
5364 instruct loadConL(rRegL dst, immL src)
5365 %{
5366   match(Set dst src);
5367 
5368   ins_cost(150);
5369   format %{ "movq    $dst, $src\t# long" %}
5370   ins_encode(load_immL(dst, src));
5371   ins_pipe(ialu_reg);
5372 %}
5373 
5374 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5375 %{
5376   match(Set dst src);
5377   effect(KILL cr);
5378 
5379   ins_cost(50);
5380   format %{ "xorl    $dst, $dst\t# long" %}
5381   opcode(0x33); /* + rd */
5382   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5383   ins_pipe(ialu_reg); // XXX
5384 %}
5385 
5386 instruct loadConUL32(rRegL dst, immUL32 src)
5387 %{
5388   match(Set dst src);
5389 
5390   ins_cost(60);
5391   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5392   ins_encode(load_immUL32(dst, src));
5393   ins_pipe(ialu_reg);
5394 %}
5395 
5396 instruct loadConL32(rRegL dst, immL32 src)
5397 %{
5398   match(Set dst src);
5399 
5400   ins_cost(70);
5401   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5402   ins_encode(load_immL32(dst, src));
5403   ins_pipe(ialu_reg);
5404 %}
5405 
5406 instruct loadConP(rRegP dst, immP con) %{
5407   match(Set dst con);
5408 
5409   format %{ "movq    $dst, $con\t# ptr" %}
5410   ins_encode(load_immP(dst, con));
5411   ins_pipe(ialu_reg_fat); // XXX
5412 %}
5413 
5414 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5415 %{
5416   match(Set dst src);
5417   effect(KILL cr);
5418 
5419   ins_cost(50);
5420   format %{ "xorl    $dst, $dst\t# ptr" %}
5421   opcode(0x33); /* + rd */
5422   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5423   ins_pipe(ialu_reg);
5424 %}
5425 
5426 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5427 %{
5428   match(Set dst src);
5429   effect(KILL cr);
5430 
5431   ins_cost(60);
5432   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5433   ins_encode(load_immP31(dst, src));
5434   ins_pipe(ialu_reg);
5435 %}
5436 
5437 instruct loadConF(regF dst, immF con) %{
5438   match(Set dst con);
5439   ins_cost(125);
5440   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5441   ins_encode %{
5442     __ movflt($dst$$XMMRegister, $constantaddress($con));
5443   %}
5444   ins_pipe(pipe_slow);
5445 %}
5446 
5447 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5448   match(Set dst src);
5449   effect(KILL cr);
5450   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5451   ins_encode %{
5452     __ xorq($dst$$Register, $dst$$Register);
5453   %}
5454   ins_pipe(ialu_reg);
5455 %}
5456 
5457 instruct loadConN(rRegN dst, immN src) %{
5458   match(Set dst src);
5459 
5460   ins_cost(125);
5461   format %{ "movl    $dst, $src\t# compressed ptr" %}
5462   ins_encode %{
5463     address con = (address)$src$$constant;
5464     if (con == NULL) {
5465       ShouldNotReachHere();
5466     } else {
5467       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5468     }
5469   %}
5470   ins_pipe(ialu_reg_fat); // XXX
5471 %}
5472 
5473 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5474   match(Set dst src);
5475 
5476   ins_cost(125);
5477   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5478   ins_encode %{
5479     address con = (address)$src$$constant;
5480     if (con == NULL) {
5481       ShouldNotReachHere();
5482     } else {
5483       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5484     }
5485   %}
5486   ins_pipe(ialu_reg_fat); // XXX
5487 %}
5488 
5489 instruct loadConF0(regF dst, immF0 src)
5490 %{
5491   match(Set dst src);
5492   ins_cost(100);
5493 
5494   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5495   ins_encode %{
5496     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5497   %}
5498   ins_pipe(pipe_slow);
5499 %}
5500 
5501 // Use the same format since predicate() can not be used here.
5502 instruct loadConD(regD dst, immD con) %{
5503   match(Set dst con);
5504   ins_cost(125);
5505   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5506   ins_encode %{
5507     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5508   %}
5509   ins_pipe(pipe_slow);
5510 %}
5511 
5512 instruct loadConD0(regD dst, immD0 src)
5513 %{
5514   match(Set dst src);
5515   ins_cost(100);
5516 
5517   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5518   ins_encode %{
5519     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5520   %}
5521   ins_pipe(pipe_slow);
5522 %}
5523 
5524 instruct loadSSI(rRegI dst, stackSlotI src)
5525 %{
5526   match(Set dst src);
5527 
5528   ins_cost(125);
5529   format %{ "movl    $dst, $src\t# int stk" %}
5530   opcode(0x8B);
5531   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5532   ins_pipe(ialu_reg_mem);
5533 %}
5534 
5535 instruct loadSSL(rRegL dst, stackSlotL src)
5536 %{
5537   match(Set dst src);
5538 
5539   ins_cost(125);
5540   format %{ "movq    $dst, $src\t# long stk" %}
5541   opcode(0x8B);
5542   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5543   ins_pipe(ialu_reg_mem);
5544 %}
5545 
5546 instruct loadSSP(rRegP dst, stackSlotP src)
5547 %{
5548   match(Set dst src);
5549 
5550   ins_cost(125);
5551   format %{ "movq    $dst, $src\t# ptr stk" %}
5552   opcode(0x8B);
5553   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5554   ins_pipe(ialu_reg_mem);
5555 %}
5556 
5557 instruct loadSSF(regF dst, stackSlotF src)
5558 %{
5559   match(Set dst src);
5560 
5561   ins_cost(125);
5562   format %{ "movss   $dst, $src\t# float stk" %}
5563   ins_encode %{
5564     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5565   %}
5566   ins_pipe(pipe_slow); // XXX
5567 %}
5568 
5569 // Use the same format since predicate() can not be used here.
5570 instruct loadSSD(regD dst, stackSlotD src)
5571 %{
5572   match(Set dst src);
5573 
5574   ins_cost(125);
5575   format %{ "movsd   $dst, $src\t# double stk" %}
5576   ins_encode  %{
5577     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5578   %}
5579   ins_pipe(pipe_slow); // XXX
5580 %}
5581 
5582 // Prefetch instructions for allocation.
5583 // Must be safe to execute with invalid address (cannot fault).
5584 
5585 instruct prefetchAlloc( memory mem ) %{
5586   predicate(AllocatePrefetchInstr==3);
5587   match(PrefetchAllocation mem);
5588   ins_cost(125);
5589 
5590   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5591   ins_encode %{
5592     __ prefetchw($mem$$Address);
5593   %}
5594   ins_pipe(ialu_mem);
5595 %}
5596 
5597 instruct prefetchAllocNTA( memory mem ) %{
5598   predicate(AllocatePrefetchInstr==0);
5599   match(PrefetchAllocation mem);
5600   ins_cost(125);
5601 
5602   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5603   ins_encode %{
5604     __ prefetchnta($mem$$Address);
5605   %}
5606   ins_pipe(ialu_mem);
5607 %}
5608 
5609 instruct prefetchAllocT0( memory mem ) %{
5610   predicate(AllocatePrefetchInstr==1);
5611   match(PrefetchAllocation mem);
5612   ins_cost(125);
5613 
5614   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5615   ins_encode %{
5616     __ prefetcht0($mem$$Address);
5617   %}
5618   ins_pipe(ialu_mem);
5619 %}
5620 
5621 instruct prefetchAllocT2( memory mem ) %{
5622   predicate(AllocatePrefetchInstr==2);
5623   match(PrefetchAllocation mem);
5624   ins_cost(125);
5625 
5626   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5627   ins_encode %{
5628     __ prefetcht2($mem$$Address);
5629   %}
5630   ins_pipe(ialu_mem);
5631 %}
5632 
5633 //----------Store Instructions-------------------------------------------------
5634 
5635 // Store Byte
5636 instruct storeB(memory mem, rRegI src)
5637 %{
5638   match(Set mem (StoreB mem src));
5639 
5640   ins_cost(125); // XXX
5641   format %{ "movb    $mem, $src\t# byte" %}
5642   opcode(0x88);
5643   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5644   ins_pipe(ialu_mem_reg);
5645 %}
5646 
5647 // Store Char/Short
5648 instruct storeC(memory mem, rRegI src)
5649 %{
5650   match(Set mem (StoreC mem src));
5651 
5652   ins_cost(125); // XXX
5653   format %{ "movw    $mem, $src\t# char/short" %}
5654   opcode(0x89);
5655   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5656   ins_pipe(ialu_mem_reg);
5657 %}
5658 
5659 // Store Integer
5660 instruct storeI(memory mem, rRegI src)
5661 %{
5662   match(Set mem (StoreI mem src));
5663 
5664   ins_cost(125); // XXX
5665   format %{ "movl    $mem, $src\t# int" %}
5666   opcode(0x89);
5667   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5668   ins_pipe(ialu_mem_reg);
5669 %}
5670 
5671 // Store Long
5672 instruct storeL(memory mem, rRegL src)
5673 %{
5674   match(Set mem (StoreL mem src));
5675 
5676   ins_cost(125); // XXX
5677   format %{ "movq    $mem, $src\t# long" %}
5678   opcode(0x89);
5679   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5680   ins_pipe(ialu_mem_reg); // XXX
5681 %}
5682 
5683 // Store Pointer
5684 instruct storeP(memory mem, any_RegP src)
5685 %{
5686   match(Set mem (StoreP mem src));
5687 
5688   ins_cost(125); // XXX
5689   format %{ "movq    $mem, $src\t# ptr" %}
5690   opcode(0x89);
5691   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5692   ins_pipe(ialu_mem_reg);
5693 %}
5694 
5695 instruct storeImmP0(memory mem, immP0 zero)
5696 %{
5697   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5698   match(Set mem (StoreP mem zero));
5699 
5700   ins_cost(125); // XXX
5701   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5702   ins_encode %{
5703     __ movq($mem$$Address, r12);
5704   %}
5705   ins_pipe(ialu_mem_reg);
5706 %}
5707 
5708 // Store NULL Pointer, mark word, or other simple pointer constant.
5709 instruct storeImmP(memory mem, immP31 src)
5710 %{
5711   match(Set mem (StoreP mem src));
5712 
5713   ins_cost(150); // XXX
5714   format %{ "movq    $mem, $src\t# ptr" %}
5715   opcode(0xC7); /* C7 /0 */
5716   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5717   ins_pipe(ialu_mem_imm);
5718 %}
5719 
5720 // Store Compressed Pointer
5721 instruct storeN(memory mem, rRegN src)
5722 %{
5723   match(Set mem (StoreN mem src));
5724 
5725   ins_cost(125); // XXX
5726   format %{ "movl    $mem, $src\t# compressed ptr" %}
5727   ins_encode %{
5728     __ movl($mem$$Address, $src$$Register);
5729   %}
5730   ins_pipe(ialu_mem_reg);
5731 %}
5732 
5733 instruct storeNKlass(memory mem, rRegN src)
5734 %{
5735   match(Set mem (StoreNKlass mem src));
5736 
5737   ins_cost(125); // XXX
5738   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5739   ins_encode %{
5740     __ movl($mem$$Address, $src$$Register);
5741   %}
5742   ins_pipe(ialu_mem_reg);
5743 %}
5744 
5745 instruct storeImmN0(memory mem, immN0 zero)
5746 %{
5747   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5748   match(Set mem (StoreN mem zero));
5749 
5750   ins_cost(125); // XXX
5751   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5752   ins_encode %{
5753     __ movl($mem$$Address, r12);
5754   %}
5755   ins_pipe(ialu_mem_reg);
5756 %}
5757 
5758 instruct storeImmN(memory mem, immN src)
5759 %{
5760   match(Set mem (StoreN mem src));
5761 
5762   ins_cost(150); // XXX
5763   format %{ "movl    $mem, $src\t# compressed ptr" %}
5764   ins_encode %{
5765     address con = (address)$src$$constant;
5766     if (con == NULL) {
5767       __ movl($mem$$Address, (int32_t)0);
5768     } else {
5769       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5770     }
5771   %}
5772   ins_pipe(ialu_mem_imm);
5773 %}
5774 
5775 instruct storeImmNKlass(memory mem, immNKlass src)
5776 %{
5777   match(Set mem (StoreNKlass mem src));
5778 
5779   ins_cost(150); // XXX
5780   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5781   ins_encode %{
5782     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5783   %}
5784   ins_pipe(ialu_mem_imm);
5785 %}
5786 
5787 // Store Integer Immediate
5788 instruct storeImmI0(memory mem, immI0 zero)
5789 %{
5790   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5791   match(Set mem (StoreI mem zero));
5792 
5793   ins_cost(125); // XXX
5794   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5795   ins_encode %{
5796     __ movl($mem$$Address, r12);
5797   %}
5798   ins_pipe(ialu_mem_reg);
5799 %}
5800 
5801 instruct storeImmI(memory mem, immI src)
5802 %{
5803   match(Set mem (StoreI mem src));
5804 
5805   ins_cost(150);
5806   format %{ "movl    $mem, $src\t# int" %}
5807   opcode(0xC7); /* C7 /0 */
5808   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5809   ins_pipe(ialu_mem_imm);
5810 %}
5811 
5812 // Store Long Immediate
5813 instruct storeImmL0(memory mem, immL0 zero)
5814 %{
5815   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5816   match(Set mem (StoreL mem zero));
5817 
5818   ins_cost(125); // XXX
5819   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5820   ins_encode %{
5821     __ movq($mem$$Address, r12);
5822   %}
5823   ins_pipe(ialu_mem_reg);
5824 %}
5825 
5826 instruct storeImmL(memory mem, immL32 src)
5827 %{
5828   match(Set mem (StoreL mem src));
5829 
5830   ins_cost(150);
5831   format %{ "movq    $mem, $src\t# long" %}
5832   opcode(0xC7); /* C7 /0 */
5833   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5834   ins_pipe(ialu_mem_imm);
5835 %}
5836 
5837 // Store Short/Char Immediate
5838 instruct storeImmC0(memory mem, immI0 zero)
5839 %{
5840   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5841   match(Set mem (StoreC mem zero));
5842 
5843   ins_cost(125); // XXX
5844   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5845   ins_encode %{
5846     __ movw($mem$$Address, r12);
5847   %}
5848   ins_pipe(ialu_mem_reg);
5849 %}
5850 
5851 instruct storeImmI16(memory mem, immI16 src)
5852 %{
5853   predicate(UseStoreImmI16);
5854   match(Set mem (StoreC mem src));
5855 
5856   ins_cost(150);
5857   format %{ "movw    $mem, $src\t# short/char" %}
5858   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5859   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5860   ins_pipe(ialu_mem_imm);
5861 %}
5862 
5863 // Store Byte Immediate
5864 instruct storeImmB0(memory mem, immI0 zero)
5865 %{
5866   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5867   match(Set mem (StoreB mem zero));
5868 
5869   ins_cost(125); // XXX
5870   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5871   ins_encode %{
5872     __ movb($mem$$Address, r12);
5873   %}
5874   ins_pipe(ialu_mem_reg);
5875 %}
5876 
5877 instruct storeImmB(memory mem, immI8 src)
5878 %{
5879   match(Set mem (StoreB mem src));
5880 
5881   ins_cost(150); // XXX
5882   format %{ "movb    $mem, $src\t# byte" %}
5883   opcode(0xC6); /* C6 /0 */
5884   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5885   ins_pipe(ialu_mem_imm);
5886 %}
5887 
5888 // Store CMS card-mark Immediate
5889 instruct storeImmCM0_reg(memory mem, immI0 zero)
5890 %{
5891   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5892   match(Set mem (StoreCM mem zero));
5893 
5894   ins_cost(125); // XXX
5895   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5896   ins_encode %{
5897     __ movb($mem$$Address, r12);
5898   %}
5899   ins_pipe(ialu_mem_reg);
5900 %}
5901 
5902 instruct storeImmCM0(memory mem, immI0 src)
5903 %{
5904   match(Set mem (StoreCM mem src));
5905 
5906   ins_cost(150); // XXX
5907   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5908   opcode(0xC6); /* C6 /0 */
5909   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5910   ins_pipe(ialu_mem_imm);
5911 %}
5912 
5913 // Store Float
5914 instruct storeF(memory mem, regF src)
5915 %{
5916   match(Set mem (StoreF mem src));
5917 
5918   ins_cost(95); // XXX
5919   format %{ "movss   $mem, $src\t# float" %}
5920   ins_encode %{
5921     __ movflt($mem$$Address, $src$$XMMRegister);
5922   %}
5923   ins_pipe(pipe_slow); // XXX
5924 %}
5925 
5926 // Store immediate Float value (it is faster than store from XMM register)
5927 instruct storeF0(memory mem, immF0 zero)
5928 %{
5929   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5930   match(Set mem (StoreF mem zero));
5931 
5932   ins_cost(25); // XXX
5933   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5934   ins_encode %{
5935     __ movl($mem$$Address, r12);
5936   %}
5937   ins_pipe(ialu_mem_reg);
5938 %}
5939 
5940 instruct storeF_imm(memory mem, immF src)
5941 %{
5942   match(Set mem (StoreF mem src));
5943 
5944   ins_cost(50);
5945   format %{ "movl    $mem, $src\t# float" %}
5946   opcode(0xC7); /* C7 /0 */
5947   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5948   ins_pipe(ialu_mem_imm);
5949 %}
5950 
5951 // Store Double
5952 instruct storeD(memory mem, regD src)
5953 %{
5954   match(Set mem (StoreD mem src));
5955 
5956   ins_cost(95); // XXX
5957   format %{ "movsd   $mem, $src\t# double" %}
5958   ins_encode %{
5959     __ movdbl($mem$$Address, $src$$XMMRegister);
5960   %}
5961   ins_pipe(pipe_slow); // XXX
5962 %}
5963 
5964 // Store immediate double 0.0 (it is faster than store from XMM register)
5965 instruct storeD0_imm(memory mem, immD0 src)
5966 %{
5967   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5968   match(Set mem (StoreD mem src));
5969 
5970   ins_cost(50);
5971   format %{ "movq    $mem, $src\t# double 0." %}
5972   opcode(0xC7); /* C7 /0 */
5973   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5974   ins_pipe(ialu_mem_imm);
5975 %}
5976 
5977 instruct storeD0(memory mem, immD0 zero)
5978 %{
5979   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5980   match(Set mem (StoreD mem zero));
5981 
5982   ins_cost(25); // XXX
5983   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5984   ins_encode %{
5985     __ movq($mem$$Address, r12);
5986   %}
5987   ins_pipe(ialu_mem_reg);
5988 %}
5989 
5990 instruct storeSSI(stackSlotI dst, rRegI src)
5991 %{
5992   match(Set dst src);
5993 
5994   ins_cost(100);
5995   format %{ "movl    $dst, $src\t# int stk" %}
5996   opcode(0x89);
5997   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
5998   ins_pipe( ialu_mem_reg );
5999 %}
6000 
6001 instruct storeSSL(stackSlotL dst, rRegL src)
6002 %{
6003   match(Set dst src);
6004 
6005   ins_cost(100);
6006   format %{ "movq    $dst, $src\t# long stk" %}
6007   opcode(0x89);
6008   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6009   ins_pipe(ialu_mem_reg);
6010 %}
6011 
6012 instruct storeSSP(stackSlotP dst, rRegP src)
6013 %{
6014   match(Set dst src);
6015 
6016   ins_cost(100);
6017   format %{ "movq    $dst, $src\t# ptr stk" %}
6018   opcode(0x89);
6019   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6020   ins_pipe(ialu_mem_reg);
6021 %}
6022 
6023 instruct storeSSF(stackSlotF dst, regF src)
6024 %{
6025   match(Set dst src);
6026 
6027   ins_cost(95); // XXX
6028   format %{ "movss   $dst, $src\t# float stk" %}
6029   ins_encode %{
6030     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6031   %}
6032   ins_pipe(pipe_slow); // XXX
6033 %}
6034 
6035 instruct storeSSD(stackSlotD dst, regD src)
6036 %{
6037   match(Set dst src);
6038 
6039   ins_cost(95); // XXX
6040   format %{ "movsd   $dst, $src\t# double stk" %}
6041   ins_encode %{
6042     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6043   %}
6044   ins_pipe(pipe_slow); // XXX
6045 %}
6046 
6047 //----------BSWAP Instructions-------------------------------------------------
6048 instruct bytes_reverse_int(rRegI dst) %{
6049   match(Set dst (ReverseBytesI dst));
6050 
6051   format %{ "bswapl  $dst" %}
6052   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6053   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6054   ins_pipe( ialu_reg );
6055 %}
6056 
6057 instruct bytes_reverse_long(rRegL dst) %{
6058   match(Set dst (ReverseBytesL dst));
6059 
6060   format %{ "bswapq  $dst" %}
6061   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6062   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6063   ins_pipe( ialu_reg);
6064 %}
6065 
6066 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6067   match(Set dst (ReverseBytesUS dst));
6068   effect(KILL cr);
6069 
6070   format %{ "bswapl  $dst\n\t"
6071             "shrl    $dst,16\n\t" %}
6072   ins_encode %{
6073     __ bswapl($dst$$Register);
6074     __ shrl($dst$$Register, 16);
6075   %}
6076   ins_pipe( ialu_reg );
6077 %}
6078 
6079 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6080   match(Set dst (ReverseBytesS dst));
6081   effect(KILL cr);
6082 
6083   format %{ "bswapl  $dst\n\t"
6084             "sar     $dst,16\n\t" %}
6085   ins_encode %{
6086     __ bswapl($dst$$Register);
6087     __ sarl($dst$$Register, 16);
6088   %}
6089   ins_pipe( ialu_reg );
6090 %}
6091 
6092 //---------- Zeros Count Instructions ------------------------------------------
6093 
6094 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6095   predicate(UseCountLeadingZerosInstruction);
6096   match(Set dst (CountLeadingZerosI src));
6097   effect(KILL cr);
6098 
6099   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6100   ins_encode %{
6101     __ lzcntl($dst$$Register, $src$$Register);
6102   %}
6103   ins_pipe(ialu_reg);
6104 %}
6105 
6106 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6107   predicate(!UseCountLeadingZerosInstruction);
6108   match(Set dst (CountLeadingZerosI src));
6109   effect(KILL cr);
6110 
6111   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6112             "jnz     skip\n\t"
6113             "movl    $dst, -1\n"
6114       "skip:\n\t"
6115             "negl    $dst\n\t"
6116             "addl    $dst, 31" %}
6117   ins_encode %{
6118     Register Rdst = $dst$$Register;
6119     Register Rsrc = $src$$Register;
6120     Label skip;
6121     __ bsrl(Rdst, Rsrc);
6122     __ jccb(Assembler::notZero, skip);
6123     __ movl(Rdst, -1);
6124     __ bind(skip);
6125     __ negl(Rdst);
6126     __ addl(Rdst, BitsPerInt - 1);
6127   %}
6128   ins_pipe(ialu_reg);
6129 %}
6130 
6131 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6132   predicate(UseCountLeadingZerosInstruction);
6133   match(Set dst (CountLeadingZerosL src));
6134   effect(KILL cr);
6135 
6136   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6137   ins_encode %{
6138     __ lzcntq($dst$$Register, $src$$Register);
6139   %}
6140   ins_pipe(ialu_reg);
6141 %}
6142 
6143 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6144   predicate(!UseCountLeadingZerosInstruction);
6145   match(Set dst (CountLeadingZerosL src));
6146   effect(KILL cr);
6147 
6148   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6149             "jnz     skip\n\t"
6150             "movl    $dst, -1\n"
6151       "skip:\n\t"
6152             "negl    $dst\n\t"
6153             "addl    $dst, 63" %}
6154   ins_encode %{
6155     Register Rdst = $dst$$Register;
6156     Register Rsrc = $src$$Register;
6157     Label skip;
6158     __ bsrq(Rdst, Rsrc);
6159     __ jccb(Assembler::notZero, skip);
6160     __ movl(Rdst, -1);
6161     __ bind(skip);
6162     __ negl(Rdst);
6163     __ addl(Rdst, BitsPerLong - 1);
6164   %}
6165   ins_pipe(ialu_reg);
6166 %}
6167 
6168 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6169   predicate(UseCountTrailingZerosInstruction);
6170   match(Set dst (CountTrailingZerosI src));
6171   effect(KILL cr);
6172 
6173   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6174   ins_encode %{
6175     __ tzcntl($dst$$Register, $src$$Register);
6176   %}
6177   ins_pipe(ialu_reg);
6178 %}
6179 
6180 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6181   predicate(!UseCountTrailingZerosInstruction);
6182   match(Set dst (CountTrailingZerosI src));
6183   effect(KILL cr);
6184 
6185   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6186             "jnz     done\n\t"
6187             "movl    $dst, 32\n"
6188       "done:" %}
6189   ins_encode %{
6190     Register Rdst = $dst$$Register;
6191     Label done;
6192     __ bsfl(Rdst, $src$$Register);
6193     __ jccb(Assembler::notZero, done);
6194     __ movl(Rdst, BitsPerInt);
6195     __ bind(done);
6196   %}
6197   ins_pipe(ialu_reg);
6198 %}
6199 
6200 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6201   predicate(UseCountTrailingZerosInstruction);
6202   match(Set dst (CountTrailingZerosL src));
6203   effect(KILL cr);
6204 
6205   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6206   ins_encode %{
6207     __ tzcntq($dst$$Register, $src$$Register);
6208   %}
6209   ins_pipe(ialu_reg);
6210 %}
6211 
6212 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6213   predicate(!UseCountTrailingZerosInstruction);
6214   match(Set dst (CountTrailingZerosL src));
6215   effect(KILL cr);
6216 
6217   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6218             "jnz     done\n\t"
6219             "movl    $dst, 64\n"
6220       "done:" %}
6221   ins_encode %{
6222     Register Rdst = $dst$$Register;
6223     Label done;
6224     __ bsfq(Rdst, $src$$Register);
6225     __ jccb(Assembler::notZero, done);
6226     __ movl(Rdst, BitsPerLong);
6227     __ bind(done);
6228   %}
6229   ins_pipe(ialu_reg);
6230 %}
6231 
6232 
6233 //---------- Population Count Instructions -------------------------------------
6234 
6235 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6236   predicate(UsePopCountInstruction);
6237   match(Set dst (PopCountI src));
6238   effect(KILL cr);
6239 
6240   format %{ "popcnt  $dst, $src" %}
6241   ins_encode %{
6242     __ popcntl($dst$$Register, $src$$Register);
6243   %}
6244   ins_pipe(ialu_reg);
6245 %}
6246 
6247 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6248   predicate(UsePopCountInstruction);
6249   match(Set dst (PopCountI (LoadI mem)));
6250   effect(KILL cr);
6251 
6252   format %{ "popcnt  $dst, $mem" %}
6253   ins_encode %{
6254     __ popcntl($dst$$Register, $mem$$Address);
6255   %}
6256   ins_pipe(ialu_reg);
6257 %}
6258 
6259 // Note: Long.bitCount(long) returns an int.
6260 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6261   predicate(UsePopCountInstruction);
6262   match(Set dst (PopCountL src));
6263   effect(KILL cr);
6264 
6265   format %{ "popcnt  $dst, $src" %}
6266   ins_encode %{
6267     __ popcntq($dst$$Register, $src$$Register);
6268   %}
6269   ins_pipe(ialu_reg);
6270 %}
6271 
6272 // Note: Long.bitCount(long) returns an int.
6273 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6274   predicate(UsePopCountInstruction);
6275   match(Set dst (PopCountL (LoadL mem)));
6276   effect(KILL cr);
6277 
6278   format %{ "popcnt  $dst, $mem" %}
6279   ins_encode %{
6280     __ popcntq($dst$$Register, $mem$$Address);
6281   %}
6282   ins_pipe(ialu_reg);
6283 %}
6284 
6285 
6286 //----------MemBar Instructions-----------------------------------------------
6287 // Memory barrier flavors
6288 
6289 instruct membar_acquire()
6290 %{
6291   match(MemBarAcquire);
6292   match(LoadFence);
6293   ins_cost(0);
6294 
6295   size(0);
6296   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6297   ins_encode();
6298   ins_pipe(empty);
6299 %}
6300 
6301 instruct membar_acquire_lock()
6302 %{
6303   match(MemBarAcquireLock);
6304   ins_cost(0);
6305 
6306   size(0);
6307   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6308   ins_encode();
6309   ins_pipe(empty);
6310 %}
6311 
6312 instruct membar_release()
6313 %{
6314   match(MemBarRelease);
6315   match(StoreFence);
6316   ins_cost(0);
6317 
6318   size(0);
6319   format %{ "MEMBAR-release ! (empty encoding)" %}
6320   ins_encode();
6321   ins_pipe(empty);
6322 %}
6323 
6324 instruct membar_release_lock()
6325 %{
6326   match(MemBarReleaseLock);
6327   ins_cost(0);
6328 
6329   size(0);
6330   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6331   ins_encode();
6332   ins_pipe(empty);
6333 %}
6334 
6335 instruct membar_volatile(rFlagsReg cr) %{
6336   match(MemBarVolatile);
6337   effect(KILL cr);
6338   ins_cost(400);
6339 
6340   format %{
6341     $$template
6342     if (os::is_MP()) {
6343       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6344     } else {
6345       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6346     }
6347   %}
6348   ins_encode %{
6349     __ membar(Assembler::StoreLoad);
6350   %}
6351   ins_pipe(pipe_slow);
6352 %}
6353 
6354 instruct unnecessary_membar_volatile()
6355 %{
6356   match(MemBarVolatile);
6357   predicate(Matcher::post_store_load_barrier(n));
6358   ins_cost(0);
6359 
6360   size(0);
6361   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6362   ins_encode();
6363   ins_pipe(empty);
6364 %}
6365 
6366 instruct membar_storestore() %{
6367   match(MemBarStoreStore);
6368   ins_cost(0);
6369 
6370   size(0);
6371   format %{ "MEMBAR-storestore (empty encoding)" %}
6372   ins_encode( );
6373   ins_pipe(empty);
6374 %}
6375 
6376 //----------Move Instructions--------------------------------------------------
6377 
6378 instruct castX2P(rRegP dst, rRegL src)
6379 %{
6380   match(Set dst (CastX2P src));
6381 
6382   format %{ "movq    $dst, $src\t# long->ptr" %}
6383   ins_encode %{
6384     if ($dst$$reg != $src$$reg) {
6385       __ movptr($dst$$Register, $src$$Register);
6386     }
6387   %}
6388   ins_pipe(ialu_reg_reg); // XXX
6389 %}
6390 
6391 instruct castP2X(rRegL dst, rRegP src)
6392 %{
6393   match(Set dst (CastP2X src));
6394 
6395   format %{ "movq    $dst, $src\t# ptr -> long" %}
6396   ins_encode %{
6397     if ($dst$$reg != $src$$reg) {
6398       __ movptr($dst$$Register, $src$$Register);
6399     }
6400   %}
6401   ins_pipe(ialu_reg_reg); // XXX
6402 %}
6403 
6404 // Convert oop into int for vectors alignment masking
6405 instruct convP2I(rRegI dst, rRegP src)
6406 %{
6407   match(Set dst (ConvL2I (CastP2X src)));
6408 
6409   format %{ "movl    $dst, $src\t# ptr -> int" %}
6410   ins_encode %{
6411     __ movl($dst$$Register, $src$$Register);
6412   %}
6413   ins_pipe(ialu_reg_reg); // XXX
6414 %}
6415 
6416 // Convert compressed oop into int for vectors alignment masking
6417 // in case of 32bit oops (heap < 4Gb).
6418 instruct convN2I(rRegI dst, rRegN src)
6419 %{
6420   predicate(Universe::narrow_oop_shift() == 0);
6421   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6422 
6423   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6424   ins_encode %{
6425     __ movl($dst$$Register, $src$$Register);
6426   %}
6427   ins_pipe(ialu_reg_reg); // XXX
6428 %}
6429 
6430 // Convert oop pointer into compressed form
6431 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6432   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6433   match(Set dst (EncodeP src));
6434   effect(KILL cr);
6435   format %{ "encode_heap_oop $dst,$src" %}
6436   ins_encode %{
6437     Register s = $src$$Register;
6438     Register d = $dst$$Register;
6439     if (s != d) {
6440       __ movq(d, s);
6441     }
6442     __ encode_heap_oop(d);
6443   %}
6444   ins_pipe(ialu_reg_long);
6445 %}
6446 
6447 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6448   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6449   match(Set dst (EncodeP src));
6450   effect(KILL cr);
6451   format %{ "encode_heap_oop_not_null $dst,$src" %}
6452   ins_encode %{
6453     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6454   %}
6455   ins_pipe(ialu_reg_long);
6456 %}
6457 
6458 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6459   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6460             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6461   match(Set dst (DecodeN src));
6462   effect(KILL cr);
6463   format %{ "decode_heap_oop $dst,$src" %}
6464   ins_encode %{
6465     Register s = $src$$Register;
6466     Register d = $dst$$Register;
6467     if (s != d) {
6468       __ movq(d, s);
6469     }
6470     __ decode_heap_oop(d);
6471   %}
6472   ins_pipe(ialu_reg_long);
6473 %}
6474 
6475 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6476   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6477             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6478   match(Set dst (DecodeN src));
6479   effect(KILL cr);
6480   format %{ "decode_heap_oop_not_null $dst,$src" %}
6481   ins_encode %{
6482     Register s = $src$$Register;
6483     Register d = $dst$$Register;
6484     if (s != d) {
6485       __ decode_heap_oop_not_null(d, s);
6486     } else {
6487       __ decode_heap_oop_not_null(d);
6488     }
6489   %}
6490   ins_pipe(ialu_reg_long);
6491 %}
6492 
6493 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6494   match(Set dst (EncodePKlass src));
6495   effect(KILL cr);
6496   format %{ "encode_klass_not_null $dst,$src" %}
6497   ins_encode %{
6498     __ encode_klass_not_null($dst$$Register, $src$$Register);
6499   %}
6500   ins_pipe(ialu_reg_long);
6501 %}
6502 
6503 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6504   match(Set dst (DecodeNKlass src));
6505   effect(KILL cr);
6506   format %{ "decode_klass_not_null $dst,$src" %}
6507   ins_encode %{
6508     Register s = $src$$Register;
6509     Register d = $dst$$Register;
6510     if (s != d) {
6511       __ decode_klass_not_null(d, s);
6512     } else {
6513       __ decode_klass_not_null(d);
6514     }
6515   %}
6516   ins_pipe(ialu_reg_long);
6517 %}
6518 
6519 
6520 //----------Conditional Move---------------------------------------------------
6521 // Jump
6522 // dummy instruction for generating temp registers
6523 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6524   match(Jump (LShiftL switch_val shift));
6525   ins_cost(350);
6526   predicate(false);
6527   effect(TEMP dest);
6528 
6529   format %{ "leaq    $dest, [$constantaddress]\n\t"
6530             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6531   ins_encode %{
6532     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6533     // to do that and the compiler is using that register as one it can allocate.
6534     // So we build it all by hand.
6535     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6536     // ArrayAddress dispatch(table, index);
6537     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6538     __ lea($dest$$Register, $constantaddress);
6539     __ jmp(dispatch);
6540   %}
6541   ins_pipe(pipe_jmp);
6542 %}
6543 
6544 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6545   match(Jump (AddL (LShiftL switch_val shift) offset));
6546   ins_cost(350);
6547   effect(TEMP dest);
6548 
6549   format %{ "leaq    $dest, [$constantaddress]\n\t"
6550             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6551   ins_encode %{
6552     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6553     // to do that and the compiler is using that register as one it can allocate.
6554     // So we build it all by hand.
6555     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6556     // ArrayAddress dispatch(table, index);
6557     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6558     __ lea($dest$$Register, $constantaddress);
6559     __ jmp(dispatch);
6560   %}
6561   ins_pipe(pipe_jmp);
6562 %}
6563 
6564 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6565   match(Jump switch_val);
6566   ins_cost(350);
6567   effect(TEMP dest);
6568 
6569   format %{ "leaq    $dest, [$constantaddress]\n\t"
6570             "jmp     [$dest + $switch_val]\n\t" %}
6571   ins_encode %{
6572     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6573     // to do that and the compiler is using that register as one it can allocate.
6574     // So we build it all by hand.
6575     // Address index(noreg, switch_reg, Address::times_1);
6576     // ArrayAddress dispatch(table, index);
6577     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6578     __ lea($dest$$Register, $constantaddress);
6579     __ jmp(dispatch);
6580   %}
6581   ins_pipe(pipe_jmp);
6582 %}
6583 
6584 // Conditional move
6585 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6586 %{
6587   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6588 
6589   ins_cost(200); // XXX
6590   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6591   opcode(0x0F, 0x40);
6592   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6593   ins_pipe(pipe_cmov_reg);
6594 %}
6595 
6596 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6597   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6598 
6599   ins_cost(200); // XXX
6600   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6601   opcode(0x0F, 0x40);
6602   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6603   ins_pipe(pipe_cmov_reg);
6604 %}
6605 
6606 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6607   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6608   ins_cost(200);
6609   expand %{
6610     cmovI_regU(cop, cr, dst, src);
6611   %}
6612 %}
6613 
6614 // Conditional move
6615 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6616   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6617 
6618   ins_cost(250); // XXX
6619   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6620   opcode(0x0F, 0x40);
6621   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6622   ins_pipe(pipe_cmov_mem);
6623 %}
6624 
6625 // Conditional move
6626 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6627 %{
6628   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6629 
6630   ins_cost(250); // XXX
6631   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6632   opcode(0x0F, 0x40);
6633   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6634   ins_pipe(pipe_cmov_mem);
6635 %}
6636 
6637 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6638   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6639   ins_cost(250);
6640   expand %{
6641     cmovI_memU(cop, cr, dst, src);
6642   %}
6643 %}
6644 
6645 // Conditional move
6646 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6647 %{
6648   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6649 
6650   ins_cost(200); // XXX
6651   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6652   opcode(0x0F, 0x40);
6653   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6654   ins_pipe(pipe_cmov_reg);
6655 %}
6656 
6657 // Conditional move
6658 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6659 %{
6660   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6661 
6662   ins_cost(200); // XXX
6663   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6664   opcode(0x0F, 0x40);
6665   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6666   ins_pipe(pipe_cmov_reg);
6667 %}
6668 
6669 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6670   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6671   ins_cost(200);
6672   expand %{
6673     cmovN_regU(cop, cr, dst, src);
6674   %}
6675 %}
6676 
6677 // Conditional move
6678 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6679 %{
6680   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6681 
6682   ins_cost(200); // XXX
6683   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6684   opcode(0x0F, 0x40);
6685   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6686   ins_pipe(pipe_cmov_reg);  // XXX
6687 %}
6688 
6689 // Conditional move
6690 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6691 %{
6692   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6693 
6694   ins_cost(200); // XXX
6695   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6696   opcode(0x0F, 0x40);
6697   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6698   ins_pipe(pipe_cmov_reg); // XXX
6699 %}
6700 
6701 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6702   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6703   ins_cost(200);
6704   expand %{
6705     cmovP_regU(cop, cr, dst, src);
6706   %}
6707 %}
6708 
6709 // DISABLED: Requires the ADLC to emit a bottom_type call that
6710 // correctly meets the two pointer arguments; one is an incoming
6711 // register but the other is a memory operand.  ALSO appears to
6712 // be buggy with implicit null checks.
6713 //
6714 //// Conditional move
6715 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6716 //%{
6717 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6718 //  ins_cost(250);
6719 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6720 //  opcode(0x0F,0x40);
6721 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6722 //  ins_pipe( pipe_cmov_mem );
6723 //%}
6724 //
6725 //// Conditional move
6726 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6727 //%{
6728 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6729 //  ins_cost(250);
6730 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6731 //  opcode(0x0F,0x40);
6732 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6733 //  ins_pipe( pipe_cmov_mem );
6734 //%}
6735 
6736 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6737 %{
6738   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6739 
6740   ins_cost(200); // XXX
6741   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6742   opcode(0x0F, 0x40);
6743   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6744   ins_pipe(pipe_cmov_reg);  // XXX
6745 %}
6746 
6747 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6748 %{
6749   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6750 
6751   ins_cost(200); // XXX
6752   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6753   opcode(0x0F, 0x40);
6754   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6755   ins_pipe(pipe_cmov_mem);  // XXX
6756 %}
6757 
6758 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6759 %{
6760   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6761 
6762   ins_cost(200); // XXX
6763   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6764   opcode(0x0F, 0x40);
6765   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6766   ins_pipe(pipe_cmov_reg); // XXX
6767 %}
6768 
6769 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6770   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6771   ins_cost(200);
6772   expand %{
6773     cmovL_regU(cop, cr, dst, src);
6774   %}
6775 %}
6776 
6777 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6778 %{
6779   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6780 
6781   ins_cost(200); // XXX
6782   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6783   opcode(0x0F, 0x40);
6784   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6785   ins_pipe(pipe_cmov_mem); // XXX
6786 %}
6787 
6788 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6789   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6790   ins_cost(200);
6791   expand %{
6792     cmovL_memU(cop, cr, dst, src);
6793   %}
6794 %}
6795 
6796 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6797 %{
6798   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6799 
6800   ins_cost(200); // XXX
6801   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6802             "movss     $dst, $src\n"
6803     "skip:" %}
6804   ins_encode %{
6805     Label Lskip;
6806     // Invert sense of branch from sense of CMOV
6807     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6808     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6809     __ bind(Lskip);
6810   %}
6811   ins_pipe(pipe_slow);
6812 %}
6813 
6814 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6815 // %{
6816 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6817 
6818 //   ins_cost(200); // XXX
6819 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6820 //             "movss     $dst, $src\n"
6821 //     "skip:" %}
6822 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6823 //   ins_pipe(pipe_slow);
6824 // %}
6825 
6826 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6827 %{
6828   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6829 
6830   ins_cost(200); // XXX
6831   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6832             "movss     $dst, $src\n"
6833     "skip:" %}
6834   ins_encode %{
6835     Label Lskip;
6836     // Invert sense of branch from sense of CMOV
6837     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6838     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6839     __ bind(Lskip);
6840   %}
6841   ins_pipe(pipe_slow);
6842 %}
6843 
6844 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6845   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6846   ins_cost(200);
6847   expand %{
6848     cmovF_regU(cop, cr, dst, src);
6849   %}
6850 %}
6851 
6852 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6853 %{
6854   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6855 
6856   ins_cost(200); // XXX
6857   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6858             "movsd     $dst, $src\n"
6859     "skip:" %}
6860   ins_encode %{
6861     Label Lskip;
6862     // Invert sense of branch from sense of CMOV
6863     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6864     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6865     __ bind(Lskip);
6866   %}
6867   ins_pipe(pipe_slow);
6868 %}
6869 
6870 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6871 %{
6872   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6873 
6874   ins_cost(200); // XXX
6875   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6876             "movsd     $dst, $src\n"
6877     "skip:" %}
6878   ins_encode %{
6879     Label Lskip;
6880     // Invert sense of branch from sense of CMOV
6881     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6882     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6883     __ bind(Lskip);
6884   %}
6885   ins_pipe(pipe_slow);
6886 %}
6887 
6888 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6889   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6890   ins_cost(200);
6891   expand %{
6892     cmovD_regU(cop, cr, dst, src);
6893   %}
6894 %}
6895 
6896 //----------Arithmetic Instructions--------------------------------------------
6897 //----------Addition Instructions----------------------------------------------
6898 
6899 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6900 %{
6901   match(Set dst (AddI dst src));
6902   effect(KILL cr);
6903 
6904   format %{ "addl    $dst, $src\t# int" %}
6905   opcode(0x03);
6906   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6907   ins_pipe(ialu_reg_reg);
6908 %}
6909 
6910 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6911 %{
6912   match(Set dst (AddI dst src));
6913   effect(KILL cr);
6914 
6915   format %{ "addl    $dst, $src\t# int" %}
6916   opcode(0x81, 0x00); /* /0 id */
6917   ins_encode(OpcSErm(dst, src), Con8or32(src));
6918   ins_pipe( ialu_reg );
6919 %}
6920 
6921 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6922 %{
6923   match(Set dst (AddI dst (LoadI src)));
6924   effect(KILL cr);
6925 
6926   ins_cost(125); // XXX
6927   format %{ "addl    $dst, $src\t# int" %}
6928   opcode(0x03);
6929   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6930   ins_pipe(ialu_reg_mem);
6931 %}
6932 
6933 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6934 %{
6935   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6936   effect(KILL cr);
6937 
6938   ins_cost(150); // XXX
6939   format %{ "addl    $dst, $src\t# int" %}
6940   opcode(0x01); /* Opcode 01 /r */
6941   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6942   ins_pipe(ialu_mem_reg);
6943 %}
6944 
6945 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6946 %{
6947   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6948   effect(KILL cr);
6949 
6950   ins_cost(125); // XXX
6951   format %{ "addl    $dst, $src\t# int" %}
6952   opcode(0x81); /* Opcode 81 /0 id */
6953   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6954   ins_pipe(ialu_mem_imm);
6955 %}
6956 
6957 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6958 %{
6959   predicate(UseIncDec);
6960   match(Set dst (AddI dst src));
6961   effect(KILL cr);
6962 
6963   format %{ "incl    $dst\t# int" %}
6964   opcode(0xFF, 0x00); // FF /0
6965   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6966   ins_pipe(ialu_reg);
6967 %}
6968 
6969 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6970 %{
6971   predicate(UseIncDec);
6972   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6973   effect(KILL cr);
6974 
6975   ins_cost(125); // XXX
6976   format %{ "incl    $dst\t# int" %}
6977   opcode(0xFF); /* Opcode FF /0 */
6978   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6979   ins_pipe(ialu_mem_imm);
6980 %}
6981 
6982 // XXX why does that use AddI
6983 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6984 %{
6985   predicate(UseIncDec);
6986   match(Set dst (AddI dst src));
6987   effect(KILL cr);
6988 
6989   format %{ "decl    $dst\t# int" %}
6990   opcode(0xFF, 0x01); // FF /1
6991   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6992   ins_pipe(ialu_reg);
6993 %}
6994 
6995 // XXX why does that use AddI
6996 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
6997 %{
6998   predicate(UseIncDec);
6999   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7000   effect(KILL cr);
7001 
7002   ins_cost(125); // XXX
7003   format %{ "decl    $dst\t# int" %}
7004   opcode(0xFF); /* Opcode FF /1 */
7005   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7006   ins_pipe(ialu_mem_imm);
7007 %}
7008 
7009 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7010 %{
7011   match(Set dst (AddI src0 src1));
7012 
7013   ins_cost(110);
7014   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7015   opcode(0x8D); /* 0x8D /r */
7016   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7017   ins_pipe(ialu_reg_reg);
7018 %}
7019 
7020 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7021 %{
7022   match(Set dst (AddL dst src));
7023   effect(KILL cr);
7024 
7025   format %{ "addq    $dst, $src\t# long" %}
7026   opcode(0x03);
7027   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7028   ins_pipe(ialu_reg_reg);
7029 %}
7030 
7031 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7032 %{
7033   match(Set dst (AddL dst src));
7034   effect(KILL cr);
7035 
7036   format %{ "addq    $dst, $src\t# long" %}
7037   opcode(0x81, 0x00); /* /0 id */
7038   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7039   ins_pipe( ialu_reg );
7040 %}
7041 
7042 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7043 %{
7044   match(Set dst (AddL dst (LoadL src)));
7045   effect(KILL cr);
7046 
7047   ins_cost(125); // XXX
7048   format %{ "addq    $dst, $src\t# long" %}
7049   opcode(0x03);
7050   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7051   ins_pipe(ialu_reg_mem);
7052 %}
7053 
7054 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7055 %{
7056   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7057   effect(KILL cr);
7058 
7059   ins_cost(150); // XXX
7060   format %{ "addq    $dst, $src\t# long" %}
7061   opcode(0x01); /* Opcode 01 /r */
7062   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7063   ins_pipe(ialu_mem_reg);
7064 %}
7065 
7066 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7067 %{
7068   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7069   effect(KILL cr);
7070 
7071   ins_cost(125); // XXX
7072   format %{ "addq    $dst, $src\t# long" %}
7073   opcode(0x81); /* Opcode 81 /0 id */
7074   ins_encode(REX_mem_wide(dst),
7075              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7076   ins_pipe(ialu_mem_imm);
7077 %}
7078 
7079 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7080 %{
7081   predicate(UseIncDec);
7082   match(Set dst (AddL dst src));
7083   effect(KILL cr);
7084 
7085   format %{ "incq    $dst\t# long" %}
7086   opcode(0xFF, 0x00); // FF /0
7087   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7088   ins_pipe(ialu_reg);
7089 %}
7090 
7091 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7092 %{
7093   predicate(UseIncDec);
7094   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7095   effect(KILL cr);
7096 
7097   ins_cost(125); // XXX
7098   format %{ "incq    $dst\t# long" %}
7099   opcode(0xFF); /* Opcode FF /0 */
7100   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7101   ins_pipe(ialu_mem_imm);
7102 %}
7103 
7104 // XXX why does that use AddL
7105 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7106 %{
7107   predicate(UseIncDec);
7108   match(Set dst (AddL dst src));
7109   effect(KILL cr);
7110 
7111   format %{ "decq    $dst\t# long" %}
7112   opcode(0xFF, 0x01); // FF /1
7113   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7114   ins_pipe(ialu_reg);
7115 %}
7116 
7117 // XXX why does that use AddL
7118 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7119 %{
7120   predicate(UseIncDec);
7121   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7122   effect(KILL cr);
7123 
7124   ins_cost(125); // XXX
7125   format %{ "decq    $dst\t# long" %}
7126   opcode(0xFF); /* Opcode FF /1 */
7127   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7128   ins_pipe(ialu_mem_imm);
7129 %}
7130 
7131 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7132 %{
7133   match(Set dst (AddL src0 src1));
7134 
7135   ins_cost(110);
7136   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7137   opcode(0x8D); /* 0x8D /r */
7138   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7139   ins_pipe(ialu_reg_reg);
7140 %}
7141 
7142 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7143 %{
7144   match(Set dst (AddP dst src));
7145   effect(KILL cr);
7146 
7147   format %{ "addq    $dst, $src\t# ptr" %}
7148   opcode(0x03);
7149   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7150   ins_pipe(ialu_reg_reg);
7151 %}
7152 
7153 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7154 %{
7155   match(Set dst (AddP dst src));
7156   effect(KILL cr);
7157 
7158   format %{ "addq    $dst, $src\t# ptr" %}
7159   opcode(0x81, 0x00); /* /0 id */
7160   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7161   ins_pipe( ialu_reg );
7162 %}
7163 
7164 // XXX addP mem ops ????
7165 
7166 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7167 %{
7168   match(Set dst (AddP src0 src1));
7169 
7170   ins_cost(110);
7171   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7172   opcode(0x8D); /* 0x8D /r */
7173   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7174   ins_pipe(ialu_reg_reg);
7175 %}
7176 
7177 instruct checkCastPP(rRegP dst)
7178 %{
7179   match(Set dst (CheckCastPP dst));
7180 
7181   size(0);
7182   format %{ "# checkcastPP of $dst" %}
7183   ins_encode(/* empty encoding */);
7184   ins_pipe(empty);
7185 %}
7186 
7187 instruct castPP(rRegP dst)
7188 %{
7189   match(Set dst (CastPP dst));
7190 
7191   size(0);
7192   format %{ "# castPP of $dst" %}
7193   ins_encode(/* empty encoding */);
7194   ins_pipe(empty);
7195 %}
7196 
7197 instruct castII(rRegI dst)
7198 %{
7199   match(Set dst (CastII dst));
7200 
7201   size(0);
7202   format %{ "# castII of $dst" %}
7203   ins_encode(/* empty encoding */);
7204   ins_cost(0);
7205   ins_pipe(empty);
7206 %}
7207 
7208 // LoadP-locked same as a regular LoadP when used with compare-swap
7209 instruct loadPLocked(rRegP dst, memory mem)
7210 %{
7211   match(Set dst (LoadPLocked mem));
7212 
7213   ins_cost(125); // XXX
7214   format %{ "movq    $dst, $mem\t# ptr locked" %}
7215   opcode(0x8B);
7216   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7217   ins_pipe(ialu_reg_mem); // XXX
7218 %}
7219 
7220 // Conditional-store of the updated heap-top.
7221 // Used during allocation of the shared heap.
7222 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7223 
7224 instruct storePConditional(memory heap_top_ptr,
7225                            rax_RegP oldval, rRegP newval,
7226                            rFlagsReg cr)
7227 %{
7228   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7229 
7230   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7231             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7232   opcode(0x0F, 0xB1);
7233   ins_encode(lock_prefix,
7234              REX_reg_mem_wide(newval, heap_top_ptr),
7235              OpcP, OpcS,
7236              reg_mem(newval, heap_top_ptr));
7237   ins_pipe(pipe_cmpxchg);
7238 %}
7239 
7240 // Conditional-store of an int value.
7241 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7242 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7243 %{
7244   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7245   effect(KILL oldval);
7246 
7247   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7248   opcode(0x0F, 0xB1);
7249   ins_encode(lock_prefix,
7250              REX_reg_mem(newval, mem),
7251              OpcP, OpcS,
7252              reg_mem(newval, mem));
7253   ins_pipe(pipe_cmpxchg);
7254 %}
7255 
7256 // Conditional-store of a long value.
7257 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7258 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7259 %{
7260   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7261   effect(KILL oldval);
7262 
7263   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7264   opcode(0x0F, 0xB1);
7265   ins_encode(lock_prefix,
7266              REX_reg_mem_wide(newval, mem),
7267              OpcP, OpcS,
7268              reg_mem(newval, mem));
7269   ins_pipe(pipe_cmpxchg);
7270 %}
7271 
7272 
7273 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7274 instruct compareAndSwapP(rRegI res,
7275                          memory mem_ptr,
7276                          rax_RegP oldval, rRegP newval,
7277                          rFlagsReg cr)
7278 %{
7279   predicate(VM_Version::supports_cx8());
7280   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7281   match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
7282   effect(KILL cr, KILL oldval);
7283 
7284   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7285             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7286             "sete    $res\n\t"
7287             "movzbl  $res, $res" %}
7288   opcode(0x0F, 0xB1);
7289   ins_encode(lock_prefix,
7290              REX_reg_mem_wide(newval, mem_ptr),
7291              OpcP, OpcS,
7292              reg_mem(newval, mem_ptr),
7293              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7294              REX_reg_breg(res, res), // movzbl
7295              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7296   ins_pipe( pipe_cmpxchg );
7297 %}
7298 
7299 instruct compareAndSwapL(rRegI res,
7300                          memory mem_ptr,
7301                          rax_RegL oldval, rRegL newval,
7302                          rFlagsReg cr)
7303 %{
7304   predicate(VM_Version::supports_cx8());
7305   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7306   match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
7307   effect(KILL cr, KILL oldval);
7308 
7309   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7310             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7311             "sete    $res\n\t"
7312             "movzbl  $res, $res" %}
7313   opcode(0x0F, 0xB1);
7314   ins_encode(lock_prefix,
7315              REX_reg_mem_wide(newval, mem_ptr),
7316              OpcP, OpcS,
7317              reg_mem(newval, mem_ptr),
7318              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7319              REX_reg_breg(res, res), // movzbl
7320              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7321   ins_pipe( pipe_cmpxchg );
7322 %}
7323 
7324 instruct compareAndSwapI(rRegI res,
7325                          memory mem_ptr,
7326                          rax_RegI oldval, rRegI newval,
7327                          rFlagsReg cr)
7328 %{
7329   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7330   match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
7331   effect(KILL cr, KILL oldval);
7332 
7333   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7334             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7335             "sete    $res\n\t"
7336             "movzbl  $res, $res" %}
7337   opcode(0x0F, 0xB1);
7338   ins_encode(lock_prefix,
7339              REX_reg_mem(newval, mem_ptr),
7340              OpcP, OpcS,
7341              reg_mem(newval, mem_ptr),
7342              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7343              REX_reg_breg(res, res), // movzbl
7344              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7345   ins_pipe( pipe_cmpxchg );
7346 %}
7347 
7348 
7349 instruct compareAndSwapN(rRegI res,
7350                           memory mem_ptr,
7351                           rax_RegN oldval, rRegN newval,
7352                           rFlagsReg cr) %{
7353   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7354   match(Set res (WeakCompareAndSwapN mem_ptr (Binary oldval newval)));
7355   effect(KILL cr, KILL oldval);
7356 
7357   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7358             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7359             "sete    $res\n\t"
7360             "movzbl  $res, $res" %}
7361   opcode(0x0F, 0xB1);
7362   ins_encode(lock_prefix,
7363              REX_reg_mem(newval, mem_ptr),
7364              OpcP, OpcS,
7365              reg_mem(newval, mem_ptr),
7366              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7367              REX_reg_breg(res, res), // movzbl
7368              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7369   ins_pipe( pipe_cmpxchg );
7370 %}
7371 
7372 instruct compareAndExchangeI(
7373                          memory mem_ptr,
7374                          rax_RegI oldval, rRegI newval,
7375                          rFlagsReg cr)
7376 %{
7377   match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
7378   effect(KILL cr);
7379 
7380   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7381             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7382   opcode(0x0F, 0xB1);
7383   ins_encode(lock_prefix,
7384              REX_reg_mem(newval, mem_ptr),
7385              OpcP, OpcS,
7386              reg_mem(newval, mem_ptr) // lock cmpxchg
7387              );
7388   ins_pipe( pipe_cmpxchg );
7389 %}
7390 
7391 instruct compareAndExchangeL(
7392                          memory mem_ptr,
7393                          rax_RegL oldval, rRegL newval,
7394                          rFlagsReg cr)
7395 %{
7396   predicate(VM_Version::supports_cx8());
7397   match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
7398   effect(KILL cr);
7399 
7400   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7401             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"  %}
7402   opcode(0x0F, 0xB1);
7403   ins_encode(lock_prefix,
7404              REX_reg_mem_wide(newval, mem_ptr),
7405              OpcP, OpcS,
7406              reg_mem(newval, mem_ptr)  // lock cmpxchg
7407             );
7408   ins_pipe( pipe_cmpxchg );
7409 %}
7410 
7411 instruct compareAndExchangeN(
7412                           memory mem_ptr,
7413                           rax_RegN oldval, rRegN newval,
7414                           rFlagsReg cr) %{
7415   match(Set oldval (CompareAndExchangeN mem_ptr (Binary oldval newval)));
7416   effect(KILL cr);
7417 
7418   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7419             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7420   opcode(0x0F, 0xB1);
7421   ins_encode(lock_prefix,
7422              REX_reg_mem(newval, mem_ptr),
7423              OpcP, OpcS,
7424              reg_mem(newval, mem_ptr)  // lock cmpxchg
7425           );
7426   ins_pipe( pipe_cmpxchg );
7427 %}
7428 
7429 instruct compareAndExchangeP(
7430                          memory mem_ptr,
7431                          rax_RegP oldval, rRegP newval,
7432                          rFlagsReg cr)
7433 %{
7434   predicate(VM_Version::supports_cx8());
7435   match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
7436   effect(KILL cr);
7437 
7438   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7439             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
7440   opcode(0x0F, 0xB1);
7441   ins_encode(lock_prefix,
7442              REX_reg_mem_wide(newval, mem_ptr),
7443              OpcP, OpcS,
7444              reg_mem(newval, mem_ptr)  // lock cmpxchg
7445           );
7446   ins_pipe( pipe_cmpxchg );
7447 %}
7448 
7449 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7450   predicate(n->as_LoadStore()->result_not_used());
7451   match(Set dummy (GetAndAddI mem add));
7452   effect(KILL cr);
7453   format %{ "ADDL  [$mem],$add" %}
7454   ins_encode %{
7455     if (os::is_MP()) { __ lock(); }
7456     __ addl($mem$$Address, $add$$constant);
7457   %}
7458   ins_pipe( pipe_cmpxchg );
7459 %}
7460 
7461 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7462   match(Set newval (GetAndAddI mem newval));
7463   effect(KILL cr);
7464   format %{ "XADDL  [$mem],$newval" %}
7465   ins_encode %{
7466     if (os::is_MP()) { __ lock(); }
7467     __ xaddl($mem$$Address, $newval$$Register);
7468   %}
7469   ins_pipe( pipe_cmpxchg );
7470 %}
7471 
7472 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7473   predicate(n->as_LoadStore()->result_not_used());
7474   match(Set dummy (GetAndAddL mem add));
7475   effect(KILL cr);
7476   format %{ "ADDQ  [$mem],$add" %}
7477   ins_encode %{
7478     if (os::is_MP()) { __ lock(); }
7479     __ addq($mem$$Address, $add$$constant);
7480   %}
7481   ins_pipe( pipe_cmpxchg );
7482 %}
7483 
7484 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7485   match(Set newval (GetAndAddL mem newval));
7486   effect(KILL cr);
7487   format %{ "XADDQ  [$mem],$newval" %}
7488   ins_encode %{
7489     if (os::is_MP()) { __ lock(); }
7490     __ xaddq($mem$$Address, $newval$$Register);
7491   %}
7492   ins_pipe( pipe_cmpxchg );
7493 %}
7494 
7495 instruct xchgI( memory mem, rRegI newval) %{
7496   match(Set newval (GetAndSetI mem newval));
7497   format %{ "XCHGL  $newval,[$mem]" %}
7498   ins_encode %{
7499     __ xchgl($newval$$Register, $mem$$Address);
7500   %}
7501   ins_pipe( pipe_cmpxchg );
7502 %}
7503 
7504 instruct xchgL( memory mem, rRegL newval) %{
7505   match(Set newval (GetAndSetL mem newval));
7506   format %{ "XCHGL  $newval,[$mem]" %}
7507   ins_encode %{
7508     __ xchgq($newval$$Register, $mem$$Address);
7509   %}
7510   ins_pipe( pipe_cmpxchg );
7511 %}
7512 
7513 instruct xchgP( memory mem, rRegP newval) %{
7514   match(Set newval (GetAndSetP mem newval));
7515   format %{ "XCHGQ  $newval,[$mem]" %}
7516   ins_encode %{
7517     __ xchgq($newval$$Register, $mem$$Address);
7518   %}
7519   ins_pipe( pipe_cmpxchg );
7520 %}
7521 
7522 instruct xchgN( memory mem, rRegN newval) %{
7523   match(Set newval (GetAndSetN mem newval));
7524   format %{ "XCHGL  $newval,$mem]" %}
7525   ins_encode %{
7526     __ xchgl($newval$$Register, $mem$$Address);
7527   %}
7528   ins_pipe( pipe_cmpxchg );
7529 %}
7530 
7531 //----------Subtraction Instructions-------------------------------------------
7532 
7533 // Integer Subtraction Instructions
7534 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7535 %{
7536   match(Set dst (SubI dst src));
7537   effect(KILL cr);
7538 
7539   format %{ "subl    $dst, $src\t# int" %}
7540   opcode(0x2B);
7541   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7542   ins_pipe(ialu_reg_reg);
7543 %}
7544 
7545 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7546 %{
7547   match(Set dst (SubI dst src));
7548   effect(KILL cr);
7549 
7550   format %{ "subl    $dst, $src\t# int" %}
7551   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7552   ins_encode(OpcSErm(dst, src), Con8or32(src));
7553   ins_pipe(ialu_reg);
7554 %}
7555 
7556 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7557 %{
7558   match(Set dst (SubI dst (LoadI src)));
7559   effect(KILL cr);
7560 
7561   ins_cost(125);
7562   format %{ "subl    $dst, $src\t# int" %}
7563   opcode(0x2B);
7564   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7565   ins_pipe(ialu_reg_mem);
7566 %}
7567 
7568 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7569 %{
7570   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7571   effect(KILL cr);
7572 
7573   ins_cost(150);
7574   format %{ "subl    $dst, $src\t# int" %}
7575   opcode(0x29); /* Opcode 29 /r */
7576   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7577   ins_pipe(ialu_mem_reg);
7578 %}
7579 
7580 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7581 %{
7582   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7583   effect(KILL cr);
7584 
7585   ins_cost(125); // XXX
7586   format %{ "subl    $dst, $src\t# int" %}
7587   opcode(0x81); /* Opcode 81 /5 id */
7588   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7589   ins_pipe(ialu_mem_imm);
7590 %}
7591 
7592 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7593 %{
7594   match(Set dst (SubL dst src));
7595   effect(KILL cr);
7596 
7597   format %{ "subq    $dst, $src\t# long" %}
7598   opcode(0x2B);
7599   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7600   ins_pipe(ialu_reg_reg);
7601 %}
7602 
7603 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7604 %{
7605   match(Set dst (SubL dst src));
7606   effect(KILL cr);
7607 
7608   format %{ "subq    $dst, $src\t# long" %}
7609   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7610   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7611   ins_pipe(ialu_reg);
7612 %}
7613 
7614 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7615 %{
7616   match(Set dst (SubL dst (LoadL src)));
7617   effect(KILL cr);
7618 
7619   ins_cost(125);
7620   format %{ "subq    $dst, $src\t# long" %}
7621   opcode(0x2B);
7622   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7623   ins_pipe(ialu_reg_mem);
7624 %}
7625 
7626 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7627 %{
7628   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7629   effect(KILL cr);
7630 
7631   ins_cost(150);
7632   format %{ "subq    $dst, $src\t# long" %}
7633   opcode(0x29); /* Opcode 29 /r */
7634   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7635   ins_pipe(ialu_mem_reg);
7636 %}
7637 
7638 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7639 %{
7640   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7641   effect(KILL cr);
7642 
7643   ins_cost(125); // XXX
7644   format %{ "subq    $dst, $src\t# long" %}
7645   opcode(0x81); /* Opcode 81 /5 id */
7646   ins_encode(REX_mem_wide(dst),
7647              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7648   ins_pipe(ialu_mem_imm);
7649 %}
7650 
7651 // Subtract from a pointer
7652 // XXX hmpf???
7653 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7654 %{
7655   match(Set dst (AddP dst (SubI zero src)));
7656   effect(KILL cr);
7657 
7658   format %{ "subq    $dst, $src\t# ptr - int" %}
7659   opcode(0x2B);
7660   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7661   ins_pipe(ialu_reg_reg);
7662 %}
7663 
7664 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7665 %{
7666   match(Set dst (SubI zero dst));
7667   effect(KILL cr);
7668 
7669   format %{ "negl    $dst\t# int" %}
7670   opcode(0xF7, 0x03);  // Opcode F7 /3
7671   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7672   ins_pipe(ialu_reg);
7673 %}
7674 
7675 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7676 %{
7677   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7678   effect(KILL cr);
7679 
7680   format %{ "negl    $dst\t# int" %}
7681   opcode(0xF7, 0x03);  // Opcode F7 /3
7682   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7683   ins_pipe(ialu_reg);
7684 %}
7685 
7686 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7687 %{
7688   match(Set dst (SubL zero dst));
7689   effect(KILL cr);
7690 
7691   format %{ "negq    $dst\t# long" %}
7692   opcode(0xF7, 0x03);  // Opcode F7 /3
7693   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7694   ins_pipe(ialu_reg);
7695 %}
7696 
7697 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7698 %{
7699   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7700   effect(KILL cr);
7701 
7702   format %{ "negq    $dst\t# long" %}
7703   opcode(0xF7, 0x03);  // Opcode F7 /3
7704   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7705   ins_pipe(ialu_reg);
7706 %}
7707 
7708 //----------Multiplication/Division Instructions-------------------------------
7709 // Integer Multiplication Instructions
7710 // Multiply Register
7711 
7712 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7713 %{
7714   match(Set dst (MulI dst src));
7715   effect(KILL cr);
7716 
7717   ins_cost(300);
7718   format %{ "imull   $dst, $src\t# int" %}
7719   opcode(0x0F, 0xAF);
7720   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7721   ins_pipe(ialu_reg_reg_alu0);
7722 %}
7723 
7724 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7725 %{
7726   match(Set dst (MulI src imm));
7727   effect(KILL cr);
7728 
7729   ins_cost(300);
7730   format %{ "imull   $dst, $src, $imm\t# int" %}
7731   opcode(0x69); /* 69 /r id */
7732   ins_encode(REX_reg_reg(dst, src),
7733              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7734   ins_pipe(ialu_reg_reg_alu0);
7735 %}
7736 
7737 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7738 %{
7739   match(Set dst (MulI dst (LoadI src)));
7740   effect(KILL cr);
7741 
7742   ins_cost(350);
7743   format %{ "imull   $dst, $src\t# int" %}
7744   opcode(0x0F, 0xAF);
7745   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7746   ins_pipe(ialu_reg_mem_alu0);
7747 %}
7748 
7749 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7750 %{
7751   match(Set dst (MulI (LoadI src) imm));
7752   effect(KILL cr);
7753 
7754   ins_cost(300);
7755   format %{ "imull   $dst, $src, $imm\t# int" %}
7756   opcode(0x69); /* 69 /r id */
7757   ins_encode(REX_reg_mem(dst, src),
7758              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7759   ins_pipe(ialu_reg_mem_alu0);
7760 %}
7761 
7762 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7763 %{
7764   match(Set dst (MulL dst src));
7765   effect(KILL cr);
7766 
7767   ins_cost(300);
7768   format %{ "imulq   $dst, $src\t# long" %}
7769   opcode(0x0F, 0xAF);
7770   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7771   ins_pipe(ialu_reg_reg_alu0);
7772 %}
7773 
7774 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7775 %{
7776   match(Set dst (MulL src imm));
7777   effect(KILL cr);
7778 
7779   ins_cost(300);
7780   format %{ "imulq   $dst, $src, $imm\t# long" %}
7781   opcode(0x69); /* 69 /r id */
7782   ins_encode(REX_reg_reg_wide(dst, src),
7783              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7784   ins_pipe(ialu_reg_reg_alu0);
7785 %}
7786 
7787 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7788 %{
7789   match(Set dst (MulL dst (LoadL src)));
7790   effect(KILL cr);
7791 
7792   ins_cost(350);
7793   format %{ "imulq   $dst, $src\t# long" %}
7794   opcode(0x0F, 0xAF);
7795   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7796   ins_pipe(ialu_reg_mem_alu0);
7797 %}
7798 
7799 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7800 %{
7801   match(Set dst (MulL (LoadL src) imm));
7802   effect(KILL cr);
7803 
7804   ins_cost(300);
7805   format %{ "imulq   $dst, $src, $imm\t# long" %}
7806   opcode(0x69); /* 69 /r id */
7807   ins_encode(REX_reg_mem_wide(dst, src),
7808              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7809   ins_pipe(ialu_reg_mem_alu0);
7810 %}
7811 
7812 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7813 %{
7814   match(Set dst (MulHiL src rax));
7815   effect(USE_KILL rax, KILL cr);
7816 
7817   ins_cost(300);
7818   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7819   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7820   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7821   ins_pipe(ialu_reg_reg_alu0);
7822 %}
7823 
7824 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7825                    rFlagsReg cr)
7826 %{
7827   match(Set rax (DivI rax div));
7828   effect(KILL rdx, KILL cr);
7829 
7830   ins_cost(30*100+10*100); // XXX
7831   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7832             "jne,s   normal\n\t"
7833             "xorl    rdx, rdx\n\t"
7834             "cmpl    $div, -1\n\t"
7835             "je,s    done\n"
7836     "normal: cdql\n\t"
7837             "idivl   $div\n"
7838     "done:"        %}
7839   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7840   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7841   ins_pipe(ialu_reg_reg_alu0);
7842 %}
7843 
7844 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7845                    rFlagsReg cr)
7846 %{
7847   match(Set rax (DivL rax div));
7848   effect(KILL rdx, KILL cr);
7849 
7850   ins_cost(30*100+10*100); // XXX
7851   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7852             "cmpq    rax, rdx\n\t"
7853             "jne,s   normal\n\t"
7854             "xorl    rdx, rdx\n\t"
7855             "cmpq    $div, -1\n\t"
7856             "je,s    done\n"
7857     "normal: cdqq\n\t"
7858             "idivq   $div\n"
7859     "done:"        %}
7860   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7861   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7862   ins_pipe(ialu_reg_reg_alu0);
7863 %}
7864 
7865 // Integer DIVMOD with Register, both quotient and mod results
7866 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7867                              rFlagsReg cr)
7868 %{
7869   match(DivModI rax div);
7870   effect(KILL cr);
7871 
7872   ins_cost(30*100+10*100); // XXX
7873   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7874             "jne,s   normal\n\t"
7875             "xorl    rdx, rdx\n\t"
7876             "cmpl    $div, -1\n\t"
7877             "je,s    done\n"
7878     "normal: cdql\n\t"
7879             "idivl   $div\n"
7880     "done:"        %}
7881   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7882   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7883   ins_pipe(pipe_slow);
7884 %}
7885 
7886 // Long DIVMOD with Register, both quotient and mod results
7887 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7888                              rFlagsReg cr)
7889 %{
7890   match(DivModL rax div);
7891   effect(KILL cr);
7892 
7893   ins_cost(30*100+10*100); // XXX
7894   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7895             "cmpq    rax, rdx\n\t"
7896             "jne,s   normal\n\t"
7897             "xorl    rdx, rdx\n\t"
7898             "cmpq    $div, -1\n\t"
7899             "je,s    done\n"
7900     "normal: cdqq\n\t"
7901             "idivq   $div\n"
7902     "done:"        %}
7903   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7904   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7905   ins_pipe(pipe_slow);
7906 %}
7907 
7908 //----------- DivL-By-Constant-Expansions--------------------------------------
7909 // DivI cases are handled by the compiler
7910 
7911 // Magic constant, reciprocal of 10
7912 instruct loadConL_0x6666666666666667(rRegL dst)
7913 %{
7914   effect(DEF dst);
7915 
7916   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7917   ins_encode(load_immL(dst, 0x6666666666666667));
7918   ins_pipe(ialu_reg);
7919 %}
7920 
7921 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7922 %{
7923   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7924 
7925   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7926   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7927   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7928   ins_pipe(ialu_reg_reg_alu0);
7929 %}
7930 
7931 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7932 %{
7933   effect(USE_DEF dst, KILL cr);
7934 
7935   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7936   opcode(0xC1, 0x7); /* C1 /7 ib */
7937   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7938   ins_pipe(ialu_reg);
7939 %}
7940 
7941 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7942 %{
7943   effect(USE_DEF dst, KILL cr);
7944 
7945   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7946   opcode(0xC1, 0x7); /* C1 /7 ib */
7947   ins_encode(reg_opc_imm_wide(dst, 0x2));
7948   ins_pipe(ialu_reg);
7949 %}
7950 
7951 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7952 %{
7953   match(Set dst (DivL src div));
7954 
7955   ins_cost((5+8)*100);
7956   expand %{
7957     rax_RegL rax;                     // Killed temp
7958     rFlagsReg cr;                     // Killed
7959     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7960     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7961     sarL_rReg_63(src, cr);            // sarq  src, 63
7962     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7963     subL_rReg(dst, src, cr);          // subl  rdx, src
7964   %}
7965 %}
7966 
7967 //-----------------------------------------------------------------------------
7968 
7969 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7970                    rFlagsReg cr)
7971 %{
7972   match(Set rdx (ModI rax div));
7973   effect(KILL rax, KILL cr);
7974 
7975   ins_cost(300); // XXX
7976   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7977             "jne,s   normal\n\t"
7978             "xorl    rdx, rdx\n\t"
7979             "cmpl    $div, -1\n\t"
7980             "je,s    done\n"
7981     "normal: cdql\n\t"
7982             "idivl   $div\n"
7983     "done:"        %}
7984   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7985   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7986   ins_pipe(ialu_reg_reg_alu0);
7987 %}
7988 
7989 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7990                    rFlagsReg cr)
7991 %{
7992   match(Set rdx (ModL rax div));
7993   effect(KILL rax, KILL cr);
7994 
7995   ins_cost(300); // XXX
7996   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7997             "cmpq    rax, rdx\n\t"
7998             "jne,s   normal\n\t"
7999             "xorl    rdx, rdx\n\t"
8000             "cmpq    $div, -1\n\t"
8001             "je,s    done\n"
8002     "normal: cdqq\n\t"
8003             "idivq   $div\n"
8004     "done:"        %}
8005   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
8006   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
8007   ins_pipe(ialu_reg_reg_alu0);
8008 %}
8009 
8010 // Integer Shift Instructions
8011 // Shift Left by one
8012 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8013 %{
8014   match(Set dst (LShiftI dst shift));
8015   effect(KILL cr);
8016 
8017   format %{ "sall    $dst, $shift" %}
8018   opcode(0xD1, 0x4); /* D1 /4 */
8019   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8020   ins_pipe(ialu_reg);
8021 %}
8022 
8023 // Shift Left by one
8024 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8025 %{
8026   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8027   effect(KILL cr);
8028 
8029   format %{ "sall    $dst, $shift\t" %}
8030   opcode(0xD1, 0x4); /* D1 /4 */
8031   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8032   ins_pipe(ialu_mem_imm);
8033 %}
8034 
8035 // Shift Left by 8-bit immediate
8036 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8037 %{
8038   match(Set dst (LShiftI dst shift));
8039   effect(KILL cr);
8040 
8041   format %{ "sall    $dst, $shift" %}
8042   opcode(0xC1, 0x4); /* C1 /4 ib */
8043   ins_encode(reg_opc_imm(dst, shift));
8044   ins_pipe(ialu_reg);
8045 %}
8046 
8047 // Shift Left by 8-bit immediate
8048 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8049 %{
8050   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8051   effect(KILL cr);
8052 
8053   format %{ "sall    $dst, $shift" %}
8054   opcode(0xC1, 0x4); /* C1 /4 ib */
8055   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8056   ins_pipe(ialu_mem_imm);
8057 %}
8058 
8059 // Shift Left by variable
8060 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8061 %{
8062   match(Set dst (LShiftI dst shift));
8063   effect(KILL cr);
8064 
8065   format %{ "sall    $dst, $shift" %}
8066   opcode(0xD3, 0x4); /* D3 /4 */
8067   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8068   ins_pipe(ialu_reg_reg);
8069 %}
8070 
8071 // Shift Left by variable
8072 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8073 %{
8074   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8075   effect(KILL cr);
8076 
8077   format %{ "sall    $dst, $shift" %}
8078   opcode(0xD3, 0x4); /* D3 /4 */
8079   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8080   ins_pipe(ialu_mem_reg);
8081 %}
8082 
8083 // Arithmetic shift right by one
8084 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8085 %{
8086   match(Set dst (RShiftI dst shift));
8087   effect(KILL cr);
8088 
8089   format %{ "sarl    $dst, $shift" %}
8090   opcode(0xD1, 0x7); /* D1 /7 */
8091   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8092   ins_pipe(ialu_reg);
8093 %}
8094 
8095 // Arithmetic shift right by one
8096 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8097 %{
8098   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8099   effect(KILL cr);
8100 
8101   format %{ "sarl    $dst, $shift" %}
8102   opcode(0xD1, 0x7); /* D1 /7 */
8103   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8104   ins_pipe(ialu_mem_imm);
8105 %}
8106 
8107 // Arithmetic Shift Right by 8-bit immediate
8108 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8109 %{
8110   match(Set dst (RShiftI dst shift));
8111   effect(KILL cr);
8112 
8113   format %{ "sarl    $dst, $shift" %}
8114   opcode(0xC1, 0x7); /* C1 /7 ib */
8115   ins_encode(reg_opc_imm(dst, shift));
8116   ins_pipe(ialu_mem_imm);
8117 %}
8118 
8119 // Arithmetic Shift Right by 8-bit immediate
8120 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8121 %{
8122   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8123   effect(KILL cr);
8124 
8125   format %{ "sarl    $dst, $shift" %}
8126   opcode(0xC1, 0x7); /* C1 /7 ib */
8127   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8128   ins_pipe(ialu_mem_imm);
8129 %}
8130 
8131 // Arithmetic Shift Right by variable
8132 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8133 %{
8134   match(Set dst (RShiftI dst shift));
8135   effect(KILL cr);
8136 
8137   format %{ "sarl    $dst, $shift" %}
8138   opcode(0xD3, 0x7); /* D3 /7 */
8139   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8140   ins_pipe(ialu_reg_reg);
8141 %}
8142 
8143 // Arithmetic Shift Right by variable
8144 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8145 %{
8146   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8147   effect(KILL cr);
8148 
8149   format %{ "sarl    $dst, $shift" %}
8150   opcode(0xD3, 0x7); /* D3 /7 */
8151   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8152   ins_pipe(ialu_mem_reg);
8153 %}
8154 
8155 // Logical shift right by one
8156 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8157 %{
8158   match(Set dst (URShiftI dst shift));
8159   effect(KILL cr);
8160 
8161   format %{ "shrl    $dst, $shift" %}
8162   opcode(0xD1, 0x5); /* D1 /5 */
8163   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8164   ins_pipe(ialu_reg);
8165 %}
8166 
8167 // Logical shift right by one
8168 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8169 %{
8170   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8171   effect(KILL cr);
8172 
8173   format %{ "shrl    $dst, $shift" %}
8174   opcode(0xD1, 0x5); /* D1 /5 */
8175   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8176   ins_pipe(ialu_mem_imm);
8177 %}
8178 
8179 // Logical Shift Right by 8-bit immediate
8180 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8181 %{
8182   match(Set dst (URShiftI dst shift));
8183   effect(KILL cr);
8184 
8185   format %{ "shrl    $dst, $shift" %}
8186   opcode(0xC1, 0x5); /* C1 /5 ib */
8187   ins_encode(reg_opc_imm(dst, shift));
8188   ins_pipe(ialu_reg);
8189 %}
8190 
8191 // Logical Shift Right by 8-bit immediate
8192 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8193 %{
8194   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8195   effect(KILL cr);
8196 
8197   format %{ "shrl    $dst, $shift" %}
8198   opcode(0xC1, 0x5); /* C1 /5 ib */
8199   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8200   ins_pipe(ialu_mem_imm);
8201 %}
8202 
8203 // Logical Shift Right by variable
8204 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8205 %{
8206   match(Set dst (URShiftI dst shift));
8207   effect(KILL cr);
8208 
8209   format %{ "shrl    $dst, $shift" %}
8210   opcode(0xD3, 0x5); /* D3 /5 */
8211   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8212   ins_pipe(ialu_reg_reg);
8213 %}
8214 
8215 // Logical Shift Right by variable
8216 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8217 %{
8218   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8219   effect(KILL cr);
8220 
8221   format %{ "shrl    $dst, $shift" %}
8222   opcode(0xD3, 0x5); /* D3 /5 */
8223   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8224   ins_pipe(ialu_mem_reg);
8225 %}
8226 
8227 // Long Shift Instructions
8228 // Shift Left by one
8229 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8230 %{
8231   match(Set dst (LShiftL dst shift));
8232   effect(KILL cr);
8233 
8234   format %{ "salq    $dst, $shift" %}
8235   opcode(0xD1, 0x4); /* D1 /4 */
8236   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8237   ins_pipe(ialu_reg);
8238 %}
8239 
8240 // Shift Left by one
8241 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8242 %{
8243   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8244   effect(KILL cr);
8245 
8246   format %{ "salq    $dst, $shift" %}
8247   opcode(0xD1, 0x4); /* D1 /4 */
8248   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8249   ins_pipe(ialu_mem_imm);
8250 %}
8251 
8252 // Shift Left by 8-bit immediate
8253 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8254 %{
8255   match(Set dst (LShiftL dst shift));
8256   effect(KILL cr);
8257 
8258   format %{ "salq    $dst, $shift" %}
8259   opcode(0xC1, 0x4); /* C1 /4 ib */
8260   ins_encode(reg_opc_imm_wide(dst, shift));
8261   ins_pipe(ialu_reg);
8262 %}
8263 
8264 // Shift Left by 8-bit immediate
8265 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8266 %{
8267   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8268   effect(KILL cr);
8269 
8270   format %{ "salq    $dst, $shift" %}
8271   opcode(0xC1, 0x4); /* C1 /4 ib */
8272   ins_encode(REX_mem_wide(dst), OpcP,
8273              RM_opc_mem(secondary, dst), Con8or32(shift));
8274   ins_pipe(ialu_mem_imm);
8275 %}
8276 
8277 // Shift Left by variable
8278 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8279 %{
8280   match(Set dst (LShiftL dst shift));
8281   effect(KILL cr);
8282 
8283   format %{ "salq    $dst, $shift" %}
8284   opcode(0xD3, 0x4); /* D3 /4 */
8285   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8286   ins_pipe(ialu_reg_reg);
8287 %}
8288 
8289 // Shift Left by variable
8290 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8291 %{
8292   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8293   effect(KILL cr);
8294 
8295   format %{ "salq    $dst, $shift" %}
8296   opcode(0xD3, 0x4); /* D3 /4 */
8297   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8298   ins_pipe(ialu_mem_reg);
8299 %}
8300 
8301 // Arithmetic shift right by one
8302 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8303 %{
8304   match(Set dst (RShiftL dst shift));
8305   effect(KILL cr);
8306 
8307   format %{ "sarq    $dst, $shift" %}
8308   opcode(0xD1, 0x7); /* D1 /7 */
8309   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8310   ins_pipe(ialu_reg);
8311 %}
8312 
8313 // Arithmetic shift right by one
8314 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8315 %{
8316   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8317   effect(KILL cr);
8318 
8319   format %{ "sarq    $dst, $shift" %}
8320   opcode(0xD1, 0x7); /* D1 /7 */
8321   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8322   ins_pipe(ialu_mem_imm);
8323 %}
8324 
8325 // Arithmetic Shift Right by 8-bit immediate
8326 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8327 %{
8328   match(Set dst (RShiftL dst shift));
8329   effect(KILL cr);
8330 
8331   format %{ "sarq    $dst, $shift" %}
8332   opcode(0xC1, 0x7); /* C1 /7 ib */
8333   ins_encode(reg_opc_imm_wide(dst, shift));
8334   ins_pipe(ialu_mem_imm);
8335 %}
8336 
8337 // Arithmetic Shift Right by 8-bit immediate
8338 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8339 %{
8340   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8341   effect(KILL cr);
8342 
8343   format %{ "sarq    $dst, $shift" %}
8344   opcode(0xC1, 0x7); /* C1 /7 ib */
8345   ins_encode(REX_mem_wide(dst), OpcP,
8346              RM_opc_mem(secondary, dst), Con8or32(shift));
8347   ins_pipe(ialu_mem_imm);
8348 %}
8349 
8350 // Arithmetic Shift Right by variable
8351 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8352 %{
8353   match(Set dst (RShiftL dst shift));
8354   effect(KILL cr);
8355 
8356   format %{ "sarq    $dst, $shift" %}
8357   opcode(0xD3, 0x7); /* D3 /7 */
8358   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8359   ins_pipe(ialu_reg_reg);
8360 %}
8361 
8362 // Arithmetic Shift Right by variable
8363 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8364 %{
8365   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8366   effect(KILL cr);
8367 
8368   format %{ "sarq    $dst, $shift" %}
8369   opcode(0xD3, 0x7); /* D3 /7 */
8370   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8371   ins_pipe(ialu_mem_reg);
8372 %}
8373 
8374 // Logical shift right by one
8375 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8376 %{
8377   match(Set dst (URShiftL dst shift));
8378   effect(KILL cr);
8379 
8380   format %{ "shrq    $dst, $shift" %}
8381   opcode(0xD1, 0x5); /* D1 /5 */
8382   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8383   ins_pipe(ialu_reg);
8384 %}
8385 
8386 // Logical shift right by one
8387 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8388 %{
8389   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8390   effect(KILL cr);
8391 
8392   format %{ "shrq    $dst, $shift" %}
8393   opcode(0xD1, 0x5); /* D1 /5 */
8394   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8395   ins_pipe(ialu_mem_imm);
8396 %}
8397 
8398 // Logical Shift Right by 8-bit immediate
8399 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8400 %{
8401   match(Set dst (URShiftL dst shift));
8402   effect(KILL cr);
8403 
8404   format %{ "shrq    $dst, $shift" %}
8405   opcode(0xC1, 0x5); /* C1 /5 ib */
8406   ins_encode(reg_opc_imm_wide(dst, shift));
8407   ins_pipe(ialu_reg);
8408 %}
8409 
8410 
8411 // Logical Shift Right by 8-bit immediate
8412 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8413 %{
8414   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8415   effect(KILL cr);
8416 
8417   format %{ "shrq    $dst, $shift" %}
8418   opcode(0xC1, 0x5); /* C1 /5 ib */
8419   ins_encode(REX_mem_wide(dst), OpcP,
8420              RM_opc_mem(secondary, dst), Con8or32(shift));
8421   ins_pipe(ialu_mem_imm);
8422 %}
8423 
8424 // Logical Shift Right by variable
8425 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8426 %{
8427   match(Set dst (URShiftL dst shift));
8428   effect(KILL cr);
8429 
8430   format %{ "shrq    $dst, $shift" %}
8431   opcode(0xD3, 0x5); /* D3 /5 */
8432   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8433   ins_pipe(ialu_reg_reg);
8434 %}
8435 
8436 // Logical Shift Right by variable
8437 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8438 %{
8439   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8440   effect(KILL cr);
8441 
8442   format %{ "shrq    $dst, $shift" %}
8443   opcode(0xD3, 0x5); /* D3 /5 */
8444   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8445   ins_pipe(ialu_mem_reg);
8446 %}
8447 
8448 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8449 // This idiom is used by the compiler for the i2b bytecode.
8450 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8451 %{
8452   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8453 
8454   format %{ "movsbl  $dst, $src\t# i2b" %}
8455   opcode(0x0F, 0xBE);
8456   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8457   ins_pipe(ialu_reg_reg);
8458 %}
8459 
8460 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8461 // This idiom is used by the compiler the i2s bytecode.
8462 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8463 %{
8464   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8465 
8466   format %{ "movswl  $dst, $src\t# i2s" %}
8467   opcode(0x0F, 0xBF);
8468   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8469   ins_pipe(ialu_reg_reg);
8470 %}
8471 
8472 // ROL/ROR instructions
8473 
8474 // ROL expand
8475 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8476   effect(KILL cr, USE_DEF dst);
8477 
8478   format %{ "roll    $dst" %}
8479   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8480   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8481   ins_pipe(ialu_reg);
8482 %}
8483 
8484 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8485   effect(USE_DEF dst, USE shift, KILL cr);
8486 
8487   format %{ "roll    $dst, $shift" %}
8488   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8489   ins_encode( reg_opc_imm(dst, shift) );
8490   ins_pipe(ialu_reg);
8491 %}
8492 
8493 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8494 %{
8495   effect(USE_DEF dst, USE shift, KILL cr);
8496 
8497   format %{ "roll    $dst, $shift" %}
8498   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8499   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8500   ins_pipe(ialu_reg_reg);
8501 %}
8502 // end of ROL expand
8503 
8504 // Rotate Left by one
8505 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8506 %{
8507   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8508 
8509   expand %{
8510     rolI_rReg_imm1(dst, cr);
8511   %}
8512 %}
8513 
8514 // Rotate Left by 8-bit immediate
8515 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8516 %{
8517   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8518   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8519 
8520   expand %{
8521     rolI_rReg_imm8(dst, lshift, cr);
8522   %}
8523 %}
8524 
8525 // Rotate Left by variable
8526 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8527 %{
8528   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8529 
8530   expand %{
8531     rolI_rReg_CL(dst, shift, cr);
8532   %}
8533 %}
8534 
8535 // Rotate Left by variable
8536 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8537 %{
8538   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8539 
8540   expand %{
8541     rolI_rReg_CL(dst, shift, cr);
8542   %}
8543 %}
8544 
8545 // ROR expand
8546 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8547 %{
8548   effect(USE_DEF dst, KILL cr);
8549 
8550   format %{ "rorl    $dst" %}
8551   opcode(0xD1, 0x1); /* D1 /1 */
8552   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8553   ins_pipe(ialu_reg);
8554 %}
8555 
8556 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8557 %{
8558   effect(USE_DEF dst, USE shift, KILL cr);
8559 
8560   format %{ "rorl    $dst, $shift" %}
8561   opcode(0xC1, 0x1); /* C1 /1 ib */
8562   ins_encode(reg_opc_imm(dst, shift));
8563   ins_pipe(ialu_reg);
8564 %}
8565 
8566 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8567 %{
8568   effect(USE_DEF dst, USE shift, KILL cr);
8569 
8570   format %{ "rorl    $dst, $shift" %}
8571   opcode(0xD3, 0x1); /* D3 /1 */
8572   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8573   ins_pipe(ialu_reg_reg);
8574 %}
8575 // end of ROR expand
8576 
8577 // Rotate Right by one
8578 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8579 %{
8580   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8581 
8582   expand %{
8583     rorI_rReg_imm1(dst, cr);
8584   %}
8585 %}
8586 
8587 // Rotate Right by 8-bit immediate
8588 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8589 %{
8590   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8591   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8592 
8593   expand %{
8594     rorI_rReg_imm8(dst, rshift, cr);
8595   %}
8596 %}
8597 
8598 // Rotate Right by variable
8599 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8600 %{
8601   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8602 
8603   expand %{
8604     rorI_rReg_CL(dst, shift, cr);
8605   %}
8606 %}
8607 
8608 // Rotate Right by variable
8609 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8610 %{
8611   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8612 
8613   expand %{
8614     rorI_rReg_CL(dst, shift, cr);
8615   %}
8616 %}
8617 
8618 // for long rotate
8619 // ROL expand
8620 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8621   effect(USE_DEF dst, KILL cr);
8622 
8623   format %{ "rolq    $dst" %}
8624   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8625   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8626   ins_pipe(ialu_reg);
8627 %}
8628 
8629 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8630   effect(USE_DEF dst, USE shift, KILL cr);
8631 
8632   format %{ "rolq    $dst, $shift" %}
8633   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8634   ins_encode( reg_opc_imm_wide(dst, shift) );
8635   ins_pipe(ialu_reg);
8636 %}
8637 
8638 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8639 %{
8640   effect(USE_DEF dst, USE shift, KILL cr);
8641 
8642   format %{ "rolq    $dst, $shift" %}
8643   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8644   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8645   ins_pipe(ialu_reg_reg);
8646 %}
8647 // end of ROL expand
8648 
8649 // Rotate Left by one
8650 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8651 %{
8652   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8653 
8654   expand %{
8655     rolL_rReg_imm1(dst, cr);
8656   %}
8657 %}
8658 
8659 // Rotate Left by 8-bit immediate
8660 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8661 %{
8662   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8663   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8664 
8665   expand %{
8666     rolL_rReg_imm8(dst, lshift, cr);
8667   %}
8668 %}
8669 
8670 // Rotate Left by variable
8671 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8672 %{
8673   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8674 
8675   expand %{
8676     rolL_rReg_CL(dst, shift, cr);
8677   %}
8678 %}
8679 
8680 // Rotate Left by variable
8681 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8682 %{
8683   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8684 
8685   expand %{
8686     rolL_rReg_CL(dst, shift, cr);
8687   %}
8688 %}
8689 
8690 // ROR expand
8691 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8692 %{
8693   effect(USE_DEF dst, KILL cr);
8694 
8695   format %{ "rorq    $dst" %}
8696   opcode(0xD1, 0x1); /* D1 /1 */
8697   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8698   ins_pipe(ialu_reg);
8699 %}
8700 
8701 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8702 %{
8703   effect(USE_DEF dst, USE shift, KILL cr);
8704 
8705   format %{ "rorq    $dst, $shift" %}
8706   opcode(0xC1, 0x1); /* C1 /1 ib */
8707   ins_encode(reg_opc_imm_wide(dst, shift));
8708   ins_pipe(ialu_reg);
8709 %}
8710 
8711 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8712 %{
8713   effect(USE_DEF dst, USE shift, KILL cr);
8714 
8715   format %{ "rorq    $dst, $shift" %}
8716   opcode(0xD3, 0x1); /* D3 /1 */
8717   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8718   ins_pipe(ialu_reg_reg);
8719 %}
8720 // end of ROR expand
8721 
8722 // Rotate Right by one
8723 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8724 %{
8725   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8726 
8727   expand %{
8728     rorL_rReg_imm1(dst, cr);
8729   %}
8730 %}
8731 
8732 // Rotate Right by 8-bit immediate
8733 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8734 %{
8735   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8736   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8737 
8738   expand %{
8739     rorL_rReg_imm8(dst, rshift, cr);
8740   %}
8741 %}
8742 
8743 // Rotate Right by variable
8744 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8745 %{
8746   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8747 
8748   expand %{
8749     rorL_rReg_CL(dst, shift, cr);
8750   %}
8751 %}
8752 
8753 // Rotate Right by variable
8754 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8755 %{
8756   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8757 
8758   expand %{
8759     rorL_rReg_CL(dst, shift, cr);
8760   %}
8761 %}
8762 
8763 // Logical Instructions
8764 
8765 // Integer Logical Instructions
8766 
8767 // And Instructions
8768 // And Register with Register
8769 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8770 %{
8771   match(Set dst (AndI dst src));
8772   effect(KILL cr);
8773 
8774   format %{ "andl    $dst, $src\t# int" %}
8775   opcode(0x23);
8776   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8777   ins_pipe(ialu_reg_reg);
8778 %}
8779 
8780 // And Register with Immediate 255
8781 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8782 %{
8783   match(Set dst (AndI dst src));
8784 
8785   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8786   opcode(0x0F, 0xB6);
8787   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8788   ins_pipe(ialu_reg);
8789 %}
8790 
8791 // And Register with Immediate 255 and promote to long
8792 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8793 %{
8794   match(Set dst (ConvI2L (AndI src mask)));
8795 
8796   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8797   opcode(0x0F, 0xB6);
8798   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8799   ins_pipe(ialu_reg);
8800 %}
8801 
8802 // And Register with Immediate 65535
8803 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8804 %{
8805   match(Set dst (AndI dst src));
8806 
8807   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8808   opcode(0x0F, 0xB7);
8809   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8810   ins_pipe(ialu_reg);
8811 %}
8812 
8813 // And Register with Immediate 65535 and promote to long
8814 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8815 %{
8816   match(Set dst (ConvI2L (AndI src mask)));
8817 
8818   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8819   opcode(0x0F, 0xB7);
8820   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8821   ins_pipe(ialu_reg);
8822 %}
8823 
8824 // And Register with Immediate
8825 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8826 %{
8827   match(Set dst (AndI dst src));
8828   effect(KILL cr);
8829 
8830   format %{ "andl    $dst, $src\t# int" %}
8831   opcode(0x81, 0x04); /* Opcode 81 /4 */
8832   ins_encode(OpcSErm(dst, src), Con8or32(src));
8833   ins_pipe(ialu_reg);
8834 %}
8835 
8836 // And Register with Memory
8837 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8838 %{
8839   match(Set dst (AndI dst (LoadI src)));
8840   effect(KILL cr);
8841 
8842   ins_cost(125);
8843   format %{ "andl    $dst, $src\t# int" %}
8844   opcode(0x23);
8845   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8846   ins_pipe(ialu_reg_mem);
8847 %}
8848 
8849 // And Memory with Register
8850 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8851 %{
8852   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8853   effect(KILL cr);
8854 
8855   ins_cost(150);
8856   format %{ "andl    $dst, $src\t# int" %}
8857   opcode(0x21); /* Opcode 21 /r */
8858   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8859   ins_pipe(ialu_mem_reg);
8860 %}
8861 
8862 // And Memory with Immediate
8863 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8864 %{
8865   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8866   effect(KILL cr);
8867 
8868   ins_cost(125);
8869   format %{ "andl    $dst, $src\t# int" %}
8870   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8871   ins_encode(REX_mem(dst), OpcSE(src),
8872              RM_opc_mem(secondary, dst), Con8or32(src));
8873   ins_pipe(ialu_mem_imm);
8874 %}
8875 
8876 // BMI1 instructions
8877 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8878   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8879   predicate(UseBMI1Instructions);
8880   effect(KILL cr);
8881 
8882   ins_cost(125);
8883   format %{ "andnl  $dst, $src1, $src2" %}
8884 
8885   ins_encode %{
8886     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8887   %}
8888   ins_pipe(ialu_reg_mem);
8889 %}
8890 
8891 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8892   match(Set dst (AndI (XorI src1 minus_1) src2));
8893   predicate(UseBMI1Instructions);
8894   effect(KILL cr);
8895 
8896   format %{ "andnl  $dst, $src1, $src2" %}
8897 
8898   ins_encode %{
8899     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8900   %}
8901   ins_pipe(ialu_reg);
8902 %}
8903 
8904 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8905   match(Set dst (AndI (SubI imm_zero src) src));
8906   predicate(UseBMI1Instructions);
8907   effect(KILL cr);
8908 
8909   format %{ "blsil  $dst, $src" %}
8910 
8911   ins_encode %{
8912     __ blsil($dst$$Register, $src$$Register);
8913   %}
8914   ins_pipe(ialu_reg);
8915 %}
8916 
8917 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8918   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8919   predicate(UseBMI1Instructions);
8920   effect(KILL cr);
8921 
8922   ins_cost(125);
8923   format %{ "blsil  $dst, $src" %}
8924 
8925   ins_encode %{
8926     __ blsil($dst$$Register, $src$$Address);
8927   %}
8928   ins_pipe(ialu_reg_mem);
8929 %}
8930 
8931 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8932 %{
8933   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8934   predicate(UseBMI1Instructions);
8935   effect(KILL cr);
8936 
8937   ins_cost(125);
8938   format %{ "blsmskl $dst, $src" %}
8939 
8940   ins_encode %{
8941     __ blsmskl($dst$$Register, $src$$Address);
8942   %}
8943   ins_pipe(ialu_reg_mem);
8944 %}
8945 
8946 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8947 %{
8948   match(Set dst (XorI (AddI src minus_1) src));
8949   predicate(UseBMI1Instructions);
8950   effect(KILL cr);
8951 
8952   format %{ "blsmskl $dst, $src" %}
8953 
8954   ins_encode %{
8955     __ blsmskl($dst$$Register, $src$$Register);
8956   %}
8957 
8958   ins_pipe(ialu_reg);
8959 %}
8960 
8961 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8962 %{
8963   match(Set dst (AndI (AddI src minus_1) src) );
8964   predicate(UseBMI1Instructions);
8965   effect(KILL cr);
8966 
8967   format %{ "blsrl  $dst, $src" %}
8968 
8969   ins_encode %{
8970     __ blsrl($dst$$Register, $src$$Register);
8971   %}
8972 
8973   ins_pipe(ialu_reg_mem);
8974 %}
8975 
8976 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8977 %{
8978   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8979   predicate(UseBMI1Instructions);
8980   effect(KILL cr);
8981 
8982   ins_cost(125);
8983   format %{ "blsrl  $dst, $src" %}
8984 
8985   ins_encode %{
8986     __ blsrl($dst$$Register, $src$$Address);
8987   %}
8988 
8989   ins_pipe(ialu_reg);
8990 %}
8991 
8992 // Or Instructions
8993 // Or Register with Register
8994 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8995 %{
8996   match(Set dst (OrI dst src));
8997   effect(KILL cr);
8998 
8999   format %{ "orl     $dst, $src\t# int" %}
9000   opcode(0x0B);
9001   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9002   ins_pipe(ialu_reg_reg);
9003 %}
9004 
9005 // Or Register with Immediate
9006 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9007 %{
9008   match(Set dst (OrI dst src));
9009   effect(KILL cr);
9010 
9011   format %{ "orl     $dst, $src\t# int" %}
9012   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9013   ins_encode(OpcSErm(dst, src), Con8or32(src));
9014   ins_pipe(ialu_reg);
9015 %}
9016 
9017 // Or Register with Memory
9018 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9019 %{
9020   match(Set dst (OrI dst (LoadI src)));
9021   effect(KILL cr);
9022 
9023   ins_cost(125);
9024   format %{ "orl     $dst, $src\t# int" %}
9025   opcode(0x0B);
9026   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9027   ins_pipe(ialu_reg_mem);
9028 %}
9029 
9030 // Or Memory with Register
9031 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9032 %{
9033   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9034   effect(KILL cr);
9035 
9036   ins_cost(150);
9037   format %{ "orl     $dst, $src\t# int" %}
9038   opcode(0x09); /* Opcode 09 /r */
9039   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9040   ins_pipe(ialu_mem_reg);
9041 %}
9042 
9043 // Or Memory with Immediate
9044 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
9045 %{
9046   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
9047   effect(KILL cr);
9048 
9049   ins_cost(125);
9050   format %{ "orl     $dst, $src\t# int" %}
9051   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9052   ins_encode(REX_mem(dst), OpcSE(src),
9053              RM_opc_mem(secondary, dst), Con8or32(src));
9054   ins_pipe(ialu_mem_imm);
9055 %}
9056 
9057 // Xor Instructions
9058 // Xor Register with Register
9059 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9060 %{
9061   match(Set dst (XorI dst src));
9062   effect(KILL cr);
9063 
9064   format %{ "xorl    $dst, $src\t# int" %}
9065   opcode(0x33);
9066   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9067   ins_pipe(ialu_reg_reg);
9068 %}
9069 
9070 // Xor Register with Immediate -1
9071 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9072   match(Set dst (XorI dst imm));
9073 
9074   format %{ "not    $dst" %}
9075   ins_encode %{
9076      __ notl($dst$$Register);
9077   %}
9078   ins_pipe(ialu_reg);
9079 %}
9080 
9081 // Xor Register with Immediate
9082 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9083 %{
9084   match(Set dst (XorI dst src));
9085   effect(KILL cr);
9086 
9087   format %{ "xorl    $dst, $src\t# int" %}
9088   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9089   ins_encode(OpcSErm(dst, src), Con8or32(src));
9090   ins_pipe(ialu_reg);
9091 %}
9092 
9093 // Xor Register with Memory
9094 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9095 %{
9096   match(Set dst (XorI dst (LoadI src)));
9097   effect(KILL cr);
9098 
9099   ins_cost(125);
9100   format %{ "xorl    $dst, $src\t# int" %}
9101   opcode(0x33);
9102   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9103   ins_pipe(ialu_reg_mem);
9104 %}
9105 
9106 // Xor Memory with Register
9107 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9108 %{
9109   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9110   effect(KILL cr);
9111 
9112   ins_cost(150);
9113   format %{ "xorl    $dst, $src\t# int" %}
9114   opcode(0x31); /* Opcode 31 /r */
9115   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9116   ins_pipe(ialu_mem_reg);
9117 %}
9118 
9119 // Xor Memory with Immediate
9120 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9121 %{
9122   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9123   effect(KILL cr);
9124 
9125   ins_cost(125);
9126   format %{ "xorl    $dst, $src\t# int" %}
9127   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9128   ins_encode(REX_mem(dst), OpcSE(src),
9129              RM_opc_mem(secondary, dst), Con8or32(src));
9130   ins_pipe(ialu_mem_imm);
9131 %}
9132 
9133 
9134 // Long Logical Instructions
9135 
9136 // And Instructions
9137 // And Register with Register
9138 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9139 %{
9140   match(Set dst (AndL dst src));
9141   effect(KILL cr);
9142 
9143   format %{ "andq    $dst, $src\t# long" %}
9144   opcode(0x23);
9145   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9146   ins_pipe(ialu_reg_reg);
9147 %}
9148 
9149 // And Register with Immediate 255
9150 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9151 %{
9152   match(Set dst (AndL dst src));
9153 
9154   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9155   opcode(0x0F, 0xB6);
9156   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9157   ins_pipe(ialu_reg);
9158 %}
9159 
9160 // And Register with Immediate 65535
9161 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9162 %{
9163   match(Set dst (AndL dst src));
9164 
9165   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9166   opcode(0x0F, 0xB7);
9167   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9168   ins_pipe(ialu_reg);
9169 %}
9170 
9171 // And Register with Immediate
9172 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9173 %{
9174   match(Set dst (AndL dst src));
9175   effect(KILL cr);
9176 
9177   format %{ "andq    $dst, $src\t# long" %}
9178   opcode(0x81, 0x04); /* Opcode 81 /4 */
9179   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9180   ins_pipe(ialu_reg);
9181 %}
9182 
9183 // And Register with Memory
9184 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9185 %{
9186   match(Set dst (AndL dst (LoadL src)));
9187   effect(KILL cr);
9188 
9189   ins_cost(125);
9190   format %{ "andq    $dst, $src\t# long" %}
9191   opcode(0x23);
9192   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9193   ins_pipe(ialu_reg_mem);
9194 %}
9195 
9196 // And Memory with Register
9197 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9198 %{
9199   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9200   effect(KILL cr);
9201 
9202   ins_cost(150);
9203   format %{ "andq    $dst, $src\t# long" %}
9204   opcode(0x21); /* Opcode 21 /r */
9205   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9206   ins_pipe(ialu_mem_reg);
9207 %}
9208 
9209 // And Memory with Immediate
9210 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9211 %{
9212   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9213   effect(KILL cr);
9214 
9215   ins_cost(125);
9216   format %{ "andq    $dst, $src\t# long" %}
9217   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9218   ins_encode(REX_mem_wide(dst), OpcSE(src),
9219              RM_opc_mem(secondary, dst), Con8or32(src));
9220   ins_pipe(ialu_mem_imm);
9221 %}
9222 
9223 // BMI1 instructions
9224 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9225   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9226   predicate(UseBMI1Instructions);
9227   effect(KILL cr);
9228 
9229   ins_cost(125);
9230   format %{ "andnq  $dst, $src1, $src2" %}
9231 
9232   ins_encode %{
9233     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9234   %}
9235   ins_pipe(ialu_reg_mem);
9236 %}
9237 
9238 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9239   match(Set dst (AndL (XorL src1 minus_1) src2));
9240   predicate(UseBMI1Instructions);
9241   effect(KILL cr);
9242 
9243   format %{ "andnq  $dst, $src1, $src2" %}
9244 
9245   ins_encode %{
9246   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9247   %}
9248   ins_pipe(ialu_reg_mem);
9249 %}
9250 
9251 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9252   match(Set dst (AndL (SubL imm_zero src) src));
9253   predicate(UseBMI1Instructions);
9254   effect(KILL cr);
9255 
9256   format %{ "blsiq  $dst, $src" %}
9257 
9258   ins_encode %{
9259     __ blsiq($dst$$Register, $src$$Register);
9260   %}
9261   ins_pipe(ialu_reg);
9262 %}
9263 
9264 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9265   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9266   predicate(UseBMI1Instructions);
9267   effect(KILL cr);
9268 
9269   ins_cost(125);
9270   format %{ "blsiq  $dst, $src" %}
9271 
9272   ins_encode %{
9273     __ blsiq($dst$$Register, $src$$Address);
9274   %}
9275   ins_pipe(ialu_reg_mem);
9276 %}
9277 
9278 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9279 %{
9280   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9281   predicate(UseBMI1Instructions);
9282   effect(KILL cr);
9283 
9284   ins_cost(125);
9285   format %{ "blsmskq $dst, $src" %}
9286 
9287   ins_encode %{
9288     __ blsmskq($dst$$Register, $src$$Address);
9289   %}
9290   ins_pipe(ialu_reg_mem);
9291 %}
9292 
9293 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9294 %{
9295   match(Set dst (XorL (AddL src minus_1) src));
9296   predicate(UseBMI1Instructions);
9297   effect(KILL cr);
9298 
9299   format %{ "blsmskq $dst, $src" %}
9300 
9301   ins_encode %{
9302     __ blsmskq($dst$$Register, $src$$Register);
9303   %}
9304 
9305   ins_pipe(ialu_reg);
9306 %}
9307 
9308 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9309 %{
9310   match(Set dst (AndL (AddL src minus_1) src) );
9311   predicate(UseBMI1Instructions);
9312   effect(KILL cr);
9313 
9314   format %{ "blsrq  $dst, $src" %}
9315 
9316   ins_encode %{
9317     __ blsrq($dst$$Register, $src$$Register);
9318   %}
9319 
9320   ins_pipe(ialu_reg);
9321 %}
9322 
9323 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9324 %{
9325   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9326   predicate(UseBMI1Instructions);
9327   effect(KILL cr);
9328 
9329   ins_cost(125);
9330   format %{ "blsrq  $dst, $src" %}
9331 
9332   ins_encode %{
9333     __ blsrq($dst$$Register, $src$$Address);
9334   %}
9335 
9336   ins_pipe(ialu_reg);
9337 %}
9338 
9339 // Or Instructions
9340 // Or Register with Register
9341 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9342 %{
9343   match(Set dst (OrL dst src));
9344   effect(KILL cr);
9345 
9346   format %{ "orq     $dst, $src\t# long" %}
9347   opcode(0x0B);
9348   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9349   ins_pipe(ialu_reg_reg);
9350 %}
9351 
9352 // Use any_RegP to match R15 (TLS register) without spilling.
9353 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9354   match(Set dst (OrL dst (CastP2X src)));
9355   effect(KILL cr);
9356 
9357   format %{ "orq     $dst, $src\t# long" %}
9358   opcode(0x0B);
9359   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9360   ins_pipe(ialu_reg_reg);
9361 %}
9362 
9363 
9364 // Or Register with Immediate
9365 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9366 %{
9367   match(Set dst (OrL dst src));
9368   effect(KILL cr);
9369 
9370   format %{ "orq     $dst, $src\t# long" %}
9371   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9372   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9373   ins_pipe(ialu_reg);
9374 %}
9375 
9376 // Or Register with Memory
9377 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9378 %{
9379   match(Set dst (OrL dst (LoadL src)));
9380   effect(KILL cr);
9381 
9382   ins_cost(125);
9383   format %{ "orq     $dst, $src\t# long" %}
9384   opcode(0x0B);
9385   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9386   ins_pipe(ialu_reg_mem);
9387 %}
9388 
9389 // Or Memory with Register
9390 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9391 %{
9392   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9393   effect(KILL cr);
9394 
9395   ins_cost(150);
9396   format %{ "orq     $dst, $src\t# long" %}
9397   opcode(0x09); /* Opcode 09 /r */
9398   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9399   ins_pipe(ialu_mem_reg);
9400 %}
9401 
9402 // Or Memory with Immediate
9403 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9404 %{
9405   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9406   effect(KILL cr);
9407 
9408   ins_cost(125);
9409   format %{ "orq     $dst, $src\t# long" %}
9410   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9411   ins_encode(REX_mem_wide(dst), OpcSE(src),
9412              RM_opc_mem(secondary, dst), Con8or32(src));
9413   ins_pipe(ialu_mem_imm);
9414 %}
9415 
9416 // Xor Instructions
9417 // Xor Register with Register
9418 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9419 %{
9420   match(Set dst (XorL dst src));
9421   effect(KILL cr);
9422 
9423   format %{ "xorq    $dst, $src\t# long" %}
9424   opcode(0x33);
9425   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9426   ins_pipe(ialu_reg_reg);
9427 %}
9428 
9429 // Xor Register with Immediate -1
9430 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9431   match(Set dst (XorL dst imm));
9432 
9433   format %{ "notq   $dst" %}
9434   ins_encode %{
9435      __ notq($dst$$Register);
9436   %}
9437   ins_pipe(ialu_reg);
9438 %}
9439 
9440 // Xor Register with Immediate
9441 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9442 %{
9443   match(Set dst (XorL dst src));
9444   effect(KILL cr);
9445 
9446   format %{ "xorq    $dst, $src\t# long" %}
9447   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9448   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9449   ins_pipe(ialu_reg);
9450 %}
9451 
9452 // Xor Register with Memory
9453 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9454 %{
9455   match(Set dst (XorL dst (LoadL src)));
9456   effect(KILL cr);
9457 
9458   ins_cost(125);
9459   format %{ "xorq    $dst, $src\t# long" %}
9460   opcode(0x33);
9461   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9462   ins_pipe(ialu_reg_mem);
9463 %}
9464 
9465 // Xor Memory with Register
9466 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9467 %{
9468   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9469   effect(KILL cr);
9470 
9471   ins_cost(150);
9472   format %{ "xorq    $dst, $src\t# long" %}
9473   opcode(0x31); /* Opcode 31 /r */
9474   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9475   ins_pipe(ialu_mem_reg);
9476 %}
9477 
9478 // Xor Memory with Immediate
9479 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9480 %{
9481   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9482   effect(KILL cr);
9483 
9484   ins_cost(125);
9485   format %{ "xorq    $dst, $src\t# long" %}
9486   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9487   ins_encode(REX_mem_wide(dst), OpcSE(src),
9488              RM_opc_mem(secondary, dst), Con8or32(src));
9489   ins_pipe(ialu_mem_imm);
9490 %}
9491 
9492 // Convert Int to Boolean
9493 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9494 %{
9495   match(Set dst (Conv2B src));
9496   effect(KILL cr);
9497 
9498   format %{ "testl   $src, $src\t# ci2b\n\t"
9499             "setnz   $dst\n\t"
9500             "movzbl  $dst, $dst" %}
9501   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9502              setNZ_reg(dst),
9503              REX_reg_breg(dst, dst), // movzbl
9504              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9505   ins_pipe(pipe_slow); // XXX
9506 %}
9507 
9508 // Convert Pointer to Boolean
9509 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9510 %{
9511   match(Set dst (Conv2B src));
9512   effect(KILL cr);
9513 
9514   format %{ "testq   $src, $src\t# cp2b\n\t"
9515             "setnz   $dst\n\t"
9516             "movzbl  $dst, $dst" %}
9517   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9518              setNZ_reg(dst),
9519              REX_reg_breg(dst, dst), // movzbl
9520              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9521   ins_pipe(pipe_slow); // XXX
9522 %}
9523 
9524 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9525 %{
9526   match(Set dst (CmpLTMask p q));
9527   effect(KILL cr);
9528 
9529   ins_cost(400);
9530   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9531             "setlt   $dst\n\t"
9532             "movzbl  $dst, $dst\n\t"
9533             "negl    $dst" %}
9534   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9535              setLT_reg(dst),
9536              REX_reg_breg(dst, dst), // movzbl
9537              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9538              neg_reg(dst));
9539   ins_pipe(pipe_slow);
9540 %}
9541 
9542 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9543 %{
9544   match(Set dst (CmpLTMask dst zero));
9545   effect(KILL cr);
9546 
9547   ins_cost(100);
9548   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9549   ins_encode %{
9550   __ sarl($dst$$Register, 31);
9551   %}
9552   ins_pipe(ialu_reg);
9553 %}
9554 
9555 /* Better to save a register than avoid a branch */
9556 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9557 %{
9558   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9559   effect(KILL cr);
9560   ins_cost(300);
9561   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9562             "jge    done\n\t"
9563             "addl   $p,$y\n"
9564             "done:  " %}
9565   ins_encode %{
9566     Register Rp = $p$$Register;
9567     Register Rq = $q$$Register;
9568     Register Ry = $y$$Register;
9569     Label done;
9570     __ subl(Rp, Rq);
9571     __ jccb(Assembler::greaterEqual, done);
9572     __ addl(Rp, Ry);
9573     __ bind(done);
9574   %}
9575   ins_pipe(pipe_cmplt);
9576 %}
9577 
9578 /* Better to save a register than avoid a branch */
9579 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9580 %{
9581   match(Set y (AndI (CmpLTMask p q) y));
9582   effect(KILL cr);
9583 
9584   ins_cost(300);
9585 
9586   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9587             "jlt      done\n\t"
9588             "xorl     $y, $y\n"
9589             "done:  " %}
9590   ins_encode %{
9591     Register Rp = $p$$Register;
9592     Register Rq = $q$$Register;
9593     Register Ry = $y$$Register;
9594     Label done;
9595     __ cmpl(Rp, Rq);
9596     __ jccb(Assembler::less, done);
9597     __ xorl(Ry, Ry);
9598     __ bind(done);
9599   %}
9600   ins_pipe(pipe_cmplt);
9601 %}
9602 
9603 
9604 //---------- FP Instructions------------------------------------------------
9605 
9606 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9607 %{
9608   match(Set cr (CmpF src1 src2));
9609 
9610   ins_cost(145);
9611   format %{ "ucomiss $src1, $src2\n\t"
9612             "jnp,s   exit\n\t"
9613             "pushfq\t# saw NaN, set CF\n\t"
9614             "andq    [rsp], #0xffffff2b\n\t"
9615             "popfq\n"
9616     "exit:" %}
9617   ins_encode %{
9618     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9619     emit_cmpfp_fixup(_masm);
9620   %}
9621   ins_pipe(pipe_slow);
9622 %}
9623 
9624 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9625   match(Set cr (CmpF src1 src2));
9626 
9627   ins_cost(100);
9628   format %{ "ucomiss $src1, $src2" %}
9629   ins_encode %{
9630     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9631   %}
9632   ins_pipe(pipe_slow);
9633 %}
9634 
9635 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9636 %{
9637   match(Set cr (CmpF src1 (LoadF src2)));
9638 
9639   ins_cost(145);
9640   format %{ "ucomiss $src1, $src2\n\t"
9641             "jnp,s   exit\n\t"
9642             "pushfq\t# saw NaN, set CF\n\t"
9643             "andq    [rsp], #0xffffff2b\n\t"
9644             "popfq\n"
9645     "exit:" %}
9646   ins_encode %{
9647     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9648     emit_cmpfp_fixup(_masm);
9649   %}
9650   ins_pipe(pipe_slow);
9651 %}
9652 
9653 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9654   match(Set cr (CmpF src1 (LoadF src2)));
9655 
9656   ins_cost(100);
9657   format %{ "ucomiss $src1, $src2" %}
9658   ins_encode %{
9659     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9660   %}
9661   ins_pipe(pipe_slow);
9662 %}
9663 
9664 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9665   match(Set cr (CmpF src con));
9666 
9667   ins_cost(145);
9668   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9669             "jnp,s   exit\n\t"
9670             "pushfq\t# saw NaN, set CF\n\t"
9671             "andq    [rsp], #0xffffff2b\n\t"
9672             "popfq\n"
9673     "exit:" %}
9674   ins_encode %{
9675     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9676     emit_cmpfp_fixup(_masm);
9677   %}
9678   ins_pipe(pipe_slow);
9679 %}
9680 
9681 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9682   match(Set cr (CmpF src con));
9683   ins_cost(100);
9684   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9685   ins_encode %{
9686     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9687   %}
9688   ins_pipe(pipe_slow);
9689 %}
9690 
9691 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9692 %{
9693   match(Set cr (CmpD src1 src2));
9694 
9695   ins_cost(145);
9696   format %{ "ucomisd $src1, $src2\n\t"
9697             "jnp,s   exit\n\t"
9698             "pushfq\t# saw NaN, set CF\n\t"
9699             "andq    [rsp], #0xffffff2b\n\t"
9700             "popfq\n"
9701     "exit:" %}
9702   ins_encode %{
9703     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9704     emit_cmpfp_fixup(_masm);
9705   %}
9706   ins_pipe(pipe_slow);
9707 %}
9708 
9709 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9710   match(Set cr (CmpD src1 src2));
9711 
9712   ins_cost(100);
9713   format %{ "ucomisd $src1, $src2 test" %}
9714   ins_encode %{
9715     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9716   %}
9717   ins_pipe(pipe_slow);
9718 %}
9719 
9720 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9721 %{
9722   match(Set cr (CmpD src1 (LoadD src2)));
9723 
9724   ins_cost(145);
9725   format %{ "ucomisd $src1, $src2\n\t"
9726             "jnp,s   exit\n\t"
9727             "pushfq\t# saw NaN, set CF\n\t"
9728             "andq    [rsp], #0xffffff2b\n\t"
9729             "popfq\n"
9730     "exit:" %}
9731   ins_encode %{
9732     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9733     emit_cmpfp_fixup(_masm);
9734   %}
9735   ins_pipe(pipe_slow);
9736 %}
9737 
9738 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9739   match(Set cr (CmpD src1 (LoadD src2)));
9740 
9741   ins_cost(100);
9742   format %{ "ucomisd $src1, $src2" %}
9743   ins_encode %{
9744     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9745   %}
9746   ins_pipe(pipe_slow);
9747 %}
9748 
9749 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9750   match(Set cr (CmpD src con));
9751 
9752   ins_cost(145);
9753   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9754             "jnp,s   exit\n\t"
9755             "pushfq\t# saw NaN, set CF\n\t"
9756             "andq    [rsp], #0xffffff2b\n\t"
9757             "popfq\n"
9758     "exit:" %}
9759   ins_encode %{
9760     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9761     emit_cmpfp_fixup(_masm);
9762   %}
9763   ins_pipe(pipe_slow);
9764 %}
9765 
9766 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9767   match(Set cr (CmpD src con));
9768   ins_cost(100);
9769   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9770   ins_encode %{
9771     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9772   %}
9773   ins_pipe(pipe_slow);
9774 %}
9775 
9776 // Compare into -1,0,1
9777 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9778 %{
9779   match(Set dst (CmpF3 src1 src2));
9780   effect(KILL cr);
9781 
9782   ins_cost(275);
9783   format %{ "ucomiss $src1, $src2\n\t"
9784             "movl    $dst, #-1\n\t"
9785             "jp,s    done\n\t"
9786             "jb,s    done\n\t"
9787             "setne   $dst\n\t"
9788             "movzbl  $dst, $dst\n"
9789     "done:" %}
9790   ins_encode %{
9791     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9792     emit_cmpfp3(_masm, $dst$$Register);
9793   %}
9794   ins_pipe(pipe_slow);
9795 %}
9796 
9797 // Compare into -1,0,1
9798 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9799 %{
9800   match(Set dst (CmpF3 src1 (LoadF src2)));
9801   effect(KILL cr);
9802 
9803   ins_cost(275);
9804   format %{ "ucomiss $src1, $src2\n\t"
9805             "movl    $dst, #-1\n\t"
9806             "jp,s    done\n\t"
9807             "jb,s    done\n\t"
9808             "setne   $dst\n\t"
9809             "movzbl  $dst, $dst\n"
9810     "done:" %}
9811   ins_encode %{
9812     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9813     emit_cmpfp3(_masm, $dst$$Register);
9814   %}
9815   ins_pipe(pipe_slow);
9816 %}
9817 
9818 // Compare into -1,0,1
9819 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9820   match(Set dst (CmpF3 src con));
9821   effect(KILL cr);
9822 
9823   ins_cost(275);
9824   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9825             "movl    $dst, #-1\n\t"
9826             "jp,s    done\n\t"
9827             "jb,s    done\n\t"
9828             "setne   $dst\n\t"
9829             "movzbl  $dst, $dst\n"
9830     "done:" %}
9831   ins_encode %{
9832     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9833     emit_cmpfp3(_masm, $dst$$Register);
9834   %}
9835   ins_pipe(pipe_slow);
9836 %}
9837 
9838 // Compare into -1,0,1
9839 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9840 %{
9841   match(Set dst (CmpD3 src1 src2));
9842   effect(KILL cr);
9843 
9844   ins_cost(275);
9845   format %{ "ucomisd $src1, $src2\n\t"
9846             "movl    $dst, #-1\n\t"
9847             "jp,s    done\n\t"
9848             "jb,s    done\n\t"
9849             "setne   $dst\n\t"
9850             "movzbl  $dst, $dst\n"
9851     "done:" %}
9852   ins_encode %{
9853     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9854     emit_cmpfp3(_masm, $dst$$Register);
9855   %}
9856   ins_pipe(pipe_slow);
9857 %}
9858 
9859 // Compare into -1,0,1
9860 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9861 %{
9862   match(Set dst (CmpD3 src1 (LoadD src2)));
9863   effect(KILL cr);
9864 
9865   ins_cost(275);
9866   format %{ "ucomisd $src1, $src2\n\t"
9867             "movl    $dst, #-1\n\t"
9868             "jp,s    done\n\t"
9869             "jb,s    done\n\t"
9870             "setne   $dst\n\t"
9871             "movzbl  $dst, $dst\n"
9872     "done:" %}
9873   ins_encode %{
9874     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9875     emit_cmpfp3(_masm, $dst$$Register);
9876   %}
9877   ins_pipe(pipe_slow);
9878 %}
9879 
9880 // Compare into -1,0,1
9881 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9882   match(Set dst (CmpD3 src con));
9883   effect(KILL cr);
9884 
9885   ins_cost(275);
9886   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9887             "movl    $dst, #-1\n\t"
9888             "jp,s    done\n\t"
9889             "jb,s    done\n\t"
9890             "setne   $dst\n\t"
9891             "movzbl  $dst, $dst\n"
9892     "done:" %}
9893   ins_encode %{
9894     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9895     emit_cmpfp3(_masm, $dst$$Register);
9896   %}
9897   ins_pipe(pipe_slow);
9898 %}
9899 
9900 // -----------Trig and Trancendental Instructions------------------------------
9901 instruct tanD_reg(regD dst) %{
9902   match(Set dst (TanD dst));
9903 
9904   format %{ "dtan   $dst\n\t" %}
9905   ins_encode( Push_SrcXD(dst),
9906               Opcode(0xD9), Opcode(0xF2),   //fptan
9907               Opcode(0xDD), Opcode(0xD8),   //fstp st
9908               Push_ResultXD(dst) );
9909   ins_pipe( pipe_slow );
9910 %}
9911 
9912 instruct log10D_reg(regD dst) %{
9913   // The source and result Double operands in XMM registers
9914   match(Set dst (Log10D dst));
9915   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9916   // fyl2x        ; compute log_10(2) * log_2(x)
9917   format %{ "fldlg2\t\t\t#Log10\n\t"
9918             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9919          %}
9920    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9921               Push_SrcXD(dst),
9922               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9923               Push_ResultXD(dst));
9924 
9925   ins_pipe( pipe_slow );
9926 %}
9927 
9928 //----------Arithmetic Conversion Instructions---------------------------------
9929 
9930 instruct roundFloat_nop(regF dst)
9931 %{
9932   match(Set dst (RoundFloat dst));
9933 
9934   ins_cost(0);
9935   ins_encode();
9936   ins_pipe(empty);
9937 %}
9938 
9939 instruct roundDouble_nop(regD dst)
9940 %{
9941   match(Set dst (RoundDouble dst));
9942 
9943   ins_cost(0);
9944   ins_encode();
9945   ins_pipe(empty);
9946 %}
9947 
9948 instruct convF2D_reg_reg(regD dst, regF src)
9949 %{
9950   match(Set dst (ConvF2D src));
9951 
9952   format %{ "cvtss2sd $dst, $src" %}
9953   ins_encode %{
9954     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9955   %}
9956   ins_pipe(pipe_slow); // XXX
9957 %}
9958 
9959 instruct convF2D_reg_mem(regD dst, memory src)
9960 %{
9961   match(Set dst (ConvF2D (LoadF src)));
9962 
9963   format %{ "cvtss2sd $dst, $src" %}
9964   ins_encode %{
9965     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9966   %}
9967   ins_pipe(pipe_slow); // XXX
9968 %}
9969 
9970 instruct convD2F_reg_reg(regF dst, regD src)
9971 %{
9972   match(Set dst (ConvD2F src));
9973 
9974   format %{ "cvtsd2ss $dst, $src" %}
9975   ins_encode %{
9976     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9977   %}
9978   ins_pipe(pipe_slow); // XXX
9979 %}
9980 
9981 instruct convD2F_reg_mem(regF dst, memory src)
9982 %{
9983   match(Set dst (ConvD2F (LoadD src)));
9984 
9985   format %{ "cvtsd2ss $dst, $src" %}
9986   ins_encode %{
9987     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9988   %}
9989   ins_pipe(pipe_slow); // XXX
9990 %}
9991 
9992 // XXX do mem variants
9993 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9994 %{
9995   match(Set dst (ConvF2I src));
9996   effect(KILL cr);
9997 
9998   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9999             "cmpl    $dst, #0x80000000\n\t"
10000             "jne,s   done\n\t"
10001             "subq    rsp, #8\n\t"
10002             "movss   [rsp], $src\n\t"
10003             "call    f2i_fixup\n\t"
10004             "popq    $dst\n"
10005     "done:   "%}
10006   ins_encode %{
10007     Label done;
10008     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
10009     __ cmpl($dst$$Register, 0x80000000);
10010     __ jccb(Assembler::notEqual, done);
10011     __ subptr(rsp, 8);
10012     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10013     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10014     __ pop($dst$$Register);
10015     __ bind(done);
10016   %}
10017   ins_pipe(pipe_slow);
10018 %}
10019 
10020 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10021 %{
10022   match(Set dst (ConvF2L src));
10023   effect(KILL cr);
10024 
10025   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10026             "cmpq    $dst, [0x8000000000000000]\n\t"
10027             "jne,s   done\n\t"
10028             "subq    rsp, #8\n\t"
10029             "movss   [rsp], $src\n\t"
10030             "call    f2l_fixup\n\t"
10031             "popq    $dst\n"
10032     "done:   "%}
10033   ins_encode %{
10034     Label done;
10035     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10036     __ cmp64($dst$$Register,
10037              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10038     __ jccb(Assembler::notEqual, done);
10039     __ subptr(rsp, 8);
10040     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10041     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10042     __ pop($dst$$Register);
10043     __ bind(done);
10044   %}
10045   ins_pipe(pipe_slow);
10046 %}
10047 
10048 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10049 %{
10050   match(Set dst (ConvD2I src));
10051   effect(KILL cr);
10052 
10053   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10054             "cmpl    $dst, #0x80000000\n\t"
10055             "jne,s   done\n\t"
10056             "subq    rsp, #8\n\t"
10057             "movsd   [rsp], $src\n\t"
10058             "call    d2i_fixup\n\t"
10059             "popq    $dst\n"
10060     "done:   "%}
10061   ins_encode %{
10062     Label done;
10063     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10064     __ cmpl($dst$$Register, 0x80000000);
10065     __ jccb(Assembler::notEqual, done);
10066     __ subptr(rsp, 8);
10067     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10068     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10069     __ pop($dst$$Register);
10070     __ bind(done);
10071   %}
10072   ins_pipe(pipe_slow);
10073 %}
10074 
10075 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10076 %{
10077   match(Set dst (ConvD2L src));
10078   effect(KILL cr);
10079 
10080   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10081             "cmpq    $dst, [0x8000000000000000]\n\t"
10082             "jne,s   done\n\t"
10083             "subq    rsp, #8\n\t"
10084             "movsd   [rsp], $src\n\t"
10085             "call    d2l_fixup\n\t"
10086             "popq    $dst\n"
10087     "done:   "%}
10088   ins_encode %{
10089     Label done;
10090     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10091     __ cmp64($dst$$Register,
10092              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10093     __ jccb(Assembler::notEqual, done);
10094     __ subptr(rsp, 8);
10095     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10096     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10097     __ pop($dst$$Register);
10098     __ bind(done);
10099   %}
10100   ins_pipe(pipe_slow);
10101 %}
10102 
10103 instruct convI2F_reg_reg(regF dst, rRegI src)
10104 %{
10105   predicate(!UseXmmI2F);
10106   match(Set dst (ConvI2F src));
10107 
10108   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10109   ins_encode %{
10110     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10111   %}
10112   ins_pipe(pipe_slow); // XXX
10113 %}
10114 
10115 instruct convI2F_reg_mem(regF dst, memory src)
10116 %{
10117   match(Set dst (ConvI2F (LoadI src)));
10118 
10119   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10120   ins_encode %{
10121     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10122   %}
10123   ins_pipe(pipe_slow); // XXX
10124 %}
10125 
10126 instruct convI2D_reg_reg(regD dst, rRegI src)
10127 %{
10128   predicate(!UseXmmI2D);
10129   match(Set dst (ConvI2D src));
10130 
10131   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10132   ins_encode %{
10133     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10134   %}
10135   ins_pipe(pipe_slow); // XXX
10136 %}
10137 
10138 instruct convI2D_reg_mem(regD dst, memory src)
10139 %{
10140   match(Set dst (ConvI2D (LoadI src)));
10141 
10142   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10143   ins_encode %{
10144     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10145   %}
10146   ins_pipe(pipe_slow); // XXX
10147 %}
10148 
10149 instruct convXI2F_reg(regF dst, rRegI src)
10150 %{
10151   predicate(UseXmmI2F);
10152   match(Set dst (ConvI2F src));
10153 
10154   format %{ "movdl $dst, $src\n\t"
10155             "cvtdq2psl $dst, $dst\t# i2f" %}
10156   ins_encode %{
10157     __ movdl($dst$$XMMRegister, $src$$Register);
10158     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10159   %}
10160   ins_pipe(pipe_slow); // XXX
10161 %}
10162 
10163 instruct convXI2D_reg(regD dst, rRegI src)
10164 %{
10165   predicate(UseXmmI2D);
10166   match(Set dst (ConvI2D src));
10167 
10168   format %{ "movdl $dst, $src\n\t"
10169             "cvtdq2pdl $dst, $dst\t# i2d" %}
10170   ins_encode %{
10171     __ movdl($dst$$XMMRegister, $src$$Register);
10172     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10173   %}
10174   ins_pipe(pipe_slow); // XXX
10175 %}
10176 
10177 instruct convL2F_reg_reg(regF dst, rRegL src)
10178 %{
10179   match(Set dst (ConvL2F src));
10180 
10181   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10182   ins_encode %{
10183     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10184   %}
10185   ins_pipe(pipe_slow); // XXX
10186 %}
10187 
10188 instruct convL2F_reg_mem(regF dst, memory src)
10189 %{
10190   match(Set dst (ConvL2F (LoadL src)));
10191 
10192   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10193   ins_encode %{
10194     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10195   %}
10196   ins_pipe(pipe_slow); // XXX
10197 %}
10198 
10199 instruct convL2D_reg_reg(regD dst, rRegL src)
10200 %{
10201   match(Set dst (ConvL2D src));
10202 
10203   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10204   ins_encode %{
10205     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10206   %}
10207   ins_pipe(pipe_slow); // XXX
10208 %}
10209 
10210 instruct convL2D_reg_mem(regD dst, memory src)
10211 %{
10212   match(Set dst (ConvL2D (LoadL src)));
10213 
10214   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10215   ins_encode %{
10216     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10217   %}
10218   ins_pipe(pipe_slow); // XXX
10219 %}
10220 
10221 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10222 %{
10223   match(Set dst (ConvI2L src));
10224 
10225   ins_cost(125);
10226   format %{ "movslq  $dst, $src\t# i2l" %}
10227   ins_encode %{
10228     __ movslq($dst$$Register, $src$$Register);
10229   %}
10230   ins_pipe(ialu_reg_reg);
10231 %}
10232 
10233 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10234 // %{
10235 //   match(Set dst (ConvI2L src));
10236 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10237 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10238 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10239 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10240 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10241 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10242 
10243 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10244 //   ins_encode(enc_copy(dst, src));
10245 // //   opcode(0x63); // needs REX.W
10246 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10247 //   ins_pipe(ialu_reg_reg);
10248 // %}
10249 
10250 // Zero-extend convert int to long
10251 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10252 %{
10253   match(Set dst (AndL (ConvI2L src) mask));
10254 
10255   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10256   ins_encode %{
10257     if ($dst$$reg != $src$$reg) {
10258       __ movl($dst$$Register, $src$$Register);
10259     }
10260   %}
10261   ins_pipe(ialu_reg_reg);
10262 %}
10263 
10264 // Zero-extend convert int to long
10265 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10266 %{
10267   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10268 
10269   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10270   ins_encode %{
10271     __ movl($dst$$Register, $src$$Address);
10272   %}
10273   ins_pipe(ialu_reg_mem);
10274 %}
10275 
10276 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10277 %{
10278   match(Set dst (AndL src mask));
10279 
10280   format %{ "movl    $dst, $src\t# zero-extend long" %}
10281   ins_encode %{
10282     __ movl($dst$$Register, $src$$Register);
10283   %}
10284   ins_pipe(ialu_reg_reg);
10285 %}
10286 
10287 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10288 %{
10289   match(Set dst (ConvL2I src));
10290 
10291   format %{ "movl    $dst, $src\t# l2i" %}
10292   ins_encode %{
10293     __ movl($dst$$Register, $src$$Register);
10294   %}
10295   ins_pipe(ialu_reg_reg);
10296 %}
10297 
10298 
10299 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10300   match(Set dst (MoveF2I src));
10301   effect(DEF dst, USE src);
10302 
10303   ins_cost(125);
10304   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10305   ins_encode %{
10306     __ movl($dst$$Register, Address(rsp, $src$$disp));
10307   %}
10308   ins_pipe(ialu_reg_mem);
10309 %}
10310 
10311 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10312   match(Set dst (MoveI2F src));
10313   effect(DEF dst, USE src);
10314 
10315   ins_cost(125);
10316   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10317   ins_encode %{
10318     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10319   %}
10320   ins_pipe(pipe_slow);
10321 %}
10322 
10323 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10324   match(Set dst (MoveD2L src));
10325   effect(DEF dst, USE src);
10326 
10327   ins_cost(125);
10328   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10329   ins_encode %{
10330     __ movq($dst$$Register, Address(rsp, $src$$disp));
10331   %}
10332   ins_pipe(ialu_reg_mem);
10333 %}
10334 
10335 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10336   predicate(!UseXmmLoadAndClearUpper);
10337   match(Set dst (MoveL2D src));
10338   effect(DEF dst, USE src);
10339 
10340   ins_cost(125);
10341   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10342   ins_encode %{
10343     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10344   %}
10345   ins_pipe(pipe_slow);
10346 %}
10347 
10348 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10349   predicate(UseXmmLoadAndClearUpper);
10350   match(Set dst (MoveL2D src));
10351   effect(DEF dst, USE src);
10352 
10353   ins_cost(125);
10354   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10355   ins_encode %{
10356     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10357   %}
10358   ins_pipe(pipe_slow);
10359 %}
10360 
10361 
10362 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10363   match(Set dst (MoveF2I src));
10364   effect(DEF dst, USE src);
10365 
10366   ins_cost(95); // XXX
10367   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10368   ins_encode %{
10369     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10370   %}
10371   ins_pipe(pipe_slow);
10372 %}
10373 
10374 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10375   match(Set dst (MoveI2F src));
10376   effect(DEF dst, USE src);
10377 
10378   ins_cost(100);
10379   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10380   ins_encode %{
10381     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10382   %}
10383   ins_pipe( ialu_mem_reg );
10384 %}
10385 
10386 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10387   match(Set dst (MoveD2L src));
10388   effect(DEF dst, USE src);
10389 
10390   ins_cost(95); // XXX
10391   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10392   ins_encode %{
10393     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10394   %}
10395   ins_pipe(pipe_slow);
10396 %}
10397 
10398 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10399   match(Set dst (MoveL2D src));
10400   effect(DEF dst, USE src);
10401 
10402   ins_cost(100);
10403   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10404   ins_encode %{
10405     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10406   %}
10407   ins_pipe(ialu_mem_reg);
10408 %}
10409 
10410 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10411   match(Set dst (MoveF2I src));
10412   effect(DEF dst, USE src);
10413   ins_cost(85);
10414   format %{ "movd    $dst,$src\t# MoveF2I" %}
10415   ins_encode %{
10416     __ movdl($dst$$Register, $src$$XMMRegister);
10417   %}
10418   ins_pipe( pipe_slow );
10419 %}
10420 
10421 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10422   match(Set dst (MoveD2L src));
10423   effect(DEF dst, USE src);
10424   ins_cost(85);
10425   format %{ "movd    $dst,$src\t# MoveD2L" %}
10426   ins_encode %{
10427     __ movdq($dst$$Register, $src$$XMMRegister);
10428   %}
10429   ins_pipe( pipe_slow );
10430 %}
10431 
10432 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10433   match(Set dst (MoveI2F src));
10434   effect(DEF dst, USE src);
10435   ins_cost(100);
10436   format %{ "movd    $dst,$src\t# MoveI2F" %}
10437   ins_encode %{
10438     __ movdl($dst$$XMMRegister, $src$$Register);
10439   %}
10440   ins_pipe( pipe_slow );
10441 %}
10442 
10443 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10444   match(Set dst (MoveL2D src));
10445   effect(DEF dst, USE src);
10446   ins_cost(100);
10447   format %{ "movd    $dst,$src\t# MoveL2D" %}
10448   ins_encode %{
10449      __ movdq($dst$$XMMRegister, $src$$Register);
10450   %}
10451   ins_pipe( pipe_slow );
10452 %}
10453 
10454 
10455 // =======================================================================
10456 // fast clearing of an array
10457 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10458                   rFlagsReg cr)
10459 %{
10460   predicate(!((ClearArrayNode*)n)->is_large());
10461   match(Set dummy (ClearArray cnt base));
10462   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10463 
10464   format %{ $$template
10465     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10466     $$emit$$"cmp     InitArrayShortSize,rcx\n\t"
10467     $$emit$$"jg      LARGE\n\t"
10468     $$emit$$"dec     rcx\n\t"
10469     $$emit$$"js      DONE\t# Zero length\n\t"
10470     $$emit$$"mov     rax,(rdi,rcx,8)\t# LOOP\n\t"
10471     $$emit$$"dec     rcx\n\t"
10472     $$emit$$"jge     LOOP\n\t"
10473     $$emit$$"jmp     DONE\n\t"
10474     $$emit$$"# LARGE:\n\t"
10475     if (UseFastStosb) {
10476        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10477        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--\n\t"
10478     } else {
10479        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--\n\t"
10480     }
10481     $$emit$$"# DONE"
10482   %}
10483   ins_encode %{
10484     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, false);
10485   %}
10486   ins_pipe(pipe_slow);
10487 %}
10488 
10489 instruct rep_stos_large(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10490                   rFlagsReg cr)
10491 %{
10492   predicate(((ClearArrayNode*)n)->is_large());
10493   match(Set dummy (ClearArray cnt base));
10494   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10495 
10496   format %{ $$template
10497     $$emit$$"xorq    rax, rax\t# ClearArray:\n\t"
10498     if (UseFastStosb) {
10499        $$emit$$"shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10500        $$emit$$"rep     stosb\t# Store rax to *rdi++ while rcx--"
10501     } else {
10502        $$emit$$"rep     stosq\t# Store rax to *rdi++ while rcx--"
10503     }
10504   %}
10505   ins_encode %{
10506     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register, true);
10507   %}
10508   ins_pipe(pipe_slow);
10509 %}
10510 
10511 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10512                          rax_RegI result, regD tmp1, rFlagsReg cr)
10513 %{
10514   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10515   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10516   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10517 
10518   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10519   ins_encode %{
10520     __ string_compare($str1$$Register, $str2$$Register,
10521                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10522                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10523   %}
10524   ins_pipe( pipe_slow );
10525 %}
10526 
10527 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10528                          rax_RegI result, regD tmp1, rFlagsReg cr)
10529 %{
10530   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10531   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10532   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10533 
10534   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10535   ins_encode %{
10536     __ string_compare($str1$$Register, $str2$$Register,
10537                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10538                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10539   %}
10540   ins_pipe( pipe_slow );
10541 %}
10542 
10543 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10544                           rax_RegI result, regD tmp1, rFlagsReg cr)
10545 %{
10546   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10547   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10548   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10549 
10550   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10551   ins_encode %{
10552     __ string_compare($str1$$Register, $str2$$Register,
10553                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10554                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10555   %}
10556   ins_pipe( pipe_slow );
10557 %}
10558 
10559 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10560                           rax_RegI result, regD tmp1, rFlagsReg cr)
10561 %{
10562   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10563   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10564   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10565 
10566   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10567   ins_encode %{
10568     __ string_compare($str2$$Register, $str1$$Register,
10569                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10570                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10571   %}
10572   ins_pipe( pipe_slow );
10573 %}
10574 
10575 // fast search of substring with known size.
10576 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10577                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10578 %{
10579   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10580   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10581   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10582 
10583   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10584   ins_encode %{
10585     int icnt2 = (int)$int_cnt2$$constant;
10586     if (icnt2 >= 16) {
10587       // IndexOf for constant substrings with size >= 16 elements
10588       // which don't need to be loaded through stack.
10589       __ string_indexofC8($str1$$Register, $str2$$Register,
10590                           $cnt1$$Register, $cnt2$$Register,
10591                           icnt2, $result$$Register,
10592                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10593     } else {
10594       // Small strings are loaded through stack if they cross page boundary.
10595       __ string_indexof($str1$$Register, $str2$$Register,
10596                         $cnt1$$Register, $cnt2$$Register,
10597                         icnt2, $result$$Register,
10598                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10599     }
10600   %}
10601   ins_pipe( pipe_slow );
10602 %}
10603 
10604 // fast search of substring with known size.
10605 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10606                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10607 %{
10608   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10609   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10610   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10611 
10612   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10613   ins_encode %{
10614     int icnt2 = (int)$int_cnt2$$constant;
10615     if (icnt2 >= 8) {
10616       // IndexOf for constant substrings with size >= 8 elements
10617       // which don't need to be loaded through stack.
10618       __ string_indexofC8($str1$$Register, $str2$$Register,
10619                           $cnt1$$Register, $cnt2$$Register,
10620                           icnt2, $result$$Register,
10621                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10622     } else {
10623       // Small strings are loaded through stack if they cross page boundary.
10624       __ string_indexof($str1$$Register, $str2$$Register,
10625                         $cnt1$$Register, $cnt2$$Register,
10626                         icnt2, $result$$Register,
10627                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10628     }
10629   %}
10630   ins_pipe( pipe_slow );
10631 %}
10632 
10633 // fast search of substring with known size.
10634 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10635                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10636 %{
10637   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10638   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10639   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10640 
10641   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10642   ins_encode %{
10643     int icnt2 = (int)$int_cnt2$$constant;
10644     if (icnt2 >= 8) {
10645       // IndexOf for constant substrings with size >= 8 elements
10646       // which don't need to be loaded through stack.
10647       __ string_indexofC8($str1$$Register, $str2$$Register,
10648                           $cnt1$$Register, $cnt2$$Register,
10649                           icnt2, $result$$Register,
10650                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10651     } else {
10652       // Small strings are loaded through stack if they cross page boundary.
10653       __ string_indexof($str1$$Register, $str2$$Register,
10654                         $cnt1$$Register, $cnt2$$Register,
10655                         icnt2, $result$$Register,
10656                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10657     }
10658   %}
10659   ins_pipe( pipe_slow );
10660 %}
10661 
10662 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10663                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10664 %{
10665   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10666   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10667   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10668 
10669   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10670   ins_encode %{
10671     __ string_indexof($str1$$Register, $str2$$Register,
10672                       $cnt1$$Register, $cnt2$$Register,
10673                       (-1), $result$$Register,
10674                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10675   %}
10676   ins_pipe( pipe_slow );
10677 %}
10678 
10679 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10680                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10681 %{
10682   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10683   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10684   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10685 
10686   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10687   ins_encode %{
10688     __ string_indexof($str1$$Register, $str2$$Register,
10689                       $cnt1$$Register, $cnt2$$Register,
10690                       (-1), $result$$Register,
10691                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10692   %}
10693   ins_pipe( pipe_slow );
10694 %}
10695 
10696 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10697                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10698 %{
10699   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10700   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10701   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10702 
10703   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10704   ins_encode %{
10705     __ string_indexof($str1$$Register, $str2$$Register,
10706                       $cnt1$$Register, $cnt2$$Register,
10707                       (-1), $result$$Register,
10708                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10709   %}
10710   ins_pipe( pipe_slow );
10711 %}
10712 
10713 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10714                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10715 %{
10716   predicate(UseSSE42Intrinsics);
10717   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10718   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10719   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10720   ins_encode %{
10721     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10722                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10723   %}
10724   ins_pipe( pipe_slow );
10725 %}
10726 
10727 // fast string equals
10728 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10729                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10730 %{
10731   match(Set result (StrEquals (Binary str1 str2) cnt));
10732   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10733 
10734   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10735   ins_encode %{
10736     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10737                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10738                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10739   %}
10740   ins_pipe( pipe_slow );
10741 %}
10742 
10743 // fast array equals
10744 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10745                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10746 %{
10747   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10748   match(Set result (AryEq ary1 ary2));
10749   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10750 
10751   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10752   ins_encode %{
10753     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10754                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10755                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10756   %}
10757   ins_pipe( pipe_slow );
10758 %}
10759 
10760 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10761                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10762 %{
10763   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10764   match(Set result (AryEq ary1 ary2));
10765   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10766 
10767   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10768   ins_encode %{
10769     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10770                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10771                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10772   %}
10773   ins_pipe( pipe_slow );
10774 %}
10775 
10776 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10777                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10778 %{
10779   match(Set result (HasNegatives ary1 len));
10780   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10781 
10782   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10783   ins_encode %{
10784     __ has_negatives($ary1$$Register, $len$$Register,
10785                      $result$$Register, $tmp3$$Register,
10786                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10787   %}
10788   ins_pipe( pipe_slow );
10789 %}
10790 
10791 // fast char[] to byte[] compression
10792 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10793                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10794   match(Set result (StrCompressedCopy src (Binary dst len)));
10795   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10796 
10797   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10798   ins_encode %{
10799     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10800                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10801                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10802   %}
10803   ins_pipe( pipe_slow );
10804 %}
10805 
10806 // fast byte[] to char[] inflation
10807 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10808                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10809   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10810   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10811 
10812   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10813   ins_encode %{
10814     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10815                           $tmp1$$XMMRegister, $tmp2$$Register);
10816   %}
10817   ins_pipe( pipe_slow );
10818 %}
10819 
10820 // encode char[] to byte[] in ISO_8859_1
10821 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10822                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10823                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10824   match(Set result (EncodeISOArray src (Binary dst len)));
10825   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10826 
10827   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10828   ins_encode %{
10829     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10830                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10831                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10832   %}
10833   ins_pipe( pipe_slow );
10834 %}
10835 
10836 //----------Overflow Math Instructions-----------------------------------------
10837 
10838 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10839 %{
10840   match(Set cr (OverflowAddI op1 op2));
10841   effect(DEF cr, USE_KILL op1, USE op2);
10842 
10843   format %{ "addl    $op1, $op2\t# overflow check int" %}
10844 
10845   ins_encode %{
10846     __ addl($op1$$Register, $op2$$Register);
10847   %}
10848   ins_pipe(ialu_reg_reg);
10849 %}
10850 
10851 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10852 %{
10853   match(Set cr (OverflowAddI op1 op2));
10854   effect(DEF cr, USE_KILL op1, USE op2);
10855 
10856   format %{ "addl    $op1, $op2\t# overflow check int" %}
10857 
10858   ins_encode %{
10859     __ addl($op1$$Register, $op2$$constant);
10860   %}
10861   ins_pipe(ialu_reg_reg);
10862 %}
10863 
10864 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10865 %{
10866   match(Set cr (OverflowAddL op1 op2));
10867   effect(DEF cr, USE_KILL op1, USE op2);
10868 
10869   format %{ "addq    $op1, $op2\t# overflow check long" %}
10870   ins_encode %{
10871     __ addq($op1$$Register, $op2$$Register);
10872   %}
10873   ins_pipe(ialu_reg_reg);
10874 %}
10875 
10876 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10877 %{
10878   match(Set cr (OverflowAddL op1 op2));
10879   effect(DEF cr, USE_KILL op1, USE op2);
10880 
10881   format %{ "addq    $op1, $op2\t# overflow check long" %}
10882   ins_encode %{
10883     __ addq($op1$$Register, $op2$$constant);
10884   %}
10885   ins_pipe(ialu_reg_reg);
10886 %}
10887 
10888 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10889 %{
10890   match(Set cr (OverflowSubI op1 op2));
10891 
10892   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10893   ins_encode %{
10894     __ cmpl($op1$$Register, $op2$$Register);
10895   %}
10896   ins_pipe(ialu_reg_reg);
10897 %}
10898 
10899 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10900 %{
10901   match(Set cr (OverflowSubI op1 op2));
10902 
10903   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10904   ins_encode %{
10905     __ cmpl($op1$$Register, $op2$$constant);
10906   %}
10907   ins_pipe(ialu_reg_reg);
10908 %}
10909 
10910 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10911 %{
10912   match(Set cr (OverflowSubL op1 op2));
10913 
10914   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10915   ins_encode %{
10916     __ cmpq($op1$$Register, $op2$$Register);
10917   %}
10918   ins_pipe(ialu_reg_reg);
10919 %}
10920 
10921 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10922 %{
10923   match(Set cr (OverflowSubL op1 op2));
10924 
10925   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10926   ins_encode %{
10927     __ cmpq($op1$$Register, $op2$$constant);
10928   %}
10929   ins_pipe(ialu_reg_reg);
10930 %}
10931 
10932 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10933 %{
10934   match(Set cr (OverflowSubI zero op2));
10935   effect(DEF cr, USE_KILL op2);
10936 
10937   format %{ "negl    $op2\t# overflow check int" %}
10938   ins_encode %{
10939     __ negl($op2$$Register);
10940   %}
10941   ins_pipe(ialu_reg_reg);
10942 %}
10943 
10944 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10945 %{
10946   match(Set cr (OverflowSubL zero op2));
10947   effect(DEF cr, USE_KILL op2);
10948 
10949   format %{ "negq    $op2\t# overflow check long" %}
10950   ins_encode %{
10951     __ negq($op2$$Register);
10952   %}
10953   ins_pipe(ialu_reg_reg);
10954 %}
10955 
10956 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10957 %{
10958   match(Set cr (OverflowMulI op1 op2));
10959   effect(DEF cr, USE_KILL op1, USE op2);
10960 
10961   format %{ "imull    $op1, $op2\t# overflow check int" %}
10962   ins_encode %{
10963     __ imull($op1$$Register, $op2$$Register);
10964   %}
10965   ins_pipe(ialu_reg_reg_alu0);
10966 %}
10967 
10968 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10969 %{
10970   match(Set cr (OverflowMulI op1 op2));
10971   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10972 
10973   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10974   ins_encode %{
10975     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10976   %}
10977   ins_pipe(ialu_reg_reg_alu0);
10978 %}
10979 
10980 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10981 %{
10982   match(Set cr (OverflowMulL op1 op2));
10983   effect(DEF cr, USE_KILL op1, USE op2);
10984 
10985   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10986   ins_encode %{
10987     __ imulq($op1$$Register, $op2$$Register);
10988   %}
10989   ins_pipe(ialu_reg_reg_alu0);
10990 %}
10991 
10992 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10993 %{
10994   match(Set cr (OverflowMulL op1 op2));
10995   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10996 
10997   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10998   ins_encode %{
10999     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
11000   %}
11001   ins_pipe(ialu_reg_reg_alu0);
11002 %}
11003 
11004 
11005 //----------Control Flow Instructions------------------------------------------
11006 // Signed compare Instructions
11007 
11008 // XXX more variants!!
11009 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
11010 %{
11011   match(Set cr (CmpI op1 op2));
11012   effect(DEF cr, USE op1, USE op2);
11013 
11014   format %{ "cmpl    $op1, $op2" %}
11015   opcode(0x3B);  /* Opcode 3B /r */
11016   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11017   ins_pipe(ialu_cr_reg_reg);
11018 %}
11019 
11020 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
11021 %{
11022   match(Set cr (CmpI op1 op2));
11023 
11024   format %{ "cmpl    $op1, $op2" %}
11025   opcode(0x81, 0x07); /* Opcode 81 /7 */
11026   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11027   ins_pipe(ialu_cr_reg_imm);
11028 %}
11029 
11030 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
11031 %{
11032   match(Set cr (CmpI op1 (LoadI op2)));
11033 
11034   ins_cost(500); // XXX
11035   format %{ "cmpl    $op1, $op2" %}
11036   opcode(0x3B); /* Opcode 3B /r */
11037   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11038   ins_pipe(ialu_cr_reg_mem);
11039 %}
11040 
11041 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
11042 %{
11043   match(Set cr (CmpI src zero));
11044 
11045   format %{ "testl   $src, $src" %}
11046   opcode(0x85);
11047   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11048   ins_pipe(ialu_cr_reg_imm);
11049 %}
11050 
11051 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11052 %{
11053   match(Set cr (CmpI (AndI src con) zero));
11054 
11055   format %{ "testl   $src, $con" %}
11056   opcode(0xF7, 0x00);
11057   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11058   ins_pipe(ialu_cr_reg_imm);
11059 %}
11060 
11061 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11062 %{
11063   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11064 
11065   format %{ "testl   $src, $mem" %}
11066   opcode(0x85);
11067   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11068   ins_pipe(ialu_cr_reg_mem);
11069 %}
11070 
11071 // Unsigned compare Instructions; really, same as signed except they
11072 // produce an rFlagsRegU instead of rFlagsReg.
11073 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11074 %{
11075   match(Set cr (CmpU op1 op2));
11076 
11077   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11078   opcode(0x3B); /* Opcode 3B /r */
11079   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11080   ins_pipe(ialu_cr_reg_reg);
11081 %}
11082 
11083 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11084 %{
11085   match(Set cr (CmpU op1 op2));
11086 
11087   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11088   opcode(0x81,0x07); /* Opcode 81 /7 */
11089   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11090   ins_pipe(ialu_cr_reg_imm);
11091 %}
11092 
11093 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11094 %{
11095   match(Set cr (CmpU op1 (LoadI op2)));
11096 
11097   ins_cost(500); // XXX
11098   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11099   opcode(0x3B); /* Opcode 3B /r */
11100   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11101   ins_pipe(ialu_cr_reg_mem);
11102 %}
11103 
11104 // // // Cisc-spilled version of cmpU_rReg
11105 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11106 // //%{
11107 // //  match(Set cr (CmpU (LoadI op1) op2));
11108 // //
11109 // //  format %{ "CMPu   $op1,$op2" %}
11110 // //  ins_cost(500);
11111 // //  opcode(0x39);  /* Opcode 39 /r */
11112 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11113 // //%}
11114 
11115 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11116 %{
11117   match(Set cr (CmpU src zero));
11118 
11119   format %{ "testl  $src, $src\t# unsigned" %}
11120   opcode(0x85);
11121   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11122   ins_pipe(ialu_cr_reg_imm);
11123 %}
11124 
11125 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11126 %{
11127   match(Set cr (CmpP op1 op2));
11128 
11129   format %{ "cmpq    $op1, $op2\t# ptr" %}
11130   opcode(0x3B); /* Opcode 3B /r */
11131   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11132   ins_pipe(ialu_cr_reg_reg);
11133 %}
11134 
11135 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11136 %{
11137   match(Set cr (CmpP op1 (LoadP op2)));
11138 
11139   ins_cost(500); // XXX
11140   format %{ "cmpq    $op1, $op2\t# ptr" %}
11141   opcode(0x3B); /* Opcode 3B /r */
11142   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11143   ins_pipe(ialu_cr_reg_mem);
11144 %}
11145 
11146 // // // Cisc-spilled version of cmpP_rReg
11147 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11148 // //%{
11149 // //  match(Set cr (CmpP (LoadP op1) op2));
11150 // //
11151 // //  format %{ "CMPu   $op1,$op2" %}
11152 // //  ins_cost(500);
11153 // //  opcode(0x39);  /* Opcode 39 /r */
11154 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11155 // //%}
11156 
11157 // XXX this is generalized by compP_rReg_mem???
11158 // Compare raw pointer (used in out-of-heap check).
11159 // Only works because non-oop pointers must be raw pointers
11160 // and raw pointers have no anti-dependencies.
11161 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11162 %{
11163   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11164   match(Set cr (CmpP op1 (LoadP op2)));
11165 
11166   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11167   opcode(0x3B); /* Opcode 3B /r */
11168   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11169   ins_pipe(ialu_cr_reg_mem);
11170 %}
11171 
11172 // This will generate a signed flags result. This should be OK since
11173 // any compare to a zero should be eq/neq.
11174 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11175 %{
11176   match(Set cr (CmpP src zero));
11177 
11178   format %{ "testq   $src, $src\t# ptr" %}
11179   opcode(0x85);
11180   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11181   ins_pipe(ialu_cr_reg_imm);
11182 %}
11183 
11184 // This will generate a signed flags result. This should be OK since
11185 // any compare to a zero should be eq/neq.
11186 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11187 %{
11188   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11189   match(Set cr (CmpP (LoadP op) zero));
11190 
11191   ins_cost(500); // XXX
11192   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11193   opcode(0xF7); /* Opcode F7 /0 */
11194   ins_encode(REX_mem_wide(op),
11195              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11196   ins_pipe(ialu_cr_reg_imm);
11197 %}
11198 
11199 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11200 %{
11201   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11202   match(Set cr (CmpP (LoadP mem) zero));
11203 
11204   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11205   ins_encode %{
11206     __ cmpq(r12, $mem$$Address);
11207   %}
11208   ins_pipe(ialu_cr_reg_mem);
11209 %}
11210 
11211 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11212 %{
11213   match(Set cr (CmpN op1 op2));
11214 
11215   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11216   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11217   ins_pipe(ialu_cr_reg_reg);
11218 %}
11219 
11220 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11221 %{
11222   match(Set cr (CmpN src (LoadN mem)));
11223 
11224   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11225   ins_encode %{
11226     __ cmpl($src$$Register, $mem$$Address);
11227   %}
11228   ins_pipe(ialu_cr_reg_mem);
11229 %}
11230 
11231 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11232   match(Set cr (CmpN op1 op2));
11233 
11234   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11235   ins_encode %{
11236     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11237   %}
11238   ins_pipe(ialu_cr_reg_imm);
11239 %}
11240 
11241 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11242 %{
11243   match(Set cr (CmpN src (LoadN mem)));
11244 
11245   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11246   ins_encode %{
11247     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11248   %}
11249   ins_pipe(ialu_cr_reg_mem);
11250 %}
11251 
11252 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11253   match(Set cr (CmpN op1 op2));
11254 
11255   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11256   ins_encode %{
11257     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11258   %}
11259   ins_pipe(ialu_cr_reg_imm);
11260 %}
11261 
11262 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11263 %{
11264   match(Set cr (CmpN src (LoadNKlass mem)));
11265 
11266   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11267   ins_encode %{
11268     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11269   %}
11270   ins_pipe(ialu_cr_reg_mem);
11271 %}
11272 
11273 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11274   match(Set cr (CmpN src zero));
11275 
11276   format %{ "testl   $src, $src\t# compressed ptr" %}
11277   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11278   ins_pipe(ialu_cr_reg_imm);
11279 %}
11280 
11281 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11282 %{
11283   predicate(Universe::narrow_oop_base() != NULL);
11284   match(Set cr (CmpN (LoadN mem) zero));
11285 
11286   ins_cost(500); // XXX
11287   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11288   ins_encode %{
11289     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11290   %}
11291   ins_pipe(ialu_cr_reg_mem);
11292 %}
11293 
11294 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11295 %{
11296   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11297   match(Set cr (CmpN (LoadN mem) zero));
11298 
11299   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11300   ins_encode %{
11301     __ cmpl(r12, $mem$$Address);
11302   %}
11303   ins_pipe(ialu_cr_reg_mem);
11304 %}
11305 
11306 // Yanked all unsigned pointer compare operations.
11307 // Pointer compares are done with CmpP which is already unsigned.
11308 
11309 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11310 %{
11311   match(Set cr (CmpL op1 op2));
11312 
11313   format %{ "cmpq    $op1, $op2" %}
11314   opcode(0x3B);  /* Opcode 3B /r */
11315   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11316   ins_pipe(ialu_cr_reg_reg);
11317 %}
11318 
11319 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11320 %{
11321   match(Set cr (CmpL op1 op2));
11322 
11323   format %{ "cmpq    $op1, $op2" %}
11324   opcode(0x81, 0x07); /* Opcode 81 /7 */
11325   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11326   ins_pipe(ialu_cr_reg_imm);
11327 %}
11328 
11329 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11330 %{
11331   match(Set cr (CmpL op1 (LoadL op2)));
11332 
11333   format %{ "cmpq    $op1, $op2" %}
11334   opcode(0x3B); /* Opcode 3B /r */
11335   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11336   ins_pipe(ialu_cr_reg_mem);
11337 %}
11338 
11339 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11340 %{
11341   match(Set cr (CmpL src zero));
11342 
11343   format %{ "testq   $src, $src" %}
11344   opcode(0x85);
11345   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11346   ins_pipe(ialu_cr_reg_imm);
11347 %}
11348 
11349 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11350 %{
11351   match(Set cr (CmpL (AndL src con) zero));
11352 
11353   format %{ "testq   $src, $con\t# long" %}
11354   opcode(0xF7, 0x00);
11355   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11356   ins_pipe(ialu_cr_reg_imm);
11357 %}
11358 
11359 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11360 %{
11361   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11362 
11363   format %{ "testq   $src, $mem" %}
11364   opcode(0x85);
11365   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11366   ins_pipe(ialu_cr_reg_mem);
11367 %}
11368 
11369 // Manifest a CmpL result in an integer register.  Very painful.
11370 // This is the test to avoid.
11371 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11372 %{
11373   match(Set dst (CmpL3 src1 src2));
11374   effect(KILL flags);
11375 
11376   ins_cost(275); // XXX
11377   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11378             "movl    $dst, -1\n\t"
11379             "jl,s    done\n\t"
11380             "setne   $dst\n\t"
11381             "movzbl  $dst, $dst\n\t"
11382     "done:" %}
11383   ins_encode(cmpl3_flag(src1, src2, dst));
11384   ins_pipe(pipe_slow);
11385 %}
11386 
11387 //----------Max and Min--------------------------------------------------------
11388 // Min Instructions
11389 
11390 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11391 %{
11392   effect(USE_DEF dst, USE src, USE cr);
11393 
11394   format %{ "cmovlgt $dst, $src\t# min" %}
11395   opcode(0x0F, 0x4F);
11396   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11397   ins_pipe(pipe_cmov_reg);
11398 %}
11399 
11400 
11401 instruct minI_rReg(rRegI dst, rRegI src)
11402 %{
11403   match(Set dst (MinI dst src));
11404 
11405   ins_cost(200);
11406   expand %{
11407     rFlagsReg cr;
11408     compI_rReg(cr, dst, src);
11409     cmovI_reg_g(dst, src, cr);
11410   %}
11411 %}
11412 
11413 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11414 %{
11415   effect(USE_DEF dst, USE src, USE cr);
11416 
11417   format %{ "cmovllt $dst, $src\t# max" %}
11418   opcode(0x0F, 0x4C);
11419   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11420   ins_pipe(pipe_cmov_reg);
11421 %}
11422 
11423 
11424 instruct maxI_rReg(rRegI dst, rRegI src)
11425 %{
11426   match(Set dst (MaxI dst src));
11427 
11428   ins_cost(200);
11429   expand %{
11430     rFlagsReg cr;
11431     compI_rReg(cr, dst, src);
11432     cmovI_reg_l(dst, src, cr);
11433   %}
11434 %}
11435 
11436 // ============================================================================
11437 // Branch Instructions
11438 
11439 // Jump Direct - Label defines a relative address from JMP+1
11440 instruct jmpDir(label labl)
11441 %{
11442   match(Goto);
11443   effect(USE labl);
11444 
11445   ins_cost(300);
11446   format %{ "jmp     $labl" %}
11447   size(5);
11448   ins_encode %{
11449     Label* L = $labl$$label;
11450     __ jmp(*L, false); // Always long jump
11451   %}
11452   ins_pipe(pipe_jmp);
11453 %}
11454 
11455 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11456 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11457 %{
11458   match(If cop cr);
11459   effect(USE labl);
11460 
11461   ins_cost(300);
11462   format %{ "j$cop     $labl" %}
11463   size(6);
11464   ins_encode %{
11465     Label* L = $labl$$label;
11466     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11467   %}
11468   ins_pipe(pipe_jcc);
11469 %}
11470 
11471 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11472 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11473 %{
11474   match(CountedLoopEnd cop cr);
11475   effect(USE labl);
11476 
11477   ins_cost(300);
11478   format %{ "j$cop     $labl\t# loop end" %}
11479   size(6);
11480   ins_encode %{
11481     Label* L = $labl$$label;
11482     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11483   %}
11484   ins_pipe(pipe_jcc);
11485 %}
11486 
11487 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11488 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11489   match(CountedLoopEnd cop cmp);
11490   effect(USE labl);
11491 
11492   ins_cost(300);
11493   format %{ "j$cop,u   $labl\t# loop end" %}
11494   size(6);
11495   ins_encode %{
11496     Label* L = $labl$$label;
11497     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11498   %}
11499   ins_pipe(pipe_jcc);
11500 %}
11501 
11502 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11503   match(CountedLoopEnd cop cmp);
11504   effect(USE labl);
11505 
11506   ins_cost(200);
11507   format %{ "j$cop,u   $labl\t# loop end" %}
11508   size(6);
11509   ins_encode %{
11510     Label* L = $labl$$label;
11511     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11512   %}
11513   ins_pipe(pipe_jcc);
11514 %}
11515 
11516 // Jump Direct Conditional - using unsigned comparison
11517 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11518   match(If cop cmp);
11519   effect(USE labl);
11520 
11521   ins_cost(300);
11522   format %{ "j$cop,u  $labl" %}
11523   size(6);
11524   ins_encode %{
11525     Label* L = $labl$$label;
11526     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11527   %}
11528   ins_pipe(pipe_jcc);
11529 %}
11530 
11531 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11532   match(If cop cmp);
11533   effect(USE labl);
11534 
11535   ins_cost(200);
11536   format %{ "j$cop,u  $labl" %}
11537   size(6);
11538   ins_encode %{
11539     Label* L = $labl$$label;
11540     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11541   %}
11542   ins_pipe(pipe_jcc);
11543 %}
11544 
11545 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11546   match(If cop cmp);
11547   effect(USE labl);
11548 
11549   ins_cost(200);
11550   format %{ $$template
11551     if ($cop$$cmpcode == Assembler::notEqual) {
11552       $$emit$$"jp,u   $labl\n\t"
11553       $$emit$$"j$cop,u   $labl"
11554     } else {
11555       $$emit$$"jp,u   done\n\t"
11556       $$emit$$"j$cop,u   $labl\n\t"
11557       $$emit$$"done:"
11558     }
11559   %}
11560   ins_encode %{
11561     Label* l = $labl$$label;
11562     if ($cop$$cmpcode == Assembler::notEqual) {
11563       __ jcc(Assembler::parity, *l, false);
11564       __ jcc(Assembler::notEqual, *l, false);
11565     } else if ($cop$$cmpcode == Assembler::equal) {
11566       Label done;
11567       __ jccb(Assembler::parity, done);
11568       __ jcc(Assembler::equal, *l, false);
11569       __ bind(done);
11570     } else {
11571        ShouldNotReachHere();
11572     }
11573   %}
11574   ins_pipe(pipe_jcc);
11575 %}
11576 
11577 // ============================================================================
11578 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11579 // superklass array for an instance of the superklass.  Set a hidden
11580 // internal cache on a hit (cache is checked with exposed code in
11581 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11582 // encoding ALSO sets flags.
11583 
11584 instruct partialSubtypeCheck(rdi_RegP result,
11585                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11586                              rFlagsReg cr)
11587 %{
11588   match(Set result (PartialSubtypeCheck sub super));
11589   effect(KILL rcx, KILL cr);
11590 
11591   ins_cost(1100);  // slightly larger than the next version
11592   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11593             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11594             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11595             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11596             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11597             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11598             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11599     "miss:\t" %}
11600 
11601   opcode(0x1); // Force a XOR of RDI
11602   ins_encode(enc_PartialSubtypeCheck());
11603   ins_pipe(pipe_slow);
11604 %}
11605 
11606 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11607                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11608                                      immP0 zero,
11609                                      rdi_RegP result)
11610 %{
11611   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11612   effect(KILL rcx, KILL result);
11613 
11614   ins_cost(1000);
11615   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11616             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11617             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11618             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11619             "jne,s   miss\t\t# Missed: flags nz\n\t"
11620             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11621     "miss:\t" %}
11622 
11623   opcode(0x0); // No need to XOR RDI
11624   ins_encode(enc_PartialSubtypeCheck());
11625   ins_pipe(pipe_slow);
11626 %}
11627 
11628 // ============================================================================
11629 // Branch Instructions -- short offset versions
11630 //
11631 // These instructions are used to replace jumps of a long offset (the default
11632 // match) with jumps of a shorter offset.  These instructions are all tagged
11633 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11634 // match rules in general matching.  Instead, the ADLC generates a conversion
11635 // method in the MachNode which can be used to do in-place replacement of the
11636 // long variant with the shorter variant.  The compiler will determine if a
11637 // branch can be taken by the is_short_branch_offset() predicate in the machine
11638 // specific code section of the file.
11639 
11640 // Jump Direct - Label defines a relative address from JMP+1
11641 instruct jmpDir_short(label labl) %{
11642   match(Goto);
11643   effect(USE labl);
11644 
11645   ins_cost(300);
11646   format %{ "jmp,s   $labl" %}
11647   size(2);
11648   ins_encode %{
11649     Label* L = $labl$$label;
11650     __ jmpb(*L);
11651   %}
11652   ins_pipe(pipe_jmp);
11653   ins_short_branch(1);
11654 %}
11655 
11656 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11657 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11658   match(If cop cr);
11659   effect(USE labl);
11660 
11661   ins_cost(300);
11662   format %{ "j$cop,s   $labl" %}
11663   size(2);
11664   ins_encode %{
11665     Label* L = $labl$$label;
11666     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11667   %}
11668   ins_pipe(pipe_jcc);
11669   ins_short_branch(1);
11670 %}
11671 
11672 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11673 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11674   match(CountedLoopEnd cop cr);
11675   effect(USE labl);
11676 
11677   ins_cost(300);
11678   format %{ "j$cop,s   $labl\t# loop end" %}
11679   size(2);
11680   ins_encode %{
11681     Label* L = $labl$$label;
11682     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11683   %}
11684   ins_pipe(pipe_jcc);
11685   ins_short_branch(1);
11686 %}
11687 
11688 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11689 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11690   match(CountedLoopEnd cop cmp);
11691   effect(USE labl);
11692 
11693   ins_cost(300);
11694   format %{ "j$cop,us  $labl\t# loop end" %}
11695   size(2);
11696   ins_encode %{
11697     Label* L = $labl$$label;
11698     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11699   %}
11700   ins_pipe(pipe_jcc);
11701   ins_short_branch(1);
11702 %}
11703 
11704 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11705   match(CountedLoopEnd cop cmp);
11706   effect(USE labl);
11707 
11708   ins_cost(300);
11709   format %{ "j$cop,us  $labl\t# loop end" %}
11710   size(2);
11711   ins_encode %{
11712     Label* L = $labl$$label;
11713     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11714   %}
11715   ins_pipe(pipe_jcc);
11716   ins_short_branch(1);
11717 %}
11718 
11719 // Jump Direct Conditional - using unsigned comparison
11720 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11721   match(If cop cmp);
11722   effect(USE labl);
11723 
11724   ins_cost(300);
11725   format %{ "j$cop,us  $labl" %}
11726   size(2);
11727   ins_encode %{
11728     Label* L = $labl$$label;
11729     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11730   %}
11731   ins_pipe(pipe_jcc);
11732   ins_short_branch(1);
11733 %}
11734 
11735 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11736   match(If cop cmp);
11737   effect(USE labl);
11738 
11739   ins_cost(300);
11740   format %{ "j$cop,us  $labl" %}
11741   size(2);
11742   ins_encode %{
11743     Label* L = $labl$$label;
11744     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11745   %}
11746   ins_pipe(pipe_jcc);
11747   ins_short_branch(1);
11748 %}
11749 
11750 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11751   match(If cop cmp);
11752   effect(USE labl);
11753 
11754   ins_cost(300);
11755   format %{ $$template
11756     if ($cop$$cmpcode == Assembler::notEqual) {
11757       $$emit$$"jp,u,s   $labl\n\t"
11758       $$emit$$"j$cop,u,s   $labl"
11759     } else {
11760       $$emit$$"jp,u,s   done\n\t"
11761       $$emit$$"j$cop,u,s  $labl\n\t"
11762       $$emit$$"done:"
11763     }
11764   %}
11765   size(4);
11766   ins_encode %{
11767     Label* l = $labl$$label;
11768     if ($cop$$cmpcode == Assembler::notEqual) {
11769       __ jccb(Assembler::parity, *l);
11770       __ jccb(Assembler::notEqual, *l);
11771     } else if ($cop$$cmpcode == Assembler::equal) {
11772       Label done;
11773       __ jccb(Assembler::parity, done);
11774       __ jccb(Assembler::equal, *l);
11775       __ bind(done);
11776     } else {
11777        ShouldNotReachHere();
11778     }
11779   %}
11780   ins_pipe(pipe_jcc);
11781   ins_short_branch(1);
11782 %}
11783 
11784 // ============================================================================
11785 // inlined locking and unlocking
11786 
11787 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11788   predicate(Compile::current()->use_rtm());
11789   match(Set cr (FastLock object box));
11790   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11791   ins_cost(300);
11792   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11793   ins_encode %{
11794     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11795                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11796                  _counters, _rtm_counters, _stack_rtm_counters,
11797                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11798                  true, ra_->C->profile_rtm());
11799   %}
11800   ins_pipe(pipe_slow);
11801 %}
11802 
11803 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11804   predicate(!Compile::current()->use_rtm());
11805   match(Set cr (FastLock object box));
11806   effect(TEMP tmp, TEMP scr, USE_KILL box);
11807   ins_cost(300);
11808   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11809   ins_encode %{
11810     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11811                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11812   %}
11813   ins_pipe(pipe_slow);
11814 %}
11815 
11816 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11817   match(Set cr (FastUnlock object box));
11818   effect(TEMP tmp, USE_KILL box);
11819   ins_cost(300);
11820   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11821   ins_encode %{
11822     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11823   %}
11824   ins_pipe(pipe_slow);
11825 %}
11826 
11827 
11828 // ============================================================================
11829 // Safepoint Instructions
11830 instruct safePoint_poll(rFlagsReg cr)
11831 %{
11832   predicate(!Assembler::is_polling_page_far());
11833   match(SafePoint);
11834   effect(KILL cr);
11835 
11836   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11837             "# Safepoint: poll for GC" %}
11838   ins_cost(125);
11839   ins_encode %{
11840     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11841     __ testl(rax, addr);
11842   %}
11843   ins_pipe(ialu_reg_mem);
11844 %}
11845 
11846 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11847 %{
11848   predicate(Assembler::is_polling_page_far());
11849   match(SafePoint poll);
11850   effect(KILL cr, USE poll);
11851 
11852   format %{ "testl  rax, [$poll]\t"
11853             "# Safepoint: poll for GC" %}
11854   ins_cost(125);
11855   ins_encode %{
11856     __ relocate(relocInfo::poll_type);
11857     __ testl(rax, Address($poll$$Register, 0));
11858   %}
11859   ins_pipe(ialu_reg_mem);
11860 %}
11861 
11862 // ============================================================================
11863 // Procedure Call/Return Instructions
11864 // Call Java Static Instruction
11865 // Note: If this code changes, the corresponding ret_addr_offset() and
11866 //       compute_padding() functions will have to be adjusted.
11867 instruct CallStaticJavaDirect(method meth) %{
11868   match(CallStaticJava);
11869   effect(USE meth);
11870 
11871   ins_cost(300);
11872   format %{ "call,static " %}
11873   opcode(0xE8); /* E8 cd */
11874   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11875   ins_pipe(pipe_slow);
11876   ins_alignment(4);
11877 %}
11878 
11879 // Call Java Dynamic Instruction
11880 // Note: If this code changes, the corresponding ret_addr_offset() and
11881 //       compute_padding() functions will have to be adjusted.
11882 instruct CallDynamicJavaDirect(method meth)
11883 %{
11884   match(CallDynamicJava);
11885   effect(USE meth);
11886 
11887   ins_cost(300);
11888   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11889             "call,dynamic " %}
11890   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11891   ins_pipe(pipe_slow);
11892   ins_alignment(4);
11893 %}
11894 
11895 // Call Runtime Instruction
11896 instruct CallRuntimeDirect(method meth)
11897 %{
11898   match(CallRuntime);
11899   effect(USE meth);
11900 
11901   ins_cost(300);
11902   format %{ "call,runtime " %}
11903   ins_encode(clear_avx, Java_To_Runtime(meth));
11904   ins_pipe(pipe_slow);
11905 %}
11906 
11907 // Call runtime without safepoint
11908 instruct CallLeafDirect(method meth)
11909 %{
11910   match(CallLeaf);
11911   effect(USE meth);
11912 
11913   ins_cost(300);
11914   format %{ "call_leaf,runtime " %}
11915   ins_encode(clear_avx, Java_To_Runtime(meth));
11916   ins_pipe(pipe_slow);
11917 %}
11918 
11919 // Call runtime without safepoint
11920 instruct CallLeafNoFPDirect(method meth)
11921 %{
11922   match(CallLeafNoFP);
11923   effect(USE meth);
11924 
11925   ins_cost(300);
11926   format %{ "call_leaf_nofp,runtime " %}
11927   ins_encode(Java_To_Runtime(meth));
11928   ins_pipe(pipe_slow);
11929 %}
11930 
11931 // Return Instruction
11932 // Remove the return address & jump to it.
11933 // Notice: We always emit a nop after a ret to make sure there is room
11934 // for safepoint patching
11935 instruct Ret()
11936 %{
11937   match(Return);
11938 
11939   format %{ "ret" %}
11940   opcode(0xC3);
11941   ins_encode(OpcP);
11942   ins_pipe(pipe_jmp);
11943 %}
11944 
11945 // Tail Call; Jump from runtime stub to Java code.
11946 // Also known as an 'interprocedural jump'.
11947 // Target of jump will eventually return to caller.
11948 // TailJump below removes the return address.
11949 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11950 %{
11951   match(TailCall jump_target method_oop);
11952 
11953   ins_cost(300);
11954   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11955   opcode(0xFF, 0x4); /* Opcode FF /4 */
11956   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11957   ins_pipe(pipe_jmp);
11958 %}
11959 
11960 // Tail Jump; remove the return address; jump to target.
11961 // TailCall above leaves the return address around.
11962 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11963 %{
11964   match(TailJump jump_target ex_oop);
11965 
11966   ins_cost(300);
11967   format %{ "popq    rdx\t# pop return address\n\t"
11968             "jmp     $jump_target" %}
11969   opcode(0xFF, 0x4); /* Opcode FF /4 */
11970   ins_encode(Opcode(0x5a), // popq rdx
11971              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11972   ins_pipe(pipe_jmp);
11973 %}
11974 
11975 // Create exception oop: created by stack-crawling runtime code.
11976 // Created exception is now available to this handler, and is setup
11977 // just prior to jumping to this handler.  No code emitted.
11978 instruct CreateException(rax_RegP ex_oop)
11979 %{
11980   match(Set ex_oop (CreateEx));
11981 
11982   size(0);
11983   // use the following format syntax
11984   format %{ "# exception oop is in rax; no code emitted" %}
11985   ins_encode();
11986   ins_pipe(empty);
11987 %}
11988 
11989 // Rethrow exception:
11990 // The exception oop will come in the first argument position.
11991 // Then JUMP (not call) to the rethrow stub code.
11992 instruct RethrowException()
11993 %{
11994   match(Rethrow);
11995 
11996   // use the following format syntax
11997   format %{ "jmp     rethrow_stub" %}
11998   ins_encode(enc_rethrow);
11999   ins_pipe(pipe_jmp);
12000 %}
12001 
12002 
12003 // ============================================================================
12004 // This name is KNOWN by the ADLC and cannot be changed.
12005 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
12006 // for this guy.
12007 instruct tlsLoadP(r15_RegP dst) %{
12008   match(Set dst (ThreadLocal));
12009   effect(DEF dst);
12010 
12011   size(0);
12012   format %{ "# TLS is in R15" %}
12013   ins_encode( /*empty encoding*/ );
12014   ins_pipe(ialu_reg_reg);
12015 %}
12016 
12017 
12018 //----------PEEPHOLE RULES-----------------------------------------------------
12019 // These must follow all instruction definitions as they use the names
12020 // defined in the instructions definitions.
12021 //
12022 // peepmatch ( root_instr_name [preceding_instruction]* );
12023 //
12024 // peepconstraint %{
12025 // (instruction_number.operand_name relational_op instruction_number.operand_name
12026 //  [, ...] );
12027 // // instruction numbers are zero-based using left to right order in peepmatch
12028 //
12029 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
12030 // // provide an instruction_number.operand_name for each operand that appears
12031 // // in the replacement instruction's match rule
12032 //
12033 // ---------VM FLAGS---------------------------------------------------------
12034 //
12035 // All peephole optimizations can be turned off using -XX:-OptoPeephole
12036 //
12037 // Each peephole rule is given an identifying number starting with zero and
12038 // increasing by one in the order seen by the parser.  An individual peephole
12039 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
12040 // on the command-line.
12041 //
12042 // ---------CURRENT LIMITATIONS----------------------------------------------
12043 //
12044 // Only match adjacent instructions in same basic block
12045 // Only equality constraints
12046 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12047 // Only one replacement instruction
12048 //
12049 // ---------EXAMPLE----------------------------------------------------------
12050 //
12051 // // pertinent parts of existing instructions in architecture description
12052 // instruct movI(rRegI dst, rRegI src)
12053 // %{
12054 //   match(Set dst (CopyI src));
12055 // %}
12056 //
12057 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12058 // %{
12059 //   match(Set dst (AddI dst src));
12060 //   effect(KILL cr);
12061 // %}
12062 //
12063 // // Change (inc mov) to lea
12064 // peephole %{
12065 //   // increment preceeded by register-register move
12066 //   peepmatch ( incI_rReg movI );
12067 //   // require that the destination register of the increment
12068 //   // match the destination register of the move
12069 //   peepconstraint ( 0.dst == 1.dst );
12070 //   // construct a replacement instruction that sets
12071 //   // the destination to ( move's source register + one )
12072 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12073 // %}
12074 //
12075 
12076 // Implementation no longer uses movX instructions since
12077 // machine-independent system no longer uses CopyX nodes.
12078 //
12079 // peephole
12080 // %{
12081 //   peepmatch (incI_rReg movI);
12082 //   peepconstraint (0.dst == 1.dst);
12083 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12084 // %}
12085 
12086 // peephole
12087 // %{
12088 //   peepmatch (decI_rReg movI);
12089 //   peepconstraint (0.dst == 1.dst);
12090 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12091 // %}
12092 
12093 // peephole
12094 // %{
12095 //   peepmatch (addI_rReg_imm movI);
12096 //   peepconstraint (0.dst == 1.dst);
12097 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12098 // %}
12099 
12100 // peephole
12101 // %{
12102 //   peepmatch (incL_rReg movL);
12103 //   peepconstraint (0.dst == 1.dst);
12104 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12105 // %}
12106 
12107 // peephole
12108 // %{
12109 //   peepmatch (decL_rReg movL);
12110 //   peepconstraint (0.dst == 1.dst);
12111 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12112 // %}
12113 
12114 // peephole
12115 // %{
12116 //   peepmatch (addL_rReg_imm movL);
12117 //   peepconstraint (0.dst == 1.dst);
12118 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12119 // %}
12120 
12121 // peephole
12122 // %{
12123 //   peepmatch (addP_rReg_imm movP);
12124 //   peepconstraint (0.dst == 1.dst);
12125 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12126 // %}
12127 
12128 // // Change load of spilled value to only a spill
12129 // instruct storeI(memory mem, rRegI src)
12130 // %{
12131 //   match(Set mem (StoreI mem src));
12132 // %}
12133 //
12134 // instruct loadI(rRegI dst, memory mem)
12135 // %{
12136 //   match(Set dst (LoadI mem));
12137 // %}
12138 //
12139 
12140 peephole
12141 %{
12142   peepmatch (loadI storeI);
12143   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12144   peepreplace (storeI(1.mem 1.mem 1.src));
12145 %}
12146 
12147 peephole
12148 %{
12149   peepmatch (loadL storeL);
12150   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12151   peepreplace (storeL(1.mem 1.mem 1.src));
12152 %}
12153 
12154 //----------SMARTSPILL RULES---------------------------------------------------
12155 // These must follow all instruction definitions as they use the names
12156 // defined in the instructions definitions.