1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 540 }
 541 
 542 // !!!!! Special hack to get all types of calls to specify the byte offset
 543 //       from the start of the call to the point where the return address
 544 //       will point.
 545 int MachCallStaticJavaNode::ret_addr_offset()
 546 {
 547   int offset = 5; // 5 bytes from start of call to where return address points
 548   offset += clear_avx_size();
 549   return offset;
 550 }
 551 
 552 int MachCallDynamicJavaNode::ret_addr_offset()
 553 {
 554   int offset = 15; // 15 bytes from start of call to where return address points
 555   offset += clear_avx_size();
 556   return offset;
 557 }
 558 
 559 int MachCallRuntimeNode::ret_addr_offset() {
 560   int offset = 13; // movq r10,#addr; callq (r10)
 561   offset += clear_avx_size();
 562   return offset;
 563 }
 564 
 565 // Indicate if the safepoint node needs the polling page as an input,
 566 // it does if the polling page is more than disp32 away.
 567 bool SafePointNode::needs_polling_address_input()
 568 {
 569   return Assembler::is_polling_page_far();
 570 }
 571 
 572 //
 573 // Compute padding required for nodes which need alignment
 574 //
 575 
 576 // The address of the call instruction needs to be 4-byte aligned to
 577 // ensure that it does not span a cache line so that it can be patched.
 578 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 579 {
 580   current_offset += clear_avx_size(); // skip vzeroupper
 581   current_offset += 1; // skip call opcode byte
 582   return round_to(current_offset, alignment_required()) - current_offset;
 583 }
 584 
 585 // The address of the call instruction needs to be 4-byte aligned to
 586 // ensure that it does not span a cache line so that it can be patched.
 587 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 588 {
 589   current_offset += clear_avx_size(); // skip vzeroupper
 590   current_offset += 11; // skip movq instruction + call opcode byte
 591   return round_to(current_offset, alignment_required()) - current_offset;
 592 }
 593 
 594 // EMIT_RM()
 595 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 596   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 597   cbuf.insts()->emit_int8(c);
 598 }
 599 
 600 // EMIT_CC()
 601 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 602   unsigned char c = (unsigned char) (f1 | f2);
 603   cbuf.insts()->emit_int8(c);
 604 }
 605 
 606 // EMIT_OPCODE()
 607 void emit_opcode(CodeBuffer &cbuf, int code) {
 608   cbuf.insts()->emit_int8((unsigned char) code);
 609 }
 610 
 611 // EMIT_OPCODE() w/ relocation information
 612 void emit_opcode(CodeBuffer &cbuf,
 613                  int code, relocInfo::relocType reloc, int offset, int format)
 614 {
 615   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 616   emit_opcode(cbuf, code);
 617 }
 618 
 619 // EMIT_D8()
 620 void emit_d8(CodeBuffer &cbuf, int d8) {
 621   cbuf.insts()->emit_int8((unsigned char) d8);
 622 }
 623 
 624 // EMIT_D16()
 625 void emit_d16(CodeBuffer &cbuf, int d16) {
 626   cbuf.insts()->emit_int16(d16);
 627 }
 628 
 629 // EMIT_D32()
 630 void emit_d32(CodeBuffer &cbuf, int d32) {
 631   cbuf.insts()->emit_int32(d32);
 632 }
 633 
 634 // EMIT_D64()
 635 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 636   cbuf.insts()->emit_int64(d64);
 637 }
 638 
 639 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 640 void emit_d32_reloc(CodeBuffer& cbuf,
 641                     int d32,
 642                     relocInfo::relocType reloc,
 643                     int format)
 644 {
 645   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 646   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 647   cbuf.insts()->emit_int32(d32);
 648 }
 649 
 650 // emit 32 bit value and construct relocation entry from RelocationHolder
 651 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 652 #ifdef ASSERT
 653   if (rspec.reloc()->type() == relocInfo::oop_type &&
 654       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 655     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 656     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 657   }
 658 #endif
 659   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 660   cbuf.insts()->emit_int32(d32);
 661 }
 662 
 663 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 664   address next_ip = cbuf.insts_end() + 4;
 665   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 666                  external_word_Relocation::spec(addr),
 667                  RELOC_DISP32);
 668 }
 669 
 670 
 671 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 672 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 673   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 674   cbuf.insts()->emit_int64(d64);
 675 }
 676 
 677 // emit 64 bit value and construct relocation entry from RelocationHolder
 678 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 679 #ifdef ASSERT
 680   if (rspec.reloc()->type() == relocInfo::oop_type &&
 681       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 682     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 683     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 684            "cannot embed scavengable oops in code");
 685   }
 686 #endif
 687   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 688   cbuf.insts()->emit_int64(d64);
 689 }
 690 
 691 // Access stack slot for load or store
 692 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 693 {
 694   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 695   if (-0x80 <= disp && disp < 0x80) {
 696     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 697     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 698     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 699   } else {
 700     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d32(cbuf, disp);     // Displacement // R/M byte
 703   }
 704 }
 705 
 706    // rRegI ereg, memory mem) %{    // emit_reg_mem
 707 void encode_RegMem(CodeBuffer &cbuf,
 708                    int reg,
 709                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 710 {
 711   assert(disp_reloc == relocInfo::none, "cannot have disp");
 712   int regenc = reg & 7;
 713   int baseenc = base & 7;
 714   int indexenc = index & 7;
 715 
 716   // There is no index & no scale, use form without SIB byte
 717   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 718     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 719     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 720       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 721     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 722       // If 8-bit displacement, mode 0x1
 723       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 724       emit_d8(cbuf, disp);
 725     } else {
 726       // If 32-bit displacement
 727       if (base == -1) { // Special flag for absolute address
 728         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 729         if (disp_reloc != relocInfo::none) {
 730           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 731         } else {
 732           emit_d32(cbuf, disp);
 733         }
 734       } else {
 735         // Normal base + offset
 736         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 737         if (disp_reloc != relocInfo::none) {
 738           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 739         } else {
 740           emit_d32(cbuf, disp);
 741         }
 742       }
 743     }
 744   } else {
 745     // Else, encode with the SIB byte
 746     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 747     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 748       // If no displacement
 749       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 750       emit_rm(cbuf, scale, indexenc, baseenc);
 751     } else {
 752       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 753         // If 8-bit displacement, mode 0x1
 754         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 755         emit_rm(cbuf, scale, indexenc, baseenc);
 756         emit_d8(cbuf, disp);
 757       } else {
 758         // If 32-bit displacement
 759         if (base == 0x04 ) {
 760           emit_rm(cbuf, 0x2, regenc, 0x4);
 761           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 762         } else {
 763           emit_rm(cbuf, 0x2, regenc, 0x4);
 764           emit_rm(cbuf, scale, indexenc, baseenc); // *
 765         }
 766         if (disp_reloc != relocInfo::none) {
 767           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 768         } else {
 769           emit_d32(cbuf, disp);
 770         }
 771       }
 772     }
 773   }
 774 }
 775 
 776 // This could be in MacroAssembler but it's fairly C2 specific
 777 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 778   Label exit;
 779   __ jccb(Assembler::noParity, exit);
 780   __ pushf();
 781   //
 782   // comiss/ucomiss instructions set ZF,PF,CF flags and
 783   // zero OF,AF,SF for NaN values.
 784   // Fixup flags by zeroing ZF,PF so that compare of NaN
 785   // values returns 'less than' result (CF is set).
 786   // Leave the rest of flags unchanged.
 787   //
 788   //    7 6 5 4 3 2 1 0
 789   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 790   //    0 0 1 0 1 0 1 1   (0x2B)
 791   //
 792   __ andq(Address(rsp, 0), 0xffffff2b);
 793   __ popf();
 794   __ bind(exit);
 795 }
 796 
 797 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 798   Label done;
 799   __ movl(dst, -1);
 800   __ jcc(Assembler::parity, done);
 801   __ jcc(Assembler::below, done);
 802   __ setb(Assembler::notEqual, dst);
 803   __ movzbl(dst, dst);
 804   __ bind(done);
 805 }
 806 
 807 
 808 //=============================================================================
 809 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 810 
 811 int Compile::ConstantTable::calculate_table_base_offset() const {
 812   return 0;  // absolute addressing, no offset
 813 }
 814 
 815 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 816 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 817   ShouldNotReachHere();
 818 }
 819 
 820 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 821   // Empty encoding
 822 }
 823 
 824 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 825   return 0;
 826 }
 827 
 828 #ifndef PRODUCT
 829 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 830   st->print("# MachConstantBaseNode (empty encoding)");
 831 }
 832 #endif
 833 
 834 
 835 //=============================================================================
 836 #ifndef PRODUCT
 837 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 838   Compile* C = ra_->C;
 839 
 840   int framesize = C->frame_size_in_bytes();
 841   int bangsize = C->bang_size_in_bytes();
 842   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 843   // Remove wordSize for return addr which is already pushed.
 844   framesize -= wordSize;
 845 
 846   if (C->need_stack_bang(bangsize)) {
 847     framesize -= wordSize;
 848     st->print("# stack bang (%d bytes)", bangsize);
 849     st->print("\n\t");
 850     st->print("pushq   rbp\t# Save rbp");
 851     if (PreserveFramePointer) {
 852         st->print("\n\t");
 853         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 854     }
 855     if (framesize) {
 856       st->print("\n\t");
 857       st->print("subq    rsp, #%d\t# Create frame",framesize);
 858     }
 859   } else {
 860     st->print("subq    rsp, #%d\t# Create frame",framesize);
 861     st->print("\n\t");
 862     framesize -= wordSize;
 863     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 864     if (PreserveFramePointer) {
 865       st->print("\n\t");
 866       st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 867       if (framesize > 0) {
 868         st->print("\n\t");
 869         st->print("addq    rbp, #%d", framesize);
 870       }
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   if (C->max_vector_size() > 16) {
 957     // Clear upper bits of YMM registers when current compiled code uses
 958     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 959     MacroAssembler _masm(&cbuf);
 960     __ vzeroupper();
 961   }
 962 
 963   int framesize = C->frame_size_in_bytes();
 964   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 965   // Remove word for return adr already pushed
 966   // and RBP
 967   framesize -= 2*wordSize;
 968 
 969   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 970 
 971   if (framesize) {
 972     emit_opcode(cbuf, Assembler::REX_W);
 973     if (framesize < 0x80) {
 974       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 975       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 976       emit_d8(cbuf, framesize);
 977     } else {
 978       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d32(cbuf, framesize);
 981     }
 982   }
 983 
 984   // popq rbp
 985   emit_opcode(cbuf, 0x58 | RBP_enc);
 986 
 987   if (do_polling() && C->is_method_compilation()) {
 988     MacroAssembler _masm(&cbuf);
 989     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 990     if (Assembler::is_polling_page_far()) {
 991       __ lea(rscratch1, polling_page);
 992       __ relocate(relocInfo::poll_return_type);
 993       __ testl(rax, Address(rscratch1, 0));
 994     } else {
 995       __ testl(rax, polling_page);
 996     }
 997   }
 998 }
 999 
1000 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1001 {
1002   return MachNode::size(ra_); // too many variables; just compute it
1003                               // the hard way
1004 }
1005 
1006 int MachEpilogNode::reloc() const
1007 {
1008   return 2; // a large enough number
1009 }
1010 
1011 const Pipeline* MachEpilogNode::pipeline() const
1012 {
1013   return MachNode::pipeline_class();
1014 }
1015 
1016 int MachEpilogNode::safepoint_offset() const
1017 {
1018   return 0;
1019 }
1020 
1021 //=============================================================================
1022 
1023 enum RC {
1024   rc_bad,
1025   rc_int,
1026   rc_float,
1027   rc_stack
1028 };
1029 
1030 static enum RC rc_class(OptoReg::Name reg)
1031 {
1032   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1033 
1034   if (OptoReg::is_stack(reg)) return rc_stack;
1035 
1036   VMReg r = OptoReg::as_VMReg(reg);
1037 
1038   if (r->is_Register()) return rc_int;
1039 
1040   assert(r->is_XMMRegister(), "must be");
1041   return rc_float;
1042 }
1043 
1044 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1045 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1046                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1047 
1048 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1049                             int stack_offset, int reg, uint ireg, outputStream* st);
1050 
1051 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1052                                       int dst_offset, uint ireg, outputStream* st) {
1053   if (cbuf) {
1054     MacroAssembler _masm(cbuf);
1055     switch (ireg) {
1056     case Op_VecS:
1057       __ movq(Address(rsp, -8), rax);
1058       __ movl(rax, Address(rsp, src_offset));
1059       __ movl(Address(rsp, dst_offset), rax);
1060       __ movq(rax, Address(rsp, -8));
1061       break;
1062     case Op_VecD:
1063       __ pushq(Address(rsp, src_offset));
1064       __ popq (Address(rsp, dst_offset));
1065       break;
1066     case Op_VecX:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       __ pushq(Address(rsp, src_offset+8));
1070       __ popq (Address(rsp, dst_offset+8));
1071       break;
1072     case Op_VecY:
1073       __ vmovdqu(Address(rsp, -32), xmm0);
1074       __ vmovdqu(xmm0, Address(rsp, src_offset));
1075       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1076       __ vmovdqu(xmm0, Address(rsp, -32));
1077     case Op_VecZ:
1078       __ evmovdqul(Address(rsp, -64), xmm0, 2);
1079       __ evmovdqul(xmm0, Address(rsp, src_offset), 2);
1080       __ evmovdqul(Address(rsp, dst_offset), xmm0, 2);
1081       __ evmovdqul(xmm0, Address(rsp, -64), 2);
1082       break;
1083     default:
1084       ShouldNotReachHere();
1085     }
1086 #ifndef PRODUCT
1087   } else {
1088     switch (ireg) {
1089     case Op_VecS:
1090       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1091                 "movl    rax, [rsp + #%d]\n\t"
1092                 "movl    [rsp + #%d], rax\n\t"
1093                 "movq    rax, [rsp - #8]",
1094                 src_offset, dst_offset);
1095       break;
1096     case Op_VecD:
1097       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1098                 "popq    [rsp + #%d]",
1099                 src_offset, dst_offset);
1100       break;
1101      case Op_VecX:
1102       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1103                 "popq    [rsp + #%d]\n\t"
1104                 "pushq   [rsp + #%d]\n\t"
1105                 "popq    [rsp + #%d]",
1106                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1107       break;
1108     case Op_VecY:
1109       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1110                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1111                 "vmovdqu [rsp + #%d], xmm0\n\t"
1112                 "vmovdqu xmm0, [rsp - #32]",
1113                 src_offset, dst_offset);
1114       break;
1115     case Op_VecZ:
1116       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1117                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1118                 "vmovdqu [rsp + #%d], xmm0\n\t"
1119                 "vmovdqu xmm0, [rsp - #64]",
1120                 src_offset, dst_offset);
1121       break;
1122     default:
1123       ShouldNotReachHere();
1124     }
1125 #endif
1126   }
1127 }
1128 
1129 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1130                                        PhaseRegAlloc* ra_,
1131                                        bool do_size,
1132                                        outputStream* st) const {
1133   assert(cbuf != NULL || st  != NULL, "sanity");
1134   // Get registers to move
1135   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1136   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1137   OptoReg::Name dst_second = ra_->get_reg_second(this);
1138   OptoReg::Name dst_first = ra_->get_reg_first(this);
1139 
1140   enum RC src_second_rc = rc_class(src_second);
1141   enum RC src_first_rc = rc_class(src_first);
1142   enum RC dst_second_rc = rc_class(dst_second);
1143   enum RC dst_first_rc = rc_class(dst_first);
1144 
1145   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1146          "must move at least 1 register" );
1147 
1148   if (src_first == dst_first && src_second == dst_second) {
1149     // Self copy, no move
1150     return 0;
1151   }
1152   if (bottom_type()->isa_vect() != NULL) {
1153     uint ireg = ideal_reg();
1154     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1155     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1156     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1157       // mem -> mem
1158       int src_offset = ra_->reg2offset(src_first);
1159       int dst_offset = ra_->reg2offset(dst_first);
1160       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1161     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1162       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1163     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1164       int stack_offset = ra_->reg2offset(dst_first);
1165       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1166     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1167       int stack_offset = ra_->reg2offset(src_first);
1168       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1169     } else {
1170       ShouldNotReachHere();
1171     }
1172     return 0;
1173   }
1174   if (src_first_rc == rc_stack) {
1175     // mem ->
1176     if (dst_first_rc == rc_stack) {
1177       // mem -> mem
1178       assert(src_second != dst_first, "overlap");
1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1181         // 64-bit
1182         int src_offset = ra_->reg2offset(src_first);
1183         int dst_offset = ra_->reg2offset(dst_first);
1184         if (cbuf) {
1185           MacroAssembler _masm(cbuf);
1186           __ pushq(Address(rsp, src_offset));
1187           __ popq (Address(rsp, dst_offset));
1188 #ifndef PRODUCT
1189         } else {
1190           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1191                     "popq    [rsp + #%d]",
1192                      src_offset, dst_offset);
1193 #endif
1194         }
1195       } else {
1196         // 32-bit
1197         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1198         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1199         // No pushl/popl, so:
1200         int src_offset = ra_->reg2offset(src_first);
1201         int dst_offset = ra_->reg2offset(dst_first);
1202         if (cbuf) {
1203           MacroAssembler _masm(cbuf);
1204           __ movq(Address(rsp, -8), rax);
1205           __ movl(rax, Address(rsp, src_offset));
1206           __ movl(Address(rsp, dst_offset), rax);
1207           __ movq(rax, Address(rsp, -8));
1208 #ifndef PRODUCT
1209         } else {
1210           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1211                     "movl    rax, [rsp + #%d]\n\t"
1212                     "movl    [rsp + #%d], rax\n\t"
1213                     "movq    rax, [rsp - #8]",
1214                      src_offset, dst_offset);
1215 #endif
1216         }
1217       }
1218       return 0;
1219     } else if (dst_first_rc == rc_int) {
1220       // mem -> gpr
1221       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1222           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1223         // 64-bit
1224         int offset = ra_->reg2offset(src_first);
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1228 #ifndef PRODUCT
1229         } else {
1230           st->print("movq    %s, [rsp + #%d]\t# spill",
1231                      Matcher::regName[dst_first],
1232                      offset);
1233 #endif
1234         }
1235       } else {
1236         // 32-bit
1237         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1238         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1239         int offset = ra_->reg2offset(src_first);
1240         if (cbuf) {
1241           MacroAssembler _masm(cbuf);
1242           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1243 #ifndef PRODUCT
1244         } else {
1245           st->print("movl    %s, [rsp + #%d]\t# spill",
1246                      Matcher::regName[dst_first],
1247                      offset);
1248 #endif
1249         }
1250       }
1251       return 0;
1252     } else if (dst_first_rc == rc_float) {
1253       // mem-> xmm
1254       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1255           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1256         // 64-bit
1257         int offset = ra_->reg2offset(src_first);
1258         if (cbuf) {
1259           MacroAssembler _masm(cbuf);
1260           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1261 #ifndef PRODUCT
1262         } else {
1263           st->print("%s  %s, [rsp + #%d]\t# spill",
1264                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1265                      Matcher::regName[dst_first],
1266                      offset);
1267 #endif
1268         }
1269       } else {
1270         // 32-bit
1271         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1272         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1273         int offset = ra_->reg2offset(src_first);
1274         if (cbuf) {
1275           MacroAssembler _masm(cbuf);
1276           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1277 #ifndef PRODUCT
1278         } else {
1279           st->print("movss   %s, [rsp + #%d]\t# spill",
1280                      Matcher::regName[dst_first],
1281                      offset);
1282 #endif
1283         }
1284       }
1285       return 0;
1286     }
1287   } else if (src_first_rc == rc_int) {
1288     // gpr ->
1289     if (dst_first_rc == rc_stack) {
1290       // gpr -> mem
1291       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1292           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1293         // 64-bit
1294         int offset = ra_->reg2offset(dst_first);
1295         if (cbuf) {
1296           MacroAssembler _masm(cbuf);
1297           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1298 #ifndef PRODUCT
1299         } else {
1300           st->print("movq    [rsp + #%d], %s\t# spill",
1301                      offset,
1302                      Matcher::regName[src_first]);
1303 #endif
1304         }
1305       } else {
1306         // 32-bit
1307         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1308         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1309         int offset = ra_->reg2offset(dst_first);
1310         if (cbuf) {
1311           MacroAssembler _masm(cbuf);
1312           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1313 #ifndef PRODUCT
1314         } else {
1315           st->print("movl    [rsp + #%d], %s\t# spill",
1316                      offset,
1317                      Matcher::regName[src_first]);
1318 #endif
1319         }
1320       }
1321       return 0;
1322     } else if (dst_first_rc == rc_int) {
1323       // gpr -> gpr
1324       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1325           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1326         // 64-bit
1327         if (cbuf) {
1328           MacroAssembler _masm(cbuf);
1329           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1330                   as_Register(Matcher::_regEncode[src_first]));
1331 #ifndef PRODUCT
1332         } else {
1333           st->print("movq    %s, %s\t# spill",
1334                      Matcher::regName[dst_first],
1335                      Matcher::regName[src_first]);
1336 #endif
1337         }
1338         return 0;
1339       } else {
1340         // 32-bit
1341         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1342         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1343         if (cbuf) {
1344           MacroAssembler _masm(cbuf);
1345           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1346                   as_Register(Matcher::_regEncode[src_first]));
1347 #ifndef PRODUCT
1348         } else {
1349           st->print("movl    %s, %s\t# spill",
1350                      Matcher::regName[dst_first],
1351                      Matcher::regName[src_first]);
1352 #endif
1353         }
1354         return 0;
1355       }
1356     } else if (dst_first_rc == rc_float) {
1357       // gpr -> xmm
1358       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1359           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1360         // 64-bit
1361         if (cbuf) {
1362           MacroAssembler _masm(cbuf);
1363           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1364 #ifndef PRODUCT
1365         } else {
1366           st->print("movdq   %s, %s\t# spill",
1367                      Matcher::regName[dst_first],
1368                      Matcher::regName[src_first]);
1369 #endif
1370         }
1371       } else {
1372         // 32-bit
1373         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1374         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1378 #ifndef PRODUCT
1379         } else {
1380           st->print("movdl   %s, %s\t# spill",
1381                      Matcher::regName[dst_first],
1382                      Matcher::regName[src_first]);
1383 #endif
1384         }
1385       }
1386       return 0;
1387     }
1388   } else if (src_first_rc == rc_float) {
1389     // xmm ->
1390     if (dst_first_rc == rc_stack) {
1391       // xmm -> mem
1392       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1393           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1394         // 64-bit
1395         int offset = ra_->reg2offset(dst_first);
1396         if (cbuf) {
1397           MacroAssembler _masm(cbuf);
1398           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1399 #ifndef PRODUCT
1400         } else {
1401           st->print("movsd   [rsp + #%d], %s\t# spill",
1402                      offset,
1403                      Matcher::regName[src_first]);
1404 #endif
1405         }
1406       } else {
1407         // 32-bit
1408         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1409         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1410         int offset = ra_->reg2offset(dst_first);
1411         if (cbuf) {
1412           MacroAssembler _masm(cbuf);
1413           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1414 #ifndef PRODUCT
1415         } else {
1416           st->print("movss   [rsp + #%d], %s\t# spill",
1417                      offset,
1418                      Matcher::regName[src_first]);
1419 #endif
1420         }
1421       }
1422       return 0;
1423     } else if (dst_first_rc == rc_int) {
1424       // xmm -> gpr
1425       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1426           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1427         // 64-bit
1428         if (cbuf) {
1429           MacroAssembler _masm(cbuf);
1430           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1431 #ifndef PRODUCT
1432         } else {
1433           st->print("movdq   %s, %s\t# spill",
1434                      Matcher::regName[dst_first],
1435                      Matcher::regName[src_first]);
1436 #endif
1437         }
1438       } else {
1439         // 32-bit
1440         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1441         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1445 #ifndef PRODUCT
1446         } else {
1447           st->print("movdl   %s, %s\t# spill",
1448                      Matcher::regName[dst_first],
1449                      Matcher::regName[src_first]);
1450 #endif
1451         }
1452       }
1453       return 0;
1454     } else if (dst_first_rc == rc_float) {
1455       // xmm -> xmm
1456       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1457           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1458         // 64-bit
1459         if (cbuf) {
1460           MacroAssembler _masm(cbuf);
1461           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1462 #ifndef PRODUCT
1463         } else {
1464           st->print("%s  %s, %s\t# spill",
1465                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1466                      Matcher::regName[dst_first],
1467                      Matcher::regName[src_first]);
1468 #endif
1469         }
1470       } else {
1471         // 32-bit
1472         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1473         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1474         if (cbuf) {
1475           MacroAssembler _masm(cbuf);
1476           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1477 #ifndef PRODUCT
1478         } else {
1479           st->print("%s  %s, %s\t# spill",
1480                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1481                      Matcher::regName[dst_first],
1482                      Matcher::regName[src_first]);
1483 #endif
1484         }
1485       }
1486       return 0;
1487     }
1488   }
1489 
1490   assert(0," foo ");
1491   Unimplemented();
1492   return 0;
1493 }
1494 
1495 #ifndef PRODUCT
1496 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1497   implementation(NULL, ra_, false, st);
1498 }
1499 #endif
1500 
1501 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1502   implementation(&cbuf, ra_, false, NULL);
1503 }
1504 
1505 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1506   return MachNode::size(ra_);
1507 }
1508 
1509 //=============================================================================
1510 #ifndef PRODUCT
1511 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1512 {
1513   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1514   int reg = ra_->get_reg_first(this);
1515   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1516             Matcher::regName[reg], offset);
1517 }
1518 #endif
1519 
1520 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1521 {
1522   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1523   int reg = ra_->get_encode(this);
1524   if (offset >= 0x80) {
1525     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1526     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1527     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1528     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1529     emit_d32(cbuf, offset);
1530   } else {
1531     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1532     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1533     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1534     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1535     emit_d8(cbuf, offset);
1536   }
1537 }
1538 
1539 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1540 {
1541   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1542   return (offset < 0x80) ? 5 : 8; // REX
1543 }
1544 
1545 //=============================================================================
1546 #ifndef PRODUCT
1547 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1548 {
1549   if (UseCompressedClassPointers) {
1550     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1551     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1552     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1553   } else {
1554     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1555                  "# Inline cache check");
1556   }
1557   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1558   st->print_cr("\tnop\t# nops to align entry point");
1559 }
1560 #endif
1561 
1562 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1563 {
1564   MacroAssembler masm(&cbuf);
1565   uint insts_size = cbuf.insts_size();
1566   if (UseCompressedClassPointers) {
1567     masm.load_klass(rscratch1, j_rarg0);
1568     masm.cmpptr(rax, rscratch1);
1569   } else {
1570     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1571   }
1572 
1573   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1574 
1575   /* WARNING these NOPs are critical so that verified entry point is properly
1576      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1577   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1578   if (OptoBreakpoint) {
1579     // Leave space for int3
1580     nops_cnt -= 1;
1581   }
1582   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1583   if (nops_cnt > 0)
1584     masm.nop(nops_cnt);
1585 }
1586 
1587 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1588 {
1589   return MachNode::size(ra_); // too many variables; just compute it
1590                               // the hard way
1591 }
1592 
1593 
1594 //=============================================================================
1595 
1596 int Matcher::regnum_to_fpu_offset(int regnum)
1597 {
1598   return regnum - 32; // The FP registers are in the second chunk
1599 }
1600 
1601 // This is UltraSparc specific, true just means we have fast l2f conversion
1602 const bool Matcher::convL2FSupported(void) {
1603   return true;
1604 }
1605 
1606 // Is this branch offset short enough that a short branch can be used?
1607 //
1608 // NOTE: If the platform does not provide any short branch variants, then
1609 //       this method should return false for offset 0.
1610 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1611   // The passed offset is relative to address of the branch.
1612   // On 86 a branch displacement is calculated relative to address
1613   // of a next instruction.
1614   offset -= br_size;
1615 
1616   // the short version of jmpConUCF2 contains multiple branches,
1617   // making the reach slightly less
1618   if (rule == jmpConUCF2_rule)
1619     return (-126 <= offset && offset <= 125);
1620   return (-128 <= offset && offset <= 127);
1621 }
1622 
1623 const bool Matcher::isSimpleConstant64(jlong value) {
1624   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1625   //return value == (int) value;  // Cf. storeImmL and immL32.
1626 
1627   // Probably always true, even if a temp register is required.
1628   return true;
1629 }
1630 
1631 // The ecx parameter to rep stosq for the ClearArray node is in words.
1632 const bool Matcher::init_array_count_is_in_bytes = false;
1633 
1634 // Threshold size for cleararray.
1635 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1636 
1637 // No additional cost for CMOVL.
1638 const int Matcher::long_cmove_cost() { return 0; }
1639 
1640 // No CMOVF/CMOVD with SSE2
1641 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1642 
1643 // Does the CPU require late expand (see block.cpp for description of late expand)?
1644 const bool Matcher::require_postalloc_expand = false;
1645 
1646 // Should the Matcher clone shifts on addressing modes, expecting them
1647 // to be subsumed into complex addressing expressions or compute them
1648 // into registers?  True for Intel but false for most RISCs
1649 const bool Matcher::clone_shift_expressions = true;
1650 
1651 // Do we need to mask the count passed to shift instructions or does
1652 // the cpu only look at the lower 5/6 bits anyway?
1653 const bool Matcher::need_masked_shift_count = false;
1654 
1655 bool Matcher::narrow_oop_use_complex_address() {
1656   assert(UseCompressedOops, "only for compressed oops code");
1657   return (LogMinObjAlignmentInBytes <= 3);
1658 }
1659 
1660 bool Matcher::narrow_klass_use_complex_address() {
1661   assert(UseCompressedClassPointers, "only for compressed klass code");
1662   return (LogKlassAlignmentInBytes <= 3);
1663 }
1664 
1665 // Is it better to copy float constants, or load them directly from
1666 // memory?  Intel can load a float constant from a direct address,
1667 // requiring no extra registers.  Most RISCs will have to materialize
1668 // an address into a register first, so they would do better to copy
1669 // the constant from stack.
1670 const bool Matcher::rematerialize_float_constants = true; // XXX
1671 
1672 // If CPU can load and store mis-aligned doubles directly then no
1673 // fixup is needed.  Else we split the double into 2 integer pieces
1674 // and move it piece-by-piece.  Only happens when passing doubles into
1675 // C code as the Java calling convention forces doubles to be aligned.
1676 const bool Matcher::misaligned_doubles_ok = true;
1677 
1678 // No-op on amd64
1679 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1680 
1681 // Advertise here if the CPU requires explicit rounding operations to
1682 // implement the UseStrictFP mode.
1683 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1684 
1685 // Are floats conerted to double when stored to stack during deoptimization?
1686 // On x64 it is stored without convertion so we can use normal access.
1687 bool Matcher::float_in_double() { return false; }
1688 
1689 // Do ints take an entire long register or just half?
1690 const bool Matcher::int_in_long = true;
1691 
1692 // Return whether or not this register is ever used as an argument.
1693 // This function is used on startup to build the trampoline stubs in
1694 // generateOptoStub.  Registers not mentioned will be killed by the VM
1695 // call in the trampoline, and arguments in those registers not be
1696 // available to the callee.
1697 bool Matcher::can_be_java_arg(int reg)
1698 {
1699   return
1700     reg ==  RDI_num || reg == RDI_H_num ||
1701     reg ==  RSI_num || reg == RSI_H_num ||
1702     reg ==  RDX_num || reg == RDX_H_num ||
1703     reg ==  RCX_num || reg == RCX_H_num ||
1704     reg ==   R8_num || reg ==  R8_H_num ||
1705     reg ==   R9_num || reg ==  R9_H_num ||
1706     reg ==  R12_num || reg == R12_H_num ||
1707     reg == XMM0_num || reg == XMM0b_num ||
1708     reg == XMM1_num || reg == XMM1b_num ||
1709     reg == XMM2_num || reg == XMM2b_num ||
1710     reg == XMM3_num || reg == XMM3b_num ||
1711     reg == XMM4_num || reg == XMM4b_num ||
1712     reg == XMM5_num || reg == XMM5b_num ||
1713     reg == XMM6_num || reg == XMM6b_num ||
1714     reg == XMM7_num || reg == XMM7b_num;
1715 }
1716 
1717 bool Matcher::is_spillable_arg(int reg)
1718 {
1719   return can_be_java_arg(reg);
1720 }
1721 
1722 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1723   // In 64 bit mode a code which use multiply when
1724   // devisor is constant is faster than hardware
1725   // DIV instruction (it uses MulHiL).
1726   return false;
1727 }
1728 
1729 // Register for DIVI projection of divmodI
1730 RegMask Matcher::divI_proj_mask() {
1731   return INT_RAX_REG_mask();
1732 }
1733 
1734 // Register for MODI projection of divmodI
1735 RegMask Matcher::modI_proj_mask() {
1736   return INT_RDX_REG_mask();
1737 }
1738 
1739 // Register for DIVL projection of divmodL
1740 RegMask Matcher::divL_proj_mask() {
1741   return LONG_RAX_REG_mask();
1742 }
1743 
1744 // Register for MODL projection of divmodL
1745 RegMask Matcher::modL_proj_mask() {
1746   return LONG_RDX_REG_mask();
1747 }
1748 
1749 // Register for saving SP into on method handle invokes. Not used on x86_64.
1750 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1751     return NO_REG_mask();
1752 }
1753 
1754 %}
1755 
1756 //----------ENCODING BLOCK-----------------------------------------------------
1757 // This block specifies the encoding classes used by the compiler to
1758 // output byte streams.  Encoding classes are parameterized macros
1759 // used by Machine Instruction Nodes in order to generate the bit
1760 // encoding of the instruction.  Operands specify their base encoding
1761 // interface with the interface keyword.  There are currently
1762 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1763 // COND_INTER.  REG_INTER causes an operand to generate a function
1764 // which returns its register number when queried.  CONST_INTER causes
1765 // an operand to generate a function which returns the value of the
1766 // constant when queried.  MEMORY_INTER causes an operand to generate
1767 // four functions which return the Base Register, the Index Register,
1768 // the Scale Value, and the Offset Value of the operand when queried.
1769 // COND_INTER causes an operand to generate six functions which return
1770 // the encoding code (ie - encoding bits for the instruction)
1771 // associated with each basic boolean condition for a conditional
1772 // instruction.
1773 //
1774 // Instructions specify two basic values for encoding.  Again, a
1775 // function is available to check if the constant displacement is an
1776 // oop. They use the ins_encode keyword to specify their encoding
1777 // classes (which must be a sequence of enc_class names, and their
1778 // parameters, specified in the encoding block), and they use the
1779 // opcode keyword to specify, in order, their primary, secondary, and
1780 // tertiary opcode.  Only the opcode sections which a particular
1781 // instruction needs for encoding need to be specified.
1782 encode %{
1783   // Build emit functions for each basic byte or larger field in the
1784   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1785   // from C++ code in the enc_class source block.  Emit functions will
1786   // live in the main source block for now.  In future, we can
1787   // generalize this by adding a syntax that specifies the sizes of
1788   // fields in an order, so that the adlc can build the emit functions
1789   // automagically
1790 
1791   // Emit primary opcode
1792   enc_class OpcP
1793   %{
1794     emit_opcode(cbuf, $primary);
1795   %}
1796 
1797   // Emit secondary opcode
1798   enc_class OpcS
1799   %{
1800     emit_opcode(cbuf, $secondary);
1801   %}
1802 
1803   // Emit tertiary opcode
1804   enc_class OpcT
1805   %{
1806     emit_opcode(cbuf, $tertiary);
1807   %}
1808 
1809   // Emit opcode directly
1810   enc_class Opcode(immI d8)
1811   %{
1812     emit_opcode(cbuf, $d8$$constant);
1813   %}
1814 
1815   // Emit size prefix
1816   enc_class SizePrefix
1817   %{
1818     emit_opcode(cbuf, 0x66);
1819   %}
1820 
1821   enc_class reg(rRegI reg)
1822   %{
1823     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1824   %}
1825 
1826   enc_class reg_reg(rRegI dst, rRegI src)
1827   %{
1828     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1829   %}
1830 
1831   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1832   %{
1833     emit_opcode(cbuf, $opcode$$constant);
1834     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1835   %}
1836 
1837   enc_class cdql_enc(no_rax_rdx_RegI div)
1838   %{
1839     // Full implementation of Java idiv and irem; checks for
1840     // special case as described in JVM spec., p.243 & p.271.
1841     //
1842     //         normal case                           special case
1843     //
1844     // input : rax: dividend                         min_int
1845     //         reg: divisor                          -1
1846     //
1847     // output: rax: quotient  (= rax idiv reg)       min_int
1848     //         rdx: remainder (= rax irem reg)       0
1849     //
1850     //  Code sequnce:
1851     //
1852     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1853     //    5:   75 07/08                jne    e <normal>
1854     //    7:   33 d2                   xor    %edx,%edx
1855     //  [div >= 8 -> offset + 1]
1856     //  [REX_B]
1857     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1858     //    c:   74 03/04                je     11 <done>
1859     // 000000000000000e <normal>:
1860     //    e:   99                      cltd
1861     //  [div >= 8 -> offset + 1]
1862     //  [REX_B]
1863     //    f:   f7 f9                   idiv   $div
1864     // 0000000000000011 <done>:
1865 
1866     // cmp    $0x80000000,%eax
1867     emit_opcode(cbuf, 0x3d);
1868     emit_d8(cbuf, 0x00);
1869     emit_d8(cbuf, 0x00);
1870     emit_d8(cbuf, 0x00);
1871     emit_d8(cbuf, 0x80);
1872 
1873     // jne    e <normal>
1874     emit_opcode(cbuf, 0x75);
1875     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1876 
1877     // xor    %edx,%edx
1878     emit_opcode(cbuf, 0x33);
1879     emit_d8(cbuf, 0xD2);
1880 
1881     // cmp    $0xffffffffffffffff,%ecx
1882     if ($div$$reg >= 8) {
1883       emit_opcode(cbuf, Assembler::REX_B);
1884     }
1885     emit_opcode(cbuf, 0x83);
1886     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1887     emit_d8(cbuf, 0xFF);
1888 
1889     // je     11 <done>
1890     emit_opcode(cbuf, 0x74);
1891     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1892 
1893     // <normal>
1894     // cltd
1895     emit_opcode(cbuf, 0x99);
1896 
1897     // idivl (note: must be emitted by the user of this rule)
1898     // <done>
1899   %}
1900 
1901   enc_class cdqq_enc(no_rax_rdx_RegL div)
1902   %{
1903     // Full implementation of Java ldiv and lrem; checks for
1904     // special case as described in JVM spec., p.243 & p.271.
1905     //
1906     //         normal case                           special case
1907     //
1908     // input : rax: dividend                         min_long
1909     //         reg: divisor                          -1
1910     //
1911     // output: rax: quotient  (= rax idiv reg)       min_long
1912     //         rdx: remainder (= rax irem reg)       0
1913     //
1914     //  Code sequnce:
1915     //
1916     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1917     //    7:   00 00 80
1918     //    a:   48 39 d0                cmp    %rdx,%rax
1919     //    d:   75 08                   jne    17 <normal>
1920     //    f:   33 d2                   xor    %edx,%edx
1921     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1922     //   15:   74 05                   je     1c <done>
1923     // 0000000000000017 <normal>:
1924     //   17:   48 99                   cqto
1925     //   19:   48 f7 f9                idiv   $div
1926     // 000000000000001c <done>:
1927 
1928     // mov    $0x8000000000000000,%rdx
1929     emit_opcode(cbuf, Assembler::REX_W);
1930     emit_opcode(cbuf, 0xBA);
1931     emit_d8(cbuf, 0x00);
1932     emit_d8(cbuf, 0x00);
1933     emit_d8(cbuf, 0x00);
1934     emit_d8(cbuf, 0x00);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x80);
1939 
1940     // cmp    %rdx,%rax
1941     emit_opcode(cbuf, Assembler::REX_W);
1942     emit_opcode(cbuf, 0x39);
1943     emit_d8(cbuf, 0xD0);
1944 
1945     // jne    17 <normal>
1946     emit_opcode(cbuf, 0x75);
1947     emit_d8(cbuf, 0x08);
1948 
1949     // xor    %edx,%edx
1950     emit_opcode(cbuf, 0x33);
1951     emit_d8(cbuf, 0xD2);
1952 
1953     // cmp    $0xffffffffffffffff,$div
1954     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1955     emit_opcode(cbuf, 0x83);
1956     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1957     emit_d8(cbuf, 0xFF);
1958 
1959     // je     1e <done>
1960     emit_opcode(cbuf, 0x74);
1961     emit_d8(cbuf, 0x05);
1962 
1963     // <normal>
1964     // cqto
1965     emit_opcode(cbuf, Assembler::REX_W);
1966     emit_opcode(cbuf, 0x99);
1967 
1968     // idivq (note: must be emitted by the user of this rule)
1969     // <done>
1970   %}
1971 
1972   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1973   enc_class OpcSE(immI imm)
1974   %{
1975     // Emit primary opcode and set sign-extend bit
1976     // Check for 8-bit immediate, and set sign extend bit in opcode
1977     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1978       emit_opcode(cbuf, $primary | 0x02);
1979     } else {
1980       // 32-bit immediate
1981       emit_opcode(cbuf, $primary);
1982     }
1983   %}
1984 
1985   enc_class OpcSErm(rRegI dst, immI imm)
1986   %{
1987     // OpcSEr/m
1988     int dstenc = $dst$$reg;
1989     if (dstenc >= 8) {
1990       emit_opcode(cbuf, Assembler::REX_B);
1991       dstenc -= 8;
1992     }
1993     // Emit primary opcode and set sign-extend bit
1994     // Check for 8-bit immediate, and set sign extend bit in opcode
1995     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1996       emit_opcode(cbuf, $primary | 0x02);
1997     } else {
1998       // 32-bit immediate
1999       emit_opcode(cbuf, $primary);
2000     }
2001     // Emit r/m byte with secondary opcode, after primary opcode.
2002     emit_rm(cbuf, 0x3, $secondary, dstenc);
2003   %}
2004 
2005   enc_class OpcSErm_wide(rRegL dst, immI imm)
2006   %{
2007     // OpcSEr/m
2008     int dstenc = $dst$$reg;
2009     if (dstenc < 8) {
2010       emit_opcode(cbuf, Assembler::REX_W);
2011     } else {
2012       emit_opcode(cbuf, Assembler::REX_WB);
2013       dstenc -= 8;
2014     }
2015     // Emit primary opcode and set sign-extend bit
2016     // Check for 8-bit immediate, and set sign extend bit in opcode
2017     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2018       emit_opcode(cbuf, $primary | 0x02);
2019     } else {
2020       // 32-bit immediate
2021       emit_opcode(cbuf, $primary);
2022     }
2023     // Emit r/m byte with secondary opcode, after primary opcode.
2024     emit_rm(cbuf, 0x3, $secondary, dstenc);
2025   %}
2026 
2027   enc_class Con8or32(immI imm)
2028   %{
2029     // Check for 8-bit immediate, and set sign extend bit in opcode
2030     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2031       $$$emit8$imm$$constant;
2032     } else {
2033       // 32-bit immediate
2034       $$$emit32$imm$$constant;
2035     }
2036   %}
2037 
2038   enc_class opc2_reg(rRegI dst)
2039   %{
2040     // BSWAP
2041     emit_cc(cbuf, $secondary, $dst$$reg);
2042   %}
2043 
2044   enc_class opc3_reg(rRegI dst)
2045   %{
2046     // BSWAP
2047     emit_cc(cbuf, $tertiary, $dst$$reg);
2048   %}
2049 
2050   enc_class reg_opc(rRegI div)
2051   %{
2052     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2053     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2054   %}
2055 
2056   enc_class enc_cmov(cmpOp cop)
2057   %{
2058     // CMOV
2059     $$$emit8$primary;
2060     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2061   %}
2062 
2063   enc_class enc_PartialSubtypeCheck()
2064   %{
2065     Register Rrdi = as_Register(RDI_enc); // result register
2066     Register Rrax = as_Register(RAX_enc); // super class
2067     Register Rrcx = as_Register(RCX_enc); // killed
2068     Register Rrsi = as_Register(RSI_enc); // sub class
2069     Label miss;
2070     const bool set_cond_codes = true;
2071 
2072     MacroAssembler _masm(&cbuf);
2073     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2074                                      NULL, &miss,
2075                                      /*set_cond_codes:*/ true);
2076     if ($primary) {
2077       __ xorptr(Rrdi, Rrdi);
2078     }
2079     __ bind(miss);
2080   %}
2081 
2082   enc_class clear_avx %{
2083     debug_only(int off0 = cbuf.insts_size());
2084     if (ra_->C->max_vector_size() > 16) {
2085       // Clear upper bits of YMM registers when current compiled code uses
2086       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2087       MacroAssembler _masm(&cbuf);
2088       __ vzeroupper();
2089     }
2090     debug_only(int off1 = cbuf.insts_size());
2091     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2092   %}
2093 
2094   enc_class Java_To_Runtime(method meth) %{
2095     // No relocation needed
2096     MacroAssembler _masm(&cbuf);
2097     __ mov64(r10, (int64_t) $meth$$method);
2098     __ call(r10);
2099   %}
2100 
2101   enc_class Java_To_Interpreter(method meth)
2102   %{
2103     // CALL Java_To_Interpreter
2104     // This is the instruction starting address for relocation info.
2105     cbuf.set_insts_mark();
2106     $$$emit8$primary;
2107     // CALL directly to the runtime
2108     emit_d32_reloc(cbuf,
2109                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2110                    runtime_call_Relocation::spec(),
2111                    RELOC_DISP32);
2112   %}
2113 
2114   enc_class Java_Static_Call(method meth)
2115   %{
2116     // JAVA STATIC CALL
2117     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2118     // determine who we intended to call.
2119     cbuf.set_insts_mark();
2120     $$$emit8$primary;
2121 
2122     if (!_method) {
2123       emit_d32_reloc(cbuf,
2124                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2125                      runtime_call_Relocation::spec(),
2126                      RELOC_DISP32);
2127     } else if (_optimized_virtual) {
2128       emit_d32_reloc(cbuf,
2129                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2130                      opt_virtual_call_Relocation::spec(),
2131                      RELOC_DISP32);
2132     } else {
2133       emit_d32_reloc(cbuf,
2134                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2135                      static_call_Relocation::spec(),
2136                      RELOC_DISP32);
2137     }
2138     if (_method) {
2139       // Emit stubs for static call.
2140       address mark = cbuf.insts_mark();
2141       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
2142       if (stub == NULL) {
2143         ciEnv::current()->record_failure("CodeCache is full");
2144         return;
2145       }
2146     }
2147   %}
2148 
2149   enc_class Java_Dynamic_Call(method meth) %{
2150     MacroAssembler _masm(&cbuf);
2151     __ ic_call((address)$meth$$method);
2152   %}
2153 
2154   enc_class Java_Compiled_Call(method meth)
2155   %{
2156     // JAVA COMPILED CALL
2157     int disp = in_bytes(Method:: from_compiled_offset());
2158 
2159     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2160     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2161 
2162     // callq *disp(%rax)
2163     cbuf.set_insts_mark();
2164     $$$emit8$primary;
2165     if (disp < 0x80) {
2166       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2167       emit_d8(cbuf, disp); // Displacement
2168     } else {
2169       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2170       emit_d32(cbuf, disp); // Displacement
2171     }
2172   %}
2173 
2174   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2175   %{
2176     // SAL, SAR, SHR
2177     int dstenc = $dst$$reg;
2178     if (dstenc >= 8) {
2179       emit_opcode(cbuf, Assembler::REX_B);
2180       dstenc -= 8;
2181     }
2182     $$$emit8$primary;
2183     emit_rm(cbuf, 0x3, $secondary, dstenc);
2184     $$$emit8$shift$$constant;
2185   %}
2186 
2187   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2188   %{
2189     // SAL, SAR, SHR
2190     int dstenc = $dst$$reg;
2191     if (dstenc < 8) {
2192       emit_opcode(cbuf, Assembler::REX_W);
2193     } else {
2194       emit_opcode(cbuf, Assembler::REX_WB);
2195       dstenc -= 8;
2196     }
2197     $$$emit8$primary;
2198     emit_rm(cbuf, 0x3, $secondary, dstenc);
2199     $$$emit8$shift$$constant;
2200   %}
2201 
2202   enc_class load_immI(rRegI dst, immI src)
2203   %{
2204     int dstenc = $dst$$reg;
2205     if (dstenc >= 8) {
2206       emit_opcode(cbuf, Assembler::REX_B);
2207       dstenc -= 8;
2208     }
2209     emit_opcode(cbuf, 0xB8 | dstenc);
2210     $$$emit32$src$$constant;
2211   %}
2212 
2213   enc_class load_immL(rRegL dst, immL src)
2214   %{
2215     int dstenc = $dst$$reg;
2216     if (dstenc < 8) {
2217       emit_opcode(cbuf, Assembler::REX_W);
2218     } else {
2219       emit_opcode(cbuf, Assembler::REX_WB);
2220       dstenc -= 8;
2221     }
2222     emit_opcode(cbuf, 0xB8 | dstenc);
2223     emit_d64(cbuf, $src$$constant);
2224   %}
2225 
2226   enc_class load_immUL32(rRegL dst, immUL32 src)
2227   %{
2228     // same as load_immI, but this time we care about zeroes in the high word
2229     int dstenc = $dst$$reg;
2230     if (dstenc >= 8) {
2231       emit_opcode(cbuf, Assembler::REX_B);
2232       dstenc -= 8;
2233     }
2234     emit_opcode(cbuf, 0xB8 | dstenc);
2235     $$$emit32$src$$constant;
2236   %}
2237 
2238   enc_class load_immL32(rRegL dst, immL32 src)
2239   %{
2240     int dstenc = $dst$$reg;
2241     if (dstenc < 8) {
2242       emit_opcode(cbuf, Assembler::REX_W);
2243     } else {
2244       emit_opcode(cbuf, Assembler::REX_WB);
2245       dstenc -= 8;
2246     }
2247     emit_opcode(cbuf, 0xC7);
2248     emit_rm(cbuf, 0x03, 0x00, dstenc);
2249     $$$emit32$src$$constant;
2250   %}
2251 
2252   enc_class load_immP31(rRegP dst, immP32 src)
2253   %{
2254     // same as load_immI, but this time we care about zeroes in the high word
2255     int dstenc = $dst$$reg;
2256     if (dstenc >= 8) {
2257       emit_opcode(cbuf, Assembler::REX_B);
2258       dstenc -= 8;
2259     }
2260     emit_opcode(cbuf, 0xB8 | dstenc);
2261     $$$emit32$src$$constant;
2262   %}
2263 
2264   enc_class load_immP(rRegP dst, immP src)
2265   %{
2266     int dstenc = $dst$$reg;
2267     if (dstenc < 8) {
2268       emit_opcode(cbuf, Assembler::REX_W);
2269     } else {
2270       emit_opcode(cbuf, Assembler::REX_WB);
2271       dstenc -= 8;
2272     }
2273     emit_opcode(cbuf, 0xB8 | dstenc);
2274     // This next line should be generated from ADLC
2275     if ($src->constant_reloc() != relocInfo::none) {
2276       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2277     } else {
2278       emit_d64(cbuf, $src$$constant);
2279     }
2280   %}
2281 
2282   enc_class Con32(immI src)
2283   %{
2284     // Output immediate
2285     $$$emit32$src$$constant;
2286   %}
2287 
2288   enc_class Con32F_as_bits(immF src)
2289   %{
2290     // Output Float immediate bits
2291     jfloat jf = $src$$constant;
2292     jint jf_as_bits = jint_cast(jf);
2293     emit_d32(cbuf, jf_as_bits);
2294   %}
2295 
2296   enc_class Con16(immI src)
2297   %{
2298     // Output immediate
2299     $$$emit16$src$$constant;
2300   %}
2301 
2302   // How is this different from Con32??? XXX
2303   enc_class Con_d32(immI src)
2304   %{
2305     emit_d32(cbuf,$src$$constant);
2306   %}
2307 
2308   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2309     // Output immediate memory reference
2310     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2311     emit_d32(cbuf, 0x00);
2312   %}
2313 
2314   enc_class lock_prefix()
2315   %{
2316     if (os::is_MP()) {
2317       emit_opcode(cbuf, 0xF0); // lock
2318     }
2319   %}
2320 
2321   enc_class REX_mem(memory mem)
2322   %{
2323     if ($mem$$base >= 8) {
2324       if ($mem$$index < 8) {
2325         emit_opcode(cbuf, Assembler::REX_B);
2326       } else {
2327         emit_opcode(cbuf, Assembler::REX_XB);
2328       }
2329     } else {
2330       if ($mem$$index >= 8) {
2331         emit_opcode(cbuf, Assembler::REX_X);
2332       }
2333     }
2334   %}
2335 
2336   enc_class REX_mem_wide(memory mem)
2337   %{
2338     if ($mem$$base >= 8) {
2339       if ($mem$$index < 8) {
2340         emit_opcode(cbuf, Assembler::REX_WB);
2341       } else {
2342         emit_opcode(cbuf, Assembler::REX_WXB);
2343       }
2344     } else {
2345       if ($mem$$index < 8) {
2346         emit_opcode(cbuf, Assembler::REX_W);
2347       } else {
2348         emit_opcode(cbuf, Assembler::REX_WX);
2349       }
2350     }
2351   %}
2352 
2353   // for byte regs
2354   enc_class REX_breg(rRegI reg)
2355   %{
2356     if ($reg$$reg >= 4) {
2357       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2358     }
2359   %}
2360 
2361   // for byte regs
2362   enc_class REX_reg_breg(rRegI dst, rRegI src)
2363   %{
2364     if ($dst$$reg < 8) {
2365       if ($src$$reg >= 4) {
2366         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2367       }
2368     } else {
2369       if ($src$$reg < 8) {
2370         emit_opcode(cbuf, Assembler::REX_R);
2371       } else {
2372         emit_opcode(cbuf, Assembler::REX_RB);
2373       }
2374     }
2375   %}
2376 
2377   // for byte regs
2378   enc_class REX_breg_mem(rRegI reg, memory mem)
2379   %{
2380     if ($reg$$reg < 8) {
2381       if ($mem$$base < 8) {
2382         if ($mem$$index >= 8) {
2383           emit_opcode(cbuf, Assembler::REX_X);
2384         } else if ($reg$$reg >= 4) {
2385           emit_opcode(cbuf, Assembler::REX);
2386         }
2387       } else {
2388         if ($mem$$index < 8) {
2389           emit_opcode(cbuf, Assembler::REX_B);
2390         } else {
2391           emit_opcode(cbuf, Assembler::REX_XB);
2392         }
2393       }
2394     } else {
2395       if ($mem$$base < 8) {
2396         if ($mem$$index < 8) {
2397           emit_opcode(cbuf, Assembler::REX_R);
2398         } else {
2399           emit_opcode(cbuf, Assembler::REX_RX);
2400         }
2401       } else {
2402         if ($mem$$index < 8) {
2403           emit_opcode(cbuf, Assembler::REX_RB);
2404         } else {
2405           emit_opcode(cbuf, Assembler::REX_RXB);
2406         }
2407       }
2408     }
2409   %}
2410 
2411   enc_class REX_reg(rRegI reg)
2412   %{
2413     if ($reg$$reg >= 8) {
2414       emit_opcode(cbuf, Assembler::REX_B);
2415     }
2416   %}
2417 
2418   enc_class REX_reg_wide(rRegI reg)
2419   %{
2420     if ($reg$$reg < 8) {
2421       emit_opcode(cbuf, Assembler::REX_W);
2422     } else {
2423       emit_opcode(cbuf, Assembler::REX_WB);
2424     }
2425   %}
2426 
2427   enc_class REX_reg_reg(rRegI dst, rRegI src)
2428   %{
2429     if ($dst$$reg < 8) {
2430       if ($src$$reg >= 8) {
2431         emit_opcode(cbuf, Assembler::REX_B);
2432       }
2433     } else {
2434       if ($src$$reg < 8) {
2435         emit_opcode(cbuf, Assembler::REX_R);
2436       } else {
2437         emit_opcode(cbuf, Assembler::REX_RB);
2438       }
2439     }
2440   %}
2441 
2442   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2443   %{
2444     if ($dst$$reg < 8) {
2445       if ($src$$reg < 8) {
2446         emit_opcode(cbuf, Assembler::REX_W);
2447       } else {
2448         emit_opcode(cbuf, Assembler::REX_WB);
2449       }
2450     } else {
2451       if ($src$$reg < 8) {
2452         emit_opcode(cbuf, Assembler::REX_WR);
2453       } else {
2454         emit_opcode(cbuf, Assembler::REX_WRB);
2455       }
2456     }
2457   %}
2458 
2459   enc_class REX_reg_mem(rRegI reg, memory mem)
2460   %{
2461     if ($reg$$reg < 8) {
2462       if ($mem$$base < 8) {
2463         if ($mem$$index >= 8) {
2464           emit_opcode(cbuf, Assembler::REX_X);
2465         }
2466       } else {
2467         if ($mem$$index < 8) {
2468           emit_opcode(cbuf, Assembler::REX_B);
2469         } else {
2470           emit_opcode(cbuf, Assembler::REX_XB);
2471         }
2472       }
2473     } else {
2474       if ($mem$$base < 8) {
2475         if ($mem$$index < 8) {
2476           emit_opcode(cbuf, Assembler::REX_R);
2477         } else {
2478           emit_opcode(cbuf, Assembler::REX_RX);
2479         }
2480       } else {
2481         if ($mem$$index < 8) {
2482           emit_opcode(cbuf, Assembler::REX_RB);
2483         } else {
2484           emit_opcode(cbuf, Assembler::REX_RXB);
2485         }
2486       }
2487     }
2488   %}
2489 
2490   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2491   %{
2492     if ($reg$$reg < 8) {
2493       if ($mem$$base < 8) {
2494         if ($mem$$index < 8) {
2495           emit_opcode(cbuf, Assembler::REX_W);
2496         } else {
2497           emit_opcode(cbuf, Assembler::REX_WX);
2498         }
2499       } else {
2500         if ($mem$$index < 8) {
2501           emit_opcode(cbuf, Assembler::REX_WB);
2502         } else {
2503           emit_opcode(cbuf, Assembler::REX_WXB);
2504         }
2505       }
2506     } else {
2507       if ($mem$$base < 8) {
2508         if ($mem$$index < 8) {
2509           emit_opcode(cbuf, Assembler::REX_WR);
2510         } else {
2511           emit_opcode(cbuf, Assembler::REX_WRX);
2512         }
2513       } else {
2514         if ($mem$$index < 8) {
2515           emit_opcode(cbuf, Assembler::REX_WRB);
2516         } else {
2517           emit_opcode(cbuf, Assembler::REX_WRXB);
2518         }
2519       }
2520     }
2521   %}
2522 
2523   enc_class reg_mem(rRegI ereg, memory mem)
2524   %{
2525     // High registers handle in encode_RegMem
2526     int reg = $ereg$$reg;
2527     int base = $mem$$base;
2528     int index = $mem$$index;
2529     int scale = $mem$$scale;
2530     int disp = $mem$$disp;
2531     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2532 
2533     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2534   %}
2535 
2536   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2537   %{
2538     int rm_byte_opcode = $rm_opcode$$constant;
2539 
2540     // High registers handle in encode_RegMem
2541     int base = $mem$$base;
2542     int index = $mem$$index;
2543     int scale = $mem$$scale;
2544     int displace = $mem$$disp;
2545 
2546     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2547                                             // working with static
2548                                             // globals
2549     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2550                   disp_reloc);
2551   %}
2552 
2553   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2554   %{
2555     int reg_encoding = $dst$$reg;
2556     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2557     int index        = 0x04;            // 0x04 indicates no index
2558     int scale        = 0x00;            // 0x00 indicates no scale
2559     int displace     = $src1$$constant; // 0x00 indicates no displacement
2560     relocInfo::relocType disp_reloc = relocInfo::none;
2561     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2562                   disp_reloc);
2563   %}
2564 
2565   enc_class neg_reg(rRegI dst)
2566   %{
2567     int dstenc = $dst$$reg;
2568     if (dstenc >= 8) {
2569       emit_opcode(cbuf, Assembler::REX_B);
2570       dstenc -= 8;
2571     }
2572     // NEG $dst
2573     emit_opcode(cbuf, 0xF7);
2574     emit_rm(cbuf, 0x3, 0x03, dstenc);
2575   %}
2576 
2577   enc_class neg_reg_wide(rRegI dst)
2578   %{
2579     int dstenc = $dst$$reg;
2580     if (dstenc < 8) {
2581       emit_opcode(cbuf, Assembler::REX_W);
2582     } else {
2583       emit_opcode(cbuf, Assembler::REX_WB);
2584       dstenc -= 8;
2585     }
2586     // NEG $dst
2587     emit_opcode(cbuf, 0xF7);
2588     emit_rm(cbuf, 0x3, 0x03, dstenc);
2589   %}
2590 
2591   enc_class setLT_reg(rRegI dst)
2592   %{
2593     int dstenc = $dst$$reg;
2594     if (dstenc >= 8) {
2595       emit_opcode(cbuf, Assembler::REX_B);
2596       dstenc -= 8;
2597     } else if (dstenc >= 4) {
2598       emit_opcode(cbuf, Assembler::REX);
2599     }
2600     // SETLT $dst
2601     emit_opcode(cbuf, 0x0F);
2602     emit_opcode(cbuf, 0x9C);
2603     emit_rm(cbuf, 0x3, 0x0, dstenc);
2604   %}
2605 
2606   enc_class setNZ_reg(rRegI dst)
2607   %{
2608     int dstenc = $dst$$reg;
2609     if (dstenc >= 8) {
2610       emit_opcode(cbuf, Assembler::REX_B);
2611       dstenc -= 8;
2612     } else if (dstenc >= 4) {
2613       emit_opcode(cbuf, Assembler::REX);
2614     }
2615     // SETNZ $dst
2616     emit_opcode(cbuf, 0x0F);
2617     emit_opcode(cbuf, 0x95);
2618     emit_rm(cbuf, 0x3, 0x0, dstenc);
2619   %}
2620 
2621 
2622   // Compare the lonogs and set -1, 0, or 1 into dst
2623   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2624   %{
2625     int src1enc = $src1$$reg;
2626     int src2enc = $src2$$reg;
2627     int dstenc = $dst$$reg;
2628 
2629     // cmpq $src1, $src2
2630     if (src1enc < 8) {
2631       if (src2enc < 8) {
2632         emit_opcode(cbuf, Assembler::REX_W);
2633       } else {
2634         emit_opcode(cbuf, Assembler::REX_WB);
2635       }
2636     } else {
2637       if (src2enc < 8) {
2638         emit_opcode(cbuf, Assembler::REX_WR);
2639       } else {
2640         emit_opcode(cbuf, Assembler::REX_WRB);
2641       }
2642     }
2643     emit_opcode(cbuf, 0x3B);
2644     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2645 
2646     // movl $dst, -1
2647     if (dstenc >= 8) {
2648       emit_opcode(cbuf, Assembler::REX_B);
2649     }
2650     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2651     emit_d32(cbuf, -1);
2652 
2653     // jl,s done
2654     emit_opcode(cbuf, 0x7C);
2655     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2656 
2657     // setne $dst
2658     if (dstenc >= 4) {
2659       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2660     }
2661     emit_opcode(cbuf, 0x0F);
2662     emit_opcode(cbuf, 0x95);
2663     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2664 
2665     // movzbl $dst, $dst
2666     if (dstenc >= 4) {
2667       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2668     }
2669     emit_opcode(cbuf, 0x0F);
2670     emit_opcode(cbuf, 0xB6);
2671     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2672   %}
2673 
2674   enc_class Push_ResultXD(regD dst) %{
2675     MacroAssembler _masm(&cbuf);
2676     __ fstp_d(Address(rsp, 0));
2677     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2678     __ addptr(rsp, 8);
2679   %}
2680 
2681   enc_class Push_SrcXD(regD src) %{
2682     MacroAssembler _masm(&cbuf);
2683     __ subptr(rsp, 8);
2684     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2685     __ fld_d(Address(rsp, 0));
2686   %}
2687 
2688 
2689   enc_class enc_rethrow()
2690   %{
2691     cbuf.set_insts_mark();
2692     emit_opcode(cbuf, 0xE9); // jmp entry
2693     emit_d32_reloc(cbuf,
2694                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2695                    runtime_call_Relocation::spec(),
2696                    RELOC_DISP32);
2697   %}
2698 
2699 %}
2700 
2701 
2702 
2703 //----------FRAME--------------------------------------------------------------
2704 // Definition of frame structure and management information.
2705 //
2706 //  S T A C K   L A Y O U T    Allocators stack-slot number
2707 //                             |   (to get allocators register number
2708 //  G  Owned by    |        |  v    add OptoReg::stack0())
2709 //  r   CALLER     |        |
2710 //  o     |        +--------+      pad to even-align allocators stack-slot
2711 //  w     V        |  pad0  |        numbers; owned by CALLER
2712 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2713 //  h     ^        |   in   |  5
2714 //        |        |  args  |  4   Holes in incoming args owned by SELF
2715 //  |     |        |        |  3
2716 //  |     |        +--------+
2717 //  V     |        | old out|      Empty on Intel, window on Sparc
2718 //        |    old |preserve|      Must be even aligned.
2719 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2720 //        |        |   in   |  3   area for Intel ret address
2721 //     Owned by    |preserve|      Empty on Sparc.
2722 //       SELF      +--------+
2723 //        |        |  pad2  |  2   pad to align old SP
2724 //        |        +--------+  1
2725 //        |        | locks  |  0
2726 //        |        +--------+----> OptoReg::stack0(), even aligned
2727 //        |        |  pad1  | 11   pad to align new SP
2728 //        |        +--------+
2729 //        |        |        | 10
2730 //        |        | spills |  9   spills
2731 //        V        |        |  8   (pad0 slot for callee)
2732 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2733 //        ^        |  out   |  7
2734 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2735 //     Owned by    +--------+
2736 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2737 //        |    new |preserve|      Must be even-aligned.
2738 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2739 //        |        |        |
2740 //
2741 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2742 //         known from SELF's arguments and the Java calling convention.
2743 //         Region 6-7 is determined per call site.
2744 // Note 2: If the calling convention leaves holes in the incoming argument
2745 //         area, those holes are owned by SELF.  Holes in the outgoing area
2746 //         are owned by the CALLEE.  Holes should not be nessecary in the
2747 //         incoming area, as the Java calling convention is completely under
2748 //         the control of the AD file.  Doubles can be sorted and packed to
2749 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2750 //         varargs C calling conventions.
2751 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2752 //         even aligned with pad0 as needed.
2753 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2754 //         region 6-11 is even aligned; it may be padded out more so that
2755 //         the region from SP to FP meets the minimum stack alignment.
2756 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2757 //         alignment.  Region 11, pad1, may be dynamically extended so that
2758 //         SP meets the minimum alignment.
2759 
2760 frame
2761 %{
2762   // What direction does stack grow in (assumed to be same for C & Java)
2763   stack_direction(TOWARDS_LOW);
2764 
2765   // These three registers define part of the calling convention
2766   // between compiled code and the interpreter.
2767   inline_cache_reg(RAX);                // Inline Cache Register
2768   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2769                                         // calling interpreter
2770 
2771   // Optional: name the operand used by cisc-spilling to access
2772   // [stack_pointer + offset]
2773   cisc_spilling_operand_name(indOffset32);
2774 
2775   // Number of stack slots consumed by locking an object
2776   sync_stack_slots(2);
2777 
2778   // Compiled code's Frame Pointer
2779   frame_pointer(RSP);
2780 
2781   // Interpreter stores its frame pointer in a register which is
2782   // stored to the stack by I2CAdaptors.
2783   // I2CAdaptors convert from interpreted java to compiled java.
2784   interpreter_frame_pointer(RBP);
2785 
2786   // Stack alignment requirement
2787   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2788 
2789   // Number of stack slots between incoming argument block and the start of
2790   // a new frame.  The PROLOG must add this many slots to the stack.  The
2791   // EPILOG must remove this many slots.  amd64 needs two slots for
2792   // return address.
2793   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2794 
2795   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2796   // for calls to C.  Supports the var-args backing area for register parms.
2797   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2798 
2799   // The after-PROLOG location of the return address.  Location of
2800   // return address specifies a type (REG or STACK) and a number
2801   // representing the register number (i.e. - use a register name) or
2802   // stack slot.
2803   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2804   // Otherwise, it is above the locks and verification slot and alignment word
2805   return_addr(STACK - 2 +
2806               round_to((Compile::current()->in_preserve_stack_slots() +
2807                         Compile::current()->fixed_slots()),
2808                        stack_alignment_in_slots()));
2809 
2810   // Body of function which returns an integer array locating
2811   // arguments either in registers or in stack slots.  Passed an array
2812   // of ideal registers called "sig" and a "length" count.  Stack-slot
2813   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2814   // arguments for a CALLEE.  Incoming stack arguments are
2815   // automatically biased by the preserve_stack_slots field above.
2816 
2817   calling_convention
2818   %{
2819     // No difference between ingoing/outgoing just pass false
2820     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2821   %}
2822 
2823   c_calling_convention
2824   %{
2825     // This is obviously always outgoing
2826     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2827   %}
2828 
2829   // Location of compiled Java return values.  Same as C for now.
2830   return_value
2831   %{
2832     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2833            "only return normal values");
2834 
2835     static const int lo[Op_RegL + 1] = {
2836       0,
2837       0,
2838       RAX_num,  // Op_RegN
2839       RAX_num,  // Op_RegI
2840       RAX_num,  // Op_RegP
2841       XMM0_num, // Op_RegF
2842       XMM0_num, // Op_RegD
2843       RAX_num   // Op_RegL
2844     };
2845     static const int hi[Op_RegL + 1] = {
2846       0,
2847       0,
2848       OptoReg::Bad, // Op_RegN
2849       OptoReg::Bad, // Op_RegI
2850       RAX_H_num,    // Op_RegP
2851       OptoReg::Bad, // Op_RegF
2852       XMM0b_num,    // Op_RegD
2853       RAX_H_num     // Op_RegL
2854     };
2855     // Excluded flags and vector registers.
2856     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2857     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2858   %}
2859 %}
2860 
2861 //----------ATTRIBUTES---------------------------------------------------------
2862 //----------Operand Attributes-------------------------------------------------
2863 op_attrib op_cost(0);        // Required cost attribute
2864 
2865 //----------Instruction Attributes---------------------------------------------
2866 ins_attrib ins_cost(100);       // Required cost attribute
2867 ins_attrib ins_size(8);         // Required size attribute (in bits)
2868 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2869                                 // a non-matching short branch variant
2870                                 // of some long branch?
2871 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2872                                 // be a power of 2) specifies the
2873                                 // alignment that some part of the
2874                                 // instruction (not necessarily the
2875                                 // start) requires.  If > 1, a
2876                                 // compute_padding() function must be
2877                                 // provided for the instruction
2878 
2879 //----------OPERANDS-----------------------------------------------------------
2880 // Operand definitions must precede instruction definitions for correct parsing
2881 // in the ADLC because operands constitute user defined types which are used in
2882 // instruction definitions.
2883 
2884 //----------Simple Operands----------------------------------------------------
2885 // Immediate Operands
2886 // Integer Immediate
2887 operand immI()
2888 %{
2889   match(ConI);
2890 
2891   op_cost(10);
2892   format %{ %}
2893   interface(CONST_INTER);
2894 %}
2895 
2896 // Constant for test vs zero
2897 operand immI0()
2898 %{
2899   predicate(n->get_int() == 0);
2900   match(ConI);
2901 
2902   op_cost(0);
2903   format %{ %}
2904   interface(CONST_INTER);
2905 %}
2906 
2907 // Constant for increment
2908 operand immI1()
2909 %{
2910   predicate(n->get_int() == 1);
2911   match(ConI);
2912 
2913   op_cost(0);
2914   format %{ %}
2915   interface(CONST_INTER);
2916 %}
2917 
2918 // Constant for decrement
2919 operand immI_M1()
2920 %{
2921   predicate(n->get_int() == -1);
2922   match(ConI);
2923 
2924   op_cost(0);
2925   format %{ %}
2926   interface(CONST_INTER);
2927 %}
2928 
2929 // Valid scale values for addressing modes
2930 operand immI2()
2931 %{
2932   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2933   match(ConI);
2934 
2935   format %{ %}
2936   interface(CONST_INTER);
2937 %}
2938 
2939 operand immI8()
2940 %{
2941   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2942   match(ConI);
2943 
2944   op_cost(5);
2945   format %{ %}
2946   interface(CONST_INTER);
2947 %}
2948 
2949 operand immI16()
2950 %{
2951   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2952   match(ConI);
2953 
2954   op_cost(10);
2955   format %{ %}
2956   interface(CONST_INTER);
2957 %}
2958 
2959 // Int Immediate non-negative
2960 operand immU31()
2961 %{
2962   predicate(n->get_int() >= 0);
2963   match(ConI);
2964 
2965   op_cost(0);
2966   format %{ %}
2967   interface(CONST_INTER);
2968 %}
2969 
2970 // Constant for long shifts
2971 operand immI_32()
2972 %{
2973   predicate( n->get_int() == 32 );
2974   match(ConI);
2975 
2976   op_cost(0);
2977   format %{ %}
2978   interface(CONST_INTER);
2979 %}
2980 
2981 // Constant for long shifts
2982 operand immI_64()
2983 %{
2984   predicate( n->get_int() == 64 );
2985   match(ConI);
2986 
2987   op_cost(0);
2988   format %{ %}
2989   interface(CONST_INTER);
2990 %}
2991 
2992 // Pointer Immediate
2993 operand immP()
2994 %{
2995   match(ConP);
2996 
2997   op_cost(10);
2998   format %{ %}
2999   interface(CONST_INTER);
3000 %}
3001 
3002 // NULL Pointer Immediate
3003 operand immP0()
3004 %{
3005   predicate(n->get_ptr() == 0);
3006   match(ConP);
3007 
3008   op_cost(5);
3009   format %{ %}
3010   interface(CONST_INTER);
3011 %}
3012 
3013 // Pointer Immediate
3014 operand immN() %{
3015   match(ConN);
3016 
3017   op_cost(10);
3018   format %{ %}
3019   interface(CONST_INTER);
3020 %}
3021 
3022 operand immNKlass() %{
3023   match(ConNKlass);
3024 
3025   op_cost(10);
3026   format %{ %}
3027   interface(CONST_INTER);
3028 %}
3029 
3030 // NULL Pointer Immediate
3031 operand immN0() %{
3032   predicate(n->get_narrowcon() == 0);
3033   match(ConN);
3034 
3035   op_cost(5);
3036   format %{ %}
3037   interface(CONST_INTER);
3038 %}
3039 
3040 operand immP31()
3041 %{
3042   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3043             && (n->get_ptr() >> 31) == 0);
3044   match(ConP);
3045 
3046   op_cost(5);
3047   format %{ %}
3048   interface(CONST_INTER);
3049 %}
3050 
3051 
3052 // Long Immediate
3053 operand immL()
3054 %{
3055   match(ConL);
3056 
3057   op_cost(20);
3058   format %{ %}
3059   interface(CONST_INTER);
3060 %}
3061 
3062 // Long Immediate 8-bit
3063 operand immL8()
3064 %{
3065   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3066   match(ConL);
3067 
3068   op_cost(5);
3069   format %{ %}
3070   interface(CONST_INTER);
3071 %}
3072 
3073 // Long Immediate 32-bit unsigned
3074 operand immUL32()
3075 %{
3076   predicate(n->get_long() == (unsigned int) (n->get_long()));
3077   match(ConL);
3078 
3079   op_cost(10);
3080   format %{ %}
3081   interface(CONST_INTER);
3082 %}
3083 
3084 // Long Immediate 32-bit signed
3085 operand immL32()
3086 %{
3087   predicate(n->get_long() == (int) (n->get_long()));
3088   match(ConL);
3089 
3090   op_cost(15);
3091   format %{ %}
3092   interface(CONST_INTER);
3093 %}
3094 
3095 // Long Immediate zero
3096 operand immL0()
3097 %{
3098   predicate(n->get_long() == 0L);
3099   match(ConL);
3100 
3101   op_cost(10);
3102   format %{ %}
3103   interface(CONST_INTER);
3104 %}
3105 
3106 // Constant for increment
3107 operand immL1()
3108 %{
3109   predicate(n->get_long() == 1);
3110   match(ConL);
3111 
3112   format %{ %}
3113   interface(CONST_INTER);
3114 %}
3115 
3116 // Constant for decrement
3117 operand immL_M1()
3118 %{
3119   predicate(n->get_long() == -1);
3120   match(ConL);
3121 
3122   format %{ %}
3123   interface(CONST_INTER);
3124 %}
3125 
3126 // Long Immediate: the value 10
3127 operand immL10()
3128 %{
3129   predicate(n->get_long() == 10);
3130   match(ConL);
3131 
3132   format %{ %}
3133   interface(CONST_INTER);
3134 %}
3135 
3136 // Long immediate from 0 to 127.
3137 // Used for a shorter form of long mul by 10.
3138 operand immL_127()
3139 %{
3140   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3141   match(ConL);
3142 
3143   op_cost(10);
3144   format %{ %}
3145   interface(CONST_INTER);
3146 %}
3147 
3148 // Long Immediate: low 32-bit mask
3149 operand immL_32bits()
3150 %{
3151   predicate(n->get_long() == 0xFFFFFFFFL);
3152   match(ConL);
3153   op_cost(20);
3154 
3155   format %{ %}
3156   interface(CONST_INTER);
3157 %}
3158 
3159 // Float Immediate zero
3160 operand immF0()
3161 %{
3162   predicate(jint_cast(n->getf()) == 0);
3163   match(ConF);
3164 
3165   op_cost(5);
3166   format %{ %}
3167   interface(CONST_INTER);
3168 %}
3169 
3170 // Float Immediate
3171 operand immF()
3172 %{
3173   match(ConF);
3174 
3175   op_cost(15);
3176   format %{ %}
3177   interface(CONST_INTER);
3178 %}
3179 
3180 // Double Immediate zero
3181 operand immD0()
3182 %{
3183   predicate(jlong_cast(n->getd()) == 0);
3184   match(ConD);
3185 
3186   op_cost(5);
3187   format %{ %}
3188   interface(CONST_INTER);
3189 %}
3190 
3191 // Double Immediate
3192 operand immD()
3193 %{
3194   match(ConD);
3195 
3196   op_cost(15);
3197   format %{ %}
3198   interface(CONST_INTER);
3199 %}
3200 
3201 // Immediates for special shifts (sign extend)
3202 
3203 // Constants for increment
3204 operand immI_16()
3205 %{
3206   predicate(n->get_int() == 16);
3207   match(ConI);
3208 
3209   format %{ %}
3210   interface(CONST_INTER);
3211 %}
3212 
3213 operand immI_24()
3214 %{
3215   predicate(n->get_int() == 24);
3216   match(ConI);
3217 
3218   format %{ %}
3219   interface(CONST_INTER);
3220 %}
3221 
3222 // Constant for byte-wide masking
3223 operand immI_255()
3224 %{
3225   predicate(n->get_int() == 255);
3226   match(ConI);
3227 
3228   format %{ %}
3229   interface(CONST_INTER);
3230 %}
3231 
3232 // Constant for short-wide masking
3233 operand immI_65535()
3234 %{
3235   predicate(n->get_int() == 65535);
3236   match(ConI);
3237 
3238   format %{ %}
3239   interface(CONST_INTER);
3240 %}
3241 
3242 // Constant for byte-wide masking
3243 operand immL_255()
3244 %{
3245   predicate(n->get_long() == 255);
3246   match(ConL);
3247 
3248   format %{ %}
3249   interface(CONST_INTER);
3250 %}
3251 
3252 // Constant for short-wide masking
3253 operand immL_65535()
3254 %{
3255   predicate(n->get_long() == 65535);
3256   match(ConL);
3257 
3258   format %{ %}
3259   interface(CONST_INTER);
3260 %}
3261 
3262 // Register Operands
3263 // Integer Register
3264 operand rRegI()
3265 %{
3266   constraint(ALLOC_IN_RC(int_reg));
3267   match(RegI);
3268 
3269   match(rax_RegI);
3270   match(rbx_RegI);
3271   match(rcx_RegI);
3272   match(rdx_RegI);
3273   match(rdi_RegI);
3274 
3275   format %{ %}
3276   interface(REG_INTER);
3277 %}
3278 
3279 // Special Registers
3280 operand rax_RegI()
3281 %{
3282   constraint(ALLOC_IN_RC(int_rax_reg));
3283   match(RegI);
3284   match(rRegI);
3285 
3286   format %{ "RAX" %}
3287   interface(REG_INTER);
3288 %}
3289 
3290 // Special Registers
3291 operand rbx_RegI()
3292 %{
3293   constraint(ALLOC_IN_RC(int_rbx_reg));
3294   match(RegI);
3295   match(rRegI);
3296 
3297   format %{ "RBX" %}
3298   interface(REG_INTER);
3299 %}
3300 
3301 operand rcx_RegI()
3302 %{
3303   constraint(ALLOC_IN_RC(int_rcx_reg));
3304   match(RegI);
3305   match(rRegI);
3306 
3307   format %{ "RCX" %}
3308   interface(REG_INTER);
3309 %}
3310 
3311 operand rdx_RegI()
3312 %{
3313   constraint(ALLOC_IN_RC(int_rdx_reg));
3314   match(RegI);
3315   match(rRegI);
3316 
3317   format %{ "RDX" %}
3318   interface(REG_INTER);
3319 %}
3320 
3321 operand rdi_RegI()
3322 %{
3323   constraint(ALLOC_IN_RC(int_rdi_reg));
3324   match(RegI);
3325   match(rRegI);
3326 
3327   format %{ "RDI" %}
3328   interface(REG_INTER);
3329 %}
3330 
3331 operand no_rcx_RegI()
3332 %{
3333   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3334   match(RegI);
3335   match(rax_RegI);
3336   match(rbx_RegI);
3337   match(rdx_RegI);
3338   match(rdi_RegI);
3339 
3340   format %{ %}
3341   interface(REG_INTER);
3342 %}
3343 
3344 operand no_rax_rdx_RegI()
3345 %{
3346   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3347   match(RegI);
3348   match(rbx_RegI);
3349   match(rcx_RegI);
3350   match(rdi_RegI);
3351 
3352   format %{ %}
3353   interface(REG_INTER);
3354 %}
3355 
3356 // Pointer Register
3357 operand any_RegP()
3358 %{
3359   constraint(ALLOC_IN_RC(any_reg));
3360   match(RegP);
3361   match(rax_RegP);
3362   match(rbx_RegP);
3363   match(rdi_RegP);
3364   match(rsi_RegP);
3365   match(rbp_RegP);
3366   match(r15_RegP);
3367   match(rRegP);
3368 
3369   format %{ %}
3370   interface(REG_INTER);
3371 %}
3372 
3373 operand rRegP()
3374 %{
3375   constraint(ALLOC_IN_RC(ptr_reg));
3376   match(RegP);
3377   match(rax_RegP);
3378   match(rbx_RegP);
3379   match(rdi_RegP);
3380   match(rsi_RegP);
3381   match(rbp_RegP);  // See Q&A below about
3382   match(r15_RegP);  // r15_RegP and rbp_RegP.
3383 
3384   format %{ %}
3385   interface(REG_INTER);
3386 %}
3387 
3388 operand rRegN() %{
3389   constraint(ALLOC_IN_RC(int_reg));
3390   match(RegN);
3391 
3392   format %{ %}
3393   interface(REG_INTER);
3394 %}
3395 
3396 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3397 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3398 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3399 // The output of an instruction is controlled by the allocator, which respects
3400 // register class masks, not match rules.  Unless an instruction mentions
3401 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3402 // by the allocator as an input.
3403 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3404 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3405 // result, RBP is not included in the output of the instruction either.
3406 
3407 operand no_rax_RegP()
3408 %{
3409   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3410   match(RegP);
3411   match(rbx_RegP);
3412   match(rsi_RegP);
3413   match(rdi_RegP);
3414 
3415   format %{ %}
3416   interface(REG_INTER);
3417 %}
3418 
3419 // This operand is not allowed to use RBP even if
3420 // RBP is not used to hold the frame pointer.
3421 operand no_rbp_RegP()
3422 %{
3423   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3424   match(RegP);
3425   match(rbx_RegP);
3426   match(rsi_RegP);
3427   match(rdi_RegP);
3428 
3429   format %{ %}
3430   interface(REG_INTER);
3431 %}
3432 
3433 operand no_rax_rbx_RegP()
3434 %{
3435   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3436   match(RegP);
3437   match(rsi_RegP);
3438   match(rdi_RegP);
3439 
3440   format %{ %}
3441   interface(REG_INTER);
3442 %}
3443 
3444 // Special Registers
3445 // Return a pointer value
3446 operand rax_RegP()
3447 %{
3448   constraint(ALLOC_IN_RC(ptr_rax_reg));
3449   match(RegP);
3450   match(rRegP);
3451 
3452   format %{ %}
3453   interface(REG_INTER);
3454 %}
3455 
3456 // Special Registers
3457 // Return a compressed pointer value
3458 operand rax_RegN()
3459 %{
3460   constraint(ALLOC_IN_RC(int_rax_reg));
3461   match(RegN);
3462   match(rRegN);
3463 
3464   format %{ %}
3465   interface(REG_INTER);
3466 %}
3467 
3468 // Used in AtomicAdd
3469 operand rbx_RegP()
3470 %{
3471   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3472   match(RegP);
3473   match(rRegP);
3474 
3475   format %{ %}
3476   interface(REG_INTER);
3477 %}
3478 
3479 operand rsi_RegP()
3480 %{
3481   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3482   match(RegP);
3483   match(rRegP);
3484 
3485   format %{ %}
3486   interface(REG_INTER);
3487 %}
3488 
3489 // Used in rep stosq
3490 operand rdi_RegP()
3491 %{
3492   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3493   match(RegP);
3494   match(rRegP);
3495 
3496   format %{ %}
3497   interface(REG_INTER);
3498 %}
3499 
3500 operand r15_RegP()
3501 %{
3502   constraint(ALLOC_IN_RC(ptr_r15_reg));
3503   match(RegP);
3504   match(rRegP);
3505 
3506   format %{ %}
3507   interface(REG_INTER);
3508 %}
3509 
3510 operand rRegL()
3511 %{
3512   constraint(ALLOC_IN_RC(long_reg));
3513   match(RegL);
3514   match(rax_RegL);
3515   match(rdx_RegL);
3516 
3517   format %{ %}
3518   interface(REG_INTER);
3519 %}
3520 
3521 // Special Registers
3522 operand no_rax_rdx_RegL()
3523 %{
3524   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3525   match(RegL);
3526   match(rRegL);
3527 
3528   format %{ %}
3529   interface(REG_INTER);
3530 %}
3531 
3532 operand no_rax_RegL()
3533 %{
3534   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3535   match(RegL);
3536   match(rRegL);
3537   match(rdx_RegL);
3538 
3539   format %{ %}
3540   interface(REG_INTER);
3541 %}
3542 
3543 operand no_rcx_RegL()
3544 %{
3545   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3546   match(RegL);
3547   match(rRegL);
3548 
3549   format %{ %}
3550   interface(REG_INTER);
3551 %}
3552 
3553 operand rax_RegL()
3554 %{
3555   constraint(ALLOC_IN_RC(long_rax_reg));
3556   match(RegL);
3557   match(rRegL);
3558 
3559   format %{ "RAX" %}
3560   interface(REG_INTER);
3561 %}
3562 
3563 operand rcx_RegL()
3564 %{
3565   constraint(ALLOC_IN_RC(long_rcx_reg));
3566   match(RegL);
3567   match(rRegL);
3568 
3569   format %{ %}
3570   interface(REG_INTER);
3571 %}
3572 
3573 operand rdx_RegL()
3574 %{
3575   constraint(ALLOC_IN_RC(long_rdx_reg));
3576   match(RegL);
3577   match(rRegL);
3578 
3579   format %{ %}
3580   interface(REG_INTER);
3581 %}
3582 
3583 // Flags register, used as output of compare instructions
3584 operand rFlagsReg()
3585 %{
3586   constraint(ALLOC_IN_RC(int_flags));
3587   match(RegFlags);
3588 
3589   format %{ "RFLAGS" %}
3590   interface(REG_INTER);
3591 %}
3592 
3593 // Flags register, used as output of FLOATING POINT compare instructions
3594 operand rFlagsRegU()
3595 %{
3596   constraint(ALLOC_IN_RC(int_flags));
3597   match(RegFlags);
3598 
3599   format %{ "RFLAGS_U" %}
3600   interface(REG_INTER);
3601 %}
3602 
3603 operand rFlagsRegUCF() %{
3604   constraint(ALLOC_IN_RC(int_flags));
3605   match(RegFlags);
3606   predicate(false);
3607 
3608   format %{ "RFLAGS_U_CF" %}
3609   interface(REG_INTER);
3610 %}
3611 
3612 // Float register operands
3613 operand regF() %{
3614    constraint(ALLOC_IN_RC(float_reg));
3615    match(RegF);
3616 
3617    format %{ %}
3618    interface(REG_INTER);
3619 %}
3620 
3621 // Double register operands
3622 operand regD() %{
3623    constraint(ALLOC_IN_RC(double_reg));
3624    match(RegD);
3625 
3626    format %{ %}
3627    interface(REG_INTER);
3628 %}
3629 
3630 // Vectors
3631 operand vecS() %{
3632   constraint(ALLOC_IN_RC(vectors_reg));
3633   match(VecS);
3634 
3635   format %{ %}
3636   interface(REG_INTER);
3637 %}
3638 
3639 operand vecD() %{
3640   constraint(ALLOC_IN_RC(vectord_reg));
3641   match(VecD);
3642 
3643   format %{ %}
3644   interface(REG_INTER);
3645 %}
3646 
3647 operand vecX() %{
3648   constraint(ALLOC_IN_RC(vectorx_reg));
3649   match(VecX);
3650 
3651   format %{ %}
3652   interface(REG_INTER);
3653 %}
3654 
3655 operand vecY() %{
3656   constraint(ALLOC_IN_RC(vectory_reg));
3657   match(VecY);
3658 
3659   format %{ %}
3660   interface(REG_INTER);
3661 %}
3662 
3663 //----------Memory Operands----------------------------------------------------
3664 // Direct Memory Operand
3665 // operand direct(immP addr)
3666 // %{
3667 //   match(addr);
3668 
3669 //   format %{ "[$addr]" %}
3670 //   interface(MEMORY_INTER) %{
3671 //     base(0xFFFFFFFF);
3672 //     index(0x4);
3673 //     scale(0x0);
3674 //     disp($addr);
3675 //   %}
3676 // %}
3677 
3678 // Indirect Memory Operand
3679 operand indirect(any_RegP reg)
3680 %{
3681   constraint(ALLOC_IN_RC(ptr_reg));
3682   match(reg);
3683 
3684   format %{ "[$reg]" %}
3685   interface(MEMORY_INTER) %{
3686     base($reg);
3687     index(0x4);
3688     scale(0x0);
3689     disp(0x0);
3690   %}
3691 %}
3692 
3693 // Indirect Memory Plus Short Offset Operand
3694 operand indOffset8(any_RegP reg, immL8 off)
3695 %{
3696   constraint(ALLOC_IN_RC(ptr_reg));
3697   match(AddP reg off);
3698 
3699   format %{ "[$reg + $off (8-bit)]" %}
3700   interface(MEMORY_INTER) %{
3701     base($reg);
3702     index(0x4);
3703     scale(0x0);
3704     disp($off);
3705   %}
3706 %}
3707 
3708 // Indirect Memory Plus Long Offset Operand
3709 operand indOffset32(any_RegP reg, immL32 off)
3710 %{
3711   constraint(ALLOC_IN_RC(ptr_reg));
3712   match(AddP reg off);
3713 
3714   format %{ "[$reg + $off (32-bit)]" %}
3715   interface(MEMORY_INTER) %{
3716     base($reg);
3717     index(0x4);
3718     scale(0x0);
3719     disp($off);
3720   %}
3721 %}
3722 
3723 // Indirect Memory Plus Index Register Plus Offset Operand
3724 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3725 %{
3726   constraint(ALLOC_IN_RC(ptr_reg));
3727   match(AddP (AddP reg lreg) off);
3728 
3729   op_cost(10);
3730   format %{"[$reg + $off + $lreg]" %}
3731   interface(MEMORY_INTER) %{
3732     base($reg);
3733     index($lreg);
3734     scale(0x0);
3735     disp($off);
3736   %}
3737 %}
3738 
3739 // Indirect Memory Plus Index Register Plus Offset Operand
3740 operand indIndex(any_RegP reg, rRegL lreg)
3741 %{
3742   constraint(ALLOC_IN_RC(ptr_reg));
3743   match(AddP reg lreg);
3744 
3745   op_cost(10);
3746   format %{"[$reg + $lreg]" %}
3747   interface(MEMORY_INTER) %{
3748     base($reg);
3749     index($lreg);
3750     scale(0x0);
3751     disp(0x0);
3752   %}
3753 %}
3754 
3755 // Indirect Memory Times Scale Plus Index Register
3756 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3757 %{
3758   constraint(ALLOC_IN_RC(ptr_reg));
3759   match(AddP reg (LShiftL lreg scale));
3760 
3761   op_cost(10);
3762   format %{"[$reg + $lreg << $scale]" %}
3763   interface(MEMORY_INTER) %{
3764     base($reg);
3765     index($lreg);
3766     scale($scale);
3767     disp(0x0);
3768   %}
3769 %}
3770 
3771 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3772 %{
3773   constraint(ALLOC_IN_RC(ptr_reg));
3774   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3775   match(AddP reg (LShiftL (ConvI2L idx) scale));
3776 
3777   op_cost(10);
3778   format %{"[$reg + pos $idx << $scale]" %}
3779   interface(MEMORY_INTER) %{
3780     base($reg);
3781     index($idx);
3782     scale($scale);
3783     disp(0x0);
3784   %}
3785 %}
3786 
3787 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3788 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3789 %{
3790   constraint(ALLOC_IN_RC(ptr_reg));
3791   match(AddP (AddP reg (LShiftL lreg scale)) off);
3792 
3793   op_cost(10);
3794   format %{"[$reg + $off + $lreg << $scale]" %}
3795   interface(MEMORY_INTER) %{
3796     base($reg);
3797     index($lreg);
3798     scale($scale);
3799     disp($off);
3800   %}
3801 %}
3802 
3803 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3804 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3805 %{
3806   constraint(ALLOC_IN_RC(ptr_reg));
3807   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3808   match(AddP (AddP reg (ConvI2L idx)) off);
3809 
3810   op_cost(10);
3811   format %{"[$reg + $off + $idx]" %}
3812   interface(MEMORY_INTER) %{
3813     base($reg);
3814     index($idx);
3815     scale(0x0);
3816     disp($off);
3817   %}
3818 %}
3819 
3820 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3821 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3822 %{
3823   constraint(ALLOC_IN_RC(ptr_reg));
3824   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3825   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3826 
3827   op_cost(10);
3828   format %{"[$reg + $off + $idx << $scale]" %}
3829   interface(MEMORY_INTER) %{
3830     base($reg);
3831     index($idx);
3832     scale($scale);
3833     disp($off);
3834   %}
3835 %}
3836 
3837 // Indirect Narrow Oop Plus Offset Operand
3838 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3839 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3840 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3841   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3842   constraint(ALLOC_IN_RC(ptr_reg));
3843   match(AddP (DecodeN reg) off);
3844 
3845   op_cost(10);
3846   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3847   interface(MEMORY_INTER) %{
3848     base(0xc); // R12
3849     index($reg);
3850     scale(0x3);
3851     disp($off);
3852   %}
3853 %}
3854 
3855 // Indirect Memory Operand
3856 operand indirectNarrow(rRegN reg)
3857 %{
3858   predicate(Universe::narrow_oop_shift() == 0);
3859   constraint(ALLOC_IN_RC(ptr_reg));
3860   match(DecodeN reg);
3861 
3862   format %{ "[$reg]" %}
3863   interface(MEMORY_INTER) %{
3864     base($reg);
3865     index(0x4);
3866     scale(0x0);
3867     disp(0x0);
3868   %}
3869 %}
3870 
3871 // Indirect Memory Plus Short Offset Operand
3872 operand indOffset8Narrow(rRegN reg, immL8 off)
3873 %{
3874   predicate(Universe::narrow_oop_shift() == 0);
3875   constraint(ALLOC_IN_RC(ptr_reg));
3876   match(AddP (DecodeN reg) off);
3877 
3878   format %{ "[$reg + $off (8-bit)]" %}
3879   interface(MEMORY_INTER) %{
3880     base($reg);
3881     index(0x4);
3882     scale(0x0);
3883     disp($off);
3884   %}
3885 %}
3886 
3887 // Indirect Memory Plus Long Offset Operand
3888 operand indOffset32Narrow(rRegN reg, immL32 off)
3889 %{
3890   predicate(Universe::narrow_oop_shift() == 0);
3891   constraint(ALLOC_IN_RC(ptr_reg));
3892   match(AddP (DecodeN reg) off);
3893 
3894   format %{ "[$reg + $off (32-bit)]" %}
3895   interface(MEMORY_INTER) %{
3896     base($reg);
3897     index(0x4);
3898     scale(0x0);
3899     disp($off);
3900   %}
3901 %}
3902 
3903 // Indirect Memory Plus Index Register Plus Offset Operand
3904 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3905 %{
3906   predicate(Universe::narrow_oop_shift() == 0);
3907   constraint(ALLOC_IN_RC(ptr_reg));
3908   match(AddP (AddP (DecodeN reg) lreg) off);
3909 
3910   op_cost(10);
3911   format %{"[$reg + $off + $lreg]" %}
3912   interface(MEMORY_INTER) %{
3913     base($reg);
3914     index($lreg);
3915     scale(0x0);
3916     disp($off);
3917   %}
3918 %}
3919 
3920 // Indirect Memory Plus Index Register Plus Offset Operand
3921 operand indIndexNarrow(rRegN reg, rRegL lreg)
3922 %{
3923   predicate(Universe::narrow_oop_shift() == 0);
3924   constraint(ALLOC_IN_RC(ptr_reg));
3925   match(AddP (DecodeN reg) lreg);
3926 
3927   op_cost(10);
3928   format %{"[$reg + $lreg]" %}
3929   interface(MEMORY_INTER) %{
3930     base($reg);
3931     index($lreg);
3932     scale(0x0);
3933     disp(0x0);
3934   %}
3935 %}
3936 
3937 // Indirect Memory Times Scale Plus Index Register
3938 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3939 %{
3940   predicate(Universe::narrow_oop_shift() == 0);
3941   constraint(ALLOC_IN_RC(ptr_reg));
3942   match(AddP (DecodeN reg) (LShiftL lreg scale));
3943 
3944   op_cost(10);
3945   format %{"[$reg + $lreg << $scale]" %}
3946   interface(MEMORY_INTER) %{
3947     base($reg);
3948     index($lreg);
3949     scale($scale);
3950     disp(0x0);
3951   %}
3952 %}
3953 
3954 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3955 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3956 %{
3957   predicate(Universe::narrow_oop_shift() == 0);
3958   constraint(ALLOC_IN_RC(ptr_reg));
3959   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3960 
3961   op_cost(10);
3962   format %{"[$reg + $off + $lreg << $scale]" %}
3963   interface(MEMORY_INTER) %{
3964     base($reg);
3965     index($lreg);
3966     scale($scale);
3967     disp($off);
3968   %}
3969 %}
3970 
3971 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3972 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3973 %{
3974   constraint(ALLOC_IN_RC(ptr_reg));
3975   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3976   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3977 
3978   op_cost(10);
3979   format %{"[$reg + $off + $idx]" %}
3980   interface(MEMORY_INTER) %{
3981     base($reg);
3982     index($idx);
3983     scale(0x0);
3984     disp($off);
3985   %}
3986 %}
3987 
3988 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3989 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3990 %{
3991   constraint(ALLOC_IN_RC(ptr_reg));
3992   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3993   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3994 
3995   op_cost(10);
3996   format %{"[$reg + $off + $idx << $scale]" %}
3997   interface(MEMORY_INTER) %{
3998     base($reg);
3999     index($idx);
4000     scale($scale);
4001     disp($off);
4002   %}
4003 %}
4004 
4005 //----------Special Memory Operands--------------------------------------------
4006 // Stack Slot Operand - This operand is used for loading and storing temporary
4007 //                      values on the stack where a match requires a value to
4008 //                      flow through memory.
4009 operand stackSlotP(sRegP reg)
4010 %{
4011   constraint(ALLOC_IN_RC(stack_slots));
4012   // No match rule because this operand is only generated in matching
4013 
4014   format %{ "[$reg]" %}
4015   interface(MEMORY_INTER) %{
4016     base(0x4);   // RSP
4017     index(0x4);  // No Index
4018     scale(0x0);  // No Scale
4019     disp($reg);  // Stack Offset
4020   %}
4021 %}
4022 
4023 operand stackSlotI(sRegI reg)
4024 %{
4025   constraint(ALLOC_IN_RC(stack_slots));
4026   // No match rule because this operand is only generated in matching
4027 
4028   format %{ "[$reg]" %}
4029   interface(MEMORY_INTER) %{
4030     base(0x4);   // RSP
4031     index(0x4);  // No Index
4032     scale(0x0);  // No Scale
4033     disp($reg);  // Stack Offset
4034   %}
4035 %}
4036 
4037 operand stackSlotF(sRegF reg)
4038 %{
4039   constraint(ALLOC_IN_RC(stack_slots));
4040   // No match rule because this operand is only generated in matching
4041 
4042   format %{ "[$reg]" %}
4043   interface(MEMORY_INTER) %{
4044     base(0x4);   // RSP
4045     index(0x4);  // No Index
4046     scale(0x0);  // No Scale
4047     disp($reg);  // Stack Offset
4048   %}
4049 %}
4050 
4051 operand stackSlotD(sRegD reg)
4052 %{
4053   constraint(ALLOC_IN_RC(stack_slots));
4054   // No match rule because this operand is only generated in matching
4055 
4056   format %{ "[$reg]" %}
4057   interface(MEMORY_INTER) %{
4058     base(0x4);   // RSP
4059     index(0x4);  // No Index
4060     scale(0x0);  // No Scale
4061     disp($reg);  // Stack Offset
4062   %}
4063 %}
4064 operand stackSlotL(sRegL reg)
4065 %{
4066   constraint(ALLOC_IN_RC(stack_slots));
4067   // No match rule because this operand is only generated in matching
4068 
4069   format %{ "[$reg]" %}
4070   interface(MEMORY_INTER) %{
4071     base(0x4);   // RSP
4072     index(0x4);  // No Index
4073     scale(0x0);  // No Scale
4074     disp($reg);  // Stack Offset
4075   %}
4076 %}
4077 
4078 //----------Conditional Branch Operands----------------------------------------
4079 // Comparison Op  - This is the operation of the comparison, and is limited to
4080 //                  the following set of codes:
4081 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4082 //
4083 // Other attributes of the comparison, such as unsignedness, are specified
4084 // by the comparison instruction that sets a condition code flags register.
4085 // That result is represented by a flags operand whose subtype is appropriate
4086 // to the unsignedness (etc.) of the comparison.
4087 //
4088 // Later, the instruction which matches both the Comparison Op (a Bool) and
4089 // the flags (produced by the Cmp) specifies the coding of the comparison op
4090 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4091 
4092 // Comparision Code
4093 operand cmpOp()
4094 %{
4095   match(Bool);
4096 
4097   format %{ "" %}
4098   interface(COND_INTER) %{
4099     equal(0x4, "e");
4100     not_equal(0x5, "ne");
4101     less(0xC, "l");
4102     greater_equal(0xD, "ge");
4103     less_equal(0xE, "le");
4104     greater(0xF, "g");
4105     overflow(0x0, "o");
4106     no_overflow(0x1, "no");
4107   %}
4108 %}
4109 
4110 // Comparison Code, unsigned compare.  Used by FP also, with
4111 // C2 (unordered) turned into GT or LT already.  The other bits
4112 // C0 and C3 are turned into Carry & Zero flags.
4113 operand cmpOpU()
4114 %{
4115   match(Bool);
4116 
4117   format %{ "" %}
4118   interface(COND_INTER) %{
4119     equal(0x4, "e");
4120     not_equal(0x5, "ne");
4121     less(0x2, "b");
4122     greater_equal(0x3, "nb");
4123     less_equal(0x6, "be");
4124     greater(0x7, "nbe");
4125     overflow(0x0, "o");
4126     no_overflow(0x1, "no");
4127   %}
4128 %}
4129 
4130 
4131 // Floating comparisons that don't require any fixup for the unordered case
4132 operand cmpOpUCF() %{
4133   match(Bool);
4134   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4135             n->as_Bool()->_test._test == BoolTest::ge ||
4136             n->as_Bool()->_test._test == BoolTest::le ||
4137             n->as_Bool()->_test._test == BoolTest::gt);
4138   format %{ "" %}
4139   interface(COND_INTER) %{
4140     equal(0x4, "e");
4141     not_equal(0x5, "ne");
4142     less(0x2, "b");
4143     greater_equal(0x3, "nb");
4144     less_equal(0x6, "be");
4145     greater(0x7, "nbe");
4146     overflow(0x0, "o");
4147     no_overflow(0x1, "no");
4148   %}
4149 %}
4150 
4151 
4152 // Floating comparisons that can be fixed up with extra conditional jumps
4153 operand cmpOpUCF2() %{
4154   match(Bool);
4155   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4156             n->as_Bool()->_test._test == BoolTest::eq);
4157   format %{ "" %}
4158   interface(COND_INTER) %{
4159     equal(0x4, "e");
4160     not_equal(0x5, "ne");
4161     less(0x2, "b");
4162     greater_equal(0x3, "nb");
4163     less_equal(0x6, "be");
4164     greater(0x7, "nbe");
4165     overflow(0x0, "o");
4166     no_overflow(0x1, "no");
4167   %}
4168 %}
4169 
4170 
4171 //----------OPERAND CLASSES----------------------------------------------------
4172 // Operand Classes are groups of operands that are used as to simplify
4173 // instruction definitions by not requiring the AD writer to specify separate
4174 // instructions for every form of operand when the instruction accepts
4175 // multiple operand types with the same basic encoding and format.  The classic
4176 // case of this is memory operands.
4177 
4178 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4179                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4180                indCompressedOopOffset,
4181                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4182                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4183                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4184 
4185 //----------PIPELINE-----------------------------------------------------------
4186 // Rules which define the behavior of the target architectures pipeline.
4187 pipeline %{
4188 
4189 //----------ATTRIBUTES---------------------------------------------------------
4190 attributes %{
4191   variable_size_instructions;        // Fixed size instructions
4192   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4193   instruction_unit_size = 1;         // An instruction is 1 bytes long
4194   instruction_fetch_unit_size = 16;  // The processor fetches one line
4195   instruction_fetch_units = 1;       // of 16 bytes
4196 
4197   // List of nop instructions
4198   nops( MachNop );
4199 %}
4200 
4201 //----------RESOURCES----------------------------------------------------------
4202 // Resources are the functional units available to the machine
4203 
4204 // Generic P2/P3 pipeline
4205 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4206 // 3 instructions decoded per cycle.
4207 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4208 // 3 ALU op, only ALU0 handles mul instructions.
4209 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4210            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4211            BR, FPU,
4212            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4213 
4214 //----------PIPELINE DESCRIPTION-----------------------------------------------
4215 // Pipeline Description specifies the stages in the machine's pipeline
4216 
4217 // Generic P2/P3 pipeline
4218 pipe_desc(S0, S1, S2, S3, S4, S5);
4219 
4220 //----------PIPELINE CLASSES---------------------------------------------------
4221 // Pipeline Classes describe the stages in which input and output are
4222 // referenced by the hardware pipeline.
4223 
4224 // Naming convention: ialu or fpu
4225 // Then: _reg
4226 // Then: _reg if there is a 2nd register
4227 // Then: _long if it's a pair of instructions implementing a long
4228 // Then: _fat if it requires the big decoder
4229 //   Or: _mem if it requires the big decoder and a memory unit.
4230 
4231 // Integer ALU reg operation
4232 pipe_class ialu_reg(rRegI dst)
4233 %{
4234     single_instruction;
4235     dst    : S4(write);
4236     dst    : S3(read);
4237     DECODE : S0;        // any decoder
4238     ALU    : S3;        // any alu
4239 %}
4240 
4241 // Long ALU reg operation
4242 pipe_class ialu_reg_long(rRegL dst)
4243 %{
4244     instruction_count(2);
4245     dst    : S4(write);
4246     dst    : S3(read);
4247     DECODE : S0(2);     // any 2 decoders
4248     ALU    : S3(2);     // both alus
4249 %}
4250 
4251 // Integer ALU reg operation using big decoder
4252 pipe_class ialu_reg_fat(rRegI dst)
4253 %{
4254     single_instruction;
4255     dst    : S4(write);
4256     dst    : S3(read);
4257     D0     : S0;        // big decoder only
4258     ALU    : S3;        // any alu
4259 %}
4260 
4261 // Long ALU reg operation using big decoder
4262 pipe_class ialu_reg_long_fat(rRegL dst)
4263 %{
4264     instruction_count(2);
4265     dst    : S4(write);
4266     dst    : S3(read);
4267     D0     : S0(2);     // big decoder only; twice
4268     ALU    : S3(2);     // any 2 alus
4269 %}
4270 
4271 // Integer ALU reg-reg operation
4272 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4273 %{
4274     single_instruction;
4275     dst    : S4(write);
4276     src    : S3(read);
4277     DECODE : S0;        // any decoder
4278     ALU    : S3;        // any alu
4279 %}
4280 
4281 // Long ALU reg-reg operation
4282 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4283 %{
4284     instruction_count(2);
4285     dst    : S4(write);
4286     src    : S3(read);
4287     DECODE : S0(2);     // any 2 decoders
4288     ALU    : S3(2);     // both alus
4289 %}
4290 
4291 // Integer ALU reg-reg operation
4292 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4293 %{
4294     single_instruction;
4295     dst    : S4(write);
4296     src    : S3(read);
4297     D0     : S0;        // big decoder only
4298     ALU    : S3;        // any alu
4299 %}
4300 
4301 // Long ALU reg-reg operation
4302 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4303 %{
4304     instruction_count(2);
4305     dst    : S4(write);
4306     src    : S3(read);
4307     D0     : S0(2);     // big decoder only; twice
4308     ALU    : S3(2);     // both alus
4309 %}
4310 
4311 // Integer ALU reg-mem operation
4312 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4313 %{
4314     single_instruction;
4315     dst    : S5(write);
4316     mem    : S3(read);
4317     D0     : S0;        // big decoder only
4318     ALU    : S4;        // any alu
4319     MEM    : S3;        // any mem
4320 %}
4321 
4322 // Integer mem operation (prefetch)
4323 pipe_class ialu_mem(memory mem)
4324 %{
4325     single_instruction;
4326     mem    : S3(read);
4327     D0     : S0;        // big decoder only
4328     MEM    : S3;        // any mem
4329 %}
4330 
4331 // Integer Store to Memory
4332 pipe_class ialu_mem_reg(memory mem, rRegI src)
4333 %{
4334     single_instruction;
4335     mem    : S3(read);
4336     src    : S5(read);
4337     D0     : S0;        // big decoder only
4338     ALU    : S4;        // any alu
4339     MEM    : S3;
4340 %}
4341 
4342 // // Long Store to Memory
4343 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4344 // %{
4345 //     instruction_count(2);
4346 //     mem    : S3(read);
4347 //     src    : S5(read);
4348 //     D0     : S0(2);          // big decoder only; twice
4349 //     ALU    : S4(2);     // any 2 alus
4350 //     MEM    : S3(2);  // Both mems
4351 // %}
4352 
4353 // Integer Store to Memory
4354 pipe_class ialu_mem_imm(memory mem)
4355 %{
4356     single_instruction;
4357     mem    : S3(read);
4358     D0     : S0;        // big decoder only
4359     ALU    : S4;        // any alu
4360     MEM    : S3;
4361 %}
4362 
4363 // Integer ALU0 reg-reg operation
4364 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4365 %{
4366     single_instruction;
4367     dst    : S4(write);
4368     src    : S3(read);
4369     D0     : S0;        // Big decoder only
4370     ALU0   : S3;        // only alu0
4371 %}
4372 
4373 // Integer ALU0 reg-mem operation
4374 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4375 %{
4376     single_instruction;
4377     dst    : S5(write);
4378     mem    : S3(read);
4379     D0     : S0;        // big decoder only
4380     ALU0   : S4;        // ALU0 only
4381     MEM    : S3;        // any mem
4382 %}
4383 
4384 // Integer ALU reg-reg operation
4385 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4386 %{
4387     single_instruction;
4388     cr     : S4(write);
4389     src1   : S3(read);
4390     src2   : S3(read);
4391     DECODE : S0;        // any decoder
4392     ALU    : S3;        // any alu
4393 %}
4394 
4395 // Integer ALU reg-imm operation
4396 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4397 %{
4398     single_instruction;
4399     cr     : S4(write);
4400     src1   : S3(read);
4401     DECODE : S0;        // any decoder
4402     ALU    : S3;        // any alu
4403 %}
4404 
4405 // Integer ALU reg-mem operation
4406 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4407 %{
4408     single_instruction;
4409     cr     : S4(write);
4410     src1   : S3(read);
4411     src2   : S3(read);
4412     D0     : S0;        // big decoder only
4413     ALU    : S4;        // any alu
4414     MEM    : S3;
4415 %}
4416 
4417 // Conditional move reg-reg
4418 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4419 %{
4420     instruction_count(4);
4421     y      : S4(read);
4422     q      : S3(read);
4423     p      : S3(read);
4424     DECODE : S0(4);     // any decoder
4425 %}
4426 
4427 // Conditional move reg-reg
4428 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4429 %{
4430     single_instruction;
4431     dst    : S4(write);
4432     src    : S3(read);
4433     cr     : S3(read);
4434     DECODE : S0;        // any decoder
4435 %}
4436 
4437 // Conditional move reg-mem
4438 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4439 %{
4440     single_instruction;
4441     dst    : S4(write);
4442     src    : S3(read);
4443     cr     : S3(read);
4444     DECODE : S0;        // any decoder
4445     MEM    : S3;
4446 %}
4447 
4448 // Conditional move reg-reg long
4449 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4450 %{
4451     single_instruction;
4452     dst    : S4(write);
4453     src    : S3(read);
4454     cr     : S3(read);
4455     DECODE : S0(2);     // any 2 decoders
4456 %}
4457 
4458 // XXX
4459 // // Conditional move double reg-reg
4460 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4461 // %{
4462 //     single_instruction;
4463 //     dst    : S4(write);
4464 //     src    : S3(read);
4465 //     cr     : S3(read);
4466 //     DECODE : S0;     // any decoder
4467 // %}
4468 
4469 // Float reg-reg operation
4470 pipe_class fpu_reg(regD dst)
4471 %{
4472     instruction_count(2);
4473     dst    : S3(read);
4474     DECODE : S0(2);     // any 2 decoders
4475     FPU    : S3;
4476 %}
4477 
4478 // Float reg-reg operation
4479 pipe_class fpu_reg_reg(regD dst, regD src)
4480 %{
4481     instruction_count(2);
4482     dst    : S4(write);
4483     src    : S3(read);
4484     DECODE : S0(2);     // any 2 decoders
4485     FPU    : S3;
4486 %}
4487 
4488 // Float reg-reg operation
4489 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4490 %{
4491     instruction_count(3);
4492     dst    : S4(write);
4493     src1   : S3(read);
4494     src2   : S3(read);
4495     DECODE : S0(3);     // any 3 decoders
4496     FPU    : S3(2);
4497 %}
4498 
4499 // Float reg-reg operation
4500 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4501 %{
4502     instruction_count(4);
4503     dst    : S4(write);
4504     src1   : S3(read);
4505     src2   : S3(read);
4506     src3   : S3(read);
4507     DECODE : S0(4);     // any 3 decoders
4508     FPU    : S3(2);
4509 %}
4510 
4511 // Float reg-reg operation
4512 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4513 %{
4514     instruction_count(4);
4515     dst    : S4(write);
4516     src1   : S3(read);
4517     src2   : S3(read);
4518     src3   : S3(read);
4519     DECODE : S1(3);     // any 3 decoders
4520     D0     : S0;        // Big decoder only
4521     FPU    : S3(2);
4522     MEM    : S3;
4523 %}
4524 
4525 // Float reg-mem operation
4526 pipe_class fpu_reg_mem(regD dst, memory mem)
4527 %{
4528     instruction_count(2);
4529     dst    : S5(write);
4530     mem    : S3(read);
4531     D0     : S0;        // big decoder only
4532     DECODE : S1;        // any decoder for FPU POP
4533     FPU    : S4;
4534     MEM    : S3;        // any mem
4535 %}
4536 
4537 // Float reg-mem operation
4538 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4539 %{
4540     instruction_count(3);
4541     dst    : S5(write);
4542     src1   : S3(read);
4543     mem    : S3(read);
4544     D0     : S0;        // big decoder only
4545     DECODE : S1(2);     // any decoder for FPU POP
4546     FPU    : S4;
4547     MEM    : S3;        // any mem
4548 %}
4549 
4550 // Float mem-reg operation
4551 pipe_class fpu_mem_reg(memory mem, regD src)
4552 %{
4553     instruction_count(2);
4554     src    : S5(read);
4555     mem    : S3(read);
4556     DECODE : S0;        // any decoder for FPU PUSH
4557     D0     : S1;        // big decoder only
4558     FPU    : S4;
4559     MEM    : S3;        // any mem
4560 %}
4561 
4562 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4563 %{
4564     instruction_count(3);
4565     src1   : S3(read);
4566     src2   : S3(read);
4567     mem    : S3(read);
4568     DECODE : S0(2);     // any decoder for FPU PUSH
4569     D0     : S1;        // big decoder only
4570     FPU    : S4;
4571     MEM    : S3;        // any mem
4572 %}
4573 
4574 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4575 %{
4576     instruction_count(3);
4577     src1   : S3(read);
4578     src2   : S3(read);
4579     mem    : S4(read);
4580     DECODE : S0;        // any decoder for FPU PUSH
4581     D0     : S0(2);     // big decoder only
4582     FPU    : S4;
4583     MEM    : S3(2);     // any mem
4584 %}
4585 
4586 pipe_class fpu_mem_mem(memory dst, memory src1)
4587 %{
4588     instruction_count(2);
4589     src1   : S3(read);
4590     dst    : S4(read);
4591     D0     : S0(2);     // big decoder only
4592     MEM    : S3(2);     // any mem
4593 %}
4594 
4595 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4596 %{
4597     instruction_count(3);
4598     src1   : S3(read);
4599     src2   : S3(read);
4600     dst    : S4(read);
4601     D0     : S0(3);     // big decoder only
4602     FPU    : S4;
4603     MEM    : S3(3);     // any mem
4604 %}
4605 
4606 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4607 %{
4608     instruction_count(3);
4609     src1   : S4(read);
4610     mem    : S4(read);
4611     DECODE : S0;        // any decoder for FPU PUSH
4612     D0     : S0(2);     // big decoder only
4613     FPU    : S4;
4614     MEM    : S3(2);     // any mem
4615 %}
4616 
4617 // Float load constant
4618 pipe_class fpu_reg_con(regD dst)
4619 %{
4620     instruction_count(2);
4621     dst    : S5(write);
4622     D0     : S0;        // big decoder only for the load
4623     DECODE : S1;        // any decoder for FPU POP
4624     FPU    : S4;
4625     MEM    : S3;        // any mem
4626 %}
4627 
4628 // Float load constant
4629 pipe_class fpu_reg_reg_con(regD dst, regD src)
4630 %{
4631     instruction_count(3);
4632     dst    : S5(write);
4633     src    : S3(read);
4634     D0     : S0;        // big decoder only for the load
4635     DECODE : S1(2);     // any decoder for FPU POP
4636     FPU    : S4;
4637     MEM    : S3;        // any mem
4638 %}
4639 
4640 // UnConditional branch
4641 pipe_class pipe_jmp(label labl)
4642 %{
4643     single_instruction;
4644     BR   : S3;
4645 %}
4646 
4647 // Conditional branch
4648 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4649 %{
4650     single_instruction;
4651     cr    : S1(read);
4652     BR    : S3;
4653 %}
4654 
4655 // Allocation idiom
4656 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4657 %{
4658     instruction_count(1); force_serialization;
4659     fixed_latency(6);
4660     heap_ptr : S3(read);
4661     DECODE   : S0(3);
4662     D0       : S2;
4663     MEM      : S3;
4664     ALU      : S3(2);
4665     dst      : S5(write);
4666     BR       : S5;
4667 %}
4668 
4669 // Generic big/slow expanded idiom
4670 pipe_class pipe_slow()
4671 %{
4672     instruction_count(10); multiple_bundles; force_serialization;
4673     fixed_latency(100);
4674     D0  : S0(2);
4675     MEM : S3(2);
4676 %}
4677 
4678 // The real do-nothing guy
4679 pipe_class empty()
4680 %{
4681     instruction_count(0);
4682 %}
4683 
4684 // Define the class for the Nop node
4685 define
4686 %{
4687    MachNop = empty;
4688 %}
4689 
4690 %}
4691 
4692 //----------INSTRUCTIONS-------------------------------------------------------
4693 //
4694 // match      -- States which machine-independent subtree may be replaced
4695 //               by this instruction.
4696 // ins_cost   -- The estimated cost of this instruction is used by instruction
4697 //               selection to identify a minimum cost tree of machine
4698 //               instructions that matches a tree of machine-independent
4699 //               instructions.
4700 // format     -- A string providing the disassembly for this instruction.
4701 //               The value of an instruction's operand may be inserted
4702 //               by referring to it with a '$' prefix.
4703 // opcode     -- Three instruction opcodes may be provided.  These are referred
4704 //               to within an encode class as $primary, $secondary, and $tertiary
4705 //               rrspectively.  The primary opcode is commonly used to
4706 //               indicate the type of machine instruction, while secondary
4707 //               and tertiary are often used for prefix options or addressing
4708 //               modes.
4709 // ins_encode -- A list of encode classes with parameters. The encode class
4710 //               name must have been defined in an 'enc_class' specification
4711 //               in the encode section of the architecture description.
4712 
4713 
4714 //----------Load/Store/Move Instructions---------------------------------------
4715 //----------Load Instructions--------------------------------------------------
4716 
4717 // Load Byte (8 bit signed)
4718 instruct loadB(rRegI dst, memory mem)
4719 %{
4720   match(Set dst (LoadB mem));
4721 
4722   ins_cost(125);
4723   format %{ "movsbl  $dst, $mem\t# byte" %}
4724 
4725   ins_encode %{
4726     __ movsbl($dst$$Register, $mem$$Address);
4727   %}
4728 
4729   ins_pipe(ialu_reg_mem);
4730 %}
4731 
4732 // Load Byte (8 bit signed) into Long Register
4733 instruct loadB2L(rRegL dst, memory mem)
4734 %{
4735   match(Set dst (ConvI2L (LoadB mem)));
4736 
4737   ins_cost(125);
4738   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4739 
4740   ins_encode %{
4741     __ movsbq($dst$$Register, $mem$$Address);
4742   %}
4743 
4744   ins_pipe(ialu_reg_mem);
4745 %}
4746 
4747 // Load Unsigned Byte (8 bit UNsigned)
4748 instruct loadUB(rRegI dst, memory mem)
4749 %{
4750   match(Set dst (LoadUB mem));
4751 
4752   ins_cost(125);
4753   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4754 
4755   ins_encode %{
4756     __ movzbl($dst$$Register, $mem$$Address);
4757   %}
4758 
4759   ins_pipe(ialu_reg_mem);
4760 %}
4761 
4762 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4763 instruct loadUB2L(rRegL dst, memory mem)
4764 %{
4765   match(Set dst (ConvI2L (LoadUB mem)));
4766 
4767   ins_cost(125);
4768   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4769 
4770   ins_encode %{
4771     __ movzbq($dst$$Register, $mem$$Address);
4772   %}
4773 
4774   ins_pipe(ialu_reg_mem);
4775 %}
4776 
4777 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4778 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4779   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4780   effect(KILL cr);
4781 
4782   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4783             "andl    $dst, right_n_bits($mask, 8)" %}
4784   ins_encode %{
4785     Register Rdst = $dst$$Register;
4786     __ movzbq(Rdst, $mem$$Address);
4787     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4788   %}
4789   ins_pipe(ialu_reg_mem);
4790 %}
4791 
4792 // Load Short (16 bit signed)
4793 instruct loadS(rRegI dst, memory mem)
4794 %{
4795   match(Set dst (LoadS mem));
4796 
4797   ins_cost(125);
4798   format %{ "movswl $dst, $mem\t# short" %}
4799 
4800   ins_encode %{
4801     __ movswl($dst$$Register, $mem$$Address);
4802   %}
4803 
4804   ins_pipe(ialu_reg_mem);
4805 %}
4806 
4807 // Load Short (16 bit signed) to Byte (8 bit signed)
4808 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4809   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4810 
4811   ins_cost(125);
4812   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4813   ins_encode %{
4814     __ movsbl($dst$$Register, $mem$$Address);
4815   %}
4816   ins_pipe(ialu_reg_mem);
4817 %}
4818 
4819 // Load Short (16 bit signed) into Long Register
4820 instruct loadS2L(rRegL dst, memory mem)
4821 %{
4822   match(Set dst (ConvI2L (LoadS mem)));
4823 
4824   ins_cost(125);
4825   format %{ "movswq $dst, $mem\t# short -> long" %}
4826 
4827   ins_encode %{
4828     __ movswq($dst$$Register, $mem$$Address);
4829   %}
4830 
4831   ins_pipe(ialu_reg_mem);
4832 %}
4833 
4834 // Load Unsigned Short/Char (16 bit UNsigned)
4835 instruct loadUS(rRegI dst, memory mem)
4836 %{
4837   match(Set dst (LoadUS mem));
4838 
4839   ins_cost(125);
4840   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4841 
4842   ins_encode %{
4843     __ movzwl($dst$$Register, $mem$$Address);
4844   %}
4845 
4846   ins_pipe(ialu_reg_mem);
4847 %}
4848 
4849 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4850 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4851   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4852 
4853   ins_cost(125);
4854   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4855   ins_encode %{
4856     __ movsbl($dst$$Register, $mem$$Address);
4857   %}
4858   ins_pipe(ialu_reg_mem);
4859 %}
4860 
4861 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4862 instruct loadUS2L(rRegL dst, memory mem)
4863 %{
4864   match(Set dst (ConvI2L (LoadUS mem)));
4865 
4866   ins_cost(125);
4867   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4868 
4869   ins_encode %{
4870     __ movzwq($dst$$Register, $mem$$Address);
4871   %}
4872 
4873   ins_pipe(ialu_reg_mem);
4874 %}
4875 
4876 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4877 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4878   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4879 
4880   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4881   ins_encode %{
4882     __ movzbq($dst$$Register, $mem$$Address);
4883   %}
4884   ins_pipe(ialu_reg_mem);
4885 %}
4886 
4887 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4888 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4889   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4890   effect(KILL cr);
4891 
4892   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4893             "andl    $dst, right_n_bits($mask, 16)" %}
4894   ins_encode %{
4895     Register Rdst = $dst$$Register;
4896     __ movzwq(Rdst, $mem$$Address);
4897     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4898   %}
4899   ins_pipe(ialu_reg_mem);
4900 %}
4901 
4902 // Load Integer
4903 instruct loadI(rRegI dst, memory mem)
4904 %{
4905   match(Set dst (LoadI mem));
4906 
4907   ins_cost(125);
4908   format %{ "movl    $dst, $mem\t# int" %}
4909 
4910   ins_encode %{
4911     __ movl($dst$$Register, $mem$$Address);
4912   %}
4913 
4914   ins_pipe(ialu_reg_mem);
4915 %}
4916 
4917 // Load Integer (32 bit signed) to Byte (8 bit signed)
4918 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4919   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4920 
4921   ins_cost(125);
4922   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4923   ins_encode %{
4924     __ movsbl($dst$$Register, $mem$$Address);
4925   %}
4926   ins_pipe(ialu_reg_mem);
4927 %}
4928 
4929 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4930 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4931   match(Set dst (AndI (LoadI mem) mask));
4932 
4933   ins_cost(125);
4934   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4935   ins_encode %{
4936     __ movzbl($dst$$Register, $mem$$Address);
4937   %}
4938   ins_pipe(ialu_reg_mem);
4939 %}
4940 
4941 // Load Integer (32 bit signed) to Short (16 bit signed)
4942 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4943   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4944 
4945   ins_cost(125);
4946   format %{ "movswl  $dst, $mem\t# int -> short" %}
4947   ins_encode %{
4948     __ movswl($dst$$Register, $mem$$Address);
4949   %}
4950   ins_pipe(ialu_reg_mem);
4951 %}
4952 
4953 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4954 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4955   match(Set dst (AndI (LoadI mem) mask));
4956 
4957   ins_cost(125);
4958   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4959   ins_encode %{
4960     __ movzwl($dst$$Register, $mem$$Address);
4961   %}
4962   ins_pipe(ialu_reg_mem);
4963 %}
4964 
4965 // Load Integer into Long Register
4966 instruct loadI2L(rRegL dst, memory mem)
4967 %{
4968   match(Set dst (ConvI2L (LoadI mem)));
4969 
4970   ins_cost(125);
4971   format %{ "movslq  $dst, $mem\t# int -> long" %}
4972 
4973   ins_encode %{
4974     __ movslq($dst$$Register, $mem$$Address);
4975   %}
4976 
4977   ins_pipe(ialu_reg_mem);
4978 %}
4979 
4980 // Load Integer with mask 0xFF into Long Register
4981 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4982   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4983 
4984   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4985   ins_encode %{
4986     __ movzbq($dst$$Register, $mem$$Address);
4987   %}
4988   ins_pipe(ialu_reg_mem);
4989 %}
4990 
4991 // Load Integer with mask 0xFFFF into Long Register
4992 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4993   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4994 
4995   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4996   ins_encode %{
4997     __ movzwq($dst$$Register, $mem$$Address);
4998   %}
4999   ins_pipe(ialu_reg_mem);
5000 %}
5001 
5002 // Load Integer with a 31-bit mask into Long Register
5003 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5004   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5005   effect(KILL cr);
5006 
5007   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5008             "andl    $dst, $mask" %}
5009   ins_encode %{
5010     Register Rdst = $dst$$Register;
5011     __ movl(Rdst, $mem$$Address);
5012     __ andl(Rdst, $mask$$constant);
5013   %}
5014   ins_pipe(ialu_reg_mem);
5015 %}
5016 
5017 // Load Unsigned Integer into Long Register
5018 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5019 %{
5020   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5021 
5022   ins_cost(125);
5023   format %{ "movl    $dst, $mem\t# uint -> long" %}
5024 
5025   ins_encode %{
5026     __ movl($dst$$Register, $mem$$Address);
5027   %}
5028 
5029   ins_pipe(ialu_reg_mem);
5030 %}
5031 
5032 // Load Long
5033 instruct loadL(rRegL dst, memory mem)
5034 %{
5035   match(Set dst (LoadL mem));
5036 
5037   ins_cost(125);
5038   format %{ "movq    $dst, $mem\t# long" %}
5039 
5040   ins_encode %{
5041     __ movq($dst$$Register, $mem$$Address);
5042   %}
5043 
5044   ins_pipe(ialu_reg_mem); // XXX
5045 %}
5046 
5047 // Load Range
5048 instruct loadRange(rRegI dst, memory mem)
5049 %{
5050   match(Set dst (LoadRange mem));
5051 
5052   ins_cost(125); // XXX
5053   format %{ "movl    $dst, $mem\t# range" %}
5054   opcode(0x8B);
5055   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5056   ins_pipe(ialu_reg_mem);
5057 %}
5058 
5059 // Load Pointer
5060 instruct loadP(rRegP dst, memory mem)
5061 %{
5062   match(Set dst (LoadP mem));
5063 
5064   ins_cost(125); // XXX
5065   format %{ "movq    $dst, $mem\t# ptr" %}
5066   opcode(0x8B);
5067   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5068   ins_pipe(ialu_reg_mem); // XXX
5069 %}
5070 
5071 // Load Compressed Pointer
5072 instruct loadN(rRegN dst, memory mem)
5073 %{
5074    match(Set dst (LoadN mem));
5075 
5076    ins_cost(125); // XXX
5077    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5078    ins_encode %{
5079      __ movl($dst$$Register, $mem$$Address);
5080    %}
5081    ins_pipe(ialu_reg_mem); // XXX
5082 %}
5083 
5084 
5085 // Load Klass Pointer
5086 instruct loadKlass(rRegP dst, memory mem)
5087 %{
5088   match(Set dst (LoadKlass mem));
5089 
5090   ins_cost(125); // XXX
5091   format %{ "movq    $dst, $mem\t# class" %}
5092   opcode(0x8B);
5093   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5094   ins_pipe(ialu_reg_mem); // XXX
5095 %}
5096 
5097 // Load narrow Klass Pointer
5098 instruct loadNKlass(rRegN dst, memory mem)
5099 %{
5100   match(Set dst (LoadNKlass mem));
5101 
5102   ins_cost(125); // XXX
5103   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5104   ins_encode %{
5105     __ movl($dst$$Register, $mem$$Address);
5106   %}
5107   ins_pipe(ialu_reg_mem); // XXX
5108 %}
5109 
5110 // Load Float
5111 instruct loadF(regF dst, memory mem)
5112 %{
5113   match(Set dst (LoadF mem));
5114 
5115   ins_cost(145); // XXX
5116   format %{ "movss   $dst, $mem\t# float" %}
5117   ins_encode %{
5118     __ movflt($dst$$XMMRegister, $mem$$Address);
5119   %}
5120   ins_pipe(pipe_slow); // XXX
5121 %}
5122 
5123 // Load Double
5124 instruct loadD_partial(regD dst, memory mem)
5125 %{
5126   predicate(!UseXmmLoadAndClearUpper);
5127   match(Set dst (LoadD mem));
5128 
5129   ins_cost(145); // XXX
5130   format %{ "movlpd  $dst, $mem\t# double" %}
5131   ins_encode %{
5132     __ movdbl($dst$$XMMRegister, $mem$$Address);
5133   %}
5134   ins_pipe(pipe_slow); // XXX
5135 %}
5136 
5137 instruct loadD(regD dst, memory mem)
5138 %{
5139   predicate(UseXmmLoadAndClearUpper);
5140   match(Set dst (LoadD mem));
5141 
5142   ins_cost(145); // XXX
5143   format %{ "movsd   $dst, $mem\t# double" %}
5144   ins_encode %{
5145     __ movdbl($dst$$XMMRegister, $mem$$Address);
5146   %}
5147   ins_pipe(pipe_slow); // XXX
5148 %}
5149 
5150 // Load Effective Address
5151 instruct leaP8(rRegP dst, indOffset8 mem)
5152 %{
5153   match(Set dst mem);
5154 
5155   ins_cost(110); // XXX
5156   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5157   opcode(0x8D);
5158   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5159   ins_pipe(ialu_reg_reg_fat);
5160 %}
5161 
5162 instruct leaP32(rRegP dst, indOffset32 mem)
5163 %{
5164   match(Set dst mem);
5165 
5166   ins_cost(110);
5167   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5168   opcode(0x8D);
5169   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5170   ins_pipe(ialu_reg_reg_fat);
5171 %}
5172 
5173 // instruct leaPIdx(rRegP dst, indIndex mem)
5174 // %{
5175 //   match(Set dst mem);
5176 
5177 //   ins_cost(110);
5178 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5179 //   opcode(0x8D);
5180 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5181 //   ins_pipe(ialu_reg_reg_fat);
5182 // %}
5183 
5184 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5185 %{
5186   match(Set dst mem);
5187 
5188   ins_cost(110);
5189   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5190   opcode(0x8D);
5191   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5192   ins_pipe(ialu_reg_reg_fat);
5193 %}
5194 
5195 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5196 %{
5197   match(Set dst mem);
5198 
5199   ins_cost(110);
5200   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5201   opcode(0x8D);
5202   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5203   ins_pipe(ialu_reg_reg_fat);
5204 %}
5205 
5206 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5207 %{
5208   match(Set dst mem);
5209 
5210   ins_cost(110);
5211   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5212   opcode(0x8D);
5213   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5214   ins_pipe(ialu_reg_reg_fat);
5215 %}
5216 
5217 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5218 %{
5219   match(Set dst mem);
5220 
5221   ins_cost(110);
5222   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5223   opcode(0x8D);
5224   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5225   ins_pipe(ialu_reg_reg_fat);
5226 %}
5227 
5228 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5229 %{
5230   match(Set dst mem);
5231 
5232   ins_cost(110);
5233   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5234   opcode(0x8D);
5235   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5236   ins_pipe(ialu_reg_reg_fat);
5237 %}
5238 
5239 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5240 %{
5241   match(Set dst mem);
5242 
5243   ins_cost(110);
5244   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5245   opcode(0x8D);
5246   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5247   ins_pipe(ialu_reg_reg_fat);
5248 %}
5249 
5250 // Load Effective Address which uses Narrow (32-bits) oop
5251 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5252 %{
5253   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5254   match(Set dst mem);
5255 
5256   ins_cost(110);
5257   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5258   opcode(0x8D);
5259   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5260   ins_pipe(ialu_reg_reg_fat);
5261 %}
5262 
5263 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5264 %{
5265   predicate(Universe::narrow_oop_shift() == 0);
5266   match(Set dst mem);
5267 
5268   ins_cost(110); // XXX
5269   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5270   opcode(0x8D);
5271   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5272   ins_pipe(ialu_reg_reg_fat);
5273 %}
5274 
5275 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5276 %{
5277   predicate(Universe::narrow_oop_shift() == 0);
5278   match(Set dst mem);
5279 
5280   ins_cost(110);
5281   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5282   opcode(0x8D);
5283   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5284   ins_pipe(ialu_reg_reg_fat);
5285 %}
5286 
5287 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5288 %{
5289   predicate(Universe::narrow_oop_shift() == 0);
5290   match(Set dst mem);
5291 
5292   ins_cost(110);
5293   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5294   opcode(0x8D);
5295   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5296   ins_pipe(ialu_reg_reg_fat);
5297 %}
5298 
5299 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5300 %{
5301   predicate(Universe::narrow_oop_shift() == 0);
5302   match(Set dst mem);
5303 
5304   ins_cost(110);
5305   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5306   opcode(0x8D);
5307   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5308   ins_pipe(ialu_reg_reg_fat);
5309 %}
5310 
5311 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5312 %{
5313   predicate(Universe::narrow_oop_shift() == 0);
5314   match(Set dst mem);
5315 
5316   ins_cost(110);
5317   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5318   opcode(0x8D);
5319   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5320   ins_pipe(ialu_reg_reg_fat);
5321 %}
5322 
5323 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5324 %{
5325   predicate(Universe::narrow_oop_shift() == 0);
5326   match(Set dst mem);
5327 
5328   ins_cost(110);
5329   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5330   opcode(0x8D);
5331   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5332   ins_pipe(ialu_reg_reg_fat);
5333 %}
5334 
5335 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5336 %{
5337   predicate(Universe::narrow_oop_shift() == 0);
5338   match(Set dst mem);
5339 
5340   ins_cost(110);
5341   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5342   opcode(0x8D);
5343   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5344   ins_pipe(ialu_reg_reg_fat);
5345 %}
5346 
5347 instruct loadConI(rRegI dst, immI src)
5348 %{
5349   match(Set dst src);
5350 
5351   format %{ "movl    $dst, $src\t# int" %}
5352   ins_encode(load_immI(dst, src));
5353   ins_pipe(ialu_reg_fat); // XXX
5354 %}
5355 
5356 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5357 %{
5358   match(Set dst src);
5359   effect(KILL cr);
5360 
5361   ins_cost(50);
5362   format %{ "xorl    $dst, $dst\t# int" %}
5363   opcode(0x33); /* + rd */
5364   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5365   ins_pipe(ialu_reg);
5366 %}
5367 
5368 instruct loadConL(rRegL dst, immL src)
5369 %{
5370   match(Set dst src);
5371 
5372   ins_cost(150);
5373   format %{ "movq    $dst, $src\t# long" %}
5374   ins_encode(load_immL(dst, src));
5375   ins_pipe(ialu_reg);
5376 %}
5377 
5378 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5379 %{
5380   match(Set dst src);
5381   effect(KILL cr);
5382 
5383   ins_cost(50);
5384   format %{ "xorl    $dst, $dst\t# long" %}
5385   opcode(0x33); /* + rd */
5386   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5387   ins_pipe(ialu_reg); // XXX
5388 %}
5389 
5390 instruct loadConUL32(rRegL dst, immUL32 src)
5391 %{
5392   match(Set dst src);
5393 
5394   ins_cost(60);
5395   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5396   ins_encode(load_immUL32(dst, src));
5397   ins_pipe(ialu_reg);
5398 %}
5399 
5400 instruct loadConL32(rRegL dst, immL32 src)
5401 %{
5402   match(Set dst src);
5403 
5404   ins_cost(70);
5405   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5406   ins_encode(load_immL32(dst, src));
5407   ins_pipe(ialu_reg);
5408 %}
5409 
5410 instruct loadConP(rRegP dst, immP con) %{
5411   match(Set dst con);
5412 
5413   format %{ "movq    $dst, $con\t# ptr" %}
5414   ins_encode(load_immP(dst, con));
5415   ins_pipe(ialu_reg_fat); // XXX
5416 %}
5417 
5418 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5419 %{
5420   match(Set dst src);
5421   effect(KILL cr);
5422 
5423   ins_cost(50);
5424   format %{ "xorl    $dst, $dst\t# ptr" %}
5425   opcode(0x33); /* + rd */
5426   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5427   ins_pipe(ialu_reg);
5428 %}
5429 
5430 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5431 %{
5432   match(Set dst src);
5433   effect(KILL cr);
5434 
5435   ins_cost(60);
5436   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5437   ins_encode(load_immP31(dst, src));
5438   ins_pipe(ialu_reg);
5439 %}
5440 
5441 instruct loadConF(regF dst, immF con) %{
5442   match(Set dst con);
5443   ins_cost(125);
5444   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5445   ins_encode %{
5446     __ movflt($dst$$XMMRegister, $constantaddress($con));
5447   %}
5448   ins_pipe(pipe_slow);
5449 %}
5450 
5451 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5452   match(Set dst src);
5453   effect(KILL cr);
5454   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5455   ins_encode %{
5456     __ xorq($dst$$Register, $dst$$Register);
5457   %}
5458   ins_pipe(ialu_reg);
5459 %}
5460 
5461 instruct loadConN(rRegN dst, immN src) %{
5462   match(Set dst src);
5463 
5464   ins_cost(125);
5465   format %{ "movl    $dst, $src\t# compressed ptr" %}
5466   ins_encode %{
5467     address con = (address)$src$$constant;
5468     if (con == NULL) {
5469       ShouldNotReachHere();
5470     } else {
5471       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5472     }
5473   %}
5474   ins_pipe(ialu_reg_fat); // XXX
5475 %}
5476 
5477 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5478   match(Set dst src);
5479 
5480   ins_cost(125);
5481   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5482   ins_encode %{
5483     address con = (address)$src$$constant;
5484     if (con == NULL) {
5485       ShouldNotReachHere();
5486     } else {
5487       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5488     }
5489   %}
5490   ins_pipe(ialu_reg_fat); // XXX
5491 %}
5492 
5493 instruct loadConF0(regF dst, immF0 src)
5494 %{
5495   match(Set dst src);
5496   ins_cost(100);
5497 
5498   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5499   ins_encode %{
5500     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5501   %}
5502   ins_pipe(pipe_slow);
5503 %}
5504 
5505 // Use the same format since predicate() can not be used here.
5506 instruct loadConD(regD dst, immD con) %{
5507   match(Set dst con);
5508   ins_cost(125);
5509   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5510   ins_encode %{
5511     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5512   %}
5513   ins_pipe(pipe_slow);
5514 %}
5515 
5516 instruct loadConD0(regD dst, immD0 src)
5517 %{
5518   match(Set dst src);
5519   ins_cost(100);
5520 
5521   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5522   ins_encode %{
5523     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5524   %}
5525   ins_pipe(pipe_slow);
5526 %}
5527 
5528 instruct loadSSI(rRegI dst, stackSlotI src)
5529 %{
5530   match(Set dst src);
5531 
5532   ins_cost(125);
5533   format %{ "movl    $dst, $src\t# int stk" %}
5534   opcode(0x8B);
5535   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5536   ins_pipe(ialu_reg_mem);
5537 %}
5538 
5539 instruct loadSSL(rRegL dst, stackSlotL src)
5540 %{
5541   match(Set dst src);
5542 
5543   ins_cost(125);
5544   format %{ "movq    $dst, $src\t# long stk" %}
5545   opcode(0x8B);
5546   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5547   ins_pipe(ialu_reg_mem);
5548 %}
5549 
5550 instruct loadSSP(rRegP dst, stackSlotP src)
5551 %{
5552   match(Set dst src);
5553 
5554   ins_cost(125);
5555   format %{ "movq    $dst, $src\t# ptr stk" %}
5556   opcode(0x8B);
5557   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5558   ins_pipe(ialu_reg_mem);
5559 %}
5560 
5561 instruct loadSSF(regF dst, stackSlotF src)
5562 %{
5563   match(Set dst src);
5564 
5565   ins_cost(125);
5566   format %{ "movss   $dst, $src\t# float stk" %}
5567   ins_encode %{
5568     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5569   %}
5570   ins_pipe(pipe_slow); // XXX
5571 %}
5572 
5573 // Use the same format since predicate() can not be used here.
5574 instruct loadSSD(regD dst, stackSlotD src)
5575 %{
5576   match(Set dst src);
5577 
5578   ins_cost(125);
5579   format %{ "movsd   $dst, $src\t# double stk" %}
5580   ins_encode  %{
5581     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5582   %}
5583   ins_pipe(pipe_slow); // XXX
5584 %}
5585 
5586 // Prefetch instructions for allocation.
5587 // Must be safe to execute with invalid address (cannot fault).
5588 
5589 instruct prefetchAlloc( memory mem ) %{
5590   predicate(AllocatePrefetchInstr==3);
5591   match(PrefetchAllocation mem);
5592   ins_cost(125);
5593 
5594   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5595   ins_encode %{
5596     __ prefetchw($mem$$Address);
5597   %}
5598   ins_pipe(ialu_mem);
5599 %}
5600 
5601 instruct prefetchAllocNTA( memory mem ) %{
5602   predicate(AllocatePrefetchInstr==0);
5603   match(PrefetchAllocation mem);
5604   ins_cost(125);
5605 
5606   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5607   ins_encode %{
5608     __ prefetchnta($mem$$Address);
5609   %}
5610   ins_pipe(ialu_mem);
5611 %}
5612 
5613 instruct prefetchAllocT0( memory mem ) %{
5614   predicate(AllocatePrefetchInstr==1);
5615   match(PrefetchAllocation mem);
5616   ins_cost(125);
5617 
5618   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5619   ins_encode %{
5620     __ prefetcht0($mem$$Address);
5621   %}
5622   ins_pipe(ialu_mem);
5623 %}
5624 
5625 instruct prefetchAllocT2( memory mem ) %{
5626   predicate(AllocatePrefetchInstr==2);
5627   match(PrefetchAllocation mem);
5628   ins_cost(125);
5629 
5630   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5631   ins_encode %{
5632     __ prefetcht2($mem$$Address);
5633   %}
5634   ins_pipe(ialu_mem);
5635 %}
5636 
5637 //----------Store Instructions-------------------------------------------------
5638 
5639 // Store Byte
5640 instruct storeB(memory mem, rRegI src)
5641 %{
5642   match(Set mem (StoreB mem src));
5643 
5644   ins_cost(125); // XXX
5645   format %{ "movb    $mem, $src\t# byte" %}
5646   opcode(0x88);
5647   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5648   ins_pipe(ialu_mem_reg);
5649 %}
5650 
5651 // Store Char/Short
5652 instruct storeC(memory mem, rRegI src)
5653 %{
5654   match(Set mem (StoreC mem src));
5655 
5656   ins_cost(125); // XXX
5657   format %{ "movw    $mem, $src\t# char/short" %}
5658   opcode(0x89);
5659   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5660   ins_pipe(ialu_mem_reg);
5661 %}
5662 
5663 // Store Integer
5664 instruct storeI(memory mem, rRegI src)
5665 %{
5666   match(Set mem (StoreI mem src));
5667 
5668   ins_cost(125); // XXX
5669   format %{ "movl    $mem, $src\t# int" %}
5670   opcode(0x89);
5671   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5672   ins_pipe(ialu_mem_reg);
5673 %}
5674 
5675 // Store Long
5676 instruct storeL(memory mem, rRegL src)
5677 %{
5678   match(Set mem (StoreL mem src));
5679 
5680   ins_cost(125); // XXX
5681   format %{ "movq    $mem, $src\t# long" %}
5682   opcode(0x89);
5683   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5684   ins_pipe(ialu_mem_reg); // XXX
5685 %}
5686 
5687 // Store Pointer
5688 instruct storeP(memory mem, any_RegP src)
5689 %{
5690   match(Set mem (StoreP mem src));
5691 
5692   ins_cost(125); // XXX
5693   format %{ "movq    $mem, $src\t# ptr" %}
5694   opcode(0x89);
5695   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5696   ins_pipe(ialu_mem_reg);
5697 %}
5698 
5699 instruct storeImmP0(memory mem, immP0 zero)
5700 %{
5701   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5702   match(Set mem (StoreP mem zero));
5703 
5704   ins_cost(125); // XXX
5705   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5706   ins_encode %{
5707     __ movq($mem$$Address, r12);
5708   %}
5709   ins_pipe(ialu_mem_reg);
5710 %}
5711 
5712 // Store NULL Pointer, mark word, or other simple pointer constant.
5713 instruct storeImmP(memory mem, immP31 src)
5714 %{
5715   match(Set mem (StoreP mem src));
5716 
5717   ins_cost(150); // XXX
5718   format %{ "movq    $mem, $src\t# ptr" %}
5719   opcode(0xC7); /* C7 /0 */
5720   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5721   ins_pipe(ialu_mem_imm);
5722 %}
5723 
5724 // Store Compressed Pointer
5725 instruct storeN(memory mem, rRegN src)
5726 %{
5727   match(Set mem (StoreN mem src));
5728 
5729   ins_cost(125); // XXX
5730   format %{ "movl    $mem, $src\t# compressed ptr" %}
5731   ins_encode %{
5732     __ movl($mem$$Address, $src$$Register);
5733   %}
5734   ins_pipe(ialu_mem_reg);
5735 %}
5736 
5737 instruct storeNKlass(memory mem, rRegN src)
5738 %{
5739   match(Set mem (StoreNKlass mem src));
5740 
5741   ins_cost(125); // XXX
5742   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5743   ins_encode %{
5744     __ movl($mem$$Address, $src$$Register);
5745   %}
5746   ins_pipe(ialu_mem_reg);
5747 %}
5748 
5749 instruct storeImmN0(memory mem, immN0 zero)
5750 %{
5751   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5752   match(Set mem (StoreN mem zero));
5753 
5754   ins_cost(125); // XXX
5755   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5756   ins_encode %{
5757     __ movl($mem$$Address, r12);
5758   %}
5759   ins_pipe(ialu_mem_reg);
5760 %}
5761 
5762 instruct storeImmN(memory mem, immN src)
5763 %{
5764   match(Set mem (StoreN mem src));
5765 
5766   ins_cost(150); // XXX
5767   format %{ "movl    $mem, $src\t# compressed ptr" %}
5768   ins_encode %{
5769     address con = (address)$src$$constant;
5770     if (con == NULL) {
5771       __ movl($mem$$Address, (int32_t)0);
5772     } else {
5773       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5774     }
5775   %}
5776   ins_pipe(ialu_mem_imm);
5777 %}
5778 
5779 instruct storeImmNKlass(memory mem, immNKlass src)
5780 %{
5781   match(Set mem (StoreNKlass mem src));
5782 
5783   ins_cost(150); // XXX
5784   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5785   ins_encode %{
5786     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5787   %}
5788   ins_pipe(ialu_mem_imm);
5789 %}
5790 
5791 // Store Integer Immediate
5792 instruct storeImmI0(memory mem, immI0 zero)
5793 %{
5794   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5795   match(Set mem (StoreI mem zero));
5796 
5797   ins_cost(125); // XXX
5798   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5799   ins_encode %{
5800     __ movl($mem$$Address, r12);
5801   %}
5802   ins_pipe(ialu_mem_reg);
5803 %}
5804 
5805 instruct storeImmI(memory mem, immI src)
5806 %{
5807   match(Set mem (StoreI mem src));
5808 
5809   ins_cost(150);
5810   format %{ "movl    $mem, $src\t# int" %}
5811   opcode(0xC7); /* C7 /0 */
5812   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5813   ins_pipe(ialu_mem_imm);
5814 %}
5815 
5816 // Store Long Immediate
5817 instruct storeImmL0(memory mem, immL0 zero)
5818 %{
5819   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5820   match(Set mem (StoreL mem zero));
5821 
5822   ins_cost(125); // XXX
5823   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5824   ins_encode %{
5825     __ movq($mem$$Address, r12);
5826   %}
5827   ins_pipe(ialu_mem_reg);
5828 %}
5829 
5830 instruct storeImmL(memory mem, immL32 src)
5831 %{
5832   match(Set mem (StoreL mem src));
5833 
5834   ins_cost(150);
5835   format %{ "movq    $mem, $src\t# long" %}
5836   opcode(0xC7); /* C7 /0 */
5837   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5838   ins_pipe(ialu_mem_imm);
5839 %}
5840 
5841 // Store Short/Char Immediate
5842 instruct storeImmC0(memory mem, immI0 zero)
5843 %{
5844   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5845   match(Set mem (StoreC mem zero));
5846 
5847   ins_cost(125); // XXX
5848   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5849   ins_encode %{
5850     __ movw($mem$$Address, r12);
5851   %}
5852   ins_pipe(ialu_mem_reg);
5853 %}
5854 
5855 instruct storeImmI16(memory mem, immI16 src)
5856 %{
5857   predicate(UseStoreImmI16);
5858   match(Set mem (StoreC mem src));
5859 
5860   ins_cost(150);
5861   format %{ "movw    $mem, $src\t# short/char" %}
5862   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5863   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5864   ins_pipe(ialu_mem_imm);
5865 %}
5866 
5867 // Store Byte Immediate
5868 instruct storeImmB0(memory mem, immI0 zero)
5869 %{
5870   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5871   match(Set mem (StoreB mem zero));
5872 
5873   ins_cost(125); // XXX
5874   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5875   ins_encode %{
5876     __ movb($mem$$Address, r12);
5877   %}
5878   ins_pipe(ialu_mem_reg);
5879 %}
5880 
5881 instruct storeImmB(memory mem, immI8 src)
5882 %{
5883   match(Set mem (StoreB mem src));
5884 
5885   ins_cost(150); // XXX
5886   format %{ "movb    $mem, $src\t# byte" %}
5887   opcode(0xC6); /* C6 /0 */
5888   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5889   ins_pipe(ialu_mem_imm);
5890 %}
5891 
5892 // Store CMS card-mark Immediate
5893 instruct storeImmCM0_reg(memory mem, immI0 zero)
5894 %{
5895   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5896   match(Set mem (StoreCM mem zero));
5897 
5898   ins_cost(125); // XXX
5899   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5900   ins_encode %{
5901     __ movb($mem$$Address, r12);
5902   %}
5903   ins_pipe(ialu_mem_reg);
5904 %}
5905 
5906 instruct storeImmCM0(memory mem, immI0 src)
5907 %{
5908   match(Set mem (StoreCM mem src));
5909 
5910   ins_cost(150); // XXX
5911   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5912   opcode(0xC6); /* C6 /0 */
5913   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5914   ins_pipe(ialu_mem_imm);
5915 %}
5916 
5917 // Store Float
5918 instruct storeF(memory mem, regF src)
5919 %{
5920   match(Set mem (StoreF mem src));
5921 
5922   ins_cost(95); // XXX
5923   format %{ "movss   $mem, $src\t# float" %}
5924   ins_encode %{
5925     __ movflt($mem$$Address, $src$$XMMRegister);
5926   %}
5927   ins_pipe(pipe_slow); // XXX
5928 %}
5929 
5930 // Store immediate Float value (it is faster than store from XMM register)
5931 instruct storeF0(memory mem, immF0 zero)
5932 %{
5933   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5934   match(Set mem (StoreF mem zero));
5935 
5936   ins_cost(25); // XXX
5937   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5938   ins_encode %{
5939     __ movl($mem$$Address, r12);
5940   %}
5941   ins_pipe(ialu_mem_reg);
5942 %}
5943 
5944 instruct storeF_imm(memory mem, immF src)
5945 %{
5946   match(Set mem (StoreF mem src));
5947 
5948   ins_cost(50);
5949   format %{ "movl    $mem, $src\t# float" %}
5950   opcode(0xC7); /* C7 /0 */
5951   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5952   ins_pipe(ialu_mem_imm);
5953 %}
5954 
5955 // Store Double
5956 instruct storeD(memory mem, regD src)
5957 %{
5958   match(Set mem (StoreD mem src));
5959 
5960   ins_cost(95); // XXX
5961   format %{ "movsd   $mem, $src\t# double" %}
5962   ins_encode %{
5963     __ movdbl($mem$$Address, $src$$XMMRegister);
5964   %}
5965   ins_pipe(pipe_slow); // XXX
5966 %}
5967 
5968 // Store immediate double 0.0 (it is faster than store from XMM register)
5969 instruct storeD0_imm(memory mem, immD0 src)
5970 %{
5971   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5972   match(Set mem (StoreD mem src));
5973 
5974   ins_cost(50);
5975   format %{ "movq    $mem, $src\t# double 0." %}
5976   opcode(0xC7); /* C7 /0 */
5977   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5978   ins_pipe(ialu_mem_imm);
5979 %}
5980 
5981 instruct storeD0(memory mem, immD0 zero)
5982 %{
5983   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5984   match(Set mem (StoreD mem zero));
5985 
5986   ins_cost(25); // XXX
5987   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5988   ins_encode %{
5989     __ movq($mem$$Address, r12);
5990   %}
5991   ins_pipe(ialu_mem_reg);
5992 %}
5993 
5994 instruct storeSSI(stackSlotI dst, rRegI src)
5995 %{
5996   match(Set dst src);
5997 
5998   ins_cost(100);
5999   format %{ "movl    $dst, $src\t# int stk" %}
6000   opcode(0x89);
6001   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6002   ins_pipe( ialu_mem_reg );
6003 %}
6004 
6005 instruct storeSSL(stackSlotL dst, rRegL src)
6006 %{
6007   match(Set dst src);
6008 
6009   ins_cost(100);
6010   format %{ "movq    $dst, $src\t# long stk" %}
6011   opcode(0x89);
6012   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6013   ins_pipe(ialu_mem_reg);
6014 %}
6015 
6016 instruct storeSSP(stackSlotP dst, rRegP src)
6017 %{
6018   match(Set dst src);
6019 
6020   ins_cost(100);
6021   format %{ "movq    $dst, $src\t# ptr stk" %}
6022   opcode(0x89);
6023   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6024   ins_pipe(ialu_mem_reg);
6025 %}
6026 
6027 instruct storeSSF(stackSlotF dst, regF src)
6028 %{
6029   match(Set dst src);
6030 
6031   ins_cost(95); // XXX
6032   format %{ "movss   $dst, $src\t# float stk" %}
6033   ins_encode %{
6034     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6035   %}
6036   ins_pipe(pipe_slow); // XXX
6037 %}
6038 
6039 instruct storeSSD(stackSlotD dst, regD src)
6040 %{
6041   match(Set dst src);
6042 
6043   ins_cost(95); // XXX
6044   format %{ "movsd   $dst, $src\t# double stk" %}
6045   ins_encode %{
6046     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6047   %}
6048   ins_pipe(pipe_slow); // XXX
6049 %}
6050 
6051 //----------BSWAP Instructions-------------------------------------------------
6052 instruct bytes_reverse_int(rRegI dst) %{
6053   match(Set dst (ReverseBytesI dst));
6054 
6055   format %{ "bswapl  $dst" %}
6056   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6057   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6058   ins_pipe( ialu_reg );
6059 %}
6060 
6061 instruct bytes_reverse_long(rRegL dst) %{
6062   match(Set dst (ReverseBytesL dst));
6063 
6064   format %{ "bswapq  $dst" %}
6065   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6066   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6067   ins_pipe( ialu_reg);
6068 %}
6069 
6070 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6071   match(Set dst (ReverseBytesUS dst));
6072   effect(KILL cr);
6073 
6074   format %{ "bswapl  $dst\n\t"
6075             "shrl    $dst,16\n\t" %}
6076   ins_encode %{
6077     __ bswapl($dst$$Register);
6078     __ shrl($dst$$Register, 16);
6079   %}
6080   ins_pipe( ialu_reg );
6081 %}
6082 
6083 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6084   match(Set dst (ReverseBytesS dst));
6085   effect(KILL cr);
6086 
6087   format %{ "bswapl  $dst\n\t"
6088             "sar     $dst,16\n\t" %}
6089   ins_encode %{
6090     __ bswapl($dst$$Register);
6091     __ sarl($dst$$Register, 16);
6092   %}
6093   ins_pipe( ialu_reg );
6094 %}
6095 
6096 //---------- Zeros Count Instructions ------------------------------------------
6097 
6098 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6099   predicate(UseCountLeadingZerosInstruction);
6100   match(Set dst (CountLeadingZerosI src));
6101   effect(KILL cr);
6102 
6103   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6104   ins_encode %{
6105     __ lzcntl($dst$$Register, $src$$Register);
6106   %}
6107   ins_pipe(ialu_reg);
6108 %}
6109 
6110 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6111   predicate(!UseCountLeadingZerosInstruction);
6112   match(Set dst (CountLeadingZerosI src));
6113   effect(KILL cr);
6114 
6115   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6116             "jnz     skip\n\t"
6117             "movl    $dst, -1\n"
6118       "skip:\n\t"
6119             "negl    $dst\n\t"
6120             "addl    $dst, 31" %}
6121   ins_encode %{
6122     Register Rdst = $dst$$Register;
6123     Register Rsrc = $src$$Register;
6124     Label skip;
6125     __ bsrl(Rdst, Rsrc);
6126     __ jccb(Assembler::notZero, skip);
6127     __ movl(Rdst, -1);
6128     __ bind(skip);
6129     __ negl(Rdst);
6130     __ addl(Rdst, BitsPerInt - 1);
6131   %}
6132   ins_pipe(ialu_reg);
6133 %}
6134 
6135 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6136   predicate(UseCountLeadingZerosInstruction);
6137   match(Set dst (CountLeadingZerosL src));
6138   effect(KILL cr);
6139 
6140   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6141   ins_encode %{
6142     __ lzcntq($dst$$Register, $src$$Register);
6143   %}
6144   ins_pipe(ialu_reg);
6145 %}
6146 
6147 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6148   predicate(!UseCountLeadingZerosInstruction);
6149   match(Set dst (CountLeadingZerosL src));
6150   effect(KILL cr);
6151 
6152   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6153             "jnz     skip\n\t"
6154             "movl    $dst, -1\n"
6155       "skip:\n\t"
6156             "negl    $dst\n\t"
6157             "addl    $dst, 63" %}
6158   ins_encode %{
6159     Register Rdst = $dst$$Register;
6160     Register Rsrc = $src$$Register;
6161     Label skip;
6162     __ bsrq(Rdst, Rsrc);
6163     __ jccb(Assembler::notZero, skip);
6164     __ movl(Rdst, -1);
6165     __ bind(skip);
6166     __ negl(Rdst);
6167     __ addl(Rdst, BitsPerLong - 1);
6168   %}
6169   ins_pipe(ialu_reg);
6170 %}
6171 
6172 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6173   predicate(UseCountTrailingZerosInstruction);
6174   match(Set dst (CountTrailingZerosI src));
6175   effect(KILL cr);
6176 
6177   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6178   ins_encode %{
6179     __ tzcntl($dst$$Register, $src$$Register);
6180   %}
6181   ins_pipe(ialu_reg);
6182 %}
6183 
6184 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6185   predicate(!UseCountTrailingZerosInstruction);
6186   match(Set dst (CountTrailingZerosI src));
6187   effect(KILL cr);
6188 
6189   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6190             "jnz     done\n\t"
6191             "movl    $dst, 32\n"
6192       "done:" %}
6193   ins_encode %{
6194     Register Rdst = $dst$$Register;
6195     Label done;
6196     __ bsfl(Rdst, $src$$Register);
6197     __ jccb(Assembler::notZero, done);
6198     __ movl(Rdst, BitsPerInt);
6199     __ bind(done);
6200   %}
6201   ins_pipe(ialu_reg);
6202 %}
6203 
6204 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6205   predicate(UseCountTrailingZerosInstruction);
6206   match(Set dst (CountTrailingZerosL src));
6207   effect(KILL cr);
6208 
6209   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6210   ins_encode %{
6211     __ tzcntq($dst$$Register, $src$$Register);
6212   %}
6213   ins_pipe(ialu_reg);
6214 %}
6215 
6216 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6217   predicate(!UseCountTrailingZerosInstruction);
6218   match(Set dst (CountTrailingZerosL src));
6219   effect(KILL cr);
6220 
6221   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6222             "jnz     done\n\t"
6223             "movl    $dst, 64\n"
6224       "done:" %}
6225   ins_encode %{
6226     Register Rdst = $dst$$Register;
6227     Label done;
6228     __ bsfq(Rdst, $src$$Register);
6229     __ jccb(Assembler::notZero, done);
6230     __ movl(Rdst, BitsPerLong);
6231     __ bind(done);
6232   %}
6233   ins_pipe(ialu_reg);
6234 %}
6235 
6236 
6237 //---------- Population Count Instructions -------------------------------------
6238 
6239 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6240   predicate(UsePopCountInstruction);
6241   match(Set dst (PopCountI src));
6242   effect(KILL cr);
6243 
6244   format %{ "popcnt  $dst, $src" %}
6245   ins_encode %{
6246     __ popcntl($dst$$Register, $src$$Register);
6247   %}
6248   ins_pipe(ialu_reg);
6249 %}
6250 
6251 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6252   predicate(UsePopCountInstruction);
6253   match(Set dst (PopCountI (LoadI mem)));
6254   effect(KILL cr);
6255 
6256   format %{ "popcnt  $dst, $mem" %}
6257   ins_encode %{
6258     __ popcntl($dst$$Register, $mem$$Address);
6259   %}
6260   ins_pipe(ialu_reg);
6261 %}
6262 
6263 // Note: Long.bitCount(long) returns an int.
6264 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6265   predicate(UsePopCountInstruction);
6266   match(Set dst (PopCountL src));
6267   effect(KILL cr);
6268 
6269   format %{ "popcnt  $dst, $src" %}
6270   ins_encode %{
6271     __ popcntq($dst$$Register, $src$$Register);
6272   %}
6273   ins_pipe(ialu_reg);
6274 %}
6275 
6276 // Note: Long.bitCount(long) returns an int.
6277 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6278   predicate(UsePopCountInstruction);
6279   match(Set dst (PopCountL (LoadL mem)));
6280   effect(KILL cr);
6281 
6282   format %{ "popcnt  $dst, $mem" %}
6283   ins_encode %{
6284     __ popcntq($dst$$Register, $mem$$Address);
6285   %}
6286   ins_pipe(ialu_reg);
6287 %}
6288 
6289 
6290 //----------MemBar Instructions-----------------------------------------------
6291 // Memory barrier flavors
6292 
6293 instruct membar_acquire()
6294 %{
6295   match(MemBarAcquire);
6296   match(LoadFence);
6297   ins_cost(0);
6298 
6299   size(0);
6300   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6301   ins_encode();
6302   ins_pipe(empty);
6303 %}
6304 
6305 instruct membar_acquire_lock()
6306 %{
6307   match(MemBarAcquireLock);
6308   ins_cost(0);
6309 
6310   size(0);
6311   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6312   ins_encode();
6313   ins_pipe(empty);
6314 %}
6315 
6316 instruct membar_release()
6317 %{
6318   match(MemBarRelease);
6319   match(StoreFence);
6320   ins_cost(0);
6321 
6322   size(0);
6323   format %{ "MEMBAR-release ! (empty encoding)" %}
6324   ins_encode();
6325   ins_pipe(empty);
6326 %}
6327 
6328 instruct membar_release_lock()
6329 %{
6330   match(MemBarReleaseLock);
6331   ins_cost(0);
6332 
6333   size(0);
6334   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6335   ins_encode();
6336   ins_pipe(empty);
6337 %}
6338 
6339 instruct onspinwait()
6340 %{
6341   match(OnSpinWait);
6342   ins_cost(200);
6343 
6344   format %{
6345     $$template
6346     if (os::is_MP()) {
6347       $$emit$$"pause\t! membar_onspinwait"
6348     } else {
6349       $$emit$$"MEMBAR-onspinwait ! (empty encoding)"
6350     }
6351   %}
6352   ins_encode %{
6353     __ pause();
6354   %}
6355   ins_pipe(pipe_slow);
6356 %}
6357 
6358 instruct membar_volatile(rFlagsReg cr) %{
6359   match(MemBarVolatile);
6360   effect(KILL cr);
6361   ins_cost(400);
6362 
6363   format %{
6364     $$template
6365     if (os::is_MP()) {
6366       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6367     } else {
6368       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6369     }
6370   %}
6371   ins_encode %{
6372     __ membar(Assembler::StoreLoad);
6373   %}
6374   ins_pipe(pipe_slow);
6375 %}
6376 
6377 instruct unnecessary_membar_volatile()
6378 %{
6379   match(MemBarVolatile);
6380   predicate(Matcher::post_store_load_barrier(n));
6381   ins_cost(0);
6382 
6383   size(0);
6384   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6385   ins_encode();
6386   ins_pipe(empty);
6387 %}
6388 
6389 instruct membar_storestore() %{
6390   match(MemBarStoreStore);
6391   ins_cost(0);
6392 
6393   size(0);
6394   format %{ "MEMBAR-storestore (empty encoding)" %}
6395   ins_encode( );
6396   ins_pipe(empty);
6397 %}
6398 
6399 //----------Move Instructions--------------------------------------------------
6400 
6401 instruct castX2P(rRegP dst, rRegL src)
6402 %{
6403   match(Set dst (CastX2P src));
6404 
6405   format %{ "movq    $dst, $src\t# long->ptr" %}
6406   ins_encode %{
6407     if ($dst$$reg != $src$$reg) {
6408       __ movptr($dst$$Register, $src$$Register);
6409     }
6410   %}
6411   ins_pipe(ialu_reg_reg); // XXX
6412 %}
6413 
6414 instruct castP2X(rRegL dst, rRegP src)
6415 %{
6416   match(Set dst (CastP2X src));
6417 
6418   format %{ "movq    $dst, $src\t# ptr -> long" %}
6419   ins_encode %{
6420     if ($dst$$reg != $src$$reg) {
6421       __ movptr($dst$$Register, $src$$Register);
6422     }
6423   %}
6424   ins_pipe(ialu_reg_reg); // XXX
6425 %}
6426 
6427 // Convert oop into int for vectors alignment masking
6428 instruct convP2I(rRegI dst, rRegP src)
6429 %{
6430   match(Set dst (ConvL2I (CastP2X src)));
6431 
6432   format %{ "movl    $dst, $src\t# ptr -> int" %}
6433   ins_encode %{
6434     __ movl($dst$$Register, $src$$Register);
6435   %}
6436   ins_pipe(ialu_reg_reg); // XXX
6437 %}
6438 
6439 // Convert compressed oop into int for vectors alignment masking
6440 // in case of 32bit oops (heap < 4Gb).
6441 instruct convN2I(rRegI dst, rRegN src)
6442 %{
6443   predicate(Universe::narrow_oop_shift() == 0);
6444   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6445 
6446   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6447   ins_encode %{
6448     __ movl($dst$$Register, $src$$Register);
6449   %}
6450   ins_pipe(ialu_reg_reg); // XXX
6451 %}
6452 
6453 // Convert oop pointer into compressed form
6454 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6455   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6456   match(Set dst (EncodeP src));
6457   effect(KILL cr);
6458   format %{ "encode_heap_oop $dst,$src" %}
6459   ins_encode %{
6460     Register s = $src$$Register;
6461     Register d = $dst$$Register;
6462     if (s != d) {
6463       __ movq(d, s);
6464     }
6465     __ encode_heap_oop(d);
6466   %}
6467   ins_pipe(ialu_reg_long);
6468 %}
6469 
6470 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6471   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6472   match(Set dst (EncodeP src));
6473   effect(KILL cr);
6474   format %{ "encode_heap_oop_not_null $dst,$src" %}
6475   ins_encode %{
6476     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6477   %}
6478   ins_pipe(ialu_reg_long);
6479 %}
6480 
6481 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6482   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6483             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6484   match(Set dst (DecodeN src));
6485   effect(KILL cr);
6486   format %{ "decode_heap_oop $dst,$src" %}
6487   ins_encode %{
6488     Register s = $src$$Register;
6489     Register d = $dst$$Register;
6490     if (s != d) {
6491       __ movq(d, s);
6492     }
6493     __ decode_heap_oop(d);
6494   %}
6495   ins_pipe(ialu_reg_long);
6496 %}
6497 
6498 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6499   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6500             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6501   match(Set dst (DecodeN src));
6502   effect(KILL cr);
6503   format %{ "decode_heap_oop_not_null $dst,$src" %}
6504   ins_encode %{
6505     Register s = $src$$Register;
6506     Register d = $dst$$Register;
6507     if (s != d) {
6508       __ decode_heap_oop_not_null(d, s);
6509     } else {
6510       __ decode_heap_oop_not_null(d);
6511     }
6512   %}
6513   ins_pipe(ialu_reg_long);
6514 %}
6515 
6516 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6517   match(Set dst (EncodePKlass src));
6518   effect(KILL cr);
6519   format %{ "encode_klass_not_null $dst,$src" %}
6520   ins_encode %{
6521     __ encode_klass_not_null($dst$$Register, $src$$Register);
6522   %}
6523   ins_pipe(ialu_reg_long);
6524 %}
6525 
6526 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6527   match(Set dst (DecodeNKlass src));
6528   effect(KILL cr);
6529   format %{ "decode_klass_not_null $dst,$src" %}
6530   ins_encode %{
6531     Register s = $src$$Register;
6532     Register d = $dst$$Register;
6533     if (s != d) {
6534       __ decode_klass_not_null(d, s);
6535     } else {
6536       __ decode_klass_not_null(d);
6537     }
6538   %}
6539   ins_pipe(ialu_reg_long);
6540 %}
6541 
6542 
6543 //----------Conditional Move---------------------------------------------------
6544 // Jump
6545 // dummy instruction for generating temp registers
6546 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6547   match(Jump (LShiftL switch_val shift));
6548   ins_cost(350);
6549   predicate(false);
6550   effect(TEMP dest);
6551 
6552   format %{ "leaq    $dest, [$constantaddress]\n\t"
6553             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6554   ins_encode %{
6555     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6556     // to do that and the compiler is using that register as one it can allocate.
6557     // So we build it all by hand.
6558     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6559     // ArrayAddress dispatch(table, index);
6560     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6561     __ lea($dest$$Register, $constantaddress);
6562     __ jmp(dispatch);
6563   %}
6564   ins_pipe(pipe_jmp);
6565 %}
6566 
6567 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6568   match(Jump (AddL (LShiftL switch_val shift) offset));
6569   ins_cost(350);
6570   effect(TEMP dest);
6571 
6572   format %{ "leaq    $dest, [$constantaddress]\n\t"
6573             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6574   ins_encode %{
6575     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6576     // to do that and the compiler is using that register as one it can allocate.
6577     // So we build it all by hand.
6578     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6579     // ArrayAddress dispatch(table, index);
6580     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6581     __ lea($dest$$Register, $constantaddress);
6582     __ jmp(dispatch);
6583   %}
6584   ins_pipe(pipe_jmp);
6585 %}
6586 
6587 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6588   match(Jump switch_val);
6589   ins_cost(350);
6590   effect(TEMP dest);
6591 
6592   format %{ "leaq    $dest, [$constantaddress]\n\t"
6593             "jmp     [$dest + $switch_val]\n\t" %}
6594   ins_encode %{
6595     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6596     // to do that and the compiler is using that register as one it can allocate.
6597     // So we build it all by hand.
6598     // Address index(noreg, switch_reg, Address::times_1);
6599     // ArrayAddress dispatch(table, index);
6600     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6601     __ lea($dest$$Register, $constantaddress);
6602     __ jmp(dispatch);
6603   %}
6604   ins_pipe(pipe_jmp);
6605 %}
6606 
6607 // Conditional move
6608 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6609 %{
6610   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6611 
6612   ins_cost(200); // XXX
6613   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6614   opcode(0x0F, 0x40);
6615   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6616   ins_pipe(pipe_cmov_reg);
6617 %}
6618 
6619 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6620   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6621 
6622   ins_cost(200); // XXX
6623   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6624   opcode(0x0F, 0x40);
6625   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6626   ins_pipe(pipe_cmov_reg);
6627 %}
6628 
6629 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6630   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6631   ins_cost(200);
6632   expand %{
6633     cmovI_regU(cop, cr, dst, src);
6634   %}
6635 %}
6636 
6637 // Conditional move
6638 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6639   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6640 
6641   ins_cost(250); // XXX
6642   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6643   opcode(0x0F, 0x40);
6644   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6645   ins_pipe(pipe_cmov_mem);
6646 %}
6647 
6648 // Conditional move
6649 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6650 %{
6651   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6652 
6653   ins_cost(250); // XXX
6654   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6655   opcode(0x0F, 0x40);
6656   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6657   ins_pipe(pipe_cmov_mem);
6658 %}
6659 
6660 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6661   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6662   ins_cost(250);
6663   expand %{
6664     cmovI_memU(cop, cr, dst, src);
6665   %}
6666 %}
6667 
6668 // Conditional move
6669 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6670 %{
6671   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6672 
6673   ins_cost(200); // XXX
6674   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6675   opcode(0x0F, 0x40);
6676   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6677   ins_pipe(pipe_cmov_reg);
6678 %}
6679 
6680 // Conditional move
6681 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6682 %{
6683   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6684 
6685   ins_cost(200); // XXX
6686   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6687   opcode(0x0F, 0x40);
6688   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6689   ins_pipe(pipe_cmov_reg);
6690 %}
6691 
6692 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6693   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6694   ins_cost(200);
6695   expand %{
6696     cmovN_regU(cop, cr, dst, src);
6697   %}
6698 %}
6699 
6700 // Conditional move
6701 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6702 %{
6703   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6704 
6705   ins_cost(200); // XXX
6706   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6707   opcode(0x0F, 0x40);
6708   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6709   ins_pipe(pipe_cmov_reg);  // XXX
6710 %}
6711 
6712 // Conditional move
6713 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6714 %{
6715   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6716 
6717   ins_cost(200); // XXX
6718   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6719   opcode(0x0F, 0x40);
6720   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6721   ins_pipe(pipe_cmov_reg); // XXX
6722 %}
6723 
6724 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6725   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6726   ins_cost(200);
6727   expand %{
6728     cmovP_regU(cop, cr, dst, src);
6729   %}
6730 %}
6731 
6732 // DISABLED: Requires the ADLC to emit a bottom_type call that
6733 // correctly meets the two pointer arguments; one is an incoming
6734 // register but the other is a memory operand.  ALSO appears to
6735 // be buggy with implicit null checks.
6736 //
6737 //// Conditional move
6738 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6739 //%{
6740 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6741 //  ins_cost(250);
6742 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6743 //  opcode(0x0F,0x40);
6744 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6745 //  ins_pipe( pipe_cmov_mem );
6746 //%}
6747 //
6748 //// Conditional move
6749 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6750 //%{
6751 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6752 //  ins_cost(250);
6753 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6754 //  opcode(0x0F,0x40);
6755 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6756 //  ins_pipe( pipe_cmov_mem );
6757 //%}
6758 
6759 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6760 %{
6761   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6762 
6763   ins_cost(200); // XXX
6764   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6765   opcode(0x0F, 0x40);
6766   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6767   ins_pipe(pipe_cmov_reg);  // XXX
6768 %}
6769 
6770 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6771 %{
6772   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6773 
6774   ins_cost(200); // XXX
6775   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6776   opcode(0x0F, 0x40);
6777   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6778   ins_pipe(pipe_cmov_mem);  // XXX
6779 %}
6780 
6781 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6782 %{
6783   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6784 
6785   ins_cost(200); // XXX
6786   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6787   opcode(0x0F, 0x40);
6788   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6789   ins_pipe(pipe_cmov_reg); // XXX
6790 %}
6791 
6792 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6793   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6794   ins_cost(200);
6795   expand %{
6796     cmovL_regU(cop, cr, dst, src);
6797   %}
6798 %}
6799 
6800 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6801 %{
6802   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6803 
6804   ins_cost(200); // XXX
6805   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6806   opcode(0x0F, 0x40);
6807   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6808   ins_pipe(pipe_cmov_mem); // XXX
6809 %}
6810 
6811 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6812   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6813   ins_cost(200);
6814   expand %{
6815     cmovL_memU(cop, cr, dst, src);
6816   %}
6817 %}
6818 
6819 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6820 %{
6821   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6822 
6823   ins_cost(200); // XXX
6824   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6825             "movss     $dst, $src\n"
6826     "skip:" %}
6827   ins_encode %{
6828     Label Lskip;
6829     // Invert sense of branch from sense of CMOV
6830     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6831     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6832     __ bind(Lskip);
6833   %}
6834   ins_pipe(pipe_slow);
6835 %}
6836 
6837 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6838 // %{
6839 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6840 
6841 //   ins_cost(200); // XXX
6842 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6843 //             "movss     $dst, $src\n"
6844 //     "skip:" %}
6845 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6846 //   ins_pipe(pipe_slow);
6847 // %}
6848 
6849 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6850 %{
6851   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6852 
6853   ins_cost(200); // XXX
6854   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6855             "movss     $dst, $src\n"
6856     "skip:" %}
6857   ins_encode %{
6858     Label Lskip;
6859     // Invert sense of branch from sense of CMOV
6860     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6861     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6862     __ bind(Lskip);
6863   %}
6864   ins_pipe(pipe_slow);
6865 %}
6866 
6867 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6868   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6869   ins_cost(200);
6870   expand %{
6871     cmovF_regU(cop, cr, dst, src);
6872   %}
6873 %}
6874 
6875 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6876 %{
6877   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6878 
6879   ins_cost(200); // XXX
6880   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6881             "movsd     $dst, $src\n"
6882     "skip:" %}
6883   ins_encode %{
6884     Label Lskip;
6885     // Invert sense of branch from sense of CMOV
6886     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6887     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6888     __ bind(Lskip);
6889   %}
6890   ins_pipe(pipe_slow);
6891 %}
6892 
6893 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6894 %{
6895   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6896 
6897   ins_cost(200); // XXX
6898   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6899             "movsd     $dst, $src\n"
6900     "skip:" %}
6901   ins_encode %{
6902     Label Lskip;
6903     // Invert sense of branch from sense of CMOV
6904     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6905     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6906     __ bind(Lskip);
6907   %}
6908   ins_pipe(pipe_slow);
6909 %}
6910 
6911 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6912   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6913   ins_cost(200);
6914   expand %{
6915     cmovD_regU(cop, cr, dst, src);
6916   %}
6917 %}
6918 
6919 //----------Arithmetic Instructions--------------------------------------------
6920 //----------Addition Instructions----------------------------------------------
6921 
6922 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6923 %{
6924   match(Set dst (AddI dst src));
6925   effect(KILL cr);
6926 
6927   format %{ "addl    $dst, $src\t# int" %}
6928   opcode(0x03);
6929   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6930   ins_pipe(ialu_reg_reg);
6931 %}
6932 
6933 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6934 %{
6935   match(Set dst (AddI dst src));
6936   effect(KILL cr);
6937 
6938   format %{ "addl    $dst, $src\t# int" %}
6939   opcode(0x81, 0x00); /* /0 id */
6940   ins_encode(OpcSErm(dst, src), Con8or32(src));
6941   ins_pipe( ialu_reg );
6942 %}
6943 
6944 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6945 %{
6946   match(Set dst (AddI dst (LoadI src)));
6947   effect(KILL cr);
6948 
6949   ins_cost(125); // XXX
6950   format %{ "addl    $dst, $src\t# int" %}
6951   opcode(0x03);
6952   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6953   ins_pipe(ialu_reg_mem);
6954 %}
6955 
6956 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6957 %{
6958   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6959   effect(KILL cr);
6960 
6961   ins_cost(150); // XXX
6962   format %{ "addl    $dst, $src\t# int" %}
6963   opcode(0x01); /* Opcode 01 /r */
6964   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6965   ins_pipe(ialu_mem_reg);
6966 %}
6967 
6968 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6969 %{
6970   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6971   effect(KILL cr);
6972 
6973   ins_cost(125); // XXX
6974   format %{ "addl    $dst, $src\t# int" %}
6975   opcode(0x81); /* Opcode 81 /0 id */
6976   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6977   ins_pipe(ialu_mem_imm);
6978 %}
6979 
6980 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6981 %{
6982   predicate(UseIncDec);
6983   match(Set dst (AddI dst src));
6984   effect(KILL cr);
6985 
6986   format %{ "incl    $dst\t# int" %}
6987   opcode(0xFF, 0x00); // FF /0
6988   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6989   ins_pipe(ialu_reg);
6990 %}
6991 
6992 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6993 %{
6994   predicate(UseIncDec);
6995   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6996   effect(KILL cr);
6997 
6998   ins_cost(125); // XXX
6999   format %{ "incl    $dst\t# int" %}
7000   opcode(0xFF); /* Opcode FF /0 */
7001   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
7002   ins_pipe(ialu_mem_imm);
7003 %}
7004 
7005 // XXX why does that use AddI
7006 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
7007 %{
7008   predicate(UseIncDec);
7009   match(Set dst (AddI dst src));
7010   effect(KILL cr);
7011 
7012   format %{ "decl    $dst\t# int" %}
7013   opcode(0xFF, 0x01); // FF /1
7014   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7015   ins_pipe(ialu_reg);
7016 %}
7017 
7018 // XXX why does that use AddI
7019 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7020 %{
7021   predicate(UseIncDec);
7022   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7023   effect(KILL cr);
7024 
7025   ins_cost(125); // XXX
7026   format %{ "decl    $dst\t# int" %}
7027   opcode(0xFF); /* Opcode FF /1 */
7028   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7029   ins_pipe(ialu_mem_imm);
7030 %}
7031 
7032 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7033 %{
7034   match(Set dst (AddI src0 src1));
7035 
7036   ins_cost(110);
7037   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7038   opcode(0x8D); /* 0x8D /r */
7039   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7040   ins_pipe(ialu_reg_reg);
7041 %}
7042 
7043 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7044 %{
7045   match(Set dst (AddL dst src));
7046   effect(KILL cr);
7047 
7048   format %{ "addq    $dst, $src\t# long" %}
7049   opcode(0x03);
7050   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7051   ins_pipe(ialu_reg_reg);
7052 %}
7053 
7054 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7055 %{
7056   match(Set dst (AddL dst src));
7057   effect(KILL cr);
7058 
7059   format %{ "addq    $dst, $src\t# long" %}
7060   opcode(0x81, 0x00); /* /0 id */
7061   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7062   ins_pipe( ialu_reg );
7063 %}
7064 
7065 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7066 %{
7067   match(Set dst (AddL dst (LoadL src)));
7068   effect(KILL cr);
7069 
7070   ins_cost(125); // XXX
7071   format %{ "addq    $dst, $src\t# long" %}
7072   opcode(0x03);
7073   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7074   ins_pipe(ialu_reg_mem);
7075 %}
7076 
7077 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7078 %{
7079   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7080   effect(KILL cr);
7081 
7082   ins_cost(150); // XXX
7083   format %{ "addq    $dst, $src\t# long" %}
7084   opcode(0x01); /* Opcode 01 /r */
7085   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7086   ins_pipe(ialu_mem_reg);
7087 %}
7088 
7089 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7090 %{
7091   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7092   effect(KILL cr);
7093 
7094   ins_cost(125); // XXX
7095   format %{ "addq    $dst, $src\t# long" %}
7096   opcode(0x81); /* Opcode 81 /0 id */
7097   ins_encode(REX_mem_wide(dst),
7098              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7099   ins_pipe(ialu_mem_imm);
7100 %}
7101 
7102 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7103 %{
7104   predicate(UseIncDec);
7105   match(Set dst (AddL dst src));
7106   effect(KILL cr);
7107 
7108   format %{ "incq    $dst\t# long" %}
7109   opcode(0xFF, 0x00); // FF /0
7110   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7111   ins_pipe(ialu_reg);
7112 %}
7113 
7114 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7115 %{
7116   predicate(UseIncDec);
7117   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7118   effect(KILL cr);
7119 
7120   ins_cost(125); // XXX
7121   format %{ "incq    $dst\t# long" %}
7122   opcode(0xFF); /* Opcode FF /0 */
7123   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7124   ins_pipe(ialu_mem_imm);
7125 %}
7126 
7127 // XXX why does that use AddL
7128 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7129 %{
7130   predicate(UseIncDec);
7131   match(Set dst (AddL dst src));
7132   effect(KILL cr);
7133 
7134   format %{ "decq    $dst\t# long" %}
7135   opcode(0xFF, 0x01); // FF /1
7136   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7137   ins_pipe(ialu_reg);
7138 %}
7139 
7140 // XXX why does that use AddL
7141 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7142 %{
7143   predicate(UseIncDec);
7144   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7145   effect(KILL cr);
7146 
7147   ins_cost(125); // XXX
7148   format %{ "decq    $dst\t# long" %}
7149   opcode(0xFF); /* Opcode FF /1 */
7150   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7151   ins_pipe(ialu_mem_imm);
7152 %}
7153 
7154 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7155 %{
7156   match(Set dst (AddL src0 src1));
7157 
7158   ins_cost(110);
7159   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7160   opcode(0x8D); /* 0x8D /r */
7161   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7162   ins_pipe(ialu_reg_reg);
7163 %}
7164 
7165 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7166 %{
7167   match(Set dst (AddP dst src));
7168   effect(KILL cr);
7169 
7170   format %{ "addq    $dst, $src\t# ptr" %}
7171   opcode(0x03);
7172   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7173   ins_pipe(ialu_reg_reg);
7174 %}
7175 
7176 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7177 %{
7178   match(Set dst (AddP dst src));
7179   effect(KILL cr);
7180 
7181   format %{ "addq    $dst, $src\t# ptr" %}
7182   opcode(0x81, 0x00); /* /0 id */
7183   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7184   ins_pipe( ialu_reg );
7185 %}
7186 
7187 // XXX addP mem ops ????
7188 
7189 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7190 %{
7191   match(Set dst (AddP src0 src1));
7192 
7193   ins_cost(110);
7194   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7195   opcode(0x8D); /* 0x8D /r */
7196   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7197   ins_pipe(ialu_reg_reg);
7198 %}
7199 
7200 instruct checkCastPP(rRegP dst)
7201 %{
7202   match(Set dst (CheckCastPP dst));
7203 
7204   size(0);
7205   format %{ "# checkcastPP of $dst" %}
7206   ins_encode(/* empty encoding */);
7207   ins_pipe(empty);
7208 %}
7209 
7210 instruct castPP(rRegP dst)
7211 %{
7212   match(Set dst (CastPP dst));
7213 
7214   size(0);
7215   format %{ "# castPP of $dst" %}
7216   ins_encode(/* empty encoding */);
7217   ins_pipe(empty);
7218 %}
7219 
7220 instruct castII(rRegI dst)
7221 %{
7222   match(Set dst (CastII dst));
7223 
7224   size(0);
7225   format %{ "# castII of $dst" %}
7226   ins_encode(/* empty encoding */);
7227   ins_cost(0);
7228   ins_pipe(empty);
7229 %}
7230 
7231 // LoadP-locked same as a regular LoadP when used with compare-swap
7232 instruct loadPLocked(rRegP dst, memory mem)
7233 %{
7234   match(Set dst (LoadPLocked mem));
7235 
7236   ins_cost(125); // XXX
7237   format %{ "movq    $dst, $mem\t# ptr locked" %}
7238   opcode(0x8B);
7239   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7240   ins_pipe(ialu_reg_mem); // XXX
7241 %}
7242 
7243 // Conditional-store of the updated heap-top.
7244 // Used during allocation of the shared heap.
7245 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7246 
7247 instruct storePConditional(memory heap_top_ptr,
7248                            rax_RegP oldval, rRegP newval,
7249                            rFlagsReg cr)
7250 %{
7251   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7252 
7253   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7254             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7255   opcode(0x0F, 0xB1);
7256   ins_encode(lock_prefix,
7257              REX_reg_mem_wide(newval, heap_top_ptr),
7258              OpcP, OpcS,
7259              reg_mem(newval, heap_top_ptr));
7260   ins_pipe(pipe_cmpxchg);
7261 %}
7262 
7263 // Conditional-store of an int value.
7264 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7265 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7266 %{
7267   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7268   effect(KILL oldval);
7269 
7270   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7271   opcode(0x0F, 0xB1);
7272   ins_encode(lock_prefix,
7273              REX_reg_mem(newval, mem),
7274              OpcP, OpcS,
7275              reg_mem(newval, mem));
7276   ins_pipe(pipe_cmpxchg);
7277 %}
7278 
7279 // Conditional-store of a long value.
7280 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7281 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7282 %{
7283   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7284   effect(KILL oldval);
7285 
7286   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7287   opcode(0x0F, 0xB1);
7288   ins_encode(lock_prefix,
7289              REX_reg_mem_wide(newval, mem),
7290              OpcP, OpcS,
7291              reg_mem(newval, mem));
7292   ins_pipe(pipe_cmpxchg);
7293 %}
7294 
7295 
7296 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7297 instruct compareAndSwapP(rRegI res,
7298                          memory mem_ptr,
7299                          rax_RegP oldval, rRegP newval,
7300                          rFlagsReg cr)
7301 %{
7302   predicate(VM_Version::supports_cx8());
7303   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7304   effect(KILL cr, KILL oldval);
7305 
7306   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7307             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7308             "sete    $res\n\t"
7309             "movzbl  $res, $res" %}
7310   opcode(0x0F, 0xB1);
7311   ins_encode(lock_prefix,
7312              REX_reg_mem_wide(newval, mem_ptr),
7313              OpcP, OpcS,
7314              reg_mem(newval, mem_ptr),
7315              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7316              REX_reg_breg(res, res), // movzbl
7317              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7318   ins_pipe( pipe_cmpxchg );
7319 %}
7320 
7321 instruct compareAndSwapL(rRegI res,
7322                          memory mem_ptr,
7323                          rax_RegL oldval, rRegL newval,
7324                          rFlagsReg cr)
7325 %{
7326   predicate(VM_Version::supports_cx8());
7327   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7328   effect(KILL cr, KILL oldval);
7329 
7330   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7331             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7332             "sete    $res\n\t"
7333             "movzbl  $res, $res" %}
7334   opcode(0x0F, 0xB1);
7335   ins_encode(lock_prefix,
7336              REX_reg_mem_wide(newval, mem_ptr),
7337              OpcP, OpcS,
7338              reg_mem(newval, mem_ptr),
7339              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7340              REX_reg_breg(res, res), // movzbl
7341              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7342   ins_pipe( pipe_cmpxchg );
7343 %}
7344 
7345 instruct compareAndSwapI(rRegI res,
7346                          memory mem_ptr,
7347                          rax_RegI oldval, rRegI newval,
7348                          rFlagsReg cr)
7349 %{
7350   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7351   effect(KILL cr, KILL oldval);
7352 
7353   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7354             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7355             "sete    $res\n\t"
7356             "movzbl  $res, $res" %}
7357   opcode(0x0F, 0xB1);
7358   ins_encode(lock_prefix,
7359              REX_reg_mem(newval, mem_ptr),
7360              OpcP, OpcS,
7361              reg_mem(newval, mem_ptr),
7362              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7363              REX_reg_breg(res, res), // movzbl
7364              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7365   ins_pipe( pipe_cmpxchg );
7366 %}
7367 
7368 
7369 instruct compareAndSwapN(rRegI res,
7370                           memory mem_ptr,
7371                           rax_RegN oldval, rRegN newval,
7372                           rFlagsReg cr) %{
7373   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7374   effect(KILL cr, KILL oldval);
7375 
7376   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7377             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7378             "sete    $res\n\t"
7379             "movzbl  $res, $res" %}
7380   opcode(0x0F, 0xB1);
7381   ins_encode(lock_prefix,
7382              REX_reg_mem(newval, mem_ptr),
7383              OpcP, OpcS,
7384              reg_mem(newval, mem_ptr),
7385              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7386              REX_reg_breg(res, res), // movzbl
7387              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7388   ins_pipe( pipe_cmpxchg );
7389 %}
7390 
7391 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7392   predicate(n->as_LoadStore()->result_not_used());
7393   match(Set dummy (GetAndAddI mem add));
7394   effect(KILL cr);
7395   format %{ "ADDL  [$mem],$add" %}
7396   ins_encode %{
7397     if (os::is_MP()) { __ lock(); }
7398     __ addl($mem$$Address, $add$$constant);
7399   %}
7400   ins_pipe( pipe_cmpxchg );
7401 %}
7402 
7403 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7404   match(Set newval (GetAndAddI mem newval));
7405   effect(KILL cr);
7406   format %{ "XADDL  [$mem],$newval" %}
7407   ins_encode %{
7408     if (os::is_MP()) { __ lock(); }
7409     __ xaddl($mem$$Address, $newval$$Register);
7410   %}
7411   ins_pipe( pipe_cmpxchg );
7412 %}
7413 
7414 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7415   predicate(n->as_LoadStore()->result_not_used());
7416   match(Set dummy (GetAndAddL mem add));
7417   effect(KILL cr);
7418   format %{ "ADDQ  [$mem],$add" %}
7419   ins_encode %{
7420     if (os::is_MP()) { __ lock(); }
7421     __ addq($mem$$Address, $add$$constant);
7422   %}
7423   ins_pipe( pipe_cmpxchg );
7424 %}
7425 
7426 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7427   match(Set newval (GetAndAddL mem newval));
7428   effect(KILL cr);
7429   format %{ "XADDQ  [$mem],$newval" %}
7430   ins_encode %{
7431     if (os::is_MP()) { __ lock(); }
7432     __ xaddq($mem$$Address, $newval$$Register);
7433   %}
7434   ins_pipe( pipe_cmpxchg );
7435 %}
7436 
7437 instruct xchgI( memory mem, rRegI newval) %{
7438   match(Set newval (GetAndSetI mem newval));
7439   format %{ "XCHGL  $newval,[$mem]" %}
7440   ins_encode %{
7441     __ xchgl($newval$$Register, $mem$$Address);
7442   %}
7443   ins_pipe( pipe_cmpxchg );
7444 %}
7445 
7446 instruct xchgL( memory mem, rRegL newval) %{
7447   match(Set newval (GetAndSetL mem newval));
7448   format %{ "XCHGL  $newval,[$mem]" %}
7449   ins_encode %{
7450     __ xchgq($newval$$Register, $mem$$Address);
7451   %}
7452   ins_pipe( pipe_cmpxchg );
7453 %}
7454 
7455 instruct xchgP( memory mem, rRegP newval) %{
7456   match(Set newval (GetAndSetP mem newval));
7457   format %{ "XCHGQ  $newval,[$mem]" %}
7458   ins_encode %{
7459     __ xchgq($newval$$Register, $mem$$Address);
7460   %}
7461   ins_pipe( pipe_cmpxchg );
7462 %}
7463 
7464 instruct xchgN( memory mem, rRegN newval) %{
7465   match(Set newval (GetAndSetN mem newval));
7466   format %{ "XCHGL  $newval,$mem]" %}
7467   ins_encode %{
7468     __ xchgl($newval$$Register, $mem$$Address);
7469   %}
7470   ins_pipe( pipe_cmpxchg );
7471 %}
7472 
7473 //----------Subtraction Instructions-------------------------------------------
7474 
7475 // Integer Subtraction Instructions
7476 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7477 %{
7478   match(Set dst (SubI dst src));
7479   effect(KILL cr);
7480 
7481   format %{ "subl    $dst, $src\t# int" %}
7482   opcode(0x2B);
7483   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7484   ins_pipe(ialu_reg_reg);
7485 %}
7486 
7487 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7488 %{
7489   match(Set dst (SubI dst src));
7490   effect(KILL cr);
7491 
7492   format %{ "subl    $dst, $src\t# int" %}
7493   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7494   ins_encode(OpcSErm(dst, src), Con8or32(src));
7495   ins_pipe(ialu_reg);
7496 %}
7497 
7498 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7499 %{
7500   match(Set dst (SubI dst (LoadI src)));
7501   effect(KILL cr);
7502 
7503   ins_cost(125);
7504   format %{ "subl    $dst, $src\t# int" %}
7505   opcode(0x2B);
7506   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7507   ins_pipe(ialu_reg_mem);
7508 %}
7509 
7510 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7511 %{
7512   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7513   effect(KILL cr);
7514 
7515   ins_cost(150);
7516   format %{ "subl    $dst, $src\t# int" %}
7517   opcode(0x29); /* Opcode 29 /r */
7518   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7519   ins_pipe(ialu_mem_reg);
7520 %}
7521 
7522 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7523 %{
7524   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7525   effect(KILL cr);
7526 
7527   ins_cost(125); // XXX
7528   format %{ "subl    $dst, $src\t# int" %}
7529   opcode(0x81); /* Opcode 81 /5 id */
7530   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7531   ins_pipe(ialu_mem_imm);
7532 %}
7533 
7534 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7535 %{
7536   match(Set dst (SubL dst src));
7537   effect(KILL cr);
7538 
7539   format %{ "subq    $dst, $src\t# long" %}
7540   opcode(0x2B);
7541   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7542   ins_pipe(ialu_reg_reg);
7543 %}
7544 
7545 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7546 %{
7547   match(Set dst (SubL dst src));
7548   effect(KILL cr);
7549 
7550   format %{ "subq    $dst, $src\t# long" %}
7551   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7552   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7553   ins_pipe(ialu_reg);
7554 %}
7555 
7556 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7557 %{
7558   match(Set dst (SubL dst (LoadL src)));
7559   effect(KILL cr);
7560 
7561   ins_cost(125);
7562   format %{ "subq    $dst, $src\t# long" %}
7563   opcode(0x2B);
7564   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7565   ins_pipe(ialu_reg_mem);
7566 %}
7567 
7568 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7569 %{
7570   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7571   effect(KILL cr);
7572 
7573   ins_cost(150);
7574   format %{ "subq    $dst, $src\t# long" %}
7575   opcode(0x29); /* Opcode 29 /r */
7576   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7577   ins_pipe(ialu_mem_reg);
7578 %}
7579 
7580 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7581 %{
7582   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7583   effect(KILL cr);
7584 
7585   ins_cost(125); // XXX
7586   format %{ "subq    $dst, $src\t# long" %}
7587   opcode(0x81); /* Opcode 81 /5 id */
7588   ins_encode(REX_mem_wide(dst),
7589              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7590   ins_pipe(ialu_mem_imm);
7591 %}
7592 
7593 // Subtract from a pointer
7594 // XXX hmpf???
7595 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7596 %{
7597   match(Set dst (AddP dst (SubI zero src)));
7598   effect(KILL cr);
7599 
7600   format %{ "subq    $dst, $src\t# ptr - int" %}
7601   opcode(0x2B);
7602   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7603   ins_pipe(ialu_reg_reg);
7604 %}
7605 
7606 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7607 %{
7608   match(Set dst (SubI zero dst));
7609   effect(KILL cr);
7610 
7611   format %{ "negl    $dst\t# int" %}
7612   opcode(0xF7, 0x03);  // Opcode F7 /3
7613   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7614   ins_pipe(ialu_reg);
7615 %}
7616 
7617 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7618 %{
7619   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7620   effect(KILL cr);
7621 
7622   format %{ "negl    $dst\t# int" %}
7623   opcode(0xF7, 0x03);  // Opcode F7 /3
7624   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7625   ins_pipe(ialu_reg);
7626 %}
7627 
7628 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7629 %{
7630   match(Set dst (SubL zero dst));
7631   effect(KILL cr);
7632 
7633   format %{ "negq    $dst\t# long" %}
7634   opcode(0xF7, 0x03);  // Opcode F7 /3
7635   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7636   ins_pipe(ialu_reg);
7637 %}
7638 
7639 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7640 %{
7641   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7642   effect(KILL cr);
7643 
7644   format %{ "negq    $dst\t# long" %}
7645   opcode(0xF7, 0x03);  // Opcode F7 /3
7646   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7647   ins_pipe(ialu_reg);
7648 %}
7649 
7650 //----------Multiplication/Division Instructions-------------------------------
7651 // Integer Multiplication Instructions
7652 // Multiply Register
7653 
7654 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7655 %{
7656   match(Set dst (MulI dst src));
7657   effect(KILL cr);
7658 
7659   ins_cost(300);
7660   format %{ "imull   $dst, $src\t# int" %}
7661   opcode(0x0F, 0xAF);
7662   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7663   ins_pipe(ialu_reg_reg_alu0);
7664 %}
7665 
7666 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7667 %{
7668   match(Set dst (MulI src imm));
7669   effect(KILL cr);
7670 
7671   ins_cost(300);
7672   format %{ "imull   $dst, $src, $imm\t# int" %}
7673   opcode(0x69); /* 69 /r id */
7674   ins_encode(REX_reg_reg(dst, src),
7675              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7676   ins_pipe(ialu_reg_reg_alu0);
7677 %}
7678 
7679 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7680 %{
7681   match(Set dst (MulI dst (LoadI src)));
7682   effect(KILL cr);
7683 
7684   ins_cost(350);
7685   format %{ "imull   $dst, $src\t# int" %}
7686   opcode(0x0F, 0xAF);
7687   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7688   ins_pipe(ialu_reg_mem_alu0);
7689 %}
7690 
7691 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7692 %{
7693   match(Set dst (MulI (LoadI src) imm));
7694   effect(KILL cr);
7695 
7696   ins_cost(300);
7697   format %{ "imull   $dst, $src, $imm\t# int" %}
7698   opcode(0x69); /* 69 /r id */
7699   ins_encode(REX_reg_mem(dst, src),
7700              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7701   ins_pipe(ialu_reg_mem_alu0);
7702 %}
7703 
7704 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7705 %{
7706   match(Set dst (MulL dst src));
7707   effect(KILL cr);
7708 
7709   ins_cost(300);
7710   format %{ "imulq   $dst, $src\t# long" %}
7711   opcode(0x0F, 0xAF);
7712   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7713   ins_pipe(ialu_reg_reg_alu0);
7714 %}
7715 
7716 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7717 %{
7718   match(Set dst (MulL src imm));
7719   effect(KILL cr);
7720 
7721   ins_cost(300);
7722   format %{ "imulq   $dst, $src, $imm\t# long" %}
7723   opcode(0x69); /* 69 /r id */
7724   ins_encode(REX_reg_reg_wide(dst, src),
7725              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7726   ins_pipe(ialu_reg_reg_alu0);
7727 %}
7728 
7729 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7730 %{
7731   match(Set dst (MulL dst (LoadL src)));
7732   effect(KILL cr);
7733 
7734   ins_cost(350);
7735   format %{ "imulq   $dst, $src\t# long" %}
7736   opcode(0x0F, 0xAF);
7737   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7738   ins_pipe(ialu_reg_mem_alu0);
7739 %}
7740 
7741 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7742 %{
7743   match(Set dst (MulL (LoadL src) imm));
7744   effect(KILL cr);
7745 
7746   ins_cost(300);
7747   format %{ "imulq   $dst, $src, $imm\t# long" %}
7748   opcode(0x69); /* 69 /r id */
7749   ins_encode(REX_reg_mem_wide(dst, src),
7750              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7751   ins_pipe(ialu_reg_mem_alu0);
7752 %}
7753 
7754 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7755 %{
7756   match(Set dst (MulHiL src rax));
7757   effect(USE_KILL rax, KILL cr);
7758 
7759   ins_cost(300);
7760   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7761   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7762   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7763   ins_pipe(ialu_reg_reg_alu0);
7764 %}
7765 
7766 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7767                    rFlagsReg cr)
7768 %{
7769   match(Set rax (DivI rax div));
7770   effect(KILL rdx, KILL cr);
7771 
7772   ins_cost(30*100+10*100); // XXX
7773   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7774             "jne,s   normal\n\t"
7775             "xorl    rdx, rdx\n\t"
7776             "cmpl    $div, -1\n\t"
7777             "je,s    done\n"
7778     "normal: cdql\n\t"
7779             "idivl   $div\n"
7780     "done:"        %}
7781   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7782   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7783   ins_pipe(ialu_reg_reg_alu0);
7784 %}
7785 
7786 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7787                    rFlagsReg cr)
7788 %{
7789   match(Set rax (DivL rax div));
7790   effect(KILL rdx, KILL cr);
7791 
7792   ins_cost(30*100+10*100); // XXX
7793   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7794             "cmpq    rax, rdx\n\t"
7795             "jne,s   normal\n\t"
7796             "xorl    rdx, rdx\n\t"
7797             "cmpq    $div, -1\n\t"
7798             "je,s    done\n"
7799     "normal: cdqq\n\t"
7800             "idivq   $div\n"
7801     "done:"        %}
7802   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7803   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7804   ins_pipe(ialu_reg_reg_alu0);
7805 %}
7806 
7807 // Integer DIVMOD with Register, both quotient and mod results
7808 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7809                              rFlagsReg cr)
7810 %{
7811   match(DivModI rax div);
7812   effect(KILL cr);
7813 
7814   ins_cost(30*100+10*100); // XXX
7815   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7816             "jne,s   normal\n\t"
7817             "xorl    rdx, rdx\n\t"
7818             "cmpl    $div, -1\n\t"
7819             "je,s    done\n"
7820     "normal: cdql\n\t"
7821             "idivl   $div\n"
7822     "done:"        %}
7823   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7824   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7825   ins_pipe(pipe_slow);
7826 %}
7827 
7828 // Long DIVMOD with Register, both quotient and mod results
7829 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7830                              rFlagsReg cr)
7831 %{
7832   match(DivModL rax div);
7833   effect(KILL cr);
7834 
7835   ins_cost(30*100+10*100); // XXX
7836   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7837             "cmpq    rax, rdx\n\t"
7838             "jne,s   normal\n\t"
7839             "xorl    rdx, rdx\n\t"
7840             "cmpq    $div, -1\n\t"
7841             "je,s    done\n"
7842     "normal: cdqq\n\t"
7843             "idivq   $div\n"
7844     "done:"        %}
7845   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7846   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7847   ins_pipe(pipe_slow);
7848 %}
7849 
7850 //----------- DivL-By-Constant-Expansions--------------------------------------
7851 // DivI cases are handled by the compiler
7852 
7853 // Magic constant, reciprocal of 10
7854 instruct loadConL_0x6666666666666667(rRegL dst)
7855 %{
7856   effect(DEF dst);
7857 
7858   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7859   ins_encode(load_immL(dst, 0x6666666666666667));
7860   ins_pipe(ialu_reg);
7861 %}
7862 
7863 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7864 %{
7865   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7866 
7867   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7868   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7869   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7870   ins_pipe(ialu_reg_reg_alu0);
7871 %}
7872 
7873 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7874 %{
7875   effect(USE_DEF dst, KILL cr);
7876 
7877   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7878   opcode(0xC1, 0x7); /* C1 /7 ib */
7879   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7880   ins_pipe(ialu_reg);
7881 %}
7882 
7883 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7884 %{
7885   effect(USE_DEF dst, KILL cr);
7886 
7887   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7888   opcode(0xC1, 0x7); /* C1 /7 ib */
7889   ins_encode(reg_opc_imm_wide(dst, 0x2));
7890   ins_pipe(ialu_reg);
7891 %}
7892 
7893 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7894 %{
7895   match(Set dst (DivL src div));
7896 
7897   ins_cost((5+8)*100);
7898   expand %{
7899     rax_RegL rax;                     // Killed temp
7900     rFlagsReg cr;                     // Killed
7901     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7902     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7903     sarL_rReg_63(src, cr);            // sarq  src, 63
7904     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7905     subL_rReg(dst, src, cr);          // subl  rdx, src
7906   %}
7907 %}
7908 
7909 //-----------------------------------------------------------------------------
7910 
7911 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7912                    rFlagsReg cr)
7913 %{
7914   match(Set rdx (ModI rax div));
7915   effect(KILL rax, KILL cr);
7916 
7917   ins_cost(300); // XXX
7918   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7919             "jne,s   normal\n\t"
7920             "xorl    rdx, rdx\n\t"
7921             "cmpl    $div, -1\n\t"
7922             "je,s    done\n"
7923     "normal: cdql\n\t"
7924             "idivl   $div\n"
7925     "done:"        %}
7926   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7927   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7928   ins_pipe(ialu_reg_reg_alu0);
7929 %}
7930 
7931 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7932                    rFlagsReg cr)
7933 %{
7934   match(Set rdx (ModL rax div));
7935   effect(KILL rax, KILL cr);
7936 
7937   ins_cost(300); // XXX
7938   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7939             "cmpq    rax, rdx\n\t"
7940             "jne,s   normal\n\t"
7941             "xorl    rdx, rdx\n\t"
7942             "cmpq    $div, -1\n\t"
7943             "je,s    done\n"
7944     "normal: cdqq\n\t"
7945             "idivq   $div\n"
7946     "done:"        %}
7947   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7948   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7949   ins_pipe(ialu_reg_reg_alu0);
7950 %}
7951 
7952 // Integer Shift Instructions
7953 // Shift Left by one
7954 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7955 %{
7956   match(Set dst (LShiftI dst shift));
7957   effect(KILL cr);
7958 
7959   format %{ "sall    $dst, $shift" %}
7960   opcode(0xD1, 0x4); /* D1 /4 */
7961   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7962   ins_pipe(ialu_reg);
7963 %}
7964 
7965 // Shift Left by one
7966 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7967 %{
7968   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7969   effect(KILL cr);
7970 
7971   format %{ "sall    $dst, $shift\t" %}
7972   opcode(0xD1, 0x4); /* D1 /4 */
7973   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7974   ins_pipe(ialu_mem_imm);
7975 %}
7976 
7977 // Shift Left by 8-bit immediate
7978 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7979 %{
7980   match(Set dst (LShiftI dst shift));
7981   effect(KILL cr);
7982 
7983   format %{ "sall    $dst, $shift" %}
7984   opcode(0xC1, 0x4); /* C1 /4 ib */
7985   ins_encode(reg_opc_imm(dst, shift));
7986   ins_pipe(ialu_reg);
7987 %}
7988 
7989 // Shift Left by 8-bit immediate
7990 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7991 %{
7992   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7993   effect(KILL cr);
7994 
7995   format %{ "sall    $dst, $shift" %}
7996   opcode(0xC1, 0x4); /* C1 /4 ib */
7997   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7998   ins_pipe(ialu_mem_imm);
7999 %}
8000 
8001 // Shift Left by variable
8002 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8003 %{
8004   match(Set dst (LShiftI dst shift));
8005   effect(KILL cr);
8006 
8007   format %{ "sall    $dst, $shift" %}
8008   opcode(0xD3, 0x4); /* D3 /4 */
8009   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8010   ins_pipe(ialu_reg_reg);
8011 %}
8012 
8013 // Shift Left by variable
8014 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8015 %{
8016   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
8017   effect(KILL cr);
8018 
8019   format %{ "sall    $dst, $shift" %}
8020   opcode(0xD3, 0x4); /* D3 /4 */
8021   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8022   ins_pipe(ialu_mem_reg);
8023 %}
8024 
8025 // Arithmetic shift right by one
8026 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8027 %{
8028   match(Set dst (RShiftI dst shift));
8029   effect(KILL cr);
8030 
8031   format %{ "sarl    $dst, $shift" %}
8032   opcode(0xD1, 0x7); /* D1 /7 */
8033   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8034   ins_pipe(ialu_reg);
8035 %}
8036 
8037 // Arithmetic shift right by one
8038 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8039 %{
8040   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8041   effect(KILL cr);
8042 
8043   format %{ "sarl    $dst, $shift" %}
8044   opcode(0xD1, 0x7); /* D1 /7 */
8045   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8046   ins_pipe(ialu_mem_imm);
8047 %}
8048 
8049 // Arithmetic Shift Right by 8-bit immediate
8050 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8051 %{
8052   match(Set dst (RShiftI dst shift));
8053   effect(KILL cr);
8054 
8055   format %{ "sarl    $dst, $shift" %}
8056   opcode(0xC1, 0x7); /* C1 /7 ib */
8057   ins_encode(reg_opc_imm(dst, shift));
8058   ins_pipe(ialu_mem_imm);
8059 %}
8060 
8061 // Arithmetic Shift Right by 8-bit immediate
8062 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8063 %{
8064   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8065   effect(KILL cr);
8066 
8067   format %{ "sarl    $dst, $shift" %}
8068   opcode(0xC1, 0x7); /* C1 /7 ib */
8069   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8070   ins_pipe(ialu_mem_imm);
8071 %}
8072 
8073 // Arithmetic Shift Right by variable
8074 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8075 %{
8076   match(Set dst (RShiftI dst shift));
8077   effect(KILL cr);
8078 
8079   format %{ "sarl    $dst, $shift" %}
8080   opcode(0xD3, 0x7); /* D3 /7 */
8081   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8082   ins_pipe(ialu_reg_reg);
8083 %}
8084 
8085 // Arithmetic Shift Right by variable
8086 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8087 %{
8088   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8089   effect(KILL cr);
8090 
8091   format %{ "sarl    $dst, $shift" %}
8092   opcode(0xD3, 0x7); /* D3 /7 */
8093   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8094   ins_pipe(ialu_mem_reg);
8095 %}
8096 
8097 // Logical shift right by one
8098 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8099 %{
8100   match(Set dst (URShiftI dst shift));
8101   effect(KILL cr);
8102 
8103   format %{ "shrl    $dst, $shift" %}
8104   opcode(0xD1, 0x5); /* D1 /5 */
8105   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8106   ins_pipe(ialu_reg);
8107 %}
8108 
8109 // Logical shift right by one
8110 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8111 %{
8112   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8113   effect(KILL cr);
8114 
8115   format %{ "shrl    $dst, $shift" %}
8116   opcode(0xD1, 0x5); /* D1 /5 */
8117   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8118   ins_pipe(ialu_mem_imm);
8119 %}
8120 
8121 // Logical Shift Right by 8-bit immediate
8122 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8123 %{
8124   match(Set dst (URShiftI dst shift));
8125   effect(KILL cr);
8126 
8127   format %{ "shrl    $dst, $shift" %}
8128   opcode(0xC1, 0x5); /* C1 /5 ib */
8129   ins_encode(reg_opc_imm(dst, shift));
8130   ins_pipe(ialu_reg);
8131 %}
8132 
8133 // Logical Shift Right by 8-bit immediate
8134 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8135 %{
8136   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8137   effect(KILL cr);
8138 
8139   format %{ "shrl    $dst, $shift" %}
8140   opcode(0xC1, 0x5); /* C1 /5 ib */
8141   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8142   ins_pipe(ialu_mem_imm);
8143 %}
8144 
8145 // Logical Shift Right by variable
8146 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8147 %{
8148   match(Set dst (URShiftI dst shift));
8149   effect(KILL cr);
8150 
8151   format %{ "shrl    $dst, $shift" %}
8152   opcode(0xD3, 0x5); /* D3 /5 */
8153   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8154   ins_pipe(ialu_reg_reg);
8155 %}
8156 
8157 // Logical Shift Right by variable
8158 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8159 %{
8160   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8161   effect(KILL cr);
8162 
8163   format %{ "shrl    $dst, $shift" %}
8164   opcode(0xD3, 0x5); /* D3 /5 */
8165   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8166   ins_pipe(ialu_mem_reg);
8167 %}
8168 
8169 // Long Shift Instructions
8170 // Shift Left by one
8171 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8172 %{
8173   match(Set dst (LShiftL dst shift));
8174   effect(KILL cr);
8175 
8176   format %{ "salq    $dst, $shift" %}
8177   opcode(0xD1, 0x4); /* D1 /4 */
8178   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8179   ins_pipe(ialu_reg);
8180 %}
8181 
8182 // Shift Left by one
8183 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8184 %{
8185   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8186   effect(KILL cr);
8187 
8188   format %{ "salq    $dst, $shift" %}
8189   opcode(0xD1, 0x4); /* D1 /4 */
8190   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8191   ins_pipe(ialu_mem_imm);
8192 %}
8193 
8194 // Shift Left by 8-bit immediate
8195 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8196 %{
8197   match(Set dst (LShiftL dst shift));
8198   effect(KILL cr);
8199 
8200   format %{ "salq    $dst, $shift" %}
8201   opcode(0xC1, 0x4); /* C1 /4 ib */
8202   ins_encode(reg_opc_imm_wide(dst, shift));
8203   ins_pipe(ialu_reg);
8204 %}
8205 
8206 // Shift Left by 8-bit immediate
8207 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8208 %{
8209   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8210   effect(KILL cr);
8211 
8212   format %{ "salq    $dst, $shift" %}
8213   opcode(0xC1, 0x4); /* C1 /4 ib */
8214   ins_encode(REX_mem_wide(dst), OpcP,
8215              RM_opc_mem(secondary, dst), Con8or32(shift));
8216   ins_pipe(ialu_mem_imm);
8217 %}
8218 
8219 // Shift Left by variable
8220 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8221 %{
8222   match(Set dst (LShiftL dst shift));
8223   effect(KILL cr);
8224 
8225   format %{ "salq    $dst, $shift" %}
8226   opcode(0xD3, 0x4); /* D3 /4 */
8227   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8228   ins_pipe(ialu_reg_reg);
8229 %}
8230 
8231 // Shift Left by variable
8232 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8233 %{
8234   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8235   effect(KILL cr);
8236 
8237   format %{ "salq    $dst, $shift" %}
8238   opcode(0xD3, 0x4); /* D3 /4 */
8239   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8240   ins_pipe(ialu_mem_reg);
8241 %}
8242 
8243 // Arithmetic shift right by one
8244 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8245 %{
8246   match(Set dst (RShiftL dst shift));
8247   effect(KILL cr);
8248 
8249   format %{ "sarq    $dst, $shift" %}
8250   opcode(0xD1, 0x7); /* D1 /7 */
8251   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8252   ins_pipe(ialu_reg);
8253 %}
8254 
8255 // Arithmetic shift right by one
8256 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8257 %{
8258   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8259   effect(KILL cr);
8260 
8261   format %{ "sarq    $dst, $shift" %}
8262   opcode(0xD1, 0x7); /* D1 /7 */
8263   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8264   ins_pipe(ialu_mem_imm);
8265 %}
8266 
8267 // Arithmetic Shift Right by 8-bit immediate
8268 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8269 %{
8270   match(Set dst (RShiftL dst shift));
8271   effect(KILL cr);
8272 
8273   format %{ "sarq    $dst, $shift" %}
8274   opcode(0xC1, 0x7); /* C1 /7 ib */
8275   ins_encode(reg_opc_imm_wide(dst, shift));
8276   ins_pipe(ialu_mem_imm);
8277 %}
8278 
8279 // Arithmetic Shift Right by 8-bit immediate
8280 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8281 %{
8282   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8283   effect(KILL cr);
8284 
8285   format %{ "sarq    $dst, $shift" %}
8286   opcode(0xC1, 0x7); /* C1 /7 ib */
8287   ins_encode(REX_mem_wide(dst), OpcP,
8288              RM_opc_mem(secondary, dst), Con8or32(shift));
8289   ins_pipe(ialu_mem_imm);
8290 %}
8291 
8292 // Arithmetic Shift Right by variable
8293 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8294 %{
8295   match(Set dst (RShiftL dst shift));
8296   effect(KILL cr);
8297 
8298   format %{ "sarq    $dst, $shift" %}
8299   opcode(0xD3, 0x7); /* D3 /7 */
8300   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8301   ins_pipe(ialu_reg_reg);
8302 %}
8303 
8304 // Arithmetic Shift Right by variable
8305 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8306 %{
8307   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8308   effect(KILL cr);
8309 
8310   format %{ "sarq    $dst, $shift" %}
8311   opcode(0xD3, 0x7); /* D3 /7 */
8312   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8313   ins_pipe(ialu_mem_reg);
8314 %}
8315 
8316 // Logical shift right by one
8317 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8318 %{
8319   match(Set dst (URShiftL dst shift));
8320   effect(KILL cr);
8321 
8322   format %{ "shrq    $dst, $shift" %}
8323   opcode(0xD1, 0x5); /* D1 /5 */
8324   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8325   ins_pipe(ialu_reg);
8326 %}
8327 
8328 // Logical shift right by one
8329 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8330 %{
8331   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8332   effect(KILL cr);
8333 
8334   format %{ "shrq    $dst, $shift" %}
8335   opcode(0xD1, 0x5); /* D1 /5 */
8336   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8337   ins_pipe(ialu_mem_imm);
8338 %}
8339 
8340 // Logical Shift Right by 8-bit immediate
8341 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8342 %{
8343   match(Set dst (URShiftL dst shift));
8344   effect(KILL cr);
8345 
8346   format %{ "shrq    $dst, $shift" %}
8347   opcode(0xC1, 0x5); /* C1 /5 ib */
8348   ins_encode(reg_opc_imm_wide(dst, shift));
8349   ins_pipe(ialu_reg);
8350 %}
8351 
8352 
8353 // Logical Shift Right by 8-bit immediate
8354 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8355 %{
8356   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8357   effect(KILL cr);
8358 
8359   format %{ "shrq    $dst, $shift" %}
8360   opcode(0xC1, 0x5); /* C1 /5 ib */
8361   ins_encode(REX_mem_wide(dst), OpcP,
8362              RM_opc_mem(secondary, dst), Con8or32(shift));
8363   ins_pipe(ialu_mem_imm);
8364 %}
8365 
8366 // Logical Shift Right by variable
8367 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8368 %{
8369   match(Set dst (URShiftL dst shift));
8370   effect(KILL cr);
8371 
8372   format %{ "shrq    $dst, $shift" %}
8373   opcode(0xD3, 0x5); /* D3 /5 */
8374   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8375   ins_pipe(ialu_reg_reg);
8376 %}
8377 
8378 // Logical Shift Right by variable
8379 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8380 %{
8381   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8382   effect(KILL cr);
8383 
8384   format %{ "shrq    $dst, $shift" %}
8385   opcode(0xD3, 0x5); /* D3 /5 */
8386   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8387   ins_pipe(ialu_mem_reg);
8388 %}
8389 
8390 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8391 // This idiom is used by the compiler for the i2b bytecode.
8392 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8393 %{
8394   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8395 
8396   format %{ "movsbl  $dst, $src\t# i2b" %}
8397   opcode(0x0F, 0xBE);
8398   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8399   ins_pipe(ialu_reg_reg);
8400 %}
8401 
8402 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8403 // This idiom is used by the compiler the i2s bytecode.
8404 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8405 %{
8406   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8407 
8408   format %{ "movswl  $dst, $src\t# i2s" %}
8409   opcode(0x0F, 0xBF);
8410   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8411   ins_pipe(ialu_reg_reg);
8412 %}
8413 
8414 // ROL/ROR instructions
8415 
8416 // ROL expand
8417 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8418   effect(KILL cr, USE_DEF dst);
8419 
8420   format %{ "roll    $dst" %}
8421   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8422   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8423   ins_pipe(ialu_reg);
8424 %}
8425 
8426 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8427   effect(USE_DEF dst, USE shift, KILL cr);
8428 
8429   format %{ "roll    $dst, $shift" %}
8430   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8431   ins_encode( reg_opc_imm(dst, shift) );
8432   ins_pipe(ialu_reg);
8433 %}
8434 
8435 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8436 %{
8437   effect(USE_DEF dst, USE shift, KILL cr);
8438 
8439   format %{ "roll    $dst, $shift" %}
8440   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8441   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8442   ins_pipe(ialu_reg_reg);
8443 %}
8444 // end of ROL expand
8445 
8446 // Rotate Left by one
8447 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8448 %{
8449   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8450 
8451   expand %{
8452     rolI_rReg_imm1(dst, cr);
8453   %}
8454 %}
8455 
8456 // Rotate Left by 8-bit immediate
8457 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8458 %{
8459   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8460   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8461 
8462   expand %{
8463     rolI_rReg_imm8(dst, lshift, cr);
8464   %}
8465 %}
8466 
8467 // Rotate Left by variable
8468 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8469 %{
8470   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8471 
8472   expand %{
8473     rolI_rReg_CL(dst, shift, cr);
8474   %}
8475 %}
8476 
8477 // Rotate Left by variable
8478 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8479 %{
8480   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8481 
8482   expand %{
8483     rolI_rReg_CL(dst, shift, cr);
8484   %}
8485 %}
8486 
8487 // ROR expand
8488 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8489 %{
8490   effect(USE_DEF dst, KILL cr);
8491 
8492   format %{ "rorl    $dst" %}
8493   opcode(0xD1, 0x1); /* D1 /1 */
8494   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8495   ins_pipe(ialu_reg);
8496 %}
8497 
8498 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8499 %{
8500   effect(USE_DEF dst, USE shift, KILL cr);
8501 
8502   format %{ "rorl    $dst, $shift" %}
8503   opcode(0xC1, 0x1); /* C1 /1 ib */
8504   ins_encode(reg_opc_imm(dst, shift));
8505   ins_pipe(ialu_reg);
8506 %}
8507 
8508 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8509 %{
8510   effect(USE_DEF dst, USE shift, KILL cr);
8511 
8512   format %{ "rorl    $dst, $shift" %}
8513   opcode(0xD3, 0x1); /* D3 /1 */
8514   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8515   ins_pipe(ialu_reg_reg);
8516 %}
8517 // end of ROR expand
8518 
8519 // Rotate Right by one
8520 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8521 %{
8522   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8523 
8524   expand %{
8525     rorI_rReg_imm1(dst, cr);
8526   %}
8527 %}
8528 
8529 // Rotate Right by 8-bit immediate
8530 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8531 %{
8532   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8533   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8534 
8535   expand %{
8536     rorI_rReg_imm8(dst, rshift, cr);
8537   %}
8538 %}
8539 
8540 // Rotate Right by variable
8541 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8542 %{
8543   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8544 
8545   expand %{
8546     rorI_rReg_CL(dst, shift, cr);
8547   %}
8548 %}
8549 
8550 // Rotate Right by variable
8551 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8552 %{
8553   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8554 
8555   expand %{
8556     rorI_rReg_CL(dst, shift, cr);
8557   %}
8558 %}
8559 
8560 // for long rotate
8561 // ROL expand
8562 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8563   effect(USE_DEF dst, KILL cr);
8564 
8565   format %{ "rolq    $dst" %}
8566   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8567   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8568   ins_pipe(ialu_reg);
8569 %}
8570 
8571 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8572   effect(USE_DEF dst, USE shift, KILL cr);
8573 
8574   format %{ "rolq    $dst, $shift" %}
8575   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8576   ins_encode( reg_opc_imm_wide(dst, shift) );
8577   ins_pipe(ialu_reg);
8578 %}
8579 
8580 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8581 %{
8582   effect(USE_DEF dst, USE shift, KILL cr);
8583 
8584   format %{ "rolq    $dst, $shift" %}
8585   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8586   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8587   ins_pipe(ialu_reg_reg);
8588 %}
8589 // end of ROL expand
8590 
8591 // Rotate Left by one
8592 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8593 %{
8594   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8595 
8596   expand %{
8597     rolL_rReg_imm1(dst, cr);
8598   %}
8599 %}
8600 
8601 // Rotate Left by 8-bit immediate
8602 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8603 %{
8604   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8605   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8606 
8607   expand %{
8608     rolL_rReg_imm8(dst, lshift, cr);
8609   %}
8610 %}
8611 
8612 // Rotate Left by variable
8613 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8614 %{
8615   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8616 
8617   expand %{
8618     rolL_rReg_CL(dst, shift, cr);
8619   %}
8620 %}
8621 
8622 // Rotate Left by variable
8623 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8624 %{
8625   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8626 
8627   expand %{
8628     rolL_rReg_CL(dst, shift, cr);
8629   %}
8630 %}
8631 
8632 // ROR expand
8633 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8634 %{
8635   effect(USE_DEF dst, KILL cr);
8636 
8637   format %{ "rorq    $dst" %}
8638   opcode(0xD1, 0x1); /* D1 /1 */
8639   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8640   ins_pipe(ialu_reg);
8641 %}
8642 
8643 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8644 %{
8645   effect(USE_DEF dst, USE shift, KILL cr);
8646 
8647   format %{ "rorq    $dst, $shift" %}
8648   opcode(0xC1, 0x1); /* C1 /1 ib */
8649   ins_encode(reg_opc_imm_wide(dst, shift));
8650   ins_pipe(ialu_reg);
8651 %}
8652 
8653 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8654 %{
8655   effect(USE_DEF dst, USE shift, KILL cr);
8656 
8657   format %{ "rorq    $dst, $shift" %}
8658   opcode(0xD3, 0x1); /* D3 /1 */
8659   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8660   ins_pipe(ialu_reg_reg);
8661 %}
8662 // end of ROR expand
8663 
8664 // Rotate Right by one
8665 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8666 %{
8667   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8668 
8669   expand %{
8670     rorL_rReg_imm1(dst, cr);
8671   %}
8672 %}
8673 
8674 // Rotate Right by 8-bit immediate
8675 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8676 %{
8677   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8678   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8679 
8680   expand %{
8681     rorL_rReg_imm8(dst, rshift, cr);
8682   %}
8683 %}
8684 
8685 // Rotate Right by variable
8686 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8687 %{
8688   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8689 
8690   expand %{
8691     rorL_rReg_CL(dst, shift, cr);
8692   %}
8693 %}
8694 
8695 // Rotate Right by variable
8696 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8697 %{
8698   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8699 
8700   expand %{
8701     rorL_rReg_CL(dst, shift, cr);
8702   %}
8703 %}
8704 
8705 // Logical Instructions
8706 
8707 // Integer Logical Instructions
8708 
8709 // And Instructions
8710 // And Register with Register
8711 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8712 %{
8713   match(Set dst (AndI dst src));
8714   effect(KILL cr);
8715 
8716   format %{ "andl    $dst, $src\t# int" %}
8717   opcode(0x23);
8718   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8719   ins_pipe(ialu_reg_reg);
8720 %}
8721 
8722 // And Register with Immediate 255
8723 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8724 %{
8725   match(Set dst (AndI dst src));
8726 
8727   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8728   opcode(0x0F, 0xB6);
8729   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8730   ins_pipe(ialu_reg);
8731 %}
8732 
8733 // And Register with Immediate 255 and promote to long
8734 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8735 %{
8736   match(Set dst (ConvI2L (AndI src mask)));
8737 
8738   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8739   opcode(0x0F, 0xB6);
8740   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8741   ins_pipe(ialu_reg);
8742 %}
8743 
8744 // And Register with Immediate 65535
8745 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8746 %{
8747   match(Set dst (AndI dst src));
8748 
8749   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8750   opcode(0x0F, 0xB7);
8751   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8752   ins_pipe(ialu_reg);
8753 %}
8754 
8755 // And Register with Immediate 65535 and promote to long
8756 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8757 %{
8758   match(Set dst (ConvI2L (AndI src mask)));
8759 
8760   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8761   opcode(0x0F, 0xB7);
8762   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8763   ins_pipe(ialu_reg);
8764 %}
8765 
8766 // And Register with Immediate
8767 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8768 %{
8769   match(Set dst (AndI dst src));
8770   effect(KILL cr);
8771 
8772   format %{ "andl    $dst, $src\t# int" %}
8773   opcode(0x81, 0x04); /* Opcode 81 /4 */
8774   ins_encode(OpcSErm(dst, src), Con8or32(src));
8775   ins_pipe(ialu_reg);
8776 %}
8777 
8778 // And Register with Memory
8779 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8780 %{
8781   match(Set dst (AndI dst (LoadI src)));
8782   effect(KILL cr);
8783 
8784   ins_cost(125);
8785   format %{ "andl    $dst, $src\t# int" %}
8786   opcode(0x23);
8787   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8788   ins_pipe(ialu_reg_mem);
8789 %}
8790 
8791 // And Memory with Register
8792 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8793 %{
8794   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8795   effect(KILL cr);
8796 
8797   ins_cost(150);
8798   format %{ "andl    $dst, $src\t# int" %}
8799   opcode(0x21); /* Opcode 21 /r */
8800   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8801   ins_pipe(ialu_mem_reg);
8802 %}
8803 
8804 // And Memory with Immediate
8805 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8806 %{
8807   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8808   effect(KILL cr);
8809 
8810   ins_cost(125);
8811   format %{ "andl    $dst, $src\t# int" %}
8812   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8813   ins_encode(REX_mem(dst), OpcSE(src),
8814              RM_opc_mem(secondary, dst), Con8or32(src));
8815   ins_pipe(ialu_mem_imm);
8816 %}
8817 
8818 // BMI1 instructions
8819 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8820   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8821   predicate(UseBMI1Instructions);
8822   effect(KILL cr);
8823 
8824   ins_cost(125);
8825   format %{ "andnl  $dst, $src1, $src2" %}
8826 
8827   ins_encode %{
8828     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8829   %}
8830   ins_pipe(ialu_reg_mem);
8831 %}
8832 
8833 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8834   match(Set dst (AndI (XorI src1 minus_1) src2));
8835   predicate(UseBMI1Instructions);
8836   effect(KILL cr);
8837 
8838   format %{ "andnl  $dst, $src1, $src2" %}
8839 
8840   ins_encode %{
8841     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8842   %}
8843   ins_pipe(ialu_reg);
8844 %}
8845 
8846 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8847   match(Set dst (AndI (SubI imm_zero src) src));
8848   predicate(UseBMI1Instructions);
8849   effect(KILL cr);
8850 
8851   format %{ "blsil  $dst, $src" %}
8852 
8853   ins_encode %{
8854     __ blsil($dst$$Register, $src$$Register);
8855   %}
8856   ins_pipe(ialu_reg);
8857 %}
8858 
8859 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8860   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8861   predicate(UseBMI1Instructions);
8862   effect(KILL cr);
8863 
8864   ins_cost(125);
8865   format %{ "blsil  $dst, $src" %}
8866 
8867   ins_encode %{
8868     __ blsil($dst$$Register, $src$$Address);
8869   %}
8870   ins_pipe(ialu_reg_mem);
8871 %}
8872 
8873 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8874 %{
8875   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8876   predicate(UseBMI1Instructions);
8877   effect(KILL cr);
8878 
8879   ins_cost(125);
8880   format %{ "blsmskl $dst, $src" %}
8881 
8882   ins_encode %{
8883     __ blsmskl($dst$$Register, $src$$Address);
8884   %}
8885   ins_pipe(ialu_reg_mem);
8886 %}
8887 
8888 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8889 %{
8890   match(Set dst (XorI (AddI src minus_1) src));
8891   predicate(UseBMI1Instructions);
8892   effect(KILL cr);
8893 
8894   format %{ "blsmskl $dst, $src" %}
8895 
8896   ins_encode %{
8897     __ blsmskl($dst$$Register, $src$$Register);
8898   %}
8899 
8900   ins_pipe(ialu_reg);
8901 %}
8902 
8903 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8904 %{
8905   match(Set dst (AndI (AddI src minus_1) src) );
8906   predicate(UseBMI1Instructions);
8907   effect(KILL cr);
8908 
8909   format %{ "blsrl  $dst, $src" %}
8910 
8911   ins_encode %{
8912     __ blsrl($dst$$Register, $src$$Register);
8913   %}
8914 
8915   ins_pipe(ialu_reg_mem);
8916 %}
8917 
8918 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8919 %{
8920   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8921   predicate(UseBMI1Instructions);
8922   effect(KILL cr);
8923 
8924   ins_cost(125);
8925   format %{ "blsrl  $dst, $src" %}
8926 
8927   ins_encode %{
8928     __ blsrl($dst$$Register, $src$$Address);
8929   %}
8930 
8931   ins_pipe(ialu_reg);
8932 %}
8933 
8934 // Or Instructions
8935 // Or Register with Register
8936 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8937 %{
8938   match(Set dst (OrI dst src));
8939   effect(KILL cr);
8940 
8941   format %{ "orl     $dst, $src\t# int" %}
8942   opcode(0x0B);
8943   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8944   ins_pipe(ialu_reg_reg);
8945 %}
8946 
8947 // Or Register with Immediate
8948 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8949 %{
8950   match(Set dst (OrI dst src));
8951   effect(KILL cr);
8952 
8953   format %{ "orl     $dst, $src\t# int" %}
8954   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8955   ins_encode(OpcSErm(dst, src), Con8or32(src));
8956   ins_pipe(ialu_reg);
8957 %}
8958 
8959 // Or Register with Memory
8960 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8961 %{
8962   match(Set dst (OrI dst (LoadI src)));
8963   effect(KILL cr);
8964 
8965   ins_cost(125);
8966   format %{ "orl     $dst, $src\t# int" %}
8967   opcode(0x0B);
8968   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8969   ins_pipe(ialu_reg_mem);
8970 %}
8971 
8972 // Or Memory with Register
8973 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8974 %{
8975   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8976   effect(KILL cr);
8977 
8978   ins_cost(150);
8979   format %{ "orl     $dst, $src\t# int" %}
8980   opcode(0x09); /* Opcode 09 /r */
8981   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8982   ins_pipe(ialu_mem_reg);
8983 %}
8984 
8985 // Or Memory with Immediate
8986 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8987 %{
8988   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8989   effect(KILL cr);
8990 
8991   ins_cost(125);
8992   format %{ "orl     $dst, $src\t# int" %}
8993   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8994   ins_encode(REX_mem(dst), OpcSE(src),
8995              RM_opc_mem(secondary, dst), Con8or32(src));
8996   ins_pipe(ialu_mem_imm);
8997 %}
8998 
8999 // Xor Instructions
9000 // Xor Register with Register
9001 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
9002 %{
9003   match(Set dst (XorI dst src));
9004   effect(KILL cr);
9005 
9006   format %{ "xorl    $dst, $src\t# int" %}
9007   opcode(0x33);
9008   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
9009   ins_pipe(ialu_reg_reg);
9010 %}
9011 
9012 // Xor Register with Immediate -1
9013 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
9014   match(Set dst (XorI dst imm));
9015 
9016   format %{ "not    $dst" %}
9017   ins_encode %{
9018      __ notl($dst$$Register);
9019   %}
9020   ins_pipe(ialu_reg);
9021 %}
9022 
9023 // Xor Register with Immediate
9024 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9025 %{
9026   match(Set dst (XorI dst src));
9027   effect(KILL cr);
9028 
9029   format %{ "xorl    $dst, $src\t# int" %}
9030   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9031   ins_encode(OpcSErm(dst, src), Con8or32(src));
9032   ins_pipe(ialu_reg);
9033 %}
9034 
9035 // Xor Register with Memory
9036 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9037 %{
9038   match(Set dst (XorI dst (LoadI src)));
9039   effect(KILL cr);
9040 
9041   ins_cost(125);
9042   format %{ "xorl    $dst, $src\t# int" %}
9043   opcode(0x33);
9044   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9045   ins_pipe(ialu_reg_mem);
9046 %}
9047 
9048 // Xor Memory with Register
9049 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9050 %{
9051   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9052   effect(KILL cr);
9053 
9054   ins_cost(150);
9055   format %{ "xorl    $dst, $src\t# int" %}
9056   opcode(0x31); /* Opcode 31 /r */
9057   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9058   ins_pipe(ialu_mem_reg);
9059 %}
9060 
9061 // Xor Memory with Immediate
9062 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9063 %{
9064   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9065   effect(KILL cr);
9066 
9067   ins_cost(125);
9068   format %{ "xorl    $dst, $src\t# int" %}
9069   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9070   ins_encode(REX_mem(dst), OpcSE(src),
9071              RM_opc_mem(secondary, dst), Con8or32(src));
9072   ins_pipe(ialu_mem_imm);
9073 %}
9074 
9075 
9076 // Long Logical Instructions
9077 
9078 // And Instructions
9079 // And Register with Register
9080 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9081 %{
9082   match(Set dst (AndL dst src));
9083   effect(KILL cr);
9084 
9085   format %{ "andq    $dst, $src\t# long" %}
9086   opcode(0x23);
9087   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9088   ins_pipe(ialu_reg_reg);
9089 %}
9090 
9091 // And Register with Immediate 255
9092 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9093 %{
9094   match(Set dst (AndL dst src));
9095 
9096   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9097   opcode(0x0F, 0xB6);
9098   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9099   ins_pipe(ialu_reg);
9100 %}
9101 
9102 // And Register with Immediate 65535
9103 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9104 %{
9105   match(Set dst (AndL dst src));
9106 
9107   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9108   opcode(0x0F, 0xB7);
9109   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9110   ins_pipe(ialu_reg);
9111 %}
9112 
9113 // And Register with Immediate
9114 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9115 %{
9116   match(Set dst (AndL dst src));
9117   effect(KILL cr);
9118 
9119   format %{ "andq    $dst, $src\t# long" %}
9120   opcode(0x81, 0x04); /* Opcode 81 /4 */
9121   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9122   ins_pipe(ialu_reg);
9123 %}
9124 
9125 // And Register with Memory
9126 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9127 %{
9128   match(Set dst (AndL dst (LoadL src)));
9129   effect(KILL cr);
9130 
9131   ins_cost(125);
9132   format %{ "andq    $dst, $src\t# long" %}
9133   opcode(0x23);
9134   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9135   ins_pipe(ialu_reg_mem);
9136 %}
9137 
9138 // And Memory with Register
9139 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9140 %{
9141   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9142   effect(KILL cr);
9143 
9144   ins_cost(150);
9145   format %{ "andq    $dst, $src\t# long" %}
9146   opcode(0x21); /* Opcode 21 /r */
9147   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9148   ins_pipe(ialu_mem_reg);
9149 %}
9150 
9151 // And Memory with Immediate
9152 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9153 %{
9154   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9155   effect(KILL cr);
9156 
9157   ins_cost(125);
9158   format %{ "andq    $dst, $src\t# long" %}
9159   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9160   ins_encode(REX_mem_wide(dst), OpcSE(src),
9161              RM_opc_mem(secondary, dst), Con8or32(src));
9162   ins_pipe(ialu_mem_imm);
9163 %}
9164 
9165 // BMI1 instructions
9166 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9167   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9168   predicate(UseBMI1Instructions);
9169   effect(KILL cr);
9170 
9171   ins_cost(125);
9172   format %{ "andnq  $dst, $src1, $src2" %}
9173 
9174   ins_encode %{
9175     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9176   %}
9177   ins_pipe(ialu_reg_mem);
9178 %}
9179 
9180 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9181   match(Set dst (AndL (XorL src1 minus_1) src2));
9182   predicate(UseBMI1Instructions);
9183   effect(KILL cr);
9184 
9185   format %{ "andnq  $dst, $src1, $src2" %}
9186 
9187   ins_encode %{
9188   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9189   %}
9190   ins_pipe(ialu_reg_mem);
9191 %}
9192 
9193 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9194   match(Set dst (AndL (SubL imm_zero src) src));
9195   predicate(UseBMI1Instructions);
9196   effect(KILL cr);
9197 
9198   format %{ "blsiq  $dst, $src" %}
9199 
9200   ins_encode %{
9201     __ blsiq($dst$$Register, $src$$Register);
9202   %}
9203   ins_pipe(ialu_reg);
9204 %}
9205 
9206 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9207   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9208   predicate(UseBMI1Instructions);
9209   effect(KILL cr);
9210 
9211   ins_cost(125);
9212   format %{ "blsiq  $dst, $src" %}
9213 
9214   ins_encode %{
9215     __ blsiq($dst$$Register, $src$$Address);
9216   %}
9217   ins_pipe(ialu_reg_mem);
9218 %}
9219 
9220 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9221 %{
9222   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9223   predicate(UseBMI1Instructions);
9224   effect(KILL cr);
9225 
9226   ins_cost(125);
9227   format %{ "blsmskq $dst, $src" %}
9228 
9229   ins_encode %{
9230     __ blsmskq($dst$$Register, $src$$Address);
9231   %}
9232   ins_pipe(ialu_reg_mem);
9233 %}
9234 
9235 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9236 %{
9237   match(Set dst (XorL (AddL src minus_1) src));
9238   predicate(UseBMI1Instructions);
9239   effect(KILL cr);
9240 
9241   format %{ "blsmskq $dst, $src" %}
9242 
9243   ins_encode %{
9244     __ blsmskq($dst$$Register, $src$$Register);
9245   %}
9246 
9247   ins_pipe(ialu_reg);
9248 %}
9249 
9250 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9251 %{
9252   match(Set dst (AndL (AddL src minus_1) src) );
9253   predicate(UseBMI1Instructions);
9254   effect(KILL cr);
9255 
9256   format %{ "blsrq  $dst, $src" %}
9257 
9258   ins_encode %{
9259     __ blsrq($dst$$Register, $src$$Register);
9260   %}
9261 
9262   ins_pipe(ialu_reg);
9263 %}
9264 
9265 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9266 %{
9267   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9268   predicate(UseBMI1Instructions);
9269   effect(KILL cr);
9270 
9271   ins_cost(125);
9272   format %{ "blsrq  $dst, $src" %}
9273 
9274   ins_encode %{
9275     __ blsrq($dst$$Register, $src$$Address);
9276   %}
9277 
9278   ins_pipe(ialu_reg);
9279 %}
9280 
9281 // Or Instructions
9282 // Or Register with Register
9283 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9284 %{
9285   match(Set dst (OrL dst src));
9286   effect(KILL cr);
9287 
9288   format %{ "orq     $dst, $src\t# long" %}
9289   opcode(0x0B);
9290   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9291   ins_pipe(ialu_reg_reg);
9292 %}
9293 
9294 // Use any_RegP to match R15 (TLS register) without spilling.
9295 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9296   match(Set dst (OrL dst (CastP2X src)));
9297   effect(KILL cr);
9298 
9299   format %{ "orq     $dst, $src\t# long" %}
9300   opcode(0x0B);
9301   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9302   ins_pipe(ialu_reg_reg);
9303 %}
9304 
9305 
9306 // Or Register with Immediate
9307 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9308 %{
9309   match(Set dst (OrL dst src));
9310   effect(KILL cr);
9311 
9312   format %{ "orq     $dst, $src\t# long" %}
9313   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9314   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9315   ins_pipe(ialu_reg);
9316 %}
9317 
9318 // Or Register with Memory
9319 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9320 %{
9321   match(Set dst (OrL dst (LoadL src)));
9322   effect(KILL cr);
9323 
9324   ins_cost(125);
9325   format %{ "orq     $dst, $src\t# long" %}
9326   opcode(0x0B);
9327   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9328   ins_pipe(ialu_reg_mem);
9329 %}
9330 
9331 // Or Memory with Register
9332 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9333 %{
9334   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9335   effect(KILL cr);
9336 
9337   ins_cost(150);
9338   format %{ "orq     $dst, $src\t# long" %}
9339   opcode(0x09); /* Opcode 09 /r */
9340   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9341   ins_pipe(ialu_mem_reg);
9342 %}
9343 
9344 // Or Memory with Immediate
9345 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9346 %{
9347   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9348   effect(KILL cr);
9349 
9350   ins_cost(125);
9351   format %{ "orq     $dst, $src\t# long" %}
9352   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9353   ins_encode(REX_mem_wide(dst), OpcSE(src),
9354              RM_opc_mem(secondary, dst), Con8or32(src));
9355   ins_pipe(ialu_mem_imm);
9356 %}
9357 
9358 // Xor Instructions
9359 // Xor Register with Register
9360 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9361 %{
9362   match(Set dst (XorL dst src));
9363   effect(KILL cr);
9364 
9365   format %{ "xorq    $dst, $src\t# long" %}
9366   opcode(0x33);
9367   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9368   ins_pipe(ialu_reg_reg);
9369 %}
9370 
9371 // Xor Register with Immediate -1
9372 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9373   match(Set dst (XorL dst imm));
9374 
9375   format %{ "notq   $dst" %}
9376   ins_encode %{
9377      __ notq($dst$$Register);
9378   %}
9379   ins_pipe(ialu_reg);
9380 %}
9381 
9382 // Xor Register with Immediate
9383 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9384 %{
9385   match(Set dst (XorL dst src));
9386   effect(KILL cr);
9387 
9388   format %{ "xorq    $dst, $src\t# long" %}
9389   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9390   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9391   ins_pipe(ialu_reg);
9392 %}
9393 
9394 // Xor Register with Memory
9395 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9396 %{
9397   match(Set dst (XorL dst (LoadL src)));
9398   effect(KILL cr);
9399 
9400   ins_cost(125);
9401   format %{ "xorq    $dst, $src\t# long" %}
9402   opcode(0x33);
9403   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9404   ins_pipe(ialu_reg_mem);
9405 %}
9406 
9407 // Xor Memory with Register
9408 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9409 %{
9410   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9411   effect(KILL cr);
9412 
9413   ins_cost(150);
9414   format %{ "xorq    $dst, $src\t# long" %}
9415   opcode(0x31); /* Opcode 31 /r */
9416   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9417   ins_pipe(ialu_mem_reg);
9418 %}
9419 
9420 // Xor Memory with Immediate
9421 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9422 %{
9423   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9424   effect(KILL cr);
9425 
9426   ins_cost(125);
9427   format %{ "xorq    $dst, $src\t# long" %}
9428   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9429   ins_encode(REX_mem_wide(dst), OpcSE(src),
9430              RM_opc_mem(secondary, dst), Con8or32(src));
9431   ins_pipe(ialu_mem_imm);
9432 %}
9433 
9434 // Convert Int to Boolean
9435 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9436 %{
9437   match(Set dst (Conv2B src));
9438   effect(KILL cr);
9439 
9440   format %{ "testl   $src, $src\t# ci2b\n\t"
9441             "setnz   $dst\n\t"
9442             "movzbl  $dst, $dst" %}
9443   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9444              setNZ_reg(dst),
9445              REX_reg_breg(dst, dst), // movzbl
9446              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9447   ins_pipe(pipe_slow); // XXX
9448 %}
9449 
9450 // Convert Pointer to Boolean
9451 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9452 %{
9453   match(Set dst (Conv2B src));
9454   effect(KILL cr);
9455 
9456   format %{ "testq   $src, $src\t# cp2b\n\t"
9457             "setnz   $dst\n\t"
9458             "movzbl  $dst, $dst" %}
9459   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9460              setNZ_reg(dst),
9461              REX_reg_breg(dst, dst), // movzbl
9462              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9463   ins_pipe(pipe_slow); // XXX
9464 %}
9465 
9466 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9467 %{
9468   match(Set dst (CmpLTMask p q));
9469   effect(KILL cr);
9470 
9471   ins_cost(400);
9472   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9473             "setlt   $dst\n\t"
9474             "movzbl  $dst, $dst\n\t"
9475             "negl    $dst" %}
9476   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9477              setLT_reg(dst),
9478              REX_reg_breg(dst, dst), // movzbl
9479              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9480              neg_reg(dst));
9481   ins_pipe(pipe_slow);
9482 %}
9483 
9484 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9485 %{
9486   match(Set dst (CmpLTMask dst zero));
9487   effect(KILL cr);
9488 
9489   ins_cost(100);
9490   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9491   ins_encode %{
9492   __ sarl($dst$$Register, 31);
9493   %}
9494   ins_pipe(ialu_reg);
9495 %}
9496 
9497 /* Better to save a register than avoid a branch */
9498 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9499 %{
9500   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9501   effect(KILL cr);
9502   ins_cost(300);
9503   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9504             "jge    done\n\t"
9505             "addl   $p,$y\n"
9506             "done:  " %}
9507   ins_encode %{
9508     Register Rp = $p$$Register;
9509     Register Rq = $q$$Register;
9510     Register Ry = $y$$Register;
9511     Label done;
9512     __ subl(Rp, Rq);
9513     __ jccb(Assembler::greaterEqual, done);
9514     __ addl(Rp, Ry);
9515     __ bind(done);
9516   %}
9517   ins_pipe(pipe_cmplt);
9518 %}
9519 
9520 /* Better to save a register than avoid a branch */
9521 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9522 %{
9523   match(Set y (AndI (CmpLTMask p q) y));
9524   effect(KILL cr);
9525 
9526   ins_cost(300);
9527 
9528   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9529             "jlt      done\n\t"
9530             "xorl     $y, $y\n"
9531             "done:  " %}
9532   ins_encode %{
9533     Register Rp = $p$$Register;
9534     Register Rq = $q$$Register;
9535     Register Ry = $y$$Register;
9536     Label done;
9537     __ cmpl(Rp, Rq);
9538     __ jccb(Assembler::less, done);
9539     __ xorl(Ry, Ry);
9540     __ bind(done);
9541   %}
9542   ins_pipe(pipe_cmplt);
9543 %}
9544 
9545 
9546 //---------- FP Instructions------------------------------------------------
9547 
9548 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9549 %{
9550   match(Set cr (CmpF src1 src2));
9551 
9552   ins_cost(145);
9553   format %{ "ucomiss $src1, $src2\n\t"
9554             "jnp,s   exit\n\t"
9555             "pushfq\t# saw NaN, set CF\n\t"
9556             "andq    [rsp], #0xffffff2b\n\t"
9557             "popfq\n"
9558     "exit:" %}
9559   ins_encode %{
9560     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9561     emit_cmpfp_fixup(_masm);
9562   %}
9563   ins_pipe(pipe_slow);
9564 %}
9565 
9566 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9567   match(Set cr (CmpF src1 src2));
9568 
9569   ins_cost(100);
9570   format %{ "ucomiss $src1, $src2" %}
9571   ins_encode %{
9572     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9573   %}
9574   ins_pipe(pipe_slow);
9575 %}
9576 
9577 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9578 %{
9579   match(Set cr (CmpF src1 (LoadF src2)));
9580 
9581   ins_cost(145);
9582   format %{ "ucomiss $src1, $src2\n\t"
9583             "jnp,s   exit\n\t"
9584             "pushfq\t# saw NaN, set CF\n\t"
9585             "andq    [rsp], #0xffffff2b\n\t"
9586             "popfq\n"
9587     "exit:" %}
9588   ins_encode %{
9589     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9590     emit_cmpfp_fixup(_masm);
9591   %}
9592   ins_pipe(pipe_slow);
9593 %}
9594 
9595 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9596   match(Set cr (CmpF src1 (LoadF src2)));
9597 
9598   ins_cost(100);
9599   format %{ "ucomiss $src1, $src2" %}
9600   ins_encode %{
9601     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9602   %}
9603   ins_pipe(pipe_slow);
9604 %}
9605 
9606 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9607   match(Set cr (CmpF src con));
9608 
9609   ins_cost(145);
9610   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9611             "jnp,s   exit\n\t"
9612             "pushfq\t# saw NaN, set CF\n\t"
9613             "andq    [rsp], #0xffffff2b\n\t"
9614             "popfq\n"
9615     "exit:" %}
9616   ins_encode %{
9617     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9618     emit_cmpfp_fixup(_masm);
9619   %}
9620   ins_pipe(pipe_slow);
9621 %}
9622 
9623 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9624   match(Set cr (CmpF src con));
9625   ins_cost(100);
9626   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9627   ins_encode %{
9628     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9629   %}
9630   ins_pipe(pipe_slow);
9631 %}
9632 
9633 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9634 %{
9635   match(Set cr (CmpD src1 src2));
9636 
9637   ins_cost(145);
9638   format %{ "ucomisd $src1, $src2\n\t"
9639             "jnp,s   exit\n\t"
9640             "pushfq\t# saw NaN, set CF\n\t"
9641             "andq    [rsp], #0xffffff2b\n\t"
9642             "popfq\n"
9643     "exit:" %}
9644   ins_encode %{
9645     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9646     emit_cmpfp_fixup(_masm);
9647   %}
9648   ins_pipe(pipe_slow);
9649 %}
9650 
9651 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9652   match(Set cr (CmpD src1 src2));
9653 
9654   ins_cost(100);
9655   format %{ "ucomisd $src1, $src2 test" %}
9656   ins_encode %{
9657     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9658   %}
9659   ins_pipe(pipe_slow);
9660 %}
9661 
9662 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9663 %{
9664   match(Set cr (CmpD src1 (LoadD src2)));
9665 
9666   ins_cost(145);
9667   format %{ "ucomisd $src1, $src2\n\t"
9668             "jnp,s   exit\n\t"
9669             "pushfq\t# saw NaN, set CF\n\t"
9670             "andq    [rsp], #0xffffff2b\n\t"
9671             "popfq\n"
9672     "exit:" %}
9673   ins_encode %{
9674     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9675     emit_cmpfp_fixup(_masm);
9676   %}
9677   ins_pipe(pipe_slow);
9678 %}
9679 
9680 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9681   match(Set cr (CmpD src1 (LoadD src2)));
9682 
9683   ins_cost(100);
9684   format %{ "ucomisd $src1, $src2" %}
9685   ins_encode %{
9686     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9687   %}
9688   ins_pipe(pipe_slow);
9689 %}
9690 
9691 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9692   match(Set cr (CmpD src con));
9693 
9694   ins_cost(145);
9695   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9696             "jnp,s   exit\n\t"
9697             "pushfq\t# saw NaN, set CF\n\t"
9698             "andq    [rsp], #0xffffff2b\n\t"
9699             "popfq\n"
9700     "exit:" %}
9701   ins_encode %{
9702     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9703     emit_cmpfp_fixup(_masm);
9704   %}
9705   ins_pipe(pipe_slow);
9706 %}
9707 
9708 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9709   match(Set cr (CmpD src con));
9710   ins_cost(100);
9711   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9712   ins_encode %{
9713     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9714   %}
9715   ins_pipe(pipe_slow);
9716 %}
9717 
9718 // Compare into -1,0,1
9719 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9720 %{
9721   match(Set dst (CmpF3 src1 src2));
9722   effect(KILL cr);
9723 
9724   ins_cost(275);
9725   format %{ "ucomiss $src1, $src2\n\t"
9726             "movl    $dst, #-1\n\t"
9727             "jp,s    done\n\t"
9728             "jb,s    done\n\t"
9729             "setne   $dst\n\t"
9730             "movzbl  $dst, $dst\n"
9731     "done:" %}
9732   ins_encode %{
9733     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9734     emit_cmpfp3(_masm, $dst$$Register);
9735   %}
9736   ins_pipe(pipe_slow);
9737 %}
9738 
9739 // Compare into -1,0,1
9740 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9741 %{
9742   match(Set dst (CmpF3 src1 (LoadF src2)));
9743   effect(KILL cr);
9744 
9745   ins_cost(275);
9746   format %{ "ucomiss $src1, $src2\n\t"
9747             "movl    $dst, #-1\n\t"
9748             "jp,s    done\n\t"
9749             "jb,s    done\n\t"
9750             "setne   $dst\n\t"
9751             "movzbl  $dst, $dst\n"
9752     "done:" %}
9753   ins_encode %{
9754     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9755     emit_cmpfp3(_masm, $dst$$Register);
9756   %}
9757   ins_pipe(pipe_slow);
9758 %}
9759 
9760 // Compare into -1,0,1
9761 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9762   match(Set dst (CmpF3 src con));
9763   effect(KILL cr);
9764 
9765   ins_cost(275);
9766   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9767             "movl    $dst, #-1\n\t"
9768             "jp,s    done\n\t"
9769             "jb,s    done\n\t"
9770             "setne   $dst\n\t"
9771             "movzbl  $dst, $dst\n"
9772     "done:" %}
9773   ins_encode %{
9774     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9775     emit_cmpfp3(_masm, $dst$$Register);
9776   %}
9777   ins_pipe(pipe_slow);
9778 %}
9779 
9780 // Compare into -1,0,1
9781 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9782 %{
9783   match(Set dst (CmpD3 src1 src2));
9784   effect(KILL cr);
9785 
9786   ins_cost(275);
9787   format %{ "ucomisd $src1, $src2\n\t"
9788             "movl    $dst, #-1\n\t"
9789             "jp,s    done\n\t"
9790             "jb,s    done\n\t"
9791             "setne   $dst\n\t"
9792             "movzbl  $dst, $dst\n"
9793     "done:" %}
9794   ins_encode %{
9795     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9796     emit_cmpfp3(_masm, $dst$$Register);
9797   %}
9798   ins_pipe(pipe_slow);
9799 %}
9800 
9801 // Compare into -1,0,1
9802 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9803 %{
9804   match(Set dst (CmpD3 src1 (LoadD src2)));
9805   effect(KILL cr);
9806 
9807   ins_cost(275);
9808   format %{ "ucomisd $src1, $src2\n\t"
9809             "movl    $dst, #-1\n\t"
9810             "jp,s    done\n\t"
9811             "jb,s    done\n\t"
9812             "setne   $dst\n\t"
9813             "movzbl  $dst, $dst\n"
9814     "done:" %}
9815   ins_encode %{
9816     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9817     emit_cmpfp3(_masm, $dst$$Register);
9818   %}
9819   ins_pipe(pipe_slow);
9820 %}
9821 
9822 // Compare into -1,0,1
9823 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9824   match(Set dst (CmpD3 src con));
9825   effect(KILL cr);
9826 
9827   ins_cost(275);
9828   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9829             "movl    $dst, #-1\n\t"
9830             "jp,s    done\n\t"
9831             "jb,s    done\n\t"
9832             "setne   $dst\n\t"
9833             "movzbl  $dst, $dst\n"
9834     "done:" %}
9835   ins_encode %{
9836     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9837     emit_cmpfp3(_masm, $dst$$Register);
9838   %}
9839   ins_pipe(pipe_slow);
9840 %}
9841 
9842 // -----------Trig and Trancendental Instructions------------------------------
9843 instruct cosD_reg(regD dst) %{
9844   match(Set dst (CosD dst));
9845 
9846   format %{ "dcos   $dst\n\t" %}
9847   opcode(0xD9, 0xFF);
9848   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9849   ins_pipe( pipe_slow );
9850 %}
9851 
9852 instruct sinD_reg(regD dst) %{
9853   match(Set dst (SinD dst));
9854 
9855   format %{ "dsin   $dst\n\t" %}
9856   opcode(0xD9, 0xFE);
9857   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9858   ins_pipe( pipe_slow );
9859 %}
9860 
9861 instruct tanD_reg(regD dst) %{
9862   match(Set dst (TanD dst));
9863 
9864   format %{ "dtan   $dst\n\t" %}
9865   ins_encode( Push_SrcXD(dst),
9866               Opcode(0xD9), Opcode(0xF2),   //fptan
9867               Opcode(0xDD), Opcode(0xD8),   //fstp st
9868               Push_ResultXD(dst) );
9869   ins_pipe( pipe_slow );
9870 %}
9871 
9872 instruct log10D_reg(regD dst) %{
9873   // The source and result Double operands in XMM registers
9874   match(Set dst (Log10D dst));
9875   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9876   // fyl2x        ; compute log_10(2) * log_2(x)
9877   format %{ "fldlg2\t\t\t#Log10\n\t"
9878             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9879          %}
9880    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9881               Push_SrcXD(dst),
9882               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9883               Push_ResultXD(dst));
9884 
9885   ins_pipe( pipe_slow );
9886 %}
9887 
9888 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9889   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9890   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9891   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9892   ins_encode %{
9893     __ subptr(rsp, 8);
9894     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9895     __ fld_d(Address(rsp, 0));
9896     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9897     __ fld_d(Address(rsp, 0));
9898     __ fast_pow();
9899     __ fstp_d(Address(rsp, 0));
9900     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9901     __ addptr(rsp, 8);
9902   %}
9903   ins_pipe( pipe_slow );
9904 %}
9905 
9906 //----------Arithmetic Conversion Instructions---------------------------------
9907 
9908 instruct roundFloat_nop(regF dst)
9909 %{
9910   match(Set dst (RoundFloat dst));
9911 
9912   ins_cost(0);
9913   ins_encode();
9914   ins_pipe(empty);
9915 %}
9916 
9917 instruct roundDouble_nop(regD dst)
9918 %{
9919   match(Set dst (RoundDouble dst));
9920 
9921   ins_cost(0);
9922   ins_encode();
9923   ins_pipe(empty);
9924 %}
9925 
9926 instruct convF2D_reg_reg(regD dst, regF src)
9927 %{
9928   match(Set dst (ConvF2D src));
9929 
9930   format %{ "cvtss2sd $dst, $src" %}
9931   ins_encode %{
9932     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9933   %}
9934   ins_pipe(pipe_slow); // XXX
9935 %}
9936 
9937 instruct convF2D_reg_mem(regD dst, memory src)
9938 %{
9939   match(Set dst (ConvF2D (LoadF src)));
9940 
9941   format %{ "cvtss2sd $dst, $src" %}
9942   ins_encode %{
9943     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9944   %}
9945   ins_pipe(pipe_slow); // XXX
9946 %}
9947 
9948 instruct convD2F_reg_reg(regF dst, regD src)
9949 %{
9950   match(Set dst (ConvD2F src));
9951 
9952   format %{ "cvtsd2ss $dst, $src" %}
9953   ins_encode %{
9954     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9955   %}
9956   ins_pipe(pipe_slow); // XXX
9957 %}
9958 
9959 instruct convD2F_reg_mem(regF dst, memory src)
9960 %{
9961   match(Set dst (ConvD2F (LoadD src)));
9962 
9963   format %{ "cvtsd2ss $dst, $src" %}
9964   ins_encode %{
9965     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9966   %}
9967   ins_pipe(pipe_slow); // XXX
9968 %}
9969 
9970 // XXX do mem variants
9971 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9972 %{
9973   match(Set dst (ConvF2I src));
9974   effect(KILL cr);
9975 
9976   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9977             "cmpl    $dst, #0x80000000\n\t"
9978             "jne,s   done\n\t"
9979             "subq    rsp, #8\n\t"
9980             "movss   [rsp], $src\n\t"
9981             "call    f2i_fixup\n\t"
9982             "popq    $dst\n"
9983     "done:   "%}
9984   ins_encode %{
9985     Label done;
9986     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9987     __ cmpl($dst$$Register, 0x80000000);
9988     __ jccb(Assembler::notEqual, done);
9989     __ subptr(rsp, 8);
9990     __ movflt(Address(rsp, 0), $src$$XMMRegister);
9991     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
9992     __ pop($dst$$Register);
9993     __ bind(done);
9994   %}
9995   ins_pipe(pipe_slow);
9996 %}
9997 
9998 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
9999 %{
10000   match(Set dst (ConvF2L src));
10001   effect(KILL cr);
10002 
10003   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10004             "cmpq    $dst, [0x8000000000000000]\n\t"
10005             "jne,s   done\n\t"
10006             "subq    rsp, #8\n\t"
10007             "movss   [rsp], $src\n\t"
10008             "call    f2l_fixup\n\t"
10009             "popq    $dst\n"
10010     "done:   "%}
10011   ins_encode %{
10012     Label done;
10013     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10014     __ cmp64($dst$$Register,
10015              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10016     __ jccb(Assembler::notEqual, done);
10017     __ subptr(rsp, 8);
10018     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10019     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10020     __ pop($dst$$Register);
10021     __ bind(done);
10022   %}
10023   ins_pipe(pipe_slow);
10024 %}
10025 
10026 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10027 %{
10028   match(Set dst (ConvD2I src));
10029   effect(KILL cr);
10030 
10031   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10032             "cmpl    $dst, #0x80000000\n\t"
10033             "jne,s   done\n\t"
10034             "subq    rsp, #8\n\t"
10035             "movsd   [rsp], $src\n\t"
10036             "call    d2i_fixup\n\t"
10037             "popq    $dst\n"
10038     "done:   "%}
10039   ins_encode %{
10040     Label done;
10041     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10042     __ cmpl($dst$$Register, 0x80000000);
10043     __ jccb(Assembler::notEqual, done);
10044     __ subptr(rsp, 8);
10045     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10046     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10047     __ pop($dst$$Register);
10048     __ bind(done);
10049   %}
10050   ins_pipe(pipe_slow);
10051 %}
10052 
10053 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10054 %{
10055   match(Set dst (ConvD2L src));
10056   effect(KILL cr);
10057 
10058   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10059             "cmpq    $dst, [0x8000000000000000]\n\t"
10060             "jne,s   done\n\t"
10061             "subq    rsp, #8\n\t"
10062             "movsd   [rsp], $src\n\t"
10063             "call    d2l_fixup\n\t"
10064             "popq    $dst\n"
10065     "done:   "%}
10066   ins_encode %{
10067     Label done;
10068     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10069     __ cmp64($dst$$Register,
10070              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10071     __ jccb(Assembler::notEqual, done);
10072     __ subptr(rsp, 8);
10073     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10074     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10075     __ pop($dst$$Register);
10076     __ bind(done);
10077   %}
10078   ins_pipe(pipe_slow);
10079 %}
10080 
10081 instruct convI2F_reg_reg(regF dst, rRegI src)
10082 %{
10083   predicate(!UseXmmI2F);
10084   match(Set dst (ConvI2F src));
10085 
10086   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10087   ins_encode %{
10088     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10089   %}
10090   ins_pipe(pipe_slow); // XXX
10091 %}
10092 
10093 instruct convI2F_reg_mem(regF dst, memory src)
10094 %{
10095   match(Set dst (ConvI2F (LoadI src)));
10096 
10097   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10098   ins_encode %{
10099     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10100   %}
10101   ins_pipe(pipe_slow); // XXX
10102 %}
10103 
10104 instruct convI2D_reg_reg(regD dst, rRegI src)
10105 %{
10106   predicate(!UseXmmI2D);
10107   match(Set dst (ConvI2D src));
10108 
10109   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10110   ins_encode %{
10111     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10112   %}
10113   ins_pipe(pipe_slow); // XXX
10114 %}
10115 
10116 instruct convI2D_reg_mem(regD dst, memory src)
10117 %{
10118   match(Set dst (ConvI2D (LoadI src)));
10119 
10120   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10121   ins_encode %{
10122     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10123   %}
10124   ins_pipe(pipe_slow); // XXX
10125 %}
10126 
10127 instruct convXI2F_reg(regF dst, rRegI src)
10128 %{
10129   predicate(UseXmmI2F);
10130   match(Set dst (ConvI2F src));
10131 
10132   format %{ "movdl $dst, $src\n\t"
10133             "cvtdq2psl $dst, $dst\t# i2f" %}
10134   ins_encode %{
10135     __ movdl($dst$$XMMRegister, $src$$Register);
10136     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10137   %}
10138   ins_pipe(pipe_slow); // XXX
10139 %}
10140 
10141 instruct convXI2D_reg(regD dst, rRegI src)
10142 %{
10143   predicate(UseXmmI2D);
10144   match(Set dst (ConvI2D src));
10145 
10146   format %{ "movdl $dst, $src\n\t"
10147             "cvtdq2pdl $dst, $dst\t# i2d" %}
10148   ins_encode %{
10149     __ movdl($dst$$XMMRegister, $src$$Register);
10150     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10151   %}
10152   ins_pipe(pipe_slow); // XXX
10153 %}
10154 
10155 instruct convL2F_reg_reg(regF dst, rRegL src)
10156 %{
10157   match(Set dst (ConvL2F src));
10158 
10159   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10160   ins_encode %{
10161     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10162   %}
10163   ins_pipe(pipe_slow); // XXX
10164 %}
10165 
10166 instruct convL2F_reg_mem(regF dst, memory src)
10167 %{
10168   match(Set dst (ConvL2F (LoadL src)));
10169 
10170   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10171   ins_encode %{
10172     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10173   %}
10174   ins_pipe(pipe_slow); // XXX
10175 %}
10176 
10177 instruct convL2D_reg_reg(regD dst, rRegL src)
10178 %{
10179   match(Set dst (ConvL2D src));
10180 
10181   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10182   ins_encode %{
10183     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10184   %}
10185   ins_pipe(pipe_slow); // XXX
10186 %}
10187 
10188 instruct convL2D_reg_mem(regD dst, memory src)
10189 %{
10190   match(Set dst (ConvL2D (LoadL src)));
10191 
10192   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10193   ins_encode %{
10194     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10195   %}
10196   ins_pipe(pipe_slow); // XXX
10197 %}
10198 
10199 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10200 %{
10201   match(Set dst (ConvI2L src));
10202 
10203   ins_cost(125);
10204   format %{ "movslq  $dst, $src\t# i2l" %}
10205   ins_encode %{
10206     __ movslq($dst$$Register, $src$$Register);
10207   %}
10208   ins_pipe(ialu_reg_reg);
10209 %}
10210 
10211 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10212 // %{
10213 //   match(Set dst (ConvI2L src));
10214 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10215 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10216 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10217 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10218 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10219 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10220 
10221 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10222 //   ins_encode(enc_copy(dst, src));
10223 // //   opcode(0x63); // needs REX.W
10224 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10225 //   ins_pipe(ialu_reg_reg);
10226 // %}
10227 
10228 // Zero-extend convert int to long
10229 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10230 %{
10231   match(Set dst (AndL (ConvI2L src) mask));
10232 
10233   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10234   ins_encode %{
10235     if ($dst$$reg != $src$$reg) {
10236       __ movl($dst$$Register, $src$$Register);
10237     }
10238   %}
10239   ins_pipe(ialu_reg_reg);
10240 %}
10241 
10242 // Zero-extend convert int to long
10243 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10244 %{
10245   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10246 
10247   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10248   ins_encode %{
10249     __ movl($dst$$Register, $src$$Address);
10250   %}
10251   ins_pipe(ialu_reg_mem);
10252 %}
10253 
10254 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10255 %{
10256   match(Set dst (AndL src mask));
10257 
10258   format %{ "movl    $dst, $src\t# zero-extend long" %}
10259   ins_encode %{
10260     __ movl($dst$$Register, $src$$Register);
10261   %}
10262   ins_pipe(ialu_reg_reg);
10263 %}
10264 
10265 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10266 %{
10267   match(Set dst (ConvL2I src));
10268 
10269   format %{ "movl    $dst, $src\t# l2i" %}
10270   ins_encode %{
10271     __ movl($dst$$Register, $src$$Register);
10272   %}
10273   ins_pipe(ialu_reg_reg);
10274 %}
10275 
10276 
10277 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10278   match(Set dst (MoveF2I src));
10279   effect(DEF dst, USE src);
10280 
10281   ins_cost(125);
10282   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10283   ins_encode %{
10284     __ movl($dst$$Register, Address(rsp, $src$$disp));
10285   %}
10286   ins_pipe(ialu_reg_mem);
10287 %}
10288 
10289 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10290   match(Set dst (MoveI2F src));
10291   effect(DEF dst, USE src);
10292 
10293   ins_cost(125);
10294   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10295   ins_encode %{
10296     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10297   %}
10298   ins_pipe(pipe_slow);
10299 %}
10300 
10301 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10302   match(Set dst (MoveD2L src));
10303   effect(DEF dst, USE src);
10304 
10305   ins_cost(125);
10306   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10307   ins_encode %{
10308     __ movq($dst$$Register, Address(rsp, $src$$disp));
10309   %}
10310   ins_pipe(ialu_reg_mem);
10311 %}
10312 
10313 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10314   predicate(!UseXmmLoadAndClearUpper);
10315   match(Set dst (MoveL2D src));
10316   effect(DEF dst, USE src);
10317 
10318   ins_cost(125);
10319   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10320   ins_encode %{
10321     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10322   %}
10323   ins_pipe(pipe_slow);
10324 %}
10325 
10326 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10327   predicate(UseXmmLoadAndClearUpper);
10328   match(Set dst (MoveL2D src));
10329   effect(DEF dst, USE src);
10330 
10331   ins_cost(125);
10332   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10333   ins_encode %{
10334     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10335   %}
10336   ins_pipe(pipe_slow);
10337 %}
10338 
10339 
10340 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10341   match(Set dst (MoveF2I src));
10342   effect(DEF dst, USE src);
10343 
10344   ins_cost(95); // XXX
10345   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10346   ins_encode %{
10347     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10348   %}
10349   ins_pipe(pipe_slow);
10350 %}
10351 
10352 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10353   match(Set dst (MoveI2F src));
10354   effect(DEF dst, USE src);
10355 
10356   ins_cost(100);
10357   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10358   ins_encode %{
10359     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10360   %}
10361   ins_pipe( ialu_mem_reg );
10362 %}
10363 
10364 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10365   match(Set dst (MoveD2L src));
10366   effect(DEF dst, USE src);
10367 
10368   ins_cost(95); // XXX
10369   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10370   ins_encode %{
10371     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10372   %}
10373   ins_pipe(pipe_slow);
10374 %}
10375 
10376 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10377   match(Set dst (MoveL2D src));
10378   effect(DEF dst, USE src);
10379 
10380   ins_cost(100);
10381   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10382   ins_encode %{
10383     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10384   %}
10385   ins_pipe(ialu_mem_reg);
10386 %}
10387 
10388 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10389   match(Set dst (MoveF2I src));
10390   effect(DEF dst, USE src);
10391   ins_cost(85);
10392   format %{ "movd    $dst,$src\t# MoveF2I" %}
10393   ins_encode %{
10394     __ movdl($dst$$Register, $src$$XMMRegister);
10395   %}
10396   ins_pipe( pipe_slow );
10397 %}
10398 
10399 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10400   match(Set dst (MoveD2L src));
10401   effect(DEF dst, USE src);
10402   ins_cost(85);
10403   format %{ "movd    $dst,$src\t# MoveD2L" %}
10404   ins_encode %{
10405     __ movdq($dst$$Register, $src$$XMMRegister);
10406   %}
10407   ins_pipe( pipe_slow );
10408 %}
10409 
10410 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10411   match(Set dst (MoveI2F src));
10412   effect(DEF dst, USE src);
10413   ins_cost(100);
10414   format %{ "movd    $dst,$src\t# MoveI2F" %}
10415   ins_encode %{
10416     __ movdl($dst$$XMMRegister, $src$$Register);
10417   %}
10418   ins_pipe( pipe_slow );
10419 %}
10420 
10421 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10422   match(Set dst (MoveL2D src));
10423   effect(DEF dst, USE src);
10424   ins_cost(100);
10425   format %{ "movd    $dst,$src\t# MoveL2D" %}
10426   ins_encode %{
10427      __ movdq($dst$$XMMRegister, $src$$Register);
10428   %}
10429   ins_pipe( pipe_slow );
10430 %}
10431 
10432 
10433 // =======================================================================
10434 // fast clearing of an array
10435 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10436                   rFlagsReg cr)
10437 %{
10438   predicate(!UseFastStosb);
10439   match(Set dummy (ClearArray cnt base));
10440   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10441 
10442   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10443             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10444   ins_encode %{
10445     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10446   %}
10447   ins_pipe(pipe_slow);
10448 %}
10449 
10450 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10451                         rFlagsReg cr)
10452 %{
10453   predicate(UseFastStosb);
10454   match(Set dummy (ClearArray cnt base));
10455   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10456   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10457             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10458             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10459   ins_encode %{
10460     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10461   %}
10462   ins_pipe( pipe_slow );
10463 %}
10464 
10465 instruct string_compareL(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10466                          rax_RegI result, regD tmp1, rFlagsReg cr)
10467 %{
10468   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
10469   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10470   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10471 
10472   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10473   ins_encode %{
10474     __ string_compare($str1$$Register, $str2$$Register,
10475                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10476                       $tmp1$$XMMRegister, StrIntrinsicNode::LL);
10477   %}
10478   ins_pipe( pipe_slow );
10479 %}
10480 
10481 instruct string_compareU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10482                          rax_RegI result, regD tmp1, rFlagsReg cr)
10483 %{
10484   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
10485   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10486   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10487 
10488   format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10489   ins_encode %{
10490     __ string_compare($str1$$Register, $str2$$Register,
10491                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10492                       $tmp1$$XMMRegister, StrIntrinsicNode::UU);
10493   %}
10494   ins_pipe( pipe_slow );
10495 %}
10496 
10497 instruct string_compareLU(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10498                           rax_RegI result, regD tmp1, rFlagsReg cr)
10499 %{
10500   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
10501   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10502   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10503 
10504   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10505   ins_encode %{
10506     __ string_compare($str1$$Register, $str2$$Register,
10507                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10508                       $tmp1$$XMMRegister, StrIntrinsicNode::LU);
10509   %}
10510   ins_pipe( pipe_slow );
10511 %}
10512 
10513 instruct string_compareUL(rsi_RegP str1, rdx_RegI cnt1, rdi_RegP str2, rcx_RegI cnt2,
10514                           rax_RegI result, regD tmp1, rFlagsReg cr)
10515 %{
10516   predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
10517   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10518   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10519 
10520   format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10521   ins_encode %{
10522     __ string_compare($str2$$Register, $str1$$Register,
10523                       $cnt2$$Register, $cnt1$$Register, $result$$Register,
10524                       $tmp1$$XMMRegister, StrIntrinsicNode::UL);
10525   %}
10526   ins_pipe( pipe_slow );
10527 %}
10528 
10529 // fast search of substring with known size.
10530 instruct string_indexof_conL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10531                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10532 %{
10533   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10534   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10535   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10536 
10537   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10538   ins_encode %{
10539     int icnt2 = (int)$int_cnt2$$constant;
10540     if (icnt2 >= 16) {
10541       // IndexOf for constant substrings with size >= 16 elements
10542       // which don't need to be loaded through stack.
10543       __ string_indexofC8($str1$$Register, $str2$$Register,
10544                           $cnt1$$Register, $cnt2$$Register,
10545                           icnt2, $result$$Register,
10546                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10547     } else {
10548       // Small strings are loaded through stack if they cross page boundary.
10549       __ string_indexof($str1$$Register, $str2$$Register,
10550                         $cnt1$$Register, $cnt2$$Register,
10551                         icnt2, $result$$Register,
10552                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10553     }
10554   %}
10555   ins_pipe( pipe_slow );
10556 %}
10557 
10558 // fast search of substring with known size.
10559 instruct string_indexof_conU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10560                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10561 %{
10562   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10563   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10564   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10565 
10566   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10567   ins_encode %{
10568     int icnt2 = (int)$int_cnt2$$constant;
10569     if (icnt2 >= 8) {
10570       // IndexOf for constant substrings with size >= 8 elements
10571       // which don't need to be loaded through stack.
10572       __ string_indexofC8($str1$$Register, $str2$$Register,
10573                           $cnt1$$Register, $cnt2$$Register,
10574                           icnt2, $result$$Register,
10575                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10576     } else {
10577       // Small strings are loaded through stack if they cross page boundary.
10578       __ string_indexof($str1$$Register, $str2$$Register,
10579                         $cnt1$$Register, $cnt2$$Register,
10580                         icnt2, $result$$Register,
10581                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10582     }
10583   %}
10584   ins_pipe( pipe_slow );
10585 %}
10586 
10587 // fast search of substring with known size.
10588 instruct string_indexof_conUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10589                              rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10590 %{
10591   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10592   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10593   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10594 
10595   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10596   ins_encode %{
10597     int icnt2 = (int)$int_cnt2$$constant;
10598     if (icnt2 >= 8) {
10599       // IndexOf for constant substrings with size >= 8 elements
10600       // which don't need to be loaded through stack.
10601       __ string_indexofC8($str1$$Register, $str2$$Register,
10602                           $cnt1$$Register, $cnt2$$Register,
10603                           icnt2, $result$$Register,
10604                           $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10605     } else {
10606       // Small strings are loaded through stack if they cross page boundary.
10607       __ string_indexof($str1$$Register, $str2$$Register,
10608                         $cnt1$$Register, $cnt2$$Register,
10609                         icnt2, $result$$Register,
10610                         $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10611     }
10612   %}
10613   ins_pipe( pipe_slow );
10614 %}
10615 
10616 instruct string_indexofL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10617                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10618 %{
10619   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
10620   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10621   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10622 
10623   format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10624   ins_encode %{
10625     __ string_indexof($str1$$Register, $str2$$Register,
10626                       $cnt1$$Register, $cnt2$$Register,
10627                       (-1), $result$$Register,
10628                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
10629   %}
10630   ins_pipe( pipe_slow );
10631 %}
10632 
10633 instruct string_indexofU(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10634                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10635 %{
10636   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
10637   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10638   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10639 
10640   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10641   ins_encode %{
10642     __ string_indexof($str1$$Register, $str2$$Register,
10643                       $cnt1$$Register, $cnt2$$Register,
10644                       (-1), $result$$Register,
10645                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
10646   %}
10647   ins_pipe( pipe_slow );
10648 %}
10649 
10650 instruct string_indexofUL(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10651                          rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10652 %{
10653   predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
10654   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10655   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10656 
10657   format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10658   ins_encode %{
10659     __ string_indexof($str1$$Register, $str2$$Register,
10660                       $cnt1$$Register, $cnt2$$Register,
10661                       (-1), $result$$Register,
10662                       $vec$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
10663   %}
10664   ins_pipe( pipe_slow );
10665 %}
10666 
10667 instruct string_indexofU_char(rdi_RegP str1, rdx_RegI cnt1, rax_RegI ch,
10668                               rbx_RegI result, regD vec1, regD vec2, regD vec3, rcx_RegI tmp, rFlagsReg cr)
10669 %{
10670   predicate(UseSSE42Intrinsics);
10671   match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
10672   effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
10673   format %{ "String IndexOf char[] $str1,$cnt1,$ch -> $result   // KILL all" %}
10674   ins_encode %{
10675     __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
10676                            $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
10677   %}
10678   ins_pipe( pipe_slow );
10679 %}
10680 
10681 // fast string equals
10682 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10683                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10684 %{
10685   match(Set result (StrEquals (Binary str1 str2) cnt));
10686   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10687 
10688   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10689   ins_encode %{
10690     __ arrays_equals(false, $str1$$Register, $str2$$Register,
10691                      $cnt$$Register, $result$$Register, $tmp3$$Register,
10692                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10693   %}
10694   ins_pipe( pipe_slow );
10695 %}
10696 
10697 // fast array equals
10698 instruct array_equalsB(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10699                        regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10700 %{
10701   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
10702   match(Set result (AryEq ary1 ary2));
10703   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10704 
10705   format %{ "Array Equals byte[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10706   ins_encode %{
10707     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10708                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10709                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */);
10710   %}
10711   ins_pipe( pipe_slow );
10712 %}
10713 
10714 instruct array_equalsC(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10715                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10716 %{
10717   predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
10718   match(Set result (AryEq ary1 ary2));
10719   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10720 
10721   format %{ "Array Equals char[] $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10722   ins_encode %{
10723     __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
10724                      $tmp3$$Register, $result$$Register, $tmp4$$Register,
10725                      $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */);
10726   %}
10727   ins_pipe( pipe_slow );
10728 %}
10729 
10730 instruct has_negatives(rsi_RegP ary1, rcx_RegI len, rax_RegI result,
10731                       regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10732 %{
10733   match(Set result (HasNegatives ary1 len));
10734   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
10735 
10736   format %{ "has negatives byte[] $ary1,$len -> $result   // KILL $tmp1, $tmp2, $tmp3" %}
10737   ins_encode %{
10738     __ has_negatives($ary1$$Register, $len$$Register,
10739                      $result$$Register, $tmp3$$Register,
10740                      $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10741   %}
10742   ins_pipe( pipe_slow );
10743 %}
10744 
10745 // fast char[] to byte[] compression
10746 instruct string_compress(rsi_RegP src, rdi_RegP dst, rdx_RegI len, regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10747                          rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10748   match(Set result (StrCompressedCopy src (Binary dst len)));
10749   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10750 
10751   format %{ "String Compress $src,$dst -> $result    // KILL RAX, RCX, RDX" %}
10752   ins_encode %{
10753     __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
10754                            $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10755                            $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10756   %}
10757   ins_pipe( pipe_slow );
10758 %}
10759 
10760 // fast byte[] to char[] inflation
10761 instruct string_inflate(Universe dummy, rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10762                         regD tmp1, rcx_RegI tmp2, rFlagsReg cr) %{
10763   match(Set dummy (StrInflatedCopy src (Binary dst len)));
10764   effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
10765 
10766   format %{ "String Inflate $src,$dst    // KILL $tmp1, $tmp2" %}
10767   ins_encode %{
10768     __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
10769                           $tmp1$$XMMRegister, $tmp2$$Register);
10770   %}
10771   ins_pipe( pipe_slow );
10772 %}
10773 
10774 // encode char[] to byte[] in ISO_8859_1
10775 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10776                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10777                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10778   match(Set result (EncodeISOArray src (Binary dst len)));
10779   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10780 
10781   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10782   ins_encode %{
10783     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10784                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10785                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10786   %}
10787   ins_pipe( pipe_slow );
10788 %}
10789 
10790 //----------Overflow Math Instructions-----------------------------------------
10791 
10792 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10793 %{
10794   match(Set cr (OverflowAddI op1 op2));
10795   effect(DEF cr, USE_KILL op1, USE op2);
10796 
10797   format %{ "addl    $op1, $op2\t# overflow check int" %}
10798 
10799   ins_encode %{
10800     __ addl($op1$$Register, $op2$$Register);
10801   %}
10802   ins_pipe(ialu_reg_reg);
10803 %}
10804 
10805 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10806 %{
10807   match(Set cr (OverflowAddI op1 op2));
10808   effect(DEF cr, USE_KILL op1, USE op2);
10809 
10810   format %{ "addl    $op1, $op2\t# overflow check int" %}
10811 
10812   ins_encode %{
10813     __ addl($op1$$Register, $op2$$constant);
10814   %}
10815   ins_pipe(ialu_reg_reg);
10816 %}
10817 
10818 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10819 %{
10820   match(Set cr (OverflowAddL op1 op2));
10821   effect(DEF cr, USE_KILL op1, USE op2);
10822 
10823   format %{ "addq    $op1, $op2\t# overflow check long" %}
10824   ins_encode %{
10825     __ addq($op1$$Register, $op2$$Register);
10826   %}
10827   ins_pipe(ialu_reg_reg);
10828 %}
10829 
10830 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10831 %{
10832   match(Set cr (OverflowAddL op1 op2));
10833   effect(DEF cr, USE_KILL op1, USE op2);
10834 
10835   format %{ "addq    $op1, $op2\t# overflow check long" %}
10836   ins_encode %{
10837     __ addq($op1$$Register, $op2$$constant);
10838   %}
10839   ins_pipe(ialu_reg_reg);
10840 %}
10841 
10842 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10843 %{
10844   match(Set cr (OverflowSubI op1 op2));
10845 
10846   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10847   ins_encode %{
10848     __ cmpl($op1$$Register, $op2$$Register);
10849   %}
10850   ins_pipe(ialu_reg_reg);
10851 %}
10852 
10853 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10854 %{
10855   match(Set cr (OverflowSubI op1 op2));
10856 
10857   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10858   ins_encode %{
10859     __ cmpl($op1$$Register, $op2$$constant);
10860   %}
10861   ins_pipe(ialu_reg_reg);
10862 %}
10863 
10864 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10865 %{
10866   match(Set cr (OverflowSubL op1 op2));
10867 
10868   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10869   ins_encode %{
10870     __ cmpq($op1$$Register, $op2$$Register);
10871   %}
10872   ins_pipe(ialu_reg_reg);
10873 %}
10874 
10875 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10876 %{
10877   match(Set cr (OverflowSubL op1 op2));
10878 
10879   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10880   ins_encode %{
10881     __ cmpq($op1$$Register, $op2$$constant);
10882   %}
10883   ins_pipe(ialu_reg_reg);
10884 %}
10885 
10886 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10887 %{
10888   match(Set cr (OverflowSubI zero op2));
10889   effect(DEF cr, USE_KILL op2);
10890 
10891   format %{ "negl    $op2\t# overflow check int" %}
10892   ins_encode %{
10893     __ negl($op2$$Register);
10894   %}
10895   ins_pipe(ialu_reg_reg);
10896 %}
10897 
10898 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10899 %{
10900   match(Set cr (OverflowSubL zero op2));
10901   effect(DEF cr, USE_KILL op2);
10902 
10903   format %{ "negq    $op2\t# overflow check long" %}
10904   ins_encode %{
10905     __ negq($op2$$Register);
10906   %}
10907   ins_pipe(ialu_reg_reg);
10908 %}
10909 
10910 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10911 %{
10912   match(Set cr (OverflowMulI op1 op2));
10913   effect(DEF cr, USE_KILL op1, USE op2);
10914 
10915   format %{ "imull    $op1, $op2\t# overflow check int" %}
10916   ins_encode %{
10917     __ imull($op1$$Register, $op2$$Register);
10918   %}
10919   ins_pipe(ialu_reg_reg_alu0);
10920 %}
10921 
10922 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10923 %{
10924   match(Set cr (OverflowMulI op1 op2));
10925   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10926 
10927   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10928   ins_encode %{
10929     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10930   %}
10931   ins_pipe(ialu_reg_reg_alu0);
10932 %}
10933 
10934 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10935 %{
10936   match(Set cr (OverflowMulL op1 op2));
10937   effect(DEF cr, USE_KILL op1, USE op2);
10938 
10939   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10940   ins_encode %{
10941     __ imulq($op1$$Register, $op2$$Register);
10942   %}
10943   ins_pipe(ialu_reg_reg_alu0);
10944 %}
10945 
10946 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10947 %{
10948   match(Set cr (OverflowMulL op1 op2));
10949   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10950 
10951   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10952   ins_encode %{
10953     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10954   %}
10955   ins_pipe(ialu_reg_reg_alu0);
10956 %}
10957 
10958 
10959 //----------Control Flow Instructions------------------------------------------
10960 // Signed compare Instructions
10961 
10962 // XXX more variants!!
10963 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10964 %{
10965   match(Set cr (CmpI op1 op2));
10966   effect(DEF cr, USE op1, USE op2);
10967 
10968   format %{ "cmpl    $op1, $op2" %}
10969   opcode(0x3B);  /* Opcode 3B /r */
10970   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10971   ins_pipe(ialu_cr_reg_reg);
10972 %}
10973 
10974 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10975 %{
10976   match(Set cr (CmpI op1 op2));
10977 
10978   format %{ "cmpl    $op1, $op2" %}
10979   opcode(0x81, 0x07); /* Opcode 81 /7 */
10980   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10981   ins_pipe(ialu_cr_reg_imm);
10982 %}
10983 
10984 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10985 %{
10986   match(Set cr (CmpI op1 (LoadI op2)));
10987 
10988   ins_cost(500); // XXX
10989   format %{ "cmpl    $op1, $op2" %}
10990   opcode(0x3B); /* Opcode 3B /r */
10991   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10992   ins_pipe(ialu_cr_reg_mem);
10993 %}
10994 
10995 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10996 %{
10997   match(Set cr (CmpI src zero));
10998 
10999   format %{ "testl   $src, $src" %}
11000   opcode(0x85);
11001   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11002   ins_pipe(ialu_cr_reg_imm);
11003 %}
11004 
11005 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
11006 %{
11007   match(Set cr (CmpI (AndI src con) zero));
11008 
11009   format %{ "testl   $src, $con" %}
11010   opcode(0xF7, 0x00);
11011   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
11012   ins_pipe(ialu_cr_reg_imm);
11013 %}
11014 
11015 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
11016 %{
11017   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
11018 
11019   format %{ "testl   $src, $mem" %}
11020   opcode(0x85);
11021   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
11022   ins_pipe(ialu_cr_reg_mem);
11023 %}
11024 
11025 // Unsigned compare Instructions; really, same as signed except they
11026 // produce an rFlagsRegU instead of rFlagsReg.
11027 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
11028 %{
11029   match(Set cr (CmpU op1 op2));
11030 
11031   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11032   opcode(0x3B); /* Opcode 3B /r */
11033   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
11034   ins_pipe(ialu_cr_reg_reg);
11035 %}
11036 
11037 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
11038 %{
11039   match(Set cr (CmpU op1 op2));
11040 
11041   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11042   opcode(0x81,0x07); /* Opcode 81 /7 */
11043   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
11044   ins_pipe(ialu_cr_reg_imm);
11045 %}
11046 
11047 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
11048 %{
11049   match(Set cr (CmpU op1 (LoadI op2)));
11050 
11051   ins_cost(500); // XXX
11052   format %{ "cmpl    $op1, $op2\t# unsigned" %}
11053   opcode(0x3B); /* Opcode 3B /r */
11054   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
11055   ins_pipe(ialu_cr_reg_mem);
11056 %}
11057 
11058 // // // Cisc-spilled version of cmpU_rReg
11059 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
11060 // //%{
11061 // //  match(Set cr (CmpU (LoadI op1) op2));
11062 // //
11063 // //  format %{ "CMPu   $op1,$op2" %}
11064 // //  ins_cost(500);
11065 // //  opcode(0x39);  /* Opcode 39 /r */
11066 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11067 // //%}
11068 
11069 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
11070 %{
11071   match(Set cr (CmpU src zero));
11072 
11073   format %{ "testl  $src, $src\t# unsigned" %}
11074   opcode(0x85);
11075   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
11076   ins_pipe(ialu_cr_reg_imm);
11077 %}
11078 
11079 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
11080 %{
11081   match(Set cr (CmpP op1 op2));
11082 
11083   format %{ "cmpq    $op1, $op2\t# ptr" %}
11084   opcode(0x3B); /* Opcode 3B /r */
11085   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11086   ins_pipe(ialu_cr_reg_reg);
11087 %}
11088 
11089 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
11090 %{
11091   match(Set cr (CmpP op1 (LoadP op2)));
11092 
11093   ins_cost(500); // XXX
11094   format %{ "cmpq    $op1, $op2\t# ptr" %}
11095   opcode(0x3B); /* Opcode 3B /r */
11096   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11097   ins_pipe(ialu_cr_reg_mem);
11098 %}
11099 
11100 // // // Cisc-spilled version of cmpP_rReg
11101 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
11102 // //%{
11103 // //  match(Set cr (CmpP (LoadP op1) op2));
11104 // //
11105 // //  format %{ "CMPu   $op1,$op2" %}
11106 // //  ins_cost(500);
11107 // //  opcode(0x39);  /* Opcode 39 /r */
11108 // //  ins_encode( OpcP, reg_mem( op1, op2) );
11109 // //%}
11110 
11111 // XXX this is generalized by compP_rReg_mem???
11112 // Compare raw pointer (used in out-of-heap check).
11113 // Only works because non-oop pointers must be raw pointers
11114 // and raw pointers have no anti-dependencies.
11115 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
11116 %{
11117   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
11118   match(Set cr (CmpP op1 (LoadP op2)));
11119 
11120   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
11121   opcode(0x3B); /* Opcode 3B /r */
11122   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11123   ins_pipe(ialu_cr_reg_mem);
11124 %}
11125 
11126 // This will generate a signed flags result. This should be OK since
11127 // any compare to a zero should be eq/neq.
11128 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
11129 %{
11130   match(Set cr (CmpP src zero));
11131 
11132   format %{ "testq   $src, $src\t# ptr" %}
11133   opcode(0x85);
11134   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11135   ins_pipe(ialu_cr_reg_imm);
11136 %}
11137 
11138 // This will generate a signed flags result. This should be OK since
11139 // any compare to a zero should be eq/neq.
11140 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
11141 %{
11142   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
11143   match(Set cr (CmpP (LoadP op) zero));
11144 
11145   ins_cost(500); // XXX
11146   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
11147   opcode(0xF7); /* Opcode F7 /0 */
11148   ins_encode(REX_mem_wide(op),
11149              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
11150   ins_pipe(ialu_cr_reg_imm);
11151 %}
11152 
11153 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
11154 %{
11155   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
11156   match(Set cr (CmpP (LoadP mem) zero));
11157 
11158   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
11159   ins_encode %{
11160     __ cmpq(r12, $mem$$Address);
11161   %}
11162   ins_pipe(ialu_cr_reg_mem);
11163 %}
11164 
11165 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
11166 %{
11167   match(Set cr (CmpN op1 op2));
11168 
11169   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11170   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
11171   ins_pipe(ialu_cr_reg_reg);
11172 %}
11173 
11174 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
11175 %{
11176   match(Set cr (CmpN src (LoadN mem)));
11177 
11178   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
11179   ins_encode %{
11180     __ cmpl($src$$Register, $mem$$Address);
11181   %}
11182   ins_pipe(ialu_cr_reg_mem);
11183 %}
11184 
11185 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
11186   match(Set cr (CmpN op1 op2));
11187 
11188   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
11189   ins_encode %{
11190     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
11191   %}
11192   ins_pipe(ialu_cr_reg_imm);
11193 %}
11194 
11195 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
11196 %{
11197   match(Set cr (CmpN src (LoadN mem)));
11198 
11199   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
11200   ins_encode %{
11201     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
11202   %}
11203   ins_pipe(ialu_cr_reg_mem);
11204 %}
11205 
11206 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11207   match(Set cr (CmpN op1 op2));
11208 
11209   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11210   ins_encode %{
11211     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11212   %}
11213   ins_pipe(ialu_cr_reg_imm);
11214 %}
11215 
11216 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11217 %{
11218   match(Set cr (CmpN src (LoadNKlass mem)));
11219 
11220   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11221   ins_encode %{
11222     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11223   %}
11224   ins_pipe(ialu_cr_reg_mem);
11225 %}
11226 
11227 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11228   match(Set cr (CmpN src zero));
11229 
11230   format %{ "testl   $src, $src\t# compressed ptr" %}
11231   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11232   ins_pipe(ialu_cr_reg_imm);
11233 %}
11234 
11235 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11236 %{
11237   predicate(Universe::narrow_oop_base() != NULL);
11238   match(Set cr (CmpN (LoadN mem) zero));
11239 
11240   ins_cost(500); // XXX
11241   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11242   ins_encode %{
11243     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11244   %}
11245   ins_pipe(ialu_cr_reg_mem);
11246 %}
11247 
11248 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11249 %{
11250   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11251   match(Set cr (CmpN (LoadN mem) zero));
11252 
11253   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11254   ins_encode %{
11255     __ cmpl(r12, $mem$$Address);
11256   %}
11257   ins_pipe(ialu_cr_reg_mem);
11258 %}
11259 
11260 // Yanked all unsigned pointer compare operations.
11261 // Pointer compares are done with CmpP which is already unsigned.
11262 
11263 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11264 %{
11265   match(Set cr (CmpL op1 op2));
11266 
11267   format %{ "cmpq    $op1, $op2" %}
11268   opcode(0x3B);  /* Opcode 3B /r */
11269   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11270   ins_pipe(ialu_cr_reg_reg);
11271 %}
11272 
11273 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11274 %{
11275   match(Set cr (CmpL op1 op2));
11276 
11277   format %{ "cmpq    $op1, $op2" %}
11278   opcode(0x81, 0x07); /* Opcode 81 /7 */
11279   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11280   ins_pipe(ialu_cr_reg_imm);
11281 %}
11282 
11283 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11284 %{
11285   match(Set cr (CmpL op1 (LoadL op2)));
11286 
11287   format %{ "cmpq    $op1, $op2" %}
11288   opcode(0x3B); /* Opcode 3B /r */
11289   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11290   ins_pipe(ialu_cr_reg_mem);
11291 %}
11292 
11293 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11294 %{
11295   match(Set cr (CmpL src zero));
11296 
11297   format %{ "testq   $src, $src" %}
11298   opcode(0x85);
11299   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11300   ins_pipe(ialu_cr_reg_imm);
11301 %}
11302 
11303 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11304 %{
11305   match(Set cr (CmpL (AndL src con) zero));
11306 
11307   format %{ "testq   $src, $con\t# long" %}
11308   opcode(0xF7, 0x00);
11309   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11310   ins_pipe(ialu_cr_reg_imm);
11311 %}
11312 
11313 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11314 %{
11315   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11316 
11317   format %{ "testq   $src, $mem" %}
11318   opcode(0x85);
11319   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11320   ins_pipe(ialu_cr_reg_mem);
11321 %}
11322 
11323 // Manifest a CmpL result in an integer register.  Very painful.
11324 // This is the test to avoid.
11325 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11326 %{
11327   match(Set dst (CmpL3 src1 src2));
11328   effect(KILL flags);
11329 
11330   ins_cost(275); // XXX
11331   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11332             "movl    $dst, -1\n\t"
11333             "jl,s    done\n\t"
11334             "setne   $dst\n\t"
11335             "movzbl  $dst, $dst\n\t"
11336     "done:" %}
11337   ins_encode(cmpl3_flag(src1, src2, dst));
11338   ins_pipe(pipe_slow);
11339 %}
11340 
11341 //----------Max and Min--------------------------------------------------------
11342 // Min Instructions
11343 
11344 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11345 %{
11346   effect(USE_DEF dst, USE src, USE cr);
11347 
11348   format %{ "cmovlgt $dst, $src\t# min" %}
11349   opcode(0x0F, 0x4F);
11350   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11351   ins_pipe(pipe_cmov_reg);
11352 %}
11353 
11354 
11355 instruct minI_rReg(rRegI dst, rRegI src)
11356 %{
11357   match(Set dst (MinI dst src));
11358 
11359   ins_cost(200);
11360   expand %{
11361     rFlagsReg cr;
11362     compI_rReg(cr, dst, src);
11363     cmovI_reg_g(dst, src, cr);
11364   %}
11365 %}
11366 
11367 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11368 %{
11369   effect(USE_DEF dst, USE src, USE cr);
11370 
11371   format %{ "cmovllt $dst, $src\t# max" %}
11372   opcode(0x0F, 0x4C);
11373   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11374   ins_pipe(pipe_cmov_reg);
11375 %}
11376 
11377 
11378 instruct maxI_rReg(rRegI dst, rRegI src)
11379 %{
11380   match(Set dst (MaxI dst src));
11381 
11382   ins_cost(200);
11383   expand %{
11384     rFlagsReg cr;
11385     compI_rReg(cr, dst, src);
11386     cmovI_reg_l(dst, src, cr);
11387   %}
11388 %}
11389 
11390 // ============================================================================
11391 // Branch Instructions
11392 
11393 // Jump Direct - Label defines a relative address from JMP+1
11394 instruct jmpDir(label labl)
11395 %{
11396   match(Goto);
11397   effect(USE labl);
11398 
11399   ins_cost(300);
11400   format %{ "jmp     $labl" %}
11401   size(5);
11402   ins_encode %{
11403     Label* L = $labl$$label;
11404     __ jmp(*L, false); // Always long jump
11405   %}
11406   ins_pipe(pipe_jmp);
11407 %}
11408 
11409 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11410 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11411 %{
11412   match(If cop cr);
11413   effect(USE labl);
11414 
11415   ins_cost(300);
11416   format %{ "j$cop     $labl" %}
11417   size(6);
11418   ins_encode %{
11419     Label* L = $labl$$label;
11420     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11421   %}
11422   ins_pipe(pipe_jcc);
11423 %}
11424 
11425 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11426 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11427 %{
11428   match(CountedLoopEnd cop cr);
11429   effect(USE labl);
11430 
11431   ins_cost(300);
11432   format %{ "j$cop     $labl\t# loop end" %}
11433   size(6);
11434   ins_encode %{
11435     Label* L = $labl$$label;
11436     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11437   %}
11438   ins_pipe(pipe_jcc);
11439 %}
11440 
11441 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11442 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11443   match(CountedLoopEnd cop cmp);
11444   effect(USE labl);
11445 
11446   ins_cost(300);
11447   format %{ "j$cop,u   $labl\t# loop end" %}
11448   size(6);
11449   ins_encode %{
11450     Label* L = $labl$$label;
11451     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11452   %}
11453   ins_pipe(pipe_jcc);
11454 %}
11455 
11456 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11457   match(CountedLoopEnd cop cmp);
11458   effect(USE labl);
11459 
11460   ins_cost(200);
11461   format %{ "j$cop,u   $labl\t# loop end" %}
11462   size(6);
11463   ins_encode %{
11464     Label* L = $labl$$label;
11465     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11466   %}
11467   ins_pipe(pipe_jcc);
11468 %}
11469 
11470 // Jump Direct Conditional - using unsigned comparison
11471 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11472   match(If cop cmp);
11473   effect(USE labl);
11474 
11475   ins_cost(300);
11476   format %{ "j$cop,u  $labl" %}
11477   size(6);
11478   ins_encode %{
11479     Label* L = $labl$$label;
11480     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11481   %}
11482   ins_pipe(pipe_jcc);
11483 %}
11484 
11485 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11486   match(If cop cmp);
11487   effect(USE labl);
11488 
11489   ins_cost(200);
11490   format %{ "j$cop,u  $labl" %}
11491   size(6);
11492   ins_encode %{
11493     Label* L = $labl$$label;
11494     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11495   %}
11496   ins_pipe(pipe_jcc);
11497 %}
11498 
11499 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11500   match(If cop cmp);
11501   effect(USE labl);
11502 
11503   ins_cost(200);
11504   format %{ $$template
11505     if ($cop$$cmpcode == Assembler::notEqual) {
11506       $$emit$$"jp,u   $labl\n\t"
11507       $$emit$$"j$cop,u   $labl"
11508     } else {
11509       $$emit$$"jp,u   done\n\t"
11510       $$emit$$"j$cop,u   $labl\n\t"
11511       $$emit$$"done:"
11512     }
11513   %}
11514   ins_encode %{
11515     Label* l = $labl$$label;
11516     if ($cop$$cmpcode == Assembler::notEqual) {
11517       __ jcc(Assembler::parity, *l, false);
11518       __ jcc(Assembler::notEqual, *l, false);
11519     } else if ($cop$$cmpcode == Assembler::equal) {
11520       Label done;
11521       __ jccb(Assembler::parity, done);
11522       __ jcc(Assembler::equal, *l, false);
11523       __ bind(done);
11524     } else {
11525        ShouldNotReachHere();
11526     }
11527   %}
11528   ins_pipe(pipe_jcc);
11529 %}
11530 
11531 // ============================================================================
11532 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11533 // superklass array for an instance of the superklass.  Set a hidden
11534 // internal cache on a hit (cache is checked with exposed code in
11535 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11536 // encoding ALSO sets flags.
11537 
11538 instruct partialSubtypeCheck(rdi_RegP result,
11539                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11540                              rFlagsReg cr)
11541 %{
11542   match(Set result (PartialSubtypeCheck sub super));
11543   effect(KILL rcx, KILL cr);
11544 
11545   ins_cost(1100);  // slightly larger than the next version
11546   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11547             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11548             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11549             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11550             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11551             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11552             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11553     "miss:\t" %}
11554 
11555   opcode(0x1); // Force a XOR of RDI
11556   ins_encode(enc_PartialSubtypeCheck());
11557   ins_pipe(pipe_slow);
11558 %}
11559 
11560 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11561                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11562                                      immP0 zero,
11563                                      rdi_RegP result)
11564 %{
11565   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11566   effect(KILL rcx, KILL result);
11567 
11568   ins_cost(1000);
11569   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11570             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11571             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11572             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11573             "jne,s   miss\t\t# Missed: flags nz\n\t"
11574             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11575     "miss:\t" %}
11576 
11577   opcode(0x0); // No need to XOR RDI
11578   ins_encode(enc_PartialSubtypeCheck());
11579   ins_pipe(pipe_slow);
11580 %}
11581 
11582 // ============================================================================
11583 // Branch Instructions -- short offset versions
11584 //
11585 // These instructions are used to replace jumps of a long offset (the default
11586 // match) with jumps of a shorter offset.  These instructions are all tagged
11587 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11588 // match rules in general matching.  Instead, the ADLC generates a conversion
11589 // method in the MachNode which can be used to do in-place replacement of the
11590 // long variant with the shorter variant.  The compiler will determine if a
11591 // branch can be taken by the is_short_branch_offset() predicate in the machine
11592 // specific code section of the file.
11593 
11594 // Jump Direct - Label defines a relative address from JMP+1
11595 instruct jmpDir_short(label labl) %{
11596   match(Goto);
11597   effect(USE labl);
11598 
11599   ins_cost(300);
11600   format %{ "jmp,s   $labl" %}
11601   size(2);
11602   ins_encode %{
11603     Label* L = $labl$$label;
11604     __ jmpb(*L);
11605   %}
11606   ins_pipe(pipe_jmp);
11607   ins_short_branch(1);
11608 %}
11609 
11610 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11611 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11612   match(If cop cr);
11613   effect(USE labl);
11614 
11615   ins_cost(300);
11616   format %{ "j$cop,s   $labl" %}
11617   size(2);
11618   ins_encode %{
11619     Label* L = $labl$$label;
11620     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11621   %}
11622   ins_pipe(pipe_jcc);
11623   ins_short_branch(1);
11624 %}
11625 
11626 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11627 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11628   match(CountedLoopEnd cop cr);
11629   effect(USE labl);
11630 
11631   ins_cost(300);
11632   format %{ "j$cop,s   $labl\t# loop end" %}
11633   size(2);
11634   ins_encode %{
11635     Label* L = $labl$$label;
11636     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11637   %}
11638   ins_pipe(pipe_jcc);
11639   ins_short_branch(1);
11640 %}
11641 
11642 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11643 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11644   match(CountedLoopEnd cop cmp);
11645   effect(USE labl);
11646 
11647   ins_cost(300);
11648   format %{ "j$cop,us  $labl\t# loop end" %}
11649   size(2);
11650   ins_encode %{
11651     Label* L = $labl$$label;
11652     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11653   %}
11654   ins_pipe(pipe_jcc);
11655   ins_short_branch(1);
11656 %}
11657 
11658 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11659   match(CountedLoopEnd cop cmp);
11660   effect(USE labl);
11661 
11662   ins_cost(300);
11663   format %{ "j$cop,us  $labl\t# loop end" %}
11664   size(2);
11665   ins_encode %{
11666     Label* L = $labl$$label;
11667     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11668   %}
11669   ins_pipe(pipe_jcc);
11670   ins_short_branch(1);
11671 %}
11672 
11673 // Jump Direct Conditional - using unsigned comparison
11674 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11675   match(If cop cmp);
11676   effect(USE labl);
11677 
11678   ins_cost(300);
11679   format %{ "j$cop,us  $labl" %}
11680   size(2);
11681   ins_encode %{
11682     Label* L = $labl$$label;
11683     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11684   %}
11685   ins_pipe(pipe_jcc);
11686   ins_short_branch(1);
11687 %}
11688 
11689 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11690   match(If cop cmp);
11691   effect(USE labl);
11692 
11693   ins_cost(300);
11694   format %{ "j$cop,us  $labl" %}
11695   size(2);
11696   ins_encode %{
11697     Label* L = $labl$$label;
11698     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11699   %}
11700   ins_pipe(pipe_jcc);
11701   ins_short_branch(1);
11702 %}
11703 
11704 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11705   match(If cop cmp);
11706   effect(USE labl);
11707 
11708   ins_cost(300);
11709   format %{ $$template
11710     if ($cop$$cmpcode == Assembler::notEqual) {
11711       $$emit$$"jp,u,s   $labl\n\t"
11712       $$emit$$"j$cop,u,s   $labl"
11713     } else {
11714       $$emit$$"jp,u,s   done\n\t"
11715       $$emit$$"j$cop,u,s  $labl\n\t"
11716       $$emit$$"done:"
11717     }
11718   %}
11719   size(4);
11720   ins_encode %{
11721     Label* l = $labl$$label;
11722     if ($cop$$cmpcode == Assembler::notEqual) {
11723       __ jccb(Assembler::parity, *l);
11724       __ jccb(Assembler::notEqual, *l);
11725     } else if ($cop$$cmpcode == Assembler::equal) {
11726       Label done;
11727       __ jccb(Assembler::parity, done);
11728       __ jccb(Assembler::equal, *l);
11729       __ bind(done);
11730     } else {
11731        ShouldNotReachHere();
11732     }
11733   %}
11734   ins_pipe(pipe_jcc);
11735   ins_short_branch(1);
11736 %}
11737 
11738 // ============================================================================
11739 // inlined locking and unlocking
11740 
11741 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11742   predicate(Compile::current()->use_rtm());
11743   match(Set cr (FastLock object box));
11744   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11745   ins_cost(300);
11746   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11747   ins_encode %{
11748     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11749                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11750                  _counters, _rtm_counters, _stack_rtm_counters,
11751                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11752                  true, ra_->C->profile_rtm());
11753   %}
11754   ins_pipe(pipe_slow);
11755 %}
11756 
11757 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11758   predicate(!Compile::current()->use_rtm());
11759   match(Set cr (FastLock object box));
11760   effect(TEMP tmp, TEMP scr, USE_KILL box);
11761   ins_cost(300);
11762   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11763   ins_encode %{
11764     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11765                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11766   %}
11767   ins_pipe(pipe_slow);
11768 %}
11769 
11770 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11771   match(Set cr (FastUnlock object box));
11772   effect(TEMP tmp, USE_KILL box);
11773   ins_cost(300);
11774   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11775   ins_encode %{
11776     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11777   %}
11778   ins_pipe(pipe_slow);
11779 %}
11780 
11781 
11782 // ============================================================================
11783 // Safepoint Instructions
11784 instruct safePoint_poll(rFlagsReg cr)
11785 %{
11786   predicate(!Assembler::is_polling_page_far());
11787   match(SafePoint);
11788   effect(KILL cr);
11789 
11790   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11791             "# Safepoint: poll for GC" %}
11792   ins_cost(125);
11793   ins_encode %{
11794     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11795     __ testl(rax, addr);
11796   %}
11797   ins_pipe(ialu_reg_mem);
11798 %}
11799 
11800 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11801 %{
11802   predicate(Assembler::is_polling_page_far());
11803   match(SafePoint poll);
11804   effect(KILL cr, USE poll);
11805 
11806   format %{ "testl  rax, [$poll]\t"
11807             "# Safepoint: poll for GC" %}
11808   ins_cost(125);
11809   ins_encode %{
11810     __ relocate(relocInfo::poll_type);
11811     __ testl(rax, Address($poll$$Register, 0));
11812   %}
11813   ins_pipe(ialu_reg_mem);
11814 %}
11815 
11816 // ============================================================================
11817 // Procedure Call/Return Instructions
11818 // Call Java Static Instruction
11819 // Note: If this code changes, the corresponding ret_addr_offset() and
11820 //       compute_padding() functions will have to be adjusted.
11821 instruct CallStaticJavaDirect(method meth) %{
11822   match(CallStaticJava);
11823   effect(USE meth);
11824 
11825   ins_cost(300);
11826   format %{ "call,static " %}
11827   opcode(0xE8); /* E8 cd */
11828   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11829   ins_pipe(pipe_slow);
11830   ins_alignment(4);
11831 %}
11832 
11833 // Call Java Dynamic Instruction
11834 // Note: If this code changes, the corresponding ret_addr_offset() and
11835 //       compute_padding() functions will have to be adjusted.
11836 instruct CallDynamicJavaDirect(method meth)
11837 %{
11838   match(CallDynamicJava);
11839   effect(USE meth);
11840 
11841   ins_cost(300);
11842   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11843             "call,dynamic " %}
11844   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11845   ins_pipe(pipe_slow);
11846   ins_alignment(4);
11847 %}
11848 
11849 // Call Runtime Instruction
11850 instruct CallRuntimeDirect(method meth)
11851 %{
11852   match(CallRuntime);
11853   effect(USE meth);
11854 
11855   ins_cost(300);
11856   format %{ "call,runtime " %}
11857   ins_encode(clear_avx, Java_To_Runtime(meth));
11858   ins_pipe(pipe_slow);
11859 %}
11860 
11861 // Call runtime without safepoint
11862 instruct CallLeafDirect(method meth)
11863 %{
11864   match(CallLeaf);
11865   effect(USE meth);
11866 
11867   ins_cost(300);
11868   format %{ "call_leaf,runtime " %}
11869   ins_encode(clear_avx, Java_To_Runtime(meth));
11870   ins_pipe(pipe_slow);
11871 %}
11872 
11873 // Call runtime without safepoint
11874 instruct CallLeafNoFPDirect(method meth)
11875 %{
11876   match(CallLeafNoFP);
11877   effect(USE meth);
11878 
11879   ins_cost(300);
11880   format %{ "call_leaf_nofp,runtime " %}
11881   ins_encode(Java_To_Runtime(meth));
11882   ins_pipe(pipe_slow);
11883 %}
11884 
11885 // Return Instruction
11886 // Remove the return address & jump to it.
11887 // Notice: We always emit a nop after a ret to make sure there is room
11888 // for safepoint patching
11889 instruct Ret()
11890 %{
11891   match(Return);
11892 
11893   format %{ "ret" %}
11894   opcode(0xC3);
11895   ins_encode(OpcP);
11896   ins_pipe(pipe_jmp);
11897 %}
11898 
11899 // Tail Call; Jump from runtime stub to Java code.
11900 // Also known as an 'interprocedural jump'.
11901 // Target of jump will eventually return to caller.
11902 // TailJump below removes the return address.
11903 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11904 %{
11905   match(TailCall jump_target method_oop);
11906 
11907   ins_cost(300);
11908   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11909   opcode(0xFF, 0x4); /* Opcode FF /4 */
11910   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11911   ins_pipe(pipe_jmp);
11912 %}
11913 
11914 // Tail Jump; remove the return address; jump to target.
11915 // TailCall above leaves the return address around.
11916 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11917 %{
11918   match(TailJump jump_target ex_oop);
11919 
11920   ins_cost(300);
11921   format %{ "popq    rdx\t# pop return address\n\t"
11922             "jmp     $jump_target" %}
11923   opcode(0xFF, 0x4); /* Opcode FF /4 */
11924   ins_encode(Opcode(0x5a), // popq rdx
11925              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11926   ins_pipe(pipe_jmp);
11927 %}
11928 
11929 // Create exception oop: created by stack-crawling runtime code.
11930 // Created exception is now available to this handler, and is setup
11931 // just prior to jumping to this handler.  No code emitted.
11932 instruct CreateException(rax_RegP ex_oop)
11933 %{
11934   match(Set ex_oop (CreateEx));
11935 
11936   size(0);
11937   // use the following format syntax
11938   format %{ "# exception oop is in rax; no code emitted" %}
11939   ins_encode();
11940   ins_pipe(empty);
11941 %}
11942 
11943 // Rethrow exception:
11944 // The exception oop will come in the first argument position.
11945 // Then JUMP (not call) to the rethrow stub code.
11946 instruct RethrowException()
11947 %{
11948   match(Rethrow);
11949 
11950   // use the following format syntax
11951   format %{ "jmp     rethrow_stub" %}
11952   ins_encode(enc_rethrow);
11953   ins_pipe(pipe_jmp);
11954 %}
11955 
11956 
11957 // ============================================================================
11958 // This name is KNOWN by the ADLC and cannot be changed.
11959 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11960 // for this guy.
11961 instruct tlsLoadP(r15_RegP dst) %{
11962   match(Set dst (ThreadLocal));
11963   effect(DEF dst);
11964 
11965   size(0);
11966   format %{ "# TLS is in R15" %}
11967   ins_encode( /*empty encoding*/ );
11968   ins_pipe(ialu_reg_reg);
11969 %}
11970 
11971 
11972 //----------PEEPHOLE RULES-----------------------------------------------------
11973 // These must follow all instruction definitions as they use the names
11974 // defined in the instructions definitions.
11975 //
11976 // peepmatch ( root_instr_name [preceding_instruction]* );
11977 //
11978 // peepconstraint %{
11979 // (instruction_number.operand_name relational_op instruction_number.operand_name
11980 //  [, ...] );
11981 // // instruction numbers are zero-based using left to right order in peepmatch
11982 //
11983 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11984 // // provide an instruction_number.operand_name for each operand that appears
11985 // // in the replacement instruction's match rule
11986 //
11987 // ---------VM FLAGS---------------------------------------------------------
11988 //
11989 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11990 //
11991 // Each peephole rule is given an identifying number starting with zero and
11992 // increasing by one in the order seen by the parser.  An individual peephole
11993 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11994 // on the command-line.
11995 //
11996 // ---------CURRENT LIMITATIONS----------------------------------------------
11997 //
11998 // Only match adjacent instructions in same basic block
11999 // Only equality constraints
12000 // Only constraints between operands, not (0.dest_reg == RAX_enc)
12001 // Only one replacement instruction
12002 //
12003 // ---------EXAMPLE----------------------------------------------------------
12004 //
12005 // // pertinent parts of existing instructions in architecture description
12006 // instruct movI(rRegI dst, rRegI src)
12007 // %{
12008 //   match(Set dst (CopyI src));
12009 // %}
12010 //
12011 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
12012 // %{
12013 //   match(Set dst (AddI dst src));
12014 //   effect(KILL cr);
12015 // %}
12016 //
12017 // // Change (inc mov) to lea
12018 // peephole %{
12019 //   // increment preceeded by register-register move
12020 //   peepmatch ( incI_rReg movI );
12021 //   // require that the destination register of the increment
12022 //   // match the destination register of the move
12023 //   peepconstraint ( 0.dst == 1.dst );
12024 //   // construct a replacement instruction that sets
12025 //   // the destination to ( move's source register + one )
12026 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
12027 // %}
12028 //
12029 
12030 // Implementation no longer uses movX instructions since
12031 // machine-independent system no longer uses CopyX nodes.
12032 //
12033 // peephole
12034 // %{
12035 //   peepmatch (incI_rReg movI);
12036 //   peepconstraint (0.dst == 1.dst);
12037 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12038 // %}
12039 
12040 // peephole
12041 // %{
12042 //   peepmatch (decI_rReg movI);
12043 //   peepconstraint (0.dst == 1.dst);
12044 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12045 // %}
12046 
12047 // peephole
12048 // %{
12049 //   peepmatch (addI_rReg_imm movI);
12050 //   peepconstraint (0.dst == 1.dst);
12051 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
12052 // %}
12053 
12054 // peephole
12055 // %{
12056 //   peepmatch (incL_rReg movL);
12057 //   peepconstraint (0.dst == 1.dst);
12058 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12059 // %}
12060 
12061 // peephole
12062 // %{
12063 //   peepmatch (decL_rReg movL);
12064 //   peepconstraint (0.dst == 1.dst);
12065 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12066 // %}
12067 
12068 // peephole
12069 // %{
12070 //   peepmatch (addL_rReg_imm movL);
12071 //   peepconstraint (0.dst == 1.dst);
12072 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
12073 // %}
12074 
12075 // peephole
12076 // %{
12077 //   peepmatch (addP_rReg_imm movP);
12078 //   peepconstraint (0.dst == 1.dst);
12079 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
12080 // %}
12081 
12082 // // Change load of spilled value to only a spill
12083 // instruct storeI(memory mem, rRegI src)
12084 // %{
12085 //   match(Set mem (StoreI mem src));
12086 // %}
12087 //
12088 // instruct loadI(rRegI dst, memory mem)
12089 // %{
12090 //   match(Set dst (LoadI mem));
12091 // %}
12092 //
12093 
12094 peephole
12095 %{
12096   peepmatch (loadI storeI);
12097   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12098   peepreplace (storeI(1.mem 1.mem 1.src));
12099 %}
12100 
12101 peephole
12102 %{
12103   peepmatch (loadL storeL);
12104   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
12105   peepreplace (storeL(1.mem 1.mem 1.src));
12106 %}
12107 
12108 //----------SMARTSPILL RULES---------------------------------------------------
12109 // These must follow all instruction definitions as they use the names
12110 // defined in the instructions definitions.