1 //
   2 // Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 //
   5 // This code is free software; you can redistribute it and/or modify it
   6 // under the terms of the GNU General Public License version 2 only, as
   7 // published by the Free Software Foundation.
   8 //
   9 // This code is distributed in the hope that it will be useful, but WITHOUT
  10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 // version 2 for more details (a copy is included in the LICENSE file that
  13 // accompanied this code).
  14 //
  15 // You should have received a copy of the GNU General Public License version
  16 // 2 along with this work; if not, write to the Free Software Foundation,
  17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18 //
  19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20 // or visit www.oracle.com if you need additional information or have any
  21 // questions.
  22 //
  23 //
  24 
  25 // AMD64 Architecture Description File
  26 
  27 //----------REGISTER DEFINITION BLOCK------------------------------------------
  28 // This information is used by the matcher and the register allocator to
  29 // describe individual registers and classes of registers within the target
  30 // archtecture.
  31 
  32 register %{
  33 //----------Architecture Description Register Definitions----------------------
  34 // General Registers
  35 // "reg_def"  name ( register save type, C convention save type,
  36 //                   ideal register type, encoding );
  37 // Register Save Types:
  38 //
  39 // NS  = No-Save:       The register allocator assumes that these registers
  40 //                      can be used without saving upon entry to the method, &
  41 //                      that they do not need to be saved at call sites.
  42 //
  43 // SOC = Save-On-Call:  The register allocator assumes that these registers
  44 //                      can be used without saving upon entry to the method,
  45 //                      but that they must be saved at call sites.
  46 //
  47 // SOE = Save-On-Entry: The register allocator assumes that these registers
  48 //                      must be saved before using them upon entry to the
  49 //                      method, but they do not need to be saved at call
  50 //                      sites.
  51 //
  52 // AS  = Always-Save:   The register allocator assumes that these registers
  53 //                      must be saved before using them upon entry to the
  54 //                      method, & that they must be saved at call sites.
  55 //
  56 // Ideal Register Type is used to determine how to save & restore a
  57 // register.  Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
  58 // spilled with LoadP/StoreP.  If the register supports both, use Op_RegI.
  59 //
  60 // The encoding number is the actual bit-pattern placed into the opcodes.
  61 
  62 // General Registers
  63 // R8-R15 must be encoded with REX.  (RSP, RBP, RSI, RDI need REX when
  64 // used as byte registers)
  65 
  66 // Previously set RBX, RSI, and RDI as save-on-entry for java code
  67 // Turn off SOE in java-code due to frequent use of uncommon-traps.
  68 // Now that allocator is better, turn on RSI and RDI as SOE registers.
  69 
  70 reg_def RAX  (SOC, SOC, Op_RegI,  0, rax->as_VMReg());
  71 reg_def RAX_H(SOC, SOC, Op_RegI,  0, rax->as_VMReg()->next());
  72 
  73 reg_def RCX  (SOC, SOC, Op_RegI,  1, rcx->as_VMReg());
  74 reg_def RCX_H(SOC, SOC, Op_RegI,  1, rcx->as_VMReg()->next());
  75 
  76 reg_def RDX  (SOC, SOC, Op_RegI,  2, rdx->as_VMReg());
  77 reg_def RDX_H(SOC, SOC, Op_RegI,  2, rdx->as_VMReg()->next());
  78 
  79 reg_def RBX  (SOC, SOE, Op_RegI,  3, rbx->as_VMReg());
  80 reg_def RBX_H(SOC, SOE, Op_RegI,  3, rbx->as_VMReg()->next());
  81 
  82 reg_def RSP  (NS,  NS,  Op_RegI,  4, rsp->as_VMReg());
  83 reg_def RSP_H(NS,  NS,  Op_RegI,  4, rsp->as_VMReg()->next());
  84 
  85 // now that adapter frames are gone RBP is always saved and restored by the prolog/epilog code
  86 reg_def RBP  (NS, SOE, Op_RegI,  5, rbp->as_VMReg());
  87 reg_def RBP_H(NS, SOE, Op_RegI,  5, rbp->as_VMReg()->next());
  88 
  89 #ifdef _WIN64
  90 
  91 reg_def RSI  (SOC, SOE, Op_RegI,  6, rsi->as_VMReg());
  92 reg_def RSI_H(SOC, SOE, Op_RegI,  6, rsi->as_VMReg()->next());
  93 
  94 reg_def RDI  (SOC, SOE, Op_RegI,  7, rdi->as_VMReg());
  95 reg_def RDI_H(SOC, SOE, Op_RegI,  7, rdi->as_VMReg()->next());
  96 
  97 #else
  98 
  99 reg_def RSI  (SOC, SOC, Op_RegI,  6, rsi->as_VMReg());
 100 reg_def RSI_H(SOC, SOC, Op_RegI,  6, rsi->as_VMReg()->next());
 101 
 102 reg_def RDI  (SOC, SOC, Op_RegI,  7, rdi->as_VMReg());
 103 reg_def RDI_H(SOC, SOC, Op_RegI,  7, rdi->as_VMReg()->next());
 104 
 105 #endif
 106 
 107 reg_def R8   (SOC, SOC, Op_RegI,  8, r8->as_VMReg());
 108 reg_def R8_H (SOC, SOC, Op_RegI,  8, r8->as_VMReg()->next());
 109 
 110 reg_def R9   (SOC, SOC, Op_RegI,  9, r9->as_VMReg());
 111 reg_def R9_H (SOC, SOC, Op_RegI,  9, r9->as_VMReg()->next());
 112 
 113 reg_def R10  (SOC, SOC, Op_RegI, 10, r10->as_VMReg());
 114 reg_def R10_H(SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
 115 
 116 reg_def R11  (SOC, SOC, Op_RegI, 11, r11->as_VMReg());
 117 reg_def R11_H(SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
 118 
 119 reg_def R12  (SOC, SOE, Op_RegI, 12, r12->as_VMReg());
 120 reg_def R12_H(SOC, SOE, Op_RegI, 12, r12->as_VMReg()->next());
 121 
 122 reg_def R13  (SOC, SOE, Op_RegI, 13, r13->as_VMReg());
 123 reg_def R13_H(SOC, SOE, Op_RegI, 13, r13->as_VMReg()->next());
 124 
 125 reg_def R14  (SOC, SOE, Op_RegI, 14, r14->as_VMReg());
 126 reg_def R14_H(SOC, SOE, Op_RegI, 14, r14->as_VMReg()->next());
 127 
 128 reg_def R15  (SOC, SOE, Op_RegI, 15, r15->as_VMReg());
 129 reg_def R15_H(SOC, SOE, Op_RegI, 15, r15->as_VMReg()->next());
 130 
 131 
 132 // Floating Point Registers
 133 
 134 // Specify priority of register selection within phases of register
 135 // allocation.  Highest priority is first.  A useful heuristic is to
 136 // give registers a low priority when they are required by machine
 137 // instructions, like EAX and EDX on I486, and choose no-save registers
 138 // before save-on-call, & save-on-call before save-on-entry.  Registers
 139 // which participate in fixed calling sequences should come last.
 140 // Registers which are used as pairs must fall on an even boundary.
 141 
 142 alloc_class chunk0(R10,         R10_H,
 143                    R11,         R11_H,
 144                    R8,          R8_H,
 145                    R9,          R9_H,
 146                    R12,         R12_H,
 147                    RCX,         RCX_H,
 148                    RBX,         RBX_H,
 149                    RDI,         RDI_H,
 150                    RDX,         RDX_H,
 151                    RSI,         RSI_H,
 152                    RAX,         RAX_H,
 153                    RBP,         RBP_H,
 154                    R13,         R13_H,
 155                    R14,         R14_H,
 156                    R15,         R15_H,
 157                    RSP,         RSP_H);
 158 
 159 
 160 //----------Architecture Description Register Classes--------------------------
 161 // Several register classes are automatically defined based upon information in
 162 // this architecture description.
 163 // 1) reg_class inline_cache_reg           ( /* as def'd in frame section */ )
 164 // 2) reg_class compiler_method_oop_reg    ( /* as def'd in frame section */ )
 165 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
 166 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
 167 //
 168 
 169 // Empty register class.
 170 reg_class no_reg();
 171 
 172 // Class for all pointer registers (including RSP and RBP)
 173 reg_class any_reg_with_rbp(RAX, RAX_H,
 174                            RDX, RDX_H,
 175                            RBP, RBP_H,
 176                            RDI, RDI_H,
 177                            RSI, RSI_H,
 178                            RCX, RCX_H,
 179                            RBX, RBX_H,
 180                            RSP, RSP_H,
 181                            R8,  R8_H,
 182                            R9,  R9_H,
 183                            R10, R10_H,
 184                            R11, R11_H,
 185                            R12, R12_H,
 186                            R13, R13_H,
 187                            R14, R14_H,
 188                            R15, R15_H);
 189 
 190 // Class for all pointer registers (including RSP, but excluding RBP)
 191 reg_class any_reg_no_rbp(RAX, RAX_H,
 192                          RDX, RDX_H,
 193                          RDI, RDI_H,
 194                          RSI, RSI_H,
 195                          RCX, RCX_H,
 196                          RBX, RBX_H,
 197                          RSP, RSP_H,
 198                          R8,  R8_H,
 199                          R9,  R9_H,
 200                          R10, R10_H,
 201                          R11, R11_H,
 202                          R12, R12_H,
 203                          R13, R13_H,
 204                          R14, R14_H,
 205                          R15, R15_H);
 206 
 207 // Dynamic register class that selects at runtime between register classes
 208 // any_reg_no_rbp and any_reg_with_rbp (depending on the value of the flag PreserveFramePointer).
 209 // Equivalent to: return PreserveFramePointer ? any_reg_no_rbp : any_reg_with_rbp;
 210 reg_class_dynamic any_reg(any_reg_no_rbp, any_reg_with_rbp, %{ PreserveFramePointer %});
 211 
 212 // Class for all pointer registers (excluding RSP)
 213 reg_class ptr_reg_with_rbp(RAX, RAX_H,
 214                            RDX, RDX_H,
 215                            RBP, RBP_H,
 216                            RDI, RDI_H,
 217                            RSI, RSI_H,
 218                            RCX, RCX_H,
 219                            RBX, RBX_H,
 220                            R8,  R8_H,
 221                            R9,  R9_H,
 222                            R10, R10_H,
 223                            R11, R11_H,
 224                            R13, R13_H,
 225                            R14, R14_H);
 226 
 227 // Class for all pointer registers (excluding RSP and RBP)
 228 reg_class ptr_reg_no_rbp(RAX, RAX_H,
 229                          RDX, RDX_H,
 230                          RDI, RDI_H,
 231                          RSI, RSI_H,
 232                          RCX, RCX_H,
 233                          RBX, RBX_H,
 234                          R8,  R8_H,
 235                          R9,  R9_H,
 236                          R10, R10_H,
 237                          R11, R11_H,
 238                          R13, R13_H,
 239                          R14, R14_H);
 240 
 241 // Dynamic register class that selects between ptr_reg_no_rbp and ptr_reg_with_rbp.
 242 reg_class_dynamic ptr_reg(ptr_reg_no_rbp, ptr_reg_with_rbp, %{ PreserveFramePointer %});
 243 
 244 // Class for all pointer registers (excluding RAX and RSP)
 245 reg_class ptr_no_rax_reg_with_rbp(RDX, RDX_H,
 246                                   RBP, RBP_H,
 247                                   RDI, RDI_H,
 248                                   RSI, RSI_H,
 249                                   RCX, RCX_H,
 250                                   RBX, RBX_H,
 251                                   R8,  R8_H,
 252                                   R9,  R9_H,
 253                                   R10, R10_H,
 254                                   R11, R11_H,
 255                                   R13, R13_H,
 256                                   R14, R14_H);
 257 
 258 // Class for all pointer registers (excluding RAX, RSP, and RBP)
 259 reg_class ptr_no_rax_reg_no_rbp(RDX, RDX_H,
 260                                 RDI, RDI_H,
 261                                 RSI, RSI_H,
 262                                 RCX, RCX_H,
 263                                 RBX, RBX_H,
 264                                 R8,  R8_H,
 265                                 R9,  R9_H,
 266                                 R10, R10_H,
 267                                 R11, R11_H,
 268                                 R13, R13_H,
 269                                 R14, R14_H);
 270 
 271 // Dynamic register class that selects between ptr_no_rax_reg_no_rbp and ptr_no_rax_reg_with_rbp.
 272 reg_class_dynamic ptr_no_rax_reg(ptr_no_rax_reg_no_rbp, ptr_no_rax_reg_with_rbp, %{ PreserveFramePointer %});
 273 
 274 // Class for all pointer registers (excluding RAX, RBX, and RSP)
 275 reg_class ptr_no_rax_rbx_reg_with_rbp(RDX, RDX_H,
 276                                       RBP, RBP_H,
 277                                       RDI, RDI_H,
 278                                       RSI, RSI_H,
 279                                       RCX, RCX_H,
 280                                       R8,  R8_H,
 281                                       R9,  R9_H,
 282                                       R10, R10_H,
 283                                       R11, R11_H,
 284                                       R13, R13_H,
 285                                       R14, R14_H);
 286 
 287 // Class for all pointer registers (excluding RAX, RBX, RSP, and RBP)
 288 reg_class ptr_no_rax_rbx_reg_no_rbp(RDX, RDX_H,
 289                                     RDI, RDI_H,
 290                                     RSI, RSI_H,
 291                                     RCX, RCX_H,
 292                                     R8,  R8_H,
 293                                     R9,  R9_H,
 294                                     R10, R10_H,
 295                                     R11, R11_H,
 296                                     R13, R13_H,
 297                                     R14, R14_H);
 298 
 299 // Dynamic register class that selects between ptr_no_rax_rbx_reg_no_rbp and ptr_no_rax_rbx_reg_with_rbp.
 300 reg_class_dynamic ptr_no_rax_rbx_reg(ptr_no_rax_rbx_reg_no_rbp, ptr_no_rax_rbx_reg_with_rbp, %{ PreserveFramePointer %});
 301 
 302 // Singleton class for RAX pointer register
 303 reg_class ptr_rax_reg(RAX, RAX_H);
 304 
 305 // Singleton class for RBX pointer register
 306 reg_class ptr_rbx_reg(RBX, RBX_H);
 307 
 308 // Singleton class for RSI pointer register
 309 reg_class ptr_rsi_reg(RSI, RSI_H);
 310 
 311 // Singleton class for RDI pointer register
 312 reg_class ptr_rdi_reg(RDI, RDI_H);
 313 
 314 // Singleton class for stack pointer
 315 reg_class ptr_rsp_reg(RSP, RSP_H);
 316 
 317 // Singleton class for TLS pointer
 318 reg_class ptr_r15_reg(R15, R15_H);
 319 
 320 // Class for all long registers (excluding RSP)
 321 reg_class long_reg_with_rbp(RAX, RAX_H,
 322                             RDX, RDX_H,
 323                             RBP, RBP_H,
 324                             RDI, RDI_H,
 325                             RSI, RSI_H,
 326                             RCX, RCX_H,
 327                             RBX, RBX_H,
 328                             R8,  R8_H,
 329                             R9,  R9_H,
 330                             R10, R10_H,
 331                             R11, R11_H,
 332                             R13, R13_H,
 333                             R14, R14_H);
 334 
 335 // Class for all long registers (excluding RSP and RBP)
 336 reg_class long_reg_no_rbp(RAX, RAX_H,
 337                           RDX, RDX_H,
 338                           RDI, RDI_H,
 339                           RSI, RSI_H,
 340                           RCX, RCX_H,
 341                           RBX, RBX_H,
 342                           R8,  R8_H,
 343                           R9,  R9_H,
 344                           R10, R10_H,
 345                           R11, R11_H,
 346                           R13, R13_H,
 347                           R14, R14_H);
 348 
 349 // Dynamic register class that selects between long_reg_no_rbp and long_reg_with_rbp.
 350 reg_class_dynamic long_reg(long_reg_no_rbp, long_reg_with_rbp, %{ PreserveFramePointer %});
 351 
 352 // Class for all long registers (excluding RAX, RDX and RSP)
 353 reg_class long_no_rax_rdx_reg_with_rbp(RBP, RBP_H,
 354                                        RDI, RDI_H,
 355                                        RSI, RSI_H,
 356                                        RCX, RCX_H,
 357                                        RBX, RBX_H,
 358                                        R8,  R8_H,
 359                                        R9,  R9_H,
 360                                        R10, R10_H,
 361                                        R11, R11_H,
 362                                        R13, R13_H,
 363                                        R14, R14_H);
 364 
 365 // Class for all long registers (excluding RAX, RDX, RSP, and RBP)
 366 reg_class long_no_rax_rdx_reg_no_rbp(RDI, RDI_H,
 367                                      RSI, RSI_H,
 368                                      RCX, RCX_H,
 369                                      RBX, RBX_H,
 370                                      R8,  R8_H,
 371                                      R9,  R9_H,
 372                                      R10, R10_H,
 373                                      R11, R11_H,
 374                                      R13, R13_H,
 375                                      R14, R14_H);
 376 
 377 // Dynamic register class that selects between long_no_rax_rdx_reg_no_rbp and long_no_rax_rdx_reg_with_rbp.
 378 reg_class_dynamic long_no_rax_rdx_reg(long_no_rax_rdx_reg_no_rbp, long_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 379 
 380 // Class for all long registers (excluding RCX and RSP)
 381 reg_class long_no_rcx_reg_with_rbp(RBP, RBP_H,
 382                                    RDI, RDI_H,
 383                                    RSI, RSI_H,
 384                                    RAX, RAX_H,
 385                                    RDX, RDX_H,
 386                                    RBX, RBX_H,
 387                                    R8,  R8_H,
 388                                    R9,  R9_H,
 389                                    R10, R10_H,
 390                                    R11, R11_H,
 391                                    R13, R13_H,
 392                                    R14, R14_H);
 393 
 394 // Class for all long registers (excluding RCX, RSP, and RBP)
 395 reg_class long_no_rcx_reg_no_rbp(RDI, RDI_H,
 396                                  RSI, RSI_H,
 397                                  RAX, RAX_H,
 398                                  RDX, RDX_H,
 399                                  RBX, RBX_H,
 400                                  R8,  R8_H,
 401                                  R9,  R9_H,
 402                                  R10, R10_H,
 403                                  R11, R11_H,
 404                                  R13, R13_H,
 405                                  R14, R14_H);
 406 
 407 // Dynamic register class that selects between long_no_rcx_reg_no_rbp and long_no_rcx_reg_with_rbp.
 408 reg_class_dynamic long_no_rcx_reg(long_no_rcx_reg_no_rbp, long_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 409 
 410 // Singleton class for RAX long register
 411 reg_class long_rax_reg(RAX, RAX_H);
 412 
 413 // Singleton class for RCX long register
 414 reg_class long_rcx_reg(RCX, RCX_H);
 415 
 416 // Singleton class for RDX long register
 417 reg_class long_rdx_reg(RDX, RDX_H);
 418 
 419 // Class for all int registers (excluding RSP)
 420 reg_class int_reg_with_rbp(RAX,
 421                            RDX,
 422                            RBP,
 423                            RDI,
 424                            RSI,
 425                            RCX,
 426                            RBX,
 427                            R8,
 428                            R9,
 429                            R10,
 430                            R11,
 431                            R13,
 432                            R14);
 433 
 434 // Class for all int registers (excluding RSP and RBP)
 435 reg_class int_reg_no_rbp(RAX,
 436                          RDX,
 437                          RDI,
 438                          RSI,
 439                          RCX,
 440                          RBX,
 441                          R8,
 442                          R9,
 443                          R10,
 444                          R11,
 445                          R13,
 446                          R14);
 447 
 448 // Dynamic register class that selects between int_reg_no_rbp and int_reg_with_rbp.
 449 reg_class_dynamic int_reg(int_reg_no_rbp, int_reg_with_rbp, %{ PreserveFramePointer %});
 450 
 451 // Class for all int registers (excluding RCX and RSP)
 452 reg_class int_no_rcx_reg_with_rbp(RAX,
 453                                   RDX,
 454                                   RBP,
 455                                   RDI,
 456                                   RSI,
 457                                   RBX,
 458                                   R8,
 459                                   R9,
 460                                   R10,
 461                                   R11,
 462                                   R13,
 463                                   R14);
 464 
 465 // Class for all int registers (excluding RCX, RSP, and RBP)
 466 reg_class int_no_rcx_reg_no_rbp(RAX,
 467                                 RDX,
 468                                 RDI,
 469                                 RSI,
 470                                 RBX,
 471                                 R8,
 472                                 R9,
 473                                 R10,
 474                                 R11,
 475                                 R13,
 476                                 R14);
 477 
 478 // Dynamic register class that selects between int_no_rcx_reg_no_rbp and int_no_rcx_reg_with_rbp.
 479 reg_class_dynamic int_no_rcx_reg(int_no_rcx_reg_no_rbp, int_no_rcx_reg_with_rbp, %{ PreserveFramePointer %});
 480 
 481 // Class for all int registers (excluding RAX, RDX, and RSP)
 482 reg_class int_no_rax_rdx_reg_with_rbp(RBP,
 483                                       RDI,
 484                                       RSI,
 485                                       RCX,
 486                                       RBX,
 487                                       R8,
 488                                       R9,
 489                                       R10,
 490                                       R11,
 491                                       R13,
 492                                       R14);
 493 
 494 // Class for all int registers (excluding RAX, RDX, RSP, and RBP)
 495 reg_class int_no_rax_rdx_reg_no_rbp(RDI,
 496                                     RSI,
 497                                     RCX,
 498                                     RBX,
 499                                     R8,
 500                                     R9,
 501                                     R10,
 502                                     R11,
 503                                     R13,
 504                                     R14);
 505 
 506 // Dynamic register class that selects between int_no_rax_rdx_reg_no_rbp and int_no_rax_rdx_reg_with_rbp.
 507 reg_class_dynamic int_no_rax_rdx_reg(int_no_rax_rdx_reg_no_rbp, int_no_rax_rdx_reg_with_rbp, %{ PreserveFramePointer %});
 508 
 509 // Singleton class for RAX int register
 510 reg_class int_rax_reg(RAX);
 511 
 512 // Singleton class for RBX int register
 513 reg_class int_rbx_reg(RBX);
 514 
 515 // Singleton class for RCX int register
 516 reg_class int_rcx_reg(RCX);
 517 
 518 // Singleton class for RCX int register
 519 reg_class int_rdx_reg(RDX);
 520 
 521 // Singleton class for RCX int register
 522 reg_class int_rdi_reg(RDI);
 523 
 524 // Singleton class for instruction pointer
 525 // reg_class ip_reg(RIP);
 526 
 527 %}
 528 
 529 //----------SOURCE BLOCK-------------------------------------------------------
 530 // This is a block of C++ code which provides values, functions, and
 531 // definitions necessary in the rest of the architecture description
 532 source %{
 533 #define   RELOC_IMM64    Assembler::imm_operand
 534 #define   RELOC_DISP32   Assembler::disp32_operand
 535 
 536 #define __ _masm.
 537 
 538 static int clear_avx_size() {
 539   if(UseAVX > 2) {
 540     return 0; // vzeroupper is ignored
 541   } else {
 542     return (Compile::current()->max_vector_size() > 16) ? 3 : 0;  // vzeroupper
 543   }
 544 }
 545 
 546 // !!!!! Special hack to get all types of calls to specify the byte offset
 547 //       from the start of the call to the point where the return address
 548 //       will point.
 549 int MachCallStaticJavaNode::ret_addr_offset()
 550 {
 551   int offset = 5; // 5 bytes from start of call to where return address points
 552   offset += clear_avx_size();
 553   return offset;
 554 }
 555 
 556 int MachCallDynamicJavaNode::ret_addr_offset()
 557 {
 558   int offset = 15; // 15 bytes from start of call to where return address points
 559   offset += clear_avx_size();
 560   return offset;
 561 }
 562 
 563 int MachCallRuntimeNode::ret_addr_offset() {
 564   int offset = 13; // movq r10,#addr; callq (r10)
 565   offset += clear_avx_size();
 566   return offset;
 567 }
 568 
 569 // Indicate if the safepoint node needs the polling page as an input,
 570 // it does if the polling page is more than disp32 away.
 571 bool SafePointNode::needs_polling_address_input()
 572 {
 573   return Assembler::is_polling_page_far();
 574 }
 575 
 576 //
 577 // Compute padding required for nodes which need alignment
 578 //
 579 
 580 // The address of the call instruction needs to be 4-byte aligned to
 581 // ensure that it does not span a cache line so that it can be patched.
 582 int CallStaticJavaDirectNode::compute_padding(int current_offset) const
 583 {
 584   current_offset += clear_avx_size(); // skip vzeroupper
 585   current_offset += 1; // skip call opcode byte
 586   return round_to(current_offset, alignment_required()) - current_offset;
 587 }
 588 
 589 // The address of the call instruction needs to be 4-byte aligned to
 590 // ensure that it does not span a cache line so that it can be patched.
 591 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 592 {
 593   current_offset += clear_avx_size(); // skip vzeroupper
 594   current_offset += 11; // skip movq instruction + call opcode byte
 595   return round_to(current_offset, alignment_required()) - current_offset;
 596 }
 597 
 598 // EMIT_RM()
 599 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 600   unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 601   cbuf.insts()->emit_int8(c);
 602 }
 603 
 604 // EMIT_CC()
 605 void emit_cc(CodeBuffer &cbuf, int f1, int f2) {
 606   unsigned char c = (unsigned char) (f1 | f2);
 607   cbuf.insts()->emit_int8(c);
 608 }
 609 
 610 // EMIT_OPCODE()
 611 void emit_opcode(CodeBuffer &cbuf, int code) {
 612   cbuf.insts()->emit_int8((unsigned char) code);
 613 }
 614 
 615 // EMIT_OPCODE() w/ relocation information
 616 void emit_opcode(CodeBuffer &cbuf,
 617                  int code, relocInfo::relocType reloc, int offset, int format)
 618 {
 619   cbuf.relocate(cbuf.insts_mark() + offset, reloc, format);
 620   emit_opcode(cbuf, code);
 621 }
 622 
 623 // EMIT_D8()
 624 void emit_d8(CodeBuffer &cbuf, int d8) {
 625   cbuf.insts()->emit_int8((unsigned char) d8);
 626 }
 627 
 628 // EMIT_D16()
 629 void emit_d16(CodeBuffer &cbuf, int d16) {
 630   cbuf.insts()->emit_int16(d16);
 631 }
 632 
 633 // EMIT_D32()
 634 void emit_d32(CodeBuffer &cbuf, int d32) {
 635   cbuf.insts()->emit_int32(d32);
 636 }
 637 
 638 // EMIT_D64()
 639 void emit_d64(CodeBuffer &cbuf, int64_t d64) {
 640   cbuf.insts()->emit_int64(d64);
 641 }
 642 
 643 // emit 32 bit value and construct relocation entry from relocInfo::relocType
 644 void emit_d32_reloc(CodeBuffer& cbuf,
 645                     int d32,
 646                     relocInfo::relocType reloc,
 647                     int format)
 648 {
 649   assert(reloc != relocInfo::external_word_type, "use 2-arg emit_d32_reloc");
 650   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 651   cbuf.insts()->emit_int32(d32);
 652 }
 653 
 654 // emit 32 bit value and construct relocation entry from RelocationHolder
 655 void emit_d32_reloc(CodeBuffer& cbuf, int d32, RelocationHolder const& rspec, int format) {
 656 #ifdef ASSERT
 657   if (rspec.reloc()->type() == relocInfo::oop_type &&
 658       d32 != 0 && d32 != (intptr_t) Universe::non_oop_word()) {
 659     assert(Universe::heap()->is_in_reserved((address)(intptr_t)d32), "should be real oop");
 660     assert(cast_to_oop((intptr_t)d32)->is_oop() && (ScavengeRootsInCode || !cast_to_oop((intptr_t)d32)->is_scavengable()), "cannot embed scavengable oops in code");
 661   }
 662 #endif
 663   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 664   cbuf.insts()->emit_int32(d32);
 665 }
 666 
 667 void emit_d32_reloc(CodeBuffer& cbuf, address addr) {
 668   address next_ip = cbuf.insts_end() + 4;
 669   emit_d32_reloc(cbuf, (int) (addr - next_ip),
 670                  external_word_Relocation::spec(addr),
 671                  RELOC_DISP32);
 672 }
 673 
 674 
 675 // emit 64 bit value and construct relocation entry from relocInfo::relocType
 676 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, relocInfo::relocType reloc, int format) {
 677   cbuf.relocate(cbuf.insts_mark(), reloc, format);
 678   cbuf.insts()->emit_int64(d64);
 679 }
 680 
 681 // emit 64 bit value and construct relocation entry from RelocationHolder
 682 void emit_d64_reloc(CodeBuffer& cbuf, int64_t d64, RelocationHolder const& rspec, int format) {
 683 #ifdef ASSERT
 684   if (rspec.reloc()->type() == relocInfo::oop_type &&
 685       d64 != 0 && d64 != (int64_t) Universe::non_oop_word()) {
 686     assert(Universe::heap()->is_in_reserved((address)d64), "should be real oop");
 687     assert(cast_to_oop(d64)->is_oop() && (ScavengeRootsInCode || !cast_to_oop(d64)->is_scavengable()),
 688            "cannot embed scavengable oops in code");
 689   }
 690 #endif
 691   cbuf.relocate(cbuf.insts_mark(), rspec, format);
 692   cbuf.insts()->emit_int64(d64);
 693 }
 694 
 695 // Access stack slot for load or store
 696 void store_to_stackslot(CodeBuffer &cbuf, int opcode, int rm_field, int disp)
 697 {
 698   emit_opcode(cbuf, opcode);                  // (e.g., FILD   [RSP+src])
 699   if (-0x80 <= disp && disp < 0x80) {
 700     emit_rm(cbuf, 0x01, rm_field, RSP_enc);   // R/M byte
 701     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 702     emit_d8(cbuf, disp);     // Displacement  // R/M byte
 703   } else {
 704     emit_rm(cbuf, 0x02, rm_field, RSP_enc);   // R/M byte
 705     emit_rm(cbuf, 0x00, RSP_enc, RSP_enc);    // SIB byte
 706     emit_d32(cbuf, disp);     // Displacement // R/M byte
 707   }
 708 }
 709 
 710    // rRegI ereg, memory mem) %{    // emit_reg_mem
 711 void encode_RegMem(CodeBuffer &cbuf,
 712                    int reg,
 713                    int base, int index, int scale, int disp, relocInfo::relocType disp_reloc)
 714 {
 715   assert(disp_reloc == relocInfo::none, "cannot have disp");
 716   int regenc = reg & 7;
 717   int baseenc = base & 7;
 718   int indexenc = index & 7;
 719 
 720   // There is no index & no scale, use form without SIB byte
 721   if (index == 0x4 && scale == 0 && base != RSP_enc && base != R12_enc) {
 722     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 723     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 724       emit_rm(cbuf, 0x0, regenc, baseenc); // *
 725     } else if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 726       // If 8-bit displacement, mode 0x1
 727       emit_rm(cbuf, 0x1, regenc, baseenc); // *
 728       emit_d8(cbuf, disp);
 729     } else {
 730       // If 32-bit displacement
 731       if (base == -1) { // Special flag for absolute address
 732         emit_rm(cbuf, 0x0, regenc, 0x5); // *
 733         if (disp_reloc != relocInfo::none) {
 734           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 735         } else {
 736           emit_d32(cbuf, disp);
 737         }
 738       } else {
 739         // Normal base + offset
 740         emit_rm(cbuf, 0x2, regenc, baseenc); // *
 741         if (disp_reloc != relocInfo::none) {
 742           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 743         } else {
 744           emit_d32(cbuf, disp);
 745         }
 746       }
 747     }
 748   } else {
 749     // Else, encode with the SIB byte
 750     // If no displacement, mode is 0x0; unless base is [RBP] or [R13]
 751     if (disp == 0 && base != RBP_enc && base != R13_enc) {
 752       // If no displacement
 753       emit_rm(cbuf, 0x0, regenc, 0x4); // *
 754       emit_rm(cbuf, scale, indexenc, baseenc);
 755     } else {
 756       if (-0x80 <= disp && disp < 0x80 && disp_reloc == relocInfo::none) {
 757         // If 8-bit displacement, mode 0x1
 758         emit_rm(cbuf, 0x1, regenc, 0x4); // *
 759         emit_rm(cbuf, scale, indexenc, baseenc);
 760         emit_d8(cbuf, disp);
 761       } else {
 762         // If 32-bit displacement
 763         if (base == 0x04 ) {
 764           emit_rm(cbuf, 0x2, regenc, 0x4);
 765           emit_rm(cbuf, scale, indexenc, 0x04); // XXX is this valid???
 766         } else {
 767           emit_rm(cbuf, 0x2, regenc, 0x4);
 768           emit_rm(cbuf, scale, indexenc, baseenc); // *
 769         }
 770         if (disp_reloc != relocInfo::none) {
 771           emit_d32_reloc(cbuf, disp, relocInfo::oop_type, RELOC_DISP32);
 772         } else {
 773           emit_d32(cbuf, disp);
 774         }
 775       }
 776     }
 777   }
 778 }
 779 
 780 // This could be in MacroAssembler but it's fairly C2 specific
 781 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 782   Label exit;
 783   __ jccb(Assembler::noParity, exit);
 784   __ pushf();
 785   //
 786   // comiss/ucomiss instructions set ZF,PF,CF flags and
 787   // zero OF,AF,SF for NaN values.
 788   // Fixup flags by zeroing ZF,PF so that compare of NaN
 789   // values returns 'less than' result (CF is set).
 790   // Leave the rest of flags unchanged.
 791   //
 792   //    7 6 5 4 3 2 1 0
 793   //   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
 794   //    0 0 1 0 1 0 1 1   (0x2B)
 795   //
 796   __ andq(Address(rsp, 0), 0xffffff2b);
 797   __ popf();
 798   __ bind(exit);
 799 }
 800 
 801 void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
 802   Label done;
 803   __ movl(dst, -1);
 804   __ jcc(Assembler::parity, done);
 805   __ jcc(Assembler::below, done);
 806   __ setb(Assembler::notEqual, dst);
 807   __ movzbl(dst, dst);
 808   __ bind(done);
 809 }
 810 
 811 
 812 //=============================================================================
 813 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 814 
 815 int Compile::ConstantTable::calculate_table_base_offset() const {
 816   return 0;  // absolute addressing, no offset
 817 }
 818 
 819 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
 820 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
 821   ShouldNotReachHere();
 822 }
 823 
 824 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
 825   // Empty encoding
 826 }
 827 
 828 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
 829   return 0;
 830 }
 831 
 832 #ifndef PRODUCT
 833 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 834   st->print("# MachConstantBaseNode (empty encoding)");
 835 }
 836 #endif
 837 
 838 
 839 //=============================================================================
 840 #ifndef PRODUCT
 841 void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 842   Compile* C = ra_->C;
 843 
 844   int framesize = C->frame_size_in_bytes();
 845   int bangsize = C->bang_size_in_bytes();
 846   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 847   // Remove wordSize for return addr which is already pushed.
 848   framesize -= wordSize;
 849 
 850   if (C->need_stack_bang(bangsize)) {
 851     framesize -= wordSize;
 852     st->print("# stack bang (%d bytes)", bangsize);
 853     st->print("\n\t");
 854     st->print("pushq   rbp\t# Save rbp");
 855     if (PreserveFramePointer) {
 856         st->print("\n\t");
 857         st->print("movq    rbp, rsp\t# Save the caller's SP into rbp");
 858     }
 859     if (framesize) {
 860       st->print("\n\t");
 861       st->print("subq    rsp, #%d\t# Create frame",framesize);
 862     }
 863   } else {
 864     st->print("subq    rsp, #%d\t# Create frame",framesize);
 865     st->print("\n\t");
 866     framesize -= wordSize;
 867     st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 868     if (PreserveFramePointer) {
 869       st->print("\n\t");
 870       st->print("movq    rbp, [rsp + #%d]\t# Save the caller's SP into rbp", (framesize + wordSize));
 871     }
 872   }
 873 
 874   if (VerifyStackAtCalls) {
 875     st->print("\n\t");
 876     framesize -= wordSize;
 877     st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
 878 #ifdef ASSERT
 879     st->print("\n\t");
 880     st->print("# stack alignment check");
 881 #endif
 882   }
 883   st->cr();
 884 }
 885 #endif
 886 
 887 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
 888   Compile* C = ra_->C;
 889   MacroAssembler _masm(&cbuf);
 890 
 891   int framesize = C->frame_size_in_bytes();
 892   int bangsize = C->bang_size_in_bytes();
 893 
 894   __ verified_entry(framesize, C->need_stack_bang(bangsize)?bangsize:0, false);
 895 
 896   C->set_frame_complete(cbuf.insts_size());
 897 
 898   if (C->has_mach_constant_base_node()) {
 899     // NOTE: We set the table base offset here because users might be
 900     // emitted before MachConstantBaseNode.
 901     Compile::ConstantTable& constant_table = C->constant_table();
 902     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
 903   }
 904 }
 905 
 906 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
 907 {
 908   return MachNode::size(ra_); // too many variables; just compute it
 909                               // the hard way
 910 }
 911 
 912 int MachPrologNode::reloc() const
 913 {
 914   return 0; // a large enough number
 915 }
 916 
 917 //=============================================================================
 918 #ifndef PRODUCT
 919 void MachEpilogNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 920 {
 921   Compile* C = ra_->C;
 922   if (C->max_vector_size() > 16) {
 923     st->print("vzeroupper");
 924     st->cr(); st->print("\t");
 925   }
 926 
 927   int framesize = C->frame_size_in_bytes();
 928   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 929   // Remove word for return adr already pushed
 930   // and RBP
 931   framesize -= 2*wordSize;
 932 
 933   if (framesize) {
 934     st->print_cr("addq    rsp, %d\t# Destroy frame", framesize);
 935     st->print("\t");
 936   }
 937 
 938   st->print_cr("popq   rbp");
 939   if (do_polling() && C->is_method_compilation()) {
 940     st->print("\t");
 941     if (Assembler::is_polling_page_far()) {
 942       st->print_cr("movq   rscratch1, #polling_page_address\n\t"
 943                    "testl  rax, [rscratch1]\t"
 944                    "# Safepoint: poll for GC");
 945     } else {
 946       st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
 947                    "# Safepoint: poll for GC");
 948     }
 949   }
 950 }
 951 #endif
 952 
 953 void MachEpilogNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 954 {
 955   Compile* C = ra_->C;
 956   if (C->max_vector_size() > 16) {
 957     // Clear upper bits of YMM registers when current compiled code uses
 958     // wide vectors to avoid AVX <-> SSE transition penalty during call.
 959     MacroAssembler _masm(&cbuf);
 960     __ vzeroupper();
 961   }
 962 
 963   int framesize = C->frame_size_in_bytes();
 964   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 965   // Remove word for return adr already pushed
 966   // and RBP
 967   framesize -= 2*wordSize;
 968 
 969   // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
 970 
 971   if (framesize) {
 972     emit_opcode(cbuf, Assembler::REX_W);
 973     if (framesize < 0x80) {
 974       emit_opcode(cbuf, 0x83); // addq rsp, #framesize
 975       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 976       emit_d8(cbuf, framesize);
 977     } else {
 978       emit_opcode(cbuf, 0x81); // addq rsp, #framesize
 979       emit_rm(cbuf, 0x3, 0x00, RSP_enc);
 980       emit_d32(cbuf, framesize);
 981     }
 982   }
 983 
 984   // popq rbp
 985   emit_opcode(cbuf, 0x58 | RBP_enc);
 986 
 987   if (do_polling() && C->is_method_compilation()) {
 988     MacroAssembler _masm(&cbuf);
 989     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 990     if (Assembler::is_polling_page_far()) {
 991       __ lea(rscratch1, polling_page);
 992       __ relocate(relocInfo::poll_return_type);
 993       __ testl(rax, Address(rscratch1, 0));
 994     } else {
 995       __ testl(rax, polling_page);
 996     }
 997   }
 998 }
 999 
1000 uint MachEpilogNode::size(PhaseRegAlloc* ra_) const
1001 {
1002   return MachNode::size(ra_); // too many variables; just compute it
1003                               // the hard way
1004 }
1005 
1006 int MachEpilogNode::reloc() const
1007 {
1008   return 2; // a large enough number
1009 }
1010 
1011 const Pipeline* MachEpilogNode::pipeline() const
1012 {
1013   return MachNode::pipeline_class();
1014 }
1015 
1016 int MachEpilogNode::safepoint_offset() const
1017 {
1018   return 0;
1019 }
1020 
1021 //=============================================================================
1022 
1023 enum RC {
1024   rc_bad,
1025   rc_int,
1026   rc_float,
1027   rc_stack
1028 };
1029 
1030 static enum RC rc_class(OptoReg::Name reg)
1031 {
1032   if( !OptoReg::is_valid(reg)  ) return rc_bad;
1033 
1034   if (OptoReg::is_stack(reg)) return rc_stack;
1035 
1036   VMReg r = OptoReg::as_VMReg(reg);
1037 
1038   if (r->is_Register()) return rc_int;
1039 
1040   assert(r->is_XMMRegister(), "must be");
1041   return rc_float;
1042 }
1043 
1044 // Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
1045 static int vec_mov_helper(CodeBuffer *cbuf, bool do_size, int src_lo, int dst_lo,
1046                           int src_hi, int dst_hi, uint ireg, outputStream* st);
1047 
1048 static int vec_spill_helper(CodeBuffer *cbuf, bool do_size, bool is_load,
1049                             int stack_offset, int reg, uint ireg, outputStream* st);
1050 
1051 static void vec_stack_to_stack_helper(CodeBuffer *cbuf, int src_offset,
1052                                       int dst_offset, uint ireg, outputStream* st) {
1053   if (cbuf) {
1054     MacroAssembler _masm(cbuf);
1055     switch (ireg) {
1056     case Op_VecS:
1057       __ movq(Address(rsp, -8), rax);
1058       __ movl(rax, Address(rsp, src_offset));
1059       __ movl(Address(rsp, dst_offset), rax);
1060       __ movq(rax, Address(rsp, -8));
1061       break;
1062     case Op_VecD:
1063       __ pushq(Address(rsp, src_offset));
1064       __ popq (Address(rsp, dst_offset));
1065       break;
1066     case Op_VecX:
1067       __ pushq(Address(rsp, src_offset));
1068       __ popq (Address(rsp, dst_offset));
1069       __ pushq(Address(rsp, src_offset+8));
1070       __ popq (Address(rsp, dst_offset+8));
1071       break;
1072     case Op_VecY:
1073       __ vmovdqu(Address(rsp, -32), xmm0);
1074       __ vmovdqu(xmm0, Address(rsp, src_offset));
1075       __ vmovdqu(Address(rsp, dst_offset), xmm0);
1076       __ vmovdqu(xmm0, Address(rsp, -32));
1077     case Op_VecZ:
1078       __ evmovdqul(Address(rsp, -64), xmm0, 2);
1079       __ evmovdqul(xmm0, Address(rsp, src_offset), 2);
1080       __ evmovdqul(Address(rsp, dst_offset), xmm0, 2);
1081       __ evmovdqul(xmm0, Address(rsp, -64), 2);
1082       break;
1083     default:
1084       ShouldNotReachHere();
1085     }
1086 #ifndef PRODUCT
1087   } else {
1088     switch (ireg) {
1089     case Op_VecS:
1090       st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1091                 "movl    rax, [rsp + #%d]\n\t"
1092                 "movl    [rsp + #%d], rax\n\t"
1093                 "movq    rax, [rsp - #8]",
1094                 src_offset, dst_offset);
1095       break;
1096     case Op_VecD:
1097       st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1098                 "popq    [rsp + #%d]",
1099                 src_offset, dst_offset);
1100       break;
1101      case Op_VecX:
1102       st->print("pushq   [rsp + #%d]\t# 128-bit mem-mem spill\n\t"
1103                 "popq    [rsp + #%d]\n\t"
1104                 "pushq   [rsp + #%d]\n\t"
1105                 "popq    [rsp + #%d]",
1106                 src_offset, dst_offset, src_offset+8, dst_offset+8);
1107       break;
1108     case Op_VecY:
1109       st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
1110                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1111                 "vmovdqu [rsp + #%d], xmm0\n\t"
1112                 "vmovdqu xmm0, [rsp - #32]",
1113                 src_offset, dst_offset);
1114       break;
1115     case Op_VecZ:
1116       st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
1117                 "vmovdqu xmm0, [rsp + #%d]\n\t"
1118                 "vmovdqu [rsp + #%d], xmm0\n\t"
1119                 "vmovdqu xmm0, [rsp - #64]",
1120                 src_offset, dst_offset);
1121       break;
1122     default:
1123       ShouldNotReachHere();
1124     }
1125 #endif
1126   }
1127 }
1128 
1129 uint MachSpillCopyNode::implementation(CodeBuffer* cbuf,
1130                                        PhaseRegAlloc* ra_,
1131                                        bool do_size,
1132                                        outputStream* st) const {
1133   assert(cbuf != NULL || st  != NULL, "sanity");
1134   // Get registers to move
1135   OptoReg::Name src_second = ra_->get_reg_second(in(1));
1136   OptoReg::Name src_first = ra_->get_reg_first(in(1));
1137   OptoReg::Name dst_second = ra_->get_reg_second(this);
1138   OptoReg::Name dst_first = ra_->get_reg_first(this);
1139 
1140   enum RC src_second_rc = rc_class(src_second);
1141   enum RC src_first_rc = rc_class(src_first);
1142   enum RC dst_second_rc = rc_class(dst_second);
1143   enum RC dst_first_rc = rc_class(dst_first);
1144 
1145   assert(OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first),
1146          "must move at least 1 register" );
1147 
1148   if (src_first == dst_first && src_second == dst_second) {
1149     // Self copy, no move
1150     return 0;
1151   }
1152   if (bottom_type()->isa_vect() != NULL) {
1153     uint ireg = ideal_reg();
1154     assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
1155     assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
1156     if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
1157       // mem -> mem
1158       int src_offset = ra_->reg2offset(src_first);
1159       int dst_offset = ra_->reg2offset(dst_first);
1160       vec_stack_to_stack_helper(cbuf, src_offset, dst_offset, ireg, st);
1161     } else if (src_first_rc == rc_float && dst_first_rc == rc_float ) {
1162       vec_mov_helper(cbuf, false, src_first, dst_first, src_second, dst_second, ireg, st);
1163     } else if (src_first_rc == rc_float && dst_first_rc == rc_stack ) {
1164       int stack_offset = ra_->reg2offset(dst_first);
1165       vec_spill_helper(cbuf, false, false, stack_offset, src_first, ireg, st);
1166     } else if (src_first_rc == rc_stack && dst_first_rc == rc_float ) {
1167       int stack_offset = ra_->reg2offset(src_first);
1168       vec_spill_helper(cbuf, false, true,  stack_offset, dst_first, ireg, st);
1169     } else {
1170       ShouldNotReachHere();
1171     }
1172     return 0;
1173   }
1174   if (src_first_rc == rc_stack) {
1175     // mem ->
1176     if (dst_first_rc == rc_stack) {
1177       // mem -> mem
1178       assert(src_second != dst_first, "overlap");
1179       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1180           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1181         // 64-bit
1182         int src_offset = ra_->reg2offset(src_first);
1183         int dst_offset = ra_->reg2offset(dst_first);
1184         if (cbuf) {
1185           MacroAssembler _masm(cbuf);
1186           __ pushq(Address(rsp, src_offset));
1187           __ popq (Address(rsp, dst_offset));
1188 #ifndef PRODUCT
1189         } else {
1190           st->print("pushq   [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
1191                     "popq    [rsp + #%d]",
1192                      src_offset, dst_offset);
1193 #endif
1194         }
1195       } else {
1196         // 32-bit
1197         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1198         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1199         // No pushl/popl, so:
1200         int src_offset = ra_->reg2offset(src_first);
1201         int dst_offset = ra_->reg2offset(dst_first);
1202         if (cbuf) {
1203           MacroAssembler _masm(cbuf);
1204           __ movq(Address(rsp, -8), rax);
1205           __ movl(rax, Address(rsp, src_offset));
1206           __ movl(Address(rsp, dst_offset), rax);
1207           __ movq(rax, Address(rsp, -8));
1208 #ifndef PRODUCT
1209         } else {
1210           st->print("movq    [rsp - #8], rax\t# 32-bit mem-mem spill\n\t"
1211                     "movl    rax, [rsp + #%d]\n\t"
1212                     "movl    [rsp + #%d], rax\n\t"
1213                     "movq    rax, [rsp - #8]",
1214                      src_offset, dst_offset);
1215 #endif
1216         }
1217       }
1218       return 0;
1219     } else if (dst_first_rc == rc_int) {
1220       // mem -> gpr
1221       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1222           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1223         // 64-bit
1224         int offset = ra_->reg2offset(src_first);
1225         if (cbuf) {
1226           MacroAssembler _masm(cbuf);
1227           __ movq(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1228 #ifndef PRODUCT
1229         } else {
1230           st->print("movq    %s, [rsp + #%d]\t# spill",
1231                      Matcher::regName[dst_first],
1232                      offset);
1233 #endif
1234         }
1235       } else {
1236         // 32-bit
1237         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1238         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1239         int offset = ra_->reg2offset(src_first);
1240         if (cbuf) {
1241           MacroAssembler _masm(cbuf);
1242           __ movl(as_Register(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1243 #ifndef PRODUCT
1244         } else {
1245           st->print("movl    %s, [rsp + #%d]\t# spill",
1246                      Matcher::regName[dst_first],
1247                      offset);
1248 #endif
1249         }
1250       }
1251       return 0;
1252     } else if (dst_first_rc == rc_float) {
1253       // mem-> xmm
1254       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1255           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1256         // 64-bit
1257         int offset = ra_->reg2offset(src_first);
1258         if (cbuf) {
1259           MacroAssembler _masm(cbuf);
1260           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1261 #ifndef PRODUCT
1262         } else {
1263           st->print("%s  %s, [rsp + #%d]\t# spill",
1264                      UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
1265                      Matcher::regName[dst_first],
1266                      offset);
1267 #endif
1268         }
1269       } else {
1270         // 32-bit
1271         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1272         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1273         int offset = ra_->reg2offset(src_first);
1274         if (cbuf) {
1275           MacroAssembler _masm(cbuf);
1276           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
1277 #ifndef PRODUCT
1278         } else {
1279           st->print("movss   %s, [rsp + #%d]\t# spill",
1280                      Matcher::regName[dst_first],
1281                      offset);
1282 #endif
1283         }
1284       }
1285       return 0;
1286     }
1287   } else if (src_first_rc == rc_int) {
1288     // gpr ->
1289     if (dst_first_rc == rc_stack) {
1290       // gpr -> mem
1291       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1292           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1293         // 64-bit
1294         int offset = ra_->reg2offset(dst_first);
1295         if (cbuf) {
1296           MacroAssembler _masm(cbuf);
1297           __ movq(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1298 #ifndef PRODUCT
1299         } else {
1300           st->print("movq    [rsp + #%d], %s\t# spill",
1301                      offset,
1302                      Matcher::regName[src_first]);
1303 #endif
1304         }
1305       } else {
1306         // 32-bit
1307         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1308         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1309         int offset = ra_->reg2offset(dst_first);
1310         if (cbuf) {
1311           MacroAssembler _masm(cbuf);
1312           __ movl(Address(rsp, offset), as_Register(Matcher::_regEncode[src_first]));
1313 #ifndef PRODUCT
1314         } else {
1315           st->print("movl    [rsp + #%d], %s\t# spill",
1316                      offset,
1317                      Matcher::regName[src_first]);
1318 #endif
1319         }
1320       }
1321       return 0;
1322     } else if (dst_first_rc == rc_int) {
1323       // gpr -> gpr
1324       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1325           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1326         // 64-bit
1327         if (cbuf) {
1328           MacroAssembler _masm(cbuf);
1329           __ movq(as_Register(Matcher::_regEncode[dst_first]),
1330                   as_Register(Matcher::_regEncode[src_first]));
1331 #ifndef PRODUCT
1332         } else {
1333           st->print("movq    %s, %s\t# spill",
1334                      Matcher::regName[dst_first],
1335                      Matcher::regName[src_first]);
1336 #endif
1337         }
1338         return 0;
1339       } else {
1340         // 32-bit
1341         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1342         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1343         if (cbuf) {
1344           MacroAssembler _masm(cbuf);
1345           __ movl(as_Register(Matcher::_regEncode[dst_first]),
1346                   as_Register(Matcher::_regEncode[src_first]));
1347 #ifndef PRODUCT
1348         } else {
1349           st->print("movl    %s, %s\t# spill",
1350                      Matcher::regName[dst_first],
1351                      Matcher::regName[src_first]);
1352 #endif
1353         }
1354         return 0;
1355       }
1356     } else if (dst_first_rc == rc_float) {
1357       // gpr -> xmm
1358       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1359           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1360         // 64-bit
1361         if (cbuf) {
1362           MacroAssembler _masm(cbuf);
1363           __ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1364 #ifndef PRODUCT
1365         } else {
1366           st->print("movdq   %s, %s\t# spill",
1367                      Matcher::regName[dst_first],
1368                      Matcher::regName[src_first]);
1369 #endif
1370         }
1371       } else {
1372         // 32-bit
1373         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1374         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1375         if (cbuf) {
1376           MacroAssembler _masm(cbuf);
1377           __ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
1378 #ifndef PRODUCT
1379         } else {
1380           st->print("movdl   %s, %s\t# spill",
1381                      Matcher::regName[dst_first],
1382                      Matcher::regName[src_first]);
1383 #endif
1384         }
1385       }
1386       return 0;
1387     }
1388   } else if (src_first_rc == rc_float) {
1389     // xmm ->
1390     if (dst_first_rc == rc_stack) {
1391       // xmm -> mem
1392       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1393           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1394         // 64-bit
1395         int offset = ra_->reg2offset(dst_first);
1396         if (cbuf) {
1397           MacroAssembler _masm(cbuf);
1398           __ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1399 #ifndef PRODUCT
1400         } else {
1401           st->print("movsd   [rsp + #%d], %s\t# spill",
1402                      offset,
1403                      Matcher::regName[src_first]);
1404 #endif
1405         }
1406       } else {
1407         // 32-bit
1408         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1409         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1410         int offset = ra_->reg2offset(dst_first);
1411         if (cbuf) {
1412           MacroAssembler _masm(cbuf);
1413           __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
1414 #ifndef PRODUCT
1415         } else {
1416           st->print("movss   [rsp + #%d], %s\t# spill",
1417                      offset,
1418                      Matcher::regName[src_first]);
1419 #endif
1420         }
1421       }
1422       return 0;
1423     } else if (dst_first_rc == rc_int) {
1424       // xmm -> gpr
1425       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1426           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1427         // 64-bit
1428         if (cbuf) {
1429           MacroAssembler _masm(cbuf);
1430           __ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1431 #ifndef PRODUCT
1432         } else {
1433           st->print("movdq   %s, %s\t# spill",
1434                      Matcher::regName[dst_first],
1435                      Matcher::regName[src_first]);
1436 #endif
1437         }
1438       } else {
1439         // 32-bit
1440         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1441         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1442         if (cbuf) {
1443           MacroAssembler _masm(cbuf);
1444           __ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1445 #ifndef PRODUCT
1446         } else {
1447           st->print("movdl   %s, %s\t# spill",
1448                      Matcher::regName[dst_first],
1449                      Matcher::regName[src_first]);
1450 #endif
1451         }
1452       }
1453       return 0;
1454     } else if (dst_first_rc == rc_float) {
1455       // xmm -> xmm
1456       if ((src_first & 1) == 0 && src_first + 1 == src_second &&
1457           (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
1458         // 64-bit
1459         if (cbuf) {
1460           MacroAssembler _masm(cbuf);
1461           __ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1462 #ifndef PRODUCT
1463         } else {
1464           st->print("%s  %s, %s\t# spill",
1465                      UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
1466                      Matcher::regName[dst_first],
1467                      Matcher::regName[src_first]);
1468 #endif
1469         }
1470       } else {
1471         // 32-bit
1472         assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
1473         assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
1474         if (cbuf) {
1475           MacroAssembler _masm(cbuf);
1476           __ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
1477 #ifndef PRODUCT
1478         } else {
1479           st->print("%s  %s, %s\t# spill",
1480                      UseXmmRegToRegMoveAll ? "movaps" : "movss ",
1481                      Matcher::regName[dst_first],
1482                      Matcher::regName[src_first]);
1483 #endif
1484         }
1485       }
1486       return 0;
1487     }
1488   }
1489 
1490   assert(0," foo ");
1491   Unimplemented();
1492   return 0;
1493 }
1494 
1495 #ifndef PRODUCT
1496 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
1497   implementation(NULL, ra_, false, st);
1498 }
1499 #endif
1500 
1501 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1502   implementation(&cbuf, ra_, false, NULL);
1503 }
1504 
1505 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1506   return MachNode::size(ra_);
1507 }
1508 
1509 //=============================================================================
1510 #ifndef PRODUCT
1511 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1512 {
1513   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1514   int reg = ra_->get_reg_first(this);
1515   st->print("leaq    %s, [rsp + #%d]\t# box lock",
1516             Matcher::regName[reg], offset);
1517 }
1518 #endif
1519 
1520 void BoxLockNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1521 {
1522   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1523   int reg = ra_->get_encode(this);
1524   if (offset >= 0x80) {
1525     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1526     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1527     emit_rm(cbuf, 0x2, reg & 7, 0x04);
1528     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1529     emit_d32(cbuf, offset);
1530   } else {
1531     emit_opcode(cbuf, reg < 8 ? Assembler::REX_W : Assembler::REX_WR);
1532     emit_opcode(cbuf, 0x8D); // LEA  reg,[SP+offset]
1533     emit_rm(cbuf, 0x1, reg & 7, 0x04);
1534     emit_rm(cbuf, 0x0, 0x04, RSP_enc);
1535     emit_d8(cbuf, offset);
1536   }
1537 }
1538 
1539 uint BoxLockNode::size(PhaseRegAlloc *ra_) const
1540 {
1541   int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1542   return (offset < 0x80) ? 5 : 8; // REX
1543 }
1544 
1545 //=============================================================================
1546 #ifndef PRODUCT
1547 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1548 {
1549   if (UseCompressedClassPointers) {
1550     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1551     st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1552     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
1553   } else {
1554     st->print_cr("\tcmpq    rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
1555                  "# Inline cache check");
1556   }
1557   st->print_cr("\tjne     SharedRuntime::_ic_miss_stub");
1558   st->print_cr("\tnop\t# nops to align entry point");
1559 }
1560 #endif
1561 
1562 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1563 {
1564   MacroAssembler masm(&cbuf);
1565   uint insts_size = cbuf.insts_size();
1566   if (UseCompressedClassPointers) {
1567     masm.load_klass(rscratch1, j_rarg0);
1568     masm.cmpptr(rax, rscratch1);
1569   } else {
1570     masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1571   }
1572 
1573   masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1574 
1575   /* WARNING these NOPs are critical so that verified entry point is properly
1576      4 bytes aligned for patching by NativeJump::patch_verified_entry() */
1577   int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
1578   if (OptoBreakpoint) {
1579     // Leave space for int3
1580     nops_cnt -= 1;
1581   }
1582   nops_cnt &= 0x3; // Do not add nops if code is aligned.
1583   if (nops_cnt > 0)
1584     masm.nop(nops_cnt);
1585 }
1586 
1587 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1588 {
1589   return MachNode::size(ra_); // too many variables; just compute it
1590                               // the hard way
1591 }
1592 
1593 
1594 //=============================================================================
1595 
1596 int Matcher::regnum_to_fpu_offset(int regnum)
1597 {
1598   return regnum - 32; // The FP registers are in the second chunk
1599 }
1600 
1601 // This is UltraSparc specific, true just means we have fast l2f conversion
1602 const bool Matcher::convL2FSupported(void) {
1603   return true;
1604 }
1605 
1606 // Is this branch offset short enough that a short branch can be used?
1607 //
1608 // NOTE: If the platform does not provide any short branch variants, then
1609 //       this method should return false for offset 0.
1610 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1611   // The passed offset is relative to address of the branch.
1612   // On 86 a branch displacement is calculated relative to address
1613   // of a next instruction.
1614   offset -= br_size;
1615 
1616   // the short version of jmpConUCF2 contains multiple branches,
1617   // making the reach slightly less
1618   if (rule == jmpConUCF2_rule)
1619     return (-126 <= offset && offset <= 125);
1620   return (-128 <= offset && offset <= 127);
1621 }
1622 
1623 const bool Matcher::isSimpleConstant64(jlong value) {
1624   // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1625   //return value == (int) value;  // Cf. storeImmL and immL32.
1626 
1627   // Probably always true, even if a temp register is required.
1628   return true;
1629 }
1630 
1631 // The ecx parameter to rep stosq for the ClearArray node is in words.
1632 const bool Matcher::init_array_count_is_in_bytes = false;
1633 
1634 // Threshold size for cleararray.
1635 const int Matcher::init_array_short_size = 8 * BytesPerLong;
1636 
1637 // No additional cost for CMOVL.
1638 const int Matcher::long_cmove_cost() { return 0; }
1639 
1640 // No CMOVF/CMOVD with SSE2
1641 const int Matcher::float_cmove_cost() { return ConditionalMoveLimit; }
1642 
1643 // Does the CPU require late expand (see block.cpp for description of late expand)?
1644 const bool Matcher::require_postalloc_expand = false;
1645 
1646 // Should the Matcher clone shifts on addressing modes, expecting them
1647 // to be subsumed into complex addressing expressions or compute them
1648 // into registers?  True for Intel but false for most RISCs
1649 const bool Matcher::clone_shift_expressions = true;
1650 
1651 // Do we need to mask the count passed to shift instructions or does
1652 // the cpu only look at the lower 5/6 bits anyway?
1653 const bool Matcher::need_masked_shift_count = false;
1654 
1655 bool Matcher::narrow_oop_use_complex_address() {
1656   assert(UseCompressedOops, "only for compressed oops code");
1657   return (LogMinObjAlignmentInBytes <= 3);
1658 }
1659 
1660 bool Matcher::narrow_klass_use_complex_address() {
1661   assert(UseCompressedClassPointers, "only for compressed klass code");
1662   return (LogKlassAlignmentInBytes <= 3);
1663 }
1664 
1665 // Is it better to copy float constants, or load them directly from
1666 // memory?  Intel can load a float constant from a direct address,
1667 // requiring no extra registers.  Most RISCs will have to materialize
1668 // an address into a register first, so they would do better to copy
1669 // the constant from stack.
1670 const bool Matcher::rematerialize_float_constants = true; // XXX
1671 
1672 // If CPU can load and store mis-aligned doubles directly then no
1673 // fixup is needed.  Else we split the double into 2 integer pieces
1674 // and move it piece-by-piece.  Only happens when passing doubles into
1675 // C code as the Java calling convention forces doubles to be aligned.
1676 const bool Matcher::misaligned_doubles_ok = true;
1677 
1678 // No-op on amd64
1679 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {}
1680 
1681 // Advertise here if the CPU requires explicit rounding operations to
1682 // implement the UseStrictFP mode.
1683 const bool Matcher::strict_fp_requires_explicit_rounding = true;
1684 
1685 // Are floats conerted to double when stored to stack during deoptimization?
1686 // On x64 it is stored without convertion so we can use normal access.
1687 bool Matcher::float_in_double() { return false; }
1688 
1689 // Do ints take an entire long register or just half?
1690 const bool Matcher::int_in_long = true;
1691 
1692 // Return whether or not this register is ever used as an argument.
1693 // This function is used on startup to build the trampoline stubs in
1694 // generateOptoStub.  Registers not mentioned will be killed by the VM
1695 // call in the trampoline, and arguments in those registers not be
1696 // available to the callee.
1697 bool Matcher::can_be_java_arg(int reg)
1698 {
1699   return
1700     reg ==  RDI_num || reg == RDI_H_num ||
1701     reg ==  RSI_num || reg == RSI_H_num ||
1702     reg ==  RDX_num || reg == RDX_H_num ||
1703     reg ==  RCX_num || reg == RCX_H_num ||
1704     reg ==   R8_num || reg ==  R8_H_num ||
1705     reg ==   R9_num || reg ==  R9_H_num ||
1706     reg ==  R12_num || reg == R12_H_num ||
1707     reg == XMM0_num || reg == XMM0b_num ||
1708     reg == XMM1_num || reg == XMM1b_num ||
1709     reg == XMM2_num || reg == XMM2b_num ||
1710     reg == XMM3_num || reg == XMM3b_num ||
1711     reg == XMM4_num || reg == XMM4b_num ||
1712     reg == XMM5_num || reg == XMM5b_num ||
1713     reg == XMM6_num || reg == XMM6b_num ||
1714     reg == XMM7_num || reg == XMM7b_num;
1715 }
1716 
1717 bool Matcher::is_spillable_arg(int reg)
1718 {
1719   return can_be_java_arg(reg);
1720 }
1721 
1722 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
1723   // In 64 bit mode a code which use multiply when
1724   // devisor is constant is faster than hardware
1725   // DIV instruction (it uses MulHiL).
1726   return false;
1727 }
1728 
1729 // Register for DIVI projection of divmodI
1730 RegMask Matcher::divI_proj_mask() {
1731   return INT_RAX_REG_mask();
1732 }
1733 
1734 // Register for MODI projection of divmodI
1735 RegMask Matcher::modI_proj_mask() {
1736   return INT_RDX_REG_mask();
1737 }
1738 
1739 // Register for DIVL projection of divmodL
1740 RegMask Matcher::divL_proj_mask() {
1741   return LONG_RAX_REG_mask();
1742 }
1743 
1744 // Register for MODL projection of divmodL
1745 RegMask Matcher::modL_proj_mask() {
1746   return LONG_RDX_REG_mask();
1747 }
1748 
1749 // Register for saving SP into on method handle invokes. Not used on x86_64.
1750 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
1751     return NO_REG_mask();
1752 }
1753 
1754 %}
1755 
1756 //----------ENCODING BLOCK-----------------------------------------------------
1757 // This block specifies the encoding classes used by the compiler to
1758 // output byte streams.  Encoding classes are parameterized macros
1759 // used by Machine Instruction Nodes in order to generate the bit
1760 // encoding of the instruction.  Operands specify their base encoding
1761 // interface with the interface keyword.  There are currently
1762 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
1763 // COND_INTER.  REG_INTER causes an operand to generate a function
1764 // which returns its register number when queried.  CONST_INTER causes
1765 // an operand to generate a function which returns the value of the
1766 // constant when queried.  MEMORY_INTER causes an operand to generate
1767 // four functions which return the Base Register, the Index Register,
1768 // the Scale Value, and the Offset Value of the operand when queried.
1769 // COND_INTER causes an operand to generate six functions which return
1770 // the encoding code (ie - encoding bits for the instruction)
1771 // associated with each basic boolean condition for a conditional
1772 // instruction.
1773 //
1774 // Instructions specify two basic values for encoding.  Again, a
1775 // function is available to check if the constant displacement is an
1776 // oop. They use the ins_encode keyword to specify their encoding
1777 // classes (which must be a sequence of enc_class names, and their
1778 // parameters, specified in the encoding block), and they use the
1779 // opcode keyword to specify, in order, their primary, secondary, and
1780 // tertiary opcode.  Only the opcode sections which a particular
1781 // instruction needs for encoding need to be specified.
1782 encode %{
1783   // Build emit functions for each basic byte or larger field in the
1784   // intel encoding scheme (opcode, rm, sib, immediate), and call them
1785   // from C++ code in the enc_class source block.  Emit functions will
1786   // live in the main source block for now.  In future, we can
1787   // generalize this by adding a syntax that specifies the sizes of
1788   // fields in an order, so that the adlc can build the emit functions
1789   // automagically
1790 
1791   // Emit primary opcode
1792   enc_class OpcP
1793   %{
1794     emit_opcode(cbuf, $primary);
1795   %}
1796 
1797   // Emit secondary opcode
1798   enc_class OpcS
1799   %{
1800     emit_opcode(cbuf, $secondary);
1801   %}
1802 
1803   // Emit tertiary opcode
1804   enc_class OpcT
1805   %{
1806     emit_opcode(cbuf, $tertiary);
1807   %}
1808 
1809   // Emit opcode directly
1810   enc_class Opcode(immI d8)
1811   %{
1812     emit_opcode(cbuf, $d8$$constant);
1813   %}
1814 
1815   // Emit size prefix
1816   enc_class SizePrefix
1817   %{
1818     emit_opcode(cbuf, 0x66);
1819   %}
1820 
1821   enc_class reg(rRegI reg)
1822   %{
1823     emit_rm(cbuf, 0x3, 0, $reg$$reg & 7);
1824   %}
1825 
1826   enc_class reg_reg(rRegI dst, rRegI src)
1827   %{
1828     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1829   %}
1830 
1831   enc_class opc_reg_reg(immI opcode, rRegI dst, rRegI src)
1832   %{
1833     emit_opcode(cbuf, $opcode$$constant);
1834     emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
1835   %}
1836 
1837   enc_class cdql_enc(no_rax_rdx_RegI div)
1838   %{
1839     // Full implementation of Java idiv and irem; checks for
1840     // special case as described in JVM spec., p.243 & p.271.
1841     //
1842     //         normal case                           special case
1843     //
1844     // input : rax: dividend                         min_int
1845     //         reg: divisor                          -1
1846     //
1847     // output: rax: quotient  (= rax idiv reg)       min_int
1848     //         rdx: remainder (= rax irem reg)       0
1849     //
1850     //  Code sequnce:
1851     //
1852     //    0:   3d 00 00 00 80          cmp    $0x80000000,%eax
1853     //    5:   75 07/08                jne    e <normal>
1854     //    7:   33 d2                   xor    %edx,%edx
1855     //  [div >= 8 -> offset + 1]
1856     //  [REX_B]
1857     //    9:   83 f9 ff                cmp    $0xffffffffffffffff,$div
1858     //    c:   74 03/04                je     11 <done>
1859     // 000000000000000e <normal>:
1860     //    e:   99                      cltd
1861     //  [div >= 8 -> offset + 1]
1862     //  [REX_B]
1863     //    f:   f7 f9                   idiv   $div
1864     // 0000000000000011 <done>:
1865 
1866     // cmp    $0x80000000,%eax
1867     emit_opcode(cbuf, 0x3d);
1868     emit_d8(cbuf, 0x00);
1869     emit_d8(cbuf, 0x00);
1870     emit_d8(cbuf, 0x00);
1871     emit_d8(cbuf, 0x80);
1872 
1873     // jne    e <normal>
1874     emit_opcode(cbuf, 0x75);
1875     emit_d8(cbuf, $div$$reg < 8 ? 0x07 : 0x08);
1876 
1877     // xor    %edx,%edx
1878     emit_opcode(cbuf, 0x33);
1879     emit_d8(cbuf, 0xD2);
1880 
1881     // cmp    $0xffffffffffffffff,%ecx
1882     if ($div$$reg >= 8) {
1883       emit_opcode(cbuf, Assembler::REX_B);
1884     }
1885     emit_opcode(cbuf, 0x83);
1886     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1887     emit_d8(cbuf, 0xFF);
1888 
1889     // je     11 <done>
1890     emit_opcode(cbuf, 0x74);
1891     emit_d8(cbuf, $div$$reg < 8 ? 0x03 : 0x04);
1892 
1893     // <normal>
1894     // cltd
1895     emit_opcode(cbuf, 0x99);
1896 
1897     // idivl (note: must be emitted by the user of this rule)
1898     // <done>
1899   %}
1900 
1901   enc_class cdqq_enc(no_rax_rdx_RegL div)
1902   %{
1903     // Full implementation of Java ldiv and lrem; checks for
1904     // special case as described in JVM spec., p.243 & p.271.
1905     //
1906     //         normal case                           special case
1907     //
1908     // input : rax: dividend                         min_long
1909     //         reg: divisor                          -1
1910     //
1911     // output: rax: quotient  (= rax idiv reg)       min_long
1912     //         rdx: remainder (= rax irem reg)       0
1913     //
1914     //  Code sequnce:
1915     //
1916     //    0:   48 ba 00 00 00 00 00    mov    $0x8000000000000000,%rdx
1917     //    7:   00 00 80
1918     //    a:   48 39 d0                cmp    %rdx,%rax
1919     //    d:   75 08                   jne    17 <normal>
1920     //    f:   33 d2                   xor    %edx,%edx
1921     //   11:   48 83 f9 ff             cmp    $0xffffffffffffffff,$div
1922     //   15:   74 05                   je     1c <done>
1923     // 0000000000000017 <normal>:
1924     //   17:   48 99                   cqto
1925     //   19:   48 f7 f9                idiv   $div
1926     // 000000000000001c <done>:
1927 
1928     // mov    $0x8000000000000000,%rdx
1929     emit_opcode(cbuf, Assembler::REX_W);
1930     emit_opcode(cbuf, 0xBA);
1931     emit_d8(cbuf, 0x00);
1932     emit_d8(cbuf, 0x00);
1933     emit_d8(cbuf, 0x00);
1934     emit_d8(cbuf, 0x00);
1935     emit_d8(cbuf, 0x00);
1936     emit_d8(cbuf, 0x00);
1937     emit_d8(cbuf, 0x00);
1938     emit_d8(cbuf, 0x80);
1939 
1940     // cmp    %rdx,%rax
1941     emit_opcode(cbuf, Assembler::REX_W);
1942     emit_opcode(cbuf, 0x39);
1943     emit_d8(cbuf, 0xD0);
1944 
1945     // jne    17 <normal>
1946     emit_opcode(cbuf, 0x75);
1947     emit_d8(cbuf, 0x08);
1948 
1949     // xor    %edx,%edx
1950     emit_opcode(cbuf, 0x33);
1951     emit_d8(cbuf, 0xD2);
1952 
1953     // cmp    $0xffffffffffffffff,$div
1954     emit_opcode(cbuf, $div$$reg < 8 ? Assembler::REX_W : Assembler::REX_WB);
1955     emit_opcode(cbuf, 0x83);
1956     emit_rm(cbuf, 0x3, 0x7, $div$$reg & 7);
1957     emit_d8(cbuf, 0xFF);
1958 
1959     // je     1e <done>
1960     emit_opcode(cbuf, 0x74);
1961     emit_d8(cbuf, 0x05);
1962 
1963     // <normal>
1964     // cqto
1965     emit_opcode(cbuf, Assembler::REX_W);
1966     emit_opcode(cbuf, 0x99);
1967 
1968     // idivq (note: must be emitted by the user of this rule)
1969     // <done>
1970   %}
1971 
1972   // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
1973   enc_class OpcSE(immI imm)
1974   %{
1975     // Emit primary opcode and set sign-extend bit
1976     // Check for 8-bit immediate, and set sign extend bit in opcode
1977     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1978       emit_opcode(cbuf, $primary | 0x02);
1979     } else {
1980       // 32-bit immediate
1981       emit_opcode(cbuf, $primary);
1982     }
1983   %}
1984 
1985   enc_class OpcSErm(rRegI dst, immI imm)
1986   %{
1987     // OpcSEr/m
1988     int dstenc = $dst$$reg;
1989     if (dstenc >= 8) {
1990       emit_opcode(cbuf, Assembler::REX_B);
1991       dstenc -= 8;
1992     }
1993     // Emit primary opcode and set sign-extend bit
1994     // Check for 8-bit immediate, and set sign extend bit in opcode
1995     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
1996       emit_opcode(cbuf, $primary | 0x02);
1997     } else {
1998       // 32-bit immediate
1999       emit_opcode(cbuf, $primary);
2000     }
2001     // Emit r/m byte with secondary opcode, after primary opcode.
2002     emit_rm(cbuf, 0x3, $secondary, dstenc);
2003   %}
2004 
2005   enc_class OpcSErm_wide(rRegL dst, immI imm)
2006   %{
2007     // OpcSEr/m
2008     int dstenc = $dst$$reg;
2009     if (dstenc < 8) {
2010       emit_opcode(cbuf, Assembler::REX_W);
2011     } else {
2012       emit_opcode(cbuf, Assembler::REX_WB);
2013       dstenc -= 8;
2014     }
2015     // Emit primary opcode and set sign-extend bit
2016     // Check for 8-bit immediate, and set sign extend bit in opcode
2017     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2018       emit_opcode(cbuf, $primary | 0x02);
2019     } else {
2020       // 32-bit immediate
2021       emit_opcode(cbuf, $primary);
2022     }
2023     // Emit r/m byte with secondary opcode, after primary opcode.
2024     emit_rm(cbuf, 0x3, $secondary, dstenc);
2025   %}
2026 
2027   enc_class Con8or32(immI imm)
2028   %{
2029     // Check for 8-bit immediate, and set sign extend bit in opcode
2030     if (-0x80 <= $imm$$constant && $imm$$constant < 0x80) {
2031       $$$emit8$imm$$constant;
2032     } else {
2033       // 32-bit immediate
2034       $$$emit32$imm$$constant;
2035     }
2036   %}
2037 
2038   enc_class opc2_reg(rRegI dst)
2039   %{
2040     // BSWAP
2041     emit_cc(cbuf, $secondary, $dst$$reg);
2042   %}
2043 
2044   enc_class opc3_reg(rRegI dst)
2045   %{
2046     // BSWAP
2047     emit_cc(cbuf, $tertiary, $dst$$reg);
2048   %}
2049 
2050   enc_class reg_opc(rRegI div)
2051   %{
2052     // INC, DEC, IDIV, IMOD, JMP indirect, ...
2053     emit_rm(cbuf, 0x3, $secondary, $div$$reg & 7);
2054   %}
2055 
2056   enc_class enc_cmov(cmpOp cop)
2057   %{
2058     // CMOV
2059     $$$emit8$primary;
2060     emit_cc(cbuf, $secondary, $cop$$cmpcode);
2061   %}
2062 
2063   enc_class enc_PartialSubtypeCheck()
2064   %{
2065     Register Rrdi = as_Register(RDI_enc); // result register
2066     Register Rrax = as_Register(RAX_enc); // super class
2067     Register Rrcx = as_Register(RCX_enc); // killed
2068     Register Rrsi = as_Register(RSI_enc); // sub class
2069     Label miss;
2070     const bool set_cond_codes = true;
2071 
2072     MacroAssembler _masm(&cbuf);
2073     __ check_klass_subtype_slow_path(Rrsi, Rrax, Rrcx, Rrdi,
2074                                      NULL, &miss,
2075                                      /*set_cond_codes:*/ true);
2076     if ($primary) {
2077       __ xorptr(Rrdi, Rrdi);
2078     }
2079     __ bind(miss);
2080   %}
2081 
2082   enc_class clear_avx %{
2083     debug_only(int off0 = cbuf.insts_size());
2084     if (ra_->C->max_vector_size() > 16) {
2085       // Clear upper bits of YMM registers when current compiled code uses
2086       // wide vectors to avoid AVX <-> SSE transition penalty during call.
2087       MacroAssembler _masm(&cbuf);
2088       __ vzeroupper();
2089     }
2090     debug_only(int off1 = cbuf.insts_size());
2091     assert(off1 - off0 == clear_avx_size(), "correct size prediction");
2092   %}
2093 
2094   enc_class Java_To_Runtime(method meth) %{
2095     // No relocation needed
2096     MacroAssembler _masm(&cbuf);
2097     __ mov64(r10, (int64_t) $meth$$method);
2098     __ call(r10);
2099   %}
2100 
2101   enc_class Java_To_Interpreter(method meth)
2102   %{
2103     // CALL Java_To_Interpreter
2104     // This is the instruction starting address for relocation info.
2105     cbuf.set_insts_mark();
2106     $$$emit8$primary;
2107     // CALL directly to the runtime
2108     emit_d32_reloc(cbuf,
2109                    (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2110                    runtime_call_Relocation::spec(),
2111                    RELOC_DISP32);
2112   %}
2113 
2114   enc_class Java_Static_Call(method meth)
2115   %{
2116     // JAVA STATIC CALL
2117     // CALL to fixup routine.  Fixup routine uses ScopeDesc info to
2118     // determine who we intended to call.
2119     cbuf.set_insts_mark();
2120     $$$emit8$primary;
2121 
2122     if (!_method) {
2123       emit_d32_reloc(cbuf,
2124                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2125                      runtime_call_Relocation::spec(),
2126                      RELOC_DISP32);
2127     } else if (_optimized_virtual) {
2128       emit_d32_reloc(cbuf,
2129                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2130                      opt_virtual_call_Relocation::spec(),
2131                      RELOC_DISP32);
2132     } else {
2133       emit_d32_reloc(cbuf,
2134                      (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
2135                      static_call_Relocation::spec(),
2136                      RELOC_DISP32);
2137     }
2138     if (_method) {
2139       // Emit stub for static call.
2140       address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
2141       if (stub == NULL) {
2142         ciEnv::current()->record_failure("CodeCache is full");
2143         return;
2144       } 
2145     }
2146   %}
2147 
2148   enc_class Java_Dynamic_Call(method meth) %{
2149     MacroAssembler _masm(&cbuf);
2150     __ ic_call((address)$meth$$method);
2151   %}
2152 
2153   enc_class Java_Compiled_Call(method meth)
2154   %{
2155     // JAVA COMPILED CALL
2156     int disp = in_bytes(Method:: from_compiled_offset());
2157 
2158     // XXX XXX offset is 128 is 1.5 NON-PRODUCT !!!
2159     // assert(-0x80 <= disp && disp < 0x80, "compiled_code_offset isn't small");
2160 
2161     // callq *disp(%rax)
2162     cbuf.set_insts_mark();
2163     $$$emit8$primary;
2164     if (disp < 0x80) {
2165       emit_rm(cbuf, 0x01, $secondary, RAX_enc); // R/M byte
2166       emit_d8(cbuf, disp); // Displacement
2167     } else {
2168       emit_rm(cbuf, 0x02, $secondary, RAX_enc); // R/M byte
2169       emit_d32(cbuf, disp); // Displacement
2170     }
2171   %}
2172 
2173   enc_class reg_opc_imm(rRegI dst, immI8 shift)
2174   %{
2175     // SAL, SAR, SHR
2176     int dstenc = $dst$$reg;
2177     if (dstenc >= 8) {
2178       emit_opcode(cbuf, Assembler::REX_B);
2179       dstenc -= 8;
2180     }
2181     $$$emit8$primary;
2182     emit_rm(cbuf, 0x3, $secondary, dstenc);
2183     $$$emit8$shift$$constant;
2184   %}
2185 
2186   enc_class reg_opc_imm_wide(rRegL dst, immI8 shift)
2187   %{
2188     // SAL, SAR, SHR
2189     int dstenc = $dst$$reg;
2190     if (dstenc < 8) {
2191       emit_opcode(cbuf, Assembler::REX_W);
2192     } else {
2193       emit_opcode(cbuf, Assembler::REX_WB);
2194       dstenc -= 8;
2195     }
2196     $$$emit8$primary;
2197     emit_rm(cbuf, 0x3, $secondary, dstenc);
2198     $$$emit8$shift$$constant;
2199   %}
2200 
2201   enc_class load_immI(rRegI dst, immI src)
2202   %{
2203     int dstenc = $dst$$reg;
2204     if (dstenc >= 8) {
2205       emit_opcode(cbuf, Assembler::REX_B);
2206       dstenc -= 8;
2207     }
2208     emit_opcode(cbuf, 0xB8 | dstenc);
2209     $$$emit32$src$$constant;
2210   %}
2211 
2212   enc_class load_immL(rRegL dst, immL src)
2213   %{
2214     int dstenc = $dst$$reg;
2215     if (dstenc < 8) {
2216       emit_opcode(cbuf, Assembler::REX_W);
2217     } else {
2218       emit_opcode(cbuf, Assembler::REX_WB);
2219       dstenc -= 8;
2220     }
2221     emit_opcode(cbuf, 0xB8 | dstenc);
2222     emit_d64(cbuf, $src$$constant);
2223   %}
2224 
2225   enc_class load_immUL32(rRegL dst, immUL32 src)
2226   %{
2227     // same as load_immI, but this time we care about zeroes in the high word
2228     int dstenc = $dst$$reg;
2229     if (dstenc >= 8) {
2230       emit_opcode(cbuf, Assembler::REX_B);
2231       dstenc -= 8;
2232     }
2233     emit_opcode(cbuf, 0xB8 | dstenc);
2234     $$$emit32$src$$constant;
2235   %}
2236 
2237   enc_class load_immL32(rRegL dst, immL32 src)
2238   %{
2239     int dstenc = $dst$$reg;
2240     if (dstenc < 8) {
2241       emit_opcode(cbuf, Assembler::REX_W);
2242     } else {
2243       emit_opcode(cbuf, Assembler::REX_WB);
2244       dstenc -= 8;
2245     }
2246     emit_opcode(cbuf, 0xC7);
2247     emit_rm(cbuf, 0x03, 0x00, dstenc);
2248     $$$emit32$src$$constant;
2249   %}
2250 
2251   enc_class load_immP31(rRegP dst, immP32 src)
2252   %{
2253     // same as load_immI, but this time we care about zeroes in the high word
2254     int dstenc = $dst$$reg;
2255     if (dstenc >= 8) {
2256       emit_opcode(cbuf, Assembler::REX_B);
2257       dstenc -= 8;
2258     }
2259     emit_opcode(cbuf, 0xB8 | dstenc);
2260     $$$emit32$src$$constant;
2261   %}
2262 
2263   enc_class load_immP(rRegP dst, immP src)
2264   %{
2265     int dstenc = $dst$$reg;
2266     if (dstenc < 8) {
2267       emit_opcode(cbuf, Assembler::REX_W);
2268     } else {
2269       emit_opcode(cbuf, Assembler::REX_WB);
2270       dstenc -= 8;
2271     }
2272     emit_opcode(cbuf, 0xB8 | dstenc);
2273     // This next line should be generated from ADLC
2274     if ($src->constant_reloc() != relocInfo::none) {
2275       emit_d64_reloc(cbuf, $src$$constant, $src->constant_reloc(), RELOC_IMM64);
2276     } else {
2277       emit_d64(cbuf, $src$$constant);
2278     }
2279   %}
2280 
2281   enc_class Con32(immI src)
2282   %{
2283     // Output immediate
2284     $$$emit32$src$$constant;
2285   %}
2286 
2287   enc_class Con32F_as_bits(immF src)
2288   %{
2289     // Output Float immediate bits
2290     jfloat jf = $src$$constant;
2291     jint jf_as_bits = jint_cast(jf);
2292     emit_d32(cbuf, jf_as_bits);
2293   %}
2294 
2295   enc_class Con16(immI src)
2296   %{
2297     // Output immediate
2298     $$$emit16$src$$constant;
2299   %}
2300 
2301   // How is this different from Con32??? XXX
2302   enc_class Con_d32(immI src)
2303   %{
2304     emit_d32(cbuf,$src$$constant);
2305   %}
2306 
2307   enc_class conmemref (rRegP t1) %{    // Con32(storeImmI)
2308     // Output immediate memory reference
2309     emit_rm(cbuf, 0x00, $t1$$reg, 0x05 );
2310     emit_d32(cbuf, 0x00);
2311   %}
2312 
2313   enc_class lock_prefix()
2314   %{
2315     if (os::is_MP()) {
2316       emit_opcode(cbuf, 0xF0); // lock
2317     }
2318   %}
2319 
2320   enc_class REX_mem(memory mem)
2321   %{
2322     if ($mem$$base >= 8) {
2323       if ($mem$$index < 8) {
2324         emit_opcode(cbuf, Assembler::REX_B);
2325       } else {
2326         emit_opcode(cbuf, Assembler::REX_XB);
2327       }
2328     } else {
2329       if ($mem$$index >= 8) {
2330         emit_opcode(cbuf, Assembler::REX_X);
2331       }
2332     }
2333   %}
2334 
2335   enc_class REX_mem_wide(memory mem)
2336   %{
2337     if ($mem$$base >= 8) {
2338       if ($mem$$index < 8) {
2339         emit_opcode(cbuf, Assembler::REX_WB);
2340       } else {
2341         emit_opcode(cbuf, Assembler::REX_WXB);
2342       }
2343     } else {
2344       if ($mem$$index < 8) {
2345         emit_opcode(cbuf, Assembler::REX_W);
2346       } else {
2347         emit_opcode(cbuf, Assembler::REX_WX);
2348       }
2349     }
2350   %}
2351 
2352   // for byte regs
2353   enc_class REX_breg(rRegI reg)
2354   %{
2355     if ($reg$$reg >= 4) {
2356       emit_opcode(cbuf, $reg$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2357     }
2358   %}
2359 
2360   // for byte regs
2361   enc_class REX_reg_breg(rRegI dst, rRegI src)
2362   %{
2363     if ($dst$$reg < 8) {
2364       if ($src$$reg >= 4) {
2365         emit_opcode(cbuf, $src$$reg < 8 ? Assembler::REX : Assembler::REX_B);
2366       }
2367     } else {
2368       if ($src$$reg < 8) {
2369         emit_opcode(cbuf, Assembler::REX_R);
2370       } else {
2371         emit_opcode(cbuf, Assembler::REX_RB);
2372       }
2373     }
2374   %}
2375 
2376   // for byte regs
2377   enc_class REX_breg_mem(rRegI reg, memory mem)
2378   %{
2379     if ($reg$$reg < 8) {
2380       if ($mem$$base < 8) {
2381         if ($mem$$index >= 8) {
2382           emit_opcode(cbuf, Assembler::REX_X);
2383         } else if ($reg$$reg >= 4) {
2384           emit_opcode(cbuf, Assembler::REX);
2385         }
2386       } else {
2387         if ($mem$$index < 8) {
2388           emit_opcode(cbuf, Assembler::REX_B);
2389         } else {
2390           emit_opcode(cbuf, Assembler::REX_XB);
2391         }
2392       }
2393     } else {
2394       if ($mem$$base < 8) {
2395         if ($mem$$index < 8) {
2396           emit_opcode(cbuf, Assembler::REX_R);
2397         } else {
2398           emit_opcode(cbuf, Assembler::REX_RX);
2399         }
2400       } else {
2401         if ($mem$$index < 8) {
2402           emit_opcode(cbuf, Assembler::REX_RB);
2403         } else {
2404           emit_opcode(cbuf, Assembler::REX_RXB);
2405         }
2406       }
2407     }
2408   %}
2409 
2410   enc_class REX_reg(rRegI reg)
2411   %{
2412     if ($reg$$reg >= 8) {
2413       emit_opcode(cbuf, Assembler::REX_B);
2414     }
2415   %}
2416 
2417   enc_class REX_reg_wide(rRegI reg)
2418   %{
2419     if ($reg$$reg < 8) {
2420       emit_opcode(cbuf, Assembler::REX_W);
2421     } else {
2422       emit_opcode(cbuf, Assembler::REX_WB);
2423     }
2424   %}
2425 
2426   enc_class REX_reg_reg(rRegI dst, rRegI src)
2427   %{
2428     if ($dst$$reg < 8) {
2429       if ($src$$reg >= 8) {
2430         emit_opcode(cbuf, Assembler::REX_B);
2431       }
2432     } else {
2433       if ($src$$reg < 8) {
2434         emit_opcode(cbuf, Assembler::REX_R);
2435       } else {
2436         emit_opcode(cbuf, Assembler::REX_RB);
2437       }
2438     }
2439   %}
2440 
2441   enc_class REX_reg_reg_wide(rRegI dst, rRegI src)
2442   %{
2443     if ($dst$$reg < 8) {
2444       if ($src$$reg < 8) {
2445         emit_opcode(cbuf, Assembler::REX_W);
2446       } else {
2447         emit_opcode(cbuf, Assembler::REX_WB);
2448       }
2449     } else {
2450       if ($src$$reg < 8) {
2451         emit_opcode(cbuf, Assembler::REX_WR);
2452       } else {
2453         emit_opcode(cbuf, Assembler::REX_WRB);
2454       }
2455     }
2456   %}
2457 
2458   enc_class REX_reg_mem(rRegI reg, memory mem)
2459   %{
2460     if ($reg$$reg < 8) {
2461       if ($mem$$base < 8) {
2462         if ($mem$$index >= 8) {
2463           emit_opcode(cbuf, Assembler::REX_X);
2464         }
2465       } else {
2466         if ($mem$$index < 8) {
2467           emit_opcode(cbuf, Assembler::REX_B);
2468         } else {
2469           emit_opcode(cbuf, Assembler::REX_XB);
2470         }
2471       }
2472     } else {
2473       if ($mem$$base < 8) {
2474         if ($mem$$index < 8) {
2475           emit_opcode(cbuf, Assembler::REX_R);
2476         } else {
2477           emit_opcode(cbuf, Assembler::REX_RX);
2478         }
2479       } else {
2480         if ($mem$$index < 8) {
2481           emit_opcode(cbuf, Assembler::REX_RB);
2482         } else {
2483           emit_opcode(cbuf, Assembler::REX_RXB);
2484         }
2485       }
2486     }
2487   %}
2488 
2489   enc_class REX_reg_mem_wide(rRegL reg, memory mem)
2490   %{
2491     if ($reg$$reg < 8) {
2492       if ($mem$$base < 8) {
2493         if ($mem$$index < 8) {
2494           emit_opcode(cbuf, Assembler::REX_W);
2495         } else {
2496           emit_opcode(cbuf, Assembler::REX_WX);
2497         }
2498       } else {
2499         if ($mem$$index < 8) {
2500           emit_opcode(cbuf, Assembler::REX_WB);
2501         } else {
2502           emit_opcode(cbuf, Assembler::REX_WXB);
2503         }
2504       }
2505     } else {
2506       if ($mem$$base < 8) {
2507         if ($mem$$index < 8) {
2508           emit_opcode(cbuf, Assembler::REX_WR);
2509         } else {
2510           emit_opcode(cbuf, Assembler::REX_WRX);
2511         }
2512       } else {
2513         if ($mem$$index < 8) {
2514           emit_opcode(cbuf, Assembler::REX_WRB);
2515         } else {
2516           emit_opcode(cbuf, Assembler::REX_WRXB);
2517         }
2518       }
2519     }
2520   %}
2521 
2522   enc_class reg_mem(rRegI ereg, memory mem)
2523   %{
2524     // High registers handle in encode_RegMem
2525     int reg = $ereg$$reg;
2526     int base = $mem$$base;
2527     int index = $mem$$index;
2528     int scale = $mem$$scale;
2529     int disp = $mem$$disp;
2530     relocInfo::relocType disp_reloc = $mem->disp_reloc();
2531 
2532     encode_RegMem(cbuf, reg, base, index, scale, disp, disp_reloc);
2533   %}
2534 
2535   enc_class RM_opc_mem(immI rm_opcode, memory mem)
2536   %{
2537     int rm_byte_opcode = $rm_opcode$$constant;
2538 
2539     // High registers handle in encode_RegMem
2540     int base = $mem$$base;
2541     int index = $mem$$index;
2542     int scale = $mem$$scale;
2543     int displace = $mem$$disp;
2544 
2545     relocInfo::relocType disp_reloc = $mem->disp_reloc();       // disp-as-oop when
2546                                             // working with static
2547                                             // globals
2548     encode_RegMem(cbuf, rm_byte_opcode, base, index, scale, displace,
2549                   disp_reloc);
2550   %}
2551 
2552   enc_class reg_lea(rRegI dst, rRegI src0, immI src1)
2553   %{
2554     int reg_encoding = $dst$$reg;
2555     int base         = $src0$$reg;      // 0xFFFFFFFF indicates no base
2556     int index        = 0x04;            // 0x04 indicates no index
2557     int scale        = 0x00;            // 0x00 indicates no scale
2558     int displace     = $src1$$constant; // 0x00 indicates no displacement
2559     relocInfo::relocType disp_reloc = relocInfo::none;
2560     encode_RegMem(cbuf, reg_encoding, base, index, scale, displace,
2561                   disp_reloc);
2562   %}
2563 
2564   enc_class neg_reg(rRegI dst)
2565   %{
2566     int dstenc = $dst$$reg;
2567     if (dstenc >= 8) {
2568       emit_opcode(cbuf, Assembler::REX_B);
2569       dstenc -= 8;
2570     }
2571     // NEG $dst
2572     emit_opcode(cbuf, 0xF7);
2573     emit_rm(cbuf, 0x3, 0x03, dstenc);
2574   %}
2575 
2576   enc_class neg_reg_wide(rRegI dst)
2577   %{
2578     int dstenc = $dst$$reg;
2579     if (dstenc < 8) {
2580       emit_opcode(cbuf, Assembler::REX_W);
2581     } else {
2582       emit_opcode(cbuf, Assembler::REX_WB);
2583       dstenc -= 8;
2584     }
2585     // NEG $dst
2586     emit_opcode(cbuf, 0xF7);
2587     emit_rm(cbuf, 0x3, 0x03, dstenc);
2588   %}
2589 
2590   enc_class setLT_reg(rRegI dst)
2591   %{
2592     int dstenc = $dst$$reg;
2593     if (dstenc >= 8) {
2594       emit_opcode(cbuf, Assembler::REX_B);
2595       dstenc -= 8;
2596     } else if (dstenc >= 4) {
2597       emit_opcode(cbuf, Assembler::REX);
2598     }
2599     // SETLT $dst
2600     emit_opcode(cbuf, 0x0F);
2601     emit_opcode(cbuf, 0x9C);
2602     emit_rm(cbuf, 0x3, 0x0, dstenc);
2603   %}
2604 
2605   enc_class setNZ_reg(rRegI dst)
2606   %{
2607     int dstenc = $dst$$reg;
2608     if (dstenc >= 8) {
2609       emit_opcode(cbuf, Assembler::REX_B);
2610       dstenc -= 8;
2611     } else if (dstenc >= 4) {
2612       emit_opcode(cbuf, Assembler::REX);
2613     }
2614     // SETNZ $dst
2615     emit_opcode(cbuf, 0x0F);
2616     emit_opcode(cbuf, 0x95);
2617     emit_rm(cbuf, 0x3, 0x0, dstenc);
2618   %}
2619 
2620 
2621   // Compare the lonogs and set -1, 0, or 1 into dst
2622   enc_class cmpl3_flag(rRegL src1, rRegL src2, rRegI dst)
2623   %{
2624     int src1enc = $src1$$reg;
2625     int src2enc = $src2$$reg;
2626     int dstenc = $dst$$reg;
2627 
2628     // cmpq $src1, $src2
2629     if (src1enc < 8) {
2630       if (src2enc < 8) {
2631         emit_opcode(cbuf, Assembler::REX_W);
2632       } else {
2633         emit_opcode(cbuf, Assembler::REX_WB);
2634       }
2635     } else {
2636       if (src2enc < 8) {
2637         emit_opcode(cbuf, Assembler::REX_WR);
2638       } else {
2639         emit_opcode(cbuf, Assembler::REX_WRB);
2640       }
2641     }
2642     emit_opcode(cbuf, 0x3B);
2643     emit_rm(cbuf, 0x3, src1enc & 7, src2enc & 7);
2644 
2645     // movl $dst, -1
2646     if (dstenc >= 8) {
2647       emit_opcode(cbuf, Assembler::REX_B);
2648     }
2649     emit_opcode(cbuf, 0xB8 | (dstenc & 7));
2650     emit_d32(cbuf, -1);
2651 
2652     // jl,s done
2653     emit_opcode(cbuf, 0x7C);
2654     emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
2655 
2656     // setne $dst
2657     if (dstenc >= 4) {
2658       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
2659     }
2660     emit_opcode(cbuf, 0x0F);
2661     emit_opcode(cbuf, 0x95);
2662     emit_opcode(cbuf, 0xC0 | (dstenc & 7));
2663 
2664     // movzbl $dst, $dst
2665     if (dstenc >= 4) {
2666       emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
2667     }
2668     emit_opcode(cbuf, 0x0F);
2669     emit_opcode(cbuf, 0xB6);
2670     emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
2671   %}
2672 
2673   enc_class Push_ResultXD(regD dst) %{
2674     MacroAssembler _masm(&cbuf);
2675     __ fstp_d(Address(rsp, 0));
2676     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
2677     __ addptr(rsp, 8);
2678   %}
2679 
2680   enc_class Push_SrcXD(regD src) %{
2681     MacroAssembler _masm(&cbuf);
2682     __ subptr(rsp, 8);
2683     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2684     __ fld_d(Address(rsp, 0));
2685   %}
2686 
2687 
2688   enc_class enc_rethrow()
2689   %{
2690     cbuf.set_insts_mark();
2691     emit_opcode(cbuf, 0xE9); // jmp entry
2692     emit_d32_reloc(cbuf,
2693                    (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
2694                    runtime_call_Relocation::spec(),
2695                    RELOC_DISP32);
2696   %}
2697 
2698 %}
2699 
2700 
2701 
2702 //----------FRAME--------------------------------------------------------------
2703 // Definition of frame structure and management information.
2704 //
2705 //  S T A C K   L A Y O U T    Allocators stack-slot number
2706 //                             |   (to get allocators register number
2707 //  G  Owned by    |        |  v    add OptoReg::stack0())
2708 //  r   CALLER     |        |
2709 //  o     |        +--------+      pad to even-align allocators stack-slot
2710 //  w     V        |  pad0  |        numbers; owned by CALLER
2711 //  t   -----------+--------+----> Matcher::_in_arg_limit, unaligned
2712 //  h     ^        |   in   |  5
2713 //        |        |  args  |  4   Holes in incoming args owned by SELF
2714 //  |     |        |        |  3
2715 //  |     |        +--------+
2716 //  V     |        | old out|      Empty on Intel, window on Sparc
2717 //        |    old |preserve|      Must be even aligned.
2718 //        |     SP-+--------+----> Matcher::_old_SP, even aligned
2719 //        |        |   in   |  3   area for Intel ret address
2720 //     Owned by    |preserve|      Empty on Sparc.
2721 //       SELF      +--------+
2722 //        |        |  pad2  |  2   pad to align old SP
2723 //        |        +--------+  1
2724 //        |        | locks  |  0
2725 //        |        +--------+----> OptoReg::stack0(), even aligned
2726 //        |        |  pad1  | 11   pad to align new SP
2727 //        |        +--------+
2728 //        |        |        | 10
2729 //        |        | spills |  9   spills
2730 //        V        |        |  8   (pad0 slot for callee)
2731 //      -----------+--------+----> Matcher::_out_arg_limit, unaligned
2732 //        ^        |  out   |  7
2733 //        |        |  args  |  6   Holes in outgoing args owned by CALLEE
2734 //     Owned by    +--------+
2735 //      CALLEE     | new out|  6   Empty on Intel, window on Sparc
2736 //        |    new |preserve|      Must be even-aligned.
2737 //        |     SP-+--------+----> Matcher::_new_SP, even aligned
2738 //        |        |        |
2739 //
2740 // Note 1: Only region 8-11 is determined by the allocator.  Region 0-5 is
2741 //         known from SELF's arguments and the Java calling convention.
2742 //         Region 6-7 is determined per call site.
2743 // Note 2: If the calling convention leaves holes in the incoming argument
2744 //         area, those holes are owned by SELF.  Holes in the outgoing area
2745 //         are owned by the CALLEE.  Holes should not be nessecary in the
2746 //         incoming area, as the Java calling convention is completely under
2747 //         the control of the AD file.  Doubles can be sorted and packed to
2748 //         avoid holes.  Holes in the outgoing arguments may be nessecary for
2749 //         varargs C calling conventions.
2750 // Note 3: Region 0-3 is even aligned, with pad2 as needed.  Region 3-5 is
2751 //         even aligned with pad0 as needed.
2752 //         Region 6 is even aligned.  Region 6-7 is NOT even aligned;
2753 //         region 6-11 is even aligned; it may be padded out more so that
2754 //         the region from SP to FP meets the minimum stack alignment.
2755 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
2756 //         alignment.  Region 11, pad1, may be dynamically extended so that
2757 //         SP meets the minimum alignment.
2758 
2759 frame
2760 %{
2761   // What direction does stack grow in (assumed to be same for C & Java)
2762   stack_direction(TOWARDS_LOW);
2763 
2764   // These three registers define part of the calling convention
2765   // between compiled code and the interpreter.
2766   inline_cache_reg(RAX);                // Inline Cache Register
2767   interpreter_method_oop_reg(RBX);      // Method Oop Register when
2768                                         // calling interpreter
2769 
2770   // Optional: name the operand used by cisc-spilling to access
2771   // [stack_pointer + offset]
2772   cisc_spilling_operand_name(indOffset32);
2773 
2774   // Number of stack slots consumed by locking an object
2775   sync_stack_slots(2);
2776 
2777   // Compiled code's Frame Pointer
2778   frame_pointer(RSP);
2779 
2780   // Interpreter stores its frame pointer in a register which is
2781   // stored to the stack by I2CAdaptors.
2782   // I2CAdaptors convert from interpreted java to compiled java.
2783   interpreter_frame_pointer(RBP);
2784 
2785   // Stack alignment requirement
2786   stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
2787 
2788   // Number of stack slots between incoming argument block and the start of
2789   // a new frame.  The PROLOG must add this many slots to the stack.  The
2790   // EPILOG must remove this many slots.  amd64 needs two slots for
2791   // return address.
2792   in_preserve_stack_slots(4 + 2 * VerifyStackAtCalls);
2793 
2794   // Number of outgoing stack slots killed above the out_preserve_stack_slots
2795   // for calls to C.  Supports the var-args backing area for register parms.
2796   varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
2797 
2798   // The after-PROLOG location of the return address.  Location of
2799   // return address specifies a type (REG or STACK) and a number
2800   // representing the register number (i.e. - use a register name) or
2801   // stack slot.
2802   // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
2803   // Otherwise, it is above the locks and verification slot and alignment word
2804   return_addr(STACK - 2 +
2805               round_to((Compile::current()->in_preserve_stack_slots() +
2806                         Compile::current()->fixed_slots()),
2807                        stack_alignment_in_slots()));
2808 
2809   // Body of function which returns an integer array locating
2810   // arguments either in registers or in stack slots.  Passed an array
2811   // of ideal registers called "sig" and a "length" count.  Stack-slot
2812   // offsets are based on outgoing arguments, i.e. a CALLER setting up
2813   // arguments for a CALLEE.  Incoming stack arguments are
2814   // automatically biased by the preserve_stack_slots field above.
2815 
2816   calling_convention
2817   %{
2818     // No difference between ingoing/outgoing just pass false
2819     SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
2820   %}
2821 
2822   c_calling_convention
2823   %{
2824     // This is obviously always outgoing
2825     (void) SharedRuntime::c_calling_convention(sig_bt, regs, /*regs2=*/NULL, length);
2826   %}
2827 
2828   // Location of compiled Java return values.  Same as C for now.
2829   return_value
2830   %{
2831     assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
2832            "only return normal values");
2833 
2834     static const int lo[Op_RegL + 1] = {
2835       0,
2836       0,
2837       RAX_num,  // Op_RegN
2838       RAX_num,  // Op_RegI
2839       RAX_num,  // Op_RegP
2840       XMM0_num, // Op_RegF
2841       XMM0_num, // Op_RegD
2842       RAX_num   // Op_RegL
2843     };
2844     static const int hi[Op_RegL + 1] = {
2845       0,
2846       0,
2847       OptoReg::Bad, // Op_RegN
2848       OptoReg::Bad, // Op_RegI
2849       RAX_H_num,    // Op_RegP
2850       OptoReg::Bad, // Op_RegF
2851       XMM0b_num,    // Op_RegD
2852       RAX_H_num     // Op_RegL
2853     };
2854     // Excluded flags and vector registers.
2855     assert(ARRAY_SIZE(hi) == _last_machine_leaf - 6, "missing type");
2856     return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
2857   %}
2858 %}
2859 
2860 //----------ATTRIBUTES---------------------------------------------------------
2861 //----------Operand Attributes-------------------------------------------------
2862 op_attrib op_cost(0);        // Required cost attribute
2863 
2864 //----------Instruction Attributes---------------------------------------------
2865 ins_attrib ins_cost(100);       // Required cost attribute
2866 ins_attrib ins_size(8);         // Required size attribute (in bits)
2867 ins_attrib ins_short_branch(0); // Required flag: is this instruction
2868                                 // a non-matching short branch variant
2869                                 // of some long branch?
2870 ins_attrib ins_alignment(1);    // Required alignment attribute (must
2871                                 // be a power of 2) specifies the
2872                                 // alignment that some part of the
2873                                 // instruction (not necessarily the
2874                                 // start) requires.  If > 1, a
2875                                 // compute_padding() function must be
2876                                 // provided for the instruction
2877 
2878 //----------OPERANDS-----------------------------------------------------------
2879 // Operand definitions must precede instruction definitions for correct parsing
2880 // in the ADLC because operands constitute user defined types which are used in
2881 // instruction definitions.
2882 
2883 //----------Simple Operands----------------------------------------------------
2884 // Immediate Operands
2885 // Integer Immediate
2886 operand immI()
2887 %{
2888   match(ConI);
2889 
2890   op_cost(10);
2891   format %{ %}
2892   interface(CONST_INTER);
2893 %}
2894 
2895 // Constant for test vs zero
2896 operand immI0()
2897 %{
2898   predicate(n->get_int() == 0);
2899   match(ConI);
2900 
2901   op_cost(0);
2902   format %{ %}
2903   interface(CONST_INTER);
2904 %}
2905 
2906 // Constant for increment
2907 operand immI1()
2908 %{
2909   predicate(n->get_int() == 1);
2910   match(ConI);
2911 
2912   op_cost(0);
2913   format %{ %}
2914   interface(CONST_INTER);
2915 %}
2916 
2917 // Constant for decrement
2918 operand immI_M1()
2919 %{
2920   predicate(n->get_int() == -1);
2921   match(ConI);
2922 
2923   op_cost(0);
2924   format %{ %}
2925   interface(CONST_INTER);
2926 %}
2927 
2928 // Valid scale values for addressing modes
2929 operand immI2()
2930 %{
2931   predicate(0 <= n->get_int() && (n->get_int() <= 3));
2932   match(ConI);
2933 
2934   format %{ %}
2935   interface(CONST_INTER);
2936 %}
2937 
2938 operand immI8()
2939 %{
2940   predicate((-0x80 <= n->get_int()) && (n->get_int() < 0x80));
2941   match(ConI);
2942 
2943   op_cost(5);
2944   format %{ %}
2945   interface(CONST_INTER);
2946 %}
2947 
2948 operand immI16()
2949 %{
2950   predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
2951   match(ConI);
2952 
2953   op_cost(10);
2954   format %{ %}
2955   interface(CONST_INTER);
2956 %}
2957 
2958 // Int Immediate non-negative
2959 operand immU31()
2960 %{
2961   predicate(n->get_int() >= 0);
2962   match(ConI);
2963 
2964   op_cost(0);
2965   format %{ %}
2966   interface(CONST_INTER);
2967 %}
2968 
2969 // Constant for long shifts
2970 operand immI_32()
2971 %{
2972   predicate( n->get_int() == 32 );
2973   match(ConI);
2974 
2975   op_cost(0);
2976   format %{ %}
2977   interface(CONST_INTER);
2978 %}
2979 
2980 // Constant for long shifts
2981 operand immI_64()
2982 %{
2983   predicate( n->get_int() == 64 );
2984   match(ConI);
2985 
2986   op_cost(0);
2987   format %{ %}
2988   interface(CONST_INTER);
2989 %}
2990 
2991 // Pointer Immediate
2992 operand immP()
2993 %{
2994   match(ConP);
2995 
2996   op_cost(10);
2997   format %{ %}
2998   interface(CONST_INTER);
2999 %}
3000 
3001 // NULL Pointer Immediate
3002 operand immP0()
3003 %{
3004   predicate(n->get_ptr() == 0);
3005   match(ConP);
3006 
3007   op_cost(5);
3008   format %{ %}
3009   interface(CONST_INTER);
3010 %}
3011 
3012 // Pointer Immediate
3013 operand immN() %{
3014   match(ConN);
3015 
3016   op_cost(10);
3017   format %{ %}
3018   interface(CONST_INTER);
3019 %}
3020 
3021 operand immNKlass() %{
3022   match(ConNKlass);
3023 
3024   op_cost(10);
3025   format %{ %}
3026   interface(CONST_INTER);
3027 %}
3028 
3029 // NULL Pointer Immediate
3030 operand immN0() %{
3031   predicate(n->get_narrowcon() == 0);
3032   match(ConN);
3033 
3034   op_cost(5);
3035   format %{ %}
3036   interface(CONST_INTER);
3037 %}
3038 
3039 operand immP31()
3040 %{
3041   predicate(n->as_Type()->type()->reloc() == relocInfo::none
3042             && (n->get_ptr() >> 31) == 0);
3043   match(ConP);
3044 
3045   op_cost(5);
3046   format %{ %}
3047   interface(CONST_INTER);
3048 %}
3049 
3050 
3051 // Long Immediate
3052 operand immL()
3053 %{
3054   match(ConL);
3055 
3056   op_cost(20);
3057   format %{ %}
3058   interface(CONST_INTER);
3059 %}
3060 
3061 // Long Immediate 8-bit
3062 operand immL8()
3063 %{
3064   predicate(-0x80L <= n->get_long() && n->get_long() < 0x80L);
3065   match(ConL);
3066 
3067   op_cost(5);
3068   format %{ %}
3069   interface(CONST_INTER);
3070 %}
3071 
3072 // Long Immediate 32-bit unsigned
3073 operand immUL32()
3074 %{
3075   predicate(n->get_long() == (unsigned int) (n->get_long()));
3076   match(ConL);
3077 
3078   op_cost(10);
3079   format %{ %}
3080   interface(CONST_INTER);
3081 %}
3082 
3083 // Long Immediate 32-bit signed
3084 operand immL32()
3085 %{
3086   predicate(n->get_long() == (int) (n->get_long()));
3087   match(ConL);
3088 
3089   op_cost(15);
3090   format %{ %}
3091   interface(CONST_INTER);
3092 %}
3093 
3094 // Long Immediate zero
3095 operand immL0()
3096 %{
3097   predicate(n->get_long() == 0L);
3098   match(ConL);
3099 
3100   op_cost(10);
3101   format %{ %}
3102   interface(CONST_INTER);
3103 %}
3104 
3105 // Constant for increment
3106 operand immL1()
3107 %{
3108   predicate(n->get_long() == 1);
3109   match(ConL);
3110 
3111   format %{ %}
3112   interface(CONST_INTER);
3113 %}
3114 
3115 // Constant for decrement
3116 operand immL_M1()
3117 %{
3118   predicate(n->get_long() == -1);
3119   match(ConL);
3120 
3121   format %{ %}
3122   interface(CONST_INTER);
3123 %}
3124 
3125 // Long Immediate: the value 10
3126 operand immL10()
3127 %{
3128   predicate(n->get_long() == 10);
3129   match(ConL);
3130 
3131   format %{ %}
3132   interface(CONST_INTER);
3133 %}
3134 
3135 // Long immediate from 0 to 127.
3136 // Used for a shorter form of long mul by 10.
3137 operand immL_127()
3138 %{
3139   predicate(0 <= n->get_long() && n->get_long() < 0x80);
3140   match(ConL);
3141 
3142   op_cost(10);
3143   format %{ %}
3144   interface(CONST_INTER);
3145 %}
3146 
3147 // Long Immediate: low 32-bit mask
3148 operand immL_32bits()
3149 %{
3150   predicate(n->get_long() == 0xFFFFFFFFL);
3151   match(ConL);
3152   op_cost(20);
3153 
3154   format %{ %}
3155   interface(CONST_INTER);
3156 %}
3157 
3158 // Float Immediate zero
3159 operand immF0()
3160 %{
3161   predicate(jint_cast(n->getf()) == 0);
3162   match(ConF);
3163 
3164   op_cost(5);
3165   format %{ %}
3166   interface(CONST_INTER);
3167 %}
3168 
3169 // Float Immediate
3170 operand immF()
3171 %{
3172   match(ConF);
3173 
3174   op_cost(15);
3175   format %{ %}
3176   interface(CONST_INTER);
3177 %}
3178 
3179 // Double Immediate zero
3180 operand immD0()
3181 %{
3182   predicate(jlong_cast(n->getd()) == 0);
3183   match(ConD);
3184 
3185   op_cost(5);
3186   format %{ %}
3187   interface(CONST_INTER);
3188 %}
3189 
3190 // Double Immediate
3191 operand immD()
3192 %{
3193   match(ConD);
3194 
3195   op_cost(15);
3196   format %{ %}
3197   interface(CONST_INTER);
3198 %}
3199 
3200 // Immediates for special shifts (sign extend)
3201 
3202 // Constants for increment
3203 operand immI_16()
3204 %{
3205   predicate(n->get_int() == 16);
3206   match(ConI);
3207 
3208   format %{ %}
3209   interface(CONST_INTER);
3210 %}
3211 
3212 operand immI_24()
3213 %{
3214   predicate(n->get_int() == 24);
3215   match(ConI);
3216 
3217   format %{ %}
3218   interface(CONST_INTER);
3219 %}
3220 
3221 // Constant for byte-wide masking
3222 operand immI_255()
3223 %{
3224   predicate(n->get_int() == 255);
3225   match(ConI);
3226 
3227   format %{ %}
3228   interface(CONST_INTER);
3229 %}
3230 
3231 // Constant for short-wide masking
3232 operand immI_65535()
3233 %{
3234   predicate(n->get_int() == 65535);
3235   match(ConI);
3236 
3237   format %{ %}
3238   interface(CONST_INTER);
3239 %}
3240 
3241 // Constant for byte-wide masking
3242 operand immL_255()
3243 %{
3244   predicate(n->get_long() == 255);
3245   match(ConL);
3246 
3247   format %{ %}
3248   interface(CONST_INTER);
3249 %}
3250 
3251 // Constant for short-wide masking
3252 operand immL_65535()
3253 %{
3254   predicate(n->get_long() == 65535);
3255   match(ConL);
3256 
3257   format %{ %}
3258   interface(CONST_INTER);
3259 %}
3260 
3261 // Register Operands
3262 // Integer Register
3263 operand rRegI()
3264 %{
3265   constraint(ALLOC_IN_RC(int_reg));
3266   match(RegI);
3267 
3268   match(rax_RegI);
3269   match(rbx_RegI);
3270   match(rcx_RegI);
3271   match(rdx_RegI);
3272   match(rdi_RegI);
3273 
3274   format %{ %}
3275   interface(REG_INTER);
3276 %}
3277 
3278 // Special Registers
3279 operand rax_RegI()
3280 %{
3281   constraint(ALLOC_IN_RC(int_rax_reg));
3282   match(RegI);
3283   match(rRegI);
3284 
3285   format %{ "RAX" %}
3286   interface(REG_INTER);
3287 %}
3288 
3289 // Special Registers
3290 operand rbx_RegI()
3291 %{
3292   constraint(ALLOC_IN_RC(int_rbx_reg));
3293   match(RegI);
3294   match(rRegI);
3295 
3296   format %{ "RBX" %}
3297   interface(REG_INTER);
3298 %}
3299 
3300 operand rcx_RegI()
3301 %{
3302   constraint(ALLOC_IN_RC(int_rcx_reg));
3303   match(RegI);
3304   match(rRegI);
3305 
3306   format %{ "RCX" %}
3307   interface(REG_INTER);
3308 %}
3309 
3310 operand rdx_RegI()
3311 %{
3312   constraint(ALLOC_IN_RC(int_rdx_reg));
3313   match(RegI);
3314   match(rRegI);
3315 
3316   format %{ "RDX" %}
3317   interface(REG_INTER);
3318 %}
3319 
3320 operand rdi_RegI()
3321 %{
3322   constraint(ALLOC_IN_RC(int_rdi_reg));
3323   match(RegI);
3324   match(rRegI);
3325 
3326   format %{ "RDI" %}
3327   interface(REG_INTER);
3328 %}
3329 
3330 operand no_rcx_RegI()
3331 %{
3332   constraint(ALLOC_IN_RC(int_no_rcx_reg));
3333   match(RegI);
3334   match(rax_RegI);
3335   match(rbx_RegI);
3336   match(rdx_RegI);
3337   match(rdi_RegI);
3338 
3339   format %{ %}
3340   interface(REG_INTER);
3341 %}
3342 
3343 operand no_rax_rdx_RegI()
3344 %{
3345   constraint(ALLOC_IN_RC(int_no_rax_rdx_reg));
3346   match(RegI);
3347   match(rbx_RegI);
3348   match(rcx_RegI);
3349   match(rdi_RegI);
3350 
3351   format %{ %}
3352   interface(REG_INTER);
3353 %}
3354 
3355 // Pointer Register
3356 operand any_RegP()
3357 %{
3358   constraint(ALLOC_IN_RC(any_reg));
3359   match(RegP);
3360   match(rax_RegP);
3361   match(rbx_RegP);
3362   match(rdi_RegP);
3363   match(rsi_RegP);
3364   match(rbp_RegP);
3365   match(r15_RegP);
3366   match(rRegP);
3367 
3368   format %{ %}
3369   interface(REG_INTER);
3370 %}
3371 
3372 operand rRegP()
3373 %{
3374   constraint(ALLOC_IN_RC(ptr_reg));
3375   match(RegP);
3376   match(rax_RegP);
3377   match(rbx_RegP);
3378   match(rdi_RegP);
3379   match(rsi_RegP);
3380   match(rbp_RegP);  // See Q&A below about
3381   match(r15_RegP);  // r15_RegP and rbp_RegP.
3382 
3383   format %{ %}
3384   interface(REG_INTER);
3385 %}
3386 
3387 operand rRegN() %{
3388   constraint(ALLOC_IN_RC(int_reg));
3389   match(RegN);
3390 
3391   format %{ %}
3392   interface(REG_INTER);
3393 %}
3394 
3395 // Question: Why is r15_RegP (the read-only TLS register) a match for rRegP?
3396 // Answer: Operand match rules govern the DFA as it processes instruction inputs.
3397 // It's fine for an instruction input that expects rRegP to match a r15_RegP.
3398 // The output of an instruction is controlled by the allocator, which respects
3399 // register class masks, not match rules.  Unless an instruction mentions
3400 // r15_RegP or any_RegP explicitly as its output, r15 will not be considered
3401 // by the allocator as an input.
3402 // The same logic applies to rbp_RegP being a match for rRegP: If PreserveFramePointer==true,
3403 // the RBP is used as a proper frame pointer and is not included in ptr_reg. As a
3404 // result, RBP is not included in the output of the instruction either.
3405 
3406 operand no_rax_RegP()
3407 %{
3408   constraint(ALLOC_IN_RC(ptr_no_rax_reg));
3409   match(RegP);
3410   match(rbx_RegP);
3411   match(rsi_RegP);
3412   match(rdi_RegP);
3413 
3414   format %{ %}
3415   interface(REG_INTER);
3416 %}
3417 
3418 // This operand is not allowed to use RBP even if
3419 // RBP is not used to hold the frame pointer.
3420 operand no_rbp_RegP()
3421 %{
3422   constraint(ALLOC_IN_RC(ptr_reg_no_rbp));
3423   match(RegP);
3424   match(rbx_RegP);
3425   match(rsi_RegP);
3426   match(rdi_RegP);
3427 
3428   format %{ %}
3429   interface(REG_INTER);
3430 %}
3431 
3432 operand no_rax_rbx_RegP()
3433 %{
3434   constraint(ALLOC_IN_RC(ptr_no_rax_rbx_reg));
3435   match(RegP);
3436   match(rsi_RegP);
3437   match(rdi_RegP);
3438 
3439   format %{ %}
3440   interface(REG_INTER);
3441 %}
3442 
3443 // Special Registers
3444 // Return a pointer value
3445 operand rax_RegP()
3446 %{
3447   constraint(ALLOC_IN_RC(ptr_rax_reg));
3448   match(RegP);
3449   match(rRegP);
3450 
3451   format %{ %}
3452   interface(REG_INTER);
3453 %}
3454 
3455 // Special Registers
3456 // Return a compressed pointer value
3457 operand rax_RegN()
3458 %{
3459   constraint(ALLOC_IN_RC(int_rax_reg));
3460   match(RegN);
3461   match(rRegN);
3462 
3463   format %{ %}
3464   interface(REG_INTER);
3465 %}
3466 
3467 // Used in AtomicAdd
3468 operand rbx_RegP()
3469 %{
3470   constraint(ALLOC_IN_RC(ptr_rbx_reg));
3471   match(RegP);
3472   match(rRegP);
3473 
3474   format %{ %}
3475   interface(REG_INTER);
3476 %}
3477 
3478 operand rsi_RegP()
3479 %{
3480   constraint(ALLOC_IN_RC(ptr_rsi_reg));
3481   match(RegP);
3482   match(rRegP);
3483 
3484   format %{ %}
3485   interface(REG_INTER);
3486 %}
3487 
3488 // Used in rep stosq
3489 operand rdi_RegP()
3490 %{
3491   constraint(ALLOC_IN_RC(ptr_rdi_reg));
3492   match(RegP);
3493   match(rRegP);
3494 
3495   format %{ %}
3496   interface(REG_INTER);
3497 %}
3498 
3499 operand r15_RegP()
3500 %{
3501   constraint(ALLOC_IN_RC(ptr_r15_reg));
3502   match(RegP);
3503   match(rRegP);
3504 
3505   format %{ %}
3506   interface(REG_INTER);
3507 %}
3508 
3509 operand rRegL()
3510 %{
3511   constraint(ALLOC_IN_RC(long_reg));
3512   match(RegL);
3513   match(rax_RegL);
3514   match(rdx_RegL);
3515 
3516   format %{ %}
3517   interface(REG_INTER);
3518 %}
3519 
3520 // Special Registers
3521 operand no_rax_rdx_RegL()
3522 %{
3523   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3524   match(RegL);
3525   match(rRegL);
3526 
3527   format %{ %}
3528   interface(REG_INTER);
3529 %}
3530 
3531 operand no_rax_RegL()
3532 %{
3533   constraint(ALLOC_IN_RC(long_no_rax_rdx_reg));
3534   match(RegL);
3535   match(rRegL);
3536   match(rdx_RegL);
3537 
3538   format %{ %}
3539   interface(REG_INTER);
3540 %}
3541 
3542 operand no_rcx_RegL()
3543 %{
3544   constraint(ALLOC_IN_RC(long_no_rcx_reg));
3545   match(RegL);
3546   match(rRegL);
3547 
3548   format %{ %}
3549   interface(REG_INTER);
3550 %}
3551 
3552 operand rax_RegL()
3553 %{
3554   constraint(ALLOC_IN_RC(long_rax_reg));
3555   match(RegL);
3556   match(rRegL);
3557 
3558   format %{ "RAX" %}
3559   interface(REG_INTER);
3560 %}
3561 
3562 operand rcx_RegL()
3563 %{
3564   constraint(ALLOC_IN_RC(long_rcx_reg));
3565   match(RegL);
3566   match(rRegL);
3567 
3568   format %{ %}
3569   interface(REG_INTER);
3570 %}
3571 
3572 operand rdx_RegL()
3573 %{
3574   constraint(ALLOC_IN_RC(long_rdx_reg));
3575   match(RegL);
3576   match(rRegL);
3577 
3578   format %{ %}
3579   interface(REG_INTER);
3580 %}
3581 
3582 // Flags register, used as output of compare instructions
3583 operand rFlagsReg()
3584 %{
3585   constraint(ALLOC_IN_RC(int_flags));
3586   match(RegFlags);
3587 
3588   format %{ "RFLAGS" %}
3589   interface(REG_INTER);
3590 %}
3591 
3592 // Flags register, used as output of FLOATING POINT compare instructions
3593 operand rFlagsRegU()
3594 %{
3595   constraint(ALLOC_IN_RC(int_flags));
3596   match(RegFlags);
3597 
3598   format %{ "RFLAGS_U" %}
3599   interface(REG_INTER);
3600 %}
3601 
3602 operand rFlagsRegUCF() %{
3603   constraint(ALLOC_IN_RC(int_flags));
3604   match(RegFlags);
3605   predicate(false);
3606 
3607   format %{ "RFLAGS_U_CF" %}
3608   interface(REG_INTER);
3609 %}
3610 
3611 // Float register operands
3612 operand regF() %{
3613    constraint(ALLOC_IN_RC(float_reg));
3614    match(RegF);
3615 
3616    format %{ %}
3617    interface(REG_INTER);
3618 %}
3619 
3620 // Double register operands
3621 operand regD() %{
3622    constraint(ALLOC_IN_RC(double_reg));
3623    match(RegD);
3624 
3625    format %{ %}
3626    interface(REG_INTER);
3627 %}
3628 
3629 // Vectors
3630 operand vecS() %{
3631   constraint(ALLOC_IN_RC(vectors_reg));
3632   match(VecS);
3633 
3634   format %{ %}
3635   interface(REG_INTER);
3636 %}
3637 
3638 operand vecD() %{
3639   constraint(ALLOC_IN_RC(vectord_reg));
3640   match(VecD);
3641 
3642   format %{ %}
3643   interface(REG_INTER);
3644 %}
3645 
3646 operand vecX() %{
3647   constraint(ALLOC_IN_RC(vectorx_reg));
3648   match(VecX);
3649 
3650   format %{ %}
3651   interface(REG_INTER);
3652 %}
3653 
3654 operand vecY() %{
3655   constraint(ALLOC_IN_RC(vectory_reg));
3656   match(VecY);
3657 
3658   format %{ %}
3659   interface(REG_INTER);
3660 %}
3661 
3662 //----------Memory Operands----------------------------------------------------
3663 // Direct Memory Operand
3664 // operand direct(immP addr)
3665 // %{
3666 //   match(addr);
3667 
3668 //   format %{ "[$addr]" %}
3669 //   interface(MEMORY_INTER) %{
3670 //     base(0xFFFFFFFF);
3671 //     index(0x4);
3672 //     scale(0x0);
3673 //     disp($addr);
3674 //   %}
3675 // %}
3676 
3677 // Indirect Memory Operand
3678 operand indirect(any_RegP reg)
3679 %{
3680   constraint(ALLOC_IN_RC(ptr_reg));
3681   match(reg);
3682 
3683   format %{ "[$reg]" %}
3684   interface(MEMORY_INTER) %{
3685     base($reg);
3686     index(0x4);
3687     scale(0x0);
3688     disp(0x0);
3689   %}
3690 %}
3691 
3692 // Indirect Memory Plus Short Offset Operand
3693 operand indOffset8(any_RegP reg, immL8 off)
3694 %{
3695   constraint(ALLOC_IN_RC(ptr_reg));
3696   match(AddP reg off);
3697 
3698   format %{ "[$reg + $off (8-bit)]" %}
3699   interface(MEMORY_INTER) %{
3700     base($reg);
3701     index(0x4);
3702     scale(0x0);
3703     disp($off);
3704   %}
3705 %}
3706 
3707 // Indirect Memory Plus Long Offset Operand
3708 operand indOffset32(any_RegP reg, immL32 off)
3709 %{
3710   constraint(ALLOC_IN_RC(ptr_reg));
3711   match(AddP reg off);
3712 
3713   format %{ "[$reg + $off (32-bit)]" %}
3714   interface(MEMORY_INTER) %{
3715     base($reg);
3716     index(0x4);
3717     scale(0x0);
3718     disp($off);
3719   %}
3720 %}
3721 
3722 // Indirect Memory Plus Index Register Plus Offset Operand
3723 operand indIndexOffset(any_RegP reg, rRegL lreg, immL32 off)
3724 %{
3725   constraint(ALLOC_IN_RC(ptr_reg));
3726   match(AddP (AddP reg lreg) off);
3727 
3728   op_cost(10);
3729   format %{"[$reg + $off + $lreg]" %}
3730   interface(MEMORY_INTER) %{
3731     base($reg);
3732     index($lreg);
3733     scale(0x0);
3734     disp($off);
3735   %}
3736 %}
3737 
3738 // Indirect Memory Plus Index Register Plus Offset Operand
3739 operand indIndex(any_RegP reg, rRegL lreg)
3740 %{
3741   constraint(ALLOC_IN_RC(ptr_reg));
3742   match(AddP reg lreg);
3743 
3744   op_cost(10);
3745   format %{"[$reg + $lreg]" %}
3746   interface(MEMORY_INTER) %{
3747     base($reg);
3748     index($lreg);
3749     scale(0x0);
3750     disp(0x0);
3751   %}
3752 %}
3753 
3754 // Indirect Memory Times Scale Plus Index Register
3755 operand indIndexScale(any_RegP reg, rRegL lreg, immI2 scale)
3756 %{
3757   constraint(ALLOC_IN_RC(ptr_reg));
3758   match(AddP reg (LShiftL lreg scale));
3759 
3760   op_cost(10);
3761   format %{"[$reg + $lreg << $scale]" %}
3762   interface(MEMORY_INTER) %{
3763     base($reg);
3764     index($lreg);
3765     scale($scale);
3766     disp(0x0);
3767   %}
3768 %}
3769 
3770 operand indPosIndexScale(any_RegP reg, rRegI idx, immI2 scale)
3771 %{
3772   constraint(ALLOC_IN_RC(ptr_reg));
3773   predicate(n->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3774   match(AddP reg (LShiftL (ConvI2L idx) scale));
3775 
3776   op_cost(10);
3777   format %{"[$reg + pos $idx << $scale]" %}
3778   interface(MEMORY_INTER) %{
3779     base($reg);
3780     index($idx);
3781     scale($scale);
3782     disp(0x0);
3783   %}
3784 %}
3785 
3786 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3787 operand indIndexScaleOffset(any_RegP reg, immL32 off, rRegL lreg, immI2 scale)
3788 %{
3789   constraint(ALLOC_IN_RC(ptr_reg));
3790   match(AddP (AddP reg (LShiftL lreg scale)) off);
3791 
3792   op_cost(10);
3793   format %{"[$reg + $off + $lreg << $scale]" %}
3794   interface(MEMORY_INTER) %{
3795     base($reg);
3796     index($lreg);
3797     scale($scale);
3798     disp($off);
3799   %}
3800 %}
3801 
3802 // Indirect Memory Plus Positive Index Register Plus Offset Operand
3803 operand indPosIndexOffset(any_RegP reg, immL32 off, rRegI idx)
3804 %{
3805   constraint(ALLOC_IN_RC(ptr_reg));
3806   predicate(n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3807   match(AddP (AddP reg (ConvI2L idx)) off);
3808 
3809   op_cost(10);
3810   format %{"[$reg + $off + $idx]" %}
3811   interface(MEMORY_INTER) %{
3812     base($reg);
3813     index($idx);
3814     scale(0x0);
3815     disp($off);
3816   %}
3817 %}
3818 
3819 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3820 operand indPosIndexScaleOffset(any_RegP reg, immL32 off, rRegI idx, immI2 scale)
3821 %{
3822   constraint(ALLOC_IN_RC(ptr_reg));
3823   predicate(n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3824   match(AddP (AddP reg (LShiftL (ConvI2L idx) scale)) off);
3825 
3826   op_cost(10);
3827   format %{"[$reg + $off + $idx << $scale]" %}
3828   interface(MEMORY_INTER) %{
3829     base($reg);
3830     index($idx);
3831     scale($scale);
3832     disp($off);
3833   %}
3834 %}
3835 
3836 // Indirect Narrow Oop Plus Offset Operand
3837 // Note: x86 architecture doesn't support "scale * index + offset" without a base
3838 // we can't free r12 even with Universe::narrow_oop_base() == NULL.
3839 operand indCompressedOopOffset(rRegN reg, immL32 off) %{
3840   predicate(UseCompressedOops && (Universe::narrow_oop_shift() == Address::times_8));
3841   constraint(ALLOC_IN_RC(ptr_reg));
3842   match(AddP (DecodeN reg) off);
3843 
3844   op_cost(10);
3845   format %{"[R12 + $reg << 3 + $off] (compressed oop addressing)" %}
3846   interface(MEMORY_INTER) %{
3847     base(0xc); // R12
3848     index($reg);
3849     scale(0x3);
3850     disp($off);
3851   %}
3852 %}
3853 
3854 // Indirect Memory Operand
3855 operand indirectNarrow(rRegN reg)
3856 %{
3857   predicate(Universe::narrow_oop_shift() == 0);
3858   constraint(ALLOC_IN_RC(ptr_reg));
3859   match(DecodeN reg);
3860 
3861   format %{ "[$reg]" %}
3862   interface(MEMORY_INTER) %{
3863     base($reg);
3864     index(0x4);
3865     scale(0x0);
3866     disp(0x0);
3867   %}
3868 %}
3869 
3870 // Indirect Memory Plus Short Offset Operand
3871 operand indOffset8Narrow(rRegN reg, immL8 off)
3872 %{
3873   predicate(Universe::narrow_oop_shift() == 0);
3874   constraint(ALLOC_IN_RC(ptr_reg));
3875   match(AddP (DecodeN reg) off);
3876 
3877   format %{ "[$reg + $off (8-bit)]" %}
3878   interface(MEMORY_INTER) %{
3879     base($reg);
3880     index(0x4);
3881     scale(0x0);
3882     disp($off);
3883   %}
3884 %}
3885 
3886 // Indirect Memory Plus Long Offset Operand
3887 operand indOffset32Narrow(rRegN reg, immL32 off)
3888 %{
3889   predicate(Universe::narrow_oop_shift() == 0);
3890   constraint(ALLOC_IN_RC(ptr_reg));
3891   match(AddP (DecodeN reg) off);
3892 
3893   format %{ "[$reg + $off (32-bit)]" %}
3894   interface(MEMORY_INTER) %{
3895     base($reg);
3896     index(0x4);
3897     scale(0x0);
3898     disp($off);
3899   %}
3900 %}
3901 
3902 // Indirect Memory Plus Index Register Plus Offset Operand
3903 operand indIndexOffsetNarrow(rRegN reg, rRegL lreg, immL32 off)
3904 %{
3905   predicate(Universe::narrow_oop_shift() == 0);
3906   constraint(ALLOC_IN_RC(ptr_reg));
3907   match(AddP (AddP (DecodeN reg) lreg) off);
3908 
3909   op_cost(10);
3910   format %{"[$reg + $off + $lreg]" %}
3911   interface(MEMORY_INTER) %{
3912     base($reg);
3913     index($lreg);
3914     scale(0x0);
3915     disp($off);
3916   %}
3917 %}
3918 
3919 // Indirect Memory Plus Index Register Plus Offset Operand
3920 operand indIndexNarrow(rRegN reg, rRegL lreg)
3921 %{
3922   predicate(Universe::narrow_oop_shift() == 0);
3923   constraint(ALLOC_IN_RC(ptr_reg));
3924   match(AddP (DecodeN reg) lreg);
3925 
3926   op_cost(10);
3927   format %{"[$reg + $lreg]" %}
3928   interface(MEMORY_INTER) %{
3929     base($reg);
3930     index($lreg);
3931     scale(0x0);
3932     disp(0x0);
3933   %}
3934 %}
3935 
3936 // Indirect Memory Times Scale Plus Index Register
3937 operand indIndexScaleNarrow(rRegN reg, rRegL lreg, immI2 scale)
3938 %{
3939   predicate(Universe::narrow_oop_shift() == 0);
3940   constraint(ALLOC_IN_RC(ptr_reg));
3941   match(AddP (DecodeN reg) (LShiftL lreg scale));
3942 
3943   op_cost(10);
3944   format %{"[$reg + $lreg << $scale]" %}
3945   interface(MEMORY_INTER) %{
3946     base($reg);
3947     index($lreg);
3948     scale($scale);
3949     disp(0x0);
3950   %}
3951 %}
3952 
3953 // Indirect Memory Times Scale Plus Index Register Plus Offset Operand
3954 operand indIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
3955 %{
3956   predicate(Universe::narrow_oop_shift() == 0);
3957   constraint(ALLOC_IN_RC(ptr_reg));
3958   match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
3959 
3960   op_cost(10);
3961   format %{"[$reg + $off + $lreg << $scale]" %}
3962   interface(MEMORY_INTER) %{
3963     base($reg);
3964     index($lreg);
3965     scale($scale);
3966     disp($off);
3967   %}
3968 %}
3969 
3970 // Indirect Memory Times Plus Positive Index Register Plus Offset Operand
3971 operand indPosIndexOffsetNarrow(rRegN reg, immL32 off, rRegI idx)
3972 %{
3973   constraint(ALLOC_IN_RC(ptr_reg));
3974   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->as_Type()->type()->is_long()->_lo >= 0);
3975   match(AddP (AddP (DecodeN reg) (ConvI2L idx)) off);
3976 
3977   op_cost(10);
3978   format %{"[$reg + $off + $idx]" %}
3979   interface(MEMORY_INTER) %{
3980     base($reg);
3981     index($idx);
3982     scale(0x0);
3983     disp($off);
3984   %}
3985 %}
3986 
3987 // Indirect Memory Times Scale Plus Positive Index Register Plus Offset Operand
3988 operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 scale)
3989 %{
3990   constraint(ALLOC_IN_RC(ptr_reg));
3991   predicate(Universe::narrow_oop_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
3992   match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L idx) scale)) off);
3993 
3994   op_cost(10);
3995   format %{"[$reg + $off + $idx << $scale]" %}
3996   interface(MEMORY_INTER) %{
3997     base($reg);
3998     index($idx);
3999     scale($scale);
4000     disp($off);
4001   %}
4002 %}
4003 
4004 //----------Special Memory Operands--------------------------------------------
4005 // Stack Slot Operand - This operand is used for loading and storing temporary
4006 //                      values on the stack where a match requires a value to
4007 //                      flow through memory.
4008 operand stackSlotP(sRegP reg)
4009 %{
4010   constraint(ALLOC_IN_RC(stack_slots));
4011   // No match rule because this operand is only generated in matching
4012 
4013   format %{ "[$reg]" %}
4014   interface(MEMORY_INTER) %{
4015     base(0x4);   // RSP
4016     index(0x4);  // No Index
4017     scale(0x0);  // No Scale
4018     disp($reg);  // Stack Offset
4019   %}
4020 %}
4021 
4022 operand stackSlotI(sRegI reg)
4023 %{
4024   constraint(ALLOC_IN_RC(stack_slots));
4025   // No match rule because this operand is only generated in matching
4026 
4027   format %{ "[$reg]" %}
4028   interface(MEMORY_INTER) %{
4029     base(0x4);   // RSP
4030     index(0x4);  // No Index
4031     scale(0x0);  // No Scale
4032     disp($reg);  // Stack Offset
4033   %}
4034 %}
4035 
4036 operand stackSlotF(sRegF reg)
4037 %{
4038   constraint(ALLOC_IN_RC(stack_slots));
4039   // No match rule because this operand is only generated in matching
4040 
4041   format %{ "[$reg]" %}
4042   interface(MEMORY_INTER) %{
4043     base(0x4);   // RSP
4044     index(0x4);  // No Index
4045     scale(0x0);  // No Scale
4046     disp($reg);  // Stack Offset
4047   %}
4048 %}
4049 
4050 operand stackSlotD(sRegD reg)
4051 %{
4052   constraint(ALLOC_IN_RC(stack_slots));
4053   // No match rule because this operand is only generated in matching
4054 
4055   format %{ "[$reg]" %}
4056   interface(MEMORY_INTER) %{
4057     base(0x4);   // RSP
4058     index(0x4);  // No Index
4059     scale(0x0);  // No Scale
4060     disp($reg);  // Stack Offset
4061   %}
4062 %}
4063 operand stackSlotL(sRegL reg)
4064 %{
4065   constraint(ALLOC_IN_RC(stack_slots));
4066   // No match rule because this operand is only generated in matching
4067 
4068   format %{ "[$reg]" %}
4069   interface(MEMORY_INTER) %{
4070     base(0x4);   // RSP
4071     index(0x4);  // No Index
4072     scale(0x0);  // No Scale
4073     disp($reg);  // Stack Offset
4074   %}
4075 %}
4076 
4077 //----------Conditional Branch Operands----------------------------------------
4078 // Comparison Op  - This is the operation of the comparison, and is limited to
4079 //                  the following set of codes:
4080 //                  L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
4081 //
4082 // Other attributes of the comparison, such as unsignedness, are specified
4083 // by the comparison instruction that sets a condition code flags register.
4084 // That result is represented by a flags operand whose subtype is appropriate
4085 // to the unsignedness (etc.) of the comparison.
4086 //
4087 // Later, the instruction which matches both the Comparison Op (a Bool) and
4088 // the flags (produced by the Cmp) specifies the coding of the comparison op
4089 // by matching a specific subtype of Bool operand below, such as cmpOpU.
4090 
4091 // Comparision Code
4092 operand cmpOp()
4093 %{
4094   match(Bool);
4095 
4096   format %{ "" %}
4097   interface(COND_INTER) %{
4098     equal(0x4, "e");
4099     not_equal(0x5, "ne");
4100     less(0xC, "l");
4101     greater_equal(0xD, "ge");
4102     less_equal(0xE, "le");
4103     greater(0xF, "g");
4104     overflow(0x0, "o");
4105     no_overflow(0x1, "no");
4106   %}
4107 %}
4108 
4109 // Comparison Code, unsigned compare.  Used by FP also, with
4110 // C2 (unordered) turned into GT or LT already.  The other bits
4111 // C0 and C3 are turned into Carry & Zero flags.
4112 operand cmpOpU()
4113 %{
4114   match(Bool);
4115 
4116   format %{ "" %}
4117   interface(COND_INTER) %{
4118     equal(0x4, "e");
4119     not_equal(0x5, "ne");
4120     less(0x2, "b");
4121     greater_equal(0x3, "nb");
4122     less_equal(0x6, "be");
4123     greater(0x7, "nbe");
4124     overflow(0x0, "o");
4125     no_overflow(0x1, "no");
4126   %}
4127 %}
4128 
4129 
4130 // Floating comparisons that don't require any fixup for the unordered case
4131 operand cmpOpUCF() %{
4132   match(Bool);
4133   predicate(n->as_Bool()->_test._test == BoolTest::lt ||
4134             n->as_Bool()->_test._test == BoolTest::ge ||
4135             n->as_Bool()->_test._test == BoolTest::le ||
4136             n->as_Bool()->_test._test == BoolTest::gt);
4137   format %{ "" %}
4138   interface(COND_INTER) %{
4139     equal(0x4, "e");
4140     not_equal(0x5, "ne");
4141     less(0x2, "b");
4142     greater_equal(0x3, "nb");
4143     less_equal(0x6, "be");
4144     greater(0x7, "nbe");
4145     overflow(0x0, "o");
4146     no_overflow(0x1, "no");
4147   %}
4148 %}
4149 
4150 
4151 // Floating comparisons that can be fixed up with extra conditional jumps
4152 operand cmpOpUCF2() %{
4153   match(Bool);
4154   predicate(n->as_Bool()->_test._test == BoolTest::ne ||
4155             n->as_Bool()->_test._test == BoolTest::eq);
4156   format %{ "" %}
4157   interface(COND_INTER) %{
4158     equal(0x4, "e");
4159     not_equal(0x5, "ne");
4160     less(0x2, "b");
4161     greater_equal(0x3, "nb");
4162     less_equal(0x6, "be");
4163     greater(0x7, "nbe");
4164     overflow(0x0, "o");
4165     no_overflow(0x1, "no");
4166   %}
4167 %}
4168 
4169 
4170 //----------OPERAND CLASSES----------------------------------------------------
4171 // Operand Classes are groups of operands that are used as to simplify
4172 // instruction definitions by not requiring the AD writer to specify separate
4173 // instructions for every form of operand when the instruction accepts
4174 // multiple operand types with the same basic encoding and format.  The classic
4175 // case of this is memory operands.
4176 
4177 opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
4178                indIndexScale, indPosIndexScale, indIndexScaleOffset, indPosIndexOffset, indPosIndexScaleOffset,
4179                indCompressedOopOffset,
4180                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
4181                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
4182                indIndexScaleOffsetNarrow, indPosIndexOffsetNarrow, indPosIndexScaleOffsetNarrow);
4183 
4184 //----------PIPELINE-----------------------------------------------------------
4185 // Rules which define the behavior of the target architectures pipeline.
4186 pipeline %{
4187 
4188 //----------ATTRIBUTES---------------------------------------------------------
4189 attributes %{
4190   variable_size_instructions;        // Fixed size instructions
4191   max_instructions_per_bundle = 3;   // Up to 3 instructions per bundle
4192   instruction_unit_size = 1;         // An instruction is 1 bytes long
4193   instruction_fetch_unit_size = 16;  // The processor fetches one line
4194   instruction_fetch_units = 1;       // of 16 bytes
4195 
4196   // List of nop instructions
4197   nops( MachNop );
4198 %}
4199 
4200 //----------RESOURCES----------------------------------------------------------
4201 // Resources are the functional units available to the machine
4202 
4203 // Generic P2/P3 pipeline
4204 // 3 decoders, only D0 handles big operands; a "bundle" is the limit of
4205 // 3 instructions decoded per cycle.
4206 // 2 load/store ops per cycle, 1 branch, 1 FPU,
4207 // 3 ALU op, only ALU0 handles mul instructions.
4208 resources( D0, D1, D2, DECODE = D0 | D1 | D2,
4209            MS0, MS1, MS2, MEM = MS0 | MS1 | MS2,
4210            BR, FPU,
4211            ALU0, ALU1, ALU2, ALU = ALU0 | ALU1 | ALU2);
4212 
4213 //----------PIPELINE DESCRIPTION-----------------------------------------------
4214 // Pipeline Description specifies the stages in the machine's pipeline
4215 
4216 // Generic P2/P3 pipeline
4217 pipe_desc(S0, S1, S2, S3, S4, S5);
4218 
4219 //----------PIPELINE CLASSES---------------------------------------------------
4220 // Pipeline Classes describe the stages in which input and output are
4221 // referenced by the hardware pipeline.
4222 
4223 // Naming convention: ialu or fpu
4224 // Then: _reg
4225 // Then: _reg if there is a 2nd register
4226 // Then: _long if it's a pair of instructions implementing a long
4227 // Then: _fat if it requires the big decoder
4228 //   Or: _mem if it requires the big decoder and a memory unit.
4229 
4230 // Integer ALU reg operation
4231 pipe_class ialu_reg(rRegI dst)
4232 %{
4233     single_instruction;
4234     dst    : S4(write);
4235     dst    : S3(read);
4236     DECODE : S0;        // any decoder
4237     ALU    : S3;        // any alu
4238 %}
4239 
4240 // Long ALU reg operation
4241 pipe_class ialu_reg_long(rRegL dst)
4242 %{
4243     instruction_count(2);
4244     dst    : S4(write);
4245     dst    : S3(read);
4246     DECODE : S0(2);     // any 2 decoders
4247     ALU    : S3(2);     // both alus
4248 %}
4249 
4250 // Integer ALU reg operation using big decoder
4251 pipe_class ialu_reg_fat(rRegI dst)
4252 %{
4253     single_instruction;
4254     dst    : S4(write);
4255     dst    : S3(read);
4256     D0     : S0;        // big decoder only
4257     ALU    : S3;        // any alu
4258 %}
4259 
4260 // Long ALU reg operation using big decoder
4261 pipe_class ialu_reg_long_fat(rRegL dst)
4262 %{
4263     instruction_count(2);
4264     dst    : S4(write);
4265     dst    : S3(read);
4266     D0     : S0(2);     // big decoder only; twice
4267     ALU    : S3(2);     // any 2 alus
4268 %}
4269 
4270 // Integer ALU reg-reg operation
4271 pipe_class ialu_reg_reg(rRegI dst, rRegI src)
4272 %{
4273     single_instruction;
4274     dst    : S4(write);
4275     src    : S3(read);
4276     DECODE : S0;        // any decoder
4277     ALU    : S3;        // any alu
4278 %}
4279 
4280 // Long ALU reg-reg operation
4281 pipe_class ialu_reg_reg_long(rRegL dst, rRegL src)
4282 %{
4283     instruction_count(2);
4284     dst    : S4(write);
4285     src    : S3(read);
4286     DECODE : S0(2);     // any 2 decoders
4287     ALU    : S3(2);     // both alus
4288 %}
4289 
4290 // Integer ALU reg-reg operation
4291 pipe_class ialu_reg_reg_fat(rRegI dst, memory src)
4292 %{
4293     single_instruction;
4294     dst    : S4(write);
4295     src    : S3(read);
4296     D0     : S0;        // big decoder only
4297     ALU    : S3;        // any alu
4298 %}
4299 
4300 // Long ALU reg-reg operation
4301 pipe_class ialu_reg_reg_long_fat(rRegL dst, rRegL src)
4302 %{
4303     instruction_count(2);
4304     dst    : S4(write);
4305     src    : S3(read);
4306     D0     : S0(2);     // big decoder only; twice
4307     ALU    : S3(2);     // both alus
4308 %}
4309 
4310 // Integer ALU reg-mem operation
4311 pipe_class ialu_reg_mem(rRegI dst, memory mem)
4312 %{
4313     single_instruction;
4314     dst    : S5(write);
4315     mem    : S3(read);
4316     D0     : S0;        // big decoder only
4317     ALU    : S4;        // any alu
4318     MEM    : S3;        // any mem
4319 %}
4320 
4321 // Integer mem operation (prefetch)
4322 pipe_class ialu_mem(memory mem)
4323 %{
4324     single_instruction;
4325     mem    : S3(read);
4326     D0     : S0;        // big decoder only
4327     MEM    : S3;        // any mem
4328 %}
4329 
4330 // Integer Store to Memory
4331 pipe_class ialu_mem_reg(memory mem, rRegI src)
4332 %{
4333     single_instruction;
4334     mem    : S3(read);
4335     src    : S5(read);
4336     D0     : S0;        // big decoder only
4337     ALU    : S4;        // any alu
4338     MEM    : S3;
4339 %}
4340 
4341 // // Long Store to Memory
4342 // pipe_class ialu_mem_long_reg(memory mem, rRegL src)
4343 // %{
4344 //     instruction_count(2);
4345 //     mem    : S3(read);
4346 //     src    : S5(read);
4347 //     D0     : S0(2);          // big decoder only; twice
4348 //     ALU    : S4(2);     // any 2 alus
4349 //     MEM    : S3(2);  // Both mems
4350 // %}
4351 
4352 // Integer Store to Memory
4353 pipe_class ialu_mem_imm(memory mem)
4354 %{
4355     single_instruction;
4356     mem    : S3(read);
4357     D0     : S0;        // big decoder only
4358     ALU    : S4;        // any alu
4359     MEM    : S3;
4360 %}
4361 
4362 // Integer ALU0 reg-reg operation
4363 pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src)
4364 %{
4365     single_instruction;
4366     dst    : S4(write);
4367     src    : S3(read);
4368     D0     : S0;        // Big decoder only
4369     ALU0   : S3;        // only alu0
4370 %}
4371 
4372 // Integer ALU0 reg-mem operation
4373 pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem)
4374 %{
4375     single_instruction;
4376     dst    : S5(write);
4377     mem    : S3(read);
4378     D0     : S0;        // big decoder only
4379     ALU0   : S4;        // ALU0 only
4380     MEM    : S3;        // any mem
4381 %}
4382 
4383 // Integer ALU reg-reg operation
4384 pipe_class ialu_cr_reg_reg(rFlagsReg cr, rRegI src1, rRegI src2)
4385 %{
4386     single_instruction;
4387     cr     : S4(write);
4388     src1   : S3(read);
4389     src2   : S3(read);
4390     DECODE : S0;        // any decoder
4391     ALU    : S3;        // any alu
4392 %}
4393 
4394 // Integer ALU reg-imm operation
4395 pipe_class ialu_cr_reg_imm(rFlagsReg cr, rRegI src1)
4396 %{
4397     single_instruction;
4398     cr     : S4(write);
4399     src1   : S3(read);
4400     DECODE : S0;        // any decoder
4401     ALU    : S3;        // any alu
4402 %}
4403 
4404 // Integer ALU reg-mem operation
4405 pipe_class ialu_cr_reg_mem(rFlagsReg cr, rRegI src1, memory src2)
4406 %{
4407     single_instruction;
4408     cr     : S4(write);
4409     src1   : S3(read);
4410     src2   : S3(read);
4411     D0     : S0;        // big decoder only
4412     ALU    : S4;        // any alu
4413     MEM    : S3;
4414 %}
4415 
4416 // Conditional move reg-reg
4417 pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y)
4418 %{
4419     instruction_count(4);
4420     y      : S4(read);
4421     q      : S3(read);
4422     p      : S3(read);
4423     DECODE : S0(4);     // any decoder
4424 %}
4425 
4426 // Conditional move reg-reg
4427 pipe_class pipe_cmov_reg( rRegI dst, rRegI src, rFlagsReg cr)
4428 %{
4429     single_instruction;
4430     dst    : S4(write);
4431     src    : S3(read);
4432     cr     : S3(read);
4433     DECODE : S0;        // any decoder
4434 %}
4435 
4436 // Conditional move reg-mem
4437 pipe_class pipe_cmov_mem( rFlagsReg cr, rRegI dst, memory src)
4438 %{
4439     single_instruction;
4440     dst    : S4(write);
4441     src    : S3(read);
4442     cr     : S3(read);
4443     DECODE : S0;        // any decoder
4444     MEM    : S3;
4445 %}
4446 
4447 // Conditional move reg-reg long
4448 pipe_class pipe_cmov_reg_long( rFlagsReg cr, rRegL dst, rRegL src)
4449 %{
4450     single_instruction;
4451     dst    : S4(write);
4452     src    : S3(read);
4453     cr     : S3(read);
4454     DECODE : S0(2);     // any 2 decoders
4455 %}
4456 
4457 // XXX
4458 // // Conditional move double reg-reg
4459 // pipe_class pipe_cmovD_reg( rFlagsReg cr, regDPR1 dst, regD src)
4460 // %{
4461 //     single_instruction;
4462 //     dst    : S4(write);
4463 //     src    : S3(read);
4464 //     cr     : S3(read);
4465 //     DECODE : S0;     // any decoder
4466 // %}
4467 
4468 // Float reg-reg operation
4469 pipe_class fpu_reg(regD dst)
4470 %{
4471     instruction_count(2);
4472     dst    : S3(read);
4473     DECODE : S0(2);     // any 2 decoders
4474     FPU    : S3;
4475 %}
4476 
4477 // Float reg-reg operation
4478 pipe_class fpu_reg_reg(regD dst, regD src)
4479 %{
4480     instruction_count(2);
4481     dst    : S4(write);
4482     src    : S3(read);
4483     DECODE : S0(2);     // any 2 decoders
4484     FPU    : S3;
4485 %}
4486 
4487 // Float reg-reg operation
4488 pipe_class fpu_reg_reg_reg(regD dst, regD src1, regD src2)
4489 %{
4490     instruction_count(3);
4491     dst    : S4(write);
4492     src1   : S3(read);
4493     src2   : S3(read);
4494     DECODE : S0(3);     // any 3 decoders
4495     FPU    : S3(2);
4496 %}
4497 
4498 // Float reg-reg operation
4499 pipe_class fpu_reg_reg_reg_reg(regD dst, regD src1, regD src2, regD src3)
4500 %{
4501     instruction_count(4);
4502     dst    : S4(write);
4503     src1   : S3(read);
4504     src2   : S3(read);
4505     src3   : S3(read);
4506     DECODE : S0(4);     // any 3 decoders
4507     FPU    : S3(2);
4508 %}
4509 
4510 // Float reg-reg operation
4511 pipe_class fpu_reg_mem_reg_reg(regD dst, memory src1, regD src2, regD src3)
4512 %{
4513     instruction_count(4);
4514     dst    : S4(write);
4515     src1   : S3(read);
4516     src2   : S3(read);
4517     src3   : S3(read);
4518     DECODE : S1(3);     // any 3 decoders
4519     D0     : S0;        // Big decoder only
4520     FPU    : S3(2);
4521     MEM    : S3;
4522 %}
4523 
4524 // Float reg-mem operation
4525 pipe_class fpu_reg_mem(regD dst, memory mem)
4526 %{
4527     instruction_count(2);
4528     dst    : S5(write);
4529     mem    : S3(read);
4530     D0     : S0;        // big decoder only
4531     DECODE : S1;        // any decoder for FPU POP
4532     FPU    : S4;
4533     MEM    : S3;        // any mem
4534 %}
4535 
4536 // Float reg-mem operation
4537 pipe_class fpu_reg_reg_mem(regD dst, regD src1, memory mem)
4538 %{
4539     instruction_count(3);
4540     dst    : S5(write);
4541     src1   : S3(read);
4542     mem    : S3(read);
4543     D0     : S0;        // big decoder only
4544     DECODE : S1(2);     // any decoder for FPU POP
4545     FPU    : S4;
4546     MEM    : S3;        // any mem
4547 %}
4548 
4549 // Float mem-reg operation
4550 pipe_class fpu_mem_reg(memory mem, regD src)
4551 %{
4552     instruction_count(2);
4553     src    : S5(read);
4554     mem    : S3(read);
4555     DECODE : S0;        // any decoder for FPU PUSH
4556     D0     : S1;        // big decoder only
4557     FPU    : S4;
4558     MEM    : S3;        // any mem
4559 %}
4560 
4561 pipe_class fpu_mem_reg_reg(memory mem, regD src1, regD src2)
4562 %{
4563     instruction_count(3);
4564     src1   : S3(read);
4565     src2   : S3(read);
4566     mem    : S3(read);
4567     DECODE : S0(2);     // any decoder for FPU PUSH
4568     D0     : S1;        // big decoder only
4569     FPU    : S4;
4570     MEM    : S3;        // any mem
4571 %}
4572 
4573 pipe_class fpu_mem_reg_mem(memory mem, regD src1, memory src2)
4574 %{
4575     instruction_count(3);
4576     src1   : S3(read);
4577     src2   : S3(read);
4578     mem    : S4(read);
4579     DECODE : S0;        // any decoder for FPU PUSH
4580     D0     : S0(2);     // big decoder only
4581     FPU    : S4;
4582     MEM    : S3(2);     // any mem
4583 %}
4584 
4585 pipe_class fpu_mem_mem(memory dst, memory src1)
4586 %{
4587     instruction_count(2);
4588     src1   : S3(read);
4589     dst    : S4(read);
4590     D0     : S0(2);     // big decoder only
4591     MEM    : S3(2);     // any mem
4592 %}
4593 
4594 pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2)
4595 %{
4596     instruction_count(3);
4597     src1   : S3(read);
4598     src2   : S3(read);
4599     dst    : S4(read);
4600     D0     : S0(3);     // big decoder only
4601     FPU    : S4;
4602     MEM    : S3(3);     // any mem
4603 %}
4604 
4605 pipe_class fpu_mem_reg_con(memory mem, regD src1)
4606 %{
4607     instruction_count(3);
4608     src1   : S4(read);
4609     mem    : S4(read);
4610     DECODE : S0;        // any decoder for FPU PUSH
4611     D0     : S0(2);     // big decoder only
4612     FPU    : S4;
4613     MEM    : S3(2);     // any mem
4614 %}
4615 
4616 // Float load constant
4617 pipe_class fpu_reg_con(regD dst)
4618 %{
4619     instruction_count(2);
4620     dst    : S5(write);
4621     D0     : S0;        // big decoder only for the load
4622     DECODE : S1;        // any decoder for FPU POP
4623     FPU    : S4;
4624     MEM    : S3;        // any mem
4625 %}
4626 
4627 // Float load constant
4628 pipe_class fpu_reg_reg_con(regD dst, regD src)
4629 %{
4630     instruction_count(3);
4631     dst    : S5(write);
4632     src    : S3(read);
4633     D0     : S0;        // big decoder only for the load
4634     DECODE : S1(2);     // any decoder for FPU POP
4635     FPU    : S4;
4636     MEM    : S3;        // any mem
4637 %}
4638 
4639 // UnConditional branch
4640 pipe_class pipe_jmp(label labl)
4641 %{
4642     single_instruction;
4643     BR   : S3;
4644 %}
4645 
4646 // Conditional branch
4647 pipe_class pipe_jcc(cmpOp cmp, rFlagsReg cr, label labl)
4648 %{
4649     single_instruction;
4650     cr    : S1(read);
4651     BR    : S3;
4652 %}
4653 
4654 // Allocation idiom
4655 pipe_class pipe_cmpxchg(rRegP dst, rRegP heap_ptr)
4656 %{
4657     instruction_count(1); force_serialization;
4658     fixed_latency(6);
4659     heap_ptr : S3(read);
4660     DECODE   : S0(3);
4661     D0       : S2;
4662     MEM      : S3;
4663     ALU      : S3(2);
4664     dst      : S5(write);
4665     BR       : S5;
4666 %}
4667 
4668 // Generic big/slow expanded idiom
4669 pipe_class pipe_slow()
4670 %{
4671     instruction_count(10); multiple_bundles; force_serialization;
4672     fixed_latency(100);
4673     D0  : S0(2);
4674     MEM : S3(2);
4675 %}
4676 
4677 // The real do-nothing guy
4678 pipe_class empty()
4679 %{
4680     instruction_count(0);
4681 %}
4682 
4683 // Define the class for the Nop node
4684 define
4685 %{
4686    MachNop = empty;
4687 %}
4688 
4689 %}
4690 
4691 //----------INSTRUCTIONS-------------------------------------------------------
4692 //
4693 // match      -- States which machine-independent subtree may be replaced
4694 //               by this instruction.
4695 // ins_cost   -- The estimated cost of this instruction is used by instruction
4696 //               selection to identify a minimum cost tree of machine
4697 //               instructions that matches a tree of machine-independent
4698 //               instructions.
4699 // format     -- A string providing the disassembly for this instruction.
4700 //               The value of an instruction's operand may be inserted
4701 //               by referring to it with a '$' prefix.
4702 // opcode     -- Three instruction opcodes may be provided.  These are referred
4703 //               to within an encode class as $primary, $secondary, and $tertiary
4704 //               rrspectively.  The primary opcode is commonly used to
4705 //               indicate the type of machine instruction, while secondary
4706 //               and tertiary are often used for prefix options or addressing
4707 //               modes.
4708 // ins_encode -- A list of encode classes with parameters. The encode class
4709 //               name must have been defined in an 'enc_class' specification
4710 //               in the encode section of the architecture description.
4711 
4712 
4713 //----------Load/Store/Move Instructions---------------------------------------
4714 //----------Load Instructions--------------------------------------------------
4715 
4716 // Load Byte (8 bit signed)
4717 instruct loadB(rRegI dst, memory mem)
4718 %{
4719   match(Set dst (LoadB mem));
4720 
4721   ins_cost(125);
4722   format %{ "movsbl  $dst, $mem\t# byte" %}
4723 
4724   ins_encode %{
4725     __ movsbl($dst$$Register, $mem$$Address);
4726   %}
4727 
4728   ins_pipe(ialu_reg_mem);
4729 %}
4730 
4731 // Load Byte (8 bit signed) into Long Register
4732 instruct loadB2L(rRegL dst, memory mem)
4733 %{
4734   match(Set dst (ConvI2L (LoadB mem)));
4735 
4736   ins_cost(125);
4737   format %{ "movsbq  $dst, $mem\t# byte -> long" %}
4738 
4739   ins_encode %{
4740     __ movsbq($dst$$Register, $mem$$Address);
4741   %}
4742 
4743   ins_pipe(ialu_reg_mem);
4744 %}
4745 
4746 // Load Unsigned Byte (8 bit UNsigned)
4747 instruct loadUB(rRegI dst, memory mem)
4748 %{
4749   match(Set dst (LoadUB mem));
4750 
4751   ins_cost(125);
4752   format %{ "movzbl  $dst, $mem\t# ubyte" %}
4753 
4754   ins_encode %{
4755     __ movzbl($dst$$Register, $mem$$Address);
4756   %}
4757 
4758   ins_pipe(ialu_reg_mem);
4759 %}
4760 
4761 // Load Unsigned Byte (8 bit UNsigned) into Long Register
4762 instruct loadUB2L(rRegL dst, memory mem)
4763 %{
4764   match(Set dst (ConvI2L (LoadUB mem)));
4765 
4766   ins_cost(125);
4767   format %{ "movzbq  $dst, $mem\t# ubyte -> long" %}
4768 
4769   ins_encode %{
4770     __ movzbq($dst$$Register, $mem$$Address);
4771   %}
4772 
4773   ins_pipe(ialu_reg_mem);
4774 %}
4775 
4776 // Load Unsigned Byte (8 bit UNsigned) with 32-bit mask into Long Register
4777 instruct loadUB2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4778   match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
4779   effect(KILL cr);
4780 
4781   format %{ "movzbq  $dst, $mem\t# ubyte & 32-bit mask -> long\n\t"
4782             "andl    $dst, right_n_bits($mask, 8)" %}
4783   ins_encode %{
4784     Register Rdst = $dst$$Register;
4785     __ movzbq(Rdst, $mem$$Address);
4786     __ andl(Rdst, $mask$$constant & right_n_bits(8));
4787   %}
4788   ins_pipe(ialu_reg_mem);
4789 %}
4790 
4791 // Load Short (16 bit signed)
4792 instruct loadS(rRegI dst, memory mem)
4793 %{
4794   match(Set dst (LoadS mem));
4795 
4796   ins_cost(125);
4797   format %{ "movswl $dst, $mem\t# short" %}
4798 
4799   ins_encode %{
4800     __ movswl($dst$$Register, $mem$$Address);
4801   %}
4802 
4803   ins_pipe(ialu_reg_mem);
4804 %}
4805 
4806 // Load Short (16 bit signed) to Byte (8 bit signed)
4807 instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4808   match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
4809 
4810   ins_cost(125);
4811   format %{ "movsbl $dst, $mem\t# short -> byte" %}
4812   ins_encode %{
4813     __ movsbl($dst$$Register, $mem$$Address);
4814   %}
4815   ins_pipe(ialu_reg_mem);
4816 %}
4817 
4818 // Load Short (16 bit signed) into Long Register
4819 instruct loadS2L(rRegL dst, memory mem)
4820 %{
4821   match(Set dst (ConvI2L (LoadS mem)));
4822 
4823   ins_cost(125);
4824   format %{ "movswq $dst, $mem\t# short -> long" %}
4825 
4826   ins_encode %{
4827     __ movswq($dst$$Register, $mem$$Address);
4828   %}
4829 
4830   ins_pipe(ialu_reg_mem);
4831 %}
4832 
4833 // Load Unsigned Short/Char (16 bit UNsigned)
4834 instruct loadUS(rRegI dst, memory mem)
4835 %{
4836   match(Set dst (LoadUS mem));
4837 
4838   ins_cost(125);
4839   format %{ "movzwl  $dst, $mem\t# ushort/char" %}
4840 
4841   ins_encode %{
4842     __ movzwl($dst$$Register, $mem$$Address);
4843   %}
4844 
4845   ins_pipe(ialu_reg_mem);
4846 %}
4847 
4848 // Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
4849 instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4850   match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
4851 
4852   ins_cost(125);
4853   format %{ "movsbl $dst, $mem\t# ushort -> byte" %}
4854   ins_encode %{
4855     __ movsbl($dst$$Register, $mem$$Address);
4856   %}
4857   ins_pipe(ialu_reg_mem);
4858 %}
4859 
4860 // Load Unsigned Short/Char (16 bit UNsigned) into Long Register
4861 instruct loadUS2L(rRegL dst, memory mem)
4862 %{
4863   match(Set dst (ConvI2L (LoadUS mem)));
4864 
4865   ins_cost(125);
4866   format %{ "movzwq  $dst, $mem\t# ushort/char -> long" %}
4867 
4868   ins_encode %{
4869     __ movzwq($dst$$Register, $mem$$Address);
4870   %}
4871 
4872   ins_pipe(ialu_reg_mem);
4873 %}
4874 
4875 // Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
4876 instruct loadUS2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4877   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4878 
4879   format %{ "movzbq  $dst, $mem\t# ushort/char & 0xFF -> long" %}
4880   ins_encode %{
4881     __ movzbq($dst$$Register, $mem$$Address);
4882   %}
4883   ins_pipe(ialu_reg_mem);
4884 %}
4885 
4886 // Load Unsigned Short/Char (16 bit UNsigned) with 32-bit mask into Long Register
4887 instruct loadUS2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
4888   match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
4889   effect(KILL cr);
4890 
4891   format %{ "movzwq  $dst, $mem\t# ushort/char & 32-bit mask -> long\n\t"
4892             "andl    $dst, right_n_bits($mask, 16)" %}
4893   ins_encode %{
4894     Register Rdst = $dst$$Register;
4895     __ movzwq(Rdst, $mem$$Address);
4896     __ andl(Rdst, $mask$$constant & right_n_bits(16));
4897   %}
4898   ins_pipe(ialu_reg_mem);
4899 %}
4900 
4901 // Load Integer
4902 instruct loadI(rRegI dst, memory mem)
4903 %{
4904   match(Set dst (LoadI mem));
4905 
4906   ins_cost(125);
4907   format %{ "movl    $dst, $mem\t# int" %}
4908 
4909   ins_encode %{
4910     __ movl($dst$$Register, $mem$$Address);
4911   %}
4912 
4913   ins_pipe(ialu_reg_mem);
4914 %}
4915 
4916 // Load Integer (32 bit signed) to Byte (8 bit signed)
4917 instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
4918   match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
4919 
4920   ins_cost(125);
4921   format %{ "movsbl  $dst, $mem\t# int -> byte" %}
4922   ins_encode %{
4923     __ movsbl($dst$$Register, $mem$$Address);
4924   %}
4925   ins_pipe(ialu_reg_mem);
4926 %}
4927 
4928 // Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
4929 instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
4930   match(Set dst (AndI (LoadI mem) mask));
4931 
4932   ins_cost(125);
4933   format %{ "movzbl  $dst, $mem\t# int -> ubyte" %}
4934   ins_encode %{
4935     __ movzbl($dst$$Register, $mem$$Address);
4936   %}
4937   ins_pipe(ialu_reg_mem);
4938 %}
4939 
4940 // Load Integer (32 bit signed) to Short (16 bit signed)
4941 instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
4942   match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
4943 
4944   ins_cost(125);
4945   format %{ "movswl  $dst, $mem\t# int -> short" %}
4946   ins_encode %{
4947     __ movswl($dst$$Register, $mem$$Address);
4948   %}
4949   ins_pipe(ialu_reg_mem);
4950 %}
4951 
4952 // Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
4953 instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
4954   match(Set dst (AndI (LoadI mem) mask));
4955 
4956   ins_cost(125);
4957   format %{ "movzwl  $dst, $mem\t# int -> ushort/char" %}
4958   ins_encode %{
4959     __ movzwl($dst$$Register, $mem$$Address);
4960   %}
4961   ins_pipe(ialu_reg_mem);
4962 %}
4963 
4964 // Load Integer into Long Register
4965 instruct loadI2L(rRegL dst, memory mem)
4966 %{
4967   match(Set dst (ConvI2L (LoadI mem)));
4968 
4969   ins_cost(125);
4970   format %{ "movslq  $dst, $mem\t# int -> long" %}
4971 
4972   ins_encode %{
4973     __ movslq($dst$$Register, $mem$$Address);
4974   %}
4975 
4976   ins_pipe(ialu_reg_mem);
4977 %}
4978 
4979 // Load Integer with mask 0xFF into Long Register
4980 instruct loadI2L_immI_255(rRegL dst, memory mem, immI_255 mask) %{
4981   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4982 
4983   format %{ "movzbq  $dst, $mem\t# int & 0xFF -> long" %}
4984   ins_encode %{
4985     __ movzbq($dst$$Register, $mem$$Address);
4986   %}
4987   ins_pipe(ialu_reg_mem);
4988 %}
4989 
4990 // Load Integer with mask 0xFFFF into Long Register
4991 instruct loadI2L_immI_65535(rRegL dst, memory mem, immI_65535 mask) %{
4992   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
4993 
4994   format %{ "movzwq  $dst, $mem\t# int & 0xFFFF -> long" %}
4995   ins_encode %{
4996     __ movzwq($dst$$Register, $mem$$Address);
4997   %}
4998   ins_pipe(ialu_reg_mem);
4999 %}
5000 
5001 // Load Integer with a 31-bit mask into Long Register
5002 instruct loadI2L_immU31(rRegL dst, memory mem, immU31 mask, rFlagsReg cr) %{
5003   match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
5004   effect(KILL cr);
5005 
5006   format %{ "movl    $dst, $mem\t# int & 31-bit mask -> long\n\t"
5007             "andl    $dst, $mask" %}
5008   ins_encode %{
5009     Register Rdst = $dst$$Register;
5010     __ movl(Rdst, $mem$$Address);
5011     __ andl(Rdst, $mask$$constant);
5012   %}
5013   ins_pipe(ialu_reg_mem);
5014 %}
5015 
5016 // Load Unsigned Integer into Long Register
5017 instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask)
5018 %{
5019   match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
5020 
5021   ins_cost(125);
5022   format %{ "movl    $dst, $mem\t# uint -> long" %}
5023 
5024   ins_encode %{
5025     __ movl($dst$$Register, $mem$$Address);
5026   %}
5027 
5028   ins_pipe(ialu_reg_mem);
5029 %}
5030 
5031 // Load Long
5032 instruct loadL(rRegL dst, memory mem)
5033 %{
5034   match(Set dst (LoadL mem));
5035 
5036   ins_cost(125);
5037   format %{ "movq    $dst, $mem\t# long" %}
5038 
5039   ins_encode %{
5040     __ movq($dst$$Register, $mem$$Address);
5041   %}
5042 
5043   ins_pipe(ialu_reg_mem); // XXX
5044 %}
5045 
5046 // Load Range
5047 instruct loadRange(rRegI dst, memory mem)
5048 %{
5049   match(Set dst (LoadRange mem));
5050 
5051   ins_cost(125); // XXX
5052   format %{ "movl    $dst, $mem\t# range" %}
5053   opcode(0x8B);
5054   ins_encode(REX_reg_mem(dst, mem), OpcP, reg_mem(dst, mem));
5055   ins_pipe(ialu_reg_mem);
5056 %}
5057 
5058 // Load Pointer
5059 instruct loadP(rRegP dst, memory mem)
5060 %{
5061   match(Set dst (LoadP mem));
5062 
5063   ins_cost(125); // XXX
5064   format %{ "movq    $dst, $mem\t# ptr" %}
5065   opcode(0x8B);
5066   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5067   ins_pipe(ialu_reg_mem); // XXX
5068 %}
5069 
5070 // Load Compressed Pointer
5071 instruct loadN(rRegN dst, memory mem)
5072 %{
5073    match(Set dst (LoadN mem));
5074 
5075    ins_cost(125); // XXX
5076    format %{ "movl    $dst, $mem\t# compressed ptr" %}
5077    ins_encode %{
5078      __ movl($dst$$Register, $mem$$Address);
5079    %}
5080    ins_pipe(ialu_reg_mem); // XXX
5081 %}
5082 
5083 
5084 // Load Klass Pointer
5085 instruct loadKlass(rRegP dst, memory mem)
5086 %{
5087   match(Set dst (LoadKlass mem));
5088 
5089   ins_cost(125); // XXX
5090   format %{ "movq    $dst, $mem\t# class" %}
5091   opcode(0x8B);
5092   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5093   ins_pipe(ialu_reg_mem); // XXX
5094 %}
5095 
5096 // Load narrow Klass Pointer
5097 instruct loadNKlass(rRegN dst, memory mem)
5098 %{
5099   match(Set dst (LoadNKlass mem));
5100 
5101   ins_cost(125); // XXX
5102   format %{ "movl    $dst, $mem\t# compressed klass ptr" %}
5103   ins_encode %{
5104     __ movl($dst$$Register, $mem$$Address);
5105   %}
5106   ins_pipe(ialu_reg_mem); // XXX
5107 %}
5108 
5109 // Load Float
5110 instruct loadF(regF dst, memory mem)
5111 %{
5112   match(Set dst (LoadF mem));
5113 
5114   ins_cost(145); // XXX
5115   format %{ "movss   $dst, $mem\t# float" %}
5116   ins_encode %{
5117     __ movflt($dst$$XMMRegister, $mem$$Address);
5118   %}
5119   ins_pipe(pipe_slow); // XXX
5120 %}
5121 
5122 // Load Double
5123 instruct loadD_partial(regD dst, memory mem)
5124 %{
5125   predicate(!UseXmmLoadAndClearUpper);
5126   match(Set dst (LoadD mem));
5127 
5128   ins_cost(145); // XXX
5129   format %{ "movlpd  $dst, $mem\t# double" %}
5130   ins_encode %{
5131     __ movdbl($dst$$XMMRegister, $mem$$Address);
5132   %}
5133   ins_pipe(pipe_slow); // XXX
5134 %}
5135 
5136 instruct loadD(regD dst, memory mem)
5137 %{
5138   predicate(UseXmmLoadAndClearUpper);
5139   match(Set dst (LoadD mem));
5140 
5141   ins_cost(145); // XXX
5142   format %{ "movsd   $dst, $mem\t# double" %}
5143   ins_encode %{
5144     __ movdbl($dst$$XMMRegister, $mem$$Address);
5145   %}
5146   ins_pipe(pipe_slow); // XXX
5147 %}
5148 
5149 // Load Effective Address
5150 instruct leaP8(rRegP dst, indOffset8 mem)
5151 %{
5152   match(Set dst mem);
5153 
5154   ins_cost(110); // XXX
5155   format %{ "leaq    $dst, $mem\t# ptr 8" %}
5156   opcode(0x8D);
5157   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5158   ins_pipe(ialu_reg_reg_fat);
5159 %}
5160 
5161 instruct leaP32(rRegP dst, indOffset32 mem)
5162 %{
5163   match(Set dst mem);
5164 
5165   ins_cost(110);
5166   format %{ "leaq    $dst, $mem\t# ptr 32" %}
5167   opcode(0x8D);
5168   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5169   ins_pipe(ialu_reg_reg_fat);
5170 %}
5171 
5172 // instruct leaPIdx(rRegP dst, indIndex mem)
5173 // %{
5174 //   match(Set dst mem);
5175 
5176 //   ins_cost(110);
5177 //   format %{ "leaq    $dst, $mem\t# ptr idx" %}
5178 //   opcode(0x8D);
5179 //   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5180 //   ins_pipe(ialu_reg_reg_fat);
5181 // %}
5182 
5183 instruct leaPIdxOff(rRegP dst, indIndexOffset mem)
5184 %{
5185   match(Set dst mem);
5186 
5187   ins_cost(110);
5188   format %{ "leaq    $dst, $mem\t# ptr idxoff" %}
5189   opcode(0x8D);
5190   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5191   ins_pipe(ialu_reg_reg_fat);
5192 %}
5193 
5194 instruct leaPIdxScale(rRegP dst, indIndexScale mem)
5195 %{
5196   match(Set dst mem);
5197 
5198   ins_cost(110);
5199   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5200   opcode(0x8D);
5201   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5202   ins_pipe(ialu_reg_reg_fat);
5203 %}
5204 
5205 instruct leaPPosIdxScale(rRegP dst, indPosIndexScale mem)
5206 %{
5207   match(Set dst mem);
5208 
5209   ins_cost(110);
5210   format %{ "leaq    $dst, $mem\t# ptr idxscale" %}
5211   opcode(0x8D);
5212   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5213   ins_pipe(ialu_reg_reg_fat);
5214 %}
5215 
5216 instruct leaPIdxScaleOff(rRegP dst, indIndexScaleOffset mem)
5217 %{
5218   match(Set dst mem);
5219 
5220   ins_cost(110);
5221   format %{ "leaq    $dst, $mem\t# ptr idxscaleoff" %}
5222   opcode(0x8D);
5223   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5224   ins_pipe(ialu_reg_reg_fat);
5225 %}
5226 
5227 instruct leaPPosIdxOff(rRegP dst, indPosIndexOffset mem)
5228 %{
5229   match(Set dst mem);
5230 
5231   ins_cost(110);
5232   format %{ "leaq    $dst, $mem\t# ptr posidxoff" %}
5233   opcode(0x8D);
5234   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5235   ins_pipe(ialu_reg_reg_fat);
5236 %}
5237 
5238 instruct leaPPosIdxScaleOff(rRegP dst, indPosIndexScaleOffset mem)
5239 %{
5240   match(Set dst mem);
5241 
5242   ins_cost(110);
5243   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoff" %}
5244   opcode(0x8D);
5245   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5246   ins_pipe(ialu_reg_reg_fat);
5247 %}
5248 
5249 // Load Effective Address which uses Narrow (32-bits) oop
5250 instruct leaPCompressedOopOffset(rRegP dst, indCompressedOopOffset mem)
5251 %{
5252   predicate(UseCompressedOops && (Universe::narrow_oop_shift() != 0));
5253   match(Set dst mem);
5254 
5255   ins_cost(110);
5256   format %{ "leaq    $dst, $mem\t# ptr compressedoopoff32" %}
5257   opcode(0x8D);
5258   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5259   ins_pipe(ialu_reg_reg_fat);
5260 %}
5261 
5262 instruct leaP8Narrow(rRegP dst, indOffset8Narrow mem)
5263 %{
5264   predicate(Universe::narrow_oop_shift() == 0);
5265   match(Set dst mem);
5266 
5267   ins_cost(110); // XXX
5268   format %{ "leaq    $dst, $mem\t# ptr off8narrow" %}
5269   opcode(0x8D);
5270   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5271   ins_pipe(ialu_reg_reg_fat);
5272 %}
5273 
5274 instruct leaP32Narrow(rRegP dst, indOffset32Narrow mem)
5275 %{
5276   predicate(Universe::narrow_oop_shift() == 0);
5277   match(Set dst mem);
5278 
5279   ins_cost(110);
5280   format %{ "leaq    $dst, $mem\t# ptr off32narrow" %}
5281   opcode(0x8D);
5282   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5283   ins_pipe(ialu_reg_reg_fat);
5284 %}
5285 
5286 instruct leaPIdxOffNarrow(rRegP dst, indIndexOffsetNarrow mem)
5287 %{
5288   predicate(Universe::narrow_oop_shift() == 0);
5289   match(Set dst mem);
5290 
5291   ins_cost(110);
5292   format %{ "leaq    $dst, $mem\t# ptr idxoffnarrow" %}
5293   opcode(0x8D);
5294   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5295   ins_pipe(ialu_reg_reg_fat);
5296 %}
5297 
5298 instruct leaPIdxScaleNarrow(rRegP dst, indIndexScaleNarrow mem)
5299 %{
5300   predicate(Universe::narrow_oop_shift() == 0);
5301   match(Set dst mem);
5302 
5303   ins_cost(110);
5304   format %{ "leaq    $dst, $mem\t# ptr idxscalenarrow" %}
5305   opcode(0x8D);
5306   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5307   ins_pipe(ialu_reg_reg_fat);
5308 %}
5309 
5310 instruct leaPIdxScaleOffNarrow(rRegP dst, indIndexScaleOffsetNarrow mem)
5311 %{
5312   predicate(Universe::narrow_oop_shift() == 0);
5313   match(Set dst mem);
5314 
5315   ins_cost(110);
5316   format %{ "leaq    $dst, $mem\t# ptr idxscaleoffnarrow" %}
5317   opcode(0x8D);
5318   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5319   ins_pipe(ialu_reg_reg_fat);
5320 %}
5321 
5322 instruct leaPPosIdxOffNarrow(rRegP dst, indPosIndexOffsetNarrow mem)
5323 %{
5324   predicate(Universe::narrow_oop_shift() == 0);
5325   match(Set dst mem);
5326 
5327   ins_cost(110);
5328   format %{ "leaq    $dst, $mem\t# ptr posidxoffnarrow" %}
5329   opcode(0x8D);
5330   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5331   ins_pipe(ialu_reg_reg_fat);
5332 %}
5333 
5334 instruct leaPPosIdxScaleOffNarrow(rRegP dst, indPosIndexScaleOffsetNarrow mem)
5335 %{
5336   predicate(Universe::narrow_oop_shift() == 0);
5337   match(Set dst mem);
5338 
5339   ins_cost(110);
5340   format %{ "leaq    $dst, $mem\t# ptr posidxscaleoffnarrow" %}
5341   opcode(0x8D);
5342   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
5343   ins_pipe(ialu_reg_reg_fat);
5344 %}
5345 
5346 instruct loadConI(rRegI dst, immI src)
5347 %{
5348   match(Set dst src);
5349 
5350   format %{ "movl    $dst, $src\t# int" %}
5351   ins_encode(load_immI(dst, src));
5352   ins_pipe(ialu_reg_fat); // XXX
5353 %}
5354 
5355 instruct loadConI0(rRegI dst, immI0 src, rFlagsReg cr)
5356 %{
5357   match(Set dst src);
5358   effect(KILL cr);
5359 
5360   ins_cost(50);
5361   format %{ "xorl    $dst, $dst\t# int" %}
5362   opcode(0x33); /* + rd */
5363   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5364   ins_pipe(ialu_reg);
5365 %}
5366 
5367 instruct loadConL(rRegL dst, immL src)
5368 %{
5369   match(Set dst src);
5370 
5371   ins_cost(150);
5372   format %{ "movq    $dst, $src\t# long" %}
5373   ins_encode(load_immL(dst, src));
5374   ins_pipe(ialu_reg);
5375 %}
5376 
5377 instruct loadConL0(rRegL dst, immL0 src, rFlagsReg cr)
5378 %{
5379   match(Set dst src);
5380   effect(KILL cr);
5381 
5382   ins_cost(50);
5383   format %{ "xorl    $dst, $dst\t# long" %}
5384   opcode(0x33); /* + rd */
5385   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5386   ins_pipe(ialu_reg); // XXX
5387 %}
5388 
5389 instruct loadConUL32(rRegL dst, immUL32 src)
5390 %{
5391   match(Set dst src);
5392 
5393   ins_cost(60);
5394   format %{ "movl    $dst, $src\t# long (unsigned 32-bit)" %}
5395   ins_encode(load_immUL32(dst, src));
5396   ins_pipe(ialu_reg);
5397 %}
5398 
5399 instruct loadConL32(rRegL dst, immL32 src)
5400 %{
5401   match(Set dst src);
5402 
5403   ins_cost(70);
5404   format %{ "movq    $dst, $src\t# long (32-bit)" %}
5405   ins_encode(load_immL32(dst, src));
5406   ins_pipe(ialu_reg);
5407 %}
5408 
5409 instruct loadConP(rRegP dst, immP con) %{
5410   match(Set dst con);
5411 
5412   format %{ "movq    $dst, $con\t# ptr" %}
5413   ins_encode(load_immP(dst, con));
5414   ins_pipe(ialu_reg_fat); // XXX
5415 %}
5416 
5417 instruct loadConP0(rRegP dst, immP0 src, rFlagsReg cr)
5418 %{
5419   match(Set dst src);
5420   effect(KILL cr);
5421 
5422   ins_cost(50);
5423   format %{ "xorl    $dst, $dst\t# ptr" %}
5424   opcode(0x33); /* + rd */
5425   ins_encode(REX_reg_reg(dst, dst), OpcP, reg_reg(dst, dst));
5426   ins_pipe(ialu_reg);
5427 %}
5428 
5429 instruct loadConP31(rRegP dst, immP31 src, rFlagsReg cr)
5430 %{
5431   match(Set dst src);
5432   effect(KILL cr);
5433 
5434   ins_cost(60);
5435   format %{ "movl    $dst, $src\t# ptr (positive 32-bit)" %}
5436   ins_encode(load_immP31(dst, src));
5437   ins_pipe(ialu_reg);
5438 %}
5439 
5440 instruct loadConF(regF dst, immF con) %{
5441   match(Set dst con);
5442   ins_cost(125);
5443   format %{ "movss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
5444   ins_encode %{
5445     __ movflt($dst$$XMMRegister, $constantaddress($con));
5446   %}
5447   ins_pipe(pipe_slow);
5448 %}
5449 
5450 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
5451   match(Set dst src);
5452   effect(KILL cr);
5453   format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
5454   ins_encode %{
5455     __ xorq($dst$$Register, $dst$$Register);
5456   %}
5457   ins_pipe(ialu_reg);
5458 %}
5459 
5460 instruct loadConN(rRegN dst, immN src) %{
5461   match(Set dst src);
5462 
5463   ins_cost(125);
5464   format %{ "movl    $dst, $src\t# compressed ptr" %}
5465   ins_encode %{
5466     address con = (address)$src$$constant;
5467     if (con == NULL) {
5468       ShouldNotReachHere();
5469     } else {
5470       __ set_narrow_oop($dst$$Register, (jobject)$src$$constant);
5471     }
5472   %}
5473   ins_pipe(ialu_reg_fat); // XXX
5474 %}
5475 
5476 instruct loadConNKlass(rRegN dst, immNKlass src) %{
5477   match(Set dst src);
5478 
5479   ins_cost(125);
5480   format %{ "movl    $dst, $src\t# compressed klass ptr" %}
5481   ins_encode %{
5482     address con = (address)$src$$constant;
5483     if (con == NULL) {
5484       ShouldNotReachHere();
5485     } else {
5486       __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
5487     }
5488   %}
5489   ins_pipe(ialu_reg_fat); // XXX
5490 %}
5491 
5492 instruct loadConF0(regF dst, immF0 src)
5493 %{
5494   match(Set dst src);
5495   ins_cost(100);
5496 
5497   format %{ "xorps   $dst, $dst\t# float 0.0" %}
5498   ins_encode %{
5499     __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
5500   %}
5501   ins_pipe(pipe_slow);
5502 %}
5503 
5504 // Use the same format since predicate() can not be used here.
5505 instruct loadConD(regD dst, immD con) %{
5506   match(Set dst con);
5507   ins_cost(125);
5508   format %{ "movsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
5509   ins_encode %{
5510     __ movdbl($dst$$XMMRegister, $constantaddress($con));
5511   %}
5512   ins_pipe(pipe_slow);
5513 %}
5514 
5515 instruct loadConD0(regD dst, immD0 src)
5516 %{
5517   match(Set dst src);
5518   ins_cost(100);
5519 
5520   format %{ "xorpd   $dst, $dst\t# double 0.0" %}
5521   ins_encode %{
5522     __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
5523   %}
5524   ins_pipe(pipe_slow);
5525 %}
5526 
5527 instruct loadSSI(rRegI dst, stackSlotI src)
5528 %{
5529   match(Set dst src);
5530 
5531   ins_cost(125);
5532   format %{ "movl    $dst, $src\t# int stk" %}
5533   opcode(0x8B);
5534   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
5535   ins_pipe(ialu_reg_mem);
5536 %}
5537 
5538 instruct loadSSL(rRegL dst, stackSlotL src)
5539 %{
5540   match(Set dst src);
5541 
5542   ins_cost(125);
5543   format %{ "movq    $dst, $src\t# long stk" %}
5544   opcode(0x8B);
5545   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5546   ins_pipe(ialu_reg_mem);
5547 %}
5548 
5549 instruct loadSSP(rRegP dst, stackSlotP src)
5550 %{
5551   match(Set dst src);
5552 
5553   ins_cost(125);
5554   format %{ "movq    $dst, $src\t# ptr stk" %}
5555   opcode(0x8B);
5556   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
5557   ins_pipe(ialu_reg_mem);
5558 %}
5559 
5560 instruct loadSSF(regF dst, stackSlotF src)
5561 %{
5562   match(Set dst src);
5563 
5564   ins_cost(125);
5565   format %{ "movss   $dst, $src\t# float stk" %}
5566   ins_encode %{
5567     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
5568   %}
5569   ins_pipe(pipe_slow); // XXX
5570 %}
5571 
5572 // Use the same format since predicate() can not be used here.
5573 instruct loadSSD(regD dst, stackSlotD src)
5574 %{
5575   match(Set dst src);
5576 
5577   ins_cost(125);
5578   format %{ "movsd   $dst, $src\t# double stk" %}
5579   ins_encode  %{
5580     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
5581   %}
5582   ins_pipe(pipe_slow); // XXX
5583 %}
5584 
5585 // Prefetch instructions for allocation.
5586 // Must be safe to execute with invalid address (cannot fault).
5587 
5588 instruct prefetchAlloc( memory mem ) %{
5589   predicate(AllocatePrefetchInstr==3);
5590   match(PrefetchAllocation mem);
5591   ins_cost(125);
5592 
5593   format %{ "PREFETCHW $mem\t# Prefetch allocation into level 1 cache and mark modified" %}
5594   ins_encode %{
5595     __ prefetchw($mem$$Address);
5596   %}
5597   ins_pipe(ialu_mem);
5598 %}
5599 
5600 instruct prefetchAllocNTA( memory mem ) %{
5601   predicate(AllocatePrefetchInstr==0);
5602   match(PrefetchAllocation mem);
5603   ins_cost(125);
5604 
5605   format %{ "PREFETCHNTA $mem\t# Prefetch allocation to non-temporal cache for write" %}
5606   ins_encode %{
5607     __ prefetchnta($mem$$Address);
5608   %}
5609   ins_pipe(ialu_mem);
5610 %}
5611 
5612 instruct prefetchAllocT0( memory mem ) %{
5613   predicate(AllocatePrefetchInstr==1);
5614   match(PrefetchAllocation mem);
5615   ins_cost(125);
5616 
5617   format %{ "PREFETCHT0 $mem\t# Prefetch allocation to level 1 and 2 caches for write" %}
5618   ins_encode %{
5619     __ prefetcht0($mem$$Address);
5620   %}
5621   ins_pipe(ialu_mem);
5622 %}
5623 
5624 instruct prefetchAllocT2( memory mem ) %{
5625   predicate(AllocatePrefetchInstr==2);
5626   match(PrefetchAllocation mem);
5627   ins_cost(125);
5628 
5629   format %{ "PREFETCHT2 $mem\t# Prefetch allocation to level 2 cache for write" %}
5630   ins_encode %{
5631     __ prefetcht2($mem$$Address);
5632   %}
5633   ins_pipe(ialu_mem);
5634 %}
5635 
5636 //----------Store Instructions-------------------------------------------------
5637 
5638 // Store Byte
5639 instruct storeB(memory mem, rRegI src)
5640 %{
5641   match(Set mem (StoreB mem src));
5642 
5643   ins_cost(125); // XXX
5644   format %{ "movb    $mem, $src\t# byte" %}
5645   opcode(0x88);
5646   ins_encode(REX_breg_mem(src, mem), OpcP, reg_mem(src, mem));
5647   ins_pipe(ialu_mem_reg);
5648 %}
5649 
5650 // Store Char/Short
5651 instruct storeC(memory mem, rRegI src)
5652 %{
5653   match(Set mem (StoreC mem src));
5654 
5655   ins_cost(125); // XXX
5656   format %{ "movw    $mem, $src\t# char/short" %}
5657   opcode(0x89);
5658   ins_encode(SizePrefix, REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5659   ins_pipe(ialu_mem_reg);
5660 %}
5661 
5662 // Store Integer
5663 instruct storeI(memory mem, rRegI src)
5664 %{
5665   match(Set mem (StoreI mem src));
5666 
5667   ins_cost(125); // XXX
5668   format %{ "movl    $mem, $src\t# int" %}
5669   opcode(0x89);
5670   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
5671   ins_pipe(ialu_mem_reg);
5672 %}
5673 
5674 // Store Long
5675 instruct storeL(memory mem, rRegL src)
5676 %{
5677   match(Set mem (StoreL mem src));
5678 
5679   ins_cost(125); // XXX
5680   format %{ "movq    $mem, $src\t# long" %}
5681   opcode(0x89);
5682   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5683   ins_pipe(ialu_mem_reg); // XXX
5684 %}
5685 
5686 // Store Pointer
5687 instruct storeP(memory mem, any_RegP src)
5688 %{
5689   match(Set mem (StoreP mem src));
5690 
5691   ins_cost(125); // XXX
5692   format %{ "movq    $mem, $src\t# ptr" %}
5693   opcode(0x89);
5694   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
5695   ins_pipe(ialu_mem_reg);
5696 %}
5697 
5698 instruct storeImmP0(memory mem, immP0 zero)
5699 %{
5700   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5701   match(Set mem (StoreP mem zero));
5702 
5703   ins_cost(125); // XXX
5704   format %{ "movq    $mem, R12\t# ptr (R12_heapbase==0)" %}
5705   ins_encode %{
5706     __ movq($mem$$Address, r12);
5707   %}
5708   ins_pipe(ialu_mem_reg);
5709 %}
5710 
5711 // Store NULL Pointer, mark word, or other simple pointer constant.
5712 instruct storeImmP(memory mem, immP31 src)
5713 %{
5714   match(Set mem (StoreP mem src));
5715 
5716   ins_cost(150); // XXX
5717   format %{ "movq    $mem, $src\t# ptr" %}
5718   opcode(0xC7); /* C7 /0 */
5719   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5720   ins_pipe(ialu_mem_imm);
5721 %}
5722 
5723 // Store Compressed Pointer
5724 instruct storeN(memory mem, rRegN src)
5725 %{
5726   match(Set mem (StoreN mem src));
5727 
5728   ins_cost(125); // XXX
5729   format %{ "movl    $mem, $src\t# compressed ptr" %}
5730   ins_encode %{
5731     __ movl($mem$$Address, $src$$Register);
5732   %}
5733   ins_pipe(ialu_mem_reg);
5734 %}
5735 
5736 instruct storeNKlass(memory mem, rRegN src)
5737 %{
5738   match(Set mem (StoreNKlass mem src));
5739 
5740   ins_cost(125); // XXX
5741   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5742   ins_encode %{
5743     __ movl($mem$$Address, $src$$Register);
5744   %}
5745   ins_pipe(ialu_mem_reg);
5746 %}
5747 
5748 instruct storeImmN0(memory mem, immN0 zero)
5749 %{
5750   predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
5751   match(Set mem (StoreN mem zero));
5752 
5753   ins_cost(125); // XXX
5754   format %{ "movl    $mem, R12\t# compressed ptr (R12_heapbase==0)" %}
5755   ins_encode %{
5756     __ movl($mem$$Address, r12);
5757   %}
5758   ins_pipe(ialu_mem_reg);
5759 %}
5760 
5761 instruct storeImmN(memory mem, immN src)
5762 %{
5763   match(Set mem (StoreN mem src));
5764 
5765   ins_cost(150); // XXX
5766   format %{ "movl    $mem, $src\t# compressed ptr" %}
5767   ins_encode %{
5768     address con = (address)$src$$constant;
5769     if (con == NULL) {
5770       __ movl($mem$$Address, (int32_t)0);
5771     } else {
5772       __ set_narrow_oop($mem$$Address, (jobject)$src$$constant);
5773     }
5774   %}
5775   ins_pipe(ialu_mem_imm);
5776 %}
5777 
5778 instruct storeImmNKlass(memory mem, immNKlass src)
5779 %{
5780   match(Set mem (StoreNKlass mem src));
5781 
5782   ins_cost(150); // XXX
5783   format %{ "movl    $mem, $src\t# compressed klass ptr" %}
5784   ins_encode %{
5785     __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
5786   %}
5787   ins_pipe(ialu_mem_imm);
5788 %}
5789 
5790 // Store Integer Immediate
5791 instruct storeImmI0(memory mem, immI0 zero)
5792 %{
5793   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5794   match(Set mem (StoreI mem zero));
5795 
5796   ins_cost(125); // XXX
5797   format %{ "movl    $mem, R12\t# int (R12_heapbase==0)" %}
5798   ins_encode %{
5799     __ movl($mem$$Address, r12);
5800   %}
5801   ins_pipe(ialu_mem_reg);
5802 %}
5803 
5804 instruct storeImmI(memory mem, immI src)
5805 %{
5806   match(Set mem (StoreI mem src));
5807 
5808   ins_cost(150);
5809   format %{ "movl    $mem, $src\t# int" %}
5810   opcode(0xC7); /* C7 /0 */
5811   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5812   ins_pipe(ialu_mem_imm);
5813 %}
5814 
5815 // Store Long Immediate
5816 instruct storeImmL0(memory mem, immL0 zero)
5817 %{
5818   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5819   match(Set mem (StoreL mem zero));
5820 
5821   ins_cost(125); // XXX
5822   format %{ "movq    $mem, R12\t# long (R12_heapbase==0)" %}
5823   ins_encode %{
5824     __ movq($mem$$Address, r12);
5825   %}
5826   ins_pipe(ialu_mem_reg);
5827 %}
5828 
5829 instruct storeImmL(memory mem, immL32 src)
5830 %{
5831   match(Set mem (StoreL mem src));
5832 
5833   ins_cost(150);
5834   format %{ "movq    $mem, $src\t# long" %}
5835   opcode(0xC7); /* C7 /0 */
5836   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32(src));
5837   ins_pipe(ialu_mem_imm);
5838 %}
5839 
5840 // Store Short/Char Immediate
5841 instruct storeImmC0(memory mem, immI0 zero)
5842 %{
5843   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5844   match(Set mem (StoreC mem zero));
5845 
5846   ins_cost(125); // XXX
5847   format %{ "movw    $mem, R12\t# short/char (R12_heapbase==0)" %}
5848   ins_encode %{
5849     __ movw($mem$$Address, r12);
5850   %}
5851   ins_pipe(ialu_mem_reg);
5852 %}
5853 
5854 instruct storeImmI16(memory mem, immI16 src)
5855 %{
5856   predicate(UseStoreImmI16);
5857   match(Set mem (StoreC mem src));
5858 
5859   ins_cost(150);
5860   format %{ "movw    $mem, $src\t# short/char" %}
5861   opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
5862   ins_encode(SizePrefix, REX_mem(mem), OpcP, RM_opc_mem(0x00, mem),Con16(src));
5863   ins_pipe(ialu_mem_imm);
5864 %}
5865 
5866 // Store Byte Immediate
5867 instruct storeImmB0(memory mem, immI0 zero)
5868 %{
5869   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5870   match(Set mem (StoreB mem zero));
5871 
5872   ins_cost(125); // XXX
5873   format %{ "movb    $mem, R12\t# short/char (R12_heapbase==0)" %}
5874   ins_encode %{
5875     __ movb($mem$$Address, r12);
5876   %}
5877   ins_pipe(ialu_mem_reg);
5878 %}
5879 
5880 instruct storeImmB(memory mem, immI8 src)
5881 %{
5882   match(Set mem (StoreB mem src));
5883 
5884   ins_cost(150); // XXX
5885   format %{ "movb    $mem, $src\t# byte" %}
5886   opcode(0xC6); /* C6 /0 */
5887   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5888   ins_pipe(ialu_mem_imm);
5889 %}
5890 
5891 // Store CMS card-mark Immediate
5892 instruct storeImmCM0_reg(memory mem, immI0 zero)
5893 %{
5894   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5895   match(Set mem (StoreCM mem zero));
5896 
5897   ins_cost(125); // XXX
5898   format %{ "movb    $mem, R12\t# CMS card-mark byte 0 (R12_heapbase==0)" %}
5899   ins_encode %{
5900     __ movb($mem$$Address, r12);
5901   %}
5902   ins_pipe(ialu_mem_reg);
5903 %}
5904 
5905 instruct storeImmCM0(memory mem, immI0 src)
5906 %{
5907   match(Set mem (StoreCM mem src));
5908 
5909   ins_cost(150); // XXX
5910   format %{ "movb    $mem, $src\t# CMS card-mark byte 0" %}
5911   opcode(0xC6); /* C6 /0 */
5912   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con8or32(src));
5913   ins_pipe(ialu_mem_imm);
5914 %}
5915 
5916 // Store Float
5917 instruct storeF(memory mem, regF src)
5918 %{
5919   match(Set mem (StoreF mem src));
5920 
5921   ins_cost(95); // XXX
5922   format %{ "movss   $mem, $src\t# float" %}
5923   ins_encode %{
5924     __ movflt($mem$$Address, $src$$XMMRegister);
5925   %}
5926   ins_pipe(pipe_slow); // XXX
5927 %}
5928 
5929 // Store immediate Float value (it is faster than store from XMM register)
5930 instruct storeF0(memory mem, immF0 zero)
5931 %{
5932   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5933   match(Set mem (StoreF mem zero));
5934 
5935   ins_cost(25); // XXX
5936   format %{ "movl    $mem, R12\t# float 0. (R12_heapbase==0)" %}
5937   ins_encode %{
5938     __ movl($mem$$Address, r12);
5939   %}
5940   ins_pipe(ialu_mem_reg);
5941 %}
5942 
5943 instruct storeF_imm(memory mem, immF src)
5944 %{
5945   match(Set mem (StoreF mem src));
5946 
5947   ins_cost(50);
5948   format %{ "movl    $mem, $src\t# float" %}
5949   opcode(0xC7); /* C7 /0 */
5950   ins_encode(REX_mem(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5951   ins_pipe(ialu_mem_imm);
5952 %}
5953 
5954 // Store Double
5955 instruct storeD(memory mem, regD src)
5956 %{
5957   match(Set mem (StoreD mem src));
5958 
5959   ins_cost(95); // XXX
5960   format %{ "movsd   $mem, $src\t# double" %}
5961   ins_encode %{
5962     __ movdbl($mem$$Address, $src$$XMMRegister);
5963   %}
5964   ins_pipe(pipe_slow); // XXX
5965 %}
5966 
5967 // Store immediate double 0.0 (it is faster than store from XMM register)
5968 instruct storeD0_imm(memory mem, immD0 src)
5969 %{
5970   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
5971   match(Set mem (StoreD mem src));
5972 
5973   ins_cost(50);
5974   format %{ "movq    $mem, $src\t# double 0." %}
5975   opcode(0xC7); /* C7 /0 */
5976   ins_encode(REX_mem_wide(mem), OpcP, RM_opc_mem(0x00, mem), Con32F_as_bits(src));
5977   ins_pipe(ialu_mem_imm);
5978 %}
5979 
5980 instruct storeD0(memory mem, immD0 zero)
5981 %{
5982   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
5983   match(Set mem (StoreD mem zero));
5984 
5985   ins_cost(25); // XXX
5986   format %{ "movq    $mem, R12\t# double 0. (R12_heapbase==0)" %}
5987   ins_encode %{
5988     __ movq($mem$$Address, r12);
5989   %}
5990   ins_pipe(ialu_mem_reg);
5991 %}
5992 
5993 instruct storeSSI(stackSlotI dst, rRegI src)
5994 %{
5995   match(Set dst src);
5996 
5997   ins_cost(100);
5998   format %{ "movl    $dst, $src\t# int stk" %}
5999   opcode(0x89);
6000   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6001   ins_pipe( ialu_mem_reg );
6002 %}
6003 
6004 instruct storeSSL(stackSlotL dst, rRegL src)
6005 %{
6006   match(Set dst src);
6007 
6008   ins_cost(100);
6009   format %{ "movq    $dst, $src\t# long stk" %}
6010   opcode(0x89);
6011   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6012   ins_pipe(ialu_mem_reg);
6013 %}
6014 
6015 instruct storeSSP(stackSlotP dst, rRegP src)
6016 %{
6017   match(Set dst src);
6018 
6019   ins_cost(100);
6020   format %{ "movq    $dst, $src\t# ptr stk" %}
6021   opcode(0x89);
6022   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
6023   ins_pipe(ialu_mem_reg);
6024 %}
6025 
6026 instruct storeSSF(stackSlotF dst, regF src)
6027 %{
6028   match(Set dst src);
6029 
6030   ins_cost(95); // XXX
6031   format %{ "movss   $dst, $src\t# float stk" %}
6032   ins_encode %{
6033     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
6034   %}
6035   ins_pipe(pipe_slow); // XXX
6036 %}
6037 
6038 instruct storeSSD(stackSlotD dst, regD src)
6039 %{
6040   match(Set dst src);
6041 
6042   ins_cost(95); // XXX
6043   format %{ "movsd   $dst, $src\t# double stk" %}
6044   ins_encode %{
6045     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
6046   %}
6047   ins_pipe(pipe_slow); // XXX
6048 %}
6049 
6050 //----------BSWAP Instructions-------------------------------------------------
6051 instruct bytes_reverse_int(rRegI dst) %{
6052   match(Set dst (ReverseBytesI dst));
6053 
6054   format %{ "bswapl  $dst" %}
6055   opcode(0x0F, 0xC8);  /*Opcode 0F /C8 */
6056   ins_encode( REX_reg(dst), OpcP, opc2_reg(dst) );
6057   ins_pipe( ialu_reg );
6058 %}
6059 
6060 instruct bytes_reverse_long(rRegL dst) %{
6061   match(Set dst (ReverseBytesL dst));
6062 
6063   format %{ "bswapq  $dst" %}
6064   opcode(0x0F, 0xC8); /* Opcode 0F /C8 */
6065   ins_encode( REX_reg_wide(dst), OpcP, opc2_reg(dst) );
6066   ins_pipe( ialu_reg);
6067 %}
6068 
6069 instruct bytes_reverse_unsigned_short(rRegI dst, rFlagsReg cr) %{
6070   match(Set dst (ReverseBytesUS dst));
6071   effect(KILL cr);
6072 
6073   format %{ "bswapl  $dst\n\t"
6074             "shrl    $dst,16\n\t" %}
6075   ins_encode %{
6076     __ bswapl($dst$$Register);
6077     __ shrl($dst$$Register, 16);
6078   %}
6079   ins_pipe( ialu_reg );
6080 %}
6081 
6082 instruct bytes_reverse_short(rRegI dst, rFlagsReg cr) %{
6083   match(Set dst (ReverseBytesS dst));
6084   effect(KILL cr);
6085 
6086   format %{ "bswapl  $dst\n\t"
6087             "sar     $dst,16\n\t" %}
6088   ins_encode %{
6089     __ bswapl($dst$$Register);
6090     __ sarl($dst$$Register, 16);
6091   %}
6092   ins_pipe( ialu_reg );
6093 %}
6094 
6095 //---------- Zeros Count Instructions ------------------------------------------
6096 
6097 instruct countLeadingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6098   predicate(UseCountLeadingZerosInstruction);
6099   match(Set dst (CountLeadingZerosI src));
6100   effect(KILL cr);
6101 
6102   format %{ "lzcntl  $dst, $src\t# count leading zeros (int)" %}
6103   ins_encode %{
6104     __ lzcntl($dst$$Register, $src$$Register);
6105   %}
6106   ins_pipe(ialu_reg);
6107 %}
6108 
6109 instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, rFlagsReg cr) %{
6110   predicate(!UseCountLeadingZerosInstruction);
6111   match(Set dst (CountLeadingZerosI src));
6112   effect(KILL cr);
6113 
6114   format %{ "bsrl    $dst, $src\t# count leading zeros (int)\n\t"
6115             "jnz     skip\n\t"
6116             "movl    $dst, -1\n"
6117       "skip:\n\t"
6118             "negl    $dst\n\t"
6119             "addl    $dst, 31" %}
6120   ins_encode %{
6121     Register Rdst = $dst$$Register;
6122     Register Rsrc = $src$$Register;
6123     Label skip;
6124     __ bsrl(Rdst, Rsrc);
6125     __ jccb(Assembler::notZero, skip);
6126     __ movl(Rdst, -1);
6127     __ bind(skip);
6128     __ negl(Rdst);
6129     __ addl(Rdst, BitsPerInt - 1);
6130   %}
6131   ins_pipe(ialu_reg);
6132 %}
6133 
6134 instruct countLeadingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6135   predicate(UseCountLeadingZerosInstruction);
6136   match(Set dst (CountLeadingZerosL src));
6137   effect(KILL cr);
6138 
6139   format %{ "lzcntq  $dst, $src\t# count leading zeros (long)" %}
6140   ins_encode %{
6141     __ lzcntq($dst$$Register, $src$$Register);
6142   %}
6143   ins_pipe(ialu_reg);
6144 %}
6145 
6146 instruct countLeadingZerosL_bsr(rRegI dst, rRegL src, rFlagsReg cr) %{
6147   predicate(!UseCountLeadingZerosInstruction);
6148   match(Set dst (CountLeadingZerosL src));
6149   effect(KILL cr);
6150 
6151   format %{ "bsrq    $dst, $src\t# count leading zeros (long)\n\t"
6152             "jnz     skip\n\t"
6153             "movl    $dst, -1\n"
6154       "skip:\n\t"
6155             "negl    $dst\n\t"
6156             "addl    $dst, 63" %}
6157   ins_encode %{
6158     Register Rdst = $dst$$Register;
6159     Register Rsrc = $src$$Register;
6160     Label skip;
6161     __ bsrq(Rdst, Rsrc);
6162     __ jccb(Assembler::notZero, skip);
6163     __ movl(Rdst, -1);
6164     __ bind(skip);
6165     __ negl(Rdst);
6166     __ addl(Rdst, BitsPerLong - 1);
6167   %}
6168   ins_pipe(ialu_reg);
6169 %}
6170 
6171 instruct countTrailingZerosI(rRegI dst, rRegI src, rFlagsReg cr) %{
6172   predicate(UseCountTrailingZerosInstruction);
6173   match(Set dst (CountTrailingZerosI src));
6174   effect(KILL cr);
6175 
6176   format %{ "tzcntl    $dst, $src\t# count trailing zeros (int)" %}
6177   ins_encode %{
6178     __ tzcntl($dst$$Register, $src$$Register);
6179   %}
6180   ins_pipe(ialu_reg);
6181 %}
6182 
6183 instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, rFlagsReg cr) %{
6184   predicate(!UseCountTrailingZerosInstruction);
6185   match(Set dst (CountTrailingZerosI src));
6186   effect(KILL cr);
6187 
6188   format %{ "bsfl    $dst, $src\t# count trailing zeros (int)\n\t"
6189             "jnz     done\n\t"
6190             "movl    $dst, 32\n"
6191       "done:" %}
6192   ins_encode %{
6193     Register Rdst = $dst$$Register;
6194     Label done;
6195     __ bsfl(Rdst, $src$$Register);
6196     __ jccb(Assembler::notZero, done);
6197     __ movl(Rdst, BitsPerInt);
6198     __ bind(done);
6199   %}
6200   ins_pipe(ialu_reg);
6201 %}
6202 
6203 instruct countTrailingZerosL(rRegI dst, rRegL src, rFlagsReg cr) %{
6204   predicate(UseCountTrailingZerosInstruction);
6205   match(Set dst (CountTrailingZerosL src));
6206   effect(KILL cr);
6207 
6208   format %{ "tzcntq    $dst, $src\t# count trailing zeros (long)" %}
6209   ins_encode %{
6210     __ tzcntq($dst$$Register, $src$$Register);
6211   %}
6212   ins_pipe(ialu_reg);
6213 %}
6214 
6215 instruct countTrailingZerosL_bsf(rRegI dst, rRegL src, rFlagsReg cr) %{
6216   predicate(!UseCountTrailingZerosInstruction);
6217   match(Set dst (CountTrailingZerosL src));
6218   effect(KILL cr);
6219 
6220   format %{ "bsfq    $dst, $src\t# count trailing zeros (long)\n\t"
6221             "jnz     done\n\t"
6222             "movl    $dst, 64\n"
6223       "done:" %}
6224   ins_encode %{
6225     Register Rdst = $dst$$Register;
6226     Label done;
6227     __ bsfq(Rdst, $src$$Register);
6228     __ jccb(Assembler::notZero, done);
6229     __ movl(Rdst, BitsPerLong);
6230     __ bind(done);
6231   %}
6232   ins_pipe(ialu_reg);
6233 %}
6234 
6235 
6236 //---------- Population Count Instructions -------------------------------------
6237 
6238 instruct popCountI(rRegI dst, rRegI src, rFlagsReg cr) %{
6239   predicate(UsePopCountInstruction);
6240   match(Set dst (PopCountI src));
6241   effect(KILL cr);
6242 
6243   format %{ "popcnt  $dst, $src" %}
6244   ins_encode %{
6245     __ popcntl($dst$$Register, $src$$Register);
6246   %}
6247   ins_pipe(ialu_reg);
6248 %}
6249 
6250 instruct popCountI_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6251   predicate(UsePopCountInstruction);
6252   match(Set dst (PopCountI (LoadI mem)));
6253   effect(KILL cr);
6254 
6255   format %{ "popcnt  $dst, $mem" %}
6256   ins_encode %{
6257     __ popcntl($dst$$Register, $mem$$Address);
6258   %}
6259   ins_pipe(ialu_reg);
6260 %}
6261 
6262 // Note: Long.bitCount(long) returns an int.
6263 instruct popCountL(rRegI dst, rRegL src, rFlagsReg cr) %{
6264   predicate(UsePopCountInstruction);
6265   match(Set dst (PopCountL src));
6266   effect(KILL cr);
6267 
6268   format %{ "popcnt  $dst, $src" %}
6269   ins_encode %{
6270     __ popcntq($dst$$Register, $src$$Register);
6271   %}
6272   ins_pipe(ialu_reg);
6273 %}
6274 
6275 // Note: Long.bitCount(long) returns an int.
6276 instruct popCountL_mem(rRegI dst, memory mem, rFlagsReg cr) %{
6277   predicate(UsePopCountInstruction);
6278   match(Set dst (PopCountL (LoadL mem)));
6279   effect(KILL cr);
6280 
6281   format %{ "popcnt  $dst, $mem" %}
6282   ins_encode %{
6283     __ popcntq($dst$$Register, $mem$$Address);
6284   %}
6285   ins_pipe(ialu_reg);
6286 %}
6287 
6288 
6289 //----------MemBar Instructions-----------------------------------------------
6290 // Memory barrier flavors
6291 
6292 instruct membar_acquire()
6293 %{
6294   match(MemBarAcquire);
6295   match(LoadFence);
6296   ins_cost(0);
6297 
6298   size(0);
6299   format %{ "MEMBAR-acquire ! (empty encoding)" %}
6300   ins_encode();
6301   ins_pipe(empty);
6302 %}
6303 
6304 instruct membar_acquire_lock()
6305 %{
6306   match(MemBarAcquireLock);
6307   ins_cost(0);
6308 
6309   size(0);
6310   format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
6311   ins_encode();
6312   ins_pipe(empty);
6313 %}
6314 
6315 instruct membar_release()
6316 %{
6317   match(MemBarRelease);
6318   match(StoreFence);
6319   ins_cost(0);
6320 
6321   size(0);
6322   format %{ "MEMBAR-release ! (empty encoding)" %}
6323   ins_encode();
6324   ins_pipe(empty);
6325 %}
6326 
6327 instruct membar_release_lock()
6328 %{
6329   match(MemBarReleaseLock);
6330   ins_cost(0);
6331 
6332   size(0);
6333   format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
6334   ins_encode();
6335   ins_pipe(empty);
6336 %}
6337 
6338 instruct membar_volatile(rFlagsReg cr) %{
6339   match(MemBarVolatile);
6340   effect(KILL cr);
6341   ins_cost(400);
6342 
6343   format %{
6344     $$template
6345     if (os::is_MP()) {
6346       $$emit$$"lock addl [rsp + #0], 0\t! membar_volatile"
6347     } else {
6348       $$emit$$"MEMBAR-volatile ! (empty encoding)"
6349     }
6350   %}
6351   ins_encode %{
6352     __ membar(Assembler::StoreLoad);
6353   %}
6354   ins_pipe(pipe_slow);
6355 %}
6356 
6357 instruct unnecessary_membar_volatile()
6358 %{
6359   match(MemBarVolatile);
6360   predicate(Matcher::post_store_load_barrier(n));
6361   ins_cost(0);
6362 
6363   size(0);
6364   format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
6365   ins_encode();
6366   ins_pipe(empty);
6367 %}
6368 
6369 instruct membar_storestore() %{
6370   match(MemBarStoreStore);
6371   ins_cost(0);
6372 
6373   size(0);
6374   format %{ "MEMBAR-storestore (empty encoding)" %}
6375   ins_encode( );
6376   ins_pipe(empty);
6377 %}
6378 
6379 //----------Move Instructions--------------------------------------------------
6380 
6381 instruct castX2P(rRegP dst, rRegL src)
6382 %{
6383   match(Set dst (CastX2P src));
6384 
6385   format %{ "movq    $dst, $src\t# long->ptr" %}
6386   ins_encode %{
6387     if ($dst$$reg != $src$$reg) {
6388       __ movptr($dst$$Register, $src$$Register);
6389     }
6390   %}
6391   ins_pipe(ialu_reg_reg); // XXX
6392 %}
6393 
6394 instruct castP2X(rRegL dst, rRegP src)
6395 %{
6396   match(Set dst (CastP2X src));
6397 
6398   format %{ "movq    $dst, $src\t# ptr -> long" %}
6399   ins_encode %{
6400     if ($dst$$reg != $src$$reg) {
6401       __ movptr($dst$$Register, $src$$Register);
6402     }
6403   %}
6404   ins_pipe(ialu_reg_reg); // XXX
6405 %}
6406 
6407 // Convert oop into int for vectors alignment masking
6408 instruct convP2I(rRegI dst, rRegP src)
6409 %{
6410   match(Set dst (ConvL2I (CastP2X src)));
6411 
6412   format %{ "movl    $dst, $src\t# ptr -> int" %}
6413   ins_encode %{
6414     __ movl($dst$$Register, $src$$Register);
6415   %}
6416   ins_pipe(ialu_reg_reg); // XXX
6417 %}
6418 
6419 // Convert compressed oop into int for vectors alignment masking
6420 // in case of 32bit oops (heap < 4Gb).
6421 instruct convN2I(rRegI dst, rRegN src)
6422 %{
6423   predicate(Universe::narrow_oop_shift() == 0);
6424   match(Set dst (ConvL2I (CastP2X (DecodeN src))));
6425 
6426   format %{ "movl    $dst, $src\t# compressed ptr -> int" %}
6427   ins_encode %{
6428     __ movl($dst$$Register, $src$$Register);
6429   %}
6430   ins_pipe(ialu_reg_reg); // XXX
6431 %}
6432 
6433 // Convert oop pointer into compressed form
6434 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
6435   predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
6436   match(Set dst (EncodeP src));
6437   effect(KILL cr);
6438   format %{ "encode_heap_oop $dst,$src" %}
6439   ins_encode %{
6440     Register s = $src$$Register;
6441     Register d = $dst$$Register;
6442     if (s != d) {
6443       __ movq(d, s);
6444     }
6445     __ encode_heap_oop(d);
6446   %}
6447   ins_pipe(ialu_reg_long);
6448 %}
6449 
6450 instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6451   predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
6452   match(Set dst (EncodeP src));
6453   effect(KILL cr);
6454   format %{ "encode_heap_oop_not_null $dst,$src" %}
6455   ins_encode %{
6456     __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
6457   %}
6458   ins_pipe(ialu_reg_long);
6459 %}
6460 
6461 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
6462   predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
6463             n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
6464   match(Set dst (DecodeN src));
6465   effect(KILL cr);
6466   format %{ "decode_heap_oop $dst,$src" %}
6467   ins_encode %{
6468     Register s = $src$$Register;
6469     Register d = $dst$$Register;
6470     if (s != d) {
6471       __ movq(d, s);
6472     }
6473     __ decode_heap_oop(d);
6474   %}
6475   ins_pipe(ialu_reg_long);
6476 %}
6477 
6478 instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6479   predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
6480             n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
6481   match(Set dst (DecodeN src));
6482   effect(KILL cr);
6483   format %{ "decode_heap_oop_not_null $dst,$src" %}
6484   ins_encode %{
6485     Register s = $src$$Register;
6486     Register d = $dst$$Register;
6487     if (s != d) {
6488       __ decode_heap_oop_not_null(d, s);
6489     } else {
6490       __ decode_heap_oop_not_null(d);
6491     }
6492   %}
6493   ins_pipe(ialu_reg_long);
6494 %}
6495 
6496 instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
6497   match(Set dst (EncodePKlass src));
6498   effect(KILL cr);
6499   format %{ "encode_klass_not_null $dst,$src" %}
6500   ins_encode %{
6501     __ encode_klass_not_null($dst$$Register, $src$$Register);
6502   %}
6503   ins_pipe(ialu_reg_long);
6504 %}
6505 
6506 instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
6507   match(Set dst (DecodeNKlass src));
6508   effect(KILL cr);
6509   format %{ "decode_klass_not_null $dst,$src" %}
6510   ins_encode %{
6511     Register s = $src$$Register;
6512     Register d = $dst$$Register;
6513     if (s != d) {
6514       __ decode_klass_not_null(d, s);
6515     } else {
6516       __ decode_klass_not_null(d);
6517     }
6518   %}
6519   ins_pipe(ialu_reg_long);
6520 %}
6521 
6522 
6523 //----------Conditional Move---------------------------------------------------
6524 // Jump
6525 // dummy instruction for generating temp registers
6526 instruct jumpXtnd_offset(rRegL switch_val, immI2 shift, rRegI dest) %{
6527   match(Jump (LShiftL switch_val shift));
6528   ins_cost(350);
6529   predicate(false);
6530   effect(TEMP dest);
6531 
6532   format %{ "leaq    $dest, [$constantaddress]\n\t"
6533             "jmp     [$dest + $switch_val << $shift]\n\t" %}
6534   ins_encode %{
6535     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6536     // to do that and the compiler is using that register as one it can allocate.
6537     // So we build it all by hand.
6538     // Address index(noreg, switch_reg, (Address::ScaleFactor)$shift$$constant);
6539     // ArrayAddress dispatch(table, index);
6540     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant);
6541     __ lea($dest$$Register, $constantaddress);
6542     __ jmp(dispatch);
6543   %}
6544   ins_pipe(pipe_jmp);
6545 %}
6546 
6547 instruct jumpXtnd_addr(rRegL switch_val, immI2 shift, immL32 offset, rRegI dest) %{
6548   match(Jump (AddL (LShiftL switch_val shift) offset));
6549   ins_cost(350);
6550   effect(TEMP dest);
6551 
6552   format %{ "leaq    $dest, [$constantaddress]\n\t"
6553             "jmp     [$dest + $switch_val << $shift + $offset]\n\t" %}
6554   ins_encode %{
6555     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6556     // to do that and the compiler is using that register as one it can allocate.
6557     // So we build it all by hand.
6558     // Address index(noreg, switch_reg, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6559     // ArrayAddress dispatch(table, index);
6560     Address dispatch($dest$$Register, $switch_val$$Register, (Address::ScaleFactor) $shift$$constant, (int) $offset$$constant);
6561     __ lea($dest$$Register, $constantaddress);
6562     __ jmp(dispatch);
6563   %}
6564   ins_pipe(pipe_jmp);
6565 %}
6566 
6567 instruct jumpXtnd(rRegL switch_val, rRegI dest) %{
6568   match(Jump switch_val);
6569   ins_cost(350);
6570   effect(TEMP dest);
6571 
6572   format %{ "leaq    $dest, [$constantaddress]\n\t"
6573             "jmp     [$dest + $switch_val]\n\t" %}
6574   ins_encode %{
6575     // We could use jump(ArrayAddress) except that the macro assembler needs to use r10
6576     // to do that and the compiler is using that register as one it can allocate.
6577     // So we build it all by hand.
6578     // Address index(noreg, switch_reg, Address::times_1);
6579     // ArrayAddress dispatch(table, index);
6580     Address dispatch($dest$$Register, $switch_val$$Register, Address::times_1);
6581     __ lea($dest$$Register, $constantaddress);
6582     __ jmp(dispatch);
6583   %}
6584   ins_pipe(pipe_jmp);
6585 %}
6586 
6587 // Conditional move
6588 instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
6589 %{
6590   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6591 
6592   ins_cost(200); // XXX
6593   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6594   opcode(0x0F, 0x40);
6595   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6596   ins_pipe(pipe_cmov_reg);
6597 %}
6598 
6599 instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
6600   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6601 
6602   ins_cost(200); // XXX
6603   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6604   opcode(0x0F, 0x40);
6605   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6606   ins_pipe(pipe_cmov_reg);
6607 %}
6608 
6609 instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
6610   match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
6611   ins_cost(200);
6612   expand %{
6613     cmovI_regU(cop, cr, dst, src);
6614   %}
6615 %}
6616 
6617 // Conditional move
6618 instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
6619   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6620 
6621   ins_cost(250); // XXX
6622   format %{ "cmovl$cop $dst, $src\t# signed, int" %}
6623   opcode(0x0F, 0x40);
6624   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6625   ins_pipe(pipe_cmov_mem);
6626 %}
6627 
6628 // Conditional move
6629 instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
6630 %{
6631   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6632 
6633   ins_cost(250); // XXX
6634   format %{ "cmovl$cop $dst, $src\t# unsigned, int" %}
6635   opcode(0x0F, 0x40);
6636   ins_encode(REX_reg_mem(dst, src), enc_cmov(cop), reg_mem(dst, src));
6637   ins_pipe(pipe_cmov_mem);
6638 %}
6639 
6640 instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
6641   match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
6642   ins_cost(250);
6643   expand %{
6644     cmovI_memU(cop, cr, dst, src);
6645   %}
6646 %}
6647 
6648 // Conditional move
6649 instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
6650 %{
6651   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6652 
6653   ins_cost(200); // XXX
6654   format %{ "cmovl$cop $dst, $src\t# signed, compressed ptr" %}
6655   opcode(0x0F, 0x40);
6656   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6657   ins_pipe(pipe_cmov_reg);
6658 %}
6659 
6660 // Conditional move
6661 instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
6662 %{
6663   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6664 
6665   ins_cost(200); // XXX
6666   format %{ "cmovl$cop $dst, $src\t# unsigned, compressed ptr" %}
6667   opcode(0x0F, 0x40);
6668   ins_encode(REX_reg_reg(dst, src), enc_cmov(cop), reg_reg(dst, src));
6669   ins_pipe(pipe_cmov_reg);
6670 %}
6671 
6672 instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
6673   match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
6674   ins_cost(200);
6675   expand %{
6676     cmovN_regU(cop, cr, dst, src);
6677   %}
6678 %}
6679 
6680 // Conditional move
6681 instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
6682 %{
6683   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6684 
6685   ins_cost(200); // XXX
6686   format %{ "cmovq$cop $dst, $src\t# signed, ptr" %}
6687   opcode(0x0F, 0x40);
6688   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6689   ins_pipe(pipe_cmov_reg);  // XXX
6690 %}
6691 
6692 // Conditional move
6693 instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
6694 %{
6695   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6696 
6697   ins_cost(200); // XXX
6698   format %{ "cmovq$cop $dst, $src\t# unsigned, ptr" %}
6699   opcode(0x0F, 0x40);
6700   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6701   ins_pipe(pipe_cmov_reg); // XXX
6702 %}
6703 
6704 instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
6705   match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
6706   ins_cost(200);
6707   expand %{
6708     cmovP_regU(cop, cr, dst, src);
6709   %}
6710 %}
6711 
6712 // DISABLED: Requires the ADLC to emit a bottom_type call that
6713 // correctly meets the two pointer arguments; one is an incoming
6714 // register but the other is a memory operand.  ALSO appears to
6715 // be buggy with implicit null checks.
6716 //
6717 //// Conditional move
6718 //instruct cmovP_mem(cmpOp cop, rFlagsReg cr, rRegP dst, memory src)
6719 //%{
6720 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6721 //  ins_cost(250);
6722 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6723 //  opcode(0x0F,0x40);
6724 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6725 //  ins_pipe( pipe_cmov_mem );
6726 //%}
6727 //
6728 //// Conditional move
6729 //instruct cmovP_memU(cmpOpU cop, rFlagsRegU cr, rRegP dst, memory src)
6730 //%{
6731 //  match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
6732 //  ins_cost(250);
6733 //  format %{ "CMOV$cop $dst,$src\t# ptr" %}
6734 //  opcode(0x0F,0x40);
6735 //  ins_encode( enc_cmov(cop), reg_mem( dst, src ) );
6736 //  ins_pipe( pipe_cmov_mem );
6737 //%}
6738 
6739 instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
6740 %{
6741   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6742 
6743   ins_cost(200); // XXX
6744   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6745   opcode(0x0F, 0x40);
6746   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6747   ins_pipe(pipe_cmov_reg);  // XXX
6748 %}
6749 
6750 instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
6751 %{
6752   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6753 
6754   ins_cost(200); // XXX
6755   format %{ "cmovq$cop $dst, $src\t# signed, long" %}
6756   opcode(0x0F, 0x40);
6757   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6758   ins_pipe(pipe_cmov_mem);  // XXX
6759 %}
6760 
6761 instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
6762 %{
6763   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6764 
6765   ins_cost(200); // XXX
6766   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6767   opcode(0x0F, 0x40);
6768   ins_encode(REX_reg_reg_wide(dst, src), enc_cmov(cop), reg_reg(dst, src));
6769   ins_pipe(pipe_cmov_reg); // XXX
6770 %}
6771 
6772 instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
6773   match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
6774   ins_cost(200);
6775   expand %{
6776     cmovL_regU(cop, cr, dst, src);
6777   %}
6778 %}
6779 
6780 instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
6781 %{
6782   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6783 
6784   ins_cost(200); // XXX
6785   format %{ "cmovq$cop $dst, $src\t# unsigned, long" %}
6786   opcode(0x0F, 0x40);
6787   ins_encode(REX_reg_mem_wide(dst, src), enc_cmov(cop), reg_mem(dst, src));
6788   ins_pipe(pipe_cmov_mem); // XXX
6789 %}
6790 
6791 instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
6792   match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
6793   ins_cost(200);
6794   expand %{
6795     cmovL_memU(cop, cr, dst, src);
6796   %}
6797 %}
6798 
6799 instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
6800 %{
6801   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6802 
6803   ins_cost(200); // XXX
6804   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6805             "movss     $dst, $src\n"
6806     "skip:" %}
6807   ins_encode %{
6808     Label Lskip;
6809     // Invert sense of branch from sense of CMOV
6810     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6811     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6812     __ bind(Lskip);
6813   %}
6814   ins_pipe(pipe_slow);
6815 %}
6816 
6817 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
6818 // %{
6819 //   match(Set dst (CMoveF (Binary cop cr) (Binary dst (LoadL src))));
6820 
6821 //   ins_cost(200); // XXX
6822 //   format %{ "jn$cop    skip\t# signed cmove float\n\t"
6823 //             "movss     $dst, $src\n"
6824 //     "skip:" %}
6825 //   ins_encode(enc_cmovf_mem_branch(cop, dst, src));
6826 //   ins_pipe(pipe_slow);
6827 // %}
6828 
6829 instruct cmovF_regU(cmpOpU cop, rFlagsRegU cr, regF dst, regF src)
6830 %{
6831   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6832 
6833   ins_cost(200); // XXX
6834   format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
6835             "movss     $dst, $src\n"
6836     "skip:" %}
6837   ins_encode %{
6838     Label Lskip;
6839     // Invert sense of branch from sense of CMOV
6840     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6841     __ movflt($dst$$XMMRegister, $src$$XMMRegister);
6842     __ bind(Lskip);
6843   %}
6844   ins_pipe(pipe_slow);
6845 %}
6846 
6847 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
6848   match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
6849   ins_cost(200);
6850   expand %{
6851     cmovF_regU(cop, cr, dst, src);
6852   %}
6853 %}
6854 
6855 instruct cmovD_reg(cmpOp cop, rFlagsReg cr, regD dst, regD src)
6856 %{
6857   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6858 
6859   ins_cost(200); // XXX
6860   format %{ "jn$cop    skip\t# signed cmove double\n\t"
6861             "movsd     $dst, $src\n"
6862     "skip:" %}
6863   ins_encode %{
6864     Label Lskip;
6865     // Invert sense of branch from sense of CMOV
6866     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6867     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6868     __ bind(Lskip);
6869   %}
6870   ins_pipe(pipe_slow);
6871 %}
6872 
6873 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
6874 %{
6875   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6876 
6877   ins_cost(200); // XXX
6878   format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
6879             "movsd     $dst, $src\n"
6880     "skip:" %}
6881   ins_encode %{
6882     Label Lskip;
6883     // Invert sense of branch from sense of CMOV
6884     __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
6885     __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
6886     __ bind(Lskip);
6887   %}
6888   ins_pipe(pipe_slow);
6889 %}
6890 
6891 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
6892   match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
6893   ins_cost(200);
6894   expand %{
6895     cmovD_regU(cop, cr, dst, src);
6896   %}
6897 %}
6898 
6899 //----------Arithmetic Instructions--------------------------------------------
6900 //----------Addition Instructions----------------------------------------------
6901 
6902 instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
6903 %{
6904   match(Set dst (AddI dst src));
6905   effect(KILL cr);
6906 
6907   format %{ "addl    $dst, $src\t# int" %}
6908   opcode(0x03);
6909   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
6910   ins_pipe(ialu_reg_reg);
6911 %}
6912 
6913 instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
6914 %{
6915   match(Set dst (AddI dst src));
6916   effect(KILL cr);
6917 
6918   format %{ "addl    $dst, $src\t# int" %}
6919   opcode(0x81, 0x00); /* /0 id */
6920   ins_encode(OpcSErm(dst, src), Con8or32(src));
6921   ins_pipe( ialu_reg );
6922 %}
6923 
6924 instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
6925 %{
6926   match(Set dst (AddI dst (LoadI src)));
6927   effect(KILL cr);
6928 
6929   ins_cost(125); // XXX
6930   format %{ "addl    $dst, $src\t# int" %}
6931   opcode(0x03);
6932   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
6933   ins_pipe(ialu_reg_mem);
6934 %}
6935 
6936 instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
6937 %{
6938   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6939   effect(KILL cr);
6940 
6941   ins_cost(150); // XXX
6942   format %{ "addl    $dst, $src\t# int" %}
6943   opcode(0x01); /* Opcode 01 /r */
6944   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
6945   ins_pipe(ialu_mem_reg);
6946 %}
6947 
6948 instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
6949 %{
6950   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6951   effect(KILL cr);
6952 
6953   ins_cost(125); // XXX
6954   format %{ "addl    $dst, $src\t# int" %}
6955   opcode(0x81); /* Opcode 81 /0 id */
6956   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
6957   ins_pipe(ialu_mem_imm);
6958 %}
6959 
6960 instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
6961 %{
6962   predicate(UseIncDec);
6963   match(Set dst (AddI dst src));
6964   effect(KILL cr);
6965 
6966   format %{ "incl    $dst\t# int" %}
6967   opcode(0xFF, 0x00); // FF /0
6968   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6969   ins_pipe(ialu_reg);
6970 %}
6971 
6972 instruct incI_mem(memory dst, immI1 src, rFlagsReg cr)
6973 %{
6974   predicate(UseIncDec);
6975   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
6976   effect(KILL cr);
6977 
6978   ins_cost(125); // XXX
6979   format %{ "incl    $dst\t# int" %}
6980   opcode(0xFF); /* Opcode FF /0 */
6981   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x00, dst));
6982   ins_pipe(ialu_mem_imm);
6983 %}
6984 
6985 // XXX why does that use AddI
6986 instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
6987 %{
6988   predicate(UseIncDec);
6989   match(Set dst (AddI dst src));
6990   effect(KILL cr);
6991 
6992   format %{ "decl    $dst\t# int" %}
6993   opcode(0xFF, 0x01); // FF /1
6994   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
6995   ins_pipe(ialu_reg);
6996 %}
6997 
6998 // XXX why does that use AddI
6999 instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
7000 %{
7001   predicate(UseIncDec);
7002   match(Set dst (StoreI dst (AddI (LoadI dst) src)));
7003   effect(KILL cr);
7004 
7005   ins_cost(125); // XXX
7006   format %{ "decl    $dst\t# int" %}
7007   opcode(0xFF); /* Opcode FF /1 */
7008   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(0x01, dst));
7009   ins_pipe(ialu_mem_imm);
7010 %}
7011 
7012 instruct leaI_rReg_immI(rRegI dst, rRegI src0, immI src1)
7013 %{
7014   match(Set dst (AddI src0 src1));
7015 
7016   ins_cost(110);
7017   format %{ "addr32 leal $dst, [$src0 + $src1]\t# int" %}
7018   opcode(0x8D); /* 0x8D /r */
7019   ins_encode(Opcode(0x67), REX_reg_reg(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7020   ins_pipe(ialu_reg_reg);
7021 %}
7022 
7023 instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7024 %{
7025   match(Set dst (AddL dst src));
7026   effect(KILL cr);
7027 
7028   format %{ "addq    $dst, $src\t# long" %}
7029   opcode(0x03);
7030   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7031   ins_pipe(ialu_reg_reg);
7032 %}
7033 
7034 instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
7035 %{
7036   match(Set dst (AddL dst src));
7037   effect(KILL cr);
7038 
7039   format %{ "addq    $dst, $src\t# long" %}
7040   opcode(0x81, 0x00); /* /0 id */
7041   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7042   ins_pipe( ialu_reg );
7043 %}
7044 
7045 instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7046 %{
7047   match(Set dst (AddL dst (LoadL src)));
7048   effect(KILL cr);
7049 
7050   ins_cost(125); // XXX
7051   format %{ "addq    $dst, $src\t# long" %}
7052   opcode(0x03);
7053   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7054   ins_pipe(ialu_reg_mem);
7055 %}
7056 
7057 instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7058 %{
7059   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7060   effect(KILL cr);
7061 
7062   ins_cost(150); // XXX
7063   format %{ "addq    $dst, $src\t# long" %}
7064   opcode(0x01); /* Opcode 01 /r */
7065   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7066   ins_pipe(ialu_mem_reg);
7067 %}
7068 
7069 instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7070 %{
7071   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7072   effect(KILL cr);
7073 
7074   ins_cost(125); // XXX
7075   format %{ "addq    $dst, $src\t# long" %}
7076   opcode(0x81); /* Opcode 81 /0 id */
7077   ins_encode(REX_mem_wide(dst),
7078              OpcSE(src), RM_opc_mem(0x00, dst), Con8or32(src));
7079   ins_pipe(ialu_mem_imm);
7080 %}
7081 
7082 instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
7083 %{
7084   predicate(UseIncDec);
7085   match(Set dst (AddL dst src));
7086   effect(KILL cr);
7087 
7088   format %{ "incq    $dst\t# long" %}
7089   opcode(0xFF, 0x00); // FF /0
7090   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7091   ins_pipe(ialu_reg);
7092 %}
7093 
7094 instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
7095 %{
7096   predicate(UseIncDec);
7097   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7098   effect(KILL cr);
7099 
7100   ins_cost(125); // XXX
7101   format %{ "incq    $dst\t# long" %}
7102   opcode(0xFF); /* Opcode FF /0 */
7103   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x00, dst));
7104   ins_pipe(ialu_mem_imm);
7105 %}
7106 
7107 // XXX why does that use AddL
7108 instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
7109 %{
7110   predicate(UseIncDec);
7111   match(Set dst (AddL dst src));
7112   effect(KILL cr);
7113 
7114   format %{ "decq    $dst\t# long" %}
7115   opcode(0xFF, 0x01); // FF /1
7116   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7117   ins_pipe(ialu_reg);
7118 %}
7119 
7120 // XXX why does that use AddL
7121 instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
7122 %{
7123   predicate(UseIncDec);
7124   match(Set dst (StoreL dst (AddL (LoadL dst) src)));
7125   effect(KILL cr);
7126 
7127   ins_cost(125); // XXX
7128   format %{ "decq    $dst\t# long" %}
7129   opcode(0xFF); /* Opcode FF /1 */
7130   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(0x01, dst));
7131   ins_pipe(ialu_mem_imm);
7132 %}
7133 
7134 instruct leaL_rReg_immL(rRegL dst, rRegL src0, immL32 src1)
7135 %{
7136   match(Set dst (AddL src0 src1));
7137 
7138   ins_cost(110);
7139   format %{ "leaq    $dst, [$src0 + $src1]\t# long" %}
7140   opcode(0x8D); /* 0x8D /r */
7141   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1)); // XXX
7142   ins_pipe(ialu_reg_reg);
7143 %}
7144 
7145 instruct addP_rReg(rRegP dst, rRegL src, rFlagsReg cr)
7146 %{
7147   match(Set dst (AddP dst src));
7148   effect(KILL cr);
7149 
7150   format %{ "addq    $dst, $src\t# ptr" %}
7151   opcode(0x03);
7152   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7153   ins_pipe(ialu_reg_reg);
7154 %}
7155 
7156 instruct addP_rReg_imm(rRegP dst, immL32 src, rFlagsReg cr)
7157 %{
7158   match(Set dst (AddP dst src));
7159   effect(KILL cr);
7160 
7161   format %{ "addq    $dst, $src\t# ptr" %}
7162   opcode(0x81, 0x00); /* /0 id */
7163   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7164   ins_pipe( ialu_reg );
7165 %}
7166 
7167 // XXX addP mem ops ????
7168 
7169 instruct leaP_rReg_imm(rRegP dst, rRegP src0, immL32 src1)
7170 %{
7171   match(Set dst (AddP src0 src1));
7172 
7173   ins_cost(110);
7174   format %{ "leaq    $dst, [$src0 + $src1]\t# ptr" %}
7175   opcode(0x8D); /* 0x8D /r */
7176   ins_encode(REX_reg_reg_wide(dst, src0), OpcP, reg_lea(dst, src0, src1));// XXX
7177   ins_pipe(ialu_reg_reg);
7178 %}
7179 
7180 instruct checkCastPP(rRegP dst)
7181 %{
7182   match(Set dst (CheckCastPP dst));
7183 
7184   size(0);
7185   format %{ "# checkcastPP of $dst" %}
7186   ins_encode(/* empty encoding */);
7187   ins_pipe(empty);
7188 %}
7189 
7190 instruct castPP(rRegP dst)
7191 %{
7192   match(Set dst (CastPP dst));
7193 
7194   size(0);
7195   format %{ "# castPP of $dst" %}
7196   ins_encode(/* empty encoding */);
7197   ins_pipe(empty);
7198 %}
7199 
7200 instruct castII(rRegI dst)
7201 %{
7202   match(Set dst (CastII dst));
7203 
7204   size(0);
7205   format %{ "# castII of $dst" %}
7206   ins_encode(/* empty encoding */);
7207   ins_cost(0);
7208   ins_pipe(empty);
7209 %}
7210 
7211 // LoadP-locked same as a regular LoadP when used with compare-swap
7212 instruct loadPLocked(rRegP dst, memory mem)
7213 %{
7214   match(Set dst (LoadPLocked mem));
7215 
7216   ins_cost(125); // XXX
7217   format %{ "movq    $dst, $mem\t# ptr locked" %}
7218   opcode(0x8B);
7219   ins_encode(REX_reg_mem_wide(dst, mem), OpcP, reg_mem(dst, mem));
7220   ins_pipe(ialu_reg_mem); // XXX
7221 %}
7222 
7223 // Conditional-store of the updated heap-top.
7224 // Used during allocation of the shared heap.
7225 // Sets flags (EQ) on success.  Implemented with a CMPXCHG on Intel.
7226 
7227 instruct storePConditional(memory heap_top_ptr,
7228                            rax_RegP oldval, rRegP newval,
7229                            rFlagsReg cr)
7230 %{
7231   match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
7232 
7233   format %{ "cmpxchgq $heap_top_ptr, $newval\t# (ptr) "
7234             "If rax == $heap_top_ptr then store $newval into $heap_top_ptr" %}
7235   opcode(0x0F, 0xB1);
7236   ins_encode(lock_prefix,
7237              REX_reg_mem_wide(newval, heap_top_ptr),
7238              OpcP, OpcS,
7239              reg_mem(newval, heap_top_ptr));
7240   ins_pipe(pipe_cmpxchg);
7241 %}
7242 
7243 // Conditional-store of an int value.
7244 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7245 instruct storeIConditional(memory mem, rax_RegI oldval, rRegI newval, rFlagsReg cr)
7246 %{
7247   match(Set cr (StoreIConditional mem (Binary oldval newval)));
7248   effect(KILL oldval);
7249 
7250   format %{ "cmpxchgl $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7251   opcode(0x0F, 0xB1);
7252   ins_encode(lock_prefix,
7253              REX_reg_mem(newval, mem),
7254              OpcP, OpcS,
7255              reg_mem(newval, mem));
7256   ins_pipe(pipe_cmpxchg);
7257 %}
7258 
7259 // Conditional-store of a long value.
7260 // ZF flag is set on success, reset otherwise.  Implemented with a CMPXCHG.
7261 instruct storeLConditional(memory mem, rax_RegL oldval, rRegL newval, rFlagsReg cr)
7262 %{
7263   match(Set cr (StoreLConditional mem (Binary oldval newval)));
7264   effect(KILL oldval);
7265 
7266   format %{ "cmpxchgq $mem, $newval\t# If rax == $mem then store $newval into $mem" %}
7267   opcode(0x0F, 0xB1);
7268   ins_encode(lock_prefix,
7269              REX_reg_mem_wide(newval, mem),
7270              OpcP, OpcS,
7271              reg_mem(newval, mem));
7272   ins_pipe(pipe_cmpxchg);
7273 %}
7274 
7275 
7276 // XXX No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
7277 instruct compareAndSwapP(rRegI res,
7278                          memory mem_ptr,
7279                          rax_RegP oldval, rRegP newval,
7280                          rFlagsReg cr)
7281 %{
7282   predicate(VM_Version::supports_cx8());
7283   match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
7284   effect(KILL cr, KILL oldval);
7285 
7286   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7287             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7288             "sete    $res\n\t"
7289             "movzbl  $res, $res" %}
7290   opcode(0x0F, 0xB1);
7291   ins_encode(lock_prefix,
7292              REX_reg_mem_wide(newval, mem_ptr),
7293              OpcP, OpcS,
7294              reg_mem(newval, mem_ptr),
7295              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7296              REX_reg_breg(res, res), // movzbl
7297              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7298   ins_pipe( pipe_cmpxchg );
7299 %}
7300 
7301 instruct compareAndSwapL(rRegI res,
7302                          memory mem_ptr,
7303                          rax_RegL oldval, rRegL newval,
7304                          rFlagsReg cr)
7305 %{
7306   predicate(VM_Version::supports_cx8());
7307   match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
7308   effect(KILL cr, KILL oldval);
7309 
7310   format %{ "cmpxchgq $mem_ptr,$newval\t# "
7311             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7312             "sete    $res\n\t"
7313             "movzbl  $res, $res" %}
7314   opcode(0x0F, 0xB1);
7315   ins_encode(lock_prefix,
7316              REX_reg_mem_wide(newval, mem_ptr),
7317              OpcP, OpcS,
7318              reg_mem(newval, mem_ptr),
7319              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7320              REX_reg_breg(res, res), // movzbl
7321              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7322   ins_pipe( pipe_cmpxchg );
7323 %}
7324 
7325 instruct compareAndSwapI(rRegI res,
7326                          memory mem_ptr,
7327                          rax_RegI oldval, rRegI newval,
7328                          rFlagsReg cr)
7329 %{
7330   match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
7331   effect(KILL cr, KILL oldval);
7332 
7333   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7334             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7335             "sete    $res\n\t"
7336             "movzbl  $res, $res" %}
7337   opcode(0x0F, 0xB1);
7338   ins_encode(lock_prefix,
7339              REX_reg_mem(newval, mem_ptr),
7340              OpcP, OpcS,
7341              reg_mem(newval, mem_ptr),
7342              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7343              REX_reg_breg(res, res), // movzbl
7344              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7345   ins_pipe( pipe_cmpxchg );
7346 %}
7347 
7348 
7349 instruct compareAndSwapN(rRegI res,
7350                           memory mem_ptr,
7351                           rax_RegN oldval, rRegN newval,
7352                           rFlagsReg cr) %{
7353   match(Set res (CompareAndSwapN mem_ptr (Binary oldval newval)));
7354   effect(KILL cr, KILL oldval);
7355 
7356   format %{ "cmpxchgl $mem_ptr,$newval\t# "
7357             "If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
7358             "sete    $res\n\t"
7359             "movzbl  $res, $res" %}
7360   opcode(0x0F, 0xB1);
7361   ins_encode(lock_prefix,
7362              REX_reg_mem(newval, mem_ptr),
7363              OpcP, OpcS,
7364              reg_mem(newval, mem_ptr),
7365              REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
7366              REX_reg_breg(res, res), // movzbl
7367              Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
7368   ins_pipe( pipe_cmpxchg );
7369 %}
7370 
7371 instruct xaddI_no_res( memory mem, Universe dummy, immI add, rFlagsReg cr) %{
7372   predicate(n->as_LoadStore()->result_not_used());
7373   match(Set dummy (GetAndAddI mem add));
7374   effect(KILL cr);
7375   format %{ "ADDL  [$mem],$add" %}
7376   ins_encode %{
7377     if (os::is_MP()) { __ lock(); }
7378     __ addl($mem$$Address, $add$$constant);
7379   %}
7380   ins_pipe( pipe_cmpxchg );
7381 %}
7382 
7383 instruct xaddI( memory mem, rRegI newval, rFlagsReg cr) %{
7384   match(Set newval (GetAndAddI mem newval));
7385   effect(KILL cr);
7386   format %{ "XADDL  [$mem],$newval" %}
7387   ins_encode %{
7388     if (os::is_MP()) { __ lock(); }
7389     __ xaddl($mem$$Address, $newval$$Register);
7390   %}
7391   ins_pipe( pipe_cmpxchg );
7392 %}
7393 
7394 instruct xaddL_no_res( memory mem, Universe dummy, immL32 add, rFlagsReg cr) %{
7395   predicate(n->as_LoadStore()->result_not_used());
7396   match(Set dummy (GetAndAddL mem add));
7397   effect(KILL cr);
7398   format %{ "ADDQ  [$mem],$add" %}
7399   ins_encode %{
7400     if (os::is_MP()) { __ lock(); }
7401     __ addq($mem$$Address, $add$$constant);
7402   %}
7403   ins_pipe( pipe_cmpxchg );
7404 %}
7405 
7406 instruct xaddL( memory mem, rRegL newval, rFlagsReg cr) %{
7407   match(Set newval (GetAndAddL mem newval));
7408   effect(KILL cr);
7409   format %{ "XADDQ  [$mem],$newval" %}
7410   ins_encode %{
7411     if (os::is_MP()) { __ lock(); }
7412     __ xaddq($mem$$Address, $newval$$Register);
7413   %}
7414   ins_pipe( pipe_cmpxchg );
7415 %}
7416 
7417 instruct xchgI( memory mem, rRegI newval) %{
7418   match(Set newval (GetAndSetI mem newval));
7419   format %{ "XCHGL  $newval,[$mem]" %}
7420   ins_encode %{
7421     __ xchgl($newval$$Register, $mem$$Address);
7422   %}
7423   ins_pipe( pipe_cmpxchg );
7424 %}
7425 
7426 instruct xchgL( memory mem, rRegL newval) %{
7427   match(Set newval (GetAndSetL mem newval));
7428   format %{ "XCHGL  $newval,[$mem]" %}
7429   ins_encode %{
7430     __ xchgq($newval$$Register, $mem$$Address);
7431   %}
7432   ins_pipe( pipe_cmpxchg );
7433 %}
7434 
7435 instruct xchgP( memory mem, rRegP newval) %{
7436   match(Set newval (GetAndSetP mem newval));
7437   format %{ "XCHGQ  $newval,[$mem]" %}
7438   ins_encode %{
7439     __ xchgq($newval$$Register, $mem$$Address);
7440   %}
7441   ins_pipe( pipe_cmpxchg );
7442 %}
7443 
7444 instruct xchgN( memory mem, rRegN newval) %{
7445   match(Set newval (GetAndSetN mem newval));
7446   format %{ "XCHGL  $newval,$mem]" %}
7447   ins_encode %{
7448     __ xchgl($newval$$Register, $mem$$Address);
7449   %}
7450   ins_pipe( pipe_cmpxchg );
7451 %}
7452 
7453 //----------Subtraction Instructions-------------------------------------------
7454 
7455 // Integer Subtraction Instructions
7456 instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7457 %{
7458   match(Set dst (SubI dst src));
7459   effect(KILL cr);
7460 
7461   format %{ "subl    $dst, $src\t# int" %}
7462   opcode(0x2B);
7463   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
7464   ins_pipe(ialu_reg_reg);
7465 %}
7466 
7467 instruct subI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
7468 %{
7469   match(Set dst (SubI dst src));
7470   effect(KILL cr);
7471 
7472   format %{ "subl    $dst, $src\t# int" %}
7473   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7474   ins_encode(OpcSErm(dst, src), Con8or32(src));
7475   ins_pipe(ialu_reg);
7476 %}
7477 
7478 instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
7479 %{
7480   match(Set dst (SubI dst (LoadI src)));
7481   effect(KILL cr);
7482 
7483   ins_cost(125);
7484   format %{ "subl    $dst, $src\t# int" %}
7485   opcode(0x2B);
7486   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
7487   ins_pipe(ialu_reg_mem);
7488 %}
7489 
7490 instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
7491 %{
7492   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7493   effect(KILL cr);
7494 
7495   ins_cost(150);
7496   format %{ "subl    $dst, $src\t# int" %}
7497   opcode(0x29); /* Opcode 29 /r */
7498   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
7499   ins_pipe(ialu_mem_reg);
7500 %}
7501 
7502 instruct subI_mem_imm(memory dst, immI src, rFlagsReg cr)
7503 %{
7504   match(Set dst (StoreI dst (SubI (LoadI dst) src)));
7505   effect(KILL cr);
7506 
7507   ins_cost(125); // XXX
7508   format %{ "subl    $dst, $src\t# int" %}
7509   opcode(0x81); /* Opcode 81 /5 id */
7510   ins_encode(REX_mem(dst), OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7511   ins_pipe(ialu_mem_imm);
7512 %}
7513 
7514 instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7515 %{
7516   match(Set dst (SubL dst src));
7517   effect(KILL cr);
7518 
7519   format %{ "subq    $dst, $src\t# long" %}
7520   opcode(0x2B);
7521   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7522   ins_pipe(ialu_reg_reg);
7523 %}
7524 
7525 instruct subL_rReg_imm(rRegI dst, immL32 src, rFlagsReg cr)
7526 %{
7527   match(Set dst (SubL dst src));
7528   effect(KILL cr);
7529 
7530   format %{ "subq    $dst, $src\t# long" %}
7531   opcode(0x81, 0x05);  /* Opcode 81 /5 */
7532   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
7533   ins_pipe(ialu_reg);
7534 %}
7535 
7536 instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
7537 %{
7538   match(Set dst (SubL dst (LoadL src)));
7539   effect(KILL cr);
7540 
7541   ins_cost(125);
7542   format %{ "subq    $dst, $src\t# long" %}
7543   opcode(0x2B);
7544   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
7545   ins_pipe(ialu_reg_mem);
7546 %}
7547 
7548 instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
7549 %{
7550   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7551   effect(KILL cr);
7552 
7553   ins_cost(150);
7554   format %{ "subq    $dst, $src\t# long" %}
7555   opcode(0x29); /* Opcode 29 /r */
7556   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
7557   ins_pipe(ialu_mem_reg);
7558 %}
7559 
7560 instruct subL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
7561 %{
7562   match(Set dst (StoreL dst (SubL (LoadL dst) src)));
7563   effect(KILL cr);
7564 
7565   ins_cost(125); // XXX
7566   format %{ "subq    $dst, $src\t# long" %}
7567   opcode(0x81); /* Opcode 81 /5 id */
7568   ins_encode(REX_mem_wide(dst),
7569              OpcSE(src), RM_opc_mem(0x05, dst), Con8or32(src));
7570   ins_pipe(ialu_mem_imm);
7571 %}
7572 
7573 // Subtract from a pointer
7574 // XXX hmpf???
7575 instruct subP_rReg(rRegP dst, rRegI src, immI0 zero, rFlagsReg cr)
7576 %{
7577   match(Set dst (AddP dst (SubI zero src)));
7578   effect(KILL cr);
7579 
7580   format %{ "subq    $dst, $src\t# ptr - int" %}
7581   opcode(0x2B);
7582   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
7583   ins_pipe(ialu_reg_reg);
7584 %}
7585 
7586 instruct negI_rReg(rRegI dst, immI0 zero, rFlagsReg cr)
7587 %{
7588   match(Set dst (SubI zero dst));
7589   effect(KILL cr);
7590 
7591   format %{ "negl    $dst\t# int" %}
7592   opcode(0xF7, 0x03);  // Opcode F7 /3
7593   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7594   ins_pipe(ialu_reg);
7595 %}
7596 
7597 instruct negI_mem(memory dst, immI0 zero, rFlagsReg cr)
7598 %{
7599   match(Set dst (StoreI dst (SubI zero (LoadI dst))));
7600   effect(KILL cr);
7601 
7602   format %{ "negl    $dst\t# int" %}
7603   opcode(0xF7, 0x03);  // Opcode F7 /3
7604   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7605   ins_pipe(ialu_reg);
7606 %}
7607 
7608 instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
7609 %{
7610   match(Set dst (SubL zero dst));
7611   effect(KILL cr);
7612 
7613   format %{ "negq    $dst\t# long" %}
7614   opcode(0xF7, 0x03);  // Opcode F7 /3
7615   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
7616   ins_pipe(ialu_reg);
7617 %}
7618 
7619 instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
7620 %{
7621   match(Set dst (StoreL dst (SubL zero (LoadL dst))));
7622   effect(KILL cr);
7623 
7624   format %{ "negq    $dst\t# long" %}
7625   opcode(0xF7, 0x03);  // Opcode F7 /3
7626   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
7627   ins_pipe(ialu_reg);
7628 %}
7629 
7630 //----------Multiplication/Division Instructions-------------------------------
7631 // Integer Multiplication Instructions
7632 // Multiply Register
7633 
7634 instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
7635 %{
7636   match(Set dst (MulI dst src));
7637   effect(KILL cr);
7638 
7639   ins_cost(300);
7640   format %{ "imull   $dst, $src\t# int" %}
7641   opcode(0x0F, 0xAF);
7642   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
7643   ins_pipe(ialu_reg_reg_alu0);
7644 %}
7645 
7646 instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
7647 %{
7648   match(Set dst (MulI src imm));
7649   effect(KILL cr);
7650 
7651   ins_cost(300);
7652   format %{ "imull   $dst, $src, $imm\t# int" %}
7653   opcode(0x69); /* 69 /r id */
7654   ins_encode(REX_reg_reg(dst, src),
7655              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7656   ins_pipe(ialu_reg_reg_alu0);
7657 %}
7658 
7659 instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
7660 %{
7661   match(Set dst (MulI dst (LoadI src)));
7662   effect(KILL cr);
7663 
7664   ins_cost(350);
7665   format %{ "imull   $dst, $src\t# int" %}
7666   opcode(0x0F, 0xAF);
7667   ins_encode(REX_reg_mem(dst, src), OpcP, OpcS, reg_mem(dst, src));
7668   ins_pipe(ialu_reg_mem_alu0);
7669 %}
7670 
7671 instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
7672 %{
7673   match(Set dst (MulI (LoadI src) imm));
7674   effect(KILL cr);
7675 
7676   ins_cost(300);
7677   format %{ "imull   $dst, $src, $imm\t# int" %}
7678   opcode(0x69); /* 69 /r id */
7679   ins_encode(REX_reg_mem(dst, src),
7680              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7681   ins_pipe(ialu_reg_mem_alu0);
7682 %}
7683 
7684 instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
7685 %{
7686   match(Set dst (MulL dst src));
7687   effect(KILL cr);
7688 
7689   ins_cost(300);
7690   format %{ "imulq   $dst, $src\t# long" %}
7691   opcode(0x0F, 0xAF);
7692   ins_encode(REX_reg_reg_wide(dst, src), OpcP, OpcS, reg_reg(dst, src));
7693   ins_pipe(ialu_reg_reg_alu0);
7694 %}
7695 
7696 instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
7697 %{
7698   match(Set dst (MulL src imm));
7699   effect(KILL cr);
7700 
7701   ins_cost(300);
7702   format %{ "imulq   $dst, $src, $imm\t# long" %}
7703   opcode(0x69); /* 69 /r id */
7704   ins_encode(REX_reg_reg_wide(dst, src),
7705              OpcSE(imm), reg_reg(dst, src), Con8or32(imm));
7706   ins_pipe(ialu_reg_reg_alu0);
7707 %}
7708 
7709 instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
7710 %{
7711   match(Set dst (MulL dst (LoadL src)));
7712   effect(KILL cr);
7713 
7714   ins_cost(350);
7715   format %{ "imulq   $dst, $src\t# long" %}
7716   opcode(0x0F, 0xAF);
7717   ins_encode(REX_reg_mem_wide(dst, src), OpcP, OpcS, reg_mem(dst, src));
7718   ins_pipe(ialu_reg_mem_alu0);
7719 %}
7720 
7721 instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
7722 %{
7723   match(Set dst (MulL (LoadL src) imm));
7724   effect(KILL cr);
7725 
7726   ins_cost(300);
7727   format %{ "imulq   $dst, $src, $imm\t# long" %}
7728   opcode(0x69); /* 69 /r id */
7729   ins_encode(REX_reg_mem_wide(dst, src),
7730              OpcSE(imm), reg_mem(dst, src), Con8or32(imm));
7731   ins_pipe(ialu_reg_mem_alu0);
7732 %}
7733 
7734 instruct mulHiL_rReg(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7735 %{
7736   match(Set dst (MulHiL src rax));
7737   effect(USE_KILL rax, KILL cr);
7738 
7739   ins_cost(300);
7740   format %{ "imulq   RDX:RAX, RAX, $src\t# mulhi" %}
7741   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7742   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7743   ins_pipe(ialu_reg_reg_alu0);
7744 %}
7745 
7746 instruct divI_rReg(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7747                    rFlagsReg cr)
7748 %{
7749   match(Set rax (DivI rax div));
7750   effect(KILL rdx, KILL cr);
7751 
7752   ins_cost(30*100+10*100); // XXX
7753   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7754             "jne,s   normal\n\t"
7755             "xorl    rdx, rdx\n\t"
7756             "cmpl    $div, -1\n\t"
7757             "je,s    done\n"
7758     "normal: cdql\n\t"
7759             "idivl   $div\n"
7760     "done:"        %}
7761   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7762   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7763   ins_pipe(ialu_reg_reg_alu0);
7764 %}
7765 
7766 instruct divL_rReg(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7767                    rFlagsReg cr)
7768 %{
7769   match(Set rax (DivL rax div));
7770   effect(KILL rdx, KILL cr);
7771 
7772   ins_cost(30*100+10*100); // XXX
7773   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7774             "cmpq    rax, rdx\n\t"
7775             "jne,s   normal\n\t"
7776             "xorl    rdx, rdx\n\t"
7777             "cmpq    $div, -1\n\t"
7778             "je,s    done\n"
7779     "normal: cdqq\n\t"
7780             "idivq   $div\n"
7781     "done:"        %}
7782   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7783   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7784   ins_pipe(ialu_reg_reg_alu0);
7785 %}
7786 
7787 // Integer DIVMOD with Register, both quotient and mod results
7788 instruct divModI_rReg_divmod(rax_RegI rax, rdx_RegI rdx, no_rax_rdx_RegI div,
7789                              rFlagsReg cr)
7790 %{
7791   match(DivModI rax div);
7792   effect(KILL cr);
7793 
7794   ins_cost(30*100+10*100); // XXX
7795   format %{ "cmpl    rax, 0x80000000\t# idiv\n\t"
7796             "jne,s   normal\n\t"
7797             "xorl    rdx, rdx\n\t"
7798             "cmpl    $div, -1\n\t"
7799             "je,s    done\n"
7800     "normal: cdql\n\t"
7801             "idivl   $div\n"
7802     "done:"        %}
7803   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7804   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7805   ins_pipe(pipe_slow);
7806 %}
7807 
7808 // Long DIVMOD with Register, both quotient and mod results
7809 instruct divModL_rReg_divmod(rax_RegL rax, rdx_RegL rdx, no_rax_rdx_RegL div,
7810                              rFlagsReg cr)
7811 %{
7812   match(DivModL rax div);
7813   effect(KILL cr);
7814 
7815   ins_cost(30*100+10*100); // XXX
7816   format %{ "movq    rdx, 0x8000000000000000\t# ldiv\n\t"
7817             "cmpq    rax, rdx\n\t"
7818             "jne,s   normal\n\t"
7819             "xorl    rdx, rdx\n\t"
7820             "cmpq    $div, -1\n\t"
7821             "je,s    done\n"
7822     "normal: cdqq\n\t"
7823             "idivq   $div\n"
7824     "done:"        %}
7825   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7826   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7827   ins_pipe(pipe_slow);
7828 %}
7829 
7830 //----------- DivL-By-Constant-Expansions--------------------------------------
7831 // DivI cases are handled by the compiler
7832 
7833 // Magic constant, reciprocal of 10
7834 instruct loadConL_0x6666666666666667(rRegL dst)
7835 %{
7836   effect(DEF dst);
7837 
7838   format %{ "movq    $dst, #0x666666666666667\t# Used in div-by-10" %}
7839   ins_encode(load_immL(dst, 0x6666666666666667));
7840   ins_pipe(ialu_reg);
7841 %}
7842 
7843 instruct mul_hi(rdx_RegL dst, no_rax_RegL src, rax_RegL rax, rFlagsReg cr)
7844 %{
7845   effect(DEF dst, USE src, USE_KILL rax, KILL cr);
7846 
7847   format %{ "imulq   rdx:rax, rax, $src\t# Used in div-by-10" %}
7848   opcode(0xF7, 0x5); /* Opcode F7 /5 */
7849   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src));
7850   ins_pipe(ialu_reg_reg_alu0);
7851 %}
7852 
7853 instruct sarL_rReg_63(rRegL dst, rFlagsReg cr)
7854 %{
7855   effect(USE_DEF dst, KILL cr);
7856 
7857   format %{ "sarq    $dst, #63\t# Used in div-by-10" %}
7858   opcode(0xC1, 0x7); /* C1 /7 ib */
7859   ins_encode(reg_opc_imm_wide(dst, 0x3F));
7860   ins_pipe(ialu_reg);
7861 %}
7862 
7863 instruct sarL_rReg_2(rRegL dst, rFlagsReg cr)
7864 %{
7865   effect(USE_DEF dst, KILL cr);
7866 
7867   format %{ "sarq    $dst, #2\t# Used in div-by-10" %}
7868   opcode(0xC1, 0x7); /* C1 /7 ib */
7869   ins_encode(reg_opc_imm_wide(dst, 0x2));
7870   ins_pipe(ialu_reg);
7871 %}
7872 
7873 instruct divL_10(rdx_RegL dst, no_rax_RegL src, immL10 div)
7874 %{
7875   match(Set dst (DivL src div));
7876 
7877   ins_cost((5+8)*100);
7878   expand %{
7879     rax_RegL rax;                     // Killed temp
7880     rFlagsReg cr;                     // Killed
7881     loadConL_0x6666666666666667(rax); // movq  rax, 0x6666666666666667
7882     mul_hi(dst, src, rax, cr);        // mulq  rdx:rax <= rax * $src
7883     sarL_rReg_63(src, cr);            // sarq  src, 63
7884     sarL_rReg_2(dst, cr);             // sarq  rdx, 2
7885     subL_rReg(dst, src, cr);          // subl  rdx, src
7886   %}
7887 %}
7888 
7889 //-----------------------------------------------------------------------------
7890 
7891 instruct modI_rReg(rdx_RegI rdx, rax_RegI rax, no_rax_rdx_RegI div,
7892                    rFlagsReg cr)
7893 %{
7894   match(Set rdx (ModI rax div));
7895   effect(KILL rax, KILL cr);
7896 
7897   ins_cost(300); // XXX
7898   format %{ "cmpl    rax, 0x80000000\t# irem\n\t"
7899             "jne,s   normal\n\t"
7900             "xorl    rdx, rdx\n\t"
7901             "cmpl    $div, -1\n\t"
7902             "je,s    done\n"
7903     "normal: cdql\n\t"
7904             "idivl   $div\n"
7905     "done:"        %}
7906   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7907   ins_encode(cdql_enc(div), REX_reg(div), OpcP, reg_opc(div));
7908   ins_pipe(ialu_reg_reg_alu0);
7909 %}
7910 
7911 instruct modL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div,
7912                    rFlagsReg cr)
7913 %{
7914   match(Set rdx (ModL rax div));
7915   effect(KILL rax, KILL cr);
7916 
7917   ins_cost(300); // XXX
7918   format %{ "movq    rdx, 0x8000000000000000\t# lrem\n\t"
7919             "cmpq    rax, rdx\n\t"
7920             "jne,s   normal\n\t"
7921             "xorl    rdx, rdx\n\t"
7922             "cmpq    $div, -1\n\t"
7923             "je,s    done\n"
7924     "normal: cdqq\n\t"
7925             "idivq   $div\n"
7926     "done:"        %}
7927   opcode(0xF7, 0x7);  /* Opcode F7 /7 */
7928   ins_encode(cdqq_enc(div), REX_reg_wide(div), OpcP, reg_opc(div));
7929   ins_pipe(ialu_reg_reg_alu0);
7930 %}
7931 
7932 // Integer Shift Instructions
7933 // Shift Left by one
7934 instruct salI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
7935 %{
7936   match(Set dst (LShiftI dst shift));
7937   effect(KILL cr);
7938 
7939   format %{ "sall    $dst, $shift" %}
7940   opcode(0xD1, 0x4); /* D1 /4 */
7941   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7942   ins_pipe(ialu_reg);
7943 %}
7944 
7945 // Shift Left by one
7946 instruct salI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
7947 %{
7948   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7949   effect(KILL cr);
7950 
7951   format %{ "sall    $dst, $shift\t" %}
7952   opcode(0xD1, 0x4); /* D1 /4 */
7953   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
7954   ins_pipe(ialu_mem_imm);
7955 %}
7956 
7957 // Shift Left by 8-bit immediate
7958 instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
7959 %{
7960   match(Set dst (LShiftI dst shift));
7961   effect(KILL cr);
7962 
7963   format %{ "sall    $dst, $shift" %}
7964   opcode(0xC1, 0x4); /* C1 /4 ib */
7965   ins_encode(reg_opc_imm(dst, shift));
7966   ins_pipe(ialu_reg);
7967 %}
7968 
7969 // Shift Left by 8-bit immediate
7970 instruct salI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
7971 %{
7972   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7973   effect(KILL cr);
7974 
7975   format %{ "sall    $dst, $shift" %}
7976   opcode(0xC1, 0x4); /* C1 /4 ib */
7977   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
7978   ins_pipe(ialu_mem_imm);
7979 %}
7980 
7981 // Shift Left by variable
7982 instruct salI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
7983 %{
7984   match(Set dst (LShiftI dst shift));
7985   effect(KILL cr);
7986 
7987   format %{ "sall    $dst, $shift" %}
7988   opcode(0xD3, 0x4); /* D3 /4 */
7989   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
7990   ins_pipe(ialu_reg_reg);
7991 %}
7992 
7993 // Shift Left by variable
7994 instruct salI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
7995 %{
7996   match(Set dst (StoreI dst (LShiftI (LoadI dst) shift)));
7997   effect(KILL cr);
7998 
7999   format %{ "sall    $dst, $shift" %}
8000   opcode(0xD3, 0x4); /* D3 /4 */
8001   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8002   ins_pipe(ialu_mem_reg);
8003 %}
8004 
8005 // Arithmetic shift right by one
8006 instruct sarI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8007 %{
8008   match(Set dst (RShiftI dst shift));
8009   effect(KILL cr);
8010 
8011   format %{ "sarl    $dst, $shift" %}
8012   opcode(0xD1, 0x7); /* D1 /7 */
8013   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8014   ins_pipe(ialu_reg);
8015 %}
8016 
8017 // Arithmetic shift right by one
8018 instruct sarI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8019 %{
8020   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8021   effect(KILL cr);
8022 
8023   format %{ "sarl    $dst, $shift" %}
8024   opcode(0xD1, 0x7); /* D1 /7 */
8025   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8026   ins_pipe(ialu_mem_imm);
8027 %}
8028 
8029 // Arithmetic Shift Right by 8-bit immediate
8030 instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8031 %{
8032   match(Set dst (RShiftI dst shift));
8033   effect(KILL cr);
8034 
8035   format %{ "sarl    $dst, $shift" %}
8036   opcode(0xC1, 0x7); /* C1 /7 ib */
8037   ins_encode(reg_opc_imm(dst, shift));
8038   ins_pipe(ialu_mem_imm);
8039 %}
8040 
8041 // Arithmetic Shift Right by 8-bit immediate
8042 instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8043 %{
8044   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8045   effect(KILL cr);
8046 
8047   format %{ "sarl    $dst, $shift" %}
8048   opcode(0xC1, 0x7); /* C1 /7 ib */
8049   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8050   ins_pipe(ialu_mem_imm);
8051 %}
8052 
8053 // Arithmetic Shift Right by variable
8054 instruct sarI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8055 %{
8056   match(Set dst (RShiftI dst shift));
8057   effect(KILL cr);
8058 
8059   format %{ "sarl    $dst, $shift" %}
8060   opcode(0xD3, 0x7); /* D3 /7 */
8061   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8062   ins_pipe(ialu_reg_reg);
8063 %}
8064 
8065 // Arithmetic Shift Right by variable
8066 instruct sarI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8067 %{
8068   match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
8069   effect(KILL cr);
8070 
8071   format %{ "sarl    $dst, $shift" %}
8072   opcode(0xD3, 0x7); /* D3 /7 */
8073   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8074   ins_pipe(ialu_mem_reg);
8075 %}
8076 
8077 // Logical shift right by one
8078 instruct shrI_rReg_1(rRegI dst, immI1 shift, rFlagsReg cr)
8079 %{
8080   match(Set dst (URShiftI dst shift));
8081   effect(KILL cr);
8082 
8083   format %{ "shrl    $dst, $shift" %}
8084   opcode(0xD1, 0x5); /* D1 /5 */
8085   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8086   ins_pipe(ialu_reg);
8087 %}
8088 
8089 // Logical shift right by one
8090 instruct shrI_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8091 %{
8092   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8093   effect(KILL cr);
8094 
8095   format %{ "shrl    $dst, $shift" %}
8096   opcode(0xD1, 0x5); /* D1 /5 */
8097   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8098   ins_pipe(ialu_mem_imm);
8099 %}
8100 
8101 // Logical Shift Right by 8-bit immediate
8102 instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
8103 %{
8104   match(Set dst (URShiftI dst shift));
8105   effect(KILL cr);
8106 
8107   format %{ "shrl    $dst, $shift" %}
8108   opcode(0xC1, 0x5); /* C1 /5 ib */
8109   ins_encode(reg_opc_imm(dst, shift));
8110   ins_pipe(ialu_reg);
8111 %}
8112 
8113 // Logical Shift Right by 8-bit immediate
8114 instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8115 %{
8116   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8117   effect(KILL cr);
8118 
8119   format %{ "shrl    $dst, $shift" %}
8120   opcode(0xC1, 0x5); /* C1 /5 ib */
8121   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst), Con8or32(shift));
8122   ins_pipe(ialu_mem_imm);
8123 %}
8124 
8125 // Logical Shift Right by variable
8126 instruct shrI_rReg_CL(rRegI dst, rcx_RegI shift, rFlagsReg cr)
8127 %{
8128   match(Set dst (URShiftI dst shift));
8129   effect(KILL cr);
8130 
8131   format %{ "shrl    $dst, $shift" %}
8132   opcode(0xD3, 0x5); /* D3 /5 */
8133   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8134   ins_pipe(ialu_reg_reg);
8135 %}
8136 
8137 // Logical Shift Right by variable
8138 instruct shrI_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8139 %{
8140   match(Set dst (StoreI dst (URShiftI (LoadI dst) shift)));
8141   effect(KILL cr);
8142 
8143   format %{ "shrl    $dst, $shift" %}
8144   opcode(0xD3, 0x5); /* D3 /5 */
8145   ins_encode(REX_mem(dst), OpcP, RM_opc_mem(secondary, dst));
8146   ins_pipe(ialu_mem_reg);
8147 %}
8148 
8149 // Long Shift Instructions
8150 // Shift Left by one
8151 instruct salL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8152 %{
8153   match(Set dst (LShiftL dst shift));
8154   effect(KILL cr);
8155 
8156   format %{ "salq    $dst, $shift" %}
8157   opcode(0xD1, 0x4); /* D1 /4 */
8158   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8159   ins_pipe(ialu_reg);
8160 %}
8161 
8162 // Shift Left by one
8163 instruct salL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8164 %{
8165   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8166   effect(KILL cr);
8167 
8168   format %{ "salq    $dst, $shift" %}
8169   opcode(0xD1, 0x4); /* D1 /4 */
8170   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8171   ins_pipe(ialu_mem_imm);
8172 %}
8173 
8174 // Shift Left by 8-bit immediate
8175 instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8176 %{
8177   match(Set dst (LShiftL dst shift));
8178   effect(KILL cr);
8179 
8180   format %{ "salq    $dst, $shift" %}
8181   opcode(0xC1, 0x4); /* C1 /4 ib */
8182   ins_encode(reg_opc_imm_wide(dst, shift));
8183   ins_pipe(ialu_reg);
8184 %}
8185 
8186 // Shift Left by 8-bit immediate
8187 instruct salL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8188 %{
8189   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8190   effect(KILL cr);
8191 
8192   format %{ "salq    $dst, $shift" %}
8193   opcode(0xC1, 0x4); /* C1 /4 ib */
8194   ins_encode(REX_mem_wide(dst), OpcP,
8195              RM_opc_mem(secondary, dst), Con8or32(shift));
8196   ins_pipe(ialu_mem_imm);
8197 %}
8198 
8199 // Shift Left by variable
8200 instruct salL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8201 %{
8202   match(Set dst (LShiftL dst shift));
8203   effect(KILL cr);
8204 
8205   format %{ "salq    $dst, $shift" %}
8206   opcode(0xD3, 0x4); /* D3 /4 */
8207   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8208   ins_pipe(ialu_reg_reg);
8209 %}
8210 
8211 // Shift Left by variable
8212 instruct salL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8213 %{
8214   match(Set dst (StoreL dst (LShiftL (LoadL dst) shift)));
8215   effect(KILL cr);
8216 
8217   format %{ "salq    $dst, $shift" %}
8218   opcode(0xD3, 0x4); /* D3 /4 */
8219   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8220   ins_pipe(ialu_mem_reg);
8221 %}
8222 
8223 // Arithmetic shift right by one
8224 instruct sarL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8225 %{
8226   match(Set dst (RShiftL dst shift));
8227   effect(KILL cr);
8228 
8229   format %{ "sarq    $dst, $shift" %}
8230   opcode(0xD1, 0x7); /* D1 /7 */
8231   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8232   ins_pipe(ialu_reg);
8233 %}
8234 
8235 // Arithmetic shift right by one
8236 instruct sarL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8237 %{
8238   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8239   effect(KILL cr);
8240 
8241   format %{ "sarq    $dst, $shift" %}
8242   opcode(0xD1, 0x7); /* D1 /7 */
8243   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8244   ins_pipe(ialu_mem_imm);
8245 %}
8246 
8247 // Arithmetic Shift Right by 8-bit immediate
8248 instruct sarL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8249 %{
8250   match(Set dst (RShiftL dst shift));
8251   effect(KILL cr);
8252 
8253   format %{ "sarq    $dst, $shift" %}
8254   opcode(0xC1, 0x7); /* C1 /7 ib */
8255   ins_encode(reg_opc_imm_wide(dst, shift));
8256   ins_pipe(ialu_mem_imm);
8257 %}
8258 
8259 // Arithmetic Shift Right by 8-bit immediate
8260 instruct sarL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8261 %{
8262   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8263   effect(KILL cr);
8264 
8265   format %{ "sarq    $dst, $shift" %}
8266   opcode(0xC1, 0x7); /* C1 /7 ib */
8267   ins_encode(REX_mem_wide(dst), OpcP,
8268              RM_opc_mem(secondary, dst), Con8or32(shift));
8269   ins_pipe(ialu_mem_imm);
8270 %}
8271 
8272 // Arithmetic Shift Right by variable
8273 instruct sarL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8274 %{
8275   match(Set dst (RShiftL dst shift));
8276   effect(KILL cr);
8277 
8278   format %{ "sarq    $dst, $shift" %}
8279   opcode(0xD3, 0x7); /* D3 /7 */
8280   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8281   ins_pipe(ialu_reg_reg);
8282 %}
8283 
8284 // Arithmetic Shift Right by variable
8285 instruct sarL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8286 %{
8287   match(Set dst (StoreL dst (RShiftL (LoadL dst) shift)));
8288   effect(KILL cr);
8289 
8290   format %{ "sarq    $dst, $shift" %}
8291   opcode(0xD3, 0x7); /* D3 /7 */
8292   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8293   ins_pipe(ialu_mem_reg);
8294 %}
8295 
8296 // Logical shift right by one
8297 instruct shrL_rReg_1(rRegL dst, immI1 shift, rFlagsReg cr)
8298 %{
8299   match(Set dst (URShiftL dst shift));
8300   effect(KILL cr);
8301 
8302   format %{ "shrq    $dst, $shift" %}
8303   opcode(0xD1, 0x5); /* D1 /5 */
8304   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst ));
8305   ins_pipe(ialu_reg);
8306 %}
8307 
8308 // Logical shift right by one
8309 instruct shrL_mem_1(memory dst, immI1 shift, rFlagsReg cr)
8310 %{
8311   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8312   effect(KILL cr);
8313 
8314   format %{ "shrq    $dst, $shift" %}
8315   opcode(0xD1, 0x5); /* D1 /5 */
8316   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8317   ins_pipe(ialu_mem_imm);
8318 %}
8319 
8320 // Logical Shift Right by 8-bit immediate
8321 instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
8322 %{
8323   match(Set dst (URShiftL dst shift));
8324   effect(KILL cr);
8325 
8326   format %{ "shrq    $dst, $shift" %}
8327   opcode(0xC1, 0x5); /* C1 /5 ib */
8328   ins_encode(reg_opc_imm_wide(dst, shift));
8329   ins_pipe(ialu_reg);
8330 %}
8331 
8332 
8333 // Logical Shift Right by 8-bit immediate
8334 instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
8335 %{
8336   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8337   effect(KILL cr);
8338 
8339   format %{ "shrq    $dst, $shift" %}
8340   opcode(0xC1, 0x5); /* C1 /5 ib */
8341   ins_encode(REX_mem_wide(dst), OpcP,
8342              RM_opc_mem(secondary, dst), Con8or32(shift));
8343   ins_pipe(ialu_mem_imm);
8344 %}
8345 
8346 // Logical Shift Right by variable
8347 instruct shrL_rReg_CL(rRegL dst, rcx_RegI shift, rFlagsReg cr)
8348 %{
8349   match(Set dst (URShiftL dst shift));
8350   effect(KILL cr);
8351 
8352   format %{ "shrq    $dst, $shift" %}
8353   opcode(0xD3, 0x5); /* D3 /5 */
8354   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8355   ins_pipe(ialu_reg_reg);
8356 %}
8357 
8358 // Logical Shift Right by variable
8359 instruct shrL_mem_CL(memory dst, rcx_RegI shift, rFlagsReg cr)
8360 %{
8361   match(Set dst (StoreL dst (URShiftL (LoadL dst) shift)));
8362   effect(KILL cr);
8363 
8364   format %{ "shrq    $dst, $shift" %}
8365   opcode(0xD3, 0x5); /* D3 /5 */
8366   ins_encode(REX_mem_wide(dst), OpcP, RM_opc_mem(secondary, dst));
8367   ins_pipe(ialu_mem_reg);
8368 %}
8369 
8370 // Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
8371 // This idiom is used by the compiler for the i2b bytecode.
8372 instruct i2b(rRegI dst, rRegI src, immI_24 twentyfour)
8373 %{
8374   match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
8375 
8376   format %{ "movsbl  $dst, $src\t# i2b" %}
8377   opcode(0x0F, 0xBE);
8378   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8379   ins_pipe(ialu_reg_reg);
8380 %}
8381 
8382 // Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
8383 // This idiom is used by the compiler the i2s bytecode.
8384 instruct i2s(rRegI dst, rRegI src, immI_16 sixteen)
8385 %{
8386   match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
8387 
8388   format %{ "movswl  $dst, $src\t# i2s" %}
8389   opcode(0x0F, 0xBF);
8390   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8391   ins_pipe(ialu_reg_reg);
8392 %}
8393 
8394 // ROL/ROR instructions
8395 
8396 // ROL expand
8397 instruct rolI_rReg_imm1(rRegI dst, rFlagsReg cr) %{
8398   effect(KILL cr, USE_DEF dst);
8399 
8400   format %{ "roll    $dst" %}
8401   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8402   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8403   ins_pipe(ialu_reg);
8404 %}
8405 
8406 instruct rolI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr) %{
8407   effect(USE_DEF dst, USE shift, KILL cr);
8408 
8409   format %{ "roll    $dst, $shift" %}
8410   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8411   ins_encode( reg_opc_imm(dst, shift) );
8412   ins_pipe(ialu_reg);
8413 %}
8414 
8415 instruct rolI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8416 %{
8417   effect(USE_DEF dst, USE shift, KILL cr);
8418 
8419   format %{ "roll    $dst, $shift" %}
8420   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8421   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8422   ins_pipe(ialu_reg_reg);
8423 %}
8424 // end of ROL expand
8425 
8426 // Rotate Left by one
8427 instruct rolI_rReg_i1(rRegI dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8428 %{
8429   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8430 
8431   expand %{
8432     rolI_rReg_imm1(dst, cr);
8433   %}
8434 %}
8435 
8436 // Rotate Left by 8-bit immediate
8437 instruct rolI_rReg_i8(rRegI dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8438 %{
8439   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8440   match(Set dst (OrI (LShiftI dst lshift) (URShiftI dst rshift)));
8441 
8442   expand %{
8443     rolI_rReg_imm8(dst, lshift, cr);
8444   %}
8445 %}
8446 
8447 // Rotate Left by variable
8448 instruct rolI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8449 %{
8450   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
8451 
8452   expand %{
8453     rolI_rReg_CL(dst, shift, cr);
8454   %}
8455 %}
8456 
8457 // Rotate Left by variable
8458 instruct rolI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8459 %{
8460   match(Set dst (OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
8461 
8462   expand %{
8463     rolI_rReg_CL(dst, shift, cr);
8464   %}
8465 %}
8466 
8467 // ROR expand
8468 instruct rorI_rReg_imm1(rRegI dst, rFlagsReg cr)
8469 %{
8470   effect(USE_DEF dst, KILL cr);
8471 
8472   format %{ "rorl    $dst" %}
8473   opcode(0xD1, 0x1); /* D1 /1 */
8474   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8475   ins_pipe(ialu_reg);
8476 %}
8477 
8478 instruct rorI_rReg_imm8(rRegI dst, immI8 shift, rFlagsReg cr)
8479 %{
8480   effect(USE_DEF dst, USE shift, KILL cr);
8481 
8482   format %{ "rorl    $dst, $shift" %}
8483   opcode(0xC1, 0x1); /* C1 /1 ib */
8484   ins_encode(reg_opc_imm(dst, shift));
8485   ins_pipe(ialu_reg);
8486 %}
8487 
8488 instruct rorI_rReg_CL(no_rcx_RegI dst, rcx_RegI shift, rFlagsReg cr)
8489 %{
8490   effect(USE_DEF dst, USE shift, KILL cr);
8491 
8492   format %{ "rorl    $dst, $shift" %}
8493   opcode(0xD3, 0x1); /* D3 /1 */
8494   ins_encode(REX_reg(dst), OpcP, reg_opc(dst));
8495   ins_pipe(ialu_reg_reg);
8496 %}
8497 // end of ROR expand
8498 
8499 // Rotate Right by one
8500 instruct rorI_rReg_i1(rRegI dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8501 %{
8502   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8503 
8504   expand %{
8505     rorI_rReg_imm1(dst, cr);
8506   %}
8507 %}
8508 
8509 // Rotate Right by 8-bit immediate
8510 instruct rorI_rReg_i8(rRegI dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8511 %{
8512   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
8513   match(Set dst (OrI (URShiftI dst rshift) (LShiftI dst lshift)));
8514 
8515   expand %{
8516     rorI_rReg_imm8(dst, rshift, cr);
8517   %}
8518 %}
8519 
8520 // Rotate Right by variable
8521 instruct rorI_rReg_Var_C0(no_rcx_RegI dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8522 %{
8523   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
8524 
8525   expand %{
8526     rorI_rReg_CL(dst, shift, cr);
8527   %}
8528 %}
8529 
8530 // Rotate Right by variable
8531 instruct rorI_rReg_Var_C32(no_rcx_RegI dst, rcx_RegI shift, immI_32 c32, rFlagsReg cr)
8532 %{
8533   match(Set dst (OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
8534 
8535   expand %{
8536     rorI_rReg_CL(dst, shift, cr);
8537   %}
8538 %}
8539 
8540 // for long rotate
8541 // ROL expand
8542 instruct rolL_rReg_imm1(rRegL dst, rFlagsReg cr) %{
8543   effect(USE_DEF dst, KILL cr);
8544 
8545   format %{ "rolq    $dst" %}
8546   opcode(0xD1, 0x0); /* Opcode  D1 /0 */
8547   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8548   ins_pipe(ialu_reg);
8549 %}
8550 
8551 instruct rolL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr) %{
8552   effect(USE_DEF dst, USE shift, KILL cr);
8553 
8554   format %{ "rolq    $dst, $shift" %}
8555   opcode(0xC1, 0x0); /* Opcode C1 /0 ib */
8556   ins_encode( reg_opc_imm_wide(dst, shift) );
8557   ins_pipe(ialu_reg);
8558 %}
8559 
8560 instruct rolL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8561 %{
8562   effect(USE_DEF dst, USE shift, KILL cr);
8563 
8564   format %{ "rolq    $dst, $shift" %}
8565   opcode(0xD3, 0x0); /* Opcode D3 /0 */
8566   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8567   ins_pipe(ialu_reg_reg);
8568 %}
8569 // end of ROL expand
8570 
8571 // Rotate Left by one
8572 instruct rolL_rReg_i1(rRegL dst, immI1 lshift, immI_M1 rshift, rFlagsReg cr)
8573 %{
8574   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8575 
8576   expand %{
8577     rolL_rReg_imm1(dst, cr);
8578   %}
8579 %}
8580 
8581 // Rotate Left by 8-bit immediate
8582 instruct rolL_rReg_i8(rRegL dst, immI8 lshift, immI8 rshift, rFlagsReg cr)
8583 %{
8584   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8585   match(Set dst (OrL (LShiftL dst lshift) (URShiftL dst rshift)));
8586 
8587   expand %{
8588     rolL_rReg_imm8(dst, lshift, cr);
8589   %}
8590 %}
8591 
8592 // Rotate Left by variable
8593 instruct rolL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8594 %{
8595   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI zero shift))));
8596 
8597   expand %{
8598     rolL_rReg_CL(dst, shift, cr);
8599   %}
8600 %}
8601 
8602 // Rotate Left by variable
8603 instruct rolL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8604 %{
8605   match(Set dst (OrL (LShiftL dst shift) (URShiftL dst (SubI c64 shift))));
8606 
8607   expand %{
8608     rolL_rReg_CL(dst, shift, cr);
8609   %}
8610 %}
8611 
8612 // ROR expand
8613 instruct rorL_rReg_imm1(rRegL dst, rFlagsReg cr)
8614 %{
8615   effect(USE_DEF dst, KILL cr);
8616 
8617   format %{ "rorq    $dst" %}
8618   opcode(0xD1, 0x1); /* D1 /1 */
8619   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8620   ins_pipe(ialu_reg);
8621 %}
8622 
8623 instruct rorL_rReg_imm8(rRegL dst, immI8 shift, rFlagsReg cr)
8624 %{
8625   effect(USE_DEF dst, USE shift, KILL cr);
8626 
8627   format %{ "rorq    $dst, $shift" %}
8628   opcode(0xC1, 0x1); /* C1 /1 ib */
8629   ins_encode(reg_opc_imm_wide(dst, shift));
8630   ins_pipe(ialu_reg);
8631 %}
8632 
8633 instruct rorL_rReg_CL(no_rcx_RegL dst, rcx_RegI shift, rFlagsReg cr)
8634 %{
8635   effect(USE_DEF dst, USE shift, KILL cr);
8636 
8637   format %{ "rorq    $dst, $shift" %}
8638   opcode(0xD3, 0x1); /* D3 /1 */
8639   ins_encode(REX_reg_wide(dst), OpcP, reg_opc(dst));
8640   ins_pipe(ialu_reg_reg);
8641 %}
8642 // end of ROR expand
8643 
8644 // Rotate Right by one
8645 instruct rorL_rReg_i1(rRegL dst, immI1 rshift, immI_M1 lshift, rFlagsReg cr)
8646 %{
8647   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8648 
8649   expand %{
8650     rorL_rReg_imm1(dst, cr);
8651   %}
8652 %}
8653 
8654 // Rotate Right by 8-bit immediate
8655 instruct rorL_rReg_i8(rRegL dst, immI8 rshift, immI8 lshift, rFlagsReg cr)
8656 %{
8657   predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x3f));
8658   match(Set dst (OrL (URShiftL dst rshift) (LShiftL dst lshift)));
8659 
8660   expand %{
8661     rorL_rReg_imm8(dst, rshift, cr);
8662   %}
8663 %}
8664 
8665 // Rotate Right by variable
8666 instruct rorL_rReg_Var_C0(no_rcx_RegL dst, rcx_RegI shift, immI0 zero, rFlagsReg cr)
8667 %{
8668   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI zero shift))));
8669 
8670   expand %{
8671     rorL_rReg_CL(dst, shift, cr);
8672   %}
8673 %}
8674 
8675 // Rotate Right by variable
8676 instruct rorL_rReg_Var_C64(no_rcx_RegL dst, rcx_RegI shift, immI_64 c64, rFlagsReg cr)
8677 %{
8678   match(Set dst (OrL (URShiftL dst shift) (LShiftL dst (SubI c64 shift))));
8679 
8680   expand %{
8681     rorL_rReg_CL(dst, shift, cr);
8682   %}
8683 %}
8684 
8685 // Logical Instructions
8686 
8687 // Integer Logical Instructions
8688 
8689 // And Instructions
8690 // And Register with Register
8691 instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8692 %{
8693   match(Set dst (AndI dst src));
8694   effect(KILL cr);
8695 
8696   format %{ "andl    $dst, $src\t# int" %}
8697   opcode(0x23);
8698   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8699   ins_pipe(ialu_reg_reg);
8700 %}
8701 
8702 // And Register with Immediate 255
8703 instruct andI_rReg_imm255(rRegI dst, immI_255 src)
8704 %{
8705   match(Set dst (AndI dst src));
8706 
8707   format %{ "movzbl  $dst, $dst\t# int & 0xFF" %}
8708   opcode(0x0F, 0xB6);
8709   ins_encode(REX_reg_breg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8710   ins_pipe(ialu_reg);
8711 %}
8712 
8713 // And Register with Immediate 255 and promote to long
8714 instruct andI2L_rReg_imm255(rRegL dst, rRegI src, immI_255 mask)
8715 %{
8716   match(Set dst (ConvI2L (AndI src mask)));
8717 
8718   format %{ "movzbl  $dst, $src\t# int & 0xFF -> long" %}
8719   opcode(0x0F, 0xB6);
8720   ins_encode(REX_reg_breg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8721   ins_pipe(ialu_reg);
8722 %}
8723 
8724 // And Register with Immediate 65535
8725 instruct andI_rReg_imm65535(rRegI dst, immI_65535 src)
8726 %{
8727   match(Set dst (AndI dst src));
8728 
8729   format %{ "movzwl  $dst, $dst\t# int & 0xFFFF" %}
8730   opcode(0x0F, 0xB7);
8731   ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
8732   ins_pipe(ialu_reg);
8733 %}
8734 
8735 // And Register with Immediate 65535 and promote to long
8736 instruct andI2L_rReg_imm65535(rRegL dst, rRegI src, immI_65535 mask)
8737 %{
8738   match(Set dst (ConvI2L (AndI src mask)));
8739 
8740   format %{ "movzwl  $dst, $src\t# int & 0xFFFF -> long" %}
8741   opcode(0x0F, 0xB7);
8742   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
8743   ins_pipe(ialu_reg);
8744 %}
8745 
8746 // And Register with Immediate
8747 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8748 %{
8749   match(Set dst (AndI dst src));
8750   effect(KILL cr);
8751 
8752   format %{ "andl    $dst, $src\t# int" %}
8753   opcode(0x81, 0x04); /* Opcode 81 /4 */
8754   ins_encode(OpcSErm(dst, src), Con8or32(src));
8755   ins_pipe(ialu_reg);
8756 %}
8757 
8758 // And Register with Memory
8759 instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8760 %{
8761   match(Set dst (AndI dst (LoadI src)));
8762   effect(KILL cr);
8763 
8764   ins_cost(125);
8765   format %{ "andl    $dst, $src\t# int" %}
8766   opcode(0x23);
8767   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8768   ins_pipe(ialu_reg_mem);
8769 %}
8770 
8771 // And Memory with Register
8772 instruct andI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8773 %{
8774   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8775   effect(KILL cr);
8776 
8777   ins_cost(150);
8778   format %{ "andl    $dst, $src\t# int" %}
8779   opcode(0x21); /* Opcode 21 /r */
8780   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8781   ins_pipe(ialu_mem_reg);
8782 %}
8783 
8784 // And Memory with Immediate
8785 instruct andI_mem_imm(memory dst, immI src, rFlagsReg cr)
8786 %{
8787   match(Set dst (StoreI dst (AndI (LoadI dst) src)));
8788   effect(KILL cr);
8789 
8790   ins_cost(125);
8791   format %{ "andl    $dst, $src\t# int" %}
8792   opcode(0x81, 0x4); /* Opcode 81 /4 id */
8793   ins_encode(REX_mem(dst), OpcSE(src),
8794              RM_opc_mem(secondary, dst), Con8or32(src));
8795   ins_pipe(ialu_mem_imm);
8796 %}
8797 
8798 // BMI1 instructions
8799 instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, rFlagsReg cr) %{
8800   match(Set dst (AndI (XorI src1 minus_1) (LoadI src2)));
8801   predicate(UseBMI1Instructions);
8802   effect(KILL cr);
8803 
8804   ins_cost(125);
8805   format %{ "andnl  $dst, $src1, $src2" %}
8806 
8807   ins_encode %{
8808     __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
8809   %}
8810   ins_pipe(ialu_reg_mem);
8811 %}
8812 
8813 instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, rFlagsReg cr) %{
8814   match(Set dst (AndI (XorI src1 minus_1) src2));
8815   predicate(UseBMI1Instructions);
8816   effect(KILL cr);
8817 
8818   format %{ "andnl  $dst, $src1, $src2" %}
8819 
8820   ins_encode %{
8821     __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
8822   %}
8823   ins_pipe(ialu_reg);
8824 %}
8825 
8826 instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI0 imm_zero, rFlagsReg cr) %{
8827   match(Set dst (AndI (SubI imm_zero src) src));
8828   predicate(UseBMI1Instructions);
8829   effect(KILL cr);
8830 
8831   format %{ "blsil  $dst, $src" %}
8832 
8833   ins_encode %{
8834     __ blsil($dst$$Register, $src$$Register);
8835   %}
8836   ins_pipe(ialu_reg);
8837 %}
8838 
8839 instruct blsiI_rReg_mem(rRegI dst, memory src, immI0 imm_zero, rFlagsReg cr) %{
8840   match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
8841   predicate(UseBMI1Instructions);
8842   effect(KILL cr);
8843 
8844   ins_cost(125);
8845   format %{ "blsil  $dst, $src" %}
8846 
8847   ins_encode %{
8848     __ blsil($dst$$Register, $src$$Address);
8849   %}
8850   ins_pipe(ialu_reg_mem);
8851 %}
8852 
8853 instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8854 %{
8855   match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ) );
8856   predicate(UseBMI1Instructions);
8857   effect(KILL cr);
8858 
8859   ins_cost(125);
8860   format %{ "blsmskl $dst, $src" %}
8861 
8862   ins_encode %{
8863     __ blsmskl($dst$$Register, $src$$Address);
8864   %}
8865   ins_pipe(ialu_reg_mem);
8866 %}
8867 
8868 instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8869 %{
8870   match(Set dst (XorI (AddI src minus_1) src));
8871   predicate(UseBMI1Instructions);
8872   effect(KILL cr);
8873 
8874   format %{ "blsmskl $dst, $src" %}
8875 
8876   ins_encode %{
8877     __ blsmskl($dst$$Register, $src$$Register);
8878   %}
8879 
8880   ins_pipe(ialu_reg);
8881 %}
8882 
8883 instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, rFlagsReg cr)
8884 %{
8885   match(Set dst (AndI (AddI src minus_1) src) );
8886   predicate(UseBMI1Instructions);
8887   effect(KILL cr);
8888 
8889   format %{ "blsrl  $dst, $src" %}
8890 
8891   ins_encode %{
8892     __ blsrl($dst$$Register, $src$$Register);
8893   %}
8894 
8895   ins_pipe(ialu_reg_mem);
8896 %}
8897 
8898 instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
8899 %{
8900   match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ) );
8901   predicate(UseBMI1Instructions);
8902   effect(KILL cr);
8903 
8904   ins_cost(125);
8905   format %{ "blsrl  $dst, $src" %}
8906 
8907   ins_encode %{
8908     __ blsrl($dst$$Register, $src$$Address);
8909   %}
8910 
8911   ins_pipe(ialu_reg);
8912 %}
8913 
8914 // Or Instructions
8915 // Or Register with Register
8916 instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8917 %{
8918   match(Set dst (OrI dst src));
8919   effect(KILL cr);
8920 
8921   format %{ "orl     $dst, $src\t# int" %}
8922   opcode(0x0B);
8923   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8924   ins_pipe(ialu_reg_reg);
8925 %}
8926 
8927 // Or Register with Immediate
8928 instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
8929 %{
8930   match(Set dst (OrI dst src));
8931   effect(KILL cr);
8932 
8933   format %{ "orl     $dst, $src\t# int" %}
8934   opcode(0x81, 0x01); /* Opcode 81 /1 id */
8935   ins_encode(OpcSErm(dst, src), Con8or32(src));
8936   ins_pipe(ialu_reg);
8937 %}
8938 
8939 // Or Register with Memory
8940 instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
8941 %{
8942   match(Set dst (OrI dst (LoadI src)));
8943   effect(KILL cr);
8944 
8945   ins_cost(125);
8946   format %{ "orl     $dst, $src\t# int" %}
8947   opcode(0x0B);
8948   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
8949   ins_pipe(ialu_reg_mem);
8950 %}
8951 
8952 // Or Memory with Register
8953 instruct orI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
8954 %{
8955   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8956   effect(KILL cr);
8957 
8958   ins_cost(150);
8959   format %{ "orl     $dst, $src\t# int" %}
8960   opcode(0x09); /* Opcode 09 /r */
8961   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
8962   ins_pipe(ialu_mem_reg);
8963 %}
8964 
8965 // Or Memory with Immediate
8966 instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
8967 %{
8968   match(Set dst (StoreI dst (OrI (LoadI dst) src)));
8969   effect(KILL cr);
8970 
8971   ins_cost(125);
8972   format %{ "orl     $dst, $src\t# int" %}
8973   opcode(0x81, 0x1); /* Opcode 81 /1 id */
8974   ins_encode(REX_mem(dst), OpcSE(src),
8975              RM_opc_mem(secondary, dst), Con8or32(src));
8976   ins_pipe(ialu_mem_imm);
8977 %}
8978 
8979 // Xor Instructions
8980 // Xor Register with Register
8981 instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
8982 %{
8983   match(Set dst (XorI dst src));
8984   effect(KILL cr);
8985 
8986   format %{ "xorl    $dst, $src\t# int" %}
8987   opcode(0x33);
8988   ins_encode(REX_reg_reg(dst, src), OpcP, reg_reg(dst, src));
8989   ins_pipe(ialu_reg_reg);
8990 %}
8991 
8992 // Xor Register with Immediate -1
8993 instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
8994   match(Set dst (XorI dst imm));
8995 
8996   format %{ "not    $dst" %}
8997   ins_encode %{
8998      __ notl($dst$$Register);
8999   %}
9000   ins_pipe(ialu_reg);
9001 %}
9002 
9003 // Xor Register with Immediate
9004 instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
9005 %{
9006   match(Set dst (XorI dst src));
9007   effect(KILL cr);
9008 
9009   format %{ "xorl    $dst, $src\t# int" %}
9010   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9011   ins_encode(OpcSErm(dst, src), Con8or32(src));
9012   ins_pipe(ialu_reg);
9013 %}
9014 
9015 // Xor Register with Memory
9016 instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
9017 %{
9018   match(Set dst (XorI dst (LoadI src)));
9019   effect(KILL cr);
9020 
9021   ins_cost(125);
9022   format %{ "xorl    $dst, $src\t# int" %}
9023   opcode(0x33);
9024   ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
9025   ins_pipe(ialu_reg_mem);
9026 %}
9027 
9028 // Xor Memory with Register
9029 instruct xorI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
9030 %{
9031   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9032   effect(KILL cr);
9033 
9034   ins_cost(150);
9035   format %{ "xorl    $dst, $src\t# int" %}
9036   opcode(0x31); /* Opcode 31 /r */
9037   ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
9038   ins_pipe(ialu_mem_reg);
9039 %}
9040 
9041 // Xor Memory with Immediate
9042 instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
9043 %{
9044   match(Set dst (StoreI dst (XorI (LoadI dst) src)));
9045   effect(KILL cr);
9046 
9047   ins_cost(125);
9048   format %{ "xorl    $dst, $src\t# int" %}
9049   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9050   ins_encode(REX_mem(dst), OpcSE(src),
9051              RM_opc_mem(secondary, dst), Con8or32(src));
9052   ins_pipe(ialu_mem_imm);
9053 %}
9054 
9055 
9056 // Long Logical Instructions
9057 
9058 // And Instructions
9059 // And Register with Register
9060 instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9061 %{
9062   match(Set dst (AndL dst src));
9063   effect(KILL cr);
9064 
9065   format %{ "andq    $dst, $src\t# long" %}
9066   opcode(0x23);
9067   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9068   ins_pipe(ialu_reg_reg);
9069 %}
9070 
9071 // And Register with Immediate 255
9072 instruct andL_rReg_imm255(rRegL dst, immL_255 src)
9073 %{
9074   match(Set dst (AndL dst src));
9075 
9076   format %{ "movzbq  $dst, $dst\t# long & 0xFF" %}
9077   opcode(0x0F, 0xB6);
9078   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9079   ins_pipe(ialu_reg);
9080 %}
9081 
9082 // And Register with Immediate 65535
9083 instruct andL_rReg_imm65535(rRegL dst, immL_65535 src)
9084 %{
9085   match(Set dst (AndL dst src));
9086 
9087   format %{ "movzwq  $dst, $dst\t# long & 0xFFFF" %}
9088   opcode(0x0F, 0xB7);
9089   ins_encode(REX_reg_reg_wide(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
9090   ins_pipe(ialu_reg);
9091 %}
9092 
9093 // And Register with Immediate
9094 instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9095 %{
9096   match(Set dst (AndL dst src));
9097   effect(KILL cr);
9098 
9099   format %{ "andq    $dst, $src\t# long" %}
9100   opcode(0x81, 0x04); /* Opcode 81 /4 */
9101   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9102   ins_pipe(ialu_reg);
9103 %}
9104 
9105 // And Register with Memory
9106 instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9107 %{
9108   match(Set dst (AndL dst (LoadL src)));
9109   effect(KILL cr);
9110 
9111   ins_cost(125);
9112   format %{ "andq    $dst, $src\t# long" %}
9113   opcode(0x23);
9114   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9115   ins_pipe(ialu_reg_mem);
9116 %}
9117 
9118 // And Memory with Register
9119 instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9120 %{
9121   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9122   effect(KILL cr);
9123 
9124   ins_cost(150);
9125   format %{ "andq    $dst, $src\t# long" %}
9126   opcode(0x21); /* Opcode 21 /r */
9127   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9128   ins_pipe(ialu_mem_reg);
9129 %}
9130 
9131 // And Memory with Immediate
9132 instruct andL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9133 %{
9134   match(Set dst (StoreL dst (AndL (LoadL dst) src)));
9135   effect(KILL cr);
9136 
9137   ins_cost(125);
9138   format %{ "andq    $dst, $src\t# long" %}
9139   opcode(0x81, 0x4); /* Opcode 81 /4 id */
9140   ins_encode(REX_mem_wide(dst), OpcSE(src),
9141              RM_opc_mem(secondary, dst), Con8or32(src));
9142   ins_pipe(ialu_mem_imm);
9143 %}
9144 
9145 // BMI1 instructions
9146 instruct andnL_rReg_rReg_mem(rRegL dst, rRegL src1, memory src2, immL_M1 minus_1, rFlagsReg cr) %{
9147   match(Set dst (AndL (XorL src1 minus_1) (LoadL src2)));
9148   predicate(UseBMI1Instructions);
9149   effect(KILL cr);
9150 
9151   ins_cost(125);
9152   format %{ "andnq  $dst, $src1, $src2" %}
9153 
9154   ins_encode %{
9155     __ andnq($dst$$Register, $src1$$Register, $src2$$Address);
9156   %}
9157   ins_pipe(ialu_reg_mem);
9158 %}
9159 
9160 instruct andnL_rReg_rReg_rReg(rRegL dst, rRegL src1, rRegL src2, immL_M1 minus_1, rFlagsReg cr) %{
9161   match(Set dst (AndL (XorL src1 minus_1) src2));
9162   predicate(UseBMI1Instructions);
9163   effect(KILL cr);
9164 
9165   format %{ "andnq  $dst, $src1, $src2" %}
9166 
9167   ins_encode %{
9168   __ andnq($dst$$Register, $src1$$Register, $src2$$Register);
9169   %}
9170   ins_pipe(ialu_reg_mem);
9171 %}
9172 
9173 instruct blsiL_rReg_rReg(rRegL dst, rRegL src, immL0 imm_zero, rFlagsReg cr) %{
9174   match(Set dst (AndL (SubL imm_zero src) src));
9175   predicate(UseBMI1Instructions);
9176   effect(KILL cr);
9177 
9178   format %{ "blsiq  $dst, $src" %}
9179 
9180   ins_encode %{
9181     __ blsiq($dst$$Register, $src$$Register);
9182   %}
9183   ins_pipe(ialu_reg);
9184 %}
9185 
9186 instruct blsiL_rReg_mem(rRegL dst, memory src, immL0 imm_zero, rFlagsReg cr) %{
9187   match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
9188   predicate(UseBMI1Instructions);
9189   effect(KILL cr);
9190 
9191   ins_cost(125);
9192   format %{ "blsiq  $dst, $src" %}
9193 
9194   ins_encode %{
9195     __ blsiq($dst$$Register, $src$$Address);
9196   %}
9197   ins_pipe(ialu_reg_mem);
9198 %}
9199 
9200 instruct blsmskL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9201 %{
9202   match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ) );
9203   predicate(UseBMI1Instructions);
9204   effect(KILL cr);
9205 
9206   ins_cost(125);
9207   format %{ "blsmskq $dst, $src" %}
9208 
9209   ins_encode %{
9210     __ blsmskq($dst$$Register, $src$$Address);
9211   %}
9212   ins_pipe(ialu_reg_mem);
9213 %}
9214 
9215 instruct blsmskL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9216 %{
9217   match(Set dst (XorL (AddL src minus_1) src));
9218   predicate(UseBMI1Instructions);
9219   effect(KILL cr);
9220 
9221   format %{ "blsmskq $dst, $src" %}
9222 
9223   ins_encode %{
9224     __ blsmskq($dst$$Register, $src$$Register);
9225   %}
9226 
9227   ins_pipe(ialu_reg);
9228 %}
9229 
9230 instruct blsrL_rReg_rReg(rRegL dst, rRegL src, immL_M1 minus_1, rFlagsReg cr)
9231 %{
9232   match(Set dst (AndL (AddL src minus_1) src) );
9233   predicate(UseBMI1Instructions);
9234   effect(KILL cr);
9235 
9236   format %{ "blsrq  $dst, $src" %}
9237 
9238   ins_encode %{
9239     __ blsrq($dst$$Register, $src$$Register);
9240   %}
9241 
9242   ins_pipe(ialu_reg);
9243 %}
9244 
9245 instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
9246 %{
9247   match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src)) );
9248   predicate(UseBMI1Instructions);
9249   effect(KILL cr);
9250 
9251   ins_cost(125);
9252   format %{ "blsrq  $dst, $src" %}
9253 
9254   ins_encode %{
9255     __ blsrq($dst$$Register, $src$$Address);
9256   %}
9257 
9258   ins_pipe(ialu_reg);
9259 %}
9260 
9261 // Or Instructions
9262 // Or Register with Register
9263 instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9264 %{
9265   match(Set dst (OrL dst src));
9266   effect(KILL cr);
9267 
9268   format %{ "orq     $dst, $src\t# long" %}
9269   opcode(0x0B);
9270   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9271   ins_pipe(ialu_reg_reg);
9272 %}
9273 
9274 // Use any_RegP to match R15 (TLS register) without spilling.
9275 instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
9276   match(Set dst (OrL dst (CastP2X src)));
9277   effect(KILL cr);
9278 
9279   format %{ "orq     $dst, $src\t# long" %}
9280   opcode(0x0B);
9281   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9282   ins_pipe(ialu_reg_reg);
9283 %}
9284 
9285 
9286 // Or Register with Immediate
9287 instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9288 %{
9289   match(Set dst (OrL dst src));
9290   effect(KILL cr);
9291 
9292   format %{ "orq     $dst, $src\t# long" %}
9293   opcode(0x81, 0x01); /* Opcode 81 /1 id */
9294   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9295   ins_pipe(ialu_reg);
9296 %}
9297 
9298 // Or Register with Memory
9299 instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9300 %{
9301   match(Set dst (OrL dst (LoadL src)));
9302   effect(KILL cr);
9303 
9304   ins_cost(125);
9305   format %{ "orq     $dst, $src\t# long" %}
9306   opcode(0x0B);
9307   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9308   ins_pipe(ialu_reg_mem);
9309 %}
9310 
9311 // Or Memory with Register
9312 instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9313 %{
9314   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9315   effect(KILL cr);
9316 
9317   ins_cost(150);
9318   format %{ "orq     $dst, $src\t# long" %}
9319   opcode(0x09); /* Opcode 09 /r */
9320   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9321   ins_pipe(ialu_mem_reg);
9322 %}
9323 
9324 // Or Memory with Immediate
9325 instruct orL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9326 %{
9327   match(Set dst (StoreL dst (OrL (LoadL dst) src)));
9328   effect(KILL cr);
9329 
9330   ins_cost(125);
9331   format %{ "orq     $dst, $src\t# long" %}
9332   opcode(0x81, 0x1); /* Opcode 81 /1 id */
9333   ins_encode(REX_mem_wide(dst), OpcSE(src),
9334              RM_opc_mem(secondary, dst), Con8or32(src));
9335   ins_pipe(ialu_mem_imm);
9336 %}
9337 
9338 // Xor Instructions
9339 // Xor Register with Register
9340 instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
9341 %{
9342   match(Set dst (XorL dst src));
9343   effect(KILL cr);
9344 
9345   format %{ "xorq    $dst, $src\t# long" %}
9346   opcode(0x33);
9347   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst, src));
9348   ins_pipe(ialu_reg_reg);
9349 %}
9350 
9351 // Xor Register with Immediate -1
9352 instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
9353   match(Set dst (XorL dst imm));
9354 
9355   format %{ "notq   $dst" %}
9356   ins_encode %{
9357      __ notq($dst$$Register);
9358   %}
9359   ins_pipe(ialu_reg);
9360 %}
9361 
9362 // Xor Register with Immediate
9363 instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
9364 %{
9365   match(Set dst (XorL dst src));
9366   effect(KILL cr);
9367 
9368   format %{ "xorq    $dst, $src\t# long" %}
9369   opcode(0x81, 0x06); /* Opcode 81 /6 id */
9370   ins_encode(OpcSErm_wide(dst, src), Con8or32(src));
9371   ins_pipe(ialu_reg);
9372 %}
9373 
9374 // Xor Register with Memory
9375 instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
9376 %{
9377   match(Set dst (XorL dst (LoadL src)));
9378   effect(KILL cr);
9379 
9380   ins_cost(125);
9381   format %{ "xorq    $dst, $src\t# long" %}
9382   opcode(0x33);
9383   ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
9384   ins_pipe(ialu_reg_mem);
9385 %}
9386 
9387 // Xor Memory with Register
9388 instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
9389 %{
9390   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9391   effect(KILL cr);
9392 
9393   ins_cost(150);
9394   format %{ "xorq    $dst, $src\t# long" %}
9395   opcode(0x31); /* Opcode 31 /r */
9396   ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
9397   ins_pipe(ialu_mem_reg);
9398 %}
9399 
9400 // Xor Memory with Immediate
9401 instruct xorL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
9402 %{
9403   match(Set dst (StoreL dst (XorL (LoadL dst) src)));
9404   effect(KILL cr);
9405 
9406   ins_cost(125);
9407   format %{ "xorq    $dst, $src\t# long" %}
9408   opcode(0x81, 0x6); /* Opcode 81 /6 id */
9409   ins_encode(REX_mem_wide(dst), OpcSE(src),
9410              RM_opc_mem(secondary, dst), Con8or32(src));
9411   ins_pipe(ialu_mem_imm);
9412 %}
9413 
9414 // Convert Int to Boolean
9415 instruct convI2B(rRegI dst, rRegI src, rFlagsReg cr)
9416 %{
9417   match(Set dst (Conv2B src));
9418   effect(KILL cr);
9419 
9420   format %{ "testl   $src, $src\t# ci2b\n\t"
9421             "setnz   $dst\n\t"
9422             "movzbl  $dst, $dst" %}
9423   ins_encode(REX_reg_reg(src, src), opc_reg_reg(0x85, src, src), // testl
9424              setNZ_reg(dst),
9425              REX_reg_breg(dst, dst), // movzbl
9426              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9427   ins_pipe(pipe_slow); // XXX
9428 %}
9429 
9430 // Convert Pointer to Boolean
9431 instruct convP2B(rRegI dst, rRegP src, rFlagsReg cr)
9432 %{
9433   match(Set dst (Conv2B src));
9434   effect(KILL cr);
9435 
9436   format %{ "testq   $src, $src\t# cp2b\n\t"
9437             "setnz   $dst\n\t"
9438             "movzbl  $dst, $dst" %}
9439   ins_encode(REX_reg_reg_wide(src, src), opc_reg_reg(0x85, src, src), // testq
9440              setNZ_reg(dst),
9441              REX_reg_breg(dst, dst), // movzbl
9442              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst));
9443   ins_pipe(pipe_slow); // XXX
9444 %}
9445 
9446 instruct cmpLTMask(rRegI dst, rRegI p, rRegI q, rFlagsReg cr)
9447 %{
9448   match(Set dst (CmpLTMask p q));
9449   effect(KILL cr);
9450 
9451   ins_cost(400);
9452   format %{ "cmpl    $p, $q\t# cmpLTMask\n\t"
9453             "setlt   $dst\n\t"
9454             "movzbl  $dst, $dst\n\t"
9455             "negl    $dst" %}
9456   ins_encode(REX_reg_reg(p, q), opc_reg_reg(0x3B, p, q), // cmpl
9457              setLT_reg(dst),
9458              REX_reg_breg(dst, dst), // movzbl
9459              Opcode(0x0F), Opcode(0xB6), reg_reg(dst, dst),
9460              neg_reg(dst));
9461   ins_pipe(pipe_slow);
9462 %}
9463 
9464 instruct cmpLTMask0(rRegI dst, immI0 zero, rFlagsReg cr)
9465 %{
9466   match(Set dst (CmpLTMask dst zero));
9467   effect(KILL cr);
9468 
9469   ins_cost(100);
9470   format %{ "sarl    $dst, #31\t# cmpLTMask0" %}
9471   ins_encode %{
9472   __ sarl($dst$$Register, 31);
9473   %}
9474   ins_pipe(ialu_reg);
9475 %}
9476 
9477 /* Better to save a register than avoid a branch */
9478 instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9479 %{
9480   match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
9481   effect(KILL cr);
9482   ins_cost(300);
9483   format %{ "subl   $p,$q\t# cadd_cmpLTMask\n\t"
9484             "jge    done\n\t"
9485             "addl   $p,$y\n"
9486             "done:  " %}
9487   ins_encode %{
9488     Register Rp = $p$$Register;
9489     Register Rq = $q$$Register;
9490     Register Ry = $y$$Register;
9491     Label done;
9492     __ subl(Rp, Rq);
9493     __ jccb(Assembler::greaterEqual, done);
9494     __ addl(Rp, Ry);
9495     __ bind(done);
9496   %}
9497   ins_pipe(pipe_cmplt);
9498 %}
9499 
9500 /* Better to save a register than avoid a branch */
9501 instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, rFlagsReg cr)
9502 %{
9503   match(Set y (AndI (CmpLTMask p q) y));
9504   effect(KILL cr);
9505 
9506   ins_cost(300);
9507 
9508   format %{ "cmpl     $p, $q\t# and_cmpLTMask\n\t"
9509             "jlt      done\n\t"
9510             "xorl     $y, $y\n"
9511             "done:  " %}
9512   ins_encode %{
9513     Register Rp = $p$$Register;
9514     Register Rq = $q$$Register;
9515     Register Ry = $y$$Register;
9516     Label done;
9517     __ cmpl(Rp, Rq);
9518     __ jccb(Assembler::less, done);
9519     __ xorl(Ry, Ry);
9520     __ bind(done);
9521   %}
9522   ins_pipe(pipe_cmplt);
9523 %}
9524 
9525 
9526 //---------- FP Instructions------------------------------------------------
9527 
9528 instruct cmpF_cc_reg(rFlagsRegU cr, regF src1, regF src2)
9529 %{
9530   match(Set cr (CmpF src1 src2));
9531 
9532   ins_cost(145);
9533   format %{ "ucomiss $src1, $src2\n\t"
9534             "jnp,s   exit\n\t"
9535             "pushfq\t# saw NaN, set CF\n\t"
9536             "andq    [rsp], #0xffffff2b\n\t"
9537             "popfq\n"
9538     "exit:" %}
9539   ins_encode %{
9540     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9541     emit_cmpfp_fixup(_masm);
9542   %}
9543   ins_pipe(pipe_slow);
9544 %}
9545 
9546 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
9547   match(Set cr (CmpF src1 src2));
9548 
9549   ins_cost(100);
9550   format %{ "ucomiss $src1, $src2" %}
9551   ins_encode %{
9552     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9553   %}
9554   ins_pipe(pipe_slow);
9555 %}
9556 
9557 instruct cmpF_cc_mem(rFlagsRegU cr, regF src1, memory src2)
9558 %{
9559   match(Set cr (CmpF src1 (LoadF src2)));
9560 
9561   ins_cost(145);
9562   format %{ "ucomiss $src1, $src2\n\t"
9563             "jnp,s   exit\n\t"
9564             "pushfq\t# saw NaN, set CF\n\t"
9565             "andq    [rsp], #0xffffff2b\n\t"
9566             "popfq\n"
9567     "exit:" %}
9568   ins_encode %{
9569     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9570     emit_cmpfp_fixup(_masm);
9571   %}
9572   ins_pipe(pipe_slow);
9573 %}
9574 
9575 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
9576   match(Set cr (CmpF src1 (LoadF src2)));
9577 
9578   ins_cost(100);
9579   format %{ "ucomiss $src1, $src2" %}
9580   ins_encode %{
9581     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9582   %}
9583   ins_pipe(pipe_slow);
9584 %}
9585 
9586 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
9587   match(Set cr (CmpF src con));
9588 
9589   ins_cost(145);
9590   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9591             "jnp,s   exit\n\t"
9592             "pushfq\t# saw NaN, set CF\n\t"
9593             "andq    [rsp], #0xffffff2b\n\t"
9594             "popfq\n"
9595     "exit:" %}
9596   ins_encode %{
9597     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9598     emit_cmpfp_fixup(_masm);
9599   %}
9600   ins_pipe(pipe_slow);
9601 %}
9602 
9603 instruct cmpF_cc_immCF(rFlagsRegUCF cr, regF src, immF con) %{
9604   match(Set cr (CmpF src con));
9605   ins_cost(100);
9606   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con" %}
9607   ins_encode %{
9608     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9609   %}
9610   ins_pipe(pipe_slow);
9611 %}
9612 
9613 instruct cmpD_cc_reg(rFlagsRegU cr, regD src1, regD src2)
9614 %{
9615   match(Set cr (CmpD src1 src2));
9616 
9617   ins_cost(145);
9618   format %{ "ucomisd $src1, $src2\n\t"
9619             "jnp,s   exit\n\t"
9620             "pushfq\t# saw NaN, set CF\n\t"
9621             "andq    [rsp], #0xffffff2b\n\t"
9622             "popfq\n"
9623     "exit:" %}
9624   ins_encode %{
9625     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9626     emit_cmpfp_fixup(_masm);
9627   %}
9628   ins_pipe(pipe_slow);
9629 %}
9630 
9631 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
9632   match(Set cr (CmpD src1 src2));
9633 
9634   ins_cost(100);
9635   format %{ "ucomisd $src1, $src2 test" %}
9636   ins_encode %{
9637     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9638   %}
9639   ins_pipe(pipe_slow);
9640 %}
9641 
9642 instruct cmpD_cc_mem(rFlagsRegU cr, regD src1, memory src2)
9643 %{
9644   match(Set cr (CmpD src1 (LoadD src2)));
9645 
9646   ins_cost(145);
9647   format %{ "ucomisd $src1, $src2\n\t"
9648             "jnp,s   exit\n\t"
9649             "pushfq\t# saw NaN, set CF\n\t"
9650             "andq    [rsp], #0xffffff2b\n\t"
9651             "popfq\n"
9652     "exit:" %}
9653   ins_encode %{
9654     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9655     emit_cmpfp_fixup(_masm);
9656   %}
9657   ins_pipe(pipe_slow);
9658 %}
9659 
9660 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
9661   match(Set cr (CmpD src1 (LoadD src2)));
9662 
9663   ins_cost(100);
9664   format %{ "ucomisd $src1, $src2" %}
9665   ins_encode %{
9666     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9667   %}
9668   ins_pipe(pipe_slow);
9669 %}
9670 
9671 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
9672   match(Set cr (CmpD src con));
9673 
9674   ins_cost(145);
9675   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9676             "jnp,s   exit\n\t"
9677             "pushfq\t# saw NaN, set CF\n\t"
9678             "andq    [rsp], #0xffffff2b\n\t"
9679             "popfq\n"
9680     "exit:" %}
9681   ins_encode %{
9682     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9683     emit_cmpfp_fixup(_masm);
9684   %}
9685   ins_pipe(pipe_slow);
9686 %}
9687 
9688 instruct cmpD_cc_immCF(rFlagsRegUCF cr, regD src, immD con) %{
9689   match(Set cr (CmpD src con));
9690   ins_cost(100);
9691   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con" %}
9692   ins_encode %{
9693     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9694   %}
9695   ins_pipe(pipe_slow);
9696 %}
9697 
9698 // Compare into -1,0,1
9699 instruct cmpF_reg(rRegI dst, regF src1, regF src2, rFlagsReg cr)
9700 %{
9701   match(Set dst (CmpF3 src1 src2));
9702   effect(KILL cr);
9703 
9704   ins_cost(275);
9705   format %{ "ucomiss $src1, $src2\n\t"
9706             "movl    $dst, #-1\n\t"
9707             "jp,s    done\n\t"
9708             "jb,s    done\n\t"
9709             "setne   $dst\n\t"
9710             "movzbl  $dst, $dst\n"
9711     "done:" %}
9712   ins_encode %{
9713     __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
9714     emit_cmpfp3(_masm, $dst$$Register);
9715   %}
9716   ins_pipe(pipe_slow);
9717 %}
9718 
9719 // Compare into -1,0,1
9720 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
9721 %{
9722   match(Set dst (CmpF3 src1 (LoadF src2)));
9723   effect(KILL cr);
9724 
9725   ins_cost(275);
9726   format %{ "ucomiss $src1, $src2\n\t"
9727             "movl    $dst, #-1\n\t"
9728             "jp,s    done\n\t"
9729             "jb,s    done\n\t"
9730             "setne   $dst\n\t"
9731             "movzbl  $dst, $dst\n"
9732     "done:" %}
9733   ins_encode %{
9734     __ ucomiss($src1$$XMMRegister, $src2$$Address);
9735     emit_cmpfp3(_masm, $dst$$Register);
9736   %}
9737   ins_pipe(pipe_slow);
9738 %}
9739 
9740 // Compare into -1,0,1
9741 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
9742   match(Set dst (CmpF3 src con));
9743   effect(KILL cr);
9744 
9745   ins_cost(275);
9746   format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
9747             "movl    $dst, #-1\n\t"
9748             "jp,s    done\n\t"
9749             "jb,s    done\n\t"
9750             "setne   $dst\n\t"
9751             "movzbl  $dst, $dst\n"
9752     "done:" %}
9753   ins_encode %{
9754     __ ucomiss($src$$XMMRegister, $constantaddress($con));
9755     emit_cmpfp3(_masm, $dst$$Register);
9756   %}
9757   ins_pipe(pipe_slow);
9758 %}
9759 
9760 // Compare into -1,0,1
9761 instruct cmpD_reg(rRegI dst, regD src1, regD src2, rFlagsReg cr)
9762 %{
9763   match(Set dst (CmpD3 src1 src2));
9764   effect(KILL cr);
9765 
9766   ins_cost(275);
9767   format %{ "ucomisd $src1, $src2\n\t"
9768             "movl    $dst, #-1\n\t"
9769             "jp,s    done\n\t"
9770             "jb,s    done\n\t"
9771             "setne   $dst\n\t"
9772             "movzbl  $dst, $dst\n"
9773     "done:" %}
9774   ins_encode %{
9775     __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
9776     emit_cmpfp3(_masm, $dst$$Register);
9777   %}
9778   ins_pipe(pipe_slow);
9779 %}
9780 
9781 // Compare into -1,0,1
9782 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
9783 %{
9784   match(Set dst (CmpD3 src1 (LoadD src2)));
9785   effect(KILL cr);
9786 
9787   ins_cost(275);
9788   format %{ "ucomisd $src1, $src2\n\t"
9789             "movl    $dst, #-1\n\t"
9790             "jp,s    done\n\t"
9791             "jb,s    done\n\t"
9792             "setne   $dst\n\t"
9793             "movzbl  $dst, $dst\n"
9794     "done:" %}
9795   ins_encode %{
9796     __ ucomisd($src1$$XMMRegister, $src2$$Address);
9797     emit_cmpfp3(_masm, $dst$$Register);
9798   %}
9799   ins_pipe(pipe_slow);
9800 %}
9801 
9802 // Compare into -1,0,1
9803 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
9804   match(Set dst (CmpD3 src con));
9805   effect(KILL cr);
9806 
9807   ins_cost(275);
9808   format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
9809             "movl    $dst, #-1\n\t"
9810             "jp,s    done\n\t"
9811             "jb,s    done\n\t"
9812             "setne   $dst\n\t"
9813             "movzbl  $dst, $dst\n"
9814     "done:" %}
9815   ins_encode %{
9816     __ ucomisd($src$$XMMRegister, $constantaddress($con));
9817     emit_cmpfp3(_masm, $dst$$Register);
9818   %}
9819   ins_pipe(pipe_slow);
9820 %}
9821 
9822 // -----------Trig and Trancendental Instructions------------------------------
9823 instruct cosD_reg(regD dst) %{
9824   match(Set dst (CosD dst));
9825 
9826   format %{ "dcos   $dst\n\t" %}
9827   opcode(0xD9, 0xFF);
9828   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9829   ins_pipe( pipe_slow );
9830 %}
9831 
9832 instruct sinD_reg(regD dst) %{
9833   match(Set dst (SinD dst));
9834 
9835   format %{ "dsin   $dst\n\t" %}
9836   opcode(0xD9, 0xFE);
9837   ins_encode( Push_SrcXD(dst), OpcP, OpcS, Push_ResultXD(dst) );
9838   ins_pipe( pipe_slow );
9839 %}
9840 
9841 instruct tanD_reg(regD dst) %{
9842   match(Set dst (TanD dst));
9843 
9844   format %{ "dtan   $dst\n\t" %}
9845   ins_encode( Push_SrcXD(dst),
9846               Opcode(0xD9), Opcode(0xF2),   //fptan
9847               Opcode(0xDD), Opcode(0xD8),   //fstp st
9848               Push_ResultXD(dst) );
9849   ins_pipe( pipe_slow );
9850 %}
9851 
9852 instruct log10D_reg(regD dst) %{
9853   // The source and result Double operands in XMM registers
9854   match(Set dst (Log10D dst));
9855   // fldlg2       ; push log_10(2) on the FPU stack; full 80-bit number
9856   // fyl2x        ; compute log_10(2) * log_2(x)
9857   format %{ "fldlg2\t\t\t#Log10\n\t"
9858             "fyl2x\t\t\t# Q=Log10*Log_2(x)\n\t"
9859          %}
9860    ins_encode(Opcode(0xD9), Opcode(0xEC),   // fldlg2
9861               Push_SrcXD(dst),
9862               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9863               Push_ResultXD(dst));
9864 
9865   ins_pipe( pipe_slow );
9866 %}
9867 
9868 instruct logD_reg(regD dst) %{
9869   // The source and result Double operands in XMM registers
9870   match(Set dst (LogD dst));
9871   // fldln2       ; push log_e(2) on the FPU stack; full 80-bit number
9872   // fyl2x        ; compute log_e(2) * log_2(x)
9873   format %{ "fldln2\t\t\t#Log_e\n\t"
9874             "fyl2x\t\t\t# Q=Log_e*Log_2(x)\n\t"
9875          %}
9876   ins_encode( Opcode(0xD9), Opcode(0xED),   // fldln2
9877               Push_SrcXD(dst),
9878               Opcode(0xD9), Opcode(0xF1),   // fyl2x
9879               Push_ResultXD(dst));
9880   ins_pipe( pipe_slow );
9881 %}
9882 
9883 instruct powD_reg(regD dst, regD src0, regD src1, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9884   match(Set dst (PowD src0 src1));  // Raise src0 to the src1'th power
9885   effect(KILL rax, KILL rdx, KILL rcx, KILL cr);
9886   format %{ "fast_pow $src0 $src1 -> $dst  // KILL $rax, $rcx, $rdx" %}
9887   ins_encode %{
9888     __ subptr(rsp, 8);
9889     __ movdbl(Address(rsp, 0), $src1$$XMMRegister);
9890     __ fld_d(Address(rsp, 0));
9891     __ movdbl(Address(rsp, 0), $src0$$XMMRegister);
9892     __ fld_d(Address(rsp, 0));
9893     __ fast_pow();
9894     __ fstp_d(Address(rsp, 0));
9895     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9896     __ addptr(rsp, 8);
9897   %}
9898   ins_pipe( pipe_slow );
9899 %}
9900 
9901 instruct expD_reg(regD dst, regD src, rax_RegI rax, rdx_RegI rdx, rcx_RegI rcx, rFlagsReg cr) %{
9902   match(Set dst (ExpD src));
9903   effect(KILL rax, KILL rcx, KILL rdx, KILL cr);
9904   format %{ "fast_exp $dst -> $src  // KILL $rax, $rcx, $rdx" %}
9905   ins_encode %{
9906     __ subptr(rsp, 8);
9907     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
9908     __ fld_d(Address(rsp, 0));
9909     __ fast_exp();
9910     __ fstp_d(Address(rsp, 0));
9911     __ movdbl($dst$$XMMRegister, Address(rsp, 0));
9912     __ addptr(rsp, 8);
9913   %}
9914   ins_pipe( pipe_slow );
9915 %}
9916 
9917 //----------Arithmetic Conversion Instructions---------------------------------
9918 
9919 instruct roundFloat_nop(regF dst)
9920 %{
9921   match(Set dst (RoundFloat dst));
9922 
9923   ins_cost(0);
9924   ins_encode();
9925   ins_pipe(empty);
9926 %}
9927 
9928 instruct roundDouble_nop(regD dst)
9929 %{
9930   match(Set dst (RoundDouble dst));
9931 
9932   ins_cost(0);
9933   ins_encode();
9934   ins_pipe(empty);
9935 %}
9936 
9937 instruct convF2D_reg_reg(regD dst, regF src)
9938 %{
9939   match(Set dst (ConvF2D src));
9940 
9941   format %{ "cvtss2sd $dst, $src" %}
9942   ins_encode %{
9943     __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
9944   %}
9945   ins_pipe(pipe_slow); // XXX
9946 %}
9947 
9948 instruct convF2D_reg_mem(regD dst, memory src)
9949 %{
9950   match(Set dst (ConvF2D (LoadF src)));
9951 
9952   format %{ "cvtss2sd $dst, $src" %}
9953   ins_encode %{
9954     __ cvtss2sd ($dst$$XMMRegister, $src$$Address);
9955   %}
9956   ins_pipe(pipe_slow); // XXX
9957 %}
9958 
9959 instruct convD2F_reg_reg(regF dst, regD src)
9960 %{
9961   match(Set dst (ConvD2F src));
9962 
9963   format %{ "cvtsd2ss $dst, $src" %}
9964   ins_encode %{
9965     __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
9966   %}
9967   ins_pipe(pipe_slow); // XXX
9968 %}
9969 
9970 instruct convD2F_reg_mem(regF dst, memory src)
9971 %{
9972   match(Set dst (ConvD2F (LoadD src)));
9973 
9974   format %{ "cvtsd2ss $dst, $src" %}
9975   ins_encode %{
9976     __ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
9977   %}
9978   ins_pipe(pipe_slow); // XXX
9979 %}
9980 
9981 // XXX do mem variants
9982 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
9983 %{
9984   match(Set dst (ConvF2I src));
9985   effect(KILL cr);
9986 
9987   format %{ "cvttss2sil $dst, $src\t# f2i\n\t"
9988             "cmpl    $dst, #0x80000000\n\t"
9989             "jne,s   done\n\t"
9990             "subq    rsp, #8\n\t"
9991             "movss   [rsp], $src\n\t"
9992             "call    f2i_fixup\n\t"
9993             "popq    $dst\n"
9994     "done:   "%}
9995   ins_encode %{
9996     Label done;
9997     __ cvttss2sil($dst$$Register, $src$$XMMRegister);
9998     __ cmpl($dst$$Register, 0x80000000);
9999     __ jccb(Assembler::notEqual, done);
10000     __ subptr(rsp, 8);
10001     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10002     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10003     __ pop($dst$$Register);
10004     __ bind(done);
10005   %}
10006   ins_pipe(pipe_slow);
10007 %}
10008 
10009 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
10010 %{
10011   match(Set dst (ConvF2L src));
10012   effect(KILL cr);
10013 
10014   format %{ "cvttss2siq $dst, $src\t# f2l\n\t"
10015             "cmpq    $dst, [0x8000000000000000]\n\t"
10016             "jne,s   done\n\t"
10017             "subq    rsp, #8\n\t"
10018             "movss   [rsp], $src\n\t"
10019             "call    f2l_fixup\n\t"
10020             "popq    $dst\n"
10021     "done:   "%}
10022   ins_encode %{
10023     Label done;
10024     __ cvttss2siq($dst$$Register, $src$$XMMRegister);
10025     __ cmp64($dst$$Register,
10026              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10027     __ jccb(Assembler::notEqual, done);
10028     __ subptr(rsp, 8);
10029     __ movflt(Address(rsp, 0), $src$$XMMRegister);
10030     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10031     __ pop($dst$$Register);
10032     __ bind(done);
10033   %}
10034   ins_pipe(pipe_slow);
10035 %}
10036 
10037 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
10038 %{
10039   match(Set dst (ConvD2I src));
10040   effect(KILL cr);
10041 
10042   format %{ "cvttsd2sil $dst, $src\t# d2i\n\t"
10043             "cmpl    $dst, #0x80000000\n\t"
10044             "jne,s   done\n\t"
10045             "subq    rsp, #8\n\t"
10046             "movsd   [rsp], $src\n\t"
10047             "call    d2i_fixup\n\t"
10048             "popq    $dst\n"
10049     "done:   "%}
10050   ins_encode %{
10051     Label done;
10052     __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
10053     __ cmpl($dst$$Register, 0x80000000);
10054     __ jccb(Assembler::notEqual, done);
10055     __ subptr(rsp, 8);
10056     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10057     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10058     __ pop($dst$$Register);
10059     __ bind(done);
10060   %}
10061   ins_pipe(pipe_slow);
10062 %}
10063 
10064 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
10065 %{
10066   match(Set dst (ConvD2L src));
10067   effect(KILL cr);
10068 
10069   format %{ "cvttsd2siq $dst, $src\t# d2l\n\t"
10070             "cmpq    $dst, [0x8000000000000000]\n\t"
10071             "jne,s   done\n\t"
10072             "subq    rsp, #8\n\t"
10073             "movsd   [rsp], $src\n\t"
10074             "call    d2l_fixup\n\t"
10075             "popq    $dst\n"
10076     "done:   "%}
10077   ins_encode %{
10078     Label done;
10079     __ cvttsd2siq($dst$$Register, $src$$XMMRegister);
10080     __ cmp64($dst$$Register,
10081              ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10082     __ jccb(Assembler::notEqual, done);
10083     __ subptr(rsp, 8);
10084     __ movdbl(Address(rsp, 0), $src$$XMMRegister);
10085     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10086     __ pop($dst$$Register);
10087     __ bind(done);
10088   %}
10089   ins_pipe(pipe_slow);
10090 %}
10091 
10092 instruct convI2F_reg_reg(regF dst, rRegI src)
10093 %{
10094   predicate(!UseXmmI2F);
10095   match(Set dst (ConvI2F src));
10096 
10097   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10098   ins_encode %{
10099     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
10100   %}
10101   ins_pipe(pipe_slow); // XXX
10102 %}
10103 
10104 instruct convI2F_reg_mem(regF dst, memory src)
10105 %{
10106   match(Set dst (ConvI2F (LoadI src)));
10107 
10108   format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
10109   ins_encode %{
10110     __ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
10111   %}
10112   ins_pipe(pipe_slow); // XXX
10113 %}
10114 
10115 instruct convI2D_reg_reg(regD dst, rRegI src)
10116 %{
10117   predicate(!UseXmmI2D);
10118   match(Set dst (ConvI2D src));
10119 
10120   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10121   ins_encode %{
10122     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
10123   %}
10124   ins_pipe(pipe_slow); // XXX
10125 %}
10126 
10127 instruct convI2D_reg_mem(regD dst, memory src)
10128 %{
10129   match(Set dst (ConvI2D (LoadI src)));
10130 
10131   format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
10132   ins_encode %{
10133     __ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
10134   %}
10135   ins_pipe(pipe_slow); // XXX
10136 %}
10137 
10138 instruct convXI2F_reg(regF dst, rRegI src)
10139 %{
10140   predicate(UseXmmI2F);
10141   match(Set dst (ConvI2F src));
10142 
10143   format %{ "movdl $dst, $src\n\t"
10144             "cvtdq2psl $dst, $dst\t# i2f" %}
10145   ins_encode %{
10146     __ movdl($dst$$XMMRegister, $src$$Register);
10147     __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
10148   %}
10149   ins_pipe(pipe_slow); // XXX
10150 %}
10151 
10152 instruct convXI2D_reg(regD dst, rRegI src)
10153 %{
10154   predicate(UseXmmI2D);
10155   match(Set dst (ConvI2D src));
10156 
10157   format %{ "movdl $dst, $src\n\t"
10158             "cvtdq2pdl $dst, $dst\t# i2d" %}
10159   ins_encode %{
10160     __ movdl($dst$$XMMRegister, $src$$Register);
10161     __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
10162   %}
10163   ins_pipe(pipe_slow); // XXX
10164 %}
10165 
10166 instruct convL2F_reg_reg(regF dst, rRegL src)
10167 %{
10168   match(Set dst (ConvL2F src));
10169 
10170   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10171   ins_encode %{
10172     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
10173   %}
10174   ins_pipe(pipe_slow); // XXX
10175 %}
10176 
10177 instruct convL2F_reg_mem(regF dst, memory src)
10178 %{
10179   match(Set dst (ConvL2F (LoadL src)));
10180 
10181   format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
10182   ins_encode %{
10183     __ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
10184   %}
10185   ins_pipe(pipe_slow); // XXX
10186 %}
10187 
10188 instruct convL2D_reg_reg(regD dst, rRegL src)
10189 %{
10190   match(Set dst (ConvL2D src));
10191 
10192   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10193   ins_encode %{
10194     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
10195   %}
10196   ins_pipe(pipe_slow); // XXX
10197 %}
10198 
10199 instruct convL2D_reg_mem(regD dst, memory src)
10200 %{
10201   match(Set dst (ConvL2D (LoadL src)));
10202 
10203   format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
10204   ins_encode %{
10205     __ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
10206   %}
10207   ins_pipe(pipe_slow); // XXX
10208 %}
10209 
10210 instruct convI2L_reg_reg(rRegL dst, rRegI src)
10211 %{
10212   match(Set dst (ConvI2L src));
10213 
10214   ins_cost(125);
10215   format %{ "movslq  $dst, $src\t# i2l" %}
10216   ins_encode %{
10217     __ movslq($dst$$Register, $src$$Register);
10218   %}
10219   ins_pipe(ialu_reg_reg);
10220 %}
10221 
10222 // instruct convI2L_reg_reg_foo(rRegL dst, rRegI src)
10223 // %{
10224 //   match(Set dst (ConvI2L src));
10225 // //   predicate(_kids[0]->_leaf->as_Type()->type()->is_int()->_lo >= 0 &&
10226 // //             _kids[0]->_leaf->as_Type()->type()->is_int()->_hi >= 0);
10227 //   predicate(((const TypeNode*) n)->type()->is_long()->_hi ==
10228 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_hi &&
10229 //             ((const TypeNode*) n)->type()->is_long()->_lo ==
10230 //             (unsigned int) ((const TypeNode*) n)->type()->is_long()->_lo);
10231 
10232 //   format %{ "movl    $dst, $src\t# unsigned i2l" %}
10233 //   ins_encode(enc_copy(dst, src));
10234 // //   opcode(0x63); // needs REX.W
10235 // //   ins_encode(REX_reg_reg_wide(dst, src), OpcP, reg_reg(dst,src));
10236 //   ins_pipe(ialu_reg_reg);
10237 // %}
10238 
10239 // Zero-extend convert int to long
10240 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
10241 %{
10242   match(Set dst (AndL (ConvI2L src) mask));
10243 
10244   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10245   ins_encode %{
10246     if ($dst$$reg != $src$$reg) {
10247       __ movl($dst$$Register, $src$$Register);
10248     }
10249   %}
10250   ins_pipe(ialu_reg_reg);
10251 %}
10252 
10253 // Zero-extend convert int to long
10254 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
10255 %{
10256   match(Set dst (AndL (ConvI2L (LoadI src)) mask));
10257 
10258   format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
10259   ins_encode %{
10260     __ movl($dst$$Register, $src$$Address);
10261   %}
10262   ins_pipe(ialu_reg_mem);
10263 %}
10264 
10265 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
10266 %{
10267   match(Set dst (AndL src mask));
10268 
10269   format %{ "movl    $dst, $src\t# zero-extend long" %}
10270   ins_encode %{
10271     __ movl($dst$$Register, $src$$Register);
10272   %}
10273   ins_pipe(ialu_reg_reg);
10274 %}
10275 
10276 instruct convL2I_reg_reg(rRegI dst, rRegL src)
10277 %{
10278   match(Set dst (ConvL2I src));
10279 
10280   format %{ "movl    $dst, $src\t# l2i" %}
10281   ins_encode %{
10282     __ movl($dst$$Register, $src$$Register);
10283   %}
10284   ins_pipe(ialu_reg_reg);
10285 %}
10286 
10287 
10288 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
10289   match(Set dst (MoveF2I src));
10290   effect(DEF dst, USE src);
10291 
10292   ins_cost(125);
10293   format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
10294   ins_encode %{
10295     __ movl($dst$$Register, Address(rsp, $src$$disp));
10296   %}
10297   ins_pipe(ialu_reg_mem);
10298 %}
10299 
10300 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
10301   match(Set dst (MoveI2F src));
10302   effect(DEF dst, USE src);
10303 
10304   ins_cost(125);
10305   format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
10306   ins_encode %{
10307     __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
10308   %}
10309   ins_pipe(pipe_slow);
10310 %}
10311 
10312 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
10313   match(Set dst (MoveD2L src));
10314   effect(DEF dst, USE src);
10315 
10316   ins_cost(125);
10317   format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
10318   ins_encode %{
10319     __ movq($dst$$Register, Address(rsp, $src$$disp));
10320   %}
10321   ins_pipe(ialu_reg_mem);
10322 %}
10323 
10324 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
10325   predicate(!UseXmmLoadAndClearUpper);
10326   match(Set dst (MoveL2D src));
10327   effect(DEF dst, USE src);
10328 
10329   ins_cost(125);
10330   format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
10331   ins_encode %{
10332     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10333   %}
10334   ins_pipe(pipe_slow);
10335 %}
10336 
10337 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
10338   predicate(UseXmmLoadAndClearUpper);
10339   match(Set dst (MoveL2D src));
10340   effect(DEF dst, USE src);
10341 
10342   ins_cost(125);
10343   format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
10344   ins_encode %{
10345     __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
10346   %}
10347   ins_pipe(pipe_slow);
10348 %}
10349 
10350 
10351 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
10352   match(Set dst (MoveF2I src));
10353   effect(DEF dst, USE src);
10354 
10355   ins_cost(95); // XXX
10356   format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
10357   ins_encode %{
10358     __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
10359   %}
10360   ins_pipe(pipe_slow);
10361 %}
10362 
10363 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
10364   match(Set dst (MoveI2F src));
10365   effect(DEF dst, USE src);
10366 
10367   ins_cost(100);
10368   format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
10369   ins_encode %{
10370     __ movl(Address(rsp, $dst$$disp), $src$$Register);
10371   %}
10372   ins_pipe( ialu_mem_reg );
10373 %}
10374 
10375 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
10376   match(Set dst (MoveD2L src));
10377   effect(DEF dst, USE src);
10378 
10379   ins_cost(95); // XXX
10380   format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
10381   ins_encode %{
10382     __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
10383   %}
10384   ins_pipe(pipe_slow);
10385 %}
10386 
10387 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
10388   match(Set dst (MoveL2D src));
10389   effect(DEF dst, USE src);
10390 
10391   ins_cost(100);
10392   format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
10393   ins_encode %{
10394     __ movq(Address(rsp, $dst$$disp), $src$$Register);
10395   %}
10396   ins_pipe(ialu_mem_reg);
10397 %}
10398 
10399 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
10400   match(Set dst (MoveF2I src));
10401   effect(DEF dst, USE src);
10402   ins_cost(85);
10403   format %{ "movd    $dst,$src\t# MoveF2I" %}
10404   ins_encode %{
10405     __ movdl($dst$$Register, $src$$XMMRegister);
10406   %}
10407   ins_pipe( pipe_slow );
10408 %}
10409 
10410 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
10411   match(Set dst (MoveD2L src));
10412   effect(DEF dst, USE src);
10413   ins_cost(85);
10414   format %{ "movd    $dst,$src\t# MoveD2L" %}
10415   ins_encode %{
10416     __ movdq($dst$$Register, $src$$XMMRegister);
10417   %}
10418   ins_pipe( pipe_slow );
10419 %}
10420 
10421 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
10422   match(Set dst (MoveI2F src));
10423   effect(DEF dst, USE src);
10424   ins_cost(100);
10425   format %{ "movd    $dst,$src\t# MoveI2F" %}
10426   ins_encode %{
10427     __ movdl($dst$$XMMRegister, $src$$Register);
10428   %}
10429   ins_pipe( pipe_slow );
10430 %}
10431 
10432 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
10433   match(Set dst (MoveL2D src));
10434   effect(DEF dst, USE src);
10435   ins_cost(100);
10436   format %{ "movd    $dst,$src\t# MoveL2D" %}
10437   ins_encode %{
10438      __ movdq($dst$$XMMRegister, $src$$Register);
10439   %}
10440   ins_pipe( pipe_slow );
10441 %}
10442 
10443 
10444 // =======================================================================
10445 // fast clearing of an array
10446 instruct rep_stos(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10447                   rFlagsReg cr)
10448 %{
10449   predicate(!UseFastStosb);
10450   match(Set dummy (ClearArray cnt base));
10451   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10452 
10453   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10454             "rep     stosq\t# Store rax to *rdi++ while rcx--" %}
10455   ins_encode %{
10456     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10457   %}
10458   ins_pipe(pipe_slow);
10459 %}
10460 
10461 instruct rep_fast_stosb(rcx_RegL cnt, rdi_RegP base, rax_RegI zero, Universe dummy,
10462                         rFlagsReg cr)
10463 %{
10464   predicate(UseFastStosb);
10465   match(Set dummy (ClearArray cnt base));
10466   effect(USE_KILL cnt, USE_KILL base, KILL zero, KILL cr);
10467   format %{ "xorq    rax, rax\t# ClearArray:\n\t"
10468             "shlq    rcx,3\t# Convert doublewords to bytes\n\t"
10469             "rep     stosb\t# Store rax to *rdi++ while rcx--" %}
10470   ins_encode %{
10471     __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register);
10472   %}
10473   ins_pipe( pipe_slow );
10474 %}
10475 
10476 instruct string_compare(rdi_RegP str1, rcx_RegI cnt1, rsi_RegP str2, rdx_RegI cnt2,
10477                         rax_RegI result, regD tmp1, rFlagsReg cr)
10478 %{
10479   match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
10480   effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
10481 
10482   format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   // KILL $tmp1" %}
10483   ins_encode %{
10484     __ string_compare($str1$$Register, $str2$$Register,
10485                       $cnt1$$Register, $cnt2$$Register, $result$$Register,
10486                       $tmp1$$XMMRegister);
10487   %}
10488   ins_pipe( pipe_slow );
10489 %}
10490 
10491 // fast search of substring with known size.
10492 instruct string_indexof_con(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, immI int_cnt2,
10493                             rbx_RegI result, regD vec, rax_RegI cnt2, rcx_RegI tmp, rFlagsReg cr)
10494 %{
10495   predicate(UseSSE42Intrinsics);
10496   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
10497   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
10498 
10499   format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result   // KILL $vec, $cnt1, $cnt2, $tmp" %}
10500   ins_encode %{
10501     int icnt2 = (int)$int_cnt2$$constant;
10502     if (icnt2 >= 8) {
10503       // IndexOf for constant substrings with size >= 8 elements
10504       // which don't need to be loaded through stack.
10505       __ string_indexofC8($str1$$Register, $str2$$Register,
10506                           $cnt1$$Register, $cnt2$$Register,
10507                           icnt2, $result$$Register,
10508                           $vec$$XMMRegister, $tmp$$Register);
10509     } else {
10510       // Small strings are loaded through stack if they cross page boundary.
10511       __ string_indexof($str1$$Register, $str2$$Register,
10512                         $cnt1$$Register, $cnt2$$Register,
10513                         icnt2, $result$$Register,
10514                         $vec$$XMMRegister, $tmp$$Register);
10515     }
10516   %}
10517   ins_pipe( pipe_slow );
10518 %}
10519 
10520 instruct string_indexof(rdi_RegP str1, rdx_RegI cnt1, rsi_RegP str2, rax_RegI cnt2,
10521                         rbx_RegI result, regD vec, rcx_RegI tmp, rFlagsReg cr)
10522 %{
10523   predicate(UseSSE42Intrinsics);
10524   match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
10525   effect(TEMP vec, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
10526 
10527   format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result   // KILL all" %}
10528   ins_encode %{
10529     __ string_indexof($str1$$Register, $str2$$Register,
10530                       $cnt1$$Register, $cnt2$$Register,
10531                       (-1), $result$$Register,
10532                       $vec$$XMMRegister, $tmp$$Register);
10533   %}
10534   ins_pipe( pipe_slow );
10535 %}
10536 
10537 // fast string equals
10538 instruct string_equals(rdi_RegP str1, rsi_RegP str2, rcx_RegI cnt, rax_RegI result,
10539                        regD tmp1, regD tmp2, rbx_RegI tmp3, rFlagsReg cr)
10540 %{
10541   match(Set result (StrEquals (Binary str1 str2) cnt));
10542   effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
10543 
10544   format %{ "String Equals $str1,$str2,$cnt -> $result    // KILL $tmp1, $tmp2, $tmp3" %}
10545   ins_encode %{
10546     __ char_arrays_equals(false, $str1$$Register, $str2$$Register,
10547                           $cnt$$Register, $result$$Register, $tmp3$$Register,
10548                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10549   %}
10550   ins_pipe( pipe_slow );
10551 %}
10552 
10553 // fast array equals
10554 instruct array_equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI result,
10555                       regD tmp1, regD tmp2, rcx_RegI tmp3, rbx_RegI tmp4, rFlagsReg cr)
10556 %{
10557   match(Set result (AryEq ary1 ary2));
10558   effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
10559   //ins_cost(300);
10560 
10561   format %{ "Array Equals $ary1,$ary2 -> $result   // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
10562   ins_encode %{
10563     __ char_arrays_equals(true, $ary1$$Register, $ary2$$Register,
10564                           $tmp3$$Register, $result$$Register, $tmp4$$Register,
10565                           $tmp1$$XMMRegister, $tmp2$$XMMRegister);
10566   %}
10567   ins_pipe( pipe_slow );
10568 %}
10569 
10570 // encode char[] to byte[] in ISO_8859_1
10571 instruct encode_iso_array(rsi_RegP src, rdi_RegP dst, rdx_RegI len,
10572                           regD tmp1, regD tmp2, regD tmp3, regD tmp4,
10573                           rcx_RegI tmp5, rax_RegI result, rFlagsReg cr) %{
10574   match(Set result (EncodeISOArray src (Binary dst len)));
10575   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
10576 
10577   format %{ "Encode array $src,$dst,$len -> $result    // KILL RCX, RDX, $tmp1, $tmp2, $tmp3, $tmp4, RSI, RDI " %}
10578   ins_encode %{
10579     __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
10580                         $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
10581                         $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register);
10582   %}
10583   ins_pipe( pipe_slow );
10584 %}
10585 
10586 //----------Overflow Math Instructions-----------------------------------------
10587 
10588 instruct overflowAddI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10589 %{
10590   match(Set cr (OverflowAddI op1 op2));
10591   effect(DEF cr, USE_KILL op1, USE op2);
10592 
10593   format %{ "addl    $op1, $op2\t# overflow check int" %}
10594 
10595   ins_encode %{
10596     __ addl($op1$$Register, $op2$$Register);
10597   %}
10598   ins_pipe(ialu_reg_reg);
10599 %}
10600 
10601 instruct overflowAddI_rReg_imm(rFlagsReg cr, rax_RegI op1, immI op2)
10602 %{
10603   match(Set cr (OverflowAddI op1 op2));
10604   effect(DEF cr, USE_KILL op1, USE op2);
10605 
10606   format %{ "addl    $op1, $op2\t# overflow check int" %}
10607 
10608   ins_encode %{
10609     __ addl($op1$$Register, $op2$$constant);
10610   %}
10611   ins_pipe(ialu_reg_reg);
10612 %}
10613 
10614 instruct overflowAddL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10615 %{
10616   match(Set cr (OverflowAddL op1 op2));
10617   effect(DEF cr, USE_KILL op1, USE op2);
10618 
10619   format %{ "addq    $op1, $op2\t# overflow check long" %}
10620   ins_encode %{
10621     __ addq($op1$$Register, $op2$$Register);
10622   %}
10623   ins_pipe(ialu_reg_reg);
10624 %}
10625 
10626 instruct overflowAddL_rReg_imm(rFlagsReg cr, rax_RegL op1, immL32 op2)
10627 %{
10628   match(Set cr (OverflowAddL op1 op2));
10629   effect(DEF cr, USE_KILL op1, USE op2);
10630 
10631   format %{ "addq    $op1, $op2\t# overflow check long" %}
10632   ins_encode %{
10633     __ addq($op1$$Register, $op2$$constant);
10634   %}
10635   ins_pipe(ialu_reg_reg);
10636 %}
10637 
10638 instruct overflowSubI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10639 %{
10640   match(Set cr (OverflowSubI op1 op2));
10641 
10642   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10643   ins_encode %{
10644     __ cmpl($op1$$Register, $op2$$Register);
10645   %}
10646   ins_pipe(ialu_reg_reg);
10647 %}
10648 
10649 instruct overflowSubI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10650 %{
10651   match(Set cr (OverflowSubI op1 op2));
10652 
10653   format %{ "cmpl    $op1, $op2\t# overflow check int" %}
10654   ins_encode %{
10655     __ cmpl($op1$$Register, $op2$$constant);
10656   %}
10657   ins_pipe(ialu_reg_reg);
10658 %}
10659 
10660 instruct overflowSubL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
10661 %{
10662   match(Set cr (OverflowSubL op1 op2));
10663 
10664   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10665   ins_encode %{
10666     __ cmpq($op1$$Register, $op2$$Register);
10667   %}
10668   ins_pipe(ialu_reg_reg);
10669 %}
10670 
10671 instruct overflowSubL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
10672 %{
10673   match(Set cr (OverflowSubL op1 op2));
10674 
10675   format %{ "cmpq    $op1, $op2\t# overflow check long" %}
10676   ins_encode %{
10677     __ cmpq($op1$$Register, $op2$$constant);
10678   %}
10679   ins_pipe(ialu_reg_reg);
10680 %}
10681 
10682 instruct overflowNegI_rReg(rFlagsReg cr, immI0 zero, rax_RegI op2)
10683 %{
10684   match(Set cr (OverflowSubI zero op2));
10685   effect(DEF cr, USE_KILL op2);
10686 
10687   format %{ "negl    $op2\t# overflow check int" %}
10688   ins_encode %{
10689     __ negl($op2$$Register);
10690   %}
10691   ins_pipe(ialu_reg_reg);
10692 %}
10693 
10694 instruct overflowNegL_rReg(rFlagsReg cr, immL0 zero, rax_RegL op2)
10695 %{
10696   match(Set cr (OverflowSubL zero op2));
10697   effect(DEF cr, USE_KILL op2);
10698 
10699   format %{ "negq    $op2\t# overflow check long" %}
10700   ins_encode %{
10701     __ negq($op2$$Register);
10702   %}
10703   ins_pipe(ialu_reg_reg);
10704 %}
10705 
10706 instruct overflowMulI_rReg(rFlagsReg cr, rax_RegI op1, rRegI op2)
10707 %{
10708   match(Set cr (OverflowMulI op1 op2));
10709   effect(DEF cr, USE_KILL op1, USE op2);
10710 
10711   format %{ "imull    $op1, $op2\t# overflow check int" %}
10712   ins_encode %{
10713     __ imull($op1$$Register, $op2$$Register);
10714   %}
10715   ins_pipe(ialu_reg_reg_alu0);
10716 %}
10717 
10718 instruct overflowMulI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
10719 %{
10720   match(Set cr (OverflowMulI op1 op2));
10721   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10722 
10723   format %{ "imull    $tmp, $op1, $op2\t# overflow check int" %}
10724   ins_encode %{
10725     __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
10726   %}
10727   ins_pipe(ialu_reg_reg_alu0);
10728 %}
10729 
10730 instruct overflowMulL_rReg(rFlagsReg cr, rax_RegL op1, rRegL op2)
10731 %{
10732   match(Set cr (OverflowMulL op1 op2));
10733   effect(DEF cr, USE_KILL op1, USE op2);
10734 
10735   format %{ "imulq    $op1, $op2\t# overflow check long" %}
10736   ins_encode %{
10737     __ imulq($op1$$Register, $op2$$Register);
10738   %}
10739   ins_pipe(ialu_reg_reg_alu0);
10740 %}
10741 
10742 instruct overflowMulL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2, rRegL tmp)
10743 %{
10744   match(Set cr (OverflowMulL op1 op2));
10745   effect(DEF cr, TEMP tmp, USE op1, USE op2);
10746 
10747   format %{ "imulq    $tmp, $op1, $op2\t# overflow check long" %}
10748   ins_encode %{
10749     __ imulq($tmp$$Register, $op1$$Register, $op2$$constant);
10750   %}
10751   ins_pipe(ialu_reg_reg_alu0);
10752 %}
10753 
10754 
10755 //----------Control Flow Instructions------------------------------------------
10756 // Signed compare Instructions
10757 
10758 // XXX more variants!!
10759 instruct compI_rReg(rFlagsReg cr, rRegI op1, rRegI op2)
10760 %{
10761   match(Set cr (CmpI op1 op2));
10762   effect(DEF cr, USE op1, USE op2);
10763 
10764   format %{ "cmpl    $op1, $op2" %}
10765   opcode(0x3B);  /* Opcode 3B /r */
10766   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10767   ins_pipe(ialu_cr_reg_reg);
10768 %}
10769 
10770 instruct compI_rReg_imm(rFlagsReg cr, rRegI op1, immI op2)
10771 %{
10772   match(Set cr (CmpI op1 op2));
10773 
10774   format %{ "cmpl    $op1, $op2" %}
10775   opcode(0x81, 0x07); /* Opcode 81 /7 */
10776   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10777   ins_pipe(ialu_cr_reg_imm);
10778 %}
10779 
10780 instruct compI_rReg_mem(rFlagsReg cr, rRegI op1, memory op2)
10781 %{
10782   match(Set cr (CmpI op1 (LoadI op2)));
10783 
10784   ins_cost(500); // XXX
10785   format %{ "cmpl    $op1, $op2" %}
10786   opcode(0x3B); /* Opcode 3B /r */
10787   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10788   ins_pipe(ialu_cr_reg_mem);
10789 %}
10790 
10791 instruct testI_reg(rFlagsReg cr, rRegI src, immI0 zero)
10792 %{
10793   match(Set cr (CmpI src zero));
10794 
10795   format %{ "testl   $src, $src" %}
10796   opcode(0x85);
10797   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10798   ins_pipe(ialu_cr_reg_imm);
10799 %}
10800 
10801 instruct testI_reg_imm(rFlagsReg cr, rRegI src, immI con, immI0 zero)
10802 %{
10803   match(Set cr (CmpI (AndI src con) zero));
10804 
10805   format %{ "testl   $src, $con" %}
10806   opcode(0xF7, 0x00);
10807   ins_encode(REX_reg(src), OpcP, reg_opc(src), Con32(con));
10808   ins_pipe(ialu_cr_reg_imm);
10809 %}
10810 
10811 instruct testI_reg_mem(rFlagsReg cr, rRegI src, memory mem, immI0 zero)
10812 %{
10813   match(Set cr (CmpI (AndI src (LoadI mem)) zero));
10814 
10815   format %{ "testl   $src, $mem" %}
10816   opcode(0x85);
10817   ins_encode(REX_reg_mem(src, mem), OpcP, reg_mem(src, mem));
10818   ins_pipe(ialu_cr_reg_mem);
10819 %}
10820 
10821 // Unsigned compare Instructions; really, same as signed except they
10822 // produce an rFlagsRegU instead of rFlagsReg.
10823 instruct compU_rReg(rFlagsRegU cr, rRegI op1, rRegI op2)
10824 %{
10825   match(Set cr (CmpU op1 op2));
10826 
10827   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10828   opcode(0x3B); /* Opcode 3B /r */
10829   ins_encode(REX_reg_reg(op1, op2), OpcP, reg_reg(op1, op2));
10830   ins_pipe(ialu_cr_reg_reg);
10831 %}
10832 
10833 instruct compU_rReg_imm(rFlagsRegU cr, rRegI op1, immI op2)
10834 %{
10835   match(Set cr (CmpU op1 op2));
10836 
10837   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10838   opcode(0x81,0x07); /* Opcode 81 /7 */
10839   ins_encode(OpcSErm(op1, op2), Con8or32(op2));
10840   ins_pipe(ialu_cr_reg_imm);
10841 %}
10842 
10843 instruct compU_rReg_mem(rFlagsRegU cr, rRegI op1, memory op2)
10844 %{
10845   match(Set cr (CmpU op1 (LoadI op2)));
10846 
10847   ins_cost(500); // XXX
10848   format %{ "cmpl    $op1, $op2\t# unsigned" %}
10849   opcode(0x3B); /* Opcode 3B /r */
10850   ins_encode(REX_reg_mem(op1, op2), OpcP, reg_mem(op1, op2));
10851   ins_pipe(ialu_cr_reg_mem);
10852 %}
10853 
10854 // // // Cisc-spilled version of cmpU_rReg
10855 // //instruct compU_mem_rReg(rFlagsRegU cr, memory op1, rRegI op2)
10856 // //%{
10857 // //  match(Set cr (CmpU (LoadI op1) op2));
10858 // //
10859 // //  format %{ "CMPu   $op1,$op2" %}
10860 // //  ins_cost(500);
10861 // //  opcode(0x39);  /* Opcode 39 /r */
10862 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10863 // //%}
10864 
10865 instruct testU_reg(rFlagsRegU cr, rRegI src, immI0 zero)
10866 %{
10867   match(Set cr (CmpU src zero));
10868 
10869   format %{ "testl  $src, $src\t# unsigned" %}
10870   opcode(0x85);
10871   ins_encode(REX_reg_reg(src, src), OpcP, reg_reg(src, src));
10872   ins_pipe(ialu_cr_reg_imm);
10873 %}
10874 
10875 instruct compP_rReg(rFlagsRegU cr, rRegP op1, rRegP op2)
10876 %{
10877   match(Set cr (CmpP op1 op2));
10878 
10879   format %{ "cmpq    $op1, $op2\t# ptr" %}
10880   opcode(0x3B); /* Opcode 3B /r */
10881   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
10882   ins_pipe(ialu_cr_reg_reg);
10883 %}
10884 
10885 instruct compP_rReg_mem(rFlagsRegU cr, rRegP op1, memory op2)
10886 %{
10887   match(Set cr (CmpP op1 (LoadP op2)));
10888 
10889   ins_cost(500); // XXX
10890   format %{ "cmpq    $op1, $op2\t# ptr" %}
10891   opcode(0x3B); /* Opcode 3B /r */
10892   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10893   ins_pipe(ialu_cr_reg_mem);
10894 %}
10895 
10896 // // // Cisc-spilled version of cmpP_rReg
10897 // //instruct compP_mem_rReg(rFlagsRegU cr, memory op1, rRegP op2)
10898 // //%{
10899 // //  match(Set cr (CmpP (LoadP op1) op2));
10900 // //
10901 // //  format %{ "CMPu   $op1,$op2" %}
10902 // //  ins_cost(500);
10903 // //  opcode(0x39);  /* Opcode 39 /r */
10904 // //  ins_encode( OpcP, reg_mem( op1, op2) );
10905 // //%}
10906 
10907 // XXX this is generalized by compP_rReg_mem???
10908 // Compare raw pointer (used in out-of-heap check).
10909 // Only works because non-oop pointers must be raw pointers
10910 // and raw pointers have no anti-dependencies.
10911 instruct compP_mem_rReg(rFlagsRegU cr, rRegP op1, memory op2)
10912 %{
10913   predicate(n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none);
10914   match(Set cr (CmpP op1 (LoadP op2)));
10915 
10916   format %{ "cmpq    $op1, $op2\t# raw ptr" %}
10917   opcode(0x3B); /* Opcode 3B /r */
10918   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
10919   ins_pipe(ialu_cr_reg_mem);
10920 %}
10921 
10922 // This will generate a signed flags result. This should be OK since
10923 // any compare to a zero should be eq/neq.
10924 instruct testP_reg(rFlagsReg cr, rRegP src, immP0 zero)
10925 %{
10926   match(Set cr (CmpP src zero));
10927 
10928   format %{ "testq   $src, $src\t# ptr" %}
10929   opcode(0x85);
10930   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
10931   ins_pipe(ialu_cr_reg_imm);
10932 %}
10933 
10934 // This will generate a signed flags result. This should be OK since
10935 // any compare to a zero should be eq/neq.
10936 instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
10937 %{
10938   predicate(!UseCompressedOops || (Universe::narrow_oop_base() != NULL));
10939   match(Set cr (CmpP (LoadP op) zero));
10940 
10941   ins_cost(500); // XXX
10942   format %{ "testq   $op, 0xffffffffffffffff\t# ptr" %}
10943   opcode(0xF7); /* Opcode F7 /0 */
10944   ins_encode(REX_mem_wide(op),
10945              OpcP, RM_opc_mem(0x00, op), Con_d32(0xFFFFFFFF));
10946   ins_pipe(ialu_cr_reg_imm);
10947 %}
10948 
10949 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
10950 %{
10951   predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
10952   match(Set cr (CmpP (LoadP mem) zero));
10953 
10954   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
10955   ins_encode %{
10956     __ cmpq(r12, $mem$$Address);
10957   %}
10958   ins_pipe(ialu_cr_reg_mem);
10959 %}
10960 
10961 instruct compN_rReg(rFlagsRegU cr, rRegN op1, rRegN op2)
10962 %{
10963   match(Set cr (CmpN op1 op2));
10964 
10965   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10966   ins_encode %{ __ cmpl($op1$$Register, $op2$$Register); %}
10967   ins_pipe(ialu_cr_reg_reg);
10968 %}
10969 
10970 instruct compN_rReg_mem(rFlagsRegU cr, rRegN src, memory mem)
10971 %{
10972   match(Set cr (CmpN src (LoadN mem)));
10973 
10974   format %{ "cmpl    $src, $mem\t# compressed ptr" %}
10975   ins_encode %{
10976     __ cmpl($src$$Register, $mem$$Address);
10977   %}
10978   ins_pipe(ialu_cr_reg_mem);
10979 %}
10980 
10981 instruct compN_rReg_imm(rFlagsRegU cr, rRegN op1, immN op2) %{
10982   match(Set cr (CmpN op1 op2));
10983 
10984   format %{ "cmpl    $op1, $op2\t# compressed ptr" %}
10985   ins_encode %{
10986     __ cmp_narrow_oop($op1$$Register, (jobject)$op2$$constant);
10987   %}
10988   ins_pipe(ialu_cr_reg_imm);
10989 %}
10990 
10991 instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
10992 %{
10993   match(Set cr (CmpN src (LoadN mem)));
10994 
10995   format %{ "cmpl    $mem, $src\t# compressed ptr" %}
10996   ins_encode %{
10997     __ cmp_narrow_oop($mem$$Address, (jobject)$src$$constant);
10998   %}
10999   ins_pipe(ialu_cr_reg_mem);
11000 %}
11001 
11002 instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
11003   match(Set cr (CmpN op1 op2));
11004 
11005   format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
11006   ins_encode %{
11007     __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
11008   %}
11009   ins_pipe(ialu_cr_reg_imm);
11010 %}
11011 
11012 instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
11013 %{
11014   match(Set cr (CmpN src (LoadNKlass mem)));
11015 
11016   format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
11017   ins_encode %{
11018     __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
11019   %}
11020   ins_pipe(ialu_cr_reg_mem);
11021 %}
11022 
11023 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
11024   match(Set cr (CmpN src zero));
11025 
11026   format %{ "testl   $src, $src\t# compressed ptr" %}
11027   ins_encode %{ __ testl($src$$Register, $src$$Register); %}
11028   ins_pipe(ialu_cr_reg_imm);
11029 %}
11030 
11031 instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
11032 %{
11033   predicate(Universe::narrow_oop_base() != NULL);
11034   match(Set cr (CmpN (LoadN mem) zero));
11035 
11036   ins_cost(500); // XXX
11037   format %{ "testl   $mem, 0xffffffff\t# compressed ptr" %}
11038   ins_encode %{
11039     __ cmpl($mem$$Address, (int)0xFFFFFFFF);
11040   %}
11041   ins_pipe(ialu_cr_reg_mem);
11042 %}
11043 
11044 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
11045 %{
11046   predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
11047   match(Set cr (CmpN (LoadN mem) zero));
11048 
11049   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}
11050   ins_encode %{
11051     __ cmpl(r12, $mem$$Address);
11052   %}
11053   ins_pipe(ialu_cr_reg_mem);
11054 %}
11055 
11056 // Yanked all unsigned pointer compare operations.
11057 // Pointer compares are done with CmpP which is already unsigned.
11058 
11059 instruct compL_rReg(rFlagsReg cr, rRegL op1, rRegL op2)
11060 %{
11061   match(Set cr (CmpL op1 op2));
11062 
11063   format %{ "cmpq    $op1, $op2" %}
11064   opcode(0x3B);  /* Opcode 3B /r */
11065   ins_encode(REX_reg_reg_wide(op1, op2), OpcP, reg_reg(op1, op2));
11066   ins_pipe(ialu_cr_reg_reg);
11067 %}
11068 
11069 instruct compL_rReg_imm(rFlagsReg cr, rRegL op1, immL32 op2)
11070 %{
11071   match(Set cr (CmpL op1 op2));
11072 
11073   format %{ "cmpq    $op1, $op2" %}
11074   opcode(0x81, 0x07); /* Opcode 81 /7 */
11075   ins_encode(OpcSErm_wide(op1, op2), Con8or32(op2));
11076   ins_pipe(ialu_cr_reg_imm);
11077 %}
11078 
11079 instruct compL_rReg_mem(rFlagsReg cr, rRegL op1, memory op2)
11080 %{
11081   match(Set cr (CmpL op1 (LoadL op2)));
11082 
11083   format %{ "cmpq    $op1, $op2" %}
11084   opcode(0x3B); /* Opcode 3B /r */
11085   ins_encode(REX_reg_mem_wide(op1, op2), OpcP, reg_mem(op1, op2));
11086   ins_pipe(ialu_cr_reg_mem);
11087 %}
11088 
11089 instruct testL_reg(rFlagsReg cr, rRegL src, immL0 zero)
11090 %{
11091   match(Set cr (CmpL src zero));
11092 
11093   format %{ "testq   $src, $src" %}
11094   opcode(0x85);
11095   ins_encode(REX_reg_reg_wide(src, src), OpcP, reg_reg(src, src));
11096   ins_pipe(ialu_cr_reg_imm);
11097 %}
11098 
11099 instruct testL_reg_imm(rFlagsReg cr, rRegL src, immL32 con, immL0 zero)
11100 %{
11101   match(Set cr (CmpL (AndL src con) zero));
11102 
11103   format %{ "testq   $src, $con\t# long" %}
11104   opcode(0xF7, 0x00);
11105   ins_encode(REX_reg_wide(src), OpcP, reg_opc(src), Con32(con));
11106   ins_pipe(ialu_cr_reg_imm);
11107 %}
11108 
11109 instruct testL_reg_mem(rFlagsReg cr, rRegL src, memory mem, immL0 zero)
11110 %{
11111   match(Set cr (CmpL (AndL src (LoadL mem)) zero));
11112 
11113   format %{ "testq   $src, $mem" %}
11114   opcode(0x85);
11115   ins_encode(REX_reg_mem_wide(src, mem), OpcP, reg_mem(src, mem));
11116   ins_pipe(ialu_cr_reg_mem);
11117 %}
11118 
11119 // Manifest a CmpL result in an integer register.  Very painful.
11120 // This is the test to avoid.
11121 instruct cmpL3_reg_reg(rRegI dst, rRegL src1, rRegL src2, rFlagsReg flags)
11122 %{
11123   match(Set dst (CmpL3 src1 src2));
11124   effect(KILL flags);
11125 
11126   ins_cost(275); // XXX
11127   format %{ "cmpq    $src1, $src2\t# CmpL3\n\t"
11128             "movl    $dst, -1\n\t"
11129             "jl,s    done\n\t"
11130             "setne   $dst\n\t"
11131             "movzbl  $dst, $dst\n\t"
11132     "done:" %}
11133   ins_encode(cmpl3_flag(src1, src2, dst));
11134   ins_pipe(pipe_slow);
11135 %}
11136 
11137 //----------Max and Min--------------------------------------------------------
11138 // Min Instructions
11139 
11140 instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
11141 %{
11142   effect(USE_DEF dst, USE src, USE cr);
11143 
11144   format %{ "cmovlgt $dst, $src\t# min" %}
11145   opcode(0x0F, 0x4F);
11146   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11147   ins_pipe(pipe_cmov_reg);
11148 %}
11149 
11150 
11151 instruct minI_rReg(rRegI dst, rRegI src)
11152 %{
11153   match(Set dst (MinI dst src));
11154 
11155   ins_cost(200);
11156   expand %{
11157     rFlagsReg cr;
11158     compI_rReg(cr, dst, src);
11159     cmovI_reg_g(dst, src, cr);
11160   %}
11161 %}
11162 
11163 instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
11164 %{
11165   effect(USE_DEF dst, USE src, USE cr);
11166 
11167   format %{ "cmovllt $dst, $src\t# max" %}
11168   opcode(0x0F, 0x4C);
11169   ins_encode(REX_reg_reg(dst, src), OpcP, OpcS, reg_reg(dst, src));
11170   ins_pipe(pipe_cmov_reg);
11171 %}
11172 
11173 
11174 instruct maxI_rReg(rRegI dst, rRegI src)
11175 %{
11176   match(Set dst (MaxI dst src));
11177 
11178   ins_cost(200);
11179   expand %{
11180     rFlagsReg cr;
11181     compI_rReg(cr, dst, src);
11182     cmovI_reg_l(dst, src, cr);
11183   %}
11184 %}
11185 
11186 // ============================================================================
11187 // Branch Instructions
11188 
11189 // Jump Direct - Label defines a relative address from JMP+1
11190 instruct jmpDir(label labl)
11191 %{
11192   match(Goto);
11193   effect(USE labl);
11194 
11195   ins_cost(300);
11196   format %{ "jmp     $labl" %}
11197   size(5);
11198   ins_encode %{
11199     Label* L = $labl$$label;
11200     __ jmp(*L, false); // Always long jump
11201   %}
11202   ins_pipe(pipe_jmp);
11203 %}
11204 
11205 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11206 instruct jmpCon(cmpOp cop, rFlagsReg cr, label labl)
11207 %{
11208   match(If cop cr);
11209   effect(USE labl);
11210 
11211   ins_cost(300);
11212   format %{ "j$cop     $labl" %}
11213   size(6);
11214   ins_encode %{
11215     Label* L = $labl$$label;
11216     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11217   %}
11218   ins_pipe(pipe_jcc);
11219 %}
11220 
11221 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11222 instruct jmpLoopEnd(cmpOp cop, rFlagsReg cr, label labl)
11223 %{
11224   match(CountedLoopEnd cop cr);
11225   effect(USE labl);
11226 
11227   ins_cost(300);
11228   format %{ "j$cop     $labl\t# loop end" %}
11229   size(6);
11230   ins_encode %{
11231     Label* L = $labl$$label;
11232     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11233   %}
11234   ins_pipe(pipe_jcc);
11235 %}
11236 
11237 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11238 instruct jmpLoopEndU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11239   match(CountedLoopEnd cop cmp);
11240   effect(USE labl);
11241 
11242   ins_cost(300);
11243   format %{ "j$cop,u   $labl\t# loop end" %}
11244   size(6);
11245   ins_encode %{
11246     Label* L = $labl$$label;
11247     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11248   %}
11249   ins_pipe(pipe_jcc);
11250 %}
11251 
11252 instruct jmpLoopEndUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11253   match(CountedLoopEnd cop cmp);
11254   effect(USE labl);
11255 
11256   ins_cost(200);
11257   format %{ "j$cop,u   $labl\t# loop end" %}
11258   size(6);
11259   ins_encode %{
11260     Label* L = $labl$$label;
11261     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11262   %}
11263   ins_pipe(pipe_jcc);
11264 %}
11265 
11266 // Jump Direct Conditional - using unsigned comparison
11267 instruct jmpConU(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11268   match(If cop cmp);
11269   effect(USE labl);
11270 
11271   ins_cost(300);
11272   format %{ "j$cop,u  $labl" %}
11273   size(6);
11274   ins_encode %{
11275     Label* L = $labl$$label;
11276     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11277   %}
11278   ins_pipe(pipe_jcc);
11279 %}
11280 
11281 instruct jmpConUCF(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11282   match(If cop cmp);
11283   effect(USE labl);
11284 
11285   ins_cost(200);
11286   format %{ "j$cop,u  $labl" %}
11287   size(6);
11288   ins_encode %{
11289     Label* L = $labl$$label;
11290     __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
11291   %}
11292   ins_pipe(pipe_jcc);
11293 %}
11294 
11295 instruct jmpConUCF2(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11296   match(If cop cmp);
11297   effect(USE labl);
11298 
11299   ins_cost(200);
11300   format %{ $$template
11301     if ($cop$$cmpcode == Assembler::notEqual) {
11302       $$emit$$"jp,u   $labl\n\t"
11303       $$emit$$"j$cop,u   $labl"
11304     } else {
11305       $$emit$$"jp,u   done\n\t"
11306       $$emit$$"j$cop,u   $labl\n\t"
11307       $$emit$$"done:"
11308     }
11309   %}
11310   ins_encode %{
11311     Label* l = $labl$$label;
11312     if ($cop$$cmpcode == Assembler::notEqual) {
11313       __ jcc(Assembler::parity, *l, false);
11314       __ jcc(Assembler::notEqual, *l, false);
11315     } else if ($cop$$cmpcode == Assembler::equal) {
11316       Label done;
11317       __ jccb(Assembler::parity, done);
11318       __ jcc(Assembler::equal, *l, false);
11319       __ bind(done);
11320     } else {
11321        ShouldNotReachHere();
11322     }
11323   %}
11324   ins_pipe(pipe_jcc);
11325 %}
11326 
11327 // ============================================================================
11328 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary
11329 // superklass array for an instance of the superklass.  Set a hidden
11330 // internal cache on a hit (cache is checked with exposed code in
11331 // gen_subtype_check()).  Return NZ for a miss or zero for a hit.  The
11332 // encoding ALSO sets flags.
11333 
11334 instruct partialSubtypeCheck(rdi_RegP result,
11335                              rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11336                              rFlagsReg cr)
11337 %{
11338   match(Set result (PartialSubtypeCheck sub super));
11339   effect(KILL rcx, KILL cr);
11340 
11341   ins_cost(1100);  // slightly larger than the next version
11342   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11343             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11344             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11345             "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
11346             "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
11347             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11348             "xorq    $result, $result\t\t Hit: rdi zero\n\t"
11349     "miss:\t" %}
11350 
11351   opcode(0x1); // Force a XOR of RDI
11352   ins_encode(enc_PartialSubtypeCheck());
11353   ins_pipe(pipe_slow);
11354 %}
11355 
11356 instruct partialSubtypeCheck_vs_Zero(rFlagsReg cr,
11357                                      rsi_RegP sub, rax_RegP super, rcx_RegI rcx,
11358                                      immP0 zero,
11359                                      rdi_RegP result)
11360 %{
11361   match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
11362   effect(KILL rcx, KILL result);
11363 
11364   ins_cost(1000);
11365   format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
11366             "movl    rcx, [rdi + Array<Klass*>::length_offset_in_bytes()]\t# length to scan\n\t"
11367             "addq    rdi, Array<Klass*>::base_offset_in_bytes()\t# Skip to start of data; set NZ in case count is zero\n\t"
11368             "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
11369             "jne,s   miss\t\t# Missed: flags nz\n\t"
11370             "movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
11371     "miss:\t" %}
11372 
11373   opcode(0x0); // No need to XOR RDI
11374   ins_encode(enc_PartialSubtypeCheck());
11375   ins_pipe(pipe_slow);
11376 %}
11377 
11378 // ============================================================================
11379 // Branch Instructions -- short offset versions
11380 //
11381 // These instructions are used to replace jumps of a long offset (the default
11382 // match) with jumps of a shorter offset.  These instructions are all tagged
11383 // with the ins_short_branch attribute, which causes the ADLC to suppress the
11384 // match rules in general matching.  Instead, the ADLC generates a conversion
11385 // method in the MachNode which can be used to do in-place replacement of the
11386 // long variant with the shorter variant.  The compiler will determine if a
11387 // branch can be taken by the is_short_branch_offset() predicate in the machine
11388 // specific code section of the file.
11389 
11390 // Jump Direct - Label defines a relative address from JMP+1
11391 instruct jmpDir_short(label labl) %{
11392   match(Goto);
11393   effect(USE labl);
11394 
11395   ins_cost(300);
11396   format %{ "jmp,s   $labl" %}
11397   size(2);
11398   ins_encode %{
11399     Label* L = $labl$$label;
11400     __ jmpb(*L);
11401   %}
11402   ins_pipe(pipe_jmp);
11403   ins_short_branch(1);
11404 %}
11405 
11406 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11407 instruct jmpCon_short(cmpOp cop, rFlagsReg cr, label labl) %{
11408   match(If cop cr);
11409   effect(USE labl);
11410 
11411   ins_cost(300);
11412   format %{ "j$cop,s   $labl" %}
11413   size(2);
11414   ins_encode %{
11415     Label* L = $labl$$label;
11416     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11417   %}
11418   ins_pipe(pipe_jcc);
11419   ins_short_branch(1);
11420 %}
11421 
11422 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11423 instruct jmpLoopEnd_short(cmpOp cop, rFlagsReg cr, label labl) %{
11424   match(CountedLoopEnd cop cr);
11425   effect(USE labl);
11426 
11427   ins_cost(300);
11428   format %{ "j$cop,s   $labl\t# loop end" %}
11429   size(2);
11430   ins_encode %{
11431     Label* L = $labl$$label;
11432     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11433   %}
11434   ins_pipe(pipe_jcc);
11435   ins_short_branch(1);
11436 %}
11437 
11438 // Jump Direct Conditional - Label defines a relative address from Jcc+1
11439 instruct jmpLoopEndU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11440   match(CountedLoopEnd cop cmp);
11441   effect(USE labl);
11442 
11443   ins_cost(300);
11444   format %{ "j$cop,us  $labl\t# loop end" %}
11445   size(2);
11446   ins_encode %{
11447     Label* L = $labl$$label;
11448     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11449   %}
11450   ins_pipe(pipe_jcc);
11451   ins_short_branch(1);
11452 %}
11453 
11454 instruct jmpLoopEndUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11455   match(CountedLoopEnd cop cmp);
11456   effect(USE labl);
11457 
11458   ins_cost(300);
11459   format %{ "j$cop,us  $labl\t# loop end" %}
11460   size(2);
11461   ins_encode %{
11462     Label* L = $labl$$label;
11463     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11464   %}
11465   ins_pipe(pipe_jcc);
11466   ins_short_branch(1);
11467 %}
11468 
11469 // Jump Direct Conditional - using unsigned comparison
11470 instruct jmpConU_short(cmpOpU cop, rFlagsRegU cmp, label labl) %{
11471   match(If cop cmp);
11472   effect(USE labl);
11473 
11474   ins_cost(300);
11475   format %{ "j$cop,us  $labl" %}
11476   size(2);
11477   ins_encode %{
11478     Label* L = $labl$$label;
11479     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11480   %}
11481   ins_pipe(pipe_jcc);
11482   ins_short_branch(1);
11483 %}
11484 
11485 instruct jmpConUCF_short(cmpOpUCF cop, rFlagsRegUCF cmp, label labl) %{
11486   match(If cop cmp);
11487   effect(USE labl);
11488 
11489   ins_cost(300);
11490   format %{ "j$cop,us  $labl" %}
11491   size(2);
11492   ins_encode %{
11493     Label* L = $labl$$label;
11494     __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
11495   %}
11496   ins_pipe(pipe_jcc);
11497   ins_short_branch(1);
11498 %}
11499 
11500 instruct jmpConUCF2_short(cmpOpUCF2 cop, rFlagsRegUCF cmp, label labl) %{
11501   match(If cop cmp);
11502   effect(USE labl);
11503 
11504   ins_cost(300);
11505   format %{ $$template
11506     if ($cop$$cmpcode == Assembler::notEqual) {
11507       $$emit$$"jp,u,s   $labl\n\t"
11508       $$emit$$"j$cop,u,s   $labl"
11509     } else {
11510       $$emit$$"jp,u,s   done\n\t"
11511       $$emit$$"j$cop,u,s  $labl\n\t"
11512       $$emit$$"done:"
11513     }
11514   %}
11515   size(4);
11516   ins_encode %{
11517     Label* l = $labl$$label;
11518     if ($cop$$cmpcode == Assembler::notEqual) {
11519       __ jccb(Assembler::parity, *l);
11520       __ jccb(Assembler::notEqual, *l);
11521     } else if ($cop$$cmpcode == Assembler::equal) {
11522       Label done;
11523       __ jccb(Assembler::parity, done);
11524       __ jccb(Assembler::equal, *l);
11525       __ bind(done);
11526     } else {
11527        ShouldNotReachHere();
11528     }
11529   %}
11530   ins_pipe(pipe_jcc);
11531   ins_short_branch(1);
11532 %}
11533 
11534 // ============================================================================
11535 // inlined locking and unlocking
11536 
11537 instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rdx_RegI scr, rRegI cx1, rRegI cx2) %{
11538   predicate(Compile::current()->use_rtm());
11539   match(Set cr (FastLock object box));
11540   effect(TEMP tmp, TEMP scr, TEMP cx1, TEMP cx2, USE_KILL box);
11541   ins_cost(300);
11542   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr,$cx1,$cx2" %}
11543   ins_encode %{
11544     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11545                  $scr$$Register, $cx1$$Register, $cx2$$Register,
11546                  _counters, _rtm_counters, _stack_rtm_counters,
11547                  ((Method*)(ra_->C->method()->constant_encoding()))->method_data(),
11548                  true, ra_->C->profile_rtm());
11549   %}
11550   ins_pipe(pipe_slow);
11551 %}
11552 
11553 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11554   predicate(!Compile::current()->use_rtm());
11555   match(Set cr (FastLock object box));
11556   effect(TEMP tmp, TEMP scr, USE_KILL box);
11557   ins_cost(300);
11558   format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11559   ins_encode %{
11560     __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
11561                  $scr$$Register, noreg, noreg, _counters, NULL, NULL, NULL, false, false);
11562   %}
11563   ins_pipe(pipe_slow);
11564 %}
11565 
11566 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11567   match(Set cr (FastUnlock object box));
11568   effect(TEMP tmp, USE_KILL box);
11569   ins_cost(300);
11570   format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11571   ins_encode %{
11572     __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, ra_->C->use_rtm());
11573   %}
11574   ins_pipe(pipe_slow);
11575 %}
11576 
11577 
11578 // ============================================================================
11579 // Safepoint Instructions
11580 instruct safePoint_poll(rFlagsReg cr)
11581 %{
11582   predicate(!Assembler::is_polling_page_far());
11583   match(SafePoint);
11584   effect(KILL cr);
11585 
11586   format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
11587             "# Safepoint: poll for GC" %}
11588   ins_cost(125);
11589   ins_encode %{
11590     AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
11591     __ testl(rax, addr);
11592   %}
11593   ins_pipe(ialu_reg_mem);
11594 %}
11595 
11596 instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
11597 %{
11598   predicate(Assembler::is_polling_page_far());
11599   match(SafePoint poll);
11600   effect(KILL cr, USE poll);
11601 
11602   format %{ "testl  rax, [$poll]\t"
11603             "# Safepoint: poll for GC" %}
11604   ins_cost(125);
11605   ins_encode %{
11606     __ relocate(relocInfo::poll_type);
11607     __ testl(rax, Address($poll$$Register, 0));
11608   %}
11609   ins_pipe(ialu_reg_mem);
11610 %}
11611 
11612 // ============================================================================
11613 // Procedure Call/Return Instructions
11614 // Call Java Static Instruction
11615 // Note: If this code changes, the corresponding ret_addr_offset() and
11616 //       compute_padding() functions will have to be adjusted.
11617 instruct CallStaticJavaDirect(method meth) %{
11618   match(CallStaticJava);
11619   effect(USE meth);
11620 
11621   ins_cost(300);
11622   format %{ "call,static " %}
11623   opcode(0xE8); /* E8 cd */
11624   ins_encode(clear_avx, Java_Static_Call(meth), call_epilog);
11625   ins_pipe(pipe_slow);
11626   ins_alignment(4);
11627 %}
11628 
11629 // Call Java Dynamic Instruction
11630 // Note: If this code changes, the corresponding ret_addr_offset() and
11631 //       compute_padding() functions will have to be adjusted.
11632 instruct CallDynamicJavaDirect(method meth)
11633 %{
11634   match(CallDynamicJava);
11635   effect(USE meth);
11636 
11637   ins_cost(300);
11638   format %{ "movq    rax, #Universe::non_oop_word()\n\t"
11639             "call,dynamic " %}
11640   ins_encode(clear_avx, Java_Dynamic_Call(meth), call_epilog);
11641   ins_pipe(pipe_slow);
11642   ins_alignment(4);
11643 %}
11644 
11645 // Call Runtime Instruction
11646 instruct CallRuntimeDirect(method meth)
11647 %{
11648   match(CallRuntime);
11649   effect(USE meth);
11650 
11651   ins_cost(300);
11652   format %{ "call,runtime " %}
11653   ins_encode(clear_avx, Java_To_Runtime(meth));
11654   ins_pipe(pipe_slow);
11655 %}
11656 
11657 // Call runtime without safepoint
11658 instruct CallLeafDirect(method meth)
11659 %{
11660   match(CallLeaf);
11661   effect(USE meth);
11662 
11663   ins_cost(300);
11664   format %{ "call_leaf,runtime " %}
11665   ins_encode(clear_avx, Java_To_Runtime(meth));
11666   ins_pipe(pipe_slow);
11667 %}
11668 
11669 // Call runtime without safepoint
11670 instruct CallLeafNoFPDirect(method meth)
11671 %{
11672   match(CallLeafNoFP);
11673   effect(USE meth);
11674 
11675   ins_cost(300);
11676   format %{ "call_leaf_nofp,runtime " %}
11677   ins_encode(Java_To_Runtime(meth));
11678   ins_pipe(pipe_slow);
11679 %}
11680 
11681 // Return Instruction
11682 // Remove the return address & jump to it.
11683 // Notice: We always emit a nop after a ret to make sure there is room
11684 // for safepoint patching
11685 instruct Ret()
11686 %{
11687   match(Return);
11688 
11689   format %{ "ret" %}
11690   opcode(0xC3);
11691   ins_encode(OpcP);
11692   ins_pipe(pipe_jmp);
11693 %}
11694 
11695 // Tail Call; Jump from runtime stub to Java code.
11696 // Also known as an 'interprocedural jump'.
11697 // Target of jump will eventually return to caller.
11698 // TailJump below removes the return address.
11699 instruct TailCalljmpInd(no_rbp_RegP jump_target, rbx_RegP method_oop)
11700 %{
11701   match(TailCall jump_target method_oop);
11702 
11703   ins_cost(300);
11704   format %{ "jmp     $jump_target\t# rbx holds method oop" %}
11705   opcode(0xFF, 0x4); /* Opcode FF /4 */
11706   ins_encode(REX_reg(jump_target), OpcP, reg_opc(jump_target));
11707   ins_pipe(pipe_jmp);
11708 %}
11709 
11710 // Tail Jump; remove the return address; jump to target.
11711 // TailCall above leaves the return address around.
11712 instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
11713 %{
11714   match(TailJump jump_target ex_oop);
11715 
11716   ins_cost(300);
11717   format %{ "popq    rdx\t# pop return address\n\t"
11718             "jmp     $jump_target" %}
11719   opcode(0xFF, 0x4); /* Opcode FF /4 */
11720   ins_encode(Opcode(0x5a), // popq rdx
11721              REX_reg(jump_target), OpcP, reg_opc(jump_target));
11722   ins_pipe(pipe_jmp);
11723 %}
11724 
11725 // Create exception oop: created by stack-crawling runtime code.
11726 // Created exception is now available to this handler, and is setup
11727 // just prior to jumping to this handler.  No code emitted.
11728 instruct CreateException(rax_RegP ex_oop)
11729 %{
11730   match(Set ex_oop (CreateEx));
11731 
11732   size(0);
11733   // use the following format syntax
11734   format %{ "# exception oop is in rax; no code emitted" %}
11735   ins_encode();
11736   ins_pipe(empty);
11737 %}
11738 
11739 // Rethrow exception:
11740 // The exception oop will come in the first argument position.
11741 // Then JUMP (not call) to the rethrow stub code.
11742 instruct RethrowException()
11743 %{
11744   match(Rethrow);
11745 
11746   // use the following format syntax
11747   format %{ "jmp     rethrow_stub" %}
11748   ins_encode(enc_rethrow);
11749   ins_pipe(pipe_jmp);
11750 %}
11751 
11752 
11753 // ============================================================================
11754 // This name is KNOWN by the ADLC and cannot be changed.
11755 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
11756 // for this guy.
11757 instruct tlsLoadP(r15_RegP dst) %{
11758   match(Set dst (ThreadLocal));
11759   effect(DEF dst);
11760 
11761   size(0);
11762   format %{ "# TLS is in R15" %}
11763   ins_encode( /*empty encoding*/ );
11764   ins_pipe(ialu_reg_reg);
11765 %}
11766 
11767 
11768 //----------PEEPHOLE RULES-----------------------------------------------------
11769 // These must follow all instruction definitions as they use the names
11770 // defined in the instructions definitions.
11771 //
11772 // peepmatch ( root_instr_name [preceding_instruction]* );
11773 //
11774 // peepconstraint %{
11775 // (instruction_number.operand_name relational_op instruction_number.operand_name
11776 //  [, ...] );
11777 // // instruction numbers are zero-based using left to right order in peepmatch
11778 //
11779 // peepreplace ( instr_name  ( [instruction_number.operand_name]* ) );
11780 // // provide an instruction_number.operand_name for each operand that appears
11781 // // in the replacement instruction's match rule
11782 //
11783 // ---------VM FLAGS---------------------------------------------------------
11784 //
11785 // All peephole optimizations can be turned off using -XX:-OptoPeephole
11786 //
11787 // Each peephole rule is given an identifying number starting with zero and
11788 // increasing by one in the order seen by the parser.  An individual peephole
11789 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
11790 // on the command-line.
11791 //
11792 // ---------CURRENT LIMITATIONS----------------------------------------------
11793 //
11794 // Only match adjacent instructions in same basic block
11795 // Only equality constraints
11796 // Only constraints between operands, not (0.dest_reg == RAX_enc)
11797 // Only one replacement instruction
11798 //
11799 // ---------EXAMPLE----------------------------------------------------------
11800 //
11801 // // pertinent parts of existing instructions in architecture description
11802 // instruct movI(rRegI dst, rRegI src)
11803 // %{
11804 //   match(Set dst (CopyI src));
11805 // %}
11806 //
11807 // instruct incI_rReg(rRegI dst, immI1 src, rFlagsReg cr)
11808 // %{
11809 //   match(Set dst (AddI dst src));
11810 //   effect(KILL cr);
11811 // %}
11812 //
11813 // // Change (inc mov) to lea
11814 // peephole %{
11815 //   // increment preceeded by register-register move
11816 //   peepmatch ( incI_rReg movI );
11817 //   // require that the destination register of the increment
11818 //   // match the destination register of the move
11819 //   peepconstraint ( 0.dst == 1.dst );
11820 //   // construct a replacement instruction that sets
11821 //   // the destination to ( move's source register + one )
11822 //   peepreplace ( leaI_rReg_immI( 0.dst 1.src 0.src ) );
11823 // %}
11824 //
11825 
11826 // Implementation no longer uses movX instructions since
11827 // machine-independent system no longer uses CopyX nodes.
11828 //
11829 // peephole
11830 // %{
11831 //   peepmatch (incI_rReg movI);
11832 //   peepconstraint (0.dst == 1.dst);
11833 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11834 // %}
11835 
11836 // peephole
11837 // %{
11838 //   peepmatch (decI_rReg movI);
11839 //   peepconstraint (0.dst == 1.dst);
11840 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11841 // %}
11842 
11843 // peephole
11844 // %{
11845 //   peepmatch (addI_rReg_imm movI);
11846 //   peepconstraint (0.dst == 1.dst);
11847 //   peepreplace (leaI_rReg_immI(0.dst 1.src 0.src));
11848 // %}
11849 
11850 // peephole
11851 // %{
11852 //   peepmatch (incL_rReg movL);
11853 //   peepconstraint (0.dst == 1.dst);
11854 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11855 // %}
11856 
11857 // peephole
11858 // %{
11859 //   peepmatch (decL_rReg movL);
11860 //   peepconstraint (0.dst == 1.dst);
11861 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11862 // %}
11863 
11864 // peephole
11865 // %{
11866 //   peepmatch (addL_rReg_imm movL);
11867 //   peepconstraint (0.dst == 1.dst);
11868 //   peepreplace (leaL_rReg_immL(0.dst 1.src 0.src));
11869 // %}
11870 
11871 // peephole
11872 // %{
11873 //   peepmatch (addP_rReg_imm movP);
11874 //   peepconstraint (0.dst == 1.dst);
11875 //   peepreplace (leaP_rReg_imm(0.dst 1.src 0.src));
11876 // %}
11877 
11878 // // Change load of spilled value to only a spill
11879 // instruct storeI(memory mem, rRegI src)
11880 // %{
11881 //   match(Set mem (StoreI mem src));
11882 // %}
11883 //
11884 // instruct loadI(rRegI dst, memory mem)
11885 // %{
11886 //   match(Set dst (LoadI mem));
11887 // %}
11888 //
11889 
11890 peephole
11891 %{
11892   peepmatch (loadI storeI);
11893   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11894   peepreplace (storeI(1.mem 1.mem 1.src));
11895 %}
11896 
11897 peephole
11898 %{
11899   peepmatch (loadL storeL);
11900   peepconstraint (1.src == 0.dst, 1.mem == 0.mem);
11901   peepreplace (storeL(1.mem 1.mem 1.src));
11902 %}
11903 
11904 //----------SMARTSPILL RULES---------------------------------------------------
11905 // These must follow all instruction definitions as they use the names
11906 // defined in the instructions definitions.